From ba67db3f13b40ca16ca904ec34b28cf19d1acef6 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 16:32:26 +0200 Subject: [PATCH 001/258] removed unused imports --- monitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monitor.py b/monitor.py index eb63e83c..cedddd4d 100644 --- a/monitor.py +++ b/monitor.py @@ -3,7 +3,7 @@ import json import re import threading from operator import itemgetter -from os import listdir, SEEK_SET, SEEK_CUR, SEEK_END +from os import listdir, SEEK_SET, SEEK_END from os.path import basename, expanduser, isdir, join from sys import platform from time import gmtime, localtime, sleep, strftime, strptime, time @@ -18,7 +18,6 @@ from companion import ship_file_name if platform=='darwin': from AppKit import NSWorkspace - from Foundation import NSSearchPathForDirectoriesInDomains, NSApplicationSupportDirectory, NSUserDomainMask from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler from fcntl import fcntl From a9f4f5d50747be092e4c79ce27e7c1d7e4ed5945 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 16:41:22 +0200 Subject: [PATCH 002/258] made all regexps raw strings ensures no weirdness from escapes --- monitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitor.py b/monitor.py index cedddd4d..985e3dce 100644 --- a/monitor.py +++ b/monitor.py @@ -130,7 +130,7 @@ class EDLogs(FileSystemEventHandler): # Latest pre-existing logfile - e.g. if E:D is already running. Assumes logs sort alphabetically. # Do this before setting up the observer in case the journal directory has gone away try: - logfiles = sorted([x for x in listdir(self.currentdir) if re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], + logfiles = sorted([x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], key=lambda x: x.split('.')[1:]) self.logfile = logfiles and join(self.currentdir, logfiles[-1]) or None except: @@ -189,7 +189,7 @@ class EDLogs(FileSystemEventHandler): def on_created(self, event): # watchdog callback, e.g. client (re)started. - if not event.is_directory and re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', basename(event.src_path)): + if not event.is_directory and re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', basename(event.src_path)): self.logfile = event.src_path def worker(self): @@ -249,7 +249,7 @@ class EDLogs(FileSystemEventHandler): else: # Poll try: - logfiles = sorted([x for x in listdir(self.currentdir) if re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], + logfiles = sorted([x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], key=lambda x: x.split('.')[1:]) newlogfile = logfiles and join(self.currentdir, logfiles[-1]) or None except: @@ -761,7 +761,7 @@ class EDLogs(FileSystemEventHandler): return ship = ship_file_name(self.state['ShipName'], self.state['ShipType']) - regexp = re.compile(re.escape(ship) + '\.\d\d\d\d\-\d\d\-\d\dT\d\d\.\d\d\.\d\d\.txt') + regexp = re.compile(re.escape(ship) + r'\.\d{4}\-\d\d\-\d\dT\d\d\.\d\d\.\d\d\.txt') oldfiles = sorted([x for x in listdir(config.get('outdir')) if regexp.match(x)]) if oldfiles: with open(join(config.get('outdir'), oldfiles[-1]), 'rU') as h: From ce7c6d4333cf6c3b87ba026816303fd942407492 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 16:43:42 +0200 Subject: [PATCH 003/258] removed unused variable --- monitor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monitor.py b/monitor.py index 985e3dce..e3b78de1 100644 --- a/monitor.py +++ b/monitor.py @@ -176,7 +176,6 @@ class EDLogs(FileSystemEventHandler): self.thread = None # Orphan the worker thread - will terminate at next poll def close(self): - thread = self.thread self.stop() if self.observer: self.observer.stop() From a40cebc749686fa7cbdd1718b154701f7fff3eca Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 17:55:14 +0200 Subject: [PATCH 004/258] Added whitespace around scope changes Helps with reading code later --- monitor.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/monitor.py b/monitor.py index e3b78de1..eac9908e 100644 --- a/monitor.py +++ b/monitor.py @@ -125,6 +125,7 @@ class EDLogs(FileSystemEventHandler): if self.currentdir and self.currentdir != logdir: self.stop() + self.currentdir = logdir # Latest pre-existing logfile - e.g. if E:D is already running. Assumes logs sort alphabetically. @@ -133,6 +134,7 @@ class EDLogs(FileSystemEventHandler): logfiles = sorted([x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], key=lambda x: x.split('.')[1:]) self.logfile = logfiles and join(self.currentdir, logfiles[-1]) or None + except: self.logfile = None return False @@ -146,6 +148,7 @@ class EDLogs(FileSystemEventHandler): self.observer = Observer() self.observer.daemon = True self.observer.start() + elif polling and self.observer: self.observer.stop() self.observer = None @@ -173,12 +176,14 @@ class EDLogs(FileSystemEventHandler): if self.observed: self.observed = None self.observer.unschedule_all() + self.thread = None # Orphan the worker thread - will terminate at next poll def close(self): self.stop() if self.observer: self.observer.stop() + if self.observer: self.observer.join() self.observer = None @@ -202,13 +207,16 @@ class EDLogs(FileSystemEventHandler): loghandle = open(logfile, 'rb', 0) # unbuffered if platform == 'darwin': fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB + for line in loghandle: try: self.parse_entry(line) # Some events are of interest even in the past + except: if __debug__: print('Invalid journal entry "%s"' % repr(line)) logpos = loghandle.tell() + else: loghandle = None @@ -225,14 +233,19 @@ class EDLogs(FileSystemEventHandler): ('SystemAddress', self.systemaddress), ('Population', self.systempopulation), ]) + if self.planet: entry['Body'] = self.planet + entry['Docked'] = bool(self.station) + if self.station: entry['StationName'] = self.station entry['StationType'] = self.stationtype entry['MarketID'] = self.station_marketid + self.event_queue.append(json.dumps(entry, separators=(', ', ':'))) + else: self.event_queue.append(None) # Generate null event to update the display (with possibly out-of-date info) self.live = False @@ -251,19 +264,24 @@ class EDLogs(FileSystemEventHandler): logfiles = sorted([x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], key=lambda x: x.split('.')[1:]) newlogfile = logfiles and join(self.currentdir, logfiles[-1]) or None + except: if __debug__: print_exc() + newlogfile = None if logfile != newlogfile: logfile = newlogfile if loghandle: loghandle.close() + if logfile: loghandle = open(logfile, 'rb', 0) # unbuffered if platform == 'darwin': fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB + logpos = 0 + if __debug__: print('New logfile "%s"' % logfile) @@ -272,8 +290,10 @@ class EDLogs(FileSystemEventHandler): loghandle.seek(logpos, SEEK_SET) # reset EOF flag for line in loghandle: self.event_queue.append(line) + if self.event_queue: self.root.event_generate('<>', when="tail") + logpos = loghandle.tell() sleep(self._POLL) @@ -287,6 +307,7 @@ class EDLogs(FileSystemEventHandler): self.event_queue.append('{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())) self.root.event_generate('<>', when="tail") self.game_was_running = False + else: self.game_was_running = self.game_running() @@ -339,8 +360,10 @@ class EDLogs(FileSystemEventHandler): 'Rebuy' : None, 'Modules' : None, } + elif entry['event'] == 'Commander': self.live = True # First event in 3.0 + elif entry['event'] == 'LoadGame': self.cmdr = entry['Commander'] self.mode = entry.get('GameMode') # 'Open', 'Solo', 'Group', or None for CQC (and Training - but no LoadGame event) @@ -366,15 +389,19 @@ class EDLogs(FileSystemEventHandler): 'Statistics' : {}, 'Role' : None, }) + elif entry['event'] == 'NewCommander': self.cmdr = entry['Name'] self.group = None + elif entry['event'] == 'SetUserShipName': self.state['ShipID'] = entry['ShipID'] if 'UserShipId' in entry: # Only present when changing the ship's ident self.state['ShipIdent'] = entry['UserShipId'] + self.state['ShipName'] = entry.get('UserShipName') self.state['ShipType'] = self.canonicalise(entry['Ship']) + elif entry['event'] == 'ShipyardBuy': self.state['ShipID'] = None self.state['ShipIdent'] = None @@ -384,6 +411,7 @@ class EDLogs(FileSystemEventHandler): self.state['ModulesValue'] = None self.state['Rebuy'] = None self.state['Modules'] = None + elif entry['event'] == 'ShipyardSwap': self.state['ShipID'] = entry['ShipID'] self.state['ShipIdent'] = None @@ -393,9 +421,11 @@ class EDLogs(FileSystemEventHandler): self.state['ModulesValue'] = None self.state['Rebuy'] = None self.state['Modules'] = None + elif (entry['event'] == 'Loadout' and not 'fighter' in self.canonicalise(entry['Ship']) and not 'buggy' in self.canonicalise(entry['Ship'])): + self.state['ShipID'] = entry['ShipID'] self.state['ShipIdent'] = entry['ShipIdent'] @@ -420,7 +450,9 @@ class EDLogs(FileSystemEventHandler): module.get('AmmoInClip') == module.get('AmmoInHopper') == 1): # lasers module.pop('AmmoInClip') module.pop('AmmoInHopper') + self.state['Modules'][module['Slot']] = module + elif entry['event'] == 'ModuleBuy': self.state['Modules'][entry['Slot']] = { 'Slot' : entry['Slot'], @@ -430,29 +462,38 @@ class EDLogs(FileSystemEventHandler): 'Health' : 1.0, 'Value' : entry['BuyPrice'], } + elif entry['event'] == 'ModuleSell': self.state['Modules'].pop(entry['Slot'], None) + elif entry['event'] == 'ModuleSwap': toitem = self.state['Modules'].get(entry['ToSlot']) self.state['Modules'][entry['ToSlot']] = self.state['Modules'][entry['FromSlot']] if toitem: self.state['Modules'][entry['FromSlot']] = toitem + else: self.state['Modules'].pop(entry['FromSlot'], None) + elif entry['event'] in ['Undocked']: self.station = None self.station_marketid = None self.stationtype = None self.stationservices = None + elif entry['event'] in ['Location', 'FSDJump', 'Docked', 'CarrierJump']: if entry['event'] in ('Location', 'CarrierJump'): self.planet = entry.get('Body') if entry.get('BodyType') == 'Planet' else None + elif entry['event'] == 'FSDJump': self.planet = None + if 'StarPos' in entry: self.coordinates = tuple(entry['StarPos']) + elif self.system != entry['StarSystem']: self.coordinates = None # Docked event doesn't include coordinates + self.systemaddress = entry.get('SystemAddress') if entry['event'] in ['Location', 'FSDJump', 'CarrierJump']: @@ -463,8 +504,10 @@ class EDLogs(FileSystemEventHandler): self.station_marketid = entry.get('MarketID') # May be None self.stationtype = entry.get('StationType') # May be None self.stationservices = entry.get('StationServices') # None under E:D < 2.4 + elif entry['event'] == 'ApproachBody': self.planet = entry['Body'] + elif entry['event'] in ['LeaveBody', 'SupercruiseEntry']: self.planet = None @@ -474,10 +517,12 @@ class EDLogs(FileSystemEventHandler): payload.pop('timestamp') for k,v in payload.items(): self.state['Rank'][k] = (v,0) + elif entry['event'] == 'Progress': for k,v in entry.items(): if k in self.state['Rank']: self.state['Rank'][k] = (self.state['Rank'][k][0], min(v, 100)) # perhaps not taken promotion mission yet + elif entry['event'] in ['Reputation', 'Statistics']: payload = OrderedDict(entry) payload.pop('event') @@ -487,6 +532,7 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'EngineerProgress': if 'Engineers' in entry: # Startup summary self.state['Engineers'] = { e['Engineer']: (e['Rank'], e.get('RankProgress', 0)) if 'Rank' in e else e['Progress'] for e in entry['Engineers'] } + else: # Promotion self.state['Engineers'][entry['Engineer']] = (entry['Rank'], entry.get('RankProgress', 0)) if 'Rank' in entry else entry['Progress'] @@ -495,15 +541,19 @@ class EDLogs(FileSystemEventHandler): if 'Inventory' not in entry: # From 3.3 full Cargo event (after the first one) is written to a separate file with open(join(self.currentdir, 'Cargo.json'), 'rb') as h: entry = json.load(h, object_pairs_hook=OrderedDict) # Preserve property order because why not? + self.state['Cargo'].update({ self.canonicalise(x['Name']): x['Count'] for x in entry['Inventory'] }) + elif entry['event'] in ['CollectCargo', 'MarketBuy', 'BuyDrones', 'MiningRefined']: commodity = self.canonicalise(entry['Type']) self.state['Cargo'][commodity] += entry.get('Count', 1) + elif entry['event'] in ['EjectCargo', 'MarketSell', 'SellDrones']: commodity = self.canonicalise(entry['Type']) self.state['Cargo'][commodity] -= entry.get('Count', 1) if self.state['Cargo'][commodity] <= 0: self.state['Cargo'].pop(commodity) + elif entry['event'] == 'SearchAndRescue': for item in entry.get('Items', []): commodity = self.canonicalise(item['Name']) @@ -515,14 +565,17 @@ class EDLogs(FileSystemEventHandler): for category in ['Raw', 'Manufactured', 'Encoded']: self.state[category] = defaultdict(int) self.state[category].update({ self.canonicalise(x['Name']): x['Count'] for x in entry.get(category, []) }) + elif entry['event'] == 'MaterialCollected': material = self.canonicalise(entry['Name']) self.state[entry['Category']][material] += entry['Count'] + elif entry['event'] in ['MaterialDiscarded', 'ScientificResearch']: material = self.canonicalise(entry['Name']) self.state[entry['Category']][material] -= entry['Count'] if self.state[entry['Category']][material] <= 0: self.state[entry['Category']].pop(material) + elif entry['event'] == 'Synthesis': for category in ['Raw', 'Manufactured', 'Encoded']: for x in entry['Materials']: @@ -531,6 +584,7 @@ class EDLogs(FileSystemEventHandler): self.state[category][material] -= x['Count'] if self.state[category][material] <= 0: self.state[category].pop(material) + elif entry['event'] == 'MaterialTrade': category = self.category(entry['Paid']['Category']) self.state[category][entry['Paid']['Material']] -= entry['Paid']['Quantity'] @@ -547,6 +601,7 @@ class EDLogs(FileSystemEventHandler): self.state[category][material] -= x['Count'] if self.state[category][material] <= 0: self.state[category].pop(material) + module = self.state['Modules'][entry['Slot']] assert(module['Item'] == self.canonicalise(entry['Module'])) module['Engineering'] = { @@ -558,9 +613,11 @@ class EDLogs(FileSystemEventHandler): 'Quality' : entry['Quality'], 'Modifiers' : entry['Modifiers'], } + if 'ExperimentalEffect' in entry: module['Engineering']['ExperimentalEffect'] = entry['ExperimentalEffect'] module['Engineering']['ExperimentalEffect_Localised'] = entry['ExperimentalEffect_Localised'] + else: module['Engineering'].pop('ExperimentalEffect', None) module['Engineering'].pop('ExperimentalEffect_Localised', None) @@ -569,17 +626,20 @@ class EDLogs(FileSystemEventHandler): for reward in entry.get('CommodityReward', []): commodity = self.canonicalise(reward['Name']) self.state['Cargo'][commodity] += reward.get('Count', 1) + for reward in entry.get('MaterialsReward', []): if 'Category' in reward: # Category not present in E:D 3.0 category = self.category(reward['Category']) material = self.canonicalise(reward['Name']) self.state[category][material] += reward.get('Count', 1) + elif entry['event'] == 'EngineerContribution': commodity = self.canonicalise(entry.get('Commodity')) if commodity: self.state['Cargo'][commodity] -= entry['Quantity'] if self.state['Cargo'][commodity] <= 0: self.state['Cargo'].pop(commodity) + material = self.canonicalise(entry.get('Material')) if material: for category in ['Raw', 'Manufactured', 'Encoded']: @@ -587,6 +647,7 @@ class EDLogs(FileSystemEventHandler): self.state[category][material] -= entry['Quantity'] if self.state[category][material] <= 0: self.state[category].pop(material) + elif entry['event'] == 'TechnologyBroker': for thing in entry.get('Ingredients', []): # 3.01 for category in ['Cargo', 'Raw', 'Manufactured', 'Encoded']: @@ -595,11 +656,13 @@ class EDLogs(FileSystemEventHandler): self.state[category][item] -= thing['Count'] if self.state[category][item] <= 0: self.state[category].pop(item) + for thing in entry.get('Commodities', []): # 3.02 commodity = self.canonicalise(thing['Name']) self.state['Cargo'][commodity] -= thing['Count'] if self.state['Cargo'][commodity] <= 0: self.state['Cargo'].pop(commodity) + for thing in entry.get('Materials', []): # 3.02 material = self.canonicalise(thing['Name']) category = thing['Category'] @@ -618,8 +681,10 @@ class EDLogs(FileSystemEventHandler): self.stationservices = None self.coordinates = None self.systemaddress = None + elif entry['event'] == 'ChangeCrewRole': self.state['Role'] = entry['Role'] + elif entry['event'] == 'QuitACrew': self.state['Captain'] = None self.state['Role'] = None @@ -643,6 +708,7 @@ class EDLogs(FileSystemEventHandler): if __debug__: print('Invalid journal entry "%s"' % repr(line)) print_exc() + return { 'event': None } # Commodities, Modules and Ships can appear in different forms e.g. "$HNShockMount_Name;", "HNShockMount", and "hnshockmount", @@ -661,6 +727,7 @@ class EDLogs(FileSystemEventHandler): def get_entry(self): if not self.event_queue: return None + else: entry = self.parse_entry(self.event_queue.pop(0)) if not self.live and entry['event'] not in [None, 'Fileheader']: @@ -678,6 +745,7 @@ class EDLogs(FileSystemEventHandler): ('StarPos', self.coordinates), ('SystemAddress', self.systemaddress), ]) + else: entry = OrderedDict([ ('timestamp', strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())), @@ -687,20 +755,21 @@ class EDLogs(FileSystemEventHandler): ('StarPos', self.coordinates), ('SystemAddress', self.systemaddress), ]) + self.event_queue.append(json.dumps(entry, separators=(', ', ':'))) + elif self.live and entry['event'] == 'Music' and entry.get('MusicTrack') == 'MainMenu': self.event_queue.append('{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())) + return entry def game_running(self): - if platform == 'darwin': for app in NSWorkspace.sharedWorkspace().runningApplications(): if app.bundleIdentifier() == 'uk.co.frontier.EliteDangerous': return True elif platform == 'win32': - def WindowTitle(h): if h: l = GetWindowTextLength(h) + 1 @@ -716,6 +785,7 @@ class EDLogs(FileSystemEventHandler): if handle: # If GetProcessHandleFromHwnd succeeds then the app is already running as this user CloseHandle(handle) return False # stop enumeration + return True return not EnumWindows(EnumWindowsProc(callback), 0) @@ -733,30 +803,36 @@ class EDLogs(FileSystemEventHandler): d = OrderedDict() if timestamped: d['timestamp'] = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()) + d['event'] = 'Loadout' d['Ship'] = self.state['ShipType'] d['ShipID'] = self.state['ShipID'] + if self.state['ShipName']: d['ShipName'] = self.state['ShipName'] + if self.state['ShipIdent']: d['ShipIdent'] = self.state['ShipIdent'] + # sort modules by slot - hardpoints, standard, internal d['Modules'] = [] + for slot in sorted(self.state['Modules'], key=lambda x: ('Hardpoint' not in x, x not in standard_order and len(standard_order) or standard_order.index(x), 'Slot' not in x, x)): module = dict(self.state['Modules'][slot]) module.pop('Health', None) module.pop('Value', None) d['Modules'].append(module) + return d # Export ship loadout as a Loadout event def export_ship(self, filename=None): string = json.dumps(self.ship(False), ensure_ascii=False, indent=2, separators=(',', ': ')) # pretty print - if filename: with open(filename, 'wt') as h: h.write(string) + return ship = ship_file_name(self.state['ShipName'], self.state['ShipType']) From 86dea84e943e4db44b0e7423f8221824b3511827 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 17:57:39 +0200 Subject: [PATCH 005/258] Removed tabs preceeding inline comments --- monitor.py | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/monitor.py b/monitor.py index eac9908e..e99e9688 100644 --- a/monitor.py +++ b/monitor.py @@ -42,7 +42,7 @@ elif platform=='win32': else: # Linux's inotify doesn't work over CIFS or NFS, so poll - FileSystemEventHandler = object # dummy + FileSystemEventHandler = object # dummy # Journal handler @@ -53,7 +53,7 @@ class EDLogs(FileSystemEventHandler): _RE_CATEGORY = re.compile(r'\$MICRORESOURCE_CATEGORY_(.+);') def __init__(self): - FileSystemEventHandler.__init__(self) # futureproofing - not need for current version of watchdog + FileSystemEventHandler.__init__(self) # futureproofing - not need for current version of watchdog self.root = None self.currentdir = None # The actual logdir that we're monitoring self.logfile = None @@ -70,7 +70,7 @@ class EDLogs(FileSystemEventHandler): # If 3 we need to inject a special 'StartUp' event since consumers won't see the LoadGame event. self.live = False - self.game_was_running = False # For generation the "ShutDown" event + self.game_was_running = False # For generation the "ShutDown" event # Context for journal handling self.version = None @@ -85,16 +85,16 @@ class EDLogs(FileSystemEventHandler): self.stationtype = None self.coordinates = None self.systemaddress = None - self.started = None # Timestamp of the LoadGame event + self.started = None # Timestamp of the LoadGame event # Cmdr state shared with EDSM and plugins # If you change anything here update PLUGINS.md documentation! self.state = { - 'Captain' : None, # On a crew + 'Captain' : None, # On a crew 'Cargo' : defaultdict(int), 'Credits' : None, - 'FID' : None, # Frontier Cmdr ID - 'Horizons' : None, # Does this user have Horizons? + 'FID' : None, # Frontier Cmdr ID + 'Horizons' : None, # Does this user have Horizons? 'Loan' : None, 'Raw' : defaultdict(int), 'Manufactured' : defaultdict(int), @@ -103,8 +103,8 @@ class EDLogs(FileSystemEventHandler): 'Rank' : {}, 'Reputation' : {}, 'Statistics' : {}, - 'Role' : None, # Crew role - None, Idle, FireCon, FighterCon - 'Friends' : set(), # Online friends + 'Role' : None, # Crew role - None, Idle, FireCon, FighterCon + 'Friends' : set(), # Online friends 'ShipID' : None, 'ShipIdent' : None, 'ShipName' : None, @@ -177,7 +177,7 @@ class EDLogs(FileSystemEventHandler): self.observed = None self.observer.unschedule_all() - self.thread = None # Orphan the worker thread - will terminate at next poll + self.thread = None # Orphan the worker thread - will terminate at next poll def close(self): self.stop() @@ -204,13 +204,13 @@ class EDLogs(FileSystemEventHandler): # Seek to the end of the latest log file logfile = self.logfile if logfile: - loghandle = open(logfile, 'rb', 0) # unbuffered + loghandle = open(logfile, 'rb', 0) # unbuffered if platform == 'darwin': - fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB + fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB for line in loghandle: try: - self.parse_entry(line) # Some events are of interest even in the past + self.parse_entry(line) # Some events are of interest even in the past except: if __debug__: @@ -247,17 +247,17 @@ class EDLogs(FileSystemEventHandler): self.event_queue.append(json.dumps(entry, separators=(', ', ':'))) else: - self.event_queue.append(None) # Generate null event to update the display (with possibly out-of-date info) + self.event_queue.append(None) # Generate null event to update the display (with possibly out-of-date info) self.live = False # Watchdog thread - emitter = self.observed and self.observer._emitter_for_watch[self.observed] # Note: Uses undocumented attribute + emitter = self.observed and self.observer._emitter_for_watch[self.observed] # Note: Uses undocumented attribute while True: # Check whether new log file started, e.g. client (re)started. if emitter and emitter.is_alive(): - newlogfile = self.logfile # updated by on_created watchdog callback + newlogfile = self.logfile # updated by on_created watchdog callback else: # Poll try: @@ -276,9 +276,9 @@ class EDLogs(FileSystemEventHandler): loghandle.close() if logfile: - loghandle = open(logfile, 'rb', 0) # unbuffered + loghandle = open(logfile, 'rb', 0) # unbuffered if platform == 'darwin': - fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB + fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB logpos = 0 @@ -286,8 +286,8 @@ class EDLogs(FileSystemEventHandler): print('New logfile "%s"' % logfile) if logfile: - loghandle.seek(0, SEEK_END) # required to make macOS notice log change over SMB - loghandle.seek(logpos, SEEK_SET) # reset EOF flag + loghandle.seek(0, SEEK_END) # required to make macOS notice log change over SMB + loghandle.seek(logpos, SEEK_SET) # reset EOF flag for line in loghandle: self.event_queue.append(line) @@ -300,7 +300,7 @@ class EDLogs(FileSystemEventHandler): # Check whether we're still supposed to be running if threading.current_thread() != self.thread: - return # Terminate + return # Terminate if self.game_was_running: if not self.game_running(): @@ -314,11 +314,11 @@ class EDLogs(FileSystemEventHandler): def parse_entry(self, line): if line is None: - return { 'event': None } # Fake startup event + return { 'event': None } # Fake startup event try: - entry = json.loads(line, object_pairs_hook=OrderedDict) # Preserve property order because why not? - entry['timestamp'] # we expect this to exist + entry = json.loads(line, object_pairs_hook=OrderedDict) # Preserve property order because why not? + entry['timestamp'] # we expect this to exist if entry['event'] == 'Fileheader': self.live = False self.version = entry['gameversion'] @@ -362,11 +362,11 @@ class EDLogs(FileSystemEventHandler): } elif entry['event'] == 'Commander': - self.live = True # First event in 3.0 + self.live = True # First event in 3.0 elif entry['event'] == 'LoadGame': self.cmdr = entry['Commander'] - self.mode = entry.get('GameMode') # 'Open', 'Solo', 'Group', or None for CQC (and Training - but no LoadGame event) + self.mode = entry.get('GameMode') # 'Open', 'Solo', 'Group', or None for CQC (and Training - but no LoadGame event) self.group = entry.get('Group') self.planet = None self.system = None @@ -377,11 +377,11 @@ class EDLogs(FileSystemEventHandler): self.coordinates = None self.systemaddress = None self.started = timegm(strptime(entry['timestamp'], '%Y-%m-%dT%H:%M:%SZ')) - self.state.update({ # Don't set Ship, ShipID etc since this will reflect Fighter or SRV if starting in those + self.state.update({ # Don't set Ship, ShipID etc since this will reflect Fighter or SRV if starting in those 'Captain' : None, 'Credits' : entry['Credits'], - 'FID' : entry.get('FID'), # From 3.3 - 'Horizons' : entry['Horizons'], # From 3.0 + 'FID' : entry.get('FID'), # From 3.3 + 'Horizons' : entry['Horizons'], # From 3.0 'Loan' : entry['Loan'], 'Engineers' : {}, 'Rank' : {}, From d877de2758ad18e250ad4f9bf3d7b9d08bd85d36 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:01:10 +0200 Subject: [PATCH 006/258] Ensured all lines are under 120 characters wide --- monitor.py | 209 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 131 insertions(+), 78 deletions(-) diff --git a/monitor.py b/monitor.py index e99e9688..38339b38 100644 --- a/monitor.py +++ b/monitor.py @@ -90,29 +90,29 @@ class EDLogs(FileSystemEventHandler): # Cmdr state shared with EDSM and plugins # If you change anything here update PLUGINS.md documentation! self.state = { - 'Captain' : None, # On a crew - 'Cargo' : defaultdict(int), - 'Credits' : None, - 'FID' : None, # Frontier Cmdr ID - 'Horizons' : None, # Does this user have Horizons? - 'Loan' : None, - 'Raw' : defaultdict(int), - 'Manufactured' : defaultdict(int), - 'Encoded' : defaultdict(int), - 'Engineers' : {}, - 'Rank' : {}, - 'Reputation' : {}, - 'Statistics' : {}, - 'Role' : None, # Crew role - None, Idle, FireCon, FighterCon - 'Friends' : set(), # Online friends - 'ShipID' : None, - 'ShipIdent' : None, - 'ShipName' : None, - 'ShipType' : None, - 'HullValue' : None, - 'ModulesValue' : None, - 'Rebuy' : None, - 'Modules' : None, + 'Captain': None, # On a crew + 'Cargo': defaultdict(int), + 'Credits': None, + 'FID': None, # Frontier Cmdr ID + 'Horizons': None, # Does this user have Horizons? + 'Loan': None, + 'Raw': defaultdict(int), + 'Manufactured': defaultdict(int), + 'Encoded': defaultdict(int), + 'Engineers': {}, + 'Rank': {}, + 'Reputation': {}, + 'Statistics': {}, + 'Role': None, # Crew role - None, Idle, FireCon, FighterCon + 'Friends': set(), # Online friends + 'ShipID': None, + 'ShipIdent': None, + 'ShipName': None, + 'ShipType': None, + 'HullValue': None, + 'ModulesValue': None, + 'Rebuy': None, + 'Modules': None, } def start(self, root): @@ -131,8 +131,11 @@ class EDLogs(FileSystemEventHandler): # Latest pre-existing logfile - e.g. if E:D is already running. Assumes logs sort alphabetically. # Do this before setting up the observer in case the journal directory has gone away try: - logfiles = sorted([x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], - key=lambda x: x.split('.')[1:]) + logfiles = sorted( + [x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], + key=lambda x: x.split('.')[1:] + ) + self.logfile = logfiles and join(self.currentdir, logfiles[-1]) or None except: @@ -170,8 +173,20 @@ class EDLogs(FileSystemEventHandler): def stop(self): if __debug__: print('Stopping monitoring Journal') + self.currentdir = None - self.version = self.mode = self.group = self.cmdr = self.planet = self.system = self.station = self.station_marketid = self.stationtype = self.stationservices = self.coordinates = self.systemaddress = None + self.version = None + self.mode = None + self.group = None + self.cmdr = None + self.planet = None + self.system = None + self.station = None + self.station_marketid = None + self.stationtype = None + self.stationservices = None + self.coordinates = None + self.systemaddress = None self.is_beta = False if self.observed: self.observed = None @@ -193,7 +208,10 @@ class EDLogs(FileSystemEventHandler): def on_created(self, event): # watchdog callback, e.g. client (re)started. - if not event.is_directory and re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', basename(event.src_path)): + if not event.is_directory and re.search( + r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', basename(event.src_path) + ): + self.logfile = event.src_path def worker(self): @@ -247,7 +265,8 @@ class EDLogs(FileSystemEventHandler): self.event_queue.append(json.dumps(entry, separators=(', ', ':'))) else: - self.event_queue.append(None) # Generate null event to update the display (with possibly out-of-date info) + # Generate null event to update the display (with possibly out-of-date info) + self.event_queue.append(None) self.live = False # Watchdog thread @@ -261,8 +280,12 @@ class EDLogs(FileSystemEventHandler): else: # Poll try: - logfiles = sorted([x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], - key=lambda x: x.split('.')[1:]) + logfiles = sorted( + [x for x in listdir(self.currentdir) if + re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], + key=lambda x: x.split('.')[1:] + ) + newlogfile = logfiles and join(self.currentdir, logfiles[-1]) or None except: @@ -304,7 +327,10 @@ class EDLogs(FileSystemEventHandler): if self.game_was_running: if not self.game_running(): - self.event_queue.append('{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())) + self.event_queue.append( + '{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()) + ) + self.root.event_generate('<>', when="tail") self.game_was_running = False @@ -336,29 +362,29 @@ class EDLogs(FileSystemEventHandler): self.systemaddress = None self.started = None self.state = { - 'Captain' : None, - 'Cargo' : defaultdict(int), - 'Credits' : None, - 'FID' : None, - 'Horizons' : None, - 'Loan' : None, - 'Raw' : defaultdict(int), - 'Manufactured' : defaultdict(int), - 'Encoded' : defaultdict(int), - 'Engineers' : {}, - 'Rank' : {}, - 'Reputation' : {}, - 'Statistics' : {}, - 'Role' : None, - 'Friends' : set(), - 'ShipID' : None, - 'ShipIdent' : None, - 'ShipName' : None, - 'ShipType' : None, - 'HullValue' : None, - 'ModulesValue' : None, - 'Rebuy' : None, - 'Modules' : None, + 'Captain': None, + 'Cargo': defaultdict(int), + 'Credits': None, + 'FID': None, + 'Horizons': None, + 'Loan': None, + 'Raw': defaultdict(int), + 'Manufactured': defaultdict(int), + 'Encoded': defaultdict(int), + 'Engineers': {}, + 'Rank': {}, + 'Reputation': {}, + 'Statistics': {}, + 'Role': None, + 'Friends': set(), + 'ShipID': None, + 'ShipIdent': None, + 'ShipName': None, + 'ShipType': None, + 'HullValue': None, + 'ModulesValue': None, + 'Rebuy': None, + 'Modules': None, } elif entry['event'] == 'Commander': @@ -366,7 +392,8 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'LoadGame': self.cmdr = entry['Commander'] - self.mode = entry.get('GameMode') # 'Open', 'Solo', 'Group', or None for CQC (and Training - but no LoadGame event) + # 'Open', 'Solo', 'Group', or None for CQC (and Training - but no LoadGame event) + self.mode = entry.get('GameMode') self.group = entry.get('Group') self.planet = None self.system = None @@ -377,17 +404,18 @@ class EDLogs(FileSystemEventHandler): self.coordinates = None self.systemaddress = None self.started = timegm(strptime(entry['timestamp'], '%Y-%m-%dT%H:%M:%SZ')) - self.state.update({ # Don't set Ship, ShipID etc since this will reflect Fighter or SRV if starting in those - 'Captain' : None, - 'Credits' : entry['Credits'], - 'FID' : entry.get('FID'), # From 3.3 - 'Horizons' : entry['Horizons'], # From 3.0 - 'Loan' : entry['Loan'], - 'Engineers' : {}, - 'Rank' : {}, - 'Reputation' : {}, - 'Statistics' : {}, - 'Role' : None, + # Don't set Ship, ShipID etc since this will reflect Fighter or SRV if starting in those + self.state.update({ + 'Captain': None, + 'Credits': entry['Credits'], + 'FID': entry.get('FID'), # From 3.3 + 'Horizons': entry['Horizons'], # From 3.0 + 'Loan': entry['Loan'], + 'Engineers': {}, + 'Rank': {}, + 'Reputation': {}, + 'Statistics': {}, + 'Role': None, }) elif entry['event'] == 'NewCommander': @@ -521,7 +549,8 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'Progress': for k,v in entry.items(): if k in self.state['Rank']: - self.state['Rank'][k] = (self.state['Rank'][k][0], min(v, 100)) # perhaps not taken promotion mission yet + # perhaps not taken promotion mission yet + self.state['Rank'][k] = (self.state['Rank'][k][0], min(v, 100)) elif entry['event'] in ['Reputation', 'Statistics']: payload = OrderedDict(entry) @@ -531,14 +560,21 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'EngineerProgress': if 'Engineers' in entry: # Startup summary - self.state['Engineers'] = { e['Engineer']: (e['Rank'], e.get('RankProgress', 0)) if 'Rank' in e else e['Progress'] for e in entry['Engineers'] } + self.state['Engineers'] = { + e['Engineer']: (e['Rank'], e.get('RankProgress', 0)) + if 'Rank' in e else e['Progress'] for e in entry['Engineers'] + } - else: # Promotion - self.state['Engineers'][entry['Engineer']] = (entry['Rank'], entry.get('RankProgress', 0)) if 'Rank' in entry else entry['Progress'] + else: # Promotion + if 'Rank' in entry: + self.state['Engineers'][entry['Engineer']] = (entry['Rank'], entry.get('RankProgress', 0)) + else: + self.state['Engineers'][entry['Engineer']] = entry['Progress'] elif entry['event'] == 'Cargo' and entry.get('Vessel') == 'Ship': self.state['Cargo'] = defaultdict(int) - if 'Inventory' not in entry: # From 3.3 full Cargo event (after the first one) is written to a separate file + # From 3.3 full Cargo event (after the first one) is written to a separate file + if 'Inventory' not in entry: with open(join(self.currentdir, 'Cargo.json'), 'rb') as h: entry = json.load(h, object_pairs_hook=OrderedDict) # Preserve property order because why not? @@ -564,7 +600,9 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'Materials': for category in ['Raw', 'Manufactured', 'Encoded']: self.state[category] = defaultdict(int) - self.state[category].update({ self.canonicalise(x['Name']): x['Count'] for x in entry.get(category, []) }) + self.state[category].update({ + self.canonicalise(x['Name']): x['Count'] for x in entry.get(category, []) + }) elif entry['event'] == 'MaterialCollected': material = self.canonicalise(entry['Name']) @@ -593,7 +631,10 @@ class EDLogs(FileSystemEventHandler): category = self.category(entry['Received']['Category']) self.state[category][entry['Received']['Material']] += entry['Received']['Quantity'] - elif entry['event'] == 'EngineerCraft' or (entry['event'] == 'EngineerLegacyConvert' and not entry.get('IsPreview')): + elif entry['event'] == 'EngineerCraft' or ( + entry['event'] == 'EngineerLegacyConvert' and not entry.get('IsPreview') + ): + for category in ['Raw', 'Manufactured', 'Encoded']: for x in entry.get('Ingredients', []): material = self.canonicalise(x['Name']) @@ -711,8 +752,9 @@ class EDLogs(FileSystemEventHandler): return { 'event': None } - # Commodities, Modules and Ships can appear in different forms e.g. "$HNShockMount_Name;", "HNShockMount", and "hnshockmount", - # "$int_cargorack_size6_class1_name;" and "Int_CargoRack_Size6_Class1", "python" and "Python", etc. + # Commodities, Modules and Ships can appear in different forms e.g. "$HNShockMount_Name;", "HNShockMount", + # and "hnshockmount", "$int_cargorack_size6_class1_name;" and "Int_CargoRack_Size6_Class1", + # "python" and "Python", etc. # This returns a simple lowercased name e.g. 'hnshockmount', 'int_cargorack_size6_class1', 'python', etc def canonicalise(self, item): if not item: return '' @@ -759,7 +801,9 @@ class EDLogs(FileSystemEventHandler): self.event_queue.append(json.dumps(entry, separators=(', ', ':'))) elif self.live and entry['event'] == 'Music' and entry.get('MusicTrack') == 'MainMenu': - self.event_queue.append('{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())) + self.event_queue.append( + '{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()) + ) return entry @@ -798,7 +842,10 @@ class EDLogs(FileSystemEventHandler): if not self.state['Modules']: return None - standard_order = ['ShipCockpit', 'CargoHatch', 'Armour', 'PowerPlant', 'MainEngines', 'FrameShiftDrive', 'LifeSupport', 'PowerDistributor', 'Radar', 'FuelTank'] + standard_order = ( + 'ShipCockpit', 'CargoHatch', 'Armour', 'PowerPlant', 'MainEngines', 'FrameShiftDrive', 'LifeSupport', + 'PowerDistributor', 'Radar', 'FuelTank' + ) d = OrderedDict() if timestamped: @@ -817,7 +864,13 @@ class EDLogs(FileSystemEventHandler): # sort modules by slot - hardpoints, standard, internal d['Modules'] = [] - for slot in sorted(self.state['Modules'], key=lambda x: ('Hardpoint' not in x, x not in standard_order and len(standard_order) or standard_order.index(x), 'Slot' not in x, x)): + for slot in sorted( + self.state['Modules'], + key=lambda x: ( + 'Hardpoint' not in x, x not in standard_order and len(standard_order) or standard_order.index(x), + 'Slot' not in x, x) + ): + module = dict(self.state['Modules'][slot]) module.pop('Health', None) module.pop('Value', None) From aeb328b31f105d7856942dd62a3e1d96d22fb3a6 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:10:18 +0200 Subject: [PATCH 007/258] Removed oneliners --- monitor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/monitor.py b/monitor.py index 38339b38..c5de435b 100644 --- a/monitor.py +++ b/monitor.py @@ -289,7 +289,8 @@ class EDLogs(FileSystemEventHandler): newlogfile = logfiles and join(self.currentdir, logfiles[-1]) or None except: - if __debug__: print_exc() + if __debug__: + print_exc() newlogfile = None @@ -757,7 +758,9 @@ class EDLogs(FileSystemEventHandler): # "python" and "Python", etc. # This returns a simple lowercased name e.g. 'hnshockmount', 'int_cargorack_size6_class1', 'python', etc def canonicalise(self, item): - if not item: return '' + if not item: + return '' + item = item.lower() match = self._RE_CANONICALISE.match(item) return match and match.group(1) or item From bf74e3647fce92144ca6d11265194781ff424d3f Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:12:36 +0200 Subject: [PATCH 008/258] Fixed various whitespace issues --- monitor.py | 126 ++++++++++++++++++++++++++--------------------------- 1 file changed, 62 insertions(+), 64 deletions(-) diff --git a/monitor.py b/monitor.py index c5de435b..9f073faf 100644 --- a/monitor.py +++ b/monitor.py @@ -16,27 +16,27 @@ from config import config from companion import ship_file_name -if platform=='darwin': +if platform == 'darwin': from AppKit import NSWorkspace from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler from fcntl import fcntl F_GLOBAL_NOCACHE = 55 -elif platform=='win32': +elif platform == 'win32': from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler import ctypes from ctypes.wintypes import * - EnumWindows = ctypes.windll.user32.EnumWindows - EnumWindowsProc = ctypes.WINFUNCTYPE(BOOL, HWND, LPARAM) + EnumWindows = ctypes.windll.user32.EnumWindows + EnumWindowsProc = ctypes.WINFUNCTYPE(BOOL, HWND, LPARAM) - CloseHandle = ctypes.windll.kernel32.CloseHandle + CloseHandle = ctypes.windll.kernel32.CloseHandle - GetWindowText = ctypes.windll.user32.GetWindowTextW + GetWindowText = ctypes.windll.user32.GetWindowTextW GetWindowText.argtypes = [HWND, LPWSTR, ctypes.c_int] - GetWindowTextLength = ctypes.windll.user32.GetWindowTextLengthW + GetWindowTextLength = ctypes.windll.user32.GetWindowTextLengthW GetProcessHandleFromHwnd = ctypes.windll.oleacc.GetProcessHandleFromHwnd @@ -164,7 +164,7 @@ class EDLogs(FileSystemEventHandler): print('Start logfile "%s"' % self.logfile) if not self.running(): - self.thread = threading.Thread(target = self.worker, name = 'Journal worker') + self.thread = threading.Thread(target=self.worker, name='Journal worker') self.thread.daemon = True self.thread.start() @@ -173,7 +173,7 @@ class EDLogs(FileSystemEventHandler): def stop(self): if __debug__: print('Stopping monitoring Journal') - + self.currentdir = None self.version = None self.mode = None @@ -233,6 +233,7 @@ class EDLogs(FileSystemEventHandler): except: if __debug__: print('Invalid journal entry "%s"' % repr(line)) + logpos = loghandle.tell() else: @@ -331,17 +332,16 @@ class EDLogs(FileSystemEventHandler): self.event_queue.append( '{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()) ) - + self.root.event_generate('<>', when="tail") self.game_was_running = False else: self.game_was_running = self.game_running() - def parse_entry(self, line): if line is None: - return { 'event': None } # Fake startup event + return {'event': None} # Fake startup event try: entry = json.loads(line, object_pairs_hook=OrderedDict) # Preserve property order because why not? @@ -424,17 +424,17 @@ class EDLogs(FileSystemEventHandler): self.group = None elif entry['event'] == 'SetUserShipName': - self.state['ShipID'] = entry['ShipID'] - if 'UserShipId' in entry: # Only present when changing the ship's ident + self.state['ShipID'] = entry['ShipID'] + if 'UserShipId' in entry: # Only present when changing the ship's ident self.state['ShipIdent'] = entry['UserShipId'] - self.state['ShipName'] = entry.get('UserShipName') - self.state['ShipType'] = self.canonicalise(entry['Ship']) + self.state['ShipName'] = entry.get('UserShipName') + self.state['ShipType'] = self.canonicalise(entry['Ship']) elif entry['event'] == 'ShipyardBuy': self.state['ShipID'] = None self.state['ShipIdent'] = None - self.state['ShipName'] = None + self.state['ShipName'] = None self.state['ShipType'] = self.canonicalise(entry['ShipType']) self.state['HullValue'] = None self.state['ModulesValue'] = None @@ -444,7 +444,7 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'ShipyardSwap': self.state['ShipID'] = entry['ShipID'] self.state['ShipIdent'] = None - self.state['ShipName'] = None + self.state['ShipName'] = None self.state['ShipType'] = self.canonicalise(entry['ShipType']) self.state['HullValue'] = None self.state['ModulesValue'] = None @@ -465,9 +465,9 @@ class EDLogs(FileSystemEventHandler): if entry['ShipName'] and entry['ShipName'] not in ('', ' '): self.state['ShipName'] = entry['ShipName'] - self.state['ShipType'] = self.canonicalise(entry['Ship']) - self.state['HullValue'] = entry.get('HullValue') # not present on exiting Outfitting - self.state['ModulesValue'] = entry.get('ModulesValue') # " + self.state['ShipType'] = self.canonicalise(entry['Ship']) + self.state['HullValue'] = entry.get('HullValue') # not present on exiting Outfitting + self.state['ModulesValue'] = entry.get('ModulesValue') # " self.state['Rebuy'] = entry.get('Rebuy') # Remove spurious differences between initial Loadout event and subsequent self.state['Modules'] = {} @@ -476,7 +476,7 @@ class EDLogs(FileSystemEventHandler): module['Item'] = self.canonicalise(module['Item']) if ('Hardpoint' in module['Slot'] and not module['Slot'].startswith('TinyHardpoint') and - module.get('AmmoInClip') == module.get('AmmoInHopper') == 1): # lasers + module.get('AmmoInClip') == module.get('AmmoInHopper') == 1): # lasers module.pop('AmmoInClip') module.pop('AmmoInHopper') @@ -484,12 +484,12 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'ModuleBuy': self.state['Modules'][entry['Slot']] = { - 'Slot' : entry['Slot'], - 'Item' : self.canonicalise(entry['BuyItem']), - 'On' : True, - 'Priority' : 1, - 'Health' : 1.0, - 'Value' : entry['BuyPrice'], + 'Slot': entry['Slot'], + 'Item': self.canonicalise(entry['BuyItem']), + 'On': True, + 'Priority': 1, + 'Health': 1.0, + 'Value': entry['BuyPrice'], } elif entry['event'] == 'ModuleSell': @@ -521,7 +521,7 @@ class EDLogs(FileSystemEventHandler): self.coordinates = tuple(entry['StarPos']) elif self.system != entry['StarSystem']: - self.coordinates = None # Docked event doesn't include coordinates + self.coordinates = None # Docked event doesn't include coordinates self.systemaddress = entry.get('SystemAddress') @@ -529,10 +529,10 @@ class EDLogs(FileSystemEventHandler): self.systempopulation = entry.get('Population') (self.system, self.station) = (entry['StarSystem'] == 'ProvingGround' and 'CQC' or entry['StarSystem'], - entry.get('StationName')) # May be None - self.station_marketid = entry.get('MarketID') # May be None - self.stationtype = entry.get('StationType') # May be None - self.stationservices = entry.get('StationServices') # None under E:D < 2.4 + entry.get('StationName')) # May be None + self.station_marketid = entry.get('MarketID') # May be None + self.stationtype = entry.get('StationType') # May be None + self.stationservices = entry.get('StationServices') # None under E:D < 2.4 elif entry['event'] == 'ApproachBody': self.planet = entry['Body'] @@ -544,11 +544,11 @@ class EDLogs(FileSystemEventHandler): payload = dict(entry) payload.pop('event') payload.pop('timestamp') - for k,v in payload.items(): - self.state['Rank'][k] = (v,0) + for k, v in payload.items(): + self.state['Rank'][k] = (v, 0) elif entry['event'] == 'Progress': - for k,v in entry.items(): + for k, v in entry.items(): if k in self.state['Rank']: # perhaps not taken promotion mission yet self.state['Rank'][k] = (self.state['Rank'][k][0], min(v, 100)) @@ -560,7 +560,7 @@ class EDLogs(FileSystemEventHandler): self.state[entry['event']] = payload elif entry['event'] == 'EngineerProgress': - if 'Engineers' in entry: # Startup summary + if 'Engineers' in entry: # Startup summary self.state['Engineers'] = { e['Engineer']: (e['Rank'], e.get('RankProgress', 0)) if 'Rank' in e else e['Progress'] for e in entry['Engineers'] @@ -577,9 +577,9 @@ class EDLogs(FileSystemEventHandler): # From 3.3 full Cargo event (after the first one) is written to a separate file if 'Inventory' not in entry: with open(join(self.currentdir, 'Cargo.json'), 'rb') as h: - entry = json.load(h, object_pairs_hook=OrderedDict) # Preserve property order because why not? + entry = json.load(h, object_pairs_hook=OrderedDict) # Preserve property order because why not? - self.state['Cargo'].update({ self.canonicalise(x['Name']): x['Count'] for x in entry['Inventory'] }) + self.state['Cargo'].update({self.canonicalise(x['Name']): x['Count'] for x in entry['Inventory']}) elif entry['event'] in ['CollectCargo', 'MarketBuy', 'BuyDrones', 'MiningRefined']: commodity = self.canonicalise(entry['Type']) @@ -647,14 +647,14 @@ class EDLogs(FileSystemEventHandler): module = self.state['Modules'][entry['Slot']] assert(module['Item'] == self.canonicalise(entry['Module'])) module['Engineering'] = { - 'Engineer' : entry['Engineer'], - 'EngineerID' : entry['EngineerID'], - 'BlueprintName' : entry['BlueprintName'], - 'BlueprintID' : entry['BlueprintID'], - 'Level' : entry['Level'], - 'Quality' : entry['Quality'], - 'Modifiers' : entry['Modifiers'], - } + 'Engineer': entry['Engineer'], + 'EngineerID': entry['EngineerID'], + 'BlueprintName': entry['BlueprintName'], + 'BlueprintID': entry['BlueprintID'], + 'Level': entry['Level'], + 'Quality': entry['Quality'], + 'Modifiers': entry['Modifiers'], + } if 'ExperimentalEffect' in entry: module['Engineering']['ExperimentalEffect'] = entry['ExperimentalEffect'] @@ -670,7 +670,7 @@ class EDLogs(FileSystemEventHandler): self.state['Cargo'][commodity] += reward.get('Count', 1) for reward in entry.get('MaterialsReward', []): - if 'Category' in reward: # Category not present in E:D 3.0 + if 'Category' in reward: # Category not present in E:D 3.0 category = self.category(reward['Category']) material = self.canonicalise(reward['Name']) self.state[category][material] += reward.get('Count', 1) @@ -691,7 +691,7 @@ class EDLogs(FileSystemEventHandler): self.state[category].pop(material) elif entry['event'] == 'TechnologyBroker': - for thing in entry.get('Ingredients', []): # 3.01 + for thing in entry.get('Ingredients', []): # 3.01 for category in ['Cargo', 'Raw', 'Manufactured', 'Encoded']: item = self.canonicalise(thing['Name']) if item in self.state[category]: @@ -699,13 +699,13 @@ class EDLogs(FileSystemEventHandler): if self.state[category][item] <= 0: self.state[category].pop(item) - for thing in entry.get('Commodities', []): # 3.02 + for thing in entry.get('Commodities', []): # 3.02 commodity = self.canonicalise(thing['Name']) self.state['Cargo'][commodity] -= thing['Count'] if self.state['Cargo'][commodity] <= 0: self.state['Cargo'].pop(commodity) - for thing in entry.get('Materials', []): # 3.02 + for thing in entry.get('Materials', []): # 3.02 material = self.canonicalise(thing['Name']) category = thing['Category'] self.state[category][material] -= thing['Count'] @@ -751,14 +751,14 @@ class EDLogs(FileSystemEventHandler): print('Invalid journal entry "%s"' % repr(line)) print_exc() - return { 'event': None } + return {'event': None} - # Commodities, Modules and Ships can appear in different forms e.g. "$HNShockMount_Name;", "HNShockMount", - # and "hnshockmount", "$int_cargorack_size6_class1_name;" and "Int_CargoRack_Size6_Class1", + # Commodities, Modules and Ships can appear in different forms e.g. "$HNShockMount_Name;", "HNShockMount", + # and "hnshockmount", "$int_cargorack_size6_class1_name;" and "Int_CargoRack_Size6_Class1", # "python" and "Python", etc. # This returns a simple lowercased name e.g. 'hnshockmount', 'int_cargorack_size6_class1', 'python', etc def canonicalise(self, item): - if not item: + if not item: return '' item = item.lower() @@ -829,9 +829,9 @@ class EDLogs(FileSystemEventHandler): name = WindowTitle(hWnd) if name and name.startswith('Elite - Dangerous'): handle = GetProcessHandleFromHwnd(hWnd) - if handle: # If GetProcessHandleFromHwnd succeeds then the app is already running as this user + if handle: # If GetProcessHandleFromHwnd succeeds then the app is already running as this user CloseHandle(handle) - return False # stop enumeration + return False # stop enumeration return True @@ -839,14 +839,13 @@ class EDLogs(FileSystemEventHandler): return False - # Return a subset of the received data describing the current ship as a Loadout event def ship(self, timestamped=True): if not self.state['Modules']: return None standard_order = ( - 'ShipCockpit', 'CargoHatch', 'Armour', 'PowerPlant', 'MainEngines', 'FrameShiftDrive', 'LifeSupport', + 'ShipCockpit', 'CargoHatch', 'Armour', 'PowerPlant', 'MainEngines', 'FrameShiftDrive', 'LifeSupport', 'PowerDistributor', 'Radar', 'FuelTank' ) @@ -870,9 +869,9 @@ class EDLogs(FileSystemEventHandler): for slot in sorted( self.state['Modules'], key=lambda x: ( - 'Hardpoint' not in x, x not in standard_order and len(standard_order) or standard_order.index(x), + 'Hardpoint' not in x, x not in standard_order and len(standard_order) or standard_order.index(x), 'Slot' not in x, x) - ): + ): module = dict(self.state['Modules'][slot]) module.pop('Health', None) @@ -881,10 +880,9 @@ class EDLogs(FileSystemEventHandler): return d - # Export ship loadout as a Loadout event def export_ship(self, filename=None): - string = json.dumps(self.ship(False), ensure_ascii=False, indent=2, separators=(',', ': ')) # pretty print + string = json.dumps(self.ship(False), ensure_ascii=False, indent=2, separators=(',', ': ')) # pretty print if filename: with open(filename, 'wt') as h: h.write(string) @@ -897,7 +895,7 @@ class EDLogs(FileSystemEventHandler): if oldfiles: with open(join(config.get('outdir'), oldfiles[-1]), 'rU') as h: if h.read() == string: - return # same as last time - don't write + return # same as last time - don't write # Write filename = join(config.get('outdir'), '%s.%s.txt' % (ship, strftime('%Y-%m-%dT%H.%M.%S', localtime(time())))) From f95bfd4280e3d2de0083158f2c7c10b07ad9f708 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:18:43 +0200 Subject: [PATCH 009/258] removed bare except clauses --- monitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitor.py b/monitor.py index 9f073faf..6e377f49 100644 --- a/monitor.py +++ b/monitor.py @@ -138,7 +138,7 @@ class EDLogs(FileSystemEventHandler): self.logfile = logfiles and join(self.currentdir, logfiles[-1]) or None - except: + except Exception: self.logfile = None return False @@ -230,7 +230,7 @@ class EDLogs(FileSystemEventHandler): try: self.parse_entry(line) # Some events are of interest even in the past - except: + except Exception: if __debug__: print('Invalid journal entry "%s"' % repr(line)) @@ -289,7 +289,7 @@ class EDLogs(FileSystemEventHandler): newlogfile = logfiles and join(self.currentdir, logfiles[-1]) or None - except: + except Exception: if __debug__: print_exc() @@ -746,7 +746,7 @@ class EDLogs(FileSystemEventHandler): self.state['Friends'].discard(entry['Name']) return entry - except: + except Exception: if __debug__: print('Invalid journal entry "%s"' % repr(line)) print_exc() From 1963140572ddcc9f6cccb50499e11184a936a82f Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:23:40 +0200 Subject: [PATCH 010/258] removed star import --- monitor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monitor.py b/monitor.py index 6e377f49..78e43fee 100644 --- a/monitor.py +++ b/monitor.py @@ -27,7 +27,7 @@ elif platform == 'win32': from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler import ctypes - from ctypes.wintypes import * + from ctypes.wintypes import BOOL, HWND, LPARAM, LPWSTR EnumWindows = ctypes.windll.user32.EnumWindows EnumWindowsProc = ctypes.WINFUNCTYPE(BOOL, HWND, LPARAM) @@ -454,7 +454,6 @@ class EDLogs(FileSystemEventHandler): elif (entry['event'] == 'Loadout' and not 'fighter' in self.canonicalise(entry['Ship']) and not 'buggy' in self.canonicalise(entry['Ship'])): - self.state['ShipID'] = entry['ShipID'] self.state['ShipIdent'] = entry['ShipIdent'] From 33f25da270b810d1a381da6154e52be0ae928405 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:31:46 +0200 Subject: [PATCH 011/258] Fixed ambiguous names and logic --- monitor.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/monitor.py b/monitor.py index 78e43fee..90de7ad6 100644 --- a/monitor.py +++ b/monitor.py @@ -234,7 +234,7 @@ class EDLogs(FileSystemEventHandler): if __debug__: print('Invalid journal entry "%s"' % repr(line)) - logpos = loghandle.tell() + log_pos = loghandle.tell() else: loghandle = None @@ -305,21 +305,21 @@ class EDLogs(FileSystemEventHandler): if platform == 'darwin': fcntl(loghandle, F_GLOBAL_NOCACHE, -1) # required to avoid corruption on macOS over SMB - logpos = 0 + log_pos = 0 if __debug__: print('New logfile "%s"' % logfile) if logfile: loghandle.seek(0, SEEK_END) # required to make macOS notice log change over SMB - loghandle.seek(logpos, SEEK_SET) # reset EOF flag + loghandle.seek(log_pos, SEEK_SET) # reset EOF flag for line in loghandle: self.event_queue.append(line) if self.event_queue: self.root.event_generate('<>', when="tail") - logpos = loghandle.tell() + log_pos = loghandle.tell() sleep(self._POLL) @@ -452,8 +452,8 @@ class EDLogs(FileSystemEventHandler): self.state['Modules'] = None elif (entry['event'] == 'Loadout' and - not 'fighter' in self.canonicalise(entry['Ship']) and - not 'buggy' in self.canonicalise(entry['Ship'])): + 'fighter' not in self.canonicalise(entry['Ship']) and + 'buggy' not in self.canonicalise(entry['Ship'])): self.state['ShipID'] = entry['ShipID'] self.state['ShipIdent'] = entry['ShipIdent'] @@ -495,10 +495,10 @@ class EDLogs(FileSystemEventHandler): self.state['Modules'].pop(entry['Slot'], None) elif entry['event'] == 'ModuleSwap': - toitem = self.state['Modules'].get(entry['ToSlot']) + to_item = self.state['Modules'].get(entry['ToSlot']) self.state['Modules'][entry['ToSlot']] = self.state['Modules'][entry['FromSlot']] - if toitem: - self.state['Modules'][entry['FromSlot']] = toitem + if to_item: + self.state['Modules'][entry['FromSlot']] = to_item else: self.state['Modules'].pop(entry['FromSlot'], None) @@ -818,9 +818,9 @@ class EDLogs(FileSystemEventHandler): elif platform == 'win32': def WindowTitle(h): if h: - l = GetWindowTextLength(h) + 1 - buf = ctypes.create_unicode_buffer(l) - if GetWindowText(h, buf, l): + length = GetWindowTextLength(h) + 1 + buf = ctypes.create_unicode_buffer(length) + if GetWindowText(h, buf, length): return buf.value return None From 05e6d498801c49dd64c451f13d4af25b71a97802 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:33:30 +0200 Subject: [PATCH 012/258] replaced list comp with generator where possible --- monitor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitor.py b/monitor.py index 90de7ad6..186c1ea5 100644 --- a/monitor.py +++ b/monitor.py @@ -132,7 +132,7 @@ class EDLogs(FileSystemEventHandler): # Do this before setting up the observer in case the journal directory has gone away try: logfiles = sorted( - [x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], + (x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)), key=lambda x: x.split('.')[1:] ) @@ -282,8 +282,8 @@ class EDLogs(FileSystemEventHandler): # Poll try: logfiles = sorted( - [x for x in listdir(self.currentdir) if - re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], + (x for x in listdir(self.currentdir) if + re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)), key=lambda x: x.split('.')[1:] ) @@ -890,7 +890,7 @@ class EDLogs(FileSystemEventHandler): ship = ship_file_name(self.state['ShipName'], self.state['ShipType']) regexp = re.compile(re.escape(ship) + r'\.\d{4}\-\d\d\-\d\dT\d\d\.\d\d\.\d\d\.txt') - oldfiles = sorted([x for x in listdir(config.get('outdir')) if regexp.match(x)]) + oldfiles = sorted((x for x in listdir(config.get('outdir')) if regexp.match(x))) if oldfiles: with open(join(config.get('outdir'), oldfiles[-1]), 'rU') as h: if h.read() == string: From ed45d59af1e54ef8db599b646aea372d7abcdee5 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 18:44:05 +0200 Subject: [PATCH 013/258] Replaced modulo formatters with .format formatters --- monitor.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/monitor.py b/monitor.py index 186c1ea5..8a39da9b 100644 --- a/monitor.py +++ b/monitor.py @@ -160,8 +160,8 @@ class EDLogs(FileSystemEventHandler): self.observed = self.observer.schedule(self, self.currentdir) if __debug__: - print('%s Journal "%s"' % (polling and 'Polling' or 'Monitoring', self.currentdir)) - print('Start logfile "%s"' % self.logfile) + print('{} Journal {!r}'.format(polling and 'Polling' or 'Monitoring', self.currentdir)) + print('Start logfile {!r}'.format(self.logfile)) if not self.running(): self.thread = threading.Thread(target=self.worker, name='Journal worker') @@ -232,7 +232,7 @@ class EDLogs(FileSystemEventHandler): except Exception: if __debug__: - print('Invalid journal entry "%s"' % repr(line)) + print('Invalid journal entry {!r}'.format(line)) log_pos = loghandle.tell() @@ -308,7 +308,7 @@ class EDLogs(FileSystemEventHandler): log_pos = 0 if __debug__: - print('New logfile "%s"' % logfile) + print('New logfile {!r}'.format(logfile)) if logfile: loghandle.seek(0, SEEK_END) # required to make macOS notice log change over SMB @@ -330,7 +330,7 @@ class EDLogs(FileSystemEventHandler): if self.game_was_running: if not self.game_running(): self.event_queue.append( - '{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()) + '{{ "timestamp":"{}", "event":"ShutDown" }}'.format(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())) ) self.root.event_generate('<>', when="tail") @@ -747,7 +747,7 @@ class EDLogs(FileSystemEventHandler): return entry except Exception: if __debug__: - print('Invalid journal entry "%s"' % repr(line)) + print('Invalid journal entry {!r}'.format(line)) print_exc() return {'event': None} @@ -804,7 +804,7 @@ class EDLogs(FileSystemEventHandler): elif self.live and entry['event'] == 'Music' and entry.get('MusicTrack') == 'MainMenu': self.event_queue.append( - '{ "timestamp":"%s", "event":"ShutDown" }' % strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()) + '{{ "timestamp":"{}", "event":"ShutDown" }}'.format(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())) ) return entry @@ -897,7 +897,9 @@ class EDLogs(FileSystemEventHandler): return # same as last time - don't write # Write - filename = join(config.get('outdir'), '%s.%s.txt' % (ship, strftime('%Y-%m-%dT%H.%M.%S', localtime(time())))) + filename = join( + config.get('outdir'), '{}.{}.txt'.format(ship, strftime('%Y-%m-%dT%H.%M.%S', localtime(time()))) + ) with open(filename, 'wt') as h: h.write(string) From b010b8015d793073610b0d0eb35e88c4bc68bae8 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 19 Jul 2020 19:08:36 +0200 Subject: [PATCH 014/258] Aligned values of large dicts helps your eyes track the changes --- monitor.py | 87 +++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/monitor.py b/monitor.py index 8a39da9b..b4b2a264 100644 --- a/monitor.py +++ b/monitor.py @@ -90,29 +90,29 @@ class EDLogs(FileSystemEventHandler): # Cmdr state shared with EDSM and plugins # If you change anything here update PLUGINS.md documentation! self.state = { - 'Captain': None, # On a crew - 'Cargo': defaultdict(int), - 'Credits': None, - 'FID': None, # Frontier Cmdr ID - 'Horizons': None, # Does this user have Horizons? - 'Loan': None, - 'Raw': defaultdict(int), + 'Captain': None, # On a crew + 'Cargo': defaultdict(int), + 'Credits': None, + 'FID': None, # Frontier Cmdr ID + 'Horizons': None, # Does this user have Horizons? + 'Loan': None, + 'Raw': defaultdict(int), 'Manufactured': defaultdict(int), - 'Encoded': defaultdict(int), - 'Engineers': {}, - 'Rank': {}, - 'Reputation': {}, - 'Statistics': {}, - 'Role': None, # Crew role - None, Idle, FireCon, FighterCon - 'Friends': set(), # Online friends - 'ShipID': None, - 'ShipIdent': None, - 'ShipName': None, - 'ShipType': None, - 'HullValue': None, + 'Encoded': defaultdict(int), + 'Engineers': {}, + 'Rank': {}, + 'Reputation': {}, + 'Statistics': {}, + 'Role': None, # Crew role - None, Idle, FireCon, FighterCon + 'Friends': set(), # Online friends + 'ShipID': None, + 'ShipIdent': None, + 'ShipName': None, + 'ShipType': None, + 'HullValue': None, 'ModulesValue': None, - 'Rebuy': None, - 'Modules': None, + 'Rebuy': None, + 'Modules': None, } def start(self, root): @@ -363,29 +363,29 @@ class EDLogs(FileSystemEventHandler): self.systemaddress = None self.started = None self.state = { - 'Captain': None, - 'Cargo': defaultdict(int), - 'Credits': None, - 'FID': None, - 'Horizons': None, - 'Loan': None, - 'Raw': defaultdict(int), + 'Captain': None, + 'Cargo': defaultdict(int), + 'Credits': None, + 'FID': None, + 'Horizons': None, + 'Loan': None, + 'Raw': defaultdict(int), 'Manufactured': defaultdict(int), - 'Encoded': defaultdict(int), - 'Engineers': {}, - 'Rank': {}, - 'Reputation': {}, - 'Statistics': {}, - 'Role': None, - 'Friends': set(), - 'ShipID': None, - 'ShipIdent': None, - 'ShipName': None, - 'ShipType': None, - 'HullValue': None, + 'Encoded': defaultdict(int), + 'Engineers': {}, + 'Rank': {}, + 'Reputation': {}, + 'Statistics': {}, + 'Role': None, + 'Friends': set(), + 'ShipID': None, + 'ShipIdent': None, + 'ShipName': None, + 'ShipType': None, + 'HullValue': None, 'ModulesValue': None, - 'Rebuy': None, - 'Modules': None, + 'Rebuy': None, + 'Modules': None, } elif entry['event'] == 'Commander': @@ -868,7 +868,8 @@ class EDLogs(FileSystemEventHandler): for slot in sorted( self.state['Modules'], key=lambda x: ( - 'Hardpoint' not in x, x not in standard_order and len(standard_order) or standard_order.index(x), + 'Hardpoint' not in x, + x not in standard_order and len(standard_order) or standard_order.index(x), 'Slot' not in x, x) ): From a6d6599c3bce49bf0a43cea01e0d526e79cb31de Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 20 Jul 2020 11:05:03 +0200 Subject: [PATCH 015/258] Moved logfile regexp to class level constant Regular expressions are expensive to recompile constantly, and while the python regexp library currently caches reuse, it only does so to a point and that is not a required behaviour. Compiling regexps once is simply best practice. On top of this, the regexp was duplicated in various places. --- monitor.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/monitor.py b/monitor.py index b4b2a264..7de3e97f 100644 --- a/monitor.py +++ b/monitor.py @@ -51,6 +51,7 @@ class EDLogs(FileSystemEventHandler): _POLL = 1 # Polling is cheap, so do it often _RE_CANONICALISE = re.compile(r'\$(.+)_name;') _RE_CATEGORY = re.compile(r'\$MICRORESOURCE_CATEGORY_(.+);') + _RE_LOGFILE = re.compile(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$') def __init__(self): FileSystemEventHandler.__init__(self) # futureproofing - not need for current version of watchdog @@ -132,7 +133,7 @@ class EDLogs(FileSystemEventHandler): # Do this before setting up the observer in case the journal directory has gone away try: logfiles = sorted( - (x for x in listdir(self.currentdir) if re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)), + (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), key=lambda x: x.split('.')[1:] ) @@ -208,9 +209,7 @@ class EDLogs(FileSystemEventHandler): def on_created(self, event): # watchdog callback, e.g. client (re)started. - if not event.is_directory and re.search( - r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', basename(event.src_path) - ): + if not event.is_directory and self._RE_LOGFILE.search(basename(event.src_path)): self.logfile = event.src_path @@ -282,8 +281,7 @@ class EDLogs(FileSystemEventHandler): # Poll try: logfiles = sorted( - (x for x in listdir(self.currentdir) if - re.search(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)), + (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), key=lambda x: x.split('.')[1:] ) From 40aa4f9e630a221a7a2b0db74edf9c03dc7b0907 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 20 Jul 2020 17:45:00 +0200 Subject: [PATCH 016/258] Added whitespace on scope changes --- monitor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/monitor.py b/monitor.py index 7de3e97f..d7030e77 100644 --- a/monitor.py +++ b/monitor.py @@ -131,7 +131,7 @@ class EDLogs(FileSystemEventHandler): # Latest pre-existing logfile - e.g. if E:D is already running. Assumes logs sort alphabetically. # Do this before setting up the observer in case the journal directory has gone away - try: + try: # TODO: This should be replaced with something specific ONLY wrapping listdir logfiles = sorted( (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), key=lambda x: x.split('.')[1:] @@ -566,6 +566,7 @@ class EDLogs(FileSystemEventHandler): else: # Promotion if 'Rank' in entry: self.state['Engineers'][entry['Engineer']] = (entry['Rank'], entry.get('RankProgress', 0)) + else: self.state['Engineers'][entry['Engineer']] = entry['Progress'] @@ -626,6 +627,7 @@ class EDLogs(FileSystemEventHandler): self.state[category][entry['Paid']['Material']] -= entry['Paid']['Quantity'] if self.state[category][entry['Paid']['Material']] <= 0: self.state[category].pop(entry['Paid']['Material']) + category = self.category(entry['Received']['Category']) self.state[category][entry['Received']['Material']] += entry['Received']['Quantity'] @@ -739,6 +741,7 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'Friends': if entry['Status'] in ['Online', 'Added']: self.state['Friends'].add(entry['Name']) + else: self.state['Friends'].discard(entry['Name']) @@ -899,6 +902,7 @@ class EDLogs(FileSystemEventHandler): filename = join( config.get('outdir'), '{}.{}.txt'.format(ship, strftime('%Y-%m-%dT%H.%M.%S', localtime(time()))) ) + with open(filename, 'wt') as h: h.write(string) From 85c27e4cd76562fb7978143f3a4811bd16453420 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 21 Jul 2020 10:54:31 +0200 Subject: [PATCH 017/258] Replaced or-based ternaries with standard ones --- monitor.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/monitor.py b/monitor.py index d7030e77..9f63e524 100644 --- a/monitor.py +++ b/monitor.py @@ -136,8 +136,8 @@ class EDLogs(FileSystemEventHandler): (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), key=lambda x: x.split('.')[1:] ) - - self.logfile = logfiles and join(self.currentdir, logfiles[-1]) or None + + self.logfile = join(self.currentdir, logfiles[-1]) if logfiles else None except Exception: self.logfile = None @@ -161,7 +161,7 @@ class EDLogs(FileSystemEventHandler): self.observed = self.observer.schedule(self, self.currentdir) if __debug__: - print('{} Journal {!r}'.format(polling and 'Polling' or 'Monitoring', self.currentdir)) + print('{} Journal {!r}'.format('Polling' if polling else 'Monitoring', self.currentdir)) print('Start logfile {!r}'.format(self.logfile)) if not self.running(): @@ -285,7 +285,7 @@ class EDLogs(FileSystemEventHandler): key=lambda x: x.split('.')[1:] ) - newlogfile = logfiles and join(self.currentdir, logfiles[-1]) or None + newlogfile = join(self.currentdir, logfiles[-1]) if logfiles else None except Exception: if __debug__: @@ -310,7 +310,7 @@ class EDLogs(FileSystemEventHandler): if logfile: loghandle.seek(0, SEEK_END) # required to make macOS notice log change over SMB - loghandle.seek(log_pos, SEEK_SET) # reset EOF flag + loghandle.seek(log_pos, SEEK_SET) # reset EOF flag # TODO: log_pos reported as possibly unbound for line in loghandle: self.event_queue.append(line) @@ -525,8 +525,8 @@ class EDLogs(FileSystemEventHandler): if entry['event'] in ['Location', 'FSDJump', 'CarrierJump']: self.systempopulation = entry.get('Population') - (self.system, self.station) = (entry['StarSystem'] == 'ProvingGround' and 'CQC' or entry['StarSystem'], - entry.get('StationName')) # May be None + self.system = 'CQC' if entry['StarSystem'] == 'ProvingGround' else entry['StarSystem'] + self.station = entry.get('StationName') # May be None self.station_marketid = entry.get('MarketID') # May be None self.stationtype = entry.get('StationType') # May be None self.stationservices = entry.get('StationServices') # None under E:D < 2.4 @@ -541,8 +541,8 @@ class EDLogs(FileSystemEventHandler): payload = dict(entry) payload.pop('event') payload.pop('timestamp') - for k, v in payload.items(): - self.state['Rank'][k] = (v, 0) + + self.state['Rank'].update({k: (v, 0) for k, v in payload.items()}) elif entry['event'] == 'Progress': for k, v in entry.items(): @@ -757,17 +757,25 @@ class EDLogs(FileSystemEventHandler): # and "hnshockmount", "$int_cargorack_size6_class1_name;" and "Int_CargoRack_Size6_Class1", # "python" and "Python", etc. # This returns a simple lowercased name e.g. 'hnshockmount', 'int_cargorack_size6_class1', 'python', etc - def canonicalise(self, item): + def canonicalise(self, item: str): if not item: return '' item = item.lower() match = self._RE_CANONICALISE.match(item) - return match and match.group(1) or item - def category(self, item): + if match: + return match.group(1) + + return item + + def category(self, item: str): match = self._RE_CATEGORY.match(item) - return (match and match.group(1) or item).capitalize() + + if match: + return match.group(1).capitalize() + + return item.capitalize() def get_entry(self): if not self.event_queue: @@ -870,8 +878,10 @@ class EDLogs(FileSystemEventHandler): self.state['Modules'], key=lambda x: ( 'Hardpoint' not in x, - x not in standard_order and len(standard_order) or standard_order.index(x), - 'Slot' not in x, x) + len(standard_order) if x not in standard_order else standard_order.index(x), + 'Slot' not in x, + x + ) ): module = dict(self.state['Modules'][slot]) From 12fdbd067864b6d7b4c12e26f463a9404dbcfc57 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 21 Jul 2020 11:00:33 +0200 Subject: [PATCH 018/258] Replaced x in list with x in tuple Tuples are immutable, so this ensures that there isn't any funny business at runtime --- monitor.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/monitor.py b/monitor.py index 9f63e524..79991027 100644 --- a/monitor.py +++ b/monitor.py @@ -501,13 +501,13 @@ class EDLogs(FileSystemEventHandler): else: self.state['Modules'].pop(entry['FromSlot'], None) - elif entry['event'] in ['Undocked']: + elif entry['event'] == 'Undocked': self.station = None self.station_marketid = None self.stationtype = None self.stationservices = None - elif entry['event'] in ['Location', 'FSDJump', 'Docked', 'CarrierJump']: + elif entry['event'] in ('Location', 'FSDJump', 'Docked', 'CarrierJump'): if entry['event'] in ('Location', 'CarrierJump'): self.planet = entry.get('Body') if entry.get('BodyType') == 'Planet' else None @@ -522,7 +522,7 @@ class EDLogs(FileSystemEventHandler): self.systemaddress = entry.get('SystemAddress') - if entry['event'] in ['Location', 'FSDJump', 'CarrierJump']: + if entry['event'] in ('Location', 'FSDJump', 'CarrierJump'): self.systempopulation = entry.get('Population') self.system = 'CQC' if entry['StarSystem'] == 'ProvingGround' else entry['StarSystem'] @@ -534,10 +534,10 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'ApproachBody': self.planet = entry['Body'] - elif entry['event'] in ['LeaveBody', 'SupercruiseEntry']: + elif entry['event'] in ('LeaveBody', 'SupercruiseEntry'): self.planet = None - elif entry['event'] in ['Rank', 'Promotion']: + elif entry['event'] in ('Rank', 'Promotion'): payload = dict(entry) payload.pop('event') payload.pop('timestamp') @@ -550,7 +550,7 @@ class EDLogs(FileSystemEventHandler): # perhaps not taken promotion mission yet self.state['Rank'][k] = (self.state['Rank'][k][0], min(v, 100)) - elif entry['event'] in ['Reputation', 'Statistics']: + elif entry['event'] in ('Reputation', 'Statistics'): payload = OrderedDict(entry) payload.pop('event') payload.pop('timestamp') @@ -579,11 +579,11 @@ class EDLogs(FileSystemEventHandler): self.state['Cargo'].update({self.canonicalise(x['Name']): x['Count'] for x in entry['Inventory']}) - elif entry['event'] in ['CollectCargo', 'MarketBuy', 'BuyDrones', 'MiningRefined']: + elif entry['event'] in ('CollectCargo', 'MarketBuy', 'BuyDrones', 'MiningRefined'): commodity = self.canonicalise(entry['Type']) self.state['Cargo'][commodity] += entry.get('Count', 1) - elif entry['event'] in ['EjectCargo', 'MarketSell', 'SellDrones']: + elif entry['event'] in ('EjectCargo', 'MarketSell', 'SellDrones'): commodity = self.canonicalise(entry['Type']) self.state['Cargo'][commodity] -= entry.get('Count', 1) if self.state['Cargo'][commodity] <= 0: @@ -597,7 +597,7 @@ class EDLogs(FileSystemEventHandler): self.state['Cargo'].pop(commodity) elif entry['event'] == 'Materials': - for category in ['Raw', 'Manufactured', 'Encoded']: + for category in ('Raw', 'Manufactured', 'Encoded'): self.state[category] = defaultdict(int) self.state[category].update({ self.canonicalise(x['Name']): x['Count'] for x in entry.get(category, []) @@ -607,14 +607,14 @@ class EDLogs(FileSystemEventHandler): material = self.canonicalise(entry['Name']) self.state[entry['Category']][material] += entry['Count'] - elif entry['event'] in ['MaterialDiscarded', 'ScientificResearch']: + elif entry['event'] in ('MaterialDiscarded', 'ScientificResearch'): material = self.canonicalise(entry['Name']) self.state[entry['Category']][material] -= entry['Count'] if self.state[entry['Category']][material] <= 0: self.state[entry['Category']].pop(material) elif entry['event'] == 'Synthesis': - for category in ['Raw', 'Manufactured', 'Encoded']: + for category in ('Raw', 'Manufactured', 'Encoded'): for x in entry['Materials']: material = self.canonicalise(x['Name']) if material in self.state[category]: @@ -635,7 +635,7 @@ class EDLogs(FileSystemEventHandler): entry['event'] == 'EngineerLegacyConvert' and not entry.get('IsPreview') ): - for category in ['Raw', 'Manufactured', 'Encoded']: + for category in ('Raw', 'Manufactured', 'Encoded'): for x in entry.get('Ingredients', []): material = self.canonicalise(x['Name']) if material in self.state[category]: @@ -683,7 +683,7 @@ class EDLogs(FileSystemEventHandler): material = self.canonicalise(entry.get('Material')) if material: - for category in ['Raw', 'Manufactured', 'Encoded']: + for category in ('Raw', 'Manufactured', 'Encoded'): if material in self.state[category]: self.state[category][material] -= entry['Quantity'] if self.state[category][material] <= 0: @@ -691,7 +691,7 @@ class EDLogs(FileSystemEventHandler): elif entry['event'] == 'TechnologyBroker': for thing in entry.get('Ingredients', []): # 3.01 - for category in ['Cargo', 'Raw', 'Manufactured', 'Encoded']: + for category in ('Cargo', 'Raw', 'Manufactured', 'Encoded'): item = self.canonicalise(thing['Name']) if item in self.state[category]: self.state[category][item] -= thing['Count'] @@ -739,7 +739,7 @@ class EDLogs(FileSystemEventHandler): self.systemaddress = None elif entry['event'] == 'Friends': - if entry['Status'] in ['Online', 'Added']: + if entry['Status'] in ('Online', 'Added'): self.state['Friends'].add(entry['Name']) else: @@ -783,7 +783,7 @@ class EDLogs(FileSystemEventHandler): else: entry = self.parse_entry(self.event_queue.pop(0)) - if not self.live and entry['event'] not in [None, 'Fileheader']: + if not self.live and entry['event'] not in (None, 'Fileheader'): # Game not running locally, but Journal has been updated self.live = True if self.station: From 21ab456e2266d70fc1b6dd758c2c24ba2b321e4f Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 21 Jul 2020 12:10:12 +0200 Subject: [PATCH 019/258] Added variables for repeatedly indexed keys Repeatedly indexing keys is in general slow. And, not only is it slow, it makes reading code hell. --- monitor.py | 144 +++++++++++++++++++++++++++++------------------------ 1 file changed, 80 insertions(+), 64 deletions(-) diff --git a/monitor.py b/monitor.py index 79991027..ac04abf0 100644 --- a/monitor.py +++ b/monitor.py @@ -8,6 +8,7 @@ from os.path import basename, expanduser, isdir, join from sys import platform from time import gmtime, localtime, sleep, strftime, strptime, time from calendar import timegm +from typing import Any, Dict if __debug__: from traceback import print_exc @@ -136,7 +137,7 @@ class EDLogs(FileSystemEventHandler): (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), key=lambda x: x.split('.')[1:] ) - + self.logfile = join(self.currentdir, logfiles[-1]) if logfiles else None except Exception: @@ -342,9 +343,11 @@ class EDLogs(FileSystemEventHandler): return {'event': None} # Fake startup event try: - entry = json.loads(line, object_pairs_hook=OrderedDict) # Preserve property order because why not? + entry: Dict[str, Any] = json.loads(line, object_pairs_hook=OrderedDict) # Preserve property order because why not? entry['timestamp'] # we expect this to exist - if entry['event'] == 'Fileheader': + + event_type = entry['event'] + if event_type == 'Fileheader': self.live = False self.version = entry['gameversion'] self.is_beta = 'beta' in entry['gameversion'].lower() @@ -386,10 +389,10 @@ class EDLogs(FileSystemEventHandler): 'Modules': None, } - elif entry['event'] == 'Commander': + elif event_type == 'Commander': self.live = True # First event in 3.0 - elif entry['event'] == 'LoadGame': + elif event_type == 'LoadGame': self.cmdr = entry['Commander'] # 'Open', 'Solo', 'Group', or None for CQC (and Training - but no LoadGame event) self.mode = entry.get('GameMode') @@ -417,11 +420,11 @@ class EDLogs(FileSystemEventHandler): 'Role': None, }) - elif entry['event'] == 'NewCommander': + elif event_type == 'NewCommander': self.cmdr = entry['Name'] self.group = None - elif entry['event'] == 'SetUserShipName': + elif event_type == 'SetUserShipName': self.state['ShipID'] = entry['ShipID'] if 'UserShipId' in entry: # Only present when changing the ship's ident self.state['ShipIdent'] = entry['UserShipId'] @@ -429,7 +432,7 @@ class EDLogs(FileSystemEventHandler): self.state['ShipName'] = entry.get('UserShipName') self.state['ShipType'] = self.canonicalise(entry['Ship']) - elif entry['event'] == 'ShipyardBuy': + elif event_type == 'ShipyardBuy': self.state['ShipID'] = None self.state['ShipIdent'] = None self.state['ShipName'] = None @@ -439,7 +442,7 @@ class EDLogs(FileSystemEventHandler): self.state['Rebuy'] = None self.state['Modules'] = None - elif entry['event'] == 'ShipyardSwap': + elif event_type == 'ShipyardSwap': self.state['ShipID'] = entry['ShipID'] self.state['ShipIdent'] = None self.state['ShipName'] = None @@ -449,7 +452,7 @@ class EDLogs(FileSystemEventHandler): self.state['Rebuy'] = None self.state['Modules'] = None - elif (entry['event'] == 'Loadout' and + elif (event_type == 'Loadout' and 'fighter' not in self.canonicalise(entry['Ship']) and 'buggy' not in self.canonicalise(entry['Ship'])): self.state['ShipID'] = entry['ShipID'] @@ -479,7 +482,7 @@ class EDLogs(FileSystemEventHandler): self.state['Modules'][module['Slot']] = module - elif entry['event'] == 'ModuleBuy': + elif event_type == 'ModuleBuy': self.state['Modules'][entry['Slot']] = { 'Slot': entry['Slot'], 'Item': self.canonicalise(entry['BuyItem']), @@ -489,29 +492,32 @@ class EDLogs(FileSystemEventHandler): 'Value': entry['BuyPrice'], } - elif entry['event'] == 'ModuleSell': + elif event_type == 'ModuleSell': self.state['Modules'].pop(entry['Slot'], None) - elif entry['event'] == 'ModuleSwap': + elif event_type == 'ModuleSwap': to_item = self.state['Modules'].get(entry['ToSlot']) - self.state['Modules'][entry['ToSlot']] = self.state['Modules'][entry['FromSlot']] + to_slot = entry['ToSlot'] + from_slot = entry['FromSlot'] + modules = self.state['Modules'] + modules[to_slot] = modules[from_slot] if to_item: - self.state['Modules'][entry['FromSlot']] = to_item + modules[from_slot] = to_item else: - self.state['Modules'].pop(entry['FromSlot'], None) + modules.pop(from_slot, None) - elif entry['event'] == 'Undocked': + elif event_type == 'Undocked': self.station = None self.station_marketid = None self.stationtype = None self.stationservices = None - elif entry['event'] in ('Location', 'FSDJump', 'Docked', 'CarrierJump'): - if entry['event'] in ('Location', 'CarrierJump'): + elif event_type in ('Location', 'FSDJump', 'Docked', 'CarrierJump'): + if event_type in ('Location', 'CarrierJump'): self.planet = entry.get('Body') if entry.get('BodyType') == 'Planet' else None - elif entry['event'] == 'FSDJump': + elif event_type == 'FSDJump': self.planet = None if 'StarPos' in entry: @@ -522,7 +528,7 @@ class EDLogs(FileSystemEventHandler): self.systemaddress = entry.get('SystemAddress') - if entry['event'] in ('Location', 'FSDJump', 'CarrierJump'): + if event_type in ('Location', 'FSDJump', 'CarrierJump'): self.systempopulation = entry.get('Population') self.system = 'CQC' if entry['StarSystem'] == 'ProvingGround' else entry['StarSystem'] @@ -531,32 +537,34 @@ class EDLogs(FileSystemEventHandler): self.stationtype = entry.get('StationType') # May be None self.stationservices = entry.get('StationServices') # None under E:D < 2.4 - elif entry['event'] == 'ApproachBody': + elif event_type == 'ApproachBody': self.planet = entry['Body'] - elif entry['event'] in ('LeaveBody', 'SupercruiseEntry'): + elif event_type in ('LeaveBody', 'SupercruiseEntry'): self.planet = None - elif entry['event'] in ('Rank', 'Promotion'): + elif event_type in ('Rank', 'Promotion'): payload = dict(entry) payload.pop('event') payload.pop('timestamp') self.state['Rank'].update({k: (v, 0) for k, v in payload.items()}) - elif entry['event'] == 'Progress': + elif event_type == 'Progress': + rank = self.state['Rank'] for k, v in entry.items(): - if k in self.state['Rank']: + if k in rank: # perhaps not taken promotion mission yet - self.state['Rank'][k] = (self.state['Rank'][k][0], min(v, 100)) + rank[k] = (rank[k][0], min(v, 100)) - elif entry['event'] in ('Reputation', 'Statistics'): + elif event_type in ('Reputation', 'Statistics'): payload = OrderedDict(entry) payload.pop('event') payload.pop('timestamp') - self.state[entry['event']] = payload + self.state[event_type] = payload - elif entry['event'] == 'EngineerProgress': + elif event_type == 'EngineerProgress': + engineers = self.state['Engineers'] if 'Engineers' in entry: # Startup summary self.state['Engineers'] = { e['Engineer']: (e['Rank'], e.get('RankProgress', 0)) @@ -564,13 +572,14 @@ class EDLogs(FileSystemEventHandler): } else: # Promotion + engineer = entry['Engineer'] if 'Rank' in entry: - self.state['Engineers'][entry['Engineer']] = (entry['Rank'], entry.get('RankProgress', 0)) + engineers[engineer] = (entry['Rank'], entry.get('RankProgress', 0)) else: - self.state['Engineers'][entry['Engineer']] = entry['Progress'] + engineers[engineer] = entry['Progress'] - elif entry['event'] == 'Cargo' and entry.get('Vessel') == 'Ship': + elif event_type == 'Cargo' and entry.get('Vessel') == 'Ship': self.state['Cargo'] = defaultdict(int) # From 3.3 full Cargo event (after the first one) is written to a separate file if 'Inventory' not in entry: @@ -579,41 +588,44 @@ class EDLogs(FileSystemEventHandler): self.state['Cargo'].update({self.canonicalise(x['Name']): x['Count'] for x in entry['Inventory']}) - elif entry['event'] in ('CollectCargo', 'MarketBuy', 'BuyDrones', 'MiningRefined'): + elif event_type in ('CollectCargo', 'MarketBuy', 'BuyDrones', 'MiningRefined'): commodity = self.canonicalise(entry['Type']) self.state['Cargo'][commodity] += entry.get('Count', 1) - elif entry['event'] in ('EjectCargo', 'MarketSell', 'SellDrones'): + elif event_type in ('EjectCargo', 'MarketSell', 'SellDrones'): commodity = self.canonicalise(entry['Type']) - self.state['Cargo'][commodity] -= entry.get('Count', 1) - if self.state['Cargo'][commodity] <= 0: - self.state['Cargo'].pop(commodity) + cargo = self.state['Cargo'] + cargo[commodity] -= entry.get('Count', 1) + if cargo[commodity] <= 0: + cargo.pop(commodity) - elif entry['event'] == 'SearchAndRescue': + elif event_type == 'SearchAndRescue': for item in entry.get('Items', []): commodity = self.canonicalise(item['Name']) - self.state['Cargo'][commodity] -= item.get('Count', 1) - if self.state['Cargo'][commodity] <= 0: - self.state['Cargo'].pop(commodity) + cargo = self.state['Cargo'] + cargo[commodity] -= item.get('Count', 1) + if cargo[commodity] <= 0: + cargo.pop(commodity) - elif entry['event'] == 'Materials': + elif event_type == 'Materials': for category in ('Raw', 'Manufactured', 'Encoded'): self.state[category] = defaultdict(int) self.state[category].update({ self.canonicalise(x['Name']): x['Count'] for x in entry.get(category, []) }) - elif entry['event'] == 'MaterialCollected': + elif event_type == 'MaterialCollected': material = self.canonicalise(entry['Name']) self.state[entry['Category']][material] += entry['Count'] - elif entry['event'] in ('MaterialDiscarded', 'ScientificResearch'): + elif event_type in ('MaterialDiscarded', 'ScientificResearch'): material = self.canonicalise(entry['Name']) - self.state[entry['Category']][material] -= entry['Count'] - if self.state[entry['Category']][material] <= 0: - self.state[entry['Category']].pop(material) + state_category = self.state[entry['Category']] + state_category[material] -= entry['Count'] + if state_category[material] <= 0: + state_category.pop(material) - elif entry['event'] == 'Synthesis': + elif event_type == 'Synthesis': for category in ('Raw', 'Manufactured', 'Encoded'): for x in entry['Materials']: material = self.canonicalise(x['Name']) @@ -622,17 +634,21 @@ class EDLogs(FileSystemEventHandler): if self.state[category][material] <= 0: self.state[category].pop(material) - elif entry['event'] == 'MaterialTrade': + elif event_type == 'MaterialTrade': category = self.category(entry['Paid']['Category']) - self.state[category][entry['Paid']['Material']] -= entry['Paid']['Quantity'] - if self.state[category][entry['Paid']['Material']] <= 0: - self.state[category].pop(entry['Paid']['Material']) + state_category = self.state[category] + paid = entry['Paid'] + received = entry['Received'] - category = self.category(entry['Received']['Category']) - self.state[category][entry['Received']['Material']] += entry['Received']['Quantity'] + state_category[paid['Material']] -= paid['Quantity'] + if state_category[paid['Material']] <= 0: + state_category.pop(paid['Material']) - elif entry['event'] == 'EngineerCraft' or ( - entry['event'] == 'EngineerLegacyConvert' and not entry.get('IsPreview') + category = self.category(received['Category']) + state_category[received['Material']] += received['Quantity'] + + elif event_type == 'EngineerCraft' or ( + event_type == 'EngineerLegacyConvert' and not entry.get('IsPreview') ): for category in ('Raw', 'Manufactured', 'Encoded'): @@ -663,7 +679,7 @@ class EDLogs(FileSystemEventHandler): module['Engineering'].pop('ExperimentalEffect', None) module['Engineering'].pop('ExperimentalEffect_Localised', None) - elif entry['event'] == 'MissionCompleted': + elif event_type == 'MissionCompleted': for reward in entry.get('CommodityReward', []): commodity = self.canonicalise(reward['Name']) self.state['Cargo'][commodity] += reward.get('Count', 1) @@ -674,7 +690,7 @@ class EDLogs(FileSystemEventHandler): material = self.canonicalise(reward['Name']) self.state[category][material] += reward.get('Count', 1) - elif entry['event'] == 'EngineerContribution': + elif event_type == 'EngineerContribution': commodity = self.canonicalise(entry.get('Commodity')) if commodity: self.state['Cargo'][commodity] -= entry['Quantity'] @@ -689,7 +705,7 @@ class EDLogs(FileSystemEventHandler): if self.state[category][material] <= 0: self.state[category].pop(material) - elif entry['event'] == 'TechnologyBroker': + elif event_type == 'TechnologyBroker': for thing in entry.get('Ingredients', []): # 3.01 for category in ('Cargo', 'Raw', 'Manufactured', 'Encoded'): item = self.canonicalise(thing['Name']) @@ -711,7 +727,7 @@ class EDLogs(FileSystemEventHandler): if self.state[category][material] <= 0: self.state[category].pop(material) - elif entry['event'] == 'JoinACrew': + elif event_type == 'JoinACrew': self.state['Captain'] = entry['Captain'] self.state['Role'] = 'Idle' self.planet = None @@ -723,10 +739,10 @@ class EDLogs(FileSystemEventHandler): self.coordinates = None self.systemaddress = None - elif entry['event'] == 'ChangeCrewRole': + elif event_type == 'ChangeCrewRole': self.state['Role'] = entry['Role'] - elif entry['event'] == 'QuitACrew': + elif event_type == 'QuitACrew': self.state['Captain'] = None self.state['Role'] = None self.planet = None @@ -738,7 +754,7 @@ class EDLogs(FileSystemEventHandler): self.coordinates = None self.systemaddress = None - elif entry['event'] == 'Friends': + elif event_type == 'Friends': if entry['Status'] in ('Online', 'Added'): self.state['Friends'].add(entry['Name']) From 9a482cb04bcb46e345f9248f7d51a488fba39411 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 14:40:08 +0200 Subject: [PATCH 020/258] Added type annotations where needed Not everywhere because they can be inferred in a lot of places. But I added them to a lot of the self.* variables --- monitor.py | 93 +++++++++++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/monitor.py b/monitor.py index ac04abf0..c5a9a41d 100644 --- a/monitor.py +++ b/monitor.py @@ -8,7 +8,10 @@ from os.path import basename, expanduser, isdir, join from sys import platform from time import gmtime, localtime, sleep, strftime, strptime, time from calendar import timegm -from typing import Any, Dict +from typing import Any, Dict, List, Optional, OrderedDict as OrderedDictT, Tuple, TYPE_CHECKING + +if TYPE_CHECKING: + import tkinter if __debug__: from traceback import print_exc @@ -47,7 +50,8 @@ else: # Journal handler -class EDLogs(FileSystemEventHandler): +class EDLogs(FileSystemEventHandler): # type: ignore # See below + # Magic with FileSystemEventHandler can confuse type checkers when they do not have access to every import _POLL = 1 # Polling is cheap, so do it often _RE_CANONICALISE = re.compile(r'\$(.+)_name;') @@ -55,13 +59,14 @@ class EDLogs(FileSystemEventHandler): _RE_LOGFILE = re.compile(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$') def __init__(self): + # TODO(A_D): A bunch of these should be switched to default values (eg '' for strings) and no longer be Optional FileSystemEventHandler.__init__(self) # futureproofing - not need for current version of watchdog self.root = None - self.currentdir = None # The actual logdir that we're monitoring - self.logfile = None + self.currentdir: Optional[str] = None # The actual logdir that we're monitoring + self.logfile: Optional[str] = None self.observer = None self.observed = None # a watchdog ObservedWatch, or None if polling - self.thread = None + self.thread: Optional[threading.Thread] = None self.event_queue = [] # For communicating journal entries back to main thread # On startup we might be: @@ -75,19 +80,19 @@ class EDLogs(FileSystemEventHandler): self.game_was_running = False # For generation the "ShutDown" event # Context for journal handling - self.version = None + self.version: Optional[str] = None self.is_beta = False - self.mode = None - self.group = None - self.cmdr = None - self.planet = None - self.system = None - self.station = None - self.station_marketid = None - self.stationtype = None - self.coordinates = None - self.systemaddress = None - self.started = None # Timestamp of the LoadGame event + self.mode: Optional[str] = None + self.group: Optional[str] = None + self.cmdr: Optional[str] = None + self.planet: Optional[str] = None + self.system: Optional[str] = None + self.station: Optional[str] = None + self.station_marketid: Optional[int] = None + self.stationtype: Optional[str] = None + self.coordinates: Optional[Tuple[int, int, int]] = None + self.systemaddress: Optional[int] = None + self.started: Optional[int] = None # Timestamp of the LoadGame event # Cmdr state shared with EDSM and plugins # If you change anything here update PLUGINS.md documentation! @@ -117,10 +122,9 @@ class EDLogs(FileSystemEventHandler): 'Modules': None, } - def start(self, root): + def start(self, root: 'tkinter.Tk'): self.root = root - logdir = expanduser(config.get('journaldir') or config.default_journal_dir) # type: ignore # config is weird - + logdir: str = config.get('journaldir') or config.default_journal_dir # type: ignore # config does weird things if not logdir or not isdir(logdir): # type: ignore # config does weird things in its get self.stop() return False @@ -134,11 +138,11 @@ class EDLogs(FileSystemEventHandler): # Do this before setting up the observer in case the journal directory has gone away try: # TODO: This should be replaced with something specific ONLY wrapping listdir logfiles = sorted( - (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), + (x for x in listdir(self.currentdir) if self._RE_LOGFILE.search(x)), # type: ignore # config is weird key=lambda x: x.split('.')[1:] ) - self.logfile = join(self.currentdir, logfiles[-1]) if logfiles else None + self.logfile = join(self.currentdir, logfiles[-1]) if logfiles else None # type: ignore # config is weird except Exception: self.logfile = None @@ -244,7 +248,7 @@ class EDLogs(FileSystemEventHandler): if self.live: if self.game_was_running: # Game is running locally - entry = OrderedDict([ + entry: OrderedDictT[str, Any] = OrderedDict([ ('timestamp', strftime('%Y-%m-%dT%H:%M:%SZ', gmtime())), ('event', 'StartUp'), ('StarSystem', self.system), @@ -286,7 +290,7 @@ class EDLogs(FileSystemEventHandler): key=lambda x: x.split('.')[1:] ) - newlogfile = join(self.currentdir, logfiles[-1]) if logfiles else None + newlogfile = join(self.currentdir, logfiles[-1]) if logfiles else None # type: ignore except Exception: if __debug__: @@ -339,11 +343,13 @@ class EDLogs(FileSystemEventHandler): self.game_was_running = self.game_running() def parse_entry(self, line): + # TODO(A_D): a bunch of these can be simplified to use if itertools.product and filters if line is None: return {'event': None} # Fake startup event try: - entry: Dict[str, Any] = json.loads(line, object_pairs_hook=OrderedDict) # Preserve property order because why not? + # Preserve property order because why not? + entry: OrderedDictT[str, Any] = json.loads(line, object_pairs_hook=OrderedDict) entry['timestamp'] # we expect this to exist event_type = entry['event'] @@ -355,11 +361,11 @@ class EDLogs(FileSystemEventHandler): self.mode = None self.group = None self.planet = None - self.system = None - self.station = None - self.station_marketid = None - self.stationtype = None - self.stationservices = None + self.system: Optional[str] = None + self.station: Optional[str] = None + self.station_marketid: Optional[int] = None + self.stationtype: Optional[str] = None + self.stationservices: Optional[List[str]] = None self.coordinates = None self.systemaddress = None self.started = None @@ -408,16 +414,16 @@ class EDLogs(FileSystemEventHandler): self.started = timegm(strptime(entry['timestamp'], '%Y-%m-%dT%H:%M:%SZ')) # Don't set Ship, ShipID etc since this will reflect Fighter or SRV if starting in those self.state.update({ - 'Captain': None, - 'Credits': entry['Credits'], - 'FID': entry.get('FID'), # From 3.3 - 'Horizons': entry['Horizons'], # From 3.0 - 'Loan': entry['Loan'], - 'Engineers': {}, - 'Rank': {}, + 'Captain': None, + 'Credits': entry['Credits'], + 'FID': entry.get('FID'), # From 3.3 + 'Horizons': entry['Horizons'], # From 3.0 + 'Loan': entry['Loan'], + 'Engineers': {}, + 'Rank': {}, 'Reputation': {}, 'Statistics': {}, - 'Role': None, + 'Role': None, }) elif event_type == 'NewCommander': @@ -583,7 +589,7 @@ class EDLogs(FileSystemEventHandler): self.state['Cargo'] = defaultdict(int) # From 3.3 full Cargo event (after the first one) is written to a separate file if 'Inventory' not in entry: - with open(join(self.currentdir, 'Cargo.json'), 'rb') as h: + with open(join(self.currentdir, 'Cargo.json'), 'rb') as h: # type: ignore entry = json.load(h, object_pairs_hook=OrderedDict) # Preserve property order because why not? self.state['Cargo'].update({self.canonicalise(x['Name']): x['Count'] for x in entry['Inventory']}) @@ -773,7 +779,7 @@ class EDLogs(FileSystemEventHandler): # and "hnshockmount", "$int_cargorack_size6_class1_name;" and "Int_CargoRack_Size6_Class1", # "python" and "Python", etc. # This returns a simple lowercased name e.g. 'hnshockmount', 'int_cargorack_size6_class1', 'python', etc - def canonicalise(self, item: str): + def canonicalise(self, item: Optional[str]): if not item: return '' @@ -909,6 +915,7 @@ class EDLogs(FileSystemEventHandler): # Export ship loadout as a Loadout event def export_ship(self, filename=None): + # TODO(A_D): Some type checking has been disabled in here due to config.get getting weird outputs string = json.dumps(self.ship(False), ensure_ascii=False, indent=2, separators=(',', ': ')) # pretty print if filename: with open(filename, 'wt') as h: @@ -918,14 +925,14 @@ class EDLogs(FileSystemEventHandler): ship = ship_file_name(self.state['ShipName'], self.state['ShipType']) regexp = re.compile(re.escape(ship) + r'\.\d{4}\-\d\d\-\d\dT\d\d\.\d\d\.\d\d\.txt') - oldfiles = sorted((x for x in listdir(config.get('outdir')) if regexp.match(x))) + oldfiles = sorted((x for x in listdir(config.get('outdir')) if regexp.match(x))) # type: ignore if oldfiles: - with open(join(config.get('outdir'), oldfiles[-1]), 'rU') as h: + with open(join(config.get('outdir'), oldfiles[-1]), 'rU') as h: # type: ignore if h.read() == string: return # same as last time - don't write # Write - filename = join( + filename = join( # type: ignore config.get('outdir'), '{}.{}.txt'.format(ship, strftime('%Y-%m-%dT%H.%M.%S', localtime(time()))) ) From d9658f2cf9a5562fbe17ce9ddd8c304c6226e9b3 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 14:46:18 +0200 Subject: [PATCH 021/258] Fixed alignment and comment grammar --- monitor.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/monitor.py b/monitor.py index c5a9a41d..552968a2 100644 --- a/monitor.py +++ b/monitor.py @@ -77,7 +77,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below # If 3 we need to inject a special 'StartUp' event since consumers won't see the LoadGame event. self.live = False - self.game_was_running = False # For generation the "ShutDown" event + self.game_was_running = False # For generation of the "ShutDown" event # Context for journal handling self.version: Optional[str] = None @@ -342,7 +342,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below else: self.game_was_running = self.game_running() - def parse_entry(self, line): + def parse_entry(self, line: str): # TODO(A_D): a bunch of these can be simplified to use if itertools.product and filters if line is None: return {'event': None} # Fake startup event @@ -350,7 +350,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below try: # Preserve property order because why not? entry: OrderedDictT[str, Any] = json.loads(line, object_pairs_hook=OrderedDict) - entry['timestamp'] # we expect this to exist + entry['timestamp'] # we expect this to exist # TODO: replace with assert? or an if key in check event_type = entry['event'] if event_type == 'Fileheader': @@ -361,11 +361,11 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.mode = None self.group = None self.planet = None - self.system: Optional[str] = None - self.station: Optional[str] = None - self.station_marketid: Optional[int] = None - self.stationtype: Optional[str] = None - self.stationservices: Optional[List[str]] = None + self.system = None + self.station = None + self.station_marketid = None + self.stationtype = None + self.stationservices = None self.coordinates = None self.systemaddress = None self.started = None @@ -490,12 +490,12 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below elif event_type == 'ModuleBuy': self.state['Modules'][entry['Slot']] = { - 'Slot': entry['Slot'], - 'Item': self.canonicalise(entry['BuyItem']), - 'On': True, + 'Slot': entry['Slot'], + 'Item': self.canonicalise(entry['BuyItem']), + 'On': True, 'Priority': 1, - 'Health': 1.0, - 'Value': entry['BuyPrice'], + 'Health': 1.0, + 'Value': entry['BuyPrice'], } elif event_type == 'ModuleSell': @@ -668,13 +668,13 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below module = self.state['Modules'][entry['Slot']] assert(module['Item'] == self.canonicalise(entry['Module'])) module['Engineering'] = { - 'Engineer': entry['Engineer'], - 'EngineerID': entry['EngineerID'], + 'Engineer': entry['Engineer'], + 'EngineerID': entry['EngineerID'], 'BlueprintName': entry['BlueprintName'], - 'BlueprintID': entry['BlueprintID'], - 'Level': entry['Level'], - 'Quality': entry['Quality'], - 'Modifiers': entry['Modifiers'], + 'BlueprintID': entry['BlueprintID'], + 'Level': entry['Level'], + 'Quality': entry['Quality'], + 'Modifiers': entry['Modifiers'], } if 'ExperimentalEffect' in entry: From ecfed2b558e2f5251d4ca04c3027f5228d36c0f9 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 14:56:11 +0200 Subject: [PATCH 022/258] Modified dict comprehension to be more clear --- monitor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitor.py b/monitor.py index 552968a2..ae2f707d 100644 --- a/monitor.py +++ b/monitor.py @@ -573,8 +573,8 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below engineers = self.state['Engineers'] if 'Engineers' in entry: # Startup summary self.state['Engineers'] = { - e['Engineer']: (e['Rank'], e.get('RankProgress', 0)) - if 'Rank' in e else e['Progress'] for e in entry['Engineers'] + e['Engineer']: ((e['Rank'], e.get('RankProgress', 0)) if 'Rank' in e else e['Progress']) + for e in entry['Engineers'] } else: # Promotion From 179e06d6e94ce71a80fb3356e33b3c9ab2814b80 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 18:03:32 +0200 Subject: [PATCH 023/258] added informational comment on private access --- monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monitor.py b/monitor.py index ae2f707d..dc1165a4 100644 --- a/monitor.py +++ b/monitor.py @@ -274,7 +274,8 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.event_queue.append(None) self.live = False - # Watchdog thread + # Watchdog thread -- there is a way to get this by using self.observer.emitters and checking for an attribute: + # watch, but that may have unforseen differences in behaviour. emitter = self.observed and self.observer._emitter_for_watch[self.observed] # Note: Uses undocumented attribute while True: From d67dd173678463595f972be7a079f774b858c52a Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 14:22:44 +0200 Subject: [PATCH 024/258] ignore editor and virtual enviroment directories Just to ensure that they're not reported by any local tools --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index e6e08332..a1e314a8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ appcast_win_*.xml appcast_mac_*.xml EDMarketConnector.VisualElementsManifest.xml *.zip + +.idea +.vscode +venv From c6e61cc3da64c4774b3dc15c03f7d6446ee2749e Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 15:13:17 +0200 Subject: [PATCH 025/258] Fix ~ not being expanded on linux --- monitor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monitor.py b/monitor.py index dc1165a4..875891e9 100644 --- a/monitor.py +++ b/monitor.py @@ -8,7 +8,7 @@ from os.path import basename, expanduser, isdir, join from sys import platform from time import gmtime, localtime, sleep, strftime, strptime, time from calendar import timegm -from typing import Any, Dict, List, Optional, OrderedDict as OrderedDictT, Tuple, TYPE_CHECKING +from typing import Any, Optional, OrderedDict as OrderedDictT, Tuple, TYPE_CHECKING if TYPE_CHECKING: import tkinter @@ -124,7 +124,8 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below def start(self, root: 'tkinter.Tk'): self.root = root - logdir: str = config.get('journaldir') or config.default_journal_dir # type: ignore # config does weird things + logdir = expanduser(config.get('journaldir') or config.default_journal_dir) # type: ignore # config is weird + if not logdir or not isdir(logdir): # type: ignore # config does weird things in its get self.stop() return False From 1f1e34d7226b48a3b9661ea8d45c5b8738e3cb33 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:01:06 +0200 Subject: [PATCH 026/258] ensure that comments have two spaces before most of the files have tabs between code and comments --- EDMC.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/EDMC.py b/EDMC.py index 1fa37f35..b928f153 100755 --- a/EDMC.py +++ b/EDMC.py @@ -36,9 +36,10 @@ sys.path.append(config.internal_plugin_dir) import eddn -SERVER_RETRY = 5 # retry pause for Companion servers [s] +SERVER_RETRY = 5 # retry pause for Companion servers [s] EXIT_SUCCESS, EXIT_SERVER, EXIT_CREDENTIALS, EXIT_VERIFICATION, EXIT_LAGGING, EXIT_SYS_ERR = range(6) + # quick and dirty version comparison assuming "strict" numeric only version numbers def versioncmp(versionstring): return list(map(int, versionstring.split('.'))) @@ -58,7 +59,7 @@ try: parser.add_argument('-d', metavar='FILE', help='write raw JSON data to FILE') parser.add_argument('-n', action='store_true', help='send data to EDDN') parser.add_argument('-p', metavar='CMDR', help='Returns data from the specified player account') - parser.add_argument('-j', help=argparse.SUPPRESS) # Import JSON dump + parser.add_argument('-j', help=argparse.SUPPRESS) # Import JSON dump args = parser.parse_args() if args.version: @@ -124,14 +125,14 @@ try: sys.stderr.write('Who are you?!\n') sys.exit(EXIT_SERVER) elif (not data.get('lastSystem', {}).get('name') or - (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked + (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked sys.stderr.write('Where are you?!\n') # Shouldn't happen sys.exit(EXIT_SERVER) elif not data.get('ship') or not data['ship'].get('modules') or not data['ship'].get('name','').strip(): - sys.stderr.write('What are you flying?!\n') # Shouldn't happen + sys.stderr.write('What are you flying?!\n') # Shouldn't happen sys.exit(EXIT_SERVER) elif args.j: - pass # Skip further validation + pass # Skip further validation elif data['commander']['name'] != monitor.cmdr: sys.stderr.write('Wrong Cmdr\n') # Companion API return doesn't match Journal sys.exit(EXIT_CREDENTIALS) @@ -167,7 +168,7 @@ try: elif not data.get('lastStarport', {}).get('name'): sys.stderr.write("Unknown station!\n") sys.exit(EXIT_LAGGING) - elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): # Ignore possibly missing shipyard info + elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): # Ignore possibly missing shipyard info sys.stderr.write("Station doesn't have anything!\n") sys.exit(EXIT_SUCCESS) else: @@ -199,7 +200,7 @@ try: # Retry for shipyard sleep(SERVER_RETRY) data2 = companion.session.station() - if (data2['commander'].get('docked') and # might have undocked while we were waiting for retry in which case station data is unreliable + if (data2['commander'].get('docked') and # might have undocked while we were waiting for retry in which case station data is unreliable data2.get('lastSystem', {}).get('name') == monitor.system and data2.get('lastStarport', {}).get('name') == monitor.station): data = data2 From 2e219605689dd9d186e5ef1265971b8a082d9501 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:03:31 +0200 Subject: [PATCH 027/258] add newlines on scope changes --- EDMC.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/EDMC.py b/EDMC.py index b928f153..2a9e94c7 100755 --- a/EDMC.py +++ b/EDMC.py @@ -77,13 +77,16 @@ try: # Import and collate from JSON dump data = json.load(open(args.j)) config.set('querytime', int(getmtime(args.j))) + else: # Get state from latest Journal file try: logdir = config.get('journaldir') or config.default_journal_dir logfiles = sorted([x for x in os.listdir(logdir) if re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], key=lambda x: x.split('.')[1:]) + logfile = join(logdir, logfiles[-1]) + with open(logfile, 'r') as loghandle: for line in loghandle: try: @@ -91,6 +94,7 @@ try: except: if __debug__: print('Invalid journal entry "%s"' % repr(line)) + except Exception as e: print("Can't read Journal file: {}\n".format(str(e)), file=sys.stderr) sys.exit(EXIT_SYS_ERR) @@ -104,18 +108,23 @@ try: cmdrs = config.get('cmdrs') or [] if args.p in cmdrs: idx = cmdrs.index(args.p) + else: for idx, cmdr in enumerate(cmdrs): if cmdr.lower() == args.p.lower(): break else: raise companion.CredentialsError() + companion.session.login(cmdrs[idx], monitor.is_beta) + else: cmdrs = config.get('cmdrs') or [] if monitor.cmdr not in cmdrs: raise companion.CredentialsError() + companion.session.login(monitor.cmdr, monitor.is_beta) + querytime = int(time()) data = companion.session.station() config.set('querytime', querytime) @@ -124,22 +133,28 @@ try: if not data.get('commander') or not data['commander'].get('name','').strip(): sys.stderr.write('Who are you?!\n') sys.exit(EXIT_SERVER) + elif (not data.get('lastSystem', {}).get('name') or (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked sys.stderr.write('Where are you?!\n') # Shouldn't happen sys.exit(EXIT_SERVER) + elif not data.get('ship') or not data['ship'].get('modules') or not data['ship'].get('name','').strip(): sys.stderr.write('What are you flying?!\n') # Shouldn't happen sys.exit(EXIT_SERVER) + elif args.j: pass # Skip further validation + elif data['commander']['name'] != monitor.cmdr: sys.stderr.write('Wrong Cmdr\n') # Companion API return doesn't match Journal sys.exit(EXIT_CREDENTIALS) + elif ((data['lastSystem']['name'] != monitor.system) or ((data['commander']['docked'] and data['lastStarport']['name'] or None) != monitor.station) or (data['ship']['id'] != monitor.state['ShipID']) or (data['ship']['name'].lower() != monitor.state['ShipType'])): + sys.stderr.write('Frontier server is lagging\n') sys.exit(EXIT_LAGGING) @@ -147,17 +162,22 @@ try: if args.d: with open(args.d, 'wb') as h: h.write(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')).encode('utf-8')) + if args.a: loadout.export(data, args.a) + if args.e: edshipyard.export(data, args.e) + if args.l: stats.export_ships(data, args.l) + if args.t: stats.export_status(data, args.t) if data['commander'].get('docked'): print('%s,%s' % (data.get('lastSystem', {}).get('name', 'Unknown'), data.get('lastStarport', {}).get('name', 'Unknown'))) + else: print(data.get('lastSystem', {}).get('name', 'Unknown')) @@ -165,12 +185,15 @@ try: if not data['commander'].get('docked'): sys.stderr.write("You're not docked at a station!\n") sys.exit(EXIT_SUCCESS) + elif not data.get('lastStarport', {}).get('name'): sys.stderr.write("Unknown station!\n") sys.exit(EXIT_LAGGING) + elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): # Ignore possibly missing shipyard info sys.stderr.write("Station doesn't have anything!\n") sys.exit(EXIT_SUCCESS) + else: sys.exit(EXIT_SUCCESS) @@ -187,12 +210,14 @@ try: # Fixup anomalies in the commodity data fixed = companion.fixup(data) commodity.export(fixed, COMMODITY_DEFAULT, args.m) + else: sys.stderr.write("Station doesn't have a market\n") if args.o: if data['lastStarport'].get('modules'): outfitting.export(data, args.o) + else: sys.stderr.write("Station doesn't supply outfitting\n") @@ -208,8 +233,10 @@ try: if args.s: if data['lastStarport'].get('ships', {}).get('shipyard_list'): shipyard.export(data, args.s) + elif not args.j and monitor.stationservices and 'Shipyard' in monitor.stationservices: sys.stderr.write("Failed to get shipyard data\n") + else: sys.stderr.write("Station doesn't have a shipyard\n") @@ -219,6 +246,7 @@ try: eddn_sender.export_commodities(data, monitor.is_beta) eddn_sender.export_outfitting(data, monitor.is_beta) eddn_sender.export_shipyard(data, monitor.is_beta) + except Exception as e: sys.stderr.write("Failed to send data to EDDN: %s\n" % unicode(e).encode('ascii', 'replace')) @@ -227,9 +255,11 @@ try: except companion.ServerError as e: sys.stderr.write('Server is down\n') sys.exit(EXIT_SERVER) + except companion.SKUError as e: sys.stderr.write('Server SKU problem\n') sys.exit(EXIT_SERVER) + except companion.CredentialsError as e: sys.stderr.write('Invalid Credentials\n') sys.exit(EXIT_CREDENTIALS) From 2592af8c9fb5f97a5b38959843861bf0033774d0 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:05:25 +0200 Subject: [PATCH 028/258] block flake8 line length on doc line --- EDMC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index 2a9e94c7..d932f5d9 100755 --- a/EDMC.py +++ b/EDMC.py @@ -47,7 +47,7 @@ def versioncmp(versionstring): try: # arg parsing - parser = argparse.ArgumentParser(prog=appcmdname, description='Prints the current system and station (if docked) to stdout and optionally writes player status, ship locations, ship loadout and/or station data to file. Requires prior setup through the accompanying GUI app.') + parser = argparse.ArgumentParser(prog=appcmdname, description='Prints the current system and station (if docked) to stdout and optionally writes player status, ship locations, ship loadout and/or station data to file. Requires prior setup through the accompanying GUI app.') # noqa:E501 parser.add_argument('-v', '--version', help='print program version and exit', action='store_const', const=True) parser.add_argument('-a', metavar='FILE', help='write ship loadout to FILE in Companion API json format') parser.add_argument('-e', metavar='FILE', help='write ship loadout to FILE in E:D Shipyard plain text format') From 9d727aaa997f28f6013fc161ec57288a0a30afc9 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:06:03 +0200 Subject: [PATCH 029/258] Remove unused variables in `as e` clauses --- EDMC.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EDMC.py b/EDMC.py index d932f5d9..683bd813 100755 --- a/EDMC.py +++ b/EDMC.py @@ -252,14 +252,14 @@ try: sys.exit(EXIT_SUCCESS) -except companion.ServerError as e: +except companion.ServerError: sys.stderr.write('Server is down\n') sys.exit(EXIT_SERVER) -except companion.SKUError as e: +except companion.SKUError: sys.stderr.write('Server SKU problem\n') sys.exit(EXIT_SERVER) -except companion.CredentialsError as e: +except companion.CredentialsError: sys.stderr.write('Invalid Credentials\n') sys.exit(EXIT_CREDENTIALS) From f9b860fd5c4cfbbe25e6839bc4d78484fe5592cd Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:10:49 +0200 Subject: [PATCH 030/258] removed bare except clause --- EDMC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index 683bd813..2f832095 100755 --- a/EDMC.py +++ b/EDMC.py @@ -91,7 +91,7 @@ try: for line in loghandle: try: monitor.parse_entry(line) - except: + except Exception: if __debug__: print('Invalid journal entry "%s"' % repr(line)) From 31c049deda83160c1ec0cc82eae92b25a9125228 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:15:23 +0200 Subject: [PATCH 031/258] Replace sys.stderr.write call with print `print()` supports files other than stdout, and will automatically add newlines for us, among other things --- EDMC.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/EDMC.py b/EDMC.py index 2f832095..6b39483b 100755 --- a/EDMC.py +++ b/EDMC.py @@ -96,11 +96,11 @@ try: print('Invalid journal entry "%s"' % repr(line)) except Exception as e: - print("Can't read Journal file: {}\n".format(str(e)), file=sys.stderr) + print("Can't read Journal file: {}".format(str(e)), file=sys.stderr) sys.exit(EXIT_SYS_ERR) if not monitor.cmdr: - sys.stderr.write('Not available while E:D is at the main menu\n') + print('Not available while E:D is at the main menu', file=sys.stderr) sys.exit(EXIT_SYS_ERR) # Get data from Companion API @@ -131,23 +131,23 @@ try: # Validation if not data.get('commander') or not data['commander'].get('name','').strip(): - sys.stderr.write('Who are you?!\n') + print('Who are you?!', file=sys.stderr) sys.exit(EXIT_SERVER) elif (not data.get('lastSystem', {}).get('name') or (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked - sys.stderr.write('Where are you?!\n') # Shouldn't happen + print('Where are you?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) elif not data.get('ship') or not data['ship'].get('modules') or not data['ship'].get('name','').strip(): - sys.stderr.write('What are you flying?!\n') # Shouldn't happen + print('What are you flying?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) elif args.j: pass # Skip further validation elif data['commander']['name'] != monitor.cmdr: - sys.stderr.write('Wrong Cmdr\n') # Companion API return doesn't match Journal + print('Wrong Cmdr', file=sys.stderr) # Companion API return doesn't match Journal sys.exit(EXIT_CREDENTIALS) elif ((data['lastSystem']['name'] != monitor.system) or @@ -155,7 +155,7 @@ try: (data['ship']['id'] != monitor.state['ShipID']) or (data['ship']['name'].lower() != monitor.state['ShipType'])): - sys.stderr.write('Frontier server is lagging\n') + print('Frontier server is lagging', file=sys.stderr) sys.exit(EXIT_LAGGING) # stuff we can do when not docked @@ -183,15 +183,15 @@ try: if (args.m or args.o or args.s or args.n or args.j): if not data['commander'].get('docked'): - sys.stderr.write("You're not docked at a station!\n") + print("You're not docked at a station!", file=sys.stderr) sys.exit(EXIT_SUCCESS) elif not data.get('lastStarport', {}).get('name'): - sys.stderr.write("Unknown station!\n") + print("Unknown station!", file=sys.stderr) sys.exit(EXIT_LAGGING) elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): # Ignore possibly missing shipyard info - sys.stderr.write("Station doesn't have anything!\n") + print("Station doesn't have anything!", file=sys.stderr) sys.exit(EXIT_SUCCESS) else: @@ -212,14 +212,14 @@ try: commodity.export(fixed, COMMODITY_DEFAULT, args.m) else: - sys.stderr.write("Station doesn't have a market\n") + print("Station doesn't have a market", file=sys.stderr) if args.o: if data['lastStarport'].get('modules'): outfitting.export(data, args.o) else: - sys.stderr.write("Station doesn't supply outfitting\n") + print("Station doesn't supply outfitting", file=sys.stderr) if (args.s or args.n) and not args.j and not data['lastStarport'].get('ships') and data['lastStarport']['services'].get('shipyard'): # Retry for shipyard @@ -235,10 +235,10 @@ try: shipyard.export(data, args.s) elif not args.j and monitor.stationservices and 'Shipyard' in monitor.stationservices: - sys.stderr.write("Failed to get shipyard data\n") + print("Failed to get shipyard data", file=sys.stderr) else: - sys.stderr.write("Station doesn't have a shipyard\n") + print("Station doesn't have a shipyard", file=sys.stderr) if args.n: try: @@ -248,18 +248,18 @@ try: eddn_sender.export_shipyard(data, monitor.is_beta) except Exception as e: - sys.stderr.write("Failed to send data to EDDN: %s\n" % unicode(e).encode('ascii', 'replace')) + print("Failed to send data to EDDN: %s" % unicode(e).encode('ascii', 'replace'), file=sys.stderr) sys.exit(EXIT_SUCCESS) except companion.ServerError: - sys.stderr.write('Server is down\n') + print('Server is down', file=sys.stderr) sys.exit(EXIT_SERVER) except companion.SKUError: - sys.stderr.write('Server SKU problem\n') + print('Server SKU problem', file=sys.stderr) sys.exit(EXIT_SERVER) except companion.CredentialsError: - sys.stderr.write('Invalid Credentials\n') + print('Invalid Credentials', file=sys.stderr) sys.exit(EXIT_CREDENTIALS) From 37c01c028c3b8cd5b69eb7c1d43b1562fd9c2c29 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:22:15 +0200 Subject: [PATCH 032/258] Replaced list comprehension with generator Creating a list here doesnt make sense when its almost instantly recreated. --- EDMC.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/EDMC.py b/EDMC.py index 6b39483b..b76a3106 100755 --- a/EDMC.py +++ b/EDMC.py @@ -82,8 +82,10 @@ try: # Get state from latest Journal file try: logdir = config.get('journaldir') or config.default_journal_dir - logfiles = sorted([x for x in os.listdir(logdir) if re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)], - key=lambda x: x.split('.')[1:]) + logfiles = sorted( + (x for x in os.listdir(logdir) if re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)), + key=lambda x: x.split('.')[1:] + ) logfile = join(logdir, logfiles[-1]) From 8117e189633a937a2ce47d03919c605da42dcee7 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:25:15 +0200 Subject: [PATCH 033/258] Don't recompile regexps where possible While the python re lib _does_ cache compiled regexps, it only does this to a point, it's better to compile once and hold a reference --- EDMC.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/EDMC.py b/EDMC.py index b76a3106..12533508 100755 --- a/EDMC.py +++ b/EDMC.py @@ -39,6 +39,8 @@ import eddn SERVER_RETRY = 5 # retry pause for Companion servers [s] EXIT_SUCCESS, EXIT_SERVER, EXIT_CREDENTIALS, EXIT_VERIFICATION, EXIT_LAGGING, EXIT_SYS_ERR = range(6) +JOURNAL_RE = re.compile(r'^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$') + # quick and dirty version comparison assuming "strict" numeric only version numbers def versioncmp(versionstring): @@ -82,10 +84,7 @@ try: # Get state from latest Journal file try: logdir = config.get('journaldir') or config.default_journal_dir - logfiles = sorted( - (x for x in os.listdir(logdir) if re.search('^Journal(Beta)?\.[0-9]{12}\.[0-9]{2}\.log$', x)), - key=lambda x: x.split('.')[1:] - ) + logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) logfile = join(logdir, logfiles[-1]) From 0dfad42de3bb296935fa8101df79c4c179669d84 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:43:59 +0200 Subject: [PATCH 034/258] Replaced .get chains with a new method `deep_get` will go down a list of keys on a dict to find either the requested key or the default if somewhere along the line the given dict doesn't have a given key --- EDMC.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/EDMC.py b/EDMC.py index 12533508..d7b2c4d9 100755 --- a/EDMC.py +++ b/EDMC.py @@ -8,6 +8,7 @@ import json import requests import sys import os +from typing import Union, Any # workaround for https://github.com/EDCD/EDMarketConnector/issues/568 os.environ["EDMC_NO_UI"] = "1" @@ -47,6 +48,18 @@ def versioncmp(versionstring): return list(map(int, versionstring.split('.'))) +def deep_get(target: dict, *args: str, default=None) -> Any: + current = target + for arg in args: + res = current.get(arg) + if res is not None: + current = res + + else: + break + + return default + try: # arg parsing parser = argparse.ArgumentParser(prog=appcmdname, description='Prints the current system and station (if docked) to stdout and optionally writes player status, ship locations, ship loadout and/or station data to file. Requires prior setup through the accompanying GUI app.') # noqa:E501 @@ -83,7 +96,7 @@ try: else: # Get state from latest Journal file try: - logdir = config.get('journaldir') or config.default_journal_dir + logdir = config.get('journaldir', config.default_journal_dir) logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) logfile = join(logdir, logfiles[-1]) @@ -106,7 +119,7 @@ try: # Get data from Companion API if args.p: - cmdrs = config.get('cmdrs') or [] + cmdrs = config.get('cmdrs', []) if args.p in cmdrs: idx = cmdrs.index(args.p) @@ -120,7 +133,7 @@ try: companion.session.login(cmdrs[idx], monitor.is_beta) else: - cmdrs = config.get('cmdrs') or [] + cmdrs = config.get('cmdrs', []) if monitor.cmdr not in cmdrs: raise companion.CredentialsError() @@ -135,12 +148,13 @@ try: print('Who are you?!', file=sys.stderr) sys.exit(EXIT_SERVER) - elif (not data.get('lastSystem', {}).get('name') or - (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked + elif not deep_get(data, 'lastSystem', 'name') or ( + data['commander'].get('docked') and not deep_get(data, 'lastStarport', 'name')): # Only care if docked + print('Where are you?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) - elif not data.get('ship') or not data['ship'].get('modules') or not data['ship'].get('name','').strip(): + elif not deep_get(data, 'ship', 'modules') or not deep_get('ship', 'name', default=''): print('What are you flying?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) @@ -177,7 +191,10 @@ try: stats.export_status(data, args.t) if data['commander'].get('docked'): - print('%s,%s' % (data.get('lastSystem', {}).get('name', 'Unknown'), data.get('lastStarport', {}).get('name', 'Unknown'))) + print('%s,%s' % ( + deep_get(data, 'lastSystem', 'name', default='Unknown'), + deep_get(data, 'lastStarport', 'name', default='Unknown') + )) else: print(data.get('lastSystem', {}).get('name', 'Unknown')) From 637f58bb0692ad40496d3aef95b74f4f6ff90e14 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:47:51 +0200 Subject: [PATCH 035/258] Replaced modulo-formatting with .format --- EDMC.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EDMC.py b/EDMC.py index d7b2c4d9..78f9da8d 100755 --- a/EDMC.py +++ b/EDMC.py @@ -107,7 +107,7 @@ try: monitor.parse_entry(line) except Exception: if __debug__: - print('Invalid journal entry "%s"' % repr(line)) + print('Invalid journal entry {!r}'.format(line)) except Exception as e: print("Can't read Journal file: {}".format(str(e)), file=sys.stderr) @@ -191,7 +191,7 @@ try: stats.export_status(data, args.t) if data['commander'].get('docked'): - print('%s,%s' % ( + print('{},{}'.format( deep_get(data, 'lastSystem', 'name', default='Unknown'), deep_get(data, 'lastStarport', 'name', default='Unknown') )) @@ -266,7 +266,7 @@ try: eddn_sender.export_shipyard(data, monitor.is_beta) except Exception as e: - print("Failed to send data to EDDN: %s" % unicode(e).encode('ascii', 'replace'), file=sys.stderr) + print("Failed to send data to EDDN: {}".format(str(e)), file=sys.stderr) sys.exit(EXIT_SUCCESS) From 1ed9225c44716a9c0ef9a125ec0e61f3ac4201d3 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:55:17 +0200 Subject: [PATCH 036/258] Added main guard Modules should _always_ check that they are main before doing any "real" work --- EDMC.py | 391 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 201 insertions(+), 190 deletions(-) diff --git a/EDMC.py b/EDMC.py index 78f9da8d..b75d9e9f 100755 --- a/EDMC.py +++ b/EDMC.py @@ -60,224 +60,235 @@ def deep_get(target: dict, *args: str, default=None) -> Any: return default -try: - # arg parsing - parser = argparse.ArgumentParser(prog=appcmdname, description='Prints the current system and station (if docked) to stdout and optionally writes player status, ship locations, ship loadout and/or station data to file. Requires prior setup through the accompanying GUI app.') # noqa:E501 - parser.add_argument('-v', '--version', help='print program version and exit', action='store_const', const=True) - parser.add_argument('-a', metavar='FILE', help='write ship loadout to FILE in Companion API json format') - parser.add_argument('-e', metavar='FILE', help='write ship loadout to FILE in E:D Shipyard plain text format') - parser.add_argument('-l', metavar='FILE', help='write ship locations to FILE in CSV format') - parser.add_argument('-m', metavar='FILE', help='write station commodity market data to FILE in CSV format') - parser.add_argument('-o', metavar='FILE', help='write station outfitting data to FILE in CSV format') - parser.add_argument('-s', metavar='FILE', help='write station shipyard data to FILE in CSV format') - parser.add_argument('-t', metavar='FILE', help='write player status to FILE in CSV format') - parser.add_argument('-d', metavar='FILE', help='write raw JSON data to FILE') - parser.add_argument('-n', action='store_true', help='send data to EDDN') - parser.add_argument('-p', metavar='CMDR', help='Returns data from the specified player account') - parser.add_argument('-j', help=argparse.SUPPRESS) # Import JSON dump - args = parser.parse_args() - if args.version: - updater = Updater(provider='internal') - newversion: EDMCVersion = updater.check_appcast() - if newversion: - print('{CURRENT} ("{UPDATE}" is available)'.format( - CURRENT=appversion, - UPDATE=newversion.title)) +def main(): + try: + # arg parsing + parser = argparse.ArgumentParser(prog=appcmdname, description='Prints the current system and station (if docked) to stdout and optionally writes player status, ship locations, ship loadout and/or station data to file. Requires prior setup through the accompanying GUI app.') # noqa:E501 + parser.add_argument('-v', '--version', help='print program version and exit', action='store_const', const=True) + parser.add_argument('-a', metavar='FILE', help='write ship loadout to FILE in Companion API json format') + parser.add_argument('-e', metavar='FILE', help='write ship loadout to FILE in E:D Shipyard plain text format') + parser.add_argument('-l', metavar='FILE', help='write ship locations to FILE in CSV format') + parser.add_argument('-m', metavar='FILE', help='write station commodity market data to FILE in CSV format') + parser.add_argument('-o', metavar='FILE', help='write station outfitting data to FILE in CSV format') + parser.add_argument('-s', metavar='FILE', help='write station shipyard data to FILE in CSV format') + parser.add_argument('-t', metavar='FILE', help='write player status to FILE in CSV format') + parser.add_argument('-d', metavar='FILE', help='write raw JSON data to FILE') + parser.add_argument('-n', action='store_true', help='send data to EDDN') + parser.add_argument('-p', metavar='CMDR', help='Returns data from the specified player account') + parser.add_argument('-j', help=argparse.SUPPRESS) # Import JSON dump + args = parser.parse_args() + + if args.version: + updater = Updater(provider='internal') + newversion: EDMCVersion = updater.check_appcast() + if newversion: + print('{CURRENT} ("{UPDATE}" is available)'.format( + CURRENT=appversion, + UPDATE=newversion.title)) + else: + print(appversion) + sys.exit(EXIT_SUCCESS) + + if args.j: + # Import and collate from JSON dump + data = json.load(open(args.j)) + config.set('querytime', int(getmtime(args.j))) + else: - print(appversion) - sys.exit(EXIT_SUCCESS) + # Get state from latest Journal file + try: + logdir = config.get('journaldir', config.default_journal_dir) + logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) - if args.j: - # Import and collate from JSON dump - data = json.load(open(args.j)) - config.set('querytime', int(getmtime(args.j))) + logfile = join(logdir, logfiles[-1]) - else: - # Get state from latest Journal file - try: - logdir = config.get('journaldir', config.default_journal_dir) - logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) + with open(logfile, 'r') as loghandle: + for line in loghandle: + try: + monitor.parse_entry(line) + except Exception: + if __debug__: + print('Invalid journal entry {!r}'.format(line)) - logfile = join(logdir, logfiles[-1]) + except Exception as e: + print("Can't read Journal file: {}".format(str(e)), file=sys.stderr) + sys.exit(EXIT_SYS_ERR) - with open(logfile, 'r') as loghandle: - for line in loghandle: - try: - monitor.parse_entry(line) - except Exception: - if __debug__: - print('Invalid journal entry {!r}'.format(line)) + if not monitor.cmdr: + print('Not available while E:D is at the main menu', file=sys.stderr) + sys.exit(EXIT_SYS_ERR) - except Exception as e: - print("Can't read Journal file: {}".format(str(e)), file=sys.stderr) - sys.exit(EXIT_SYS_ERR) + # Get data from Companion API + if args.p: + cmdrs = config.get('cmdrs', []) + if args.p in cmdrs: + idx = cmdrs.index(args.p) - if not monitor.cmdr: - print('Not available while E:D is at the main menu', file=sys.stderr) - sys.exit(EXIT_SYS_ERR) + else: + for idx, cmdr in enumerate(cmdrs): + if cmdr.lower() == args.p.lower(): + break + else: + raise companion.CredentialsError() - # Get data from Companion API - if args.p: - cmdrs = config.get('cmdrs', []) - if args.p in cmdrs: - idx = cmdrs.index(args.p) + companion.session.login(cmdrs[idx], monitor.is_beta) else: - for idx, cmdr in enumerate(cmdrs): - if cmdr.lower() == args.p.lower(): - break - else: + cmdrs = config.get('cmdrs', []) + if monitor.cmdr not in cmdrs: raise companion.CredentialsError() - companion.session.login(cmdrs[idx], monitor.is_beta) + companion.session.login(monitor.cmdr, monitor.is_beta) - else: - cmdrs = config.get('cmdrs', []) - if monitor.cmdr not in cmdrs: - raise companion.CredentialsError() + querytime = int(time()) + data = companion.session.station() + config.set('querytime', querytime) - companion.session.login(monitor.cmdr, monitor.is_beta) + # Validation + if not data.get('commander') or not data['commander'].get('name','').strip(): + print('Who are you?!', file=sys.stderr) + sys.exit(EXIT_SERVER) - querytime = int(time()) - data = companion.session.station() - config.set('querytime', querytime) + elif not deep_get(data, 'lastSystem', 'name') or ( + data['commander'].get('docked') and not deep_get(data, 'lastStarport', 'name')): # Only care if docked + + print('Where are you?!', file=sys.stderr) # Shouldn't happen + sys.exit(EXIT_SERVER) - # Validation - if not data.get('commander') or not data['commander'].get('name','').strip(): - print('Who are you?!', file=sys.stderr) - sys.exit(EXIT_SERVER) + elif not deep_get(data, 'ship', 'modules') or not deep_get('ship', 'name', default=''): + print('What are you flying?!', file=sys.stderr) # Shouldn't happen + sys.exit(EXIT_SERVER) - elif not deep_get(data, 'lastSystem', 'name') or ( - data['commander'].get('docked') and not deep_get(data, 'lastStarport', 'name')): # Only care if docked - - print('Where are you?!', file=sys.stderr) # Shouldn't happen - sys.exit(EXIT_SERVER) + elif args.j: + pass # Skip further validation - elif not deep_get(data, 'ship', 'modules') or not deep_get('ship', 'name', default=''): - print('What are you flying?!', file=sys.stderr) # Shouldn't happen - sys.exit(EXIT_SERVER) + elif data['commander']['name'] != monitor.cmdr: + print('Wrong Cmdr', file=sys.stderr) # Companion API return doesn't match Journal + sys.exit(EXIT_CREDENTIALS) - elif args.j: - pass # Skip further validation + elif ((data['lastSystem']['name'] != monitor.system) or + ((data['commander']['docked'] and data['lastStarport']['name'] or None) != monitor.station) or + (data['ship']['id'] != monitor.state['ShipID']) or + (data['ship']['name'].lower() != monitor.state['ShipType'])): - elif data['commander']['name'] != monitor.cmdr: - print('Wrong Cmdr', file=sys.stderr) # Companion API return doesn't match Journal - sys.exit(EXIT_CREDENTIALS) - - elif ((data['lastSystem']['name'] != monitor.system) or - ((data['commander']['docked'] and data['lastStarport']['name'] or None) != monitor.station) or - (data['ship']['id'] != monitor.state['ShipID']) or - (data['ship']['name'].lower() != monitor.state['ShipType'])): - - print('Frontier server is lagging', file=sys.stderr) - sys.exit(EXIT_LAGGING) - - # stuff we can do when not docked - if args.d: - with open(args.d, 'wb') as h: - h.write(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')).encode('utf-8')) - - if args.a: - loadout.export(data, args.a) - - if args.e: - edshipyard.export(data, args.e) - - if args.l: - stats.export_ships(data, args.l) - - if args.t: - stats.export_status(data, args.t) - - if data['commander'].get('docked'): - print('{},{}'.format( - deep_get(data, 'lastSystem', 'name', default='Unknown'), - deep_get(data, 'lastStarport', 'name', default='Unknown') - )) - - else: - print(data.get('lastSystem', {}).get('name', 'Unknown')) - - if (args.m or args.o or args.s or args.n or args.j): - if not data['commander'].get('docked'): - print("You're not docked at a station!", file=sys.stderr) - sys.exit(EXIT_SUCCESS) - - elif not data.get('lastStarport', {}).get('name'): - print("Unknown station!", file=sys.stderr) + print('Frontier server is lagging', file=sys.stderr) sys.exit(EXIT_LAGGING) - elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): # Ignore possibly missing shipyard info - print("Station doesn't have anything!", file=sys.stderr) + # stuff we can do when not docked + if args.d: + with open(args.d, 'wb') as h: + h.write(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')).encode('utf-8')) + + if args.a: + loadout.export(data, args.a) + + if args.e: + edshipyard.export(data, args.e) + + if args.l: + stats.export_ships(data, args.l) + + if args.t: + stats.export_status(data, args.t) + + if data['commander'].get('docked'): + print('{},{}'.format( + deep_get(data, 'lastSystem', 'name', default='Unknown'), + deep_get(data, 'lastStarport', 'name', default='Unknown') + )) + + else: + print(data.get('lastSystem', {}).get('name', 'Unknown')) + + if (args.m or args.o or args.s or args.n or args.j): + if not data['commander'].get('docked'): + print("You're not docked at a station!", file=sys.stderr) + sys.exit(EXIT_SUCCESS) + + elif not data.get('lastStarport', {}).get('name'): + print("Unknown station!", file=sys.stderr) + sys.exit(EXIT_LAGGING) + + # Ignore possibly missing shipyard info + elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): + print("Station doesn't have anything!", file=sys.stderr) + sys.exit(EXIT_SUCCESS) + + else: sys.exit(EXIT_SUCCESS) - else: + # Finally - the data looks sane and we're docked at a station + + if args.j: + # Collate from JSON dump + collate.addcommodities(data) + collate.addmodules(data) + collate.addships(data) + + if args.m: + if data['lastStarport'].get('commodities'): + # Fixup anomalies in the commodity data + fixed = companion.fixup(data) + commodity.export(fixed, COMMODITY_DEFAULT, args.m) + + else: + print("Station doesn't have a market", file=sys.stderr) + + if args.o: + if data['lastStarport'].get('modules'): + outfitting.export(data, args.o) + + else: + print("Station doesn't supply outfitting", file=sys.stderr) + + if (args.s or args.n) and not args.j and not \ + data['lastStarport'].get('ships') and data['lastStarport']['services'].get('shipyard'): + + # Retry for shipyard + sleep(SERVER_RETRY) + data2 = companion.session.station() + # might have undocked while we were waiting for retry in which case station data is unreliable + if data2['commander'].get('docked') and \ + deep_get(data2, 'lastSystem', 'name') == monitor.system and \ + deep_get(data2, 'lastStarport', 'name') == monitor.station: + + data = data2 + + if args.s: + if data['lastStarport'].get('ships', {}).get('shipyard_list'): + shipyard.export(data, args.s) + + elif not args.j and monitor.stationservices and 'Shipyard' in monitor.stationservices: + print("Failed to get shipyard data", file=sys.stderr) + + else: + print("Station doesn't have a shipyard", file=sys.stderr) + + if args.n: + try: + eddn_sender = eddn.EDDN(None) + eddn_sender.export_commodities(data, monitor.is_beta) + eddn_sender.export_outfitting(data, monitor.is_beta) + eddn_sender.export_shipyard(data, monitor.is_beta) + + except Exception as e: + print("Failed to send data to EDDN: {}".format(str(e)), file=sys.stderr) + sys.exit(EXIT_SUCCESS) - # Finally - the data looks sane and we're docked at a station + except companion.ServerError: + print('Server is down', file=sys.stderr) + sys.exit(EXIT_SERVER) - if args.j: - # Collate from JSON dump - collate.addcommodities(data) - collate.addmodules(data) - collate.addships(data) + except companion.SKUError: + print('Server SKU problem', file=sys.stderr) + sys.exit(EXIT_SERVER) - if args.m: - if data['lastStarport'].get('commodities'): - # Fixup anomalies in the commodity data - fixed = companion.fixup(data) - commodity.export(fixed, COMMODITY_DEFAULT, args.m) + except companion.CredentialsError: + print('Invalid Credentials', file=sys.stderr) + sys.exit(EXIT_CREDENTIALS) - else: - print("Station doesn't have a market", file=sys.stderr) - if args.o: - if data['lastStarport'].get('modules'): - outfitting.export(data, args.o) - - else: - print("Station doesn't supply outfitting", file=sys.stderr) - - if (args.s or args.n) and not args.j and not data['lastStarport'].get('ships') and data['lastStarport']['services'].get('shipyard'): - # Retry for shipyard - sleep(SERVER_RETRY) - data2 = companion.session.station() - if (data2['commander'].get('docked') and # might have undocked while we were waiting for retry in which case station data is unreliable - data2.get('lastSystem', {}).get('name') == monitor.system and - data2.get('lastStarport', {}).get('name') == monitor.station): - data = data2 - - if args.s: - if data['lastStarport'].get('ships', {}).get('shipyard_list'): - shipyard.export(data, args.s) - - elif not args.j and monitor.stationservices and 'Shipyard' in monitor.stationservices: - print("Failed to get shipyard data", file=sys.stderr) - - else: - print("Station doesn't have a shipyard", file=sys.stderr) - - if args.n: - try: - eddn_sender = eddn.EDDN(None) - eddn_sender.export_commodities(data, monitor.is_beta) - eddn_sender.export_outfitting(data, monitor.is_beta) - eddn_sender.export_shipyard(data, monitor.is_beta) - - except Exception as e: - print("Failed to send data to EDDN: {}".format(str(e)), file=sys.stderr) - - sys.exit(EXIT_SUCCESS) - -except companion.ServerError: - print('Server is down', file=sys.stderr) - sys.exit(EXIT_SERVER) - -except companion.SKUError: - print('Server SKU problem', file=sys.stderr) - sys.exit(EXIT_SERVER) - -except companion.CredentialsError: - print('Invalid Credentials', file=sys.stderr) - sys.exit(EXIT_CREDENTIALS) +if __name__ == '__main__': + main() From 664605a315e0c1289eeb044a91abb711854449d0 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 07:59:46 +0200 Subject: [PATCH 037/258] Cleaned up ifs where possible Removed redundant or unneeded parens, correctly added line breaks where needed --- EDMC.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/EDMC.py b/EDMC.py index b75d9e9f..1f3ae1c5 100755 --- a/EDMC.py +++ b/EDMC.py @@ -150,9 +150,10 @@ def main(): print('Who are you?!', file=sys.stderr) sys.exit(EXIT_SERVER) - elif not deep_get(data, 'lastSystem', 'name') or ( - data['commander'].get('docked') and not deep_get(data, 'lastStarport', 'name')): # Only care if docked - + elif not deep_get(data, 'lastSystem', 'name') or \ + data['commander'].get('docked') and not \ + deep_get(data, 'lastStarport', 'name'): # Only care if docked + print('Where are you?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) @@ -167,10 +168,10 @@ def main(): print('Wrong Cmdr', file=sys.stderr) # Companion API return doesn't match Journal sys.exit(EXIT_CREDENTIALS) - elif ((data['lastSystem']['name'] != monitor.system) or - ((data['commander']['docked'] and data['lastStarport']['name'] or None) != monitor.station) or - (data['ship']['id'] != monitor.state['ShipID']) or - (data['ship']['name'].lower() != monitor.state['ShipType'])): + elif data['lastSystem']['name'] != monitor.system or \ + ((data['commander']['docked'] and data['lastStarport']['name'] or None) != monitor.station) or \ + data['ship']['id'] != monitor.state['ShipID'] or \ + data['ship']['name'].lower() != monitor.state['ShipType']: print('Frontier server is lagging', file=sys.stderr) sys.exit(EXIT_LAGGING) From 4ee88694261191089073eed796e079055ad34292 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 08:10:52 +0200 Subject: [PATCH 038/258] Revert part of e510b783 Apparently config.get does _not_ take a default param. It really should. --- EDMC.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EDMC.py b/EDMC.py index 1f3ae1c5..8001dacb 100755 --- a/EDMC.py +++ b/EDMC.py @@ -98,7 +98,7 @@ def main(): else: # Get state from latest Journal file try: - logdir = config.get('journaldir', config.default_journal_dir) + logdir = config.get('journaldir') or config.default_journal_dir logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) logfile = join(logdir, logfiles[-1]) @@ -121,7 +121,7 @@ def main(): # Get data from Companion API if args.p: - cmdrs = config.get('cmdrs', []) + cmdrs = config.get('cmdrs') or [] if args.p in cmdrs: idx = cmdrs.index(args.p) @@ -135,7 +135,7 @@ def main(): companion.session.login(cmdrs[idx], monitor.is_beta) else: - cmdrs = config.get('cmdrs', []) + cmdrs = config.get('cmdrs') or [] if monitor.cmdr not in cmdrs: raise companion.CredentialsError() From 2403ed1d2f770cda37c2707fc63fa446a84e0c34 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 08:30:27 +0200 Subject: [PATCH 039/258] Replaced missed .get chains with deep_get --- EDMC.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/EDMC.py b/EDMC.py index 8001dacb..da338aea 100755 --- a/EDMC.py +++ b/EDMC.py @@ -8,7 +8,7 @@ import json import requests import sys import os -from typing import Union, Any +from typing import Any # workaround for https://github.com/EDCD/EDMarketConnector/issues/568 os.environ["EDMC_NO_UI"] = "1" @@ -146,7 +146,7 @@ def main(): config.set('querytime', querytime) # Validation - if not data.get('commander') or not data['commander'].get('name','').strip(): + if not deep_get(data, 'commander', 'name', default='').strip(): print('Who are you?!', file=sys.stderr) sys.exit(EXIT_SERVER) @@ -200,14 +200,14 @@ def main(): )) else: - print(data.get('lastSystem', {}).get('name', 'Unknown')) + print(deep_get(data, 'lastSystem', 'name', default='Unknown')) if (args.m or args.o or args.s or args.n or args.j): if not data['commander'].get('docked'): print("You're not docked at a station!", file=sys.stderr) sys.exit(EXIT_SUCCESS) - elif not data.get('lastStarport', {}).get('name'): + elif not deep_get(data, 'lastStarport', 'name'): print("Unknown station!", file=sys.stderr) sys.exit(EXIT_LAGGING) @@ -257,7 +257,7 @@ def main(): data = data2 if args.s: - if data['lastStarport'].get('ships', {}).get('shipyard_list'): + if deep_get(data, 'lastStarport', 'ships', 'shipyard_list'): shipyard.export(data, args.s) elif not args.j and monitor.stationservices and 'Shipyard' in monitor.stationservices: From b6482878f06276e6b079b5aea2436ec191f3a4ad Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 23:01:42 +0200 Subject: [PATCH 040/258] Fix deep_get not returning the correct on success I forgot to actually return the given data if we manage to index to the requested depth --- EDMC.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/EDMC.py b/EDMC.py index da338aea..0134de6b 100755 --- a/EDMC.py +++ b/EDMC.py @@ -49,16 +49,18 @@ def versioncmp(versionstring): def deep_get(target: dict, *args: str, default=None) -> Any: + if not hasattr(target, 'get'): + raise ValueError("Cannot call get on {} ({})".format(target, type(target))) + current = target for arg in args: res = current.get(arg) - if res is not None: - current = res + if res is None: + return default - else: - break + current = res - return default + return current def main(): From 3dfca91e1d87510ed1b99663c88dc2a4608db336 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 13 Jul 2020 23:02:37 +0200 Subject: [PATCH 041/258] Fixed invalid deep_get call --- EDMC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index 0134de6b..550bad04 100755 --- a/EDMC.py +++ b/EDMC.py @@ -159,7 +159,7 @@ def main(): print('Where are you?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) - elif not deep_get(data, 'ship', 'modules') or not deep_get('ship', 'name', default=''): + elif not deep_get(data, 'ship', 'modules') or not deep_get(data, 'ship', 'name', default=''): print('What are you flying?!', file=sys.stderr) # Shouldn't happen sys.exit(EXIT_SERVER) From 0e000de90a59a78ddcfb7546f2f8b61041450bf5 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 12:26:12 +0200 Subject: [PATCH 042/258] Replaced type annotation with Optional The return is Optional so the annotation on the other side must be as well, lest we have large red squiggles --- EDMC.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EDMC.py b/EDMC.py index 550bad04..65629dd9 100755 --- a/EDMC.py +++ b/EDMC.py @@ -8,7 +8,7 @@ import json import requests import sys import os -from typing import Any +from typing import Any, Optional # workaround for https://github.com/EDCD/EDMarketConnector/issues/568 os.environ["EDMC_NO_UI"] = "1" @@ -83,7 +83,7 @@ def main(): if args.version: updater = Updater(provider='internal') - newversion: EDMCVersion = updater.check_appcast() + newversion: Optional[EDMCVersion] = updater.check_appcast() if newversion: print('{CURRENT} ("{UPDATE}" is available)'.format( CURRENT=appversion, From c6d8b4eab8bb9c5ee9298b8f83d6b6dab34c1312 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 12:27:47 +0200 Subject: [PATCH 043/258] Renamed variable for clarity --- EDMC.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/EDMC.py b/EDMC.py index 65629dd9..9b880e05 100755 --- a/EDMC.py +++ b/EDMC.py @@ -250,13 +250,13 @@ def main(): # Retry for shipyard sleep(SERVER_RETRY) - data2 = companion.session.station() + new_data = companion.session.station() # might have undocked while we were waiting for retry in which case station data is unreliable - if data2['commander'].get('docked') and \ - deep_get(data2, 'lastSystem', 'name') == monitor.system and \ - deep_get(data2, 'lastStarport', 'name') == monitor.station: + if new_data['commander'].get('docked') and \ + deep_get(new_data, 'lastSystem', 'name') == monitor.system and \ + deep_get(new_data, 'lastStarport', 'name') == monitor.station: - data = data2 + data = new_data if args.s: if deep_get(data, 'lastStarport', 'ships', 'shipyard_list'): From 8b0f3e74cf24315d344b8690c6bc9d1fd1295a51 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 12:31:59 +0200 Subject: [PATCH 044/258] Replaced format directives with fstrings --- EDMC.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/EDMC.py b/EDMC.py index 9b880e05..de78ea15 100755 --- a/EDMC.py +++ b/EDMC.py @@ -5,7 +5,6 @@ import argparse import json -import requests import sys import os from typing import Any, Optional @@ -50,8 +49,8 @@ def versioncmp(versionstring): def deep_get(target: dict, *args: str, default=None) -> Any: if not hasattr(target, 'get'): - raise ValueError("Cannot call get on {} ({})".format(target, type(target))) - + raise ValueError(f"Cannot call get on {target} ({type(target)})") + current = target for arg in args: res = current.get(arg) @@ -85,9 +84,7 @@ def main(): updater = Updater(provider='internal') newversion: Optional[EDMCVersion] = updater.check_appcast() if newversion: - print('{CURRENT} ("{UPDATE}" is available)'.format( - CURRENT=appversion, - UPDATE=newversion.title)) + print(f'{appversion} ({newversion.title!r} is available)') else: print(appversion) sys.exit(EXIT_SUCCESS) @@ -111,10 +108,10 @@ def main(): monitor.parse_entry(line) except Exception: if __debug__: - print('Invalid journal entry {!r}'.format(line)) + print(f'Invalid journal entry {line!r}') except Exception as e: - print("Can't read Journal file: {}".format(str(e)), file=sys.stderr) + print(f"Can't read Journal file: {str(e)}", file=sys.stderr) sys.exit(EXIT_SYS_ERR) if not monitor.cmdr: @@ -276,7 +273,7 @@ def main(): eddn_sender.export_shipyard(data, monitor.is_beta) except Exception as e: - print("Failed to send data to EDDN: {}".format(str(e)), file=sys.stderr) + print(f"Failed to send data to EDDN: {str(e)}", file=sys.stderr) sys.exit(EXIT_SUCCESS) From 14295ce9e89950a33d62ecc8dbcfb4acce1968ff Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 12:39:12 +0200 Subject: [PATCH 045/258] Fixed scope whitespace and long lines --- EDMC.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index de78ea15..47b276ab 100755 --- a/EDMC.py +++ b/EDMC.py @@ -65,7 +65,13 @@ def deep_get(target: dict, *args: str, default=None) -> Any: def main(): try: # arg parsing - parser = argparse.ArgumentParser(prog=appcmdname, description='Prints the current system and station (if docked) to stdout and optionally writes player status, ship locations, ship loadout and/or station data to file. Requires prior setup through the accompanying GUI app.') # noqa:E501 + parser = argparse.ArgumentParser( + prog=appcmdname, + description='Prints the current system and station (if docked) to stdout and optionally writes player ' + 'status, ship locations, ship loadout and/or station data to file. ' + 'Requires prior setup through the accompanying GUI app.' + ) + parser.add_argument('-v', '--version', help='print program version and exit', action='store_const', const=True) parser.add_argument('-a', metavar='FILE', help='write ship loadout to FILE in Companion API json format') parser.add_argument('-e', metavar='FILE', help='write ship loadout to FILE in E:D Shipyard plain text format') @@ -128,6 +134,7 @@ def main(): for idx, cmdr in enumerate(cmdrs): if cmdr.lower() == args.p.lower(): break + else: raise companion.CredentialsError() From 566f52e61fb118020b036a8f1dd7fb887c036cd6 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 12:42:03 +0200 Subject: [PATCH 046/258] removed uneeded parens --- EDMC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index 47b276ab..11dc865f 100755 --- a/EDMC.py +++ b/EDMC.py @@ -218,7 +218,7 @@ def main(): sys.exit(EXIT_LAGGING) # Ignore possibly missing shipyard info - elif not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): + elif not data['lastStarport'].get('commodities') or data['lastStarport'].get('modules'): print("Station doesn't have anything!", file=sys.stderr) sys.exit(EXIT_SUCCESS) From 64b9cb39a98b853e15a9399f5bc4f424d67c7d6b Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 12:43:40 +0200 Subject: [PATCH 047/258] Replaced file.write(json.dumps()) with json.dump There's no reason to use json.dumps and directly encode it when we can let the json lib do the heavy lifting --- EDMC.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EDMC.py b/EDMC.py index 11dc865f..4287e6ae 100755 --- a/EDMC.py +++ b/EDMC.py @@ -184,8 +184,8 @@ def main(): # stuff we can do when not docked if args.d: - with open(args.d, 'wb') as h: - h.write(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')).encode('utf-8')) + with open(args.d, 'w') as f: + json.dump(data, f, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')) if args.a: loadout.export(data, args.a) From 0e0c802b04e431c46b826a8b1a653f25678221c9 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 12:22:17 +0200 Subject: [PATCH 048/258] Fixed missed whitespace after scope changes --- EDMC.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index 4287e6ae..739a4a24 100755 --- a/EDMC.py +++ b/EDMC.py @@ -91,8 +91,10 @@ def main(): newversion: Optional[EDMCVersion] = updater.check_appcast() if newversion: print(f'{appversion} ({newversion.title!r} is available)') + else: print(appversion) + sys.exit(EXIT_SUCCESS) if args.j: @@ -104,7 +106,9 @@ def main(): # Get state from latest Journal file try: logdir = config.get('journaldir') or config.default_journal_dir - logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) + logfiles = sorted( + (x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:] + ) logfile = join(logdir, logfiles[-1]) @@ -112,6 +116,7 @@ def main(): for line in loghandle: try: monitor.parse_entry(line) + except Exception: if __debug__: print(f'Invalid journal entry {line!r}') From f08d60d9b139648cfdc7c23cdf190b90e4d41e81 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 18:57:23 +0200 Subject: [PATCH 049/258] revert using json.dump windows encodings dont like the weird characters --- EDMC.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/EDMC.py b/EDMC.py index 739a4a24..2d2eb476 100755 --- a/EDMC.py +++ b/EDMC.py @@ -91,10 +91,8 @@ def main(): newversion: Optional[EDMCVersion] = updater.check_appcast() if newversion: print(f'{appversion} ({newversion.title!r} is available)') - else: print(appversion) - sys.exit(EXIT_SUCCESS) if args.j: @@ -106,9 +104,7 @@ def main(): # Get state from latest Journal file try: logdir = config.get('journaldir') or config.default_journal_dir - logfiles = sorted( - (x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:] - ) + logfiles = sorted((x for x in os.listdir(logdir) if JOURNAL_RE.search(x)), key=lambda x: x.split('.')[1:]) logfile = join(logdir, logfiles[-1]) @@ -116,7 +112,6 @@ def main(): for line in loghandle: try: monitor.parse_entry(line) - except Exception: if __debug__: print(f'Invalid journal entry {line!r}') @@ -189,8 +184,9 @@ def main(): # stuff we can do when not docked if args.d: - with open(args.d, 'w') as f: - json.dump(data, f, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')) + out = json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')) + with open(args.d, 'wb') as f: + f.write(out.encode("utf-8")) if args.a: loadout.export(data, args.a) From 87b7f639bb159f540f9cedcb8f93a709accf0ed8 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 19:24:04 +0200 Subject: [PATCH 050/258] cleanup coriolis.py --- plugins/coriolis.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/coriolis.py b/plugins/coriolis.py index 2a75662e..162dc2bd 100644 --- a/plugins/coriolis.py +++ b/plugins/coriolis.py @@ -7,17 +7,21 @@ import io # Migrate settings from <= 3.01 from config import config + if not config.get('shipyard_provider') and config.getint('shipyard'): config.set('shipyard_provider', 'Coriolis') + config.delete('shipyard') -def plugin_start3(plugin_dir): +def plugin_start3(_): return 'Coriolis' -# Return a URL for the current ship + def shipyard_url(loadout, is_beta): - string = json.dumps(loadout, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8') # most compact representation + """Return a URL for the current ship""" + # most compact representation + string = json.dumps(loadout, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8') if not string: return False @@ -25,4 +29,7 @@ def shipyard_url(loadout, is_beta): with gzip.GzipFile(fileobj=out, mode='w') as f: f.write(string) - return (is_beta and 'https://beta.coriolis.io/import?data=' or 'https://coriolis.io/import?data=') + base64.urlsafe_b64encode(out.getvalue()).decode().replace('=', '%3D') + encoded = base64.urlsafe_b64encode(out.getvalue()).decode().replace('=', '%3D') + url = 'https://beta.coriolis.io/import?data=' if is_beta else 'https://coriolis.io/import?data=' + + return f"{url}{encoded}" From 5aaf88b2816118241a8432deb1921d8b7760023d Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 19:37:28 +0200 Subject: [PATCH 051/258] fix quoting style --- plugins/coriolis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/coriolis.py b/plugins/coriolis.py index 162dc2bd..21077886 100644 --- a/plugins/coriolis.py +++ b/plugins/coriolis.py @@ -32,4 +32,4 @@ def shipyard_url(loadout, is_beta): encoded = base64.urlsafe_b64encode(out.getvalue()).decode().replace('=', '%3D') url = 'https://beta.coriolis.io/import?data=' if is_beta else 'https://coriolis.io/import?data=' - return f"{url}{encoded}" + return f'{url}{encoded}' From e1752506c5111bc6e94dc9396734eaa65224d44d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 14:51:26 +0100 Subject: [PATCH 052/258] Move "only run once" code into def enforce_single_instance() --- EDMarketConnector.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 8867dfbf..c48ed103 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -880,13 +880,12 @@ class AppWindow(object): self.theme_menubar.grid_remove() self.blank_menubar.grid(row=0, columnspan=2, sticky=tk.NSEW) -# Run the app -if __name__ == "__main__": +def enforce_single_instance() -> None: # Ensure only one copy of the app is running under this user account. OSX does this automatically. Linux TODO. if platform == 'win32': import ctypes - from ctypes.wintypes import * + from ctypes.wintypes import HWND, LPWSTR, LPCWSTR, INT, BOOL, LPARAM EnumWindows = ctypes.windll.user32.EnumWindows GetClassName = ctypes.windll.user32.GetClassNameW GetClassName.argtypes = [HWND, LPWSTR, ctypes.c_int] @@ -935,6 +934,10 @@ if __name__ == "__main__": EnumWindows(enumwindowsproc, 0) +# Run the app +if __name__ == "__main__": + + enforce_single_instance() if getattr(sys, 'frozen', False): # By default py2exe tries to write log to dirname(sys.executable) which fails when installed import tempfile From cd4216d19c197bf5d282b2ab190ef227ece685c1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 15:00:21 +0100 Subject: [PATCH 053/258] logger setup and initialisation * Initial printing of version is now a logger.info(...) --- EDMarketConnector.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index c48ed103..ee7b1271 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -942,7 +942,24 @@ if __name__ == "__main__": # By default py2exe tries to write log to dirname(sys.executable) which fails when installed import tempfile sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), '%s.log' % appname), 'wt', 1) # unbuffered not allowed for text in python3, so use line buffering - print('%s %s %s' % (applongname, appversion, strftime('%Y-%m-%dT%H:%M:%S', localtime()))) + + ########################################################################### + # Set up a logging instance + import logging + + logger_default_loglevel = logging.INFO + logger = logging.getLogger(appname) + logger.setLevel(logger_default_loglevel) + logger_ch = logging.StreamHandler() + logger_ch.setLevel(logger_default_loglevel) + logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(funcName)s: %(message)s') + logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' + logger_formatter.default_msec_format = '%s.%03d' + logger_ch.setFormatter(logger_formatter) + logger.addHandler(logger_ch) + ########################################################################### + + logger.info(f'{applongname} {appversion}') Translations.install(config.get('language') or None) # Can generate errors so wait til log set up From c971106c0e7c9cd84a0e669467f3199954522bbe Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 16:51:16 +0100 Subject: [PATCH 054/258] Convert print()s to logging & refactor help_about -> HelpAbout --- EDMarketConnector.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index ee7b1271..465ca1f8 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -207,7 +207,7 @@ class AppWindow(object): self.help_menu.add_command(command=self.help_privacy) self.help_menu.add_command(command=self.help_releases) self.help_menu.add_command(command=lambda:self.updater.checkForUpdates()) - self.help_menu.add_command(command=lambda:not self.help_about.showing and self.help_about(self.w)) + self.help_menu.add_command(command=lambda: not self.HelpAbout.showing and self.HelpAbout(self.w)) self.menubar.add_cascade(menu=self.help_menu) if platform == 'win32': @@ -409,7 +409,7 @@ class AppWindow(object): except (companion.CredentialsError, companion.ServerError, companion.ServerLagging) as e: self.status['text'] = str(e) except Exception as e: - if __debug__: print_exc() + logger.debug(f'{__class__}', exc_info=e) self.status['text'] = str(e) self.cooldown() @@ -523,7 +523,7 @@ class AppWindow(object): self.login() except Exception as e: # Including CredentialsError, ServerError - if __debug__: print_exc() + logger.debug(f'{__class__}', exc_info=e) self.status['text'] = str(e) play_bad = True @@ -538,8 +538,14 @@ class AppWindow(object): # Try again to get shipyard data and send to EDDN. Don't report errors if can't get or send the data. try: data = companion.session.station() - if __debug__: - print('Retry for shipyard - ' + (data['commander'].get('docked') and (data.get('lastStarport', {}).get('ships') and 'Success' or 'Failure') or 'Undocked!')) + if data['commander'].get('docked'): + if data.get('lastStarport', {}).get('ships'): + report = 'Success' + else: + report ='Failure' + else: + report = 'Undocked!' + logger.debug(f'{__class__}: Retry for shipyard - {report}') if not data['commander'].get('docked'): pass # might have undocked while we were waiting for retry in which case station data is unreliable elif (data.get('lastSystem', {}).get('name') == monitor.system and @@ -607,10 +613,10 @@ class AppWindow(object): # Disable WinSparkle automatic update checks, IFF configured to do so when in-game if config.getint('disable_autoappupdatecheckingame') and 1: self.updater.setAutomaticUpdatesCheck(False) - print('Monitor: Disable WinSparkle automatic update checks') + logger.info(f'{__class__}: Monitor: Disable WinSparkle automatic update checks') # Can start dashboard monitoring if not dashboard.start(self.w, monitor.started): - print("Can't start Status monitoring") + logger.info(f"{__class__}: Can't start Status monitoring") # Export loadout if entry['event'] == 'Loadout' and not monitor.state['Captain'] and config.getint('output') & config.OUT_SHIP: @@ -631,7 +637,7 @@ class AppWindow(object): # Enable WinSparkle automatic update checks # NB: Do this blindly, in case option got changed whilst in-game self.updater.setAutomaticUpdatesCheck(True) - print('Monitor: Enable WinSparkle automatic update checks') + logger.info(f'{__class__}: Monitor: Enable WinSparkle automatic update checks') # cAPI auth def auth(self, event=None): @@ -647,7 +653,7 @@ class AppWindow(object): except companion.ServerError as e: self.status['text'] = str(e) except Exception as e: - if __debug__: print_exc() + logger.debug(f'{__class__}', exc_info=e) self.status['text'] = str(e) self.cooldown() @@ -724,7 +730,7 @@ class AppWindow(object): def help_releases(self, event=None): webbrowser.open('https://github.com/EDCD/EDMarketConnector/releases') - class help_about(tk.Toplevel): + class HelpAbout(tk.Toplevel): showing = False def __init__(self, parent): @@ -802,7 +808,7 @@ class AppWindow(object): self.protocol("WM_DELETE_WINDOW", self._destroy) ############################################################ - print('Current version is {}'.format(appversion)) + logger.info(f'{__class__}: Current version is {appversion}') def apply(self): self._destroy() @@ -830,7 +836,7 @@ class AppWindow(object): except companion.ServerError as e: self.status['text'] = str(e) except Exception as e: - if __debug__: print_exc() + logger.debug(f'{__class__}', exc_info=e) self.status['text'] = str(e) def onexit(self, event=None): @@ -952,7 +958,7 @@ if __name__ == "__main__": logger.setLevel(logger_default_loglevel) logger_ch = logging.StreamHandler() logger_ch.setLevel(logger_default_loglevel) - logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(funcName)s: %(message)s') + logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s:%(lineno)d:%(funcName)s: %(message)s') logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' logger_formatter.default_msec_format = '%s.%03d' logger_ch.setFormatter(logger_formatter) From fa1443b49cb82df92ed173ee06d7461250889510 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 17:24:16 +0100 Subject: [PATCH 055/258] Contributing.md: Document use of logging. Also fixes some camelCase to be proper snake_case in an example. --- Contributing.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/Contributing.md b/Contributing.md index ff683dff..c82a4cee 100644 --- a/Contributing.md +++ b/Contributing.md @@ -184,14 +184,14 @@ Coding Conventions Yes: ```python -if somethingTrue: - Things_we_then_do() +if something_true: + one_thing_we_do() ``` No: ```python -if somethingTrue: One_thing_we_do() +if something_true: one_thing_we_do() ``` Yes, some existing code still flouts this rule. @@ -224,6 +224,26 @@ No: * Going forwards please do place [type hints](https://docs.python.org/3/library/typing.html) on the declarations of your functions, both their arguments and return types. +* Use `logging` not `print()`, and definitely not `sys.stdout.write()`! + `EDMarketConnector.py` sets up `logger` for this, so: + + from EDMarketConnector import logger + + logger.info(f'Some message with a {variable}') + + try: + something + except Exception as e: # Try to be more specific + logger.error(f'Error in ... with ...', exc_info=e) + + Also if the code is within a class definition at all then please use: + + logger.info(f'{__class__}: ...) + + so that the class name is included in the output. No `logger.Formatter` + does not support this (probably because there's no way to get at the + information for a calling function). + * In general, please follow [PEP8](https://www.python.org/dev/peps/pep-0008/). --- From fe0ff2498edb8b226164f3fab6fe31498b6b6fae Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 17:25:21 +0100 Subject: [PATCH 056/258] Moves logger definition so it's import'able --- EDMarketConnector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 465ca1f8..aa5bb0bb 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -17,6 +17,7 @@ from time import gmtime, time, localtime, strftime, strptime import _strptime # Workaround for http://bugs.python.org/issue7980 from calendar import timegm import webbrowser +import logging from config import appname, applongname, appversion, appversion_nobuild, copyright, config @@ -940,6 +941,8 @@ def enforce_single_instance() -> None: EnumWindows(enumwindowsproc, 0) +# Has to be here to be defined for other modules importing +logger = logging.getLogger(appname) # Run the app if __name__ == "__main__": @@ -951,10 +954,7 @@ if __name__ == "__main__": ########################################################################### # Set up a logging instance - import logging - logger_default_loglevel = logging.INFO - logger = logging.getLogger(appname) logger.setLevel(logger_default_loglevel) logger_ch = logging.StreamHandler() logger_ch.setLevel(logger_default_loglevel) From 23b167fe66525e3896b6d45f57f571e441b5295a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 17:31:48 +0100 Subject: [PATCH 057/258] Change startup version back to a print() This is so early it doesn't need to be a spammy log message. --- EDMarketConnector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index aa5bb0bb..bc7762b9 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -965,7 +965,8 @@ if __name__ == "__main__": logger.addHandler(logger_ch) ########################################################################### - logger.info(f'{applongname} {appversion}') + # Plain, not via `logger` + print(f'{applongname} {appversion}') Translations.install(config.get('language') or None) # Can generate errors so wait til log set up From 2e12513c6cf043ad8634c14fd08d07330ec46490 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 17:44:49 +0100 Subject: [PATCH 058/258] Changes plug.py to proper logging. This relies on the logger.Formatter setup to fill in function name rather than calling out exactly why we're logging. It should be obvious, i.e. if you're in plug.py:notify_journal_entry then the named plugin threw an un-caught exception trying to call its method for this. --- plug.py | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/plug.py b/plug.py index 56eaa884..2efc1ecb 100644 --- a/plug.py +++ b/plug.py @@ -14,6 +14,7 @@ import tkinter as tk import myNotebook as nb from config import config +from EDMarketConnector import logger from time import time as time @@ -92,7 +93,7 @@ class Plugin(object): self.module = None # None for disabled plugins. if loadfile: - sys.stdout.write('loading plugin {} from "{}"\n'.format(name.replace('.', '_'), loadfile)) + logger.info(f'loading plugin "{name.replace(".", "_")}" from "{loadfile}"') try: module = importlib.machinery.SourceFileLoader('plugin_{}'.format(name.encode(encoding='ascii', errors='replace').decode('utf-8').replace('.', '_')), loadfile).load_module() if getattr(module, 'plugin_start3', None): @@ -100,15 +101,15 @@ class Plugin(object): self.name = newname and str(newname) or name self.module = module elif getattr(module, 'plugin_start', None): - sys.stdout.write('plugin %s needs migrating\n' % name) + logger.warning(f'plugin {name} needs migrating\n') PLUGINS_not_py3.append(self) else: - sys.stdout.write('plugin %s has no plugin_start3() function\n' % name) - except: - print_exc() + logger.error(f'plugin {name} has no plugin_start3() function') + except Exception as e: + logger.error(f'{__class__}: Failed for Plugin "{name}"', exc_info=e) raise else: - sys.stdout.write('plugin %s disabled\n' % name) + logger.info(f'{__class__}: plugin {name} disabled') def _get_func(self, funcname): """ @@ -136,8 +137,8 @@ class Plugin(object): elif not isinstance(appitem, tk.Widget): raise AssertionError return appitem - except: - print_exc() + except Exception as e: + logger.error(f'{__class__}: Failed for Plugin "{self.name}"', exc_info=e) return None def get_prefs(self, parent, cmdr, is_beta): @@ -156,8 +157,8 @@ class Plugin(object): if not isinstance(frame, nb.Frame): raise AssertionError return frame - except: - print_exc() + except Exception as e: + logger.error(f'{__class__}: Failed for Plugin "{self.name}"', exc_info=e) return None @@ -174,8 +175,8 @@ def load_plugins(master): plugin = Plugin(name[:-3], os.path.join(config.internal_plugin_dir, name)) plugin.folder = None # Suppress listing in Plugins prefs tab internal.append(plugin) - except: - pass + except Exception as e: + logger.error(f'Failure loading internal Plugin "{name}"', exc_info=e) PLUGINS.extend(sorted(internal, key = lambda p: operator.attrgetter('name')(p).lower())) # Add plugin folder to load path so packages can be loaded from plugin folder @@ -195,7 +196,8 @@ def load_plugins(master): # Add plugin's folder to load path in case plugin has internal package dependencies sys.path.append(os.path.join(config.plugin_dir, name)) found.append(Plugin(name, os.path.join(config.plugin_dir, name, 'load.py'))) - except: + except Exception as e: + logger.error(f'Failure loading found Plugin "{name}"', exc_info=e) pass PLUGINS.extend(sorted(found, key = lambda p: operator.attrgetter('name')(p).lower())) @@ -240,8 +242,8 @@ def notify_stop(): try: newerror = plugin_stop() error = error or newerror - except: - print_exc() + except Exception as e: + logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) return error @@ -257,8 +259,8 @@ def notify_prefs_cmdr_changed(cmdr, is_beta): if prefs_cmdr_changed: try: prefs_cmdr_changed(cmdr, is_beta) - except: - print_exc() + except Exception as e: + logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) def notify_prefs_changed(cmdr, is_beta): @@ -275,8 +277,8 @@ def notify_prefs_changed(cmdr, is_beta): if prefs_changed: try: prefs_changed(cmdr, is_beta) - except: - print_exc() + except Exception as e: + logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) def notify_journal_entry(cmdr, is_beta, system, station, entry, state): @@ -298,8 +300,8 @@ def notify_journal_entry(cmdr, is_beta, system, station, entry, state): # Pass a copy of the journal entry in case the callee modifies it newerror = journal_entry(cmdr, is_beta, system, station, dict(entry), dict(state)) error = error or newerror - except: - print_exc() + except Exception as e: + logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) return error @@ -319,8 +321,8 @@ def notify_dashboard_entry(cmdr, is_beta, entry): # Pass a copy of the status entry in case the callee modifies it newerror = status(cmdr, is_beta, dict(entry)) error = error or newerror - except: - print_exc() + except Exception as e: + logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) return error @@ -338,8 +340,8 @@ def notify_newdata(data, is_beta): try: newerror = cmdr_data(data, is_beta) error = error or newerror - except: - print_exc() + except Exception as e: + logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) return error From debc7f97d3916feec4e941e71b55b5e58e7a0250 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 20:42:06 +0100 Subject: [PATCH 059/258] Use a logging.Filter to implement %(class)s in formatting. --- EDMarketConnector.py | 80 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index bc7762b9..6d0ff021 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -18,6 +18,8 @@ import _strptime # Workaround for http://bugs.python.org/issue7980 from calendar import timegm import webbrowser import logging +import traceback +from typing import Any, Optional from config import appname, applongname, appversion, appversion_nobuild, copyright, config @@ -941,8 +943,68 @@ def enforce_single_instance() -> None: EnumWindows(enumwindowsproc, 0) +########################################################################### +# Logging + # Has to be here to be defined for other modules importing logger = logging.getLogger(appname) + +class EDMCContextFilter(logging.Filter): + """ + logging.Filter sub-class to place the calling __class__ in the record. + """ + def filter(self, record: logging.LogRecord) -> bool: + """ + + :param record: + :return: bool - True for this record to be logged. + """ + record.__dict__['class'] = self.caller_class() + return True + + def caller_class(self, skip=5) -> str: + """ + Figure out our caller class. + + Ref: + + :param skip: How many stack frames above to look. + :return: str: The class name. + """ + import inspect + + def stack_(frame): + framelist = [] + while frame: + framelist.append(frame) + frame = frame.f_back + return framelist + + stack = stack_(sys._getframe(1)) + start = 0 + skip + if len(stack) < start + 1: + return '' + + class_name = [] + frame = stack[start] + module = inspect.getmodule(frame) + # `modname` can be None when frame is executed directly in console + # TODO(techtonik): consider using __main__ + #if module: + # class_name.append(module.__name__) + # detect classname + if 'self' in frame.f_locals: + # I don't know any way to detect call from the object method + # XXX: there seems to be no way to detect static method call - it will + # be just a function call + class_name.insert(0, frame.f_locals['self'].__class__.__qualname__) + codename = frame.f_code.co_name + #if codename != '': # top level usually + # class_name.append(codename) # function or a method + + return ".".join(class_name) +########################################################################### + # Run the app if __name__ == "__main__": @@ -954,19 +1016,33 @@ if __name__ == "__main__": ########################################################################### # Set up a logging instance - logger_default_loglevel = logging.INFO + logger_default_loglevel = logging.DEBUG logger.setLevel(logger_default_loglevel) + + # Set up filter for adding class name + logger_f = EDMCContextFilter() + logger.addFilter(logger_f) + logger_ch = logging.StreamHandler() logger_ch.setLevel(logger_default_loglevel) - logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s:%(lineno)d:%(funcName)s: %(message)s') + + logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s<.%(class)s>.%(funcName)s:%(lineno)d: %(message)s') logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' logger_formatter.default_msec_format = '%s.%03d' + logger_ch.setFormatter(logger_formatter) logger.addHandler(logger_ch) ########################################################################### # Plain, not via `logger` print(f'{applongname} {appversion}') + logger.info('Logging test from __main__') + class A(object): + class B(object): + def __init__(self): + logger.info('Test from A.B.__init__') + + ab = A.B() Translations.install(config.get('language') or None) # Can generate errors so wait til log set up From 6b9d4a11cbcc78a3059e3393022749138578436b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:02:16 +0100 Subject: [PATCH 060/258] Use a logging.Filter to implement %(qualname)s in formatting. This gets the function that called the logging, and if it's in a class it looks up the function via getattr(, ) and then uses __qualname__ to quickly get the fully qualified name. --- EDMarketConnector.py | 48 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 6d0ff021..bec122df 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -960,6 +960,7 @@ class EDMCContextFilter(logging.Filter): :return: bool - True for this record to be logged. """ record.__dict__['class'] = self.caller_class() + record.__dict__['qualname'] = self.caller_qualname() return True def caller_class(self, skip=5) -> str: @@ -987,22 +988,47 @@ class EDMCContextFilter(logging.Filter): class_name = [] frame = stack[start] - module = inspect.getmodule(frame) - # `modname` can be None when frame is executed directly in console - # TODO(techtonik): consider using __main__ - #if module: - # class_name.append(module.__name__) - # detect classname if 'self' in frame.f_locals: # I don't know any way to detect call from the object method # XXX: there seems to be no way to detect static method call - it will # be just a function call - class_name.insert(0, frame.f_locals['self'].__class__.__qualname__) - codename = frame.f_code.co_name - #if codename != '': # top level usually - # class_name.append(codename) # function or a method + class_name.append(frame.f_locals['self'].__class__.__qualname__) return ".".join(class_name) + + def caller_qualname(self, skip=5) -> str: + """ + Figure out our caller's qualname + + Ref: + + :param skip: How many stack frames above to look. + :return: str: The caller's qualname + """ + import inspect + + def stack_(frame): + framelist = [] + while frame: + framelist.append(frame) + frame = frame.f_back + return framelist + + stack = stack_(sys._getframe(1)) + start = 0 + skip + if len(stack) < start + 1: + return '' + + class_name = [] + frame = stack[start] + if 'self' in frame.f_locals: + # I don't know any way to detect call from the object method + # XXX: there seems to be no way to detect static method call - it will + # be just a function call + class_name.append(getattr(frame.f_locals['self'], frame.f_code.co_name).__qualname__) + + return ".".join(class_name) + ########################################################################### # Run the app @@ -1026,7 +1052,7 @@ if __name__ == "__main__": logger_ch = logging.StreamHandler() logger_ch.setLevel(logger_default_loglevel) - logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s<.%(class)s>.%(funcName)s:%(lineno)d: %(message)s') + logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(message)s') logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' logger_formatter.default_msec_format = '%s.%03d' From 5af87a5b982bd4a41ca297a8c7c3ed3910fed92a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:04:22 +0100 Subject: [PATCH 061/258] Clean up EDMarketConnector.py after that qualname testing --- EDMarketConnector.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index bec122df..458ebb41 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1041,7 +1041,7 @@ if __name__ == "__main__": sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), '%s.log' % appname), 'wt', 1) # unbuffered not allowed for text in python3, so use line buffering ########################################################################### - # Set up a logging instance + # Configure the logging.Logger logger_default_loglevel = logging.DEBUG logger.setLevel(logger_default_loglevel) @@ -1062,13 +1062,6 @@ if __name__ == "__main__": # Plain, not via `logger` print(f'{applongname} {appversion}') - logger.info('Logging test from __main__') - class A(object): - class B(object): - def __init__(self): - logger.info('Test from A.B.__init__') - - ab = A.B() Translations.install(config.get('language') or None) # Can generate errors so wait til log set up From 818bd89fd7e195417cb437a05cc6e0a5b3c42333 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:05:48 +0100 Subject: [PATCH 062/258] Remove un-necessary {__class__} from logging messages --- plug.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plug.py b/plug.py index 2efc1ecb..01694520 100644 --- a/plug.py +++ b/plug.py @@ -106,10 +106,10 @@ class Plugin(object): else: logger.error(f'plugin {name} has no plugin_start3() function') except Exception as e: - logger.error(f'{__class__}: Failed for Plugin "{name}"', exc_info=e) + logger.error(f': Failed for Plugin "{name}"', exc_info=e) raise else: - logger.info(f'{__class__}: plugin {name} disabled') + logger.info(f'plugin {name} disabled') def _get_func(self, funcname): """ @@ -138,7 +138,7 @@ class Plugin(object): raise AssertionError return appitem except Exception as e: - logger.error(f'{__class__}: Failed for Plugin "{self.name}"', exc_info=e) + logger.error(f'Failed for Plugin "{self.name}"', exc_info=e) return None def get_prefs(self, parent, cmdr, is_beta): @@ -158,7 +158,7 @@ class Plugin(object): raise AssertionError return frame except Exception as e: - logger.error(f'{__class__}: Failed for Plugin "{self.name}"', exc_info=e) + logger.error(f'Failed for Plugin "{self.name}"', exc_info=e) return None From 6aa409388ea9f7afbd1e049963489e3bf5771a46 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:08:53 +0100 Subject: [PATCH 063/258] plugins/eddn: Converted to proper logging --- plugins/eddn.py | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 15d7941c..5853cb13 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -15,13 +15,11 @@ from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb from prefs import prefsVersion +from EDMarketConnector import logger if sys.platform != 'win32': from fcntl import lockf, LOCK_EX, LOCK_NB -if __debug__: - from traceback import print_exc - from config import applongname, appversion, config from companion import category_map @@ -62,15 +60,15 @@ class EDDN(object): try: # Try to open existing file self.replayfile = open(filename, 'r+', buffering=1) - except: + except Exception as e: if exists(filename): raise # Couldn't open existing file else: self.replayfile = open(filename, 'w+', buffering=1) # Create file if sys.platform != 'win32': # open for writing is automatically exclusive on Windows lockf(self.replayfile, LOCK_EX|LOCK_NB) - except: - if __debug__: print_exc() + except Exception as e: + logger.debug(f'Failed opening "replay.jsonl"', exc_info=e) if self.replayfile: self.replayfile.close() self.replayfile = None @@ -104,11 +102,14 @@ class EDDN(object): ]) r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=self.TIMEOUT) - if __debug__ and r.status_code != requests.codes.ok: - print('Status\t%s' % r.status_code) - print('URL\t%s' % r.url) - print('Headers\t%s' % r.headers) - print('Content:\n%s' % r.text) + if r.status_code != requests.codes.ok: + logger.debug(f''': +Status\t{r.status_code} +URL\t{r.url} +Headers\t{r.headers}' +Content:\n{r.text} +''' + ) r.raise_for_status() def sendreplay(self): @@ -128,11 +129,9 @@ class EDDN(object): self.parent.update_idletasks() try: cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict) - except: + except json.JSONDecodeError as e: # Couldn't decode - shouldn't happen! - if __debug__: - print(self.replaylog[0]) - print_exc() + logger.debug(f'\n{self.replaylog[0]}\n', exc_info=e) self.replaylog.pop(0) # Discard and continue else: # Rewrite old schema name @@ -144,11 +143,11 @@ class EDDN(object): if not len(self.replaylog) % self.REPLAYFLUSH: self.flush() except requests.exceptions.RequestException as e: - if __debug__: print_exc() + logger.debug(f'Failed sending', exc_info=e) status['text'] = _("Error: Can't connect to EDDN") return # stop sending except Exception as e: - if __debug__: print_exc() + logger.debug(f'Failed sending', exc_info=e) status['text'] = str(e) return # stop sending @@ -443,10 +442,10 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): try: this.eddn.export_journal_entry(cmdr, is_beta, filter_localised(entry)) except requests.exceptions.RequestException as e: - if __debug__: print_exc() + logger.debug(f'Failed in export_journal_entry', exc_info=e) return _("Error: Can't connect to EDDN") except Exception as e: - if __debug__: print_exc() + logger.debug(f'Failed in export_journal_entry', exc_info=e) return str(e) elif (config.getint('output') & config.OUT_MKT_EDDN and not state['Captain'] and @@ -466,10 +465,10 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.eddn.export_journal_shipyard(cmdr, is_beta, entry) except requests.exceptions.RequestException as e: - if __debug__: print_exc() + logger.debug(f'Failed exporting {entry["event"]}', exc_info=e) return _("Error: Can't connect to EDDN") except Exception as e: - if __debug__: print_exc() + logger.debug(f'Failed exporting {entry["event"]}', exc_info=e) return str(e) def cmdr_data(data, is_beta): @@ -492,9 +491,9 @@ def cmdr_data(data, is_beta): status.update_idletasks() except requests.RequestException as e: - if __debug__: print_exc() + logger.debug(f'Failed exporting data', exc_info=e) return _("Error: Can't connect to EDDN") except Exception as e: - if __debug__: print_exc() + logger.debug(f'Failed exporting data', exc_info=e) return str(e) From 7951463fba064876c407df2fb7b1714bfc0ae363 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:17:59 +0100 Subject: [PATCH 064/258] plugins/edsm: Converted to proper logging --- plugins/edsm.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plugins/edsm.py b/plugins/edsm.py index 34b64b80..ff29e099 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -23,11 +23,10 @@ import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb +from EDMarketConnector import logger from config import appname, applongname, appversion, config import plug -if __debug__: - from traceback import print_exc EDSM_POLL = 0.1 _TIMEOUT = 20 @@ -359,7 +358,7 @@ def worker(): (msgnum, msg) = reply['msgnum'], reply['msg'] # 1xx = OK, 2xx = fatal error, 3&4xx not generated at top-level, 5xx = error but events saved for later processing if msgnum // 100 == 2: - print('EDSM\t%s %s\t%s' % (msgnum, msg, json.dumps(pending, separators = (',', ': ')))) + logger.warning(f'EDSM\t{msgnum} {msg}\t{json.dumps(pending, separators = (",", ": "))}') plug.show_error(_('Error: EDSM {MSG}').format(MSG=msg)) else: for e, r in zip(pending, reply['events']): @@ -368,12 +367,12 @@ def worker(): this.lastlookup = r this.system_link.event_generate('<>', when="tail") # calls update_status in main thread elif r['msgnum'] // 100 != 1: - print('EDSM\t%s %s\t%s' % (r['msgnum'], r['msg'], json.dumps(e, separators = (',', ': ')))) + logger.warning(f'EDSM\t{r["msgnum"]} {r["msg"]}\t{json.dumps(e, separators = (",", ": "))}') pending = [] break - except: - if __debug__: print_exc() + except Exception as e: + logger.debug(f'Sending API events', exc_info=e) retrying += 1 else: plug.show_error(_("Error: Can't connect to EDSM")) From 89cadbc0eb9f7a92304eeb2e091f9681490a7d49 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:32:21 +0100 Subject: [PATCH 065/258] Add paranoia checks to class and qualname finders --- EDMarketConnector.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 458ebb41..f280a9af 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -986,15 +986,15 @@ class EDMCContextFilter(logging.Filter): if len(stack) < start + 1: return '' - class_name = [] + class_name = '' frame = stack[start] if 'self' in frame.f_locals: - # I don't know any way to detect call from the object method - # XXX: there seems to be no way to detect static method call - it will - # be just a function call - class_name.append(frame.f_locals['self'].__class__.__qualname__) + # Paranoia checks + frame_class = frame.f_locals['self'].__class__ + if frame_class and frame_class.__qualname__: + class_name = frame_class.__qualname__ - return ".".join(class_name) + return class_name def caller_qualname(self, skip=5) -> str: """ @@ -1019,15 +1019,16 @@ class EDMCContextFilter(logging.Filter): if len(stack) < start + 1: return '' - class_name = [] + qualname = '' frame = stack[start] - if 'self' in frame.f_locals: - # I don't know any way to detect call from the object method - # XXX: there seems to be no way to detect static method call - it will - # be just a function call - class_name.append(getattr(frame.f_locals['self'], frame.f_code.co_name).__qualname__) + if frame.f_locals and 'self' in frame.f_locals: + # Paranoia checks + if frame.f_code and frame.f_code.co_name: + fn = getattr(frame.f_locals['self'], frame.f_code.co_name) + if fn and fn.__qualname__: + qualname = n.__qualname__ - return ".".join(class_name) + return qualname ########################################################################### From 4d7f81cec1ea80f8b9cd8679a1622950e1177e66 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:34:50 +0100 Subject: [PATCH 066/258] Add emergency print()s if we can't find class/qualname --- EDMarketConnector.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index f280a9af..e8f25a67 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -991,9 +991,13 @@ class EDMCContextFilter(logging.Filter): if 'self' in frame.f_locals: # Paranoia checks frame_class = frame.f_locals['self'].__class__ + if frame_class and frame_class.__qualname__: class_name = frame_class.__qualname__ + if class_name = '': + print('ALERT! Something went wrong with finding class name for logging!') + return class_name def caller_qualname(self, skip=5) -> str: @@ -1023,11 +1027,16 @@ class EDMCContextFilter(logging.Filter): frame = stack[start] if frame.f_locals and 'self' in frame.f_locals: # Paranoia checks + if frame.f_code and frame.f_code.co_name: fn = getattr(frame.f_locals['self'], frame.f_code.co_name) + if fn and fn.__qualname__: qualname = n.__qualname__ + if qualname == '': + print('ALERT! Something went wrong with finding caller qualname for logging!') + return qualname ########################################################################### From 7d060aa325e1543e70f95c3e216ea39e0e0e351f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 21:39:18 +0100 Subject: [PATCH 067/258] plugins/inara: Convert to proper logging --- plugins/inara.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index dcfd5c70..0288076c 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -15,12 +15,10 @@ import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb +from EDMarketConnector import logger from config import appname, applongname, appversion, config import plug -if __debug__: - from traceback import print_exc - _TIMEOUT = 20 FAKE = ['CQC', 'Training', 'Destination'] # Fake systems that shouldn't be sent to Inara @@ -457,7 +455,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): call() except Exception as e: - if __debug__: print_exc() + logger.debug(f'Adding events', exc_info=e) return str(e) # @@ -898,14 +896,15 @@ def worker(): callback(reply) elif status // 100 != 2: # 2xx == OK (maybe with warnings) # Log fatal errors - print('Inara\t%s %s' % (reply['header']['eventStatus'], reply['header'].get('eventStatusText', ''))) - print(json.dumps(data, indent=2, separators = (',', ': '))) + log.warning(f'Inara\t{status} {reply["header"].get("eventStatusText", "")}') + log.debug(f'JSON data:\n{json.dumps(data, indent=2, separators = (",", ": "))}') plug.show_error(_('Error: Inara {MSG}').format(MSG = reply['header'].get('eventStatusText', status))) else: # Log individual errors and warnings for data_event, reply_event in zip(data['events'], reply['events']): if reply_event['eventStatus'] != 200: - print('Inara\t%s %s\t%s' % (reply_event['eventStatus'], reply_event.get('eventStatusText', ''), json.dumps(data_event))) + logger.warning(f'Inara\t{status} {reply_event.get("eventStatusText", "")}') + logger.debug(f'JSON data:\n{json.dumps(data_event)}') if reply_event['eventStatus'] // 100 != 2: plug.show_error(_('Error: Inara {MSG}').format(MSG = '%s, %s' % (data_event['eventName'], reply_event.get('eventStatusText', reply_event['eventStatus'])))) if data_event['eventName'] in ['addCommanderTravelCarrierJump', 'addCommanderTravelDock', 'addCommanderTravelFSDJump', 'setCommanderTravelLocation']: @@ -916,8 +915,8 @@ def worker(): this.system_link.event_generate('<>', when="tail") # calls update_ship in main thread break - except: - if __debug__: print_exc() + except Exceptionas e: + logger.debug(f'Sending events', exc_info=e) retrying += 1 else: if callback: From 8e1f3b8a90fd167c52213de1b1e1727093bff365 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 22:01:29 +0100 Subject: [PATCH 068/258] plugins/inara: Fix typo on except: --- plugins/inara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index 0288076c..1fcf5911 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -915,7 +915,7 @@ def worker(): this.system_link.event_generate('<>', when="tail") # calls update_ship in main thread break - except Exceptionas e: + except Exception as e: logger.debug(f'Sending events', exc_info=e) retrying += 1 else: From 2176187be712d00787eadaa9e2f18428602bc8ef Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 22:02:19 +0100 Subject: [PATCH 069/258] Fix typos in class/qualname finders --- EDMarketConnector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index e8f25a67..1fdc1a9f 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -995,7 +995,7 @@ class EDMCContextFilter(logging.Filter): if frame_class and frame_class.__qualname__: class_name = frame_class.__qualname__ - if class_name = '': + if class_name == '': print('ALERT! Something went wrong with finding class name for logging!') return class_name @@ -1032,7 +1032,7 @@ class EDMCContextFilter(logging.Filter): fn = getattr(frame.f_locals['self'], frame.f_code.co_name) if fn and fn.__qualname__: - qualname = n.__qualname__ + qualname = fn.__qualname__ if qualname == '': print('ALERT! Something went wrong with finding caller qualname for logging!') From bae2f25e19fbe5fb837956f7f7e951ae0caab6d7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 22:17:33 +0100 Subject: [PATCH 070/258] companion.py: Convert to proper logging --- companion.py | 69 +++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/companion.py b/companion.py index 57911342..62d07282 100644 --- a/companion.py +++ b/companion.py @@ -14,12 +14,14 @@ import os from os.path import join import random import time -from traceback import print_exc import urllib.parse import webbrowser from config import appname, appversion, config from protocol import protocolhandler +import logging +logger = logging.getLogger(appname) + from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -196,18 +198,17 @@ class Auth(object): return data.get('access_token') else: - print('Auth\tCan\'t refresh token for {}'.format(self.cmdr)) + logger.error(f"Frontier CAPI Auth: Can't refresh token for \"{self.cmdr}\"") self.dump(r) - except Exception: - print('Auth\tCan\'t refresh token for {}'.format(self.cmdr)) - print_exc() + except Exception as e: + logger.error(f"Frontier CAPI Auth: Can't refresh token for \"{self.cmdr}\"", exc_info=e) else: - print('Auth\tNo token for {}'.format(self.cmdr)) + logger.error(f"Frontier CAPI Auth: No token for \"{self.cmdr}\"", exc_info=e) # New request - print('Auth\tNew authorization request') + logger.info(f'Frontier CAPI Auth: New authorization request') v = random.SystemRandom().getrandbits(8 * 32) self.verifier = self.base64URLEncode(v.to_bytes(32, byteorder='big')).encode('utf-8') s = random.SystemRandom().getrandbits(8 * 32) @@ -228,16 +229,16 @@ class Auth(object): # Handle OAuth authorization code callback. # Returns access token if successful, otherwise raises CredentialsError if '?' not in payload: - print('Auth\tMalformed response {!r}'.format(payload)) + logger.error(f'Frontier CAPI Auth: Malformed response\n{payload}\n') raise CredentialsError('malformed payload') # Not well formed data = urllib.parse.parse_qs(payload[(payload.index('?') + 1):]) if not self.state or not data.get('state') or data['state'][0] != self.state: - print('Auth\tUnexpected response {!r}'.format(payload)) + logger.error(f'Frontier CAPI Auth: Unexpected response\n{payload}\n') raise CredentialsError('Unexpected response from authorization {!r}'.format(payload)) # Unexpected reply if not data.get('code'): - print('Auth\tNegative response {!r}'.format(payload)) + logger.error(f'Frontier CAPI Auth: Negative response\n{payload}\n') error = next( (data[k] for k in ('error_description', 'error', 'message') if k in data), ('',) ) @@ -256,7 +257,7 @@ class Auth(object): r = self.session.post(SERVER_AUTH + URL_TOKEN, data=data, timeout=auth_timeout) data = r.json() if r.status_code == requests.codes.ok: - print('Auth\tNew token for {}'.format(self.cmdr)) + logger.info(f'Frontier CAPI Auth: New token for \"{self.cmdr}\"') cmdrs = config.get('cmdrs') idx = cmdrs.index(self.cmdr) tokens = config.get('fdev_apikeys') or [] @@ -268,21 +269,20 @@ class Auth(object): return data.get('access_token') except Exception as e: - print('Auth\tCan\'t get token for {}'.format(self.cmdr)) - print_exc() + logger.error(f"Frontier CAPI Auth: Can't get token for \"{self.cmdr}\"", exc_info=e) if r: self.dump(r) raise CredentialsError('unable to get token') from e - print('Auth\tCan\'t get token for {}'.format(self.cmdr)) + logger.error(f"Frontier CAPI Auth: Can't get token for \"{self.cmdr}\"") self.dump(r) error = next((data[k] for k in ('error_description', 'error', 'message') if k in data), ('',)) raise CredentialsError('Error: {!r}'.format(error)[0]) @staticmethod def invalidate(cmdr): - print('Auth\tInvalidated token for {}'.format(cmdr)) + logger.info(f'Frontier CAPI Auth: Invalidated token for "{cmdr}"') cmdrs = config.get('cmdrs') idx = cmdrs.index(cmdr) tokens = config.get('fdev_apikeys') or [] @@ -292,7 +292,7 @@ class Auth(object): config.save() # Save settings now for use by command-line app def dump(self, r): - print('Auth\t' + r.url, r.status_code, r.reason if r.reason else 'None', r.text) + logger.debug(f'Frontier CAPI Auth: {r.url} {r.status_code} {r.reason if r.reason else "None"} {r.text}') def base64URLEncode(self, text): return base64.urlsafe_b64encode(text).decode().replace('=', '') @@ -379,9 +379,7 @@ class Session(object): r = self.session.get(self.server + endpoint, timeout=timeout) except Exception as e: - if __debug__: - print_exc() - + logger.debug(f'Attempting GET', exc_info=e) raise ServerError('unable to get endpoint {}'.format(endpoint)) from e if r.url.startswith(SERVER_AUTH): @@ -402,7 +400,7 @@ class Session(object): data = r.json() # May also fail here if token expired since response is empty except (requests.HTTPError, ValueError) as e: - print_exc() + logger.error(f'Frontier CAPI Auth: GET ', exc_info=e) self.dump(r) self.close() @@ -410,6 +408,7 @@ class Session(object): self.invalidate() self.retrying = False self.login() + logger.error(f'Frontier CAPI Auth: query failed after refresh') raise CredentialsError('query failed after refresh') from e elif self.login(): # Maybe our token expired. Re-authorize in any case @@ -418,6 +417,7 @@ class Session(object): else: self.retrying = False + logger.error(f'Frontier CAPI Auth: HTTP error or invalid JSON') raise CredentialsError('HTTP error or invalid JSON') from e self.retrying = False @@ -461,9 +461,8 @@ class Session(object): try: self.session.close() - except Exception: - if __debug__: - print_exc() + except Exception as e: + logger.debug(f'Frontier CAPI Auth: closing', exc_info=e) self.session = None @@ -473,7 +472,7 @@ class Session(object): Auth.invalidate(self.credentials['cmdr']) def dump(self, r): - print('cAPI\t' + r.url, r.status_code, r.reason and r.reason or 'None', r.text) + logger.error(f'Frontier CAPI Auth: {r.url} {r.status_code} {r.reason and r.reason or "None"} {r.text}') # Returns a shallow copy of the received data suitable for export to older tools # English commodity names and anomalies fixed up @@ -497,15 +496,7 @@ def fixup(data): # But also see https://github.com/Marginal/EDMarketConnector/issues/32 for thing in ('buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket'): if not isinstance(commodity.get(thing), numbers.Number): - if __debug__: - print( - 'Invalid {!r}:{!r} ({}) for {!r}'.format( - thing, - commodity.get(thing), - type(commodity.get(thing)), - commodity.get('name', '') - ) - ) + logger.debug(f'Invalid {thing}:{commodity.get(thing)} ({type(commodity.get(thing))}) for {commodity.get("name", "")}') break else: @@ -520,20 +511,16 @@ def fixup(data): pass elif not commodity.get('categoryname'): - if __debug__: - print('Missing "categoryname" for {!r}'.format(commodity.get('name', ''))) + logger.debug(f'Missing "categoryname" for {commodity.get("name", "")}') elif not commodity.get('name'): - if __debug__: - print('Missing "name" for a commodity in {!r}'.format(commodity.get('categoryname', ''))) + logger.debug(f'Missing "name" for a commodity in {commodity.get("categoryname", "")}') elif not commodity['demandBracket'] in range(4): - if __debug__: - print('Invalid "demandBracket":{!r} for {!r}'.format(commodity['demandBracket'], commodity['name'])) + logger.debug(f'Invalid "demandBracket":{commodity["demandBracket"]} for {commodity["name"]}') elif not commodity['stockBracket'] in range(4): - if __debug__: - print('Invalid "stockBracket":{!r} for {!r}'.format(commodity['stockBracket'], commodity['name'])) + logger.debug(f'Invalid "stockBracket":{commodity["stockBracket"]} for {commodity["name"]}') else: # Rewrite text fields From 2e58d106bd9f7281a82caf0447b8915130bca6e6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 22:23:40 +0100 Subject: [PATCH 071/258] logging.getLogger(appname) instead of import from EDMarketConnector Using: from EDMarketConnector import logger causes issues if EDMarketConnector is already importing 'this' file. So just get a logger using logger.getLogger(appname) instead. `from config import appname` if needs be. --- plug.py | 8 ++++---- plugins/eddn.py | 5 +++-- plugins/edsm.py | 3 ++- plugins/inara.py | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/plug.py b/plug.py index 01694520..50d9af07 100644 --- a/plug.py +++ b/plug.py @@ -8,15 +8,15 @@ import importlib import sys import operator import threading # We don't use it, but plugins might -from traceback import print_exc - +import logging import tkinter as tk + import myNotebook as nb -from config import config -from EDMarketConnector import logger +from config import config, appname from time import time as time +logger = logging.getLogger(appname) # Dashboard Flags constants FlagsDocked = 1<<0 # on a landing pad diff --git a/plugins/eddn.py b/plugins/eddn.py index 5853cb13..e2e2b141 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -9,20 +9,21 @@ import re import requests import sys import uuid +import logging import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb from prefs import prefsVersion -from EDMarketConnector import logger if sys.platform != 'win32': from fcntl import lockf, LOCK_EX, LOCK_NB -from config import applongname, appversion, config +from config import appname, applongname, appversion, config from companion import category_map +logger = logging.getLogger(appname) this = sys.modules[__name__] # For holding module globals diff --git a/plugins/edsm.py b/plugins/edsm.py index ff29e099..eb0ab84f 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -18,15 +18,16 @@ import time import urllib.request, urllib.error, urllib.parse from queue import Queue from threading import Thread +import logging import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb -from EDMarketConnector import logger from config import appname, applongname, appversion, config import plug +logger = logging.getLogger(appname) EDSM_POLL = 0.1 _TIMEOUT = 20 diff --git a/plugins/inara.py b/plugins/inara.py index 1fcf5911..abf221f4 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -10,14 +10,15 @@ import time from operator import itemgetter from queue import Queue from threading import Thread +import logging import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb -from EDMarketConnector import logger from config import appname, applongname, appversion, config import plug +logger = logging.getLogger(appname) _TIMEOUT = 20 From a82824c0e936d90fd8e91880de977c7c5464542b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jul 2020 22:41:30 +0100 Subject: [PATCH 072/258] Contributing.md: Update logging docs for auto qualname and proper import --- Contributing.md | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/Contributing.md b/Contributing.md index c82a4cee..ef2ab460 100644 --- a/Contributing.md +++ b/Contributing.md @@ -225,9 +225,12 @@ No: types. * Use `logging` not `print()`, and definitely not `sys.stdout.write()`! - `EDMarketConnector.py` sets up `logger` for this, so: + `EDMarketConnector.py` sets up a `logging.Logger` for this under the + `appname`, so: - from EDMarketConnector import logger + import logging + from config import appname + logger = logging.getLogger(appname) logger.info(f'Some message with a {variable}') @@ -235,14 +238,26 @@ No: something except Exception as e: # Try to be more specific logger.error(f'Error in ... with ...', exc_info=e) - - Also if the code is within a class definition at all then please use: + + **DO NOT** use the following, as you might cause a circular import: - logger.info(f'{__class__}: ...) + from EDMarketConnector import logger - so that the class name is included in the output. No `logger.Formatter` - does not support this (probably because there's no way to get at the - information for a calling function). + We have implemented a `logging.Filter` that adds support for the following + in `logging.Formatter()` strings: + + 1. `%(qualname)s` which gets the full `ClassA(.ClassB...).func` of the + calling function. + 1. `%(class)s` which gets just the immediately encloding class name + of the calling function. + + So don't worry about adding anything about the class or function you're + logging from, it's taken care of. + + *Do use a pertinent message, even when using `exc_info=...` to log an + exception*. e.g. Logging will know you were in your `get_foo()` function + but you should still tell it what actually (failed to have) happened + in there. * In general, please follow [PEP8](https://www.python.org/dev/peps/pep-0008/). From c196a38e09da396bb94c5c281148671339072b47 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 25 Jul 2020 06:53:40 +0100 Subject: [PATCH 073/258] Add logging TODOs. logger creation & frame detection * Nothing should "from EDMarketConnector import logger" any more, so we can move this back inside __main__ section * We shouldn't rely on a magic number of frames to skip. Detect the proper frame automatically. --- EDMarketConnector.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 1fdc1a9f..6685ddd0 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -947,6 +947,7 @@ def enforce_single_instance() -> None: # Logging # Has to be here to be defined for other modules importing +# TODO: now nothing should import this, can it move back into __main__ block ? logger = logging.getLogger(appname) class EDMCContextFilter(logging.Filter): @@ -982,6 +983,11 @@ class EDMCContextFilter(logging.Filter): return framelist stack = stack_(sys._getframe(1)) + # TODO: Make this less fragile by not depending on a magic number of + # stack frames to skip. Should be able to find the last + # logger frame, where one of the critical, debug etc functions + # was called and then use the next frame before that. + # ALSO UPDATE caller_qualname() !!! start = 0 + skip if len(stack) < start + 1: return '' From 596527bda29266f0d015c457c8bc225f7ed7688e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 25 Jul 2020 14:46:10 +0100 Subject: [PATCH 074/258] Move logging setup to EDMCLogging.py with a class * Also now providers single caller_class_and_qualname() method to get both strings, returned as a Tuple[str, str]. Either could be empty if something went wrong. * Rather than a fragile 'skip' this now: 1. Looks for the first up-stack frame with self of logging.Logger 1. Then looks for the next up-stack frame with self NOT of logging.Logger. This should be the call site we want. --- EDMCLogging.py | 124 ++++++++++++++++++++++++++++++++++++++++ EDMarketConnector.py | 131 +++---------------------------------------- 2 files changed, 133 insertions(+), 122 deletions(-) create mode 100644 EDMCLogging.py diff --git a/EDMCLogging.py b/EDMCLogging.py new file mode 100644 index 00000000..24edac32 --- /dev/null +++ b/EDMCLogging.py @@ -0,0 +1,124 @@ +""" +TODO: blurb +""" + +import sys +import logging +from typing import Tuple + +class logger(object): + """ + TODO: desc + Wrapper class for all logging configuration and code. + """ + def __init__(self, logger_name: str, loglevel: int=logging.DEBUG): + """ + Set up a `logging.Logger` with our preferred configuration. + This includes using an EDMCContextFilter to add 'class' and 'qualname' + expansions for logging.Formatter(). + """ + self.logger = logging.getLogger(logger_name) + # Configure the logging.Logger + self.logger.setLevel(loglevel) + + # Set up filter for adding class name + self.logger_filter = EDMCContextFilter() + self.logger.addFilter(self.logger_filter) + + self.logger_channel = logging.StreamHandler() + self.logger_channel.setLevel(loglevel) + + self.logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(class)s: %(message)s') + self.logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' + self.logger_formatter.default_msec_format = '%s.%03d' + + self.logger_channel.setFormatter(self.logger_formatter) + self.logger.addHandler(self.logger_channel) + + def getLogger(self) -> logging.Logger: + return self.logger + + +class EDMCContextFilter(logging.Filter): + """ + TODO: Update this + logging.Filter sub-class to place the calling __class__ in the record. + """ + def filter(self, record: logging.LogRecord) -> bool: + """ + + :param record: + :return: bool - True for this record to be logged. + """ + # TODO: Only set these if they're not already, in case upstream + # adds them. + # TODO: Try setattr(record, 'class', ... + (class_name, qualname) = self.caller_class_and_qualname() + record.__dict__['class'] = class_name + record.__dict__['qualname'] = qualname + return True + + def caller_class_and_qualname(self, skip=4) -> Tuple[str, str]: + """ + Figure out our caller's qualname + + Ref: + + :param skip: How many stack frames above to look. + :return: str: The caller's qualname + """ + import inspect + + # TODO: Fold this into caller_class() + def stack_(frame): + framelist = [] + while frame: + framelist.append(frame) + frame = frame.f_back + return framelist + + stack = stack_(sys._getframe(0)) + # Go up through stack frames until we find the first with a + # type(f_locals.self) of logging.Logger. This should be the start + # of the frames internal to logging. + f = 0 + while stack[f]: + if type(stack[f].f_locals.get('self')) == logging.Logger: + f += 1 # Want to start on the next frame below + break + f += 1 + + # Now continue up through frames until we find the next one where + # that is *not* true, as it should be the call site of the logger + # call + while stack[f]: + if type(stack[f].f_locals.get('self')) != logging.Logger: + break # We've found the frame we want + f += 1 + + caller_qualname = caller_class_name = '' + if stack[f]: + frame = stack[f] + if frame.f_locals and 'self' in frame.f_locals: + + # Find __qualname__ of the caller + # Paranoia checks + if frame.f_code and frame.f_code.co_name: + fn = getattr(frame.f_locals['self'], frame.f_code.co_name) + + if fn and fn.__qualname__: + caller_qualname = fn.__qualname__ + + # Find immediate containing class name of caller, if any + frame_class = frame.f_locals['self'].__class__ + # Paranoia checks + if frame_class and frame_class.__qualname__: + caller_class_name = frame_class.__qualname__ + + if caller_qualname == '': + print('ALERT! Something went wrong with finding caller qualname for logging!') + + if caller_class_name == '': + print('ALERT! Something went wrong with finding caller class name for logging!') + + return (caller_class_name, caller_qualname) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 6685ddd0..4d0e6099 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -21,6 +21,7 @@ import logging import traceback from typing import Any, Optional +import EDMCLogging from config import appname, applongname, appversion, appversion_nobuild, copyright, config if getattr(sys, 'frozen', False): @@ -943,109 +944,6 @@ def enforce_single_instance() -> None: EnumWindows(enumwindowsproc, 0) -########################################################################### -# Logging - -# Has to be here to be defined for other modules importing -# TODO: now nothing should import this, can it move back into __main__ block ? -logger = logging.getLogger(appname) - -class EDMCContextFilter(logging.Filter): - """ - logging.Filter sub-class to place the calling __class__ in the record. - """ - def filter(self, record: logging.LogRecord) -> bool: - """ - - :param record: - :return: bool - True for this record to be logged. - """ - record.__dict__['class'] = self.caller_class() - record.__dict__['qualname'] = self.caller_qualname() - return True - - def caller_class(self, skip=5) -> str: - """ - Figure out our caller class. - - Ref: - - :param skip: How many stack frames above to look. - :return: str: The class name. - """ - import inspect - - def stack_(frame): - framelist = [] - while frame: - framelist.append(frame) - frame = frame.f_back - return framelist - - stack = stack_(sys._getframe(1)) - # TODO: Make this less fragile by not depending on a magic number of - # stack frames to skip. Should be able to find the last - # logger frame, where one of the critical, debug etc functions - # was called and then use the next frame before that. - # ALSO UPDATE caller_qualname() !!! - start = 0 + skip - if len(stack) < start + 1: - return '' - - class_name = '' - frame = stack[start] - if 'self' in frame.f_locals: - # Paranoia checks - frame_class = frame.f_locals['self'].__class__ - - if frame_class and frame_class.__qualname__: - class_name = frame_class.__qualname__ - - if class_name == '': - print('ALERT! Something went wrong with finding class name for logging!') - - return class_name - - def caller_qualname(self, skip=5) -> str: - """ - Figure out our caller's qualname - - Ref: - - :param skip: How many stack frames above to look. - :return: str: The caller's qualname - """ - import inspect - - def stack_(frame): - framelist = [] - while frame: - framelist.append(frame) - frame = frame.f_back - return framelist - - stack = stack_(sys._getframe(1)) - start = 0 + skip - if len(stack) < start + 1: - return '' - - qualname = '' - frame = stack[start] - if frame.f_locals and 'self' in frame.f_locals: - # Paranoia checks - - if frame.f_code and frame.f_code.co_name: - fn = getattr(frame.f_locals['self'], frame.f_code.co_name) - - if fn and fn.__qualname__: - qualname = fn.__qualname__ - - if qualname == '': - print('ALERT! Something went wrong with finding caller qualname for logging!') - - return qualname - -########################################################################### # Run the app if __name__ == "__main__": @@ -1056,29 +954,18 @@ if __name__ == "__main__": import tempfile sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), '%s.log' % appname), 'wt', 1) # unbuffered not allowed for text in python3, so use line buffering - ########################################################################### - # Configure the logging.Logger - logger_default_loglevel = logging.DEBUG - logger.setLevel(logger_default_loglevel) - - # Set up filter for adding class name - logger_f = EDMCContextFilter() - logger.addFilter(logger_f) - - logger_ch = logging.StreamHandler() - logger_ch.setLevel(logger_default_loglevel) - - logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(message)s') - logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' - logger_formatter.default_msec_format = '%s.%03d' - - logger_ch.setFormatter(logger_formatter) - logger.addHandler(logger_ch) - ########################################################################### + logger = EDMCLogging.logger(appname).getLogger() # Plain, not via `logger` print(f'{applongname} {appversion}') + class A(object): + class B(object): + def __init__(self): + logger.debug('A call from A.B.__init__') + + abinit = A.B() + Translations.install(config.get('language') or None) # Can generate errors so wait til log set up root = tk.Tk(className=appname.lower()) From 5a779a33790d24ab2ee9bb5dd7624d98d91578bc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 25 Jul 2020 15:22:22 +0100 Subject: [PATCH 075/258] Cleanups and docstrings * Added/fleshed out docstrings on file, classes and functions. * No need to use a function for the stack frame getting. * Check if LogRecord has class or qualname before setting, allowing upstream to implement them. * Use setattr()/getattr() rather than __dict__ fiddling. * Force an error string into class/qualname if we have issues finding them, rather than failing silently to ''. --- EDMCLogging.py | 76 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 24edac32..f522864e 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -1,5 +1,8 @@ """ -TODO: blurb +This module provides for a common logging-powered log facility. +Mostly it implements a logging.Filter() in order to get two extra +members on the logging.LogRecord instance for use in logging.Formatter() +strings. """ import sys @@ -8,8 +11,14 @@ from typing import Tuple class logger(object): """ - TODO: desc Wrapper class for all logging configuration and code. + + Class instantiation requires the 'logger name' and optional loglevel. + It is intended that this 'logger name' be re-used in all files/modules + that need to log. + + Users of this class should then call getLogger() to get the + logging.Logger instance. """ def __init__(self, logger_name: str, loglevel: int=logging.DEBUG): """ @@ -36,48 +45,61 @@ class logger(object): self.logger.addHandler(self.logger_channel) def getLogger(self) -> logging.Logger: + """ + :return: The logging.Logger instance. + """ return self.logger class EDMCContextFilter(logging.Filter): """ - TODO: Update this - logging.Filter sub-class to place the calling __class__ in the record. + logging.Filter sub-class to place extra attributes of the calling site + into the record. """ def filter(self, record: logging.LogRecord) -> bool: """ + Attempt to set the following in the LogRecord: - :param record: - :return: bool - True for this record to be logged. + 1. class = class name of the call site, if applicable + 2. qualname = __qualname__ of the call site. This simplifies + logging.Formatter() as you can use just this no matter if there is + a class involved or not, so you get a nice clean: + .[.classB....]. + + :param record: The LogRecord we're "filtering" + :return: bool - Always true in order for this record to be logged. """ - # TODO: Only set these if they're not already, in case upstream - # adds them. - # TODO: Try setattr(record, 'class', ... - (class_name, qualname) = self.caller_class_and_qualname() - record.__dict__['class'] = class_name - record.__dict__['qualname'] = qualname + class_name = qualname = '' + # Don't even call in if both already set. + if not getattr(record, 'class', None) or not getattr(record, 'qualname', None): + (class_name, qualname) = self.caller_class_and_qualname() + + # Only set if not already provided by logging itself + if getattr(record, 'class', None) is None: + setattr(record, 'class', class_name) + + # Only set if not already provided by logging itself + if getattr(record, 'qualname', None) is None: + setattr(record, 'qualname', qualname) + return True - def caller_class_and_qualname(self, skip=4) -> Tuple[str, str]: + def caller_class_and_qualname(self) -> Tuple[str, str]: """ - Figure out our caller's qualname + Figure out our caller's class name and qualname Ref: - :param skip: How many stack frames above to look. - :return: str: The caller's qualname + :return: Tuple[str, str]: The caller's class name and qualname """ - import inspect + # TODO: we might as well just walk this below. + # Build the stack of frames from here upwards + stack = [] + frame = sys._getframe(0) + while frame: + stack.append(frame) + frame = frame.f_back - # TODO: Fold this into caller_class() - def stack_(frame): - framelist = [] - while frame: - framelist.append(frame) - frame = frame.f_back - return framelist - - stack = stack_(sys._getframe(0)) # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start # of the frames internal to logging. @@ -117,8 +139,10 @@ class EDMCContextFilter(logging.Filter): if caller_qualname == '': print('ALERT! Something went wrong with finding caller qualname for logging!') + caller_qualname = '' if caller_class_name == '': print('ALERT! Something went wrong with finding caller class name for logging!') + caller_class_name = '' return (caller_class_name, caller_qualname) From 2eba647f17753a4e6a3b00a04af4195da658ad60 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 25 Jul 2020 15:30:34 +0100 Subject: [PATCH 076/258] Simply walk up the stack finding the frame we want. Also leave that 'A.B' test around as a hint for a unittest. Obviously that instantiation will need commenting out for a release. --- EDMCLogging.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index f522864e..0c7c629d 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -92,35 +92,26 @@ class EDMCContextFilter(logging.Filter): :return: Tuple[str, str]: The caller's class name and qualname """ - # TODO: we might as well just walk this below. - # Build the stack of frames from here upwards - stack = [] - frame = sys._getframe(0) - while frame: - stack.append(frame) - frame = frame.f_back - # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start # of the frames internal to logging. - f = 0 - while stack[f]: - if type(stack[f].f_locals.get('self')) == logging.Logger: - f += 1 # Want to start on the next frame below + frame = sys._getframe(0) + while frame: + if type(frame.f_locals.get('self')) == logging.Logger: + frame = frame.f_back # Want to start on the next frame below break - f += 1 + frame = frame.f_back # Now continue up through frames until we find the next one where # that is *not* true, as it should be the call site of the logger # call - while stack[f]: - if type(stack[f].f_locals.get('self')) != logging.Logger: + while frame: + if type(frame.f_locals.get('self')) != logging.Logger: break # We've found the frame we want - f += 1 + frame = frame.f_back caller_qualname = caller_class_name = '' - if stack[f]: - frame = stack[f] + if frame: if frame.f_locals and 'self' in frame.f_locals: # Find __qualname__ of the caller From 66e2c354c77c4afec22d06f02fd51f3984bb50f6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 25 Jul 2020 16:14:57 +0100 Subject: [PATCH 077/258] Documentation update and getLogger() -> get_logger() * Technically %(class)s can be e.g. A.B not just 'B' so say "name(s)". * To not confuse EDMCLogging.getLogger() with logging.getLogger() it's been renamed to get_logger(). * Note how we signal errors with finding class and/or qualname. * Call out EDMCLogging.py in Contributing.md. --- Contributing.md | 8 +++++--- EDMCLogging.py | 31 ++++++++++++++++++------------- EDMarketConnector.py | 2 +- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Contributing.md b/Contributing.md index ef2ab460..dfbd1b1e 100644 --- a/Contributing.md +++ b/Contributing.md @@ -246,11 +246,13 @@ No: We have implemented a `logging.Filter` that adds support for the following in `logging.Formatter()` strings: - 1. `%(qualname)s` which gets the full `ClassA(.ClassB...).func` of the - calling function. - 1. `%(class)s` which gets just the immediately encloding class name + 1. `%(qualname)s` which gets the full `.ClassA(.ClassB...).func` of the calling function. + 1. `%(class)s` which gets just the enclosing class name(s) of the calling + function. + If you want to see how we did this, check `EDMCLogging.py`. + So don't worry about adding anything about the class or function you're logging from, it's taken care of. diff --git a/EDMCLogging.py b/EDMCLogging.py index 0c7c629d..504467d6 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -44,7 +44,7 @@ class logger(object): self.logger_channel.setFormatter(self.logger_formatter) self.logger.addHandler(self.logger_channel) - def getLogger(self) -> logging.Logger: + def get_logger(self) -> logging.Logger: """ :return: The logging.Logger instance. """ @@ -60,12 +60,18 @@ class EDMCContextFilter(logging.Filter): """ Attempt to set the following in the LogRecord: - 1. class = class name of the call site, if applicable + 1. class = class name(s) of the call site, if applicable 2. qualname = __qualname__ of the call site. This simplifies logging.Formatter() as you can use just this no matter if there is a class involved or not, so you get a nice clean: .[.classB....]. + If we fail to be able to properly set either then: + + 1. Use print() to alert, to be SURE a message is seen. + 2. But also return strings noting the error, so there'll be + something in the log output if it happens. + :param record: The LogRecord we're "filtering" :return: bool - Always true in order for this record to be logged. """ @@ -86,11 +92,11 @@ class EDMCContextFilter(logging.Filter): def caller_class_and_qualname(self) -> Tuple[str, str]: """ - Figure out our caller's class name and qualname + Figure out our caller's class name(s) and qualname Ref: - :return: Tuple[str, str]: The caller's class name and qualname + :return: Tuple[str, str]: The caller's class name(s) and qualname """ # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start @@ -110,10 +116,9 @@ class EDMCContextFilter(logging.Filter): break # We've found the frame we want frame = frame.f_back - caller_qualname = caller_class_name = '' + caller_qualname = caller_class_names = '' if frame: if frame.f_locals and 'self' in frame.f_locals: - # Find __qualname__ of the caller # Paranoia checks if frame.f_code and frame.f_code.co_name: @@ -122,18 +127,18 @@ class EDMCContextFilter(logging.Filter): if fn and fn.__qualname__: caller_qualname = fn.__qualname__ - # Find immediate containing class name of caller, if any + # Find containing class name(s) of caller, if any frame_class = frame.f_locals['self'].__class__ # Paranoia checks if frame_class and frame_class.__qualname__: - caller_class_name = frame_class.__qualname__ + caller_class_names = frame_class.__qualname__ if caller_qualname == '': print('ALERT! Something went wrong with finding caller qualname for logging!') - caller_qualname = '' + caller_qualname = '' - if caller_class_name == '': - print('ALERT! Something went wrong with finding caller class name for logging!') - caller_class_name = '' + if caller_class_names == '': + print('ALERT! Something went wrong with finding caller class name(s) for logging!') + caller_class_names = '' - return (caller_class_name, caller_qualname) + return (caller_class_names, caller_qualname) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 4d0e6099..485814a0 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -954,7 +954,7 @@ if __name__ == "__main__": import tempfile sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), '%s.log' % appname), 'wt', 1) # unbuffered not allowed for text in python3, so use line buffering - logger = EDMCLogging.logger(appname).getLogger() + logger = EDMCLogging.logger(appname).get_logger() # Plain, not via `logger` print(f'{applongname} {appversion}') From 307910739a807ead6e364fa76ce5e7f5e6153a93 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 25 Jul 2020 19:55:09 +0100 Subject: [PATCH 078/258] Remove now un-necessary imports for logging/traceback --- EDMarketConnector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 485814a0..72c2b84e 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -17,9 +17,6 @@ from time import gmtime, time, localtime, strftime, strptime import _strptime # Workaround for http://bugs.python.org/issue7980 from calendar import timegm import webbrowser -import logging -import traceback -from typing import Any, Optional import EDMCLogging from config import appname, applongname, appversion, appversion_nobuild, copyright, config From 3653a1342fc6bb4d7526083e3ffbac0bdfb4f94e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 26 Jul 2020 11:51:58 +0100 Subject: [PATCH 079/258] No need to subclass object. --- EDMCLogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 504467d6..21fc39b2 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -9,7 +9,7 @@ import sys import logging from typing import Tuple -class logger(object): +class logger(): """ Wrapper class for all logging configuration and code. From f9a23cc831aa45afb170edd3b3168f814eb758c6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 09:37:10 +0100 Subject: [PATCH 080/258] Clean up flake8 output for this branch --- EDMCLogging.py | 6 ++-- EDMarketConnector.py | 38 ++++++++++++-------- companion.py | 28 +++++++-------- plug.py | 83 ++++++++++++++++++++++---------------------- plugins/eddn.py | 41 ++++++++++------------ plugins/edsm.py | 15 +++++--- plugins/inara.py | 19 +++++----- 7 files changed, 122 insertions(+), 108 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 21fc39b2..aa0a14fe 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -9,7 +9,7 @@ import sys import logging from typing import Tuple -class logger(): +class Logger: """ Wrapper class for all logging configuration and code. @@ -20,7 +20,7 @@ class logger(): Users of this class should then call getLogger() to get the logging.Logger instance. """ - def __init__(self, logger_name: str, loglevel: int=logging.DEBUG): + def __init__(self, logger_name: str, loglevel: int = logging.DEBUG): """ Set up a `logging.Logger` with our preferred configuration. This includes using an EDMCContextFilter to add 'class' and 'qualname' @@ -37,7 +37,7 @@ class logger(): self.logger_channel = logging.StreamHandler() self.logger_channel.setLevel(loglevel) - self.logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(class)s: %(message)s') + self.logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(class)s: %(message)s') # noqa: E501 self.logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' self.logger_formatter.default_msec_format = '%s.%03d' diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 72c2b84e..7ef9a9d7 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -15,7 +15,6 @@ import html import requests from time import gmtime, time, localtime, strftime, strptime import _strptime # Workaround for http://bugs.python.org/issue7980 -from calendar import timegm import webbrowser import EDMCLogging @@ -207,7 +206,7 @@ class AppWindow(object): self.help_menu.add_command(command=self.help_general) self.help_menu.add_command(command=self.help_privacy) self.help_menu.add_command(command=self.help_releases) - self.help_menu.add_command(command=lambda:self.updater.checkForUpdates()) + self.help_menu.add_command(command=lambda: self.updater.checkForUpdates()) self.help_menu.add_command(command=lambda: not self.HelpAbout.showing and self.HelpAbout(self.w)) self.menubar.add_cascade(menu=self.help_menu) @@ -528,7 +527,7 @@ class AppWindow(object): self.status['text'] = str(e) play_bad = True - if not self.status['text']: # no errors + if not self.status['text']: # no errors self.status['text'] = strftime(_('Last updated at {HH}:{MM}:{SS}').format(HH='%H', MM='%M', SS='%S'), localtime(querytime)) if play_sound and play_bad: hotkeymgr.play_bad() @@ -543,12 +542,13 @@ class AppWindow(object): if data.get('lastStarport', {}).get('ships'): report = 'Success' else: - report ='Failure' + report = 'Failure' else: report = 'Undocked!' logger.debug(f'{__class__}: Retry for shipyard - {report}') if not data['commander'].get('docked'): - pass # might have undocked while we were waiting for retry in which case station data is unreliable + # might have un-docked while we were waiting for retry in which case station data is unreliable + pass elif (data.get('lastSystem', {}).get('name') == monitor.system and data.get('lastStarport', {}).get('name') == monitor.station and data.get('lastStarport', {}).get('ships', {}).get('shipyard_list')): @@ -620,7 +620,9 @@ class AppWindow(object): logger.info(f"{__class__}: Can't start Status monitoring") # Export loadout - if entry['event'] == 'Loadout' and not monitor.state['Captain'] and config.getint('output') & config.OUT_SHIP: + if entry['event'] == 'Loadout'\ + and not monitor.state['Captain']\ + and config.getint('output') & config.OUT_SHIP: monitor.export_ship() # Plugins @@ -649,8 +651,8 @@ class AppWindow(object): self.view_menu.entryconfigure(0, state=tk.NORMAL) # Status self.file_menu.entryconfigure(0, state=tk.NORMAL) # Save Raw Data else: - self.file_menu.entryconfigure(0, state=tk.NORMAL) # Status - self.file_menu.entryconfigure(1, state=tk.NORMAL) # Save Raw Data + self.file_menu.entryconfigure(0, state=tk.NORMAL) # Status + self.file_menu.entryconfigure(1, state=tk.NORMAL) # Save Raw Data except companion.ServerError as e: self.status['text'] = str(e) except Exception as e: @@ -805,7 +807,7 @@ class AppWindow(object): row += 1 button = ttk.Button(frame, text=_('OK'), command=self.apply) button.grid(row=row, column=2, sticky=tk.E) - button.bind("", lambda event:self.apply()) + button.bind("", lambda event: self.apply()) self.protocol("WM_DELETE_WINDOW", self._destroy) ############################################################ @@ -833,7 +835,11 @@ class AppWindow(object): initialfile = '%s%s.%s.json' % (data.get('lastSystem', {}).get('name', 'Unknown'), data['commander'].get('docked') and '.'+data.get('lastStarport', {}).get('name', 'Unknown') or '', strftime('%Y-%m-%dT%H.%M.%S', localtime()))) if f: with open(f, 'wb') as h: - h.write(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')).encode('utf-8')) + h.write(json.dumps(data, + ensure_ascii=False, + indent=2, + sort_keys=True, + separators=(',', ': ')).encode('utf-8')) except companion.ServerError as e: self.status['text'] = str(e) except Exception as e: @@ -841,7 +847,8 @@ class AppWindow(object): self.status['text'] = str(e) def onexit(self, event=None): - if platform!='darwin' or self.w.winfo_rooty()>0: # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + if platform != 'darwin' or self.w.winfo_rooty() > 0: config.set('geometry', '+{1}+{2}'.format(*self.w.geometry().split('+'))) self.w.withdraw() # Following items can take a few seconds, so hide the main window while they happen protocolhandler.close() @@ -882,7 +889,7 @@ class AppWindow(object): self.theme_menubar.grid(row=0, columnspan=2, sticky=tk.NSEW) def onleave(self, event=None): - if config.getint('theme') > 1 and event.widget==self.w: + if config.getint('theme') > 1 and event.widget == self.w: self.w.attributes("-transparentcolor", 'grey4') self.theme_menubar.grid_remove() self.blank_menubar.grid(row=0, columnspan=2, sticky=tk.NSEW) @@ -931,7 +938,7 @@ def enforce_single_instance() -> None: if len(sys.argv) > 1 and sys.argv[1].startswith(protocolhandler.redirect): # Browser invoked us directly with auth response. Forward the response to the other app instance. CoInitializeEx(0, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE) - ShowWindow(hWnd, SW_RESTORE) # Wait for it to be responsive to avoid ShellExecute recursing + ShowWindow(hWnd, SW_RESTORE) # Wait for it to be responsive to avoid ShellExecute recursing ShellExecute(0, None, sys.argv[1], None, None, SW_RESTORE) else: ShowWindowAsync(hWnd, SW_RESTORE) @@ -949,9 +956,10 @@ if __name__ == "__main__": if getattr(sys, 'frozen', False): # By default py2exe tries to write log to dirname(sys.executable) which fails when installed import tempfile - sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), '%s.log' % appname), 'wt', 1) # unbuffered not allowed for text in python3, so use line buffering + # unbuffered not allowed for text in python3, so use line buffering + sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), f'{appname}.log'), 'wt', 1) - logger = EDMCLogging.logger(appname).get_logger() + logger = EDMCLogging.Logger(appname).get_logger() # Plain, not via `logger` print(f'{applongname} {appversion}') diff --git a/companion.py b/companion.py index 62d07282..d1ef0481 100644 --- a/companion.py +++ b/companion.py @@ -4,6 +4,7 @@ from builtins import object import base64 import csv import requests +from typing import TYPE_CHECKING # TODO: see https://github.com/EDCD/EDMarketConnector/issues/569 from http.cookiejar import LWPCookieJar # No longer needed but retained in case plugins use it @@ -23,7 +24,6 @@ import logging logger = logging.getLogger(appname) -from typing import TYPE_CHECKING if TYPE_CHECKING: _ = lambda x: x # noqa # to make flake8 stop complaining that the hacked in _ method doesnt exist @@ -205,21 +205,21 @@ class Auth(object): logger.error(f"Frontier CAPI Auth: Can't refresh token for \"{self.cmdr}\"", exc_info=e) else: - logger.error(f"Frontier CAPI Auth: No token for \"{self.cmdr}\"", exc_info=e) + logger.error(f"Frontier CAPI Auth: No token for \"{self.cmdr}\"") # New request - logger.info(f'Frontier CAPI Auth: New authorization request') + logger.info('Frontier CAPI Auth: New authorization request') v = random.SystemRandom().getrandbits(8 * 32) - self.verifier = self.base64URLEncode(v.to_bytes(32, byteorder='big')).encode('utf-8') + self.verifier = self.base64_url_encode(v.to_bytes(32, byteorder='big')).encode('utf-8') s = random.SystemRandom().getrandbits(8 * 32) - self.state = self.base64URLEncode(s.to_bytes(32, byteorder='big')) + self.state = self.base64_url_encode(s.to_bytes(32, byteorder='big')) # Won't work under IE: https://blogs.msdn.microsoft.com/ieinternals/2011/07/13/understanding-protocols/ webbrowser.open( '{server_auth}{url_auth}?response_type=code&audience=frontier&scope=capi&client_id={client_id}&code_challenge={challenge}&code_challenge_method=S256&state={state}&redirect_uri={redirect}'.format( # noqa: E501 # I cant make this any shorter server_auth=SERVER_AUTH, url_auth=URL_AUTH, client_id=CLIENT_ID, - challenge=self.base64URLEncode(hashlib.sha256(self.verifier).digest()), + challenge=self.base64_url_encode(hashlib.sha256(self.verifier).digest()), state=self.state, redirect=protocolhandler.redirect ) @@ -294,7 +294,7 @@ class Auth(object): def dump(self, r): logger.debug(f'Frontier CAPI Auth: {r.url} {r.status_code} {r.reason if r.reason else "None"} {r.text}') - def base64URLEncode(self, text): + def base64_url_encode(self, text): return base64.urlsafe_b64encode(text).decode().replace('=', '') @@ -379,8 +379,8 @@ class Session(object): r = self.session.get(self.server + endpoint, timeout=timeout) except Exception as e: - logger.debug(f'Attempting GET', exc_info=e) - raise ServerError('unable to get endpoint {}'.format(endpoint)) from e + logger.debug('Attempting GET', exc_info=e) + raise ServerError(f'unable to get endpoint {endpoint}') from e if r.url.startswith(SERVER_AUTH): # Redirected back to Auth server - force full re-authentication @@ -400,7 +400,7 @@ class Session(object): data = r.json() # May also fail here if token expired since response is empty except (requests.HTTPError, ValueError) as e: - logger.error(f'Frontier CAPI Auth: GET ', exc_info=e) + logger.error('Frontier CAPI Auth: GET ', exc_info=e) self.dump(r) self.close() @@ -408,7 +408,7 @@ class Session(object): self.invalidate() self.retrying = False self.login() - logger.error(f'Frontier CAPI Auth: query failed after refresh') + logger.error('Frontier CAPI Auth: query failed after refresh') raise CredentialsError('query failed after refresh') from e elif self.login(): # Maybe our token expired. Re-authorize in any case @@ -417,7 +417,7 @@ class Session(object): else: self.retrying = False - logger.error(f'Frontier CAPI Auth: HTTP error or invalid JSON') + logger.error('Frontier CAPI Auth: HTTP error or invalid JSON') raise CredentialsError('HTTP error or invalid JSON') from e self.retrying = False @@ -462,7 +462,7 @@ class Session(object): self.session.close() except Exception as e: - logger.debug(f'Frontier CAPI Auth: closing', exc_info=e) + logger.debug('Frontier CAPI Auth: closing', exc_info=e) self.session = None @@ -496,7 +496,7 @@ def fixup(data): # But also see https://github.com/Marginal/EDMarketConnector/issues/32 for thing in ('buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket'): if not isinstance(commodity.get(thing), numbers.Number): - logger.debug(f'Invalid {thing}:{commodity.get(thing)} ({type(commodity.get(thing))}) for {commodity.get("name", "")}') + logger.debug(f'Invalid {thing}:{commodity.get(thing)} ({type(commodity.get(thing))}) for {commodity.get("name", "")}') # noqa: E501 break else: diff --git a/plug.py b/plug.py index 50d9af07..a443de21 100644 --- a/plug.py +++ b/plug.py @@ -7,50 +7,49 @@ import os import importlib import sys import operator -import threading # We don't use it, but plugins might +import threading # noqa: F401 - We don't use it, but plugins might import logging import tkinter as tk import myNotebook as nb from config import config, appname -from time import time as time logger = logging.getLogger(appname) # Dashboard Flags constants -FlagsDocked = 1<<0 # on a landing pad -FlagsLanded = 1<<1 # on planet surface -FlagsLandingGearDown = 1<<2 -FlagsShieldsUp = 1<<3 -FlagsSupercruise = 1<<4 -FlagsFlightAssistOff = 1<<5 -FlagsHardpointsDeployed = 1<<6 -FlagsInWing = 1<<7 -FlagsLightsOn = 1<<8 -FlagsCargoScoopDeployed = 1<<9 -FlagsSilentRunning = 1<<10 -FlagsScoopingFuel = 1<<11 -FlagsSrvHandbrake = 1<<12 -FlagsSrvTurret = 1<<13 # using turret view -FlagsSrvUnderShip = 1<<14 # turret retracted -FlagsSrvDriveAssist = 1<<15 -FlagsFsdMassLocked = 1<<16 -FlagsFsdCharging = 1<<17 -FlagsFsdCooldown = 1<<18 -FlagsLowFuel = 1<<19 # <25% -FlagsOverHeating = 1<<20 # > 100% -FlagsHasLatLong = 1<<21 -FlagsIsInDanger = 1<<22 -FlagsBeingInterdicted = 1<<23 -FlagsInMainShip = 1<<24 -FlagsInFighter = 1<<25 -FlagsInSRV = 1<<26 -FlagsAnalysisMode = 1<<27 # Hud in Analysis mode -FlagsNightVision = 1<<28 -FlagsAverageAltitude = 1<<29 # Altitude from Average radius -FlagsFsdJump = 1<<30 -FlagsSrvHighBeam = 1<<31 +FlagsDocked = 1 << 0 # on a landing pad +FlagsLanded = 1 << 1 # on planet surface +FlagsLandingGearDown = 1 << 2 +FlagsShieldsUp = 1 << 3 +FlagsSupercruise = 1 << 4 +FlagsFlightAssistOff = 1 << 5 +FlagsHardpointsDeployed = 1 << 6 +FlagsInWing = 1 << 7 +FlagsLightsOn = 1 << 8 +FlagsCargoScoopDeployed = 1 << 9 +FlagsSilentRunning = 1 << 10 +FlagsScoopingFuel = 1 << 11 +FlagsSrvHandbrake = 1 << 12 +FlagsSrvTurret = 1 << 13 # using turret view +FlagsSrvUnderShip = 1 << 14 # turret retracted +FlagsSrvDriveAssist = 1 << 15 +FlagsFsdMassLocked = 1 << 16 +FlagsFsdCharging = 1 << 17 +FlagsFsdCooldown = 1 << 18 +FlagsLowFuel = 1 << 19 # <25% +FlagsOverHeating = 1 << 20 # > 100% +FlagsHasLatLong = 1 << 21 +FlagsIsInDanger = 1 << 22 +FlagsBeingInterdicted = 1 << 23 +FlagsInMainShip = 1 << 24 +FlagsInFighter = 1 << 25 +FlagsInSRV = 1 << 26 +FlagsAnalysisMode = 1 << 27 # Hud in Analysis mode +FlagsNightVision = 1 << 28 +FlagsAverageAltitude = 1 << 29 # Altitude from Average radius +FlagsFsdJump = 1 << 30 +FlagsSrvHighBeam = 1 << 31 # Dashboard GuiFocus constants GuiFocusNoFocus = 0 @@ -88,14 +87,16 @@ class Plugin(object): :raises Exception: Typically ImportError or OSError """ - self.name = name # Display name. - self.folder = name # basename of plugin folder. None for internal plugins. - self.module = None # None for disabled plugins. + self.name = name # Display name. + self.folder = name # basename of plugin folder. None for internal plugins. + self.module = None # None for disabled plugins. if loadfile: logger.info(f'loading plugin "{name.replace(".", "_")}" from "{loadfile}"') try: - module = importlib.machinery.SourceFileLoader('plugin_{}'.format(name.encode(encoding='ascii', errors='replace').decode('utf-8').replace('.', '_')), loadfile).load_module() + module = importlib.machinery.SourceFileLoader('plugin_{}'.format( + name.encode(encoding='ascii', errors='replace').decode('utf-8').replace('.', '_')), + loadfile).load_module() if getattr(module, 'plugin_start3', None): newname = module.plugin_start3(os.path.dirname(loadfile)) self.name = newname and str(newname) or name @@ -173,11 +174,11 @@ def load_plugins(master): if name.endswith('.py') and not name[0] in ['.', '_']: try: plugin = Plugin(name[:-3], os.path.join(config.internal_plugin_dir, name)) - plugin.folder = None # Suppress listing in Plugins prefs tab + plugin.folder = None # Suppress listing in Plugins prefs tab internal.append(plugin) except Exception as e: logger.error(f'Failure loading internal Plugin "{name}"', exc_info=e) - PLUGINS.extend(sorted(internal, key = lambda p: operator.attrgetter('name')(p).lower())) + PLUGINS.extend(sorted(internal, key=lambda p: operator.attrgetter('name')(p).lower())) # Add plugin folder to load path so packages can be loaded from plugin folder sys.path.append(config.plugin_dir) @@ -199,7 +200,7 @@ def load_plugins(master): except Exception as e: logger.error(f'Failure loading found Plugin "{name}"', exc_info=e) pass - PLUGINS.extend(sorted(found, key = lambda p: operator.attrgetter('name')(p).lower())) + PLUGINS.extend(sorted(found, key=lambda p: operator.attrgetter('name')(p).lower())) def provides(fn_name): """ diff --git a/plugins/eddn.py b/plugins/eddn.py index e2e2b141..1e979d48 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -8,7 +8,6 @@ from platform import system import re import requests import sys -import uuid import logging import tkinter as tk @@ -25,7 +24,7 @@ from companion import category_map logger = logging.getLogger(appname) -this = sys.modules[__name__] # For holding module globals +this = sys.modules[__name__] # For holding module globals # Track location to add to Journal events this.systemaddress = None @@ -61,15 +60,15 @@ class EDDN(object): try: # Try to open existing file self.replayfile = open(filename, 'r+', buffering=1) - except Exception as e: + except Exception: if exists(filename): - raise # Couldn't open existing file + raise # Couldn't open existing file else: - self.replayfile = open(filename, 'w+', buffering=1) # Create file - if sys.platform != 'win32': # open for writing is automatically exclusive on Windows - lockf(self.replayfile, LOCK_EX|LOCK_NB) + self.replayfile = open(filename, 'w+', buffering=1) # Create file + if sys.platform != 'win32': # open for writing is automatically exclusive on Windows + lockf(self.replayfile, LOCK_EX | LOCK_NB) except Exception as e: - logger.debug(f'Failed opening "replay.jsonl"', exc_info=e) + logger.debug('Failed opening "replay.jsonl"', exc_info=e) if self.replayfile: self.replayfile.close() self.replayfile = None @@ -104,18 +103,12 @@ class EDDN(object): r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=self.TIMEOUT) if r.status_code != requests.codes.ok: - logger.debug(f''': -Status\t{r.status_code} -URL\t{r.url} -Headers\t{r.headers}' -Content:\n{r.text} -''' - ) + logger.debug(f':\nStatus\t{r.status_code}URL\t{r.url}Headers\t{r.headers}Content:\n{r.text}') r.raise_for_status() def sendreplay(self): if not self.replayfile: - return # Probably closing app + return # Probably closing app status = self.parent.children['status'] @@ -148,17 +141,18 @@ Content:\n{r.text} status['text'] = _("Error: Can't connect to EDDN") return # stop sending except Exception as e: - logger.debug(f'Failed sending', exc_info=e) + logger.debug('Failed sending', exc_info=e) status['text'] = str(e) - return # stop sending + return # stop sending self.parent.after(self.REPLAYPERIOD, self.sendreplay) def export_commodities(self, data, is_beta): commodities = [] for commodity in data['lastStarport'].get('commodities') or []: - if (category_map.get(commodity['categoryname'], True) and # Check marketable - not commodity.get('legality')): # check not prohibited + # Check 'marketable' and 'not prohibited' + if (category_map.get(commodity['categoryname'], True) + and not commodity.get('legality')): commodities.append(OrderedDict([ ('name', commodity['name'].lower()), ('meanPrice', int(commodity['meanPrice'])), @@ -446,7 +440,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): logger.debug(f'Failed in export_journal_entry', exc_info=e) return _("Error: Can't connect to EDDN") except Exception as e: - logger.debug(f'Failed in export_journal_entry', exc_info=e) + logger.debug('Failed in export_journal_entry', exc_info=e) return str(e) elif (config.getint('output') & config.OUT_MKT_EDDN and not state['Captain'] and @@ -472,6 +466,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): logger.debug(f'Failed exporting {entry["event"]}', exc_info=e) return str(e) + def cmdr_data(data, is_beta): if data['commander'].get('docked') and config.getint('output') & config.OUT_MKT_EDDN: try: @@ -492,9 +487,9 @@ def cmdr_data(data, is_beta): status.update_idletasks() except requests.RequestException as e: - logger.debug(f'Failed exporting data', exc_info=e) + logger.debug('Failed exporting data', exc_info=e) return _("Error: Can't connect to EDDN") except Exception as e: - logger.debug(f'Failed exporting data', exc_info=e) + logger.debug('Failed exporting data', exc_info=e) return str(e) diff --git a/plugins/edsm.py b/plugins/edsm.py index eb0ab84f..3e59404d 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -15,7 +15,9 @@ import json import requests import sys import time -import urllib.request, urllib.error, urllib.parse +import urllib.request +import urllib.error +import urllib.parse from queue import Queue from threading import Thread import logging @@ -357,7 +359,10 @@ def worker(): r.raise_for_status() reply = r.json() (msgnum, msg) = reply['msgnum'], reply['msg'] - # 1xx = OK, 2xx = fatal error, 3&4xx not generated at top-level, 5xx = error but events saved for later processing + # 1xx = OK + # 2xx = fatal error + # 3&4xx not generated at top-level + # 5xx = error but events saved for later processing if msgnum // 100 == 2: logger.warning(f'EDSM\t{msgnum} {msg}\t{json.dumps(pending, separators = (",", ": "))}') plug.show_error(_('Error: EDSM {MSG}').format(MSG=msg)) @@ -366,9 +371,11 @@ def worker(): if not closing and e['event'] in ['StartUp', 'Location', 'FSDJump', 'CarrierJump']: # Update main window's system status this.lastlookup = r - this.system_link.event_generate('<>', when="tail") # calls update_status in main thread + # calls update_status in main thread + this.system_link.event_generate('<>', when="tail") elif r['msgnum'] // 100 != 1: - logger.warning(f'EDSM\t{r["msgnum"]} {r["msg"]}\t{json.dumps(e, separators = (",", ": "))}') + logger.warning(f'EDSM\t{r["msgnum"]} {r["msg"]}\t' + f'{json.dumps(e, separators = (",", ": "))}') pending = [] break diff --git a/plugins/inara.py b/plugins/inara.py index abf221f4..a0f33191 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -456,7 +456,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): call() except Exception as e: - logger.debug(f'Adding events', exc_info=e) + logger.debug('Adding events', exc_info=e) return str(e) # @@ -895,11 +895,11 @@ def worker(): status = reply['header']['eventStatus'] if callback: callback(reply) - elif status // 100 != 2: # 2xx == OK (maybe with warnings) + elif status // 100 != 2: # 2xx == OK (maybe with warnings) # Log fatal errors - log.warning(f'Inara\t{status} {reply["header"].get("eventStatusText", "")}') - log.debug(f'JSON data:\n{json.dumps(data, indent=2, separators = (",", ": "))}') - plug.show_error(_('Error: Inara {MSG}').format(MSG = reply['header'].get('eventStatusText', status))) + logger.warning(f'Inara\t{status} {reply["header"].get("eventStatusText", "")}') + logger.debug(f'JSON data:\n{json.dumps(data, indent=2, separators = (",", ": "))}') + plug.show_error(_('Error: Inara {MSG}').format(MSG=reply['header'].get('eventStatusText', status))) else: # Log individual errors and warnings for data_event, reply_event in zip(data['events'], reply['events']): @@ -907,17 +907,20 @@ def worker(): logger.warning(f'Inara\t{status} {reply_event.get("eventStatusText", "")}') logger.debug(f'JSON data:\n{json.dumps(data_event)}') if reply_event['eventStatus'] // 100 != 2: - plug.show_error(_('Error: Inara {MSG}').format(MSG = '%s, %s' % (data_event['eventName'], reply_event.get('eventStatusText', reply_event['eventStatus'])))) + plug.show_error(_('Error: Inara {MSG}').format( + MSG=f'{data_event["eventName"]},' + f'{reply_event.get("eventStatusText", reply_event["eventStatus"])}')) if data_event['eventName'] in ['addCommanderTravelCarrierJump', 'addCommanderTravelDock', 'addCommanderTravelFSDJump', 'setCommanderTravelLocation']: this.lastlocation = reply_event.get('eventData', {}) this.system_link.event_generate('<>', when="tail") # calls update_location in main thread elif data_event['eventName'] in ['addCommanderShip', 'setCommanderShip']: this.lastship = reply_event.get('eventData', {}) - this.system_link.event_generate('<>', when="tail") # calls update_ship in main thread + # calls update_ship in main thread + this.system_link.event_generate('<>', when="tail") break except Exception as e: - logger.debug(f'Sending events', exc_info=e) + logger.debug('Sending events', exc_info=e) retrying += 1 else: if callback: From cbba1ce28ae046ebed3493559a7e45df0349aee7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 09:44:12 +0100 Subject: [PATCH 081/258] Change time string in translations. It makes the code far too complicated to sub {HH} -> %H and the like. --- L10n/cs.strings | 2 +- L10n/de.strings | 2 +- L10n/en.template | 4 ++-- L10n/es.strings | 2 +- L10n/fi.strings | 2 +- L10n/fr.strings | 2 +- L10n/hu.strings | 2 +- L10n/it.strings | 2 +- L10n/ja.strings | 2 +- L10n/lv.strings | 2 +- L10n/nl.strings | 2 +- L10n/pl.strings | 2 +- L10n/pt-BR.strings | 2 +- L10n/pt-PT.strings | 2 +- L10n/ru.strings | 2 +- L10n/sl.strings | 2 +- L10n/sr-Latn-BA.strings | 2 +- L10n/sr-Latn.strings | 2 +- L10n/sv-SE.strings | 2 +- L10n/uk.strings | 2 +- L10n/zh-Hans.strings | 2 +- 21 files changed, 22 insertions(+), 22 deletions(-) diff --git a/L10n/cs.strings b/L10n/cs.strings index 48c58606..08a59767 100644 --- a/L10n/cs.strings +++ b/L10n/cs.strings @@ -236,7 +236,7 @@ "Language" = "Jazyk"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Poslední aktualizace {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Poslední aktualizace %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/de.strings b/L10n/de.strings index 9e50095f..301916f5 100644 --- a/L10n/de.strings +++ b/L10n/de.strings @@ -236,7 +236,7 @@ "Language" = "Sprache"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Zuletzt aktualisiert um {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Zuletzt aktualisiert um %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Kapitänleutnant"; diff --git a/L10n/en.template b/L10n/en.template index 132df0d5..97e0d8b9 100644 --- a/L10n/en.template +++ b/L10n/en.template @@ -235,8 +235,8 @@ /* Appearance setting prompt. [prefs.py] */ "Language" = "Language"; -/* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Last updated at {HH}:{MM}:{SS}"; +/* [EDMarketConnector.py] - Leave '%H:%M:%S' as-is */ +"Last updated at %H:%M:%S" = "Last updated at %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/es.strings b/L10n/es.strings index 5d4d819a..e1675de9 100644 --- a/L10n/es.strings +++ b/L10n/es.strings @@ -236,7 +236,7 @@ "Language" = "Idioma"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Última actualización: {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Última actualización: %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Teniente de navío"; diff --git a/L10n/fi.strings b/L10n/fi.strings index 7c00b1b1..dd7cbaeb 100644 --- a/L10n/fi.strings +++ b/L10n/fi.strings @@ -236,7 +236,7 @@ "Language" = "Kieli"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Päivitetty viimeksi {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Päivitetty viimeksi %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/fr.strings b/L10n/fr.strings index 97550996..081d83cd 100644 --- a/L10n/fr.strings +++ b/L10n/fr.strings @@ -236,7 +236,7 @@ "Language" = "Langue"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Dernière mise à jour à {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Dernière mise à jour à %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/hu.strings b/L10n/hu.strings index daa68919..71da38f1 100644 --- a/L10n/hu.strings +++ b/L10n/hu.strings @@ -233,7 +233,7 @@ "Language" = "Nyelv"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Utoljára frissítve {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Utoljára frissítve %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Hadnagy"; diff --git a/L10n/it.strings b/L10n/it.strings index 8ec9f094..aa21a2ba 100644 --- a/L10n/it.strings +++ b/L10n/it.strings @@ -236,7 +236,7 @@ "Language" = "Lingua"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Ultimo aggiornamento alle {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Ultimo aggiornamento alle %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/ja.strings b/L10n/ja.strings index 1810a0a7..f38bc39b 100644 --- a/L10n/ja.strings +++ b/L10n/ja.strings @@ -236,7 +236,7 @@ "Language" = "言語"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "最終更新時間 {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "最終更新時間 %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/lv.strings b/L10n/lv.strings index 3113fbbb..b60a470b 100644 --- a/L10n/lv.strings +++ b/L10n/lv.strings @@ -218,7 +218,7 @@ "Language" = "Valoda"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Pēdējo reizi atjaunināts {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Pēdējo reizi atjaunināts %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Leitnants"; diff --git a/L10n/nl.strings b/L10n/nl.strings index a3ee89e7..14a1baa1 100644 --- a/L10n/nl.strings +++ b/L10n/nl.strings @@ -236,7 +236,7 @@ "Language" = "Taal"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Voor het laatst bijgewerkt om {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Voor het laatst bijgewerkt om %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/pl.strings b/L10n/pl.strings index e2d26f8e..4fd57b59 100644 --- a/L10n/pl.strings +++ b/L10n/pl.strings @@ -236,7 +236,7 @@ "Language" = "Język"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Ostatnia aktualizacja {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Ostatnia aktualizacja %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/pt-BR.strings b/L10n/pt-BR.strings index bb0c339f..1700da85 100644 --- a/L10n/pt-BR.strings +++ b/L10n/pt-BR.strings @@ -236,7 +236,7 @@ "Language" = "Idioma"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Última atualização {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Última atualização %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Tenente"; diff --git a/L10n/pt-PT.strings b/L10n/pt-PT.strings index 249b6e29..50a86c34 100644 --- a/L10n/pt-PT.strings +++ b/L10n/pt-PT.strings @@ -236,7 +236,7 @@ "Language" = "Linguagem"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Última actualização: {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Última actualização: %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Tenente"; diff --git a/L10n/ru.strings b/L10n/ru.strings index e7f012b8..55ffd08f 100644 --- a/L10n/ru.strings +++ b/L10n/ru.strings @@ -236,7 +236,7 @@ "Language" = "Язык"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Последнее обновление в {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Последнее обновление в %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Лейтенант"; diff --git a/L10n/sl.strings b/L10n/sl.strings index fe2f7435..e01dd1fa 100644 --- a/L10n/sl.strings +++ b/L10n/sl.strings @@ -185,7 +185,7 @@ "Language" = "Jezik"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Zadnja osvežitev podatkov {HH}:{MM}:{SS} "; +"Last updated at %H:%M:%S" = "Zadnja osvežitev podatkov %H:%M:%S "; /* Federation rank. [stats.py] */ "Lieutenant" = "LIeutenant"; diff --git a/L10n/sr-Latn-BA.strings b/L10n/sr-Latn-BA.strings index 3a1dbdc2..cf72a459 100644 --- a/L10n/sr-Latn-BA.strings +++ b/L10n/sr-Latn-BA.strings @@ -236,7 +236,7 @@ "Language" = "Jezik"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Posljednji put osvježeno {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Posljednji put osvježeno %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/sr-Latn.strings b/L10n/sr-Latn.strings index 1245451b..ddeb2648 100644 --- a/L10n/sr-Latn.strings +++ b/L10n/sr-Latn.strings @@ -236,7 +236,7 @@ "Language" = "Jezik"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Poslednji put osveženo {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Poslednji put osveženo %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/sv-SE.strings b/L10n/sv-SE.strings index d707eedd..ee630ea0 100644 --- a/L10n/sv-SE.strings +++ b/L10n/sv-SE.strings @@ -236,7 +236,7 @@ "Language" = "Språk"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Senaste uppdatering: {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Senaste uppdatering: %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; diff --git a/L10n/uk.strings b/L10n/uk.strings index 09b80c27..756e8484 100644 --- a/L10n/uk.strings +++ b/L10n/uk.strings @@ -236,7 +236,7 @@ "Language" = "Мова"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "Останнє оновлення було {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "Останнє оновлення було %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Лейтенант"; diff --git a/L10n/zh-Hans.strings b/L10n/zh-Hans.strings index 2ddbeee6..a304460c 100644 --- a/L10n/zh-Hans.strings +++ b/L10n/zh-Hans.strings @@ -233,7 +233,7 @@ "Language" = "语言"; /* [EDMarketConnector.py] */ -"Last updated at {HH}:{MM}:{SS}" = "最后更新于 {HH}:{MM}:{SS}"; +"Last updated at %H:%M:%S" = "最后更新于 %H:%M:%S"; /* Federation rank. [stats.py] */ "Lieutenant" = "Lieutenant"; From d7c23724171a5ac3dd798e4f724afab0cb857d8b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 10:14:16 +0100 Subject: [PATCH 082/258] Flake8 cleanup round #2 --- EDMCLogging.py | 1 + EDMarketConnector.py | 59 +++++++++++++++++++++++++++----------------- companion.py | 2 +- plug.py | 3 ++- plugins/eddn.py | 9 ++++--- plugins/edsm.py | 5 ++-- plugins/inara.py | 8 ++++-- 7 files changed, 54 insertions(+), 33 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index aa0a14fe..69e8389b 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -9,6 +9,7 @@ import sys import logging from typing import Tuple + class Logger: """ Wrapper class for all logging configuration and code. diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 7ef9a9d7..1ea0967e 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -12,9 +12,7 @@ from os import chdir, environ from os.path import dirname, expanduser, isdir, join import re import html -import requests -from time import gmtime, time, localtime, strftime, strptime -import _strptime # Workaround for http://bugs.python.org/issue7980 +from time import time, localtime, strftime import webbrowser import EDMCLogging @@ -212,7 +210,7 @@ class AppWindow(object): self.menubar.add_cascade(menu=self.help_menu) if platform == 'win32': # Must be added after at least one "real" menu entry - self.always_ontop = tk.BooleanVar(value = config.getint('always_ontop')) + self.always_ontop = tk.BooleanVar(value=config.getint('always_ontop')) self.system_menu = tk.Menu(self.menubar, name='system', tearoff=tk.FALSE) self.system_menu.add_separator() self.system_menu.add_checkbutton(label=_('Always on top'), variable = self.always_ontop, command=self.ontop_changed) # Appearance setting @@ -528,7 +526,7 @@ class AppWindow(object): play_bad = True if not self.status['text']: # no errors - self.status['text'] = strftime(_('Last updated at {HH}:{MM}:{SS}').format(HH='%H', MM='%M', SS='%S'), localtime(querytime)) + self.status['text'] = strftime(_('Last updated at %H:%M:%S'), localtime(querytime)) if play_sound and play_bad: hotkeymgr.play_bad() @@ -553,7 +551,7 @@ class AppWindow(object): data.get('lastStarport', {}).get('name') == monitor.station and data.get('lastStarport', {}).get('ships', {}).get('shipyard_list')): self.eddn.export_shipyard(data, monitor.is_beta) - elif tries > 1: # bogus data - retry + elif tries > 1: # bogus data - retry self.w.after(int(SERVER_RETRY * 1000), lambda:self.retry_for_shipyard(tries-1)) except: pass @@ -626,7 +624,12 @@ class AppWindow(object): monitor.export_ship() # Plugins - err = plug.notify_journal_entry(monitor.cmdr, monitor.is_beta, monitor.system, monitor.station, entry, monitor.state) + err = plug.notify_journal_entry(monitor.cmdr, + monitor.is_beta, + monitor.system, + monitor.station, + entry, + monitor.state) if err: self.status['text'] = err if not config.getint('hotkey_mute'): @@ -646,10 +649,11 @@ class AppWindow(object): def auth(self, event=None): try: companion.session.auth_callback() - self.status['text'] = _('Authentication successful') # Successfully authenticated with the Frontier website + # Successfully authenticated with the Frontier website + self.status['text'] = _('Authentication successful') if platform == 'darwin': - self.view_menu.entryconfigure(0, state=tk.NORMAL) # Status - self.file_menu.entryconfigure(0, state=tk.NORMAL) # Save Raw Data + self.view_menu.entryconfigure(0, state=tk.NORMAL) # Status + self.file_menu.entryconfigure(0, state=tk.NORMAL) # Save Raw Data else: self.file_menu.entryconfigure(0, state=tk.NORMAL) # Status self.file_menu.entryconfigure(1, state=tk.NORMAL) # Save Raw Data @@ -828,11 +832,19 @@ class AppWindow(object): try: data = companion.session.station() self.status['text'] = '' - f = tkinter.filedialog.asksaveasfilename(parent = self.w, - defaultextension = platform=='darwin' and '.json' or '', - filetypes = [('JSON', '.json'), ('All Files', '*')], - initialdir = config.get('outdir'), - initialfile = '%s%s.%s.json' % (data.get('lastSystem', {}).get('name', 'Unknown'), data['commander'].get('docked') and '.'+data.get('lastStarport', {}).get('name', 'Unknown') or '', strftime('%Y-%m-%dT%H.%M.%S', localtime()))) + default_extension: str = '' + if platform == 'darwin': + default_extension = '.json' + last_system: str = data.get("lastSystem", {}).get("name", "Unknown") + last_starport: str = '' + if data['commander'].get('docked'): + last_starport = '.'+data.get('lastStarport', {}).get('name', 'Unknown') + timestamp: str = strftime('%Y-%m-%dT%H.%M.%S', localtime()) + f = tkinter.filedialog.asksaveasfilename(parent=self.w, + defaultextension=default_extension, + filetypes=[('JSON', '.json'), ('All Files', '*')], + initialdir=config.get('outdir'), + initialfile=f'{last_system}{last_starport}.{timestamp}') if f: with open(f, 'wb') as h: h.write(json.dumps(data, @@ -872,10 +884,10 @@ class AppWindow(object): self.drag_offset = None def oniconify(self, event=None): - self.w.overrideredirect(0) # Can't iconize while overrideredirect + self.w.overrideredirect(0) # Can't iconize while overrideredirect self.w.iconify() - self.w.update_idletasks() # Size and windows styles get recalculated here - self.w.wait_visibility() # Need main window to be re-created before returning + self.w.update_idletasks() # Size and windows styles get recalculated here + self.w.wait_visibility() # Need main window to be re-created before returning theme.active = None # So theme will be re-applied on map def onmap(self, event=None): @@ -923,9 +935,9 @@ def enforce_single_instance() -> None: def WindowTitle(h): if h: - l = GetWindowTextLength(h) + 1 - buf = ctypes.create_unicode_buffer(l) - if GetWindowText(h, buf, l): + text_length = GetWindowTextLength(h) + 1 + buf = ctypes.create_unicode_buffer(text_length) + if GetWindowText(h, buf, text_length): return buf.value return None @@ -933,7 +945,10 @@ def enforce_single_instance() -> None: def enumwindowsproc(hWnd, lParam): # class name limited to 256 - https://msdn.microsoft.com/en-us/library/windows/desktop/ms633576 cls = ctypes.create_unicode_buffer(257) - if GetClassName(hWnd, cls, 257) and cls.value == 'TkTopLevel' and WindowTitle(hWnd) == applongname and GetProcessHandleFromHwnd(hWnd): + if GetClassName(hWnd, cls, 257)\ + and cls.value == 'TkTopLevel'\ + and WindowTitle(hWnd) == applongname\ + and GetProcessHandleFromHwnd(hWnd): # If GetProcessHandleFromHwnd succeeds then the app is already running as this user if len(sys.argv) > 1 and sys.argv[1].startswith(protocolhandler.redirect): # Browser invoked us directly with auth response. Forward the response to the other app instance. diff --git a/companion.py b/companion.py index d1ef0481..10c6ecaf 100644 --- a/companion.py +++ b/companion.py @@ -7,7 +7,7 @@ import requests from typing import TYPE_CHECKING # TODO: see https://github.com/EDCD/EDMarketConnector/issues/569 -from http.cookiejar import LWPCookieJar # No longer needed but retained in case plugins use it +from http.cookiejar import LWPCookieJar # noqa: F401 - No longer needed but retained in case plugins use it from email.utils import parsedate import hashlib import numbers diff --git a/plug.py b/plug.py index a443de21..13f187d4 100644 --- a/plug.py +++ b/plug.py @@ -11,7 +11,7 @@ import threading # noqa: F401 - We don't use it, but plugins might import logging import tkinter as tk -import myNotebook as nb +import myNotebook as nb # noqa: N813 from config import config, appname @@ -202,6 +202,7 @@ def load_plugins(master): pass PLUGINS.extend(sorted(found, key=lambda p: operator.attrgetter('name')(p).lower())) + def provides(fn_name): """ Find plugins that provide a function diff --git a/plugins/eddn.py b/plugins/eddn.py index 1e979d48..b65ac034 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -126,7 +126,8 @@ class EDDN(object): except json.JSONDecodeError as e: # Couldn't decode - shouldn't happen! logger.debug(f'\n{self.replaylog[0]}\n', exc_info=e) - self.replaylog.pop(0) # Discard and continue + # Discard and continue + self.replaylog.pop(0) else: # Rewrite old schema name if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'): @@ -137,9 +138,9 @@ class EDDN(object): if not len(self.replaylog) % self.REPLAYFLUSH: self.flush() except requests.exceptions.RequestException as e: - logger.debug(f'Failed sending', exc_info=e) + logger.debug('Failed sending', exc_info=e) status['text'] = _("Error: Can't connect to EDDN") - return # stop sending + return # stop sending except Exception as e: logger.debug('Failed sending', exc_info=e) status['text'] = str(e) @@ -437,7 +438,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): try: this.eddn.export_journal_entry(cmdr, is_beta, filter_localised(entry)) except requests.exceptions.RequestException as e: - logger.debug(f'Failed in export_journal_entry', exc_info=e) + logger.debug('Failed in export_journal_entry', exc_info=e) return _("Error: Can't connect to EDDN") except Exception as e: logger.debug('Failed in export_journal_entry', exc_info=e) diff --git a/plugins/edsm.py b/plugins/edsm.py index 3e59404d..790f9289 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -14,7 +14,6 @@ import json import requests import sys -import time import urllib.request import urllib.error import urllib.parse @@ -24,7 +23,7 @@ import logging import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel -import myNotebook as nb +import myNotebook as nb # noqa: N813 from config import appname, applongname, appversion, config import plug @@ -380,7 +379,7 @@ def worker(): break except Exception as e: - logger.debug(f'Sending API events', exc_info=e) + logger.debug('Sending API events', exc_info=e) retrying += 1 else: plug.show_error(_("Error: Can't connect to EDSM")) diff --git a/plugins/inara.py b/plugins/inara.py index a0f33191..bfff9e5d 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -910,9 +910,13 @@ def worker(): plug.show_error(_('Error: Inara {MSG}').format( MSG=f'{data_event["eventName"]},' f'{reply_event.get("eventStatusText", reply_event["eventStatus"])}')) - if data_event['eventName'] in ['addCommanderTravelCarrierJump', 'addCommanderTravelDock', 'addCommanderTravelFSDJump', 'setCommanderTravelLocation']: + if data_event['eventName'] in ('addCommanderTravelCarrierJump', + 'addCommanderTravelDock', + 'addCommanderTravelFSDJump', + 'setCommanderTravelLocation'): this.lastlocation = reply_event.get('eventData', {}) - this.system_link.event_generate('<>', when="tail") # calls update_location in main thread + # calls update_location in main thread + this.system_link.event_generate('<>', when="tail") elif data_event['eventName'] in ['addCommanderShip', 'setCommanderShip']: this.lastship = reply_event.get('eventData', {}) # calls update_ship in main thread From 6c9139e395b48f30155810dd3ede31a0e497e12a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 10:35:08 +0100 Subject: [PATCH 083/258] TODO: unittest hint about testing logging And comment out the use of it here. --- EDMarketConnector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 1ea0967e..d6e9e579 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -979,12 +979,13 @@ if __name__ == "__main__": # Plain, not via `logger` print(f'{applongname} {appversion}') + # TODO: unittest in place of this class A(object): class B(object): def __init__(self): logger.debug('A call from A.B.__init__') - abinit = A.B() + #abinit = A.B() Translations.install(config.get('language') or None) # Can generate errors so wait til log set up From 6429cae9328193f02174625031c9dd8dd4c7b707 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 10:37:36 +0100 Subject: [PATCH 084/258] Use isinstance() for type checking --- EDMCLogging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 69e8389b..acae327d 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -104,7 +104,7 @@ class EDMCContextFilter(logging.Filter): # of the frames internal to logging. frame = sys._getframe(0) while frame: - if type(frame.f_locals.get('self')) == logging.Logger: + if isinstance(frame.f_locals.get('self'), logging.Logger): frame = frame.f_back # Want to start on the next frame below break frame = frame.f_back @@ -113,7 +113,7 @@ class EDMCContextFilter(logging.Filter): # that is *not* true, as it should be the call site of the logger # call while frame: - if type(frame.f_locals.get('self')) != logging.Logger: + if not isinstance(frame.f_locals.get('self'), logging.Logger): break # We've found the frame we want frame = frame.f_back From 19e750edddb4815496681e6a0bc722d220d15db1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 10:47:57 +0100 Subject: [PATCH 085/258] Use logger.exception() not log.error(.., exc_info=..) --- EDMarketConnector.py | 2 +- companion.py | 6 +++--- plug.py | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index d6e9e579..2c11c0ef 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -407,7 +407,7 @@ class AppWindow(object): except (companion.CredentialsError, companion.ServerError, companion.ServerLagging) as e: self.status['text'] = str(e) except Exception as e: - logger.debug(f'{__class__}', exc_info=e) + logger.debug(f'Frontier CAPI Auth', exc_info=e) self.status['text'] = str(e) self.cooldown() diff --git a/companion.py b/companion.py index 10c6ecaf..3bb23fa1 100644 --- a/companion.py +++ b/companion.py @@ -202,7 +202,7 @@ class Auth(object): self.dump(r) except Exception as e: - logger.error(f"Frontier CAPI Auth: Can't refresh token for \"{self.cmdr}\"", exc_info=e) + logger.exception(f"Frontier CAPI Auth: Can't refresh token for \"{self.cmdr}\"") else: logger.error(f"Frontier CAPI Auth: No token for \"{self.cmdr}\"") @@ -269,7 +269,7 @@ class Auth(object): return data.get('access_token') except Exception as e: - logger.error(f"Frontier CAPI Auth: Can't get token for \"{self.cmdr}\"", exc_info=e) + logger.exception(f"Frontier CAPI Auth: Can't get token for \"{self.cmdr}\"") if r: self.dump(r) @@ -400,7 +400,7 @@ class Session(object): data = r.json() # May also fail here if token expired since response is empty except (requests.HTTPError, ValueError) as e: - logger.error('Frontier CAPI Auth: GET ', exc_info=e) + logger.exception('Frontier CAPI Auth: GET ') self.dump(r) self.close() diff --git a/plug.py b/plug.py index 13f187d4..f836d7cd 100644 --- a/plug.py +++ b/plug.py @@ -107,7 +107,7 @@ class Plugin(object): else: logger.error(f'plugin {name} has no plugin_start3() function') except Exception as e: - logger.error(f': Failed for Plugin "{name}"', exc_info=e) + logger.exception(f': Failed for Plugin "{name}"') raise else: logger.info(f'plugin {name} disabled') @@ -139,7 +139,7 @@ class Plugin(object): raise AssertionError return appitem except Exception as e: - logger.error(f'Failed for Plugin "{self.name}"', exc_info=e) + logger.exception(f'Failed for Plugin "{self.name}"') return None def get_prefs(self, parent, cmdr, is_beta): @@ -159,7 +159,7 @@ class Plugin(object): raise AssertionError return frame except Exception as e: - logger.error(f'Failed for Plugin "{self.name}"', exc_info=e) + logger.exception(f'Failed for Plugin "{self.name}"') return None @@ -177,7 +177,7 @@ def load_plugins(master): plugin.folder = None # Suppress listing in Plugins prefs tab internal.append(plugin) except Exception as e: - logger.error(f'Failure loading internal Plugin "{name}"', exc_info=e) + logger.exception(f'Failure loading internal Plugin "{name}"') PLUGINS.extend(sorted(internal, key=lambda p: operator.attrgetter('name')(p).lower())) # Add plugin folder to load path so packages can be loaded from plugin folder @@ -198,7 +198,7 @@ def load_plugins(master): sys.path.append(os.path.join(config.plugin_dir, name)) found.append(Plugin(name, os.path.join(config.plugin_dir, name, 'load.py'))) except Exception as e: - logger.error(f'Failure loading found Plugin "{name}"', exc_info=e) + logger.exception(f'Failure loading found Plugin "{name}"') pass PLUGINS.extend(sorted(found, key=lambda p: operator.attrgetter('name')(p).lower())) @@ -245,7 +245,7 @@ def notify_stop(): newerror = plugin_stop() error = error or newerror except Exception as e: - logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) + logger.exception(f'Plugin "{plugin.name}" failed') return error @@ -262,7 +262,7 @@ def notify_prefs_cmdr_changed(cmdr, is_beta): try: prefs_cmdr_changed(cmdr, is_beta) except Exception as e: - logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) + logger.exception(f'Plugin "{plugin.name}" failed') def notify_prefs_changed(cmdr, is_beta): @@ -280,7 +280,7 @@ def notify_prefs_changed(cmdr, is_beta): try: prefs_changed(cmdr, is_beta) except Exception as e: - logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) + logger.exception(f'Plugin "{plugin.name}" failed') def notify_journal_entry(cmdr, is_beta, system, station, entry, state): @@ -303,7 +303,7 @@ def notify_journal_entry(cmdr, is_beta, system, station, entry, state): newerror = journal_entry(cmdr, is_beta, system, station, dict(entry), dict(state)) error = error or newerror except Exception as e: - logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) + logger.exception(f'Plugin "{plugin.name}" failed') return error @@ -324,7 +324,7 @@ def notify_dashboard_entry(cmdr, is_beta, entry): newerror = status(cmdr, is_beta, dict(entry)) error = error or newerror except Exception as e: - logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) + logger.exception(f'Plugin "{plugin.name}" failed') return error @@ -343,7 +343,7 @@ def notify_newdata(data, is_beta): newerror = cmdr_data(data, is_beta) error = error or newerror except Exception as e: - logger.error(f'Plugin "{plugin.name}" failed', exc_info=e) + logger.exception(f'Plugin "{plugin.name}" failed') return error From 63f3859af443a61b4208c3c267e3892bc98e2b18 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 10:56:57 +0100 Subject: [PATCH 086/258] Remove extraneous __class__ in logging strings --- EDMarketConnector.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 2c11c0ef..bbf2dfa0 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -520,8 +520,8 @@ class AppWindow(object): companion.session.invalidate() self.login() - except Exception as e: # Including CredentialsError, ServerError - logger.debug(f'{__class__}', exc_info=e) + except Exception as e: # Including CredentialsError, ServerError + logger.debug('"other" exception', exc_info=e) self.status['text'] = str(e) play_bad = True @@ -543,7 +543,7 @@ class AppWindow(object): report = 'Failure' else: report = 'Undocked!' - logger.debug(f'{__class__}: Retry for shipyard - {report}') + logger.debug(f'Retry for shipyard - {report}') if not data['commander'].get('docked'): # might have un-docked while we were waiting for retry in which case station data is unreliable pass @@ -612,10 +612,10 @@ class AppWindow(object): # Disable WinSparkle automatic update checks, IFF configured to do so when in-game if config.getint('disable_autoappupdatecheckingame') and 1: self.updater.setAutomaticUpdatesCheck(False) - logger.info(f'{__class__}: Monitor: Disable WinSparkle automatic update checks') + logger.info('Monitor: Disable WinSparkle automatic update checks') # Can start dashboard monitoring if not dashboard.start(self.w, monitor.started): - logger.info(f"{__class__}: Can't start Status monitoring") + logger.info("Can't start Status monitoring") # Export loadout if entry['event'] == 'Loadout'\ @@ -643,7 +643,7 @@ class AppWindow(object): # Enable WinSparkle automatic update checks # NB: Do this blindly, in case option got changed whilst in-game self.updater.setAutomaticUpdatesCheck(True) - logger.info(f'{__class__}: Monitor: Enable WinSparkle automatic update checks') + logger.info('Monitor: Enable WinSparkle automatic update checks') # cAPI auth def auth(self, event=None): @@ -660,7 +660,7 @@ class AppWindow(object): except companion.ServerError as e: self.status['text'] = str(e) except Exception as e: - logger.debug(f'{__class__}', exc_info=e) + logger.debug('Frontier CAPI Auth:', exc_info=e) self.status['text'] = str(e) self.cooldown() @@ -815,7 +815,7 @@ class AppWindow(object): self.protocol("WM_DELETE_WINDOW", self._destroy) ############################################################ - logger.info(f'{__class__}: Current version is {appversion}') + logger.info(f'Current version is {appversion}') def apply(self): self._destroy() @@ -855,7 +855,7 @@ class AppWindow(object): except companion.ServerError as e: self.status['text'] = str(e) except Exception as e: - logger.debug(f'{__class__}', exc_info=e) + logger.debug('"other" exception', exc_info=e) self.status['text'] = str(e) def onexit(self, event=None): From e0324cb9cd931a7297dc7bca58539353f5a43025 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 11:15:03 +0100 Subject: [PATCH 087/258] More flake8 cleanup --- EDMarketConnector.py | 382 ++++++++++++++++++++++++------------------- 1 file changed, 216 insertions(+), 166 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index bbf2dfa0..9c7b4a14 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -5,11 +5,9 @@ from builtins import str from builtins import object import sys from sys import platform -from collections import OrderedDict -from functools import partial import json from os import chdir, environ -from os.path import dirname, expanduser, isdir, join +from os.path import dirname, isdir, join import re import html from time import time, localtime, strftime @@ -34,7 +32,6 @@ import tkinter.messagebox from ttkHyperlinkLabel import HyperlinkLabel if __debug__: - from traceback import print_exc if platform != 'win32': import pdb import signal @@ -55,7 +52,7 @@ from dashboard import dashboard from theme import theme -SERVER_RETRY = 5 # retry pause for Companion servers [s] +SERVER_RETRY = 5 # retry pause for Companion servers [s] SHIPYARD_HTML_TEMPLATE = """ @@ -77,8 +74,8 @@ class AppWindow(object): # Tkinter Event types EVENT_KEYPRESS = 2 - EVENT_BUTTON = 4 - EVENT_VIRTUAL = 35 + EVENT_BUTTON = 4 + EVENT_VIRTUAL = 35 def __init__(self, master): @@ -97,10 +94,10 @@ class AppWindow(object): if platform == 'win32': self.w.wm_iconbitmap(default='EDMarketConnector.ico') else: - self.w.tk.call('wm', 'iconphoto', self.w, '-default', tk.PhotoImage(file = join(config.respath, 'EDMarketConnector.png'))) - self.theme_icon = tk.PhotoImage(data = 'R0lGODlhFAAQAMZQAAoKCQoKCgsKCQwKCQsLCgwLCg4LCQ4LCg0MCg8MCRAMCRANChINCREOChIOChQPChgQChgRCxwTCyYVCSoXCS0YCTkdCTseCT0fCTsjDU0jB0EnDU8lB1ElB1MnCFIoCFMoCEkrDlkqCFwrCGEuCWIuCGQvCFs0D1w1D2wyCG0yCF82D182EHE0CHM0CHQ1CGQ5EHU2CHc3CHs4CH45CIA6CIE7CJdECIdLEolMEohQE5BQE41SFJBTE5lUE5pVE5RXFKNaFKVbFLVjFbZkFrxnFr9oFsNqFsVrF8RsFshtF89xF9NzGNh1GNl2GP+KG////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////yH5BAEKAH8ALAAAAAAUABAAAAeegAGCgiGDhoeIRDiIjIZGKzmNiAQBQxkRTU6am0tPCJSGShuSAUcLoIIbRYMFra4FAUgQAQCGJz6CDQ67vAFJJBi0hjBBD0w9PMnJOkAiJhaIKEI7HRoc19ceNAolwbWDLD8uAQnl5ga1I9CHEjEBAvDxAoMtFIYCBy+kFDKHAgM3ZtgYSLAGgwkp3pEyBOJCC2ELB31QATGioAoVAwEAOw==') - self.theme_minimize = tk.BitmapImage(data = '#define im_width 16\n#define im_height 16\nstatic unsigned char im_bits[] = {\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x3f,\n 0xfc, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };\n') - self.theme_close = tk.BitmapImage(data = '#define im_width 16\n#define im_height 16\nstatic unsigned char im_bits[] = {\n 0x00, 0x00, 0x00, 0x00, 0x0c, 0x30, 0x1c, 0x38, 0x38, 0x1c, 0x70, 0x0e,\n 0xe0, 0x07, 0xc0, 0x03, 0xc0, 0x03, 0xe0, 0x07, 0x70, 0x0e, 0x38, 0x1c,\n 0x1c, 0x38, 0x0c, 0x30, 0x00, 0x00, 0x00, 0x00 };\n') + self.w.tk.call('wm', 'iconphoto', self.w, '-default', tk.PhotoImage(file=join(config.respath, 'EDMarketConnector.png'))) # noqa: E501 + self.theme_icon = tk.PhotoImage(data='R0lGODlhFAAQAMZQAAoKCQoKCgsKCQwKCQsLCgwLCg4LCQ4LCg0MCg8MCRAMCRANChINCREOChIOChQPChgQChgRCxwTCyYVCSoXCS0YCTkdCTseCT0fCTsjDU0jB0EnDU8lB1ElB1MnCFIoCFMoCEkrDlkqCFwrCGEuCWIuCGQvCFs0D1w1D2wyCG0yCF82D182EHE0CHM0CHQ1CGQ5EHU2CHc3CHs4CH45CIA6CIE7CJdECIdLEolMEohQE5BQE41SFJBTE5lUE5pVE5RXFKNaFKVbFLVjFbZkFrxnFr9oFsNqFsVrF8RsFshtF89xF9NzGNh1GNl2GP+KG////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////yH5BAEKAH8ALAAAAAAUABAAAAeegAGCgiGDhoeIRDiIjIZGKzmNiAQBQxkRTU6am0tPCJSGShuSAUcLoIIbRYMFra4FAUgQAQCGJz6CDQ67vAFJJBi0hjBBD0w9PMnJOkAiJhaIKEI7HRoc19ceNAolwbWDLD8uAQnl5ga1I9CHEjEBAvDxAoMtFIYCBy+kFDKHAgM3ZtgYSLAGgwkp3pEyBOJCC2ELB31QATGioAoVAwEAOw==') # noqa: E501 + self.theme_minimize = tk.BitmapImage(data='#define im_width 16\n#define im_height 16\nstatic unsigned char im_bits[] = {\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x3f,\n 0xfc, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };\n') # noqa: E501 + self.theme_close = tk.BitmapImage(data='#define im_width 16\n#define im_height 16\nstatic unsigned char im_bits[] = {\n 0x00, 0x00, 0x00, 0x00, 0x0c, 0x30, 0x1c, 0x38, 0x38, 0x1c, 0x70, 0x0e,\n 0xe0, 0x07, 0xc0, 0x03, 0xc0, 0x03, 0xe0, 0x07, 0x70, 0x0e, 0x38, 0x1c,\n 0x1c, 0x38, 0x0c, 0x30, 0x00, 0x00, 0x00, 0x00 };\n') # noqa: E501 frame = tk.Frame(self.w, name=appname.lower()) frame.grid(sticky=tk.NSEW) @@ -116,10 +113,10 @@ class AppWindow(object): self.system_label.grid(row=3, column=0, sticky=tk.W) self.station_label.grid(row=4, column=0, sticky=tk.W) - self.cmdr = tk.Label(frame, compound=tk.RIGHT, anchor=tk.W, name = 'cmdr') - self.ship = HyperlinkLabel(frame, compound=tk.RIGHT, url = self.shipyard_url, name = 'ship') - self.system = HyperlinkLabel(frame, compound=tk.RIGHT, url = self.system_url, popup_copy = True, name = 'system') - self.station = HyperlinkLabel(frame, compound=tk.RIGHT, url = self.station_url, name = 'station') + self.cmdr = tk.Label(frame, compound=tk.RIGHT, anchor=tk.W, name='cmdr') + self.ship = HyperlinkLabel(frame, compound=tk.RIGHT, url=self.shipyard_url, name='ship') + self.system = HyperlinkLabel(frame, compound=tk.RIGHT, url=self.system_url, popup_copy=True, name='system') + self.station = HyperlinkLabel(frame, compound=tk.RIGHT, url=self.station_url, name='station') self.cmdr.grid(row=1, column=1, sticky=tk.EW) self.ship.grid(row=2, column=1, sticky=tk.EW) @@ -129,39 +126,41 @@ class AppWindow(object): for plugin in plug.PLUGINS: appitem = plugin.get_app(frame) if appitem: - tk.Frame(frame, highlightthickness=1).grid(columnspan=2, sticky=tk.EW) # separator - if isinstance(appitem, tuple) and len(appitem)==2: + tk.Frame(frame, highlightthickness=1).grid(columnspan=2, sticky=tk.EW) # separator + if isinstance(appitem, tuple) and len(appitem) == 2: row = frame.grid_size()[1] appitem[0].grid(row=row, column=0, sticky=tk.W) appitem[1].grid(row=row, column=1, sticky=tk.EW) else: appitem.grid(columnspan=2, sticky=tk.EW) - self.button = ttk.Button(frame, text=_('Update'), width=28, default=tk.ACTIVE, state=tk.DISABLED) # Update button in main window - self.theme_button = tk.Label(frame, width = platform == 'darwin' and 32 or 28, state=tk.DISABLED) + # Update button in main window + self.button = ttk.Button(frame, text=_('Update'), width=28, default=tk.ACTIVE, state=tk.DISABLED) + self.theme_button = tk.Label(frame, width=32 if platform == 'darwin' else 28, state=tk.DISABLED) self.status = tk.Label(frame, name='status', anchor=tk.W) row = frame.grid_size()[1] self.button.grid(row=row, columnspan=2, sticky=tk.NSEW) self.theme_button.grid(row=row, columnspan=2, sticky=tk.NSEW) - theme.register_alternate((self.button, self.theme_button, self.theme_button), {'row':row, 'columnspan':2, 'sticky':tk.NSEW}) + theme.register_alternate((self.button, self.theme_button, self.theme_button), {'row': row, 'columnspan': 2, 'sticky': tk.NSEW}) # noqa: E501 self.status.grid(columnspan=2, sticky=tk.EW) self.button.bind('', self.getandsend) theme.button_bind(self.theme_button, self.getandsend) for child in frame.winfo_children(): - child.grid_configure(padx=5, pady=(platform!='win32' or isinstance(child, tk.Frame)) and 2 or 0) + child.grid_configure(padx=5, pady=(platform != 'win32' or isinstance(child, tk.Frame)) and 2 or 0) self.menubar = tk.Menu() - if platform=='darwin': + if platform == 'darwin': # Can't handle (de)iconify if topmost is set, so suppress iconify button - # http://wiki.tcl.tk/13428 and p15 of https://developer.apple.com/legacy/library/documentation/Carbon/Conceptual/HandlingWindowsControls/windowscontrols.pdf + # http://wiki.tcl.tk/13428 and p15 of + # https://developer.apple.com/legacy/library/documentation/Carbon/Conceptual/HandlingWindowsControls/windowscontrols.pdf root.call('tk::unsupported::MacWindowStyle', 'style', root, 'document', 'closeBox resizable') # https://www.tcl.tk/man/tcl/TkCmd/menu.htm self.system_menu = tk.Menu(self.menubar, name='apple') - self.system_menu.add_command(command=lambda:self.w.call('tk::mac::standardAboutPanel')) - self.system_menu.add_command(command=lambda:self.updater.checkForUpdates()) + self.system_menu.add_command(command=lambda: self.w.call('tk::mac::standardAboutPanel')) + self.system_menu.add_command(command=lambda: self.updater.checkForUpdates()) self.menubar.add_cascade(menu=self.system_menu) self.file_menu = tk.Menu(self.menubar, name='file') self.file_menu.add_command(command=self.save_raw) @@ -171,7 +170,7 @@ class AppWindow(object): self.menubar.add_cascade(menu=self.edit_menu) self.w.bind('', self.copy) self.view_menu = tk.Menu(self.menubar, name='view') - self.view_menu.add_command(command=lambda:stats.StatsDialog(self)) + self.view_menu.add_command(command=lambda: stats.StatsDialog(self)) self.menubar.add_cascade(menu=self.view_menu) window_menu = tk.Menu(self.menubar, name='window') self.menubar.add_cascade(menu=window_menu) @@ -183,17 +182,17 @@ class AppWindow(object): self.w['menu'] = self.menubar # https://www.tcl.tk/man/tcl/TkCmd/tk_mac.htm self.w.call('set', 'tk::mac::useCompatibilityMetrics', '0') - self.w.createcommand('tkAboutDialog', lambda:self.w.call('tk::mac::standardAboutPanel')) + self.w.createcommand('tkAboutDialog', lambda: self.w.call('tk::mac::standardAboutPanel')) self.w.createcommand("::tk::mac::Quit", self.onexit) - self.w.createcommand("::tk::mac::ShowPreferences", lambda:prefs.PreferencesDialog(self.w, self.postprefs)) - self.w.createcommand("::tk::mac::ReopenApplication", self.w.deiconify) # click on app in dock = restore - self.w.protocol("WM_DELETE_WINDOW", self.w.withdraw) # close button shouldn't quit app - self.w.resizable(tk.FALSE, tk.FALSE) # Can't be only resizable on one axis + self.w.createcommand("::tk::mac::ShowPreferences", lambda: prefs.PreferencesDialog(self.w, self.postprefs)) + self.w.createcommand("::tk::mac::ReopenApplication", self.w.deiconify) # click on app in dock = restore + self.w.protocol("WM_DELETE_WINDOW", self.w.withdraw) # close button shouldn't quit app + self.w.resizable(tk.FALSE, tk.FALSE) # Can't be only resizable on one axis else: self.file_menu = self.view_menu = tk.Menu(self.menubar, tearoff=tk.FALSE) - self.file_menu.add_command(command=lambda:stats.StatsDialog(self)) + self.file_menu.add_command(command=lambda: stats.StatsDialog(self)) self.file_menu.add_command(command=self.save_raw) - self.file_menu.add_command(command=lambda:prefs.PreferencesDialog(self.w, self.postprefs)) + self.file_menu.add_command(command=lambda: prefs.PreferencesDialog(self.w, self.postprefs)) self.file_menu.add_separator() self.file_menu.add_command(command=self.onexit) self.menubar.add_cascade(menu=self.file_menu) @@ -213,11 +212,13 @@ class AppWindow(object): self.always_ontop = tk.BooleanVar(value=config.getint('always_ontop')) self.system_menu = tk.Menu(self.menubar, name='system', tearoff=tk.FALSE) self.system_menu.add_separator() - self.system_menu.add_checkbutton(label=_('Always on top'), variable = self.always_ontop, command=self.ontop_changed) # Appearance setting + self.system_menu.add_checkbutton(label=_('Always on top'), + variable=self.always_ontop, + command=self.ontop_changed) # Appearance setting self.menubar.add_cascade(menu=self.system_menu) self.w.bind('', self.copy) self.w.protocol("WM_DELETE_WINDOW", self.onexit) - theme.register(self.menubar) # menus and children aren't automatically registered + theme.register(self.menubar) # menus and children aren't automatically registered theme.register(self.file_menu) theme.register(self.edit_menu) theme.register(self.help_menu) @@ -225,7 +226,9 @@ class AppWindow(object): # Alternate title bar and menu for dark theme self.theme_menubar = tk.Frame(frame) self.theme_menubar.columnconfigure(2, weight=1) - theme_titlebar = tk.Label(self.theme_menubar, text=applongname, image=self.theme_icon, cursor='fleur', anchor=tk.W, compound=tk.LEFT) + theme_titlebar = tk.Label(self.theme_menubar, text=applongname, + image=self.theme_icon, cursor='fleur', + anchor=tk.W, compound=tk.LEFT) theme_titlebar.grid(columnspan=3, padx=2, sticky=tk.NSEW) self.drag_offset = None theme_titlebar.bind('', self.drag_start) @@ -239,37 +242,49 @@ class AppWindow(object): theme.button_bind(theme_close, self.onexit, image=self.theme_close) self.theme_file_menu = tk.Label(self.theme_menubar, anchor=tk.W) self.theme_file_menu.grid(row=1, column=0, padx=5, sticky=tk.W) - theme.button_bind(self.theme_file_menu, lambda e: self.file_menu.tk_popup(e.widget.winfo_rootx(), e.widget.winfo_rooty() + e.widget.winfo_height())) + theme.button_bind(self.theme_file_menu, + lambda e: self.file_menu.tk_popup(e.widget.winfo_rootx(), + e.widget.winfo_rooty() + + e.widget.winfo_height())) self.theme_edit_menu = tk.Label(self.theme_menubar, anchor=tk.W) self.theme_edit_menu.grid(row=1, column=1, sticky=tk.W) - theme.button_bind(self.theme_edit_menu, lambda e: self.edit_menu.tk_popup(e.widget.winfo_rootx(), e.widget.winfo_rooty() + e.widget.winfo_height())) + theme.button_bind(self.theme_edit_menu, + lambda e: self.edit_menu.tk_popup(e.widget.winfo_rootx(), + e.widget.winfo_rooty() + + e.widget.winfo_height())) self.theme_help_menu = tk.Label(self.theme_menubar, anchor=tk.W) self.theme_help_menu.grid(row=1, column=2, sticky=tk.W) - theme.button_bind(self.theme_help_menu, lambda e: self.help_menu.tk_popup(e.widget.winfo_rootx(), e.widget.winfo_rooty() + e.widget.winfo_height())) + theme.button_bind(self.theme_help_menu, + lambda e: self.help_menu.tk_popup(e.widget.winfo_rootx(), + e.widget.winfo_rooty() + + e.widget.winfo_height())) tk.Frame(self.theme_menubar, highlightthickness=1).grid(columnspan=5, padx=5, sticky=tk.EW) - theme.register(self.theme_minimize) # images aren't automatically registered + theme.register(self.theme_minimize) # images aren't automatically registered theme.register(self.theme_close) self.blank_menubar = tk.Frame(frame) tk.Label(self.blank_menubar).grid() tk.Label(self.blank_menubar).grid() tk.Frame(self.blank_menubar, height=2).grid() - theme.register_alternate((self.menubar, self.theme_menubar, self.blank_menubar), {'row':0, 'columnspan':2, 'sticky':tk.NSEW}) + theme.register_alternate((self.menubar, self.theme_menubar, self.blank_menubar), + {'row': 0, 'columnspan': 2, 'sticky': tk.NSEW}) self.w.resizable(tk.TRUE, tk.FALSE) # update geometry if config.get('geometry'): - match = re.match('\+([\-\d]+)\+([\-\d]+)', config.get('geometry')) + match = re.match('\+([\-\d]+)\+([\-\d]+)', config.get('geometry')) # noqa: W605 if match: if platform == 'darwin': - if int(match.group(2)) >= 0: # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + if int(match.group(2)) >= 0: self.w.geometry(config.get('geometry')) elif platform == 'win32': # Check that the titlebar will be at least partly on screen import ctypes from ctypes.wintypes import POINT # https://msdn.microsoft.com/en-us/library/dd145064 - MONITOR_DEFAULTTONULL = 0 - if ctypes.windll.user32.MonitorFromPoint(POINT(int(match.group(1)) + 16, int(match.group(2)) + 16), MONITOR_DEFAULTTONULL): + MONITOR_DEFAULTTONULL = 0 # noqa: N806 + if ctypes.windll.user32.MonitorFromPoint(POINT(int(match.group(1)) + 16, int(match.group(2)) + 16), + MONITOR_DEFAULTTONULL): self.w.geometry(config.get('geometry')) else: self.w.geometry(config.get('geometry')) @@ -278,22 +293,22 @@ class AppWindow(object): theme.register(frame) theme.apply(self.w) - self.w.bind('', self.onmap) # Special handling for overrideredict - self.w.bind('', self.onenter) # Special handling for transparency - self.w.bind('', self.onenter) # " - self.w.bind('', self.onleave) # " - self.w.bind('', self.onleave) # " + self.w.bind('', self.onmap) # Special handling for overrideredict + self.w.bind('', self.onenter) # Special handling for transparency + self.w.bind('', self.onenter) # Special handling for transparency + self.w.bind('', self.onleave) # Special handling for transparency + self.w.bind('', self.onleave) # Special handling for transparency self.w.bind('', self.getandsend) self.w.bind('', self.getandsend) - self.w.bind_all('<>', self.getandsend) # Hotkey monitoring - self.w.bind_all('<>', self.journal_event) # Journal monitoring - self.w.bind_all('<>', self.dashboard_event) # Dashboard monitoring - self.w.bind_all('<>', self.plugin_error) # Statusbar - self.w.bind_all('<>', self.auth) # cAPI auth - self.w.bind_all('<>', self.onexit) # Updater + self.w.bind_all('<>', self.getandsend) # Hotkey monitoring + self.w.bind_all('<>', self.journal_event) # Journal monitoring + self.w.bind_all('<>', self.dashboard_event) # Dashboard monitoring + self.w.bind_all('<>', self.plugin_error) # Statusbar + self.w.bind_all('<>', self.auth) # cAPI auth + self.w.bind_all('<>', self.onexit) # Updater # Start a protocol handler to handle cAPI registration. Requires main loop to be running. - self.w.after_idle(lambda:protocolhandler.start(self.w)) + self.w.after_idle(lambda: protocolhandler.start(self.w)) # Load updater after UI creation (for WinSparkle) import update @@ -305,7 +320,7 @@ class AppWindow(object): self.updater.checkForUpdates() # Sparkle / WinSparkle does this automatically for packaged apps try: - config.get_password('') # Prod SecureStorage on Linux to initialise + config.get_password('') # Prod SecureStorage on Linux to initialise except RuntimeError: pass @@ -317,93 +332,93 @@ class AppWindow(object): config.delete('password') config.delete('logdir') - self.postprefs(False) # Companion login happens in callback from monitor - + self.postprefs(False) # Companion login happens in callback from monitor # callback after the Preferences dialog is applied def postprefs(self, dologin=True): self.prefsdialog = None - self.set_labels() # in case language has changed + self.set_labels() # in case language has changed # Reset links in case plugins changed them - self.ship.configure(url = self.shipyard_url) - self.system.configure(url = self.system_url) - self.station.configure(url = self.station_url) + self.ship.configure(url=self.shipyard_url) + self.system.configure(url=self.system_url) + self.station.configure(url=self.station_url) # (Re-)install hotkey monitoring hotkeymgr.register(self.w, config.getint('hotkey_code'), config.getint('hotkey_mods')) # (Re-)install log monitoring if not monitor.start(self.w): - self.status['text'] = 'Error: Check %s' % _('E:D journal file location') # Location of the new Journal file in E:D 2.2 + self.status['text'] = f'Error: Check {_("E:D journal file location")}' if dologin and monitor.cmdr: - self.login() # Login if not already logged in with this Cmdr + self.login() # Login if not already logged in with this Cmdr # set main window labels, e.g. after language change def set_labels(self): - self.cmdr_label['text'] = _('Cmdr') + ':' # Main window - self.ship_label['text'] = (monitor.state['Captain'] and _('Role') or # Multicrew role label in main window - _('Ship')) + ':' # Main window - self.system_label['text'] = _('System') + ':' # Main window - self.station_label['text'] = _('Station') + ':' # Main window - self.button['text'] = self.theme_button['text'] = _('Update') # Update button in main window + self.cmdr_label['text'] = _('Cmdr') + ':' # Main window + # Multicrew role label in main window + self.ship_label['text'] = (monitor.state['Captain'] and _('Role') or _('Ship')) + ':' # Main window + self.system_label['text'] = _('System') + ':' # Main window + self.station_label['text'] = _('Station') + ':' # Main window + self.button['text'] = self.theme_button['text'] = _('Update') # Update button in main window if platform == 'darwin': - self.menubar.entryconfigure(1, label=_('File')) # Menu title - self.menubar.entryconfigure(2, label=_('Edit')) # Menu title - self.menubar.entryconfigure(3, label=_('View')) # Menu title on OSX - self.menubar.entryconfigure(4, label=_('Window')) # Menu title on OSX - self.menubar.entryconfigure(5, label=_('Help')) # Menu title - self.system_menu.entryconfigure(0, label=_("About {APP}").format(APP=applongname)) # App menu entry on OSX - self.system_menu.entryconfigure(1, label=_("Check for Updates...")) # Menu item - self.file_menu.entryconfigure(0, label=_('Save Raw Data...')) # Menu item - self.view_menu.entryconfigure(0, label=_('Status')) # Menu item - self.help_menu.entryconfigure(1, label=_('Privacy Policy')) # Help menu item - self.help_menu.entryconfigure(2, label=_('Release Notes')) # Help menu item + self.menubar.entryconfigure(1, label=_('File')) # Menu title + self.menubar.entryconfigure(2, label=_('Edit')) # Menu title + self.menubar.entryconfigure(3, label=_('View')) # Menu title on OSX + self.menubar.entryconfigure(4, label=_('Window')) # Menu title on OSX + self.menubar.entryconfigure(5, label=_('Help')) # Menu title + self.system_menu.entryconfigure(0, label=_("About {APP}").format(APP=applongname)) # App menu entry on OSX + self.system_menu.entryconfigure(1, label=_("Check for Updates...")) # Menu item + self.file_menu.entryconfigure(0, label=_('Save Raw Data...')) # Menu item + self.view_menu.entryconfigure(0, label=_('Status')) # Menu item + self.help_menu.entryconfigure(1, label=_('Privacy Policy')) # Help menu item + self.help_menu.entryconfigure(2, label=_('Release Notes')) # Help menu item else: - self.menubar.entryconfigure(1, label=_('File')) # Menu title - self.menubar.entryconfigure(2, label=_('Edit')) # Menu title - self.menubar.entryconfigure(3, label=_('Help')) # Menu title - self.theme_file_menu['text'] = _('File') # Menu title - self.theme_edit_menu['text'] = _('Edit') # Menu title - self.theme_help_menu['text'] = _('Help') # Menu title + self.menubar.entryconfigure(1, label=_('File')) # Menu title + self.menubar.entryconfigure(2, label=_('Edit')) # Menu title + self.menubar.entryconfigure(3, label=_('Help')) # Menu title + self.theme_file_menu['text'] = _('File') # Menu title + self.theme_edit_menu['text'] = _('Edit') # Menu title + self.theme_help_menu['text'] = _('Help') # Menu title - ## File menu - self.file_menu.entryconfigure(0, label=_('Status')) # Menu item - self.file_menu.entryconfigure(1, label=_('Save Raw Data...')) # Menu item - self.file_menu.entryconfigure(2, label=_('Settings')) # Item in the File menu on Windows - self.file_menu.entryconfigure(4, label=_('Exit')) # Item in the File menu on Windows + # File menu + self.file_menu.entryconfigure(0, label=_('Status')) # Menu item + self.file_menu.entryconfigure(1, label=_('Save Raw Data...')) # Menu item + self.file_menu.entryconfigure(2, label=_('Settings')) # Item in the File menu on Windows + self.file_menu.entryconfigure(4, label=_('Exit')) # Item in the File menu on Windows - ## Help menu - self.help_menu.entryconfigure(0, label=_('Documentation')) # Help menu item - self.help_menu.entryconfigure(1, label=_('Privacy Policy')) # Help menu item - self.help_menu.entryconfigure(2, label=_('Release Notes')) # Help menu item - self.help_menu.entryconfigure(3, label=_('Check for Updates...')) # Menu item - self.help_menu.entryconfigure(4, label=_("About {APP}").format(APP=applongname)) # App menu entry + # Help menu + self.help_menu.entryconfigure(0, label=_('Documentation')) # Help menu item + self.help_menu.entryconfigure(1, label=_('Privacy Policy')) # Help menu item + self.help_menu.entryconfigure(2, label=_('Release Notes')) # Help menu item + self.help_menu.entryconfigure(3, label=_('Check for Updates...')) # Menu item + self.help_menu.entryconfigure(4, label=_("About {APP}").format(APP=applongname)) # App menu entry - ## Edit menu - self.edit_menu.entryconfigure(0, label=_('Copy')) # As in Copy and Paste + # Edit menu + self.edit_menu.entryconfigure(0, label=_('Copy')) # As in Copy and Paste def login(self): if not self.status['text']: self.status['text'] = _('Logging in...') self.button['state'] = self.theme_button['state'] = tk.DISABLED if platform == 'darwin': - self.view_menu.entryconfigure(0, state=tk.DISABLED) # Status - self.file_menu.entryconfigure(0, state=tk.DISABLED) # Save Raw Data + self.view_menu.entryconfigure(0, state=tk.DISABLED) # Status + self.file_menu.entryconfigure(0, state=tk.DISABLED) # Save Raw Data else: - self.file_menu.entryconfigure(0, state=tk.DISABLED) # Status - self.file_menu.entryconfigure(1, state=tk.DISABLED) # Save Raw Data + self.file_menu.entryconfigure(0, state=tk.DISABLED) # Status + self.file_menu.entryconfigure(1, state=tk.DISABLED) # Save Raw Data self.w.update_idletasks() try: if companion.session.login(monitor.cmdr, monitor.is_beta): - self.status['text'] = _('Authentication successful') # Successfully authenticated with the Frontier website + # Successfully authenticated with the Frontier website + self.status['text'] = _('Authentication successful') if platform == 'darwin': - self.view_menu.entryconfigure(0, state=tk.NORMAL) # Status - self.file_menu.entryconfigure(0, state=tk.NORMAL) # Save Raw Data + self.view_menu.entryconfigure(0, state=tk.NORMAL) # Status + self.file_menu.entryconfigure(0, state=tk.NORMAL) # Save Raw Data else: - self.file_menu.entryconfigure(0, state=tk.NORMAL) # Status - self.file_menu.entryconfigure(1, state=tk.NORMAL) # Save Raw Data + self.file_menu.entryconfigure(0, state=tk.NORMAL) # Status + self.file_menu.entryconfigure(1, state=tk.NORMAL) # Save Raw Data except (companion.CredentialsError, companion.ServerError, companion.ServerLagging) as e: self.status['text'] = str(e) except Exception as e: @@ -418,7 +433,7 @@ class AppWindow(object): play_bad = False if not monitor.cmdr or not monitor.mode or monitor.state['Captain'] or not monitor.system: - return # In CQC or on crew - do nothing + return # In CQC or on crew - do nothing if companion.session.state == companion.Session.STATE_AUTH: # Attempt another Auth @@ -426,10 +441,10 @@ class AppWindow(object): return if not retrying: - if time() < self.holdofftime: # Was invoked by key while in cooldown + if time() < self.holdofftime: # Was invoked by key while in cooldown self.status['text'] = '' if play_sound and (self.holdofftime-time()) < companion.holdoff*0.75: - hotkeymgr.play_bad() # Don't play sound in first few seconds to prevent repeats + hotkeymgr.play_bad() # Don't play sound in first few seconds to prevent repeats return elif play_sound: hotkeymgr.play_good() @@ -444,31 +459,41 @@ class AppWindow(object): # Validation if not data.get('commander', {}).get('name'): - self.status['text'] = _("Who are you?!") # Shouldn't happen - elif (not data.get('lastSystem', {}).get('name') or - (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked - self.status['text'] = _("Where are you?!") # Shouldn't happen + self.status['text'] = _("Who are you?!") # Shouldn't happen + elif (not data.get('lastSystem', {}).get('name') + or (data['commander'].get('docked') + and not data.get('lastStarport', {}).get('name'))): # Only care if docked + self.status['text'] = _("Where are you?!") # Shouldn't happen elif not data.get('ship', {}).get('name') or not data.get('ship', {}).get('modules'): - self.status['text'] = _("What are you flying?!") # Shouldn't happen + self.status['text'] = _("What are you flying?!") # Shouldn't happen elif monitor.cmdr and data['commander']['name'] != monitor.cmdr: - raise companion.CmdrError() # Companion API return doesn't match Journal - elif ((auto_update and not data['commander'].get('docked')) or - (data['lastSystem']['name'] != monitor.system) or - ((data['commander']['docked'] and data['lastStarport']['name'] or None) != monitor.station) or - (data['ship']['id'] != monitor.state['ShipID']) or - (data['ship']['name'].lower() != monitor.state['ShipType'])): + # Companion API return doesn't match Journal + raise companion.CmdrError() + elif ((auto_update and not data['commander'].get('docked')) + or (data['lastSystem']['name'] != monitor.system) + or ((data['commander']['docked'] + and data['lastStarport']['name'] or None) != monitor.station) + or (data['ship']['id'] != monitor.state['ShipID']) + or (data['ship']['name'].lower() != monitor.state['ShipType'])): raise companion.ServerLagging() else: - if __debug__: # Recording + if __debug__: # Recording if isdir('dump'): - with open('dump/%s%s.%s.json' % (data['lastSystem']['name'], data['commander'].get('docked') and '.'+data['lastStarport']['name'] or '', strftime('%Y-%m-%dT%H.%M.%S', localtime())), 'wb') as h: - h.write(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')).encode('utf-8')) + with open('dump/{system}{station}.{timestamp}.json'.format( + system=data['lastSystem']['name'], + station=data['commander'].get('docked') and '.'+data['lastStarport']['name'] or '', + timestamp=strftime('%Y-%m-%dT%H.%M.%S', localtime())), 'wb') as h: + h.write(json.dumps(data, + ensure_ascii=False, + indent=2, + sort_keys=True, + separators=(',', ': ')).encode('utf-8')) - if not monitor.state['ShipType']: # Started game in SRV or fighter + if not monitor.state['ShipType']: # Started game in SRV or fighter self.ship['text'] = companion.ship_map.get(data['ship']['name'].lower(), data['ship']['name']) - monitor.state['ShipID'] = data['ship']['id'] + monitor.state['ShipID'] = data['ship']['id'] monitor.state['ShipType'] = data['ship']['name'].lower() if data['commander'].get('credits') is not None: @@ -485,16 +510,19 @@ class AppWindow(object): if config.getint('output') & (config.OUT_STATION_ANY): if not data['commander'].get('docked'): if not self.status['text']: - # Signal as error because the user might actually be docked but the server hosting the Companion API hasn't caught up + # Signal as error because the user might actually be docked + # but the server hosting the Companion API hasn't caught up self.status['text'] = _("You're not docked at a station!") play_bad = True - elif (config.getint('output') & config.OUT_MKT_EDDN) and not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): # Ignore possibly missing shipyard info + # Ignore possibly missing shipyard info + elif (config.getint('output') & config.OUT_MKT_EDDN)\ + and not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): if not self.status['text']: self.status['text'] = _("Station doesn't have anything!") elif not data['lastStarport'].get('commodities'): if not self.status['text']: self.status['text'] = _("Station doesn't have a market!") - elif config.getint('output') & (config.OUT_MKT_CSV|config.OUT_MKT_TD): + elif config.getint('output') & (config.OUT_MKT_CSV | config.OUT_MKT_TD): # Fixup anomalies in the commodity data fixed = companion.fixup(data) if config.getint('output') & config.OUT_MKT_CSV: @@ -511,10 +539,10 @@ class AppWindow(object): play_bad = True else: # Retry once if Companion server is unresponsive - self.w.after(int(SERVER_RETRY * 1000), lambda:self.getandsend(event, True)) - return # early exit to avoid starting cooldown count + self.w.after(int(SERVER_RETRY * 1000), lambda: self.getandsend(event, True)) + return # early exit to avoid starting cooldown count - except companion.CmdrError as e: # Companion API return doesn't match Journal + except companion.CmdrError as e: # Companion API return doesn't match Journal self.status['text'] = str(e) play_bad = True companion.session.invalidate() @@ -552,8 +580,8 @@ class AppWindow(object): data.get('lastStarport', {}).get('ships', {}).get('shipyard_list')): self.eddn.export_shipyard(data, monitor.is_beta) elif tries > 1: # bogus data - retry - self.w.after(int(SERVER_RETRY * 1000), lambda:self.retry_for_shipyard(tries-1)) - except: + self.w.after(int(SERVER_RETRY * 1000), lambda: self.retry_for_shipyard(tries-1)) + except Exception: pass # Handle event(s) from the journal @@ -564,9 +592,9 @@ class AppWindow(object): return { None: '', 'Idle': '', - 'FighterCon': _('Fighter'), # Multicrew role - 'FireCon': _('Gunner'), # Multicrew role - 'FlightCon': _('Helm'), # Multicrew role + 'FighterCon': _('Fighter'), # Multicrew role + 'FireCon': _('Gunner'), # Multicrew role + 'FlightCon': _('Helm'), # Multicrew role }.get(role, role) while True: @@ -577,26 +605,43 @@ class AppWindow(object): # Update main window self.cooldown() if monitor.cmdr and monitor.state['Captain']: - self.cmdr['text'] = '%s / %s' % (monitor.cmdr, monitor.state['Captain']) - self.ship_label['text'] = _('Role') + ':' # Multicrew role label in main window - self.ship.configure(state = tk.NORMAL, text = crewroletext(monitor.state['Role']), url = None) + self.cmdr['text'] = f'{monitor.cmdr} / {monitor.state["Captain"]}' + self.ship_label['text'] = _('Role') + ':' # Multicrew role label in main window + self.ship.configure(state=tk.NORMAL, text=crewroletext(monitor.state['Role']), url=None) elif monitor.cmdr: if monitor.group: - self.cmdr['text'] = '%s / %s' % (monitor.cmdr, monitor.group) + self.cmdr['text'] = f'{monitor.cmdr} / {monitor.group}' else: self.cmdr['text'] = monitor.cmdr - self.ship_label['text'] = _('Ship') + ':' # Main window - self.ship.configure(text = monitor.state['ShipName'] or companion.ship_map.get(monitor.state['ShipType'], monitor.state['ShipType']) or '', - url = self.shipyard_url) + self.ship_label['text'] = _('Ship') + ':' # Main window + self.ship.configure( + text=monitor.state['ShipName'] + or companion.ship_map.get(monitor.state['ShipType'], monitor.state['ShipType']) + or '', + url=self.shipyard_url) else: self.cmdr['text'] = '' - self.ship_label['text'] = _('Ship') + ':' # Main window + self.ship_label['text'] = _('Ship') + ':' # Main window self.ship['text'] = '' - self.edit_menu.entryconfigure(0, state=monitor.system and tk.NORMAL or tk.DISABLED) # Copy + self.edit_menu.entryconfigure(0, state=monitor.system and tk.NORMAL or tk.DISABLED) # Copy - if entry['event'] in ['Undocked', 'StartJump', 'SetUserShipName', 'ShipyardBuy', 'ShipyardSell', 'ShipyardSwap', 'ModuleBuy', 'ModuleSell', 'MaterialCollected', 'MaterialDiscarded', 'ScientificResearch', 'EngineerCraft', 'Synthesis', 'JoinACrew']: - self.status['text'] = '' # Periodically clear any old error + if entry['event'] in ( + 'Undocked', + 'StartJump', + 'SetUserShipName', + 'ShipyardBuy', + 'ShipyardSell', + 'ShipyardSwap', + 'ModuleBuy', + 'ModuleSell', + 'MaterialCollected', + 'MaterialDiscarded', + 'ScientificResearch', + 'EngineerCraft', + 'Synthesis', + 'JoinACrew'): + self.status['text'] = '' # Periodically clear any old error self.w.update_idletasks() # Companion login @@ -606,7 +651,7 @@ class AppWindow(object): self.login() if not entry['event'] or not monitor.mode: - return # Startup or in CQC + return # Startup or in CQC if entry['event'] in ['StartUp', 'LoadGame'] and monitor.started: # Disable WinSparkle automatic update checks, IFF configured to do so when in-game @@ -618,8 +663,7 @@ class AppWindow(object): logger.info("Can't start Status monitoring") # Export loadout - if entry['event'] == 'Loadout'\ - and not monitor.state['Captain']\ + if entry['event'] == 'Loadout' and not monitor.state['Captain']\ and config.getint('output') & config.OUT_SHIP: monitor.export_ship() @@ -636,7 +680,11 @@ class AppWindow(object): hotkeymgr.play_bad() # Auto-Update after docking, but not if auth callback is pending - if entry['event'] in ['StartUp', 'Location', 'Docked'] and monitor.station and not config.getint('output') & config.OUT_MKT_MANUAL and config.getint('output') & config.OUT_STATION_ANY and companion.session.state != companion.Session.STATE_AUTH: + if entry['event'] in ('StartUp', 'Location', 'Docked')\ + and monitor.station\ + and not config.getint('output') & config.OUT_MKT_MANUAL\ + and config.getint('output') & config.OUT_STATION_ANY\ + and companion.session.state != companion.Session.STATE_AUTH: self.w.after(int(SERVER_RETRY * 1000), self.getandsend) if entry['event'] == 'ShutDown': @@ -698,7 +746,7 @@ class AppWindow(object): provider_name=html.escape(str(provider)), ship_name=html.escape(str(shipname)) ), file=f) - + return f'file://localhost/{file_name}' def system_url(self, system): @@ -709,10 +757,12 @@ class AppWindow(object): def cooldown(self): if time() < self.holdofftime: - self.button['text'] = self.theme_button['text'] = _('cooldown {SS}s').format(SS = int(self.holdofftime - time())) # Update button in main window + # Update button in main window + self.button['text'] = self.theme_button['text'] \ + = _('cooldown {SS}s').format(SS=int(self.holdofftime - time())) self.w.after(1000, self.cooldown) else: - self.button['text'] = self.theme_button['text'] = _('Update') # Update button in main window + self.button['text'] = self.theme_button['text'] = _('Update') # Update button in main window self.button['state'] = self.theme_button['state'] = (monitor.cmdr and monitor.mode and not monitor.state['Captain'] and @@ -726,7 +776,7 @@ class AppWindow(object): def copy(self, event=None): if monitor.system: self.w.clipboard_clear() - self.w.clipboard_append(monitor.station and '%s,%s' % (monitor.system, monitor.station) or monitor.system) + self.w.clipboard_append(monitor.station and f'{monitor.system},{monitor.station}' or monitor.system) def help_general(self, event=None): webbrowser.open('https://github.com/EDCD/EDMarketConnector/wiki') @@ -754,11 +804,11 @@ class AppWindow(object): self.transient(parent) # position over parent - if platform!='darwin' or parent.winfo_rooty()>0: # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 - self.geometry("+%d+%d" % (parent.winfo_rootx(), parent.winfo_rooty())) + if platform != 'darwin' or parent.winfo_rooty() > 0: # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + self.geometry(f'+{parent.winfo_rootx():d}+{parent.winfo_rooty():d}') # remove decoration - if platform=='win32': + if platform == 'win32': self.attributes('-toolwindow', tk.TRUE) self.resizable(tk.FALSE, tk.FALSE) @@ -862,7 +912,7 @@ class AppWindow(object): # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 if platform != 'darwin' or self.w.winfo_rooty() > 0: config.set('geometry', '+{1}+{2}'.format(*self.w.geometry().split('+'))) - self.w.withdraw() # Following items can take a few seconds, so hide the main window while they happen + self.w.withdraw() # Following items can take a few seconds, so hide the main window while they happen protocolhandler.close() hotkeymgr.unregister() dashboard.close() @@ -888,7 +938,7 @@ class AppWindow(object): self.w.iconify() self.w.update_idletasks() # Size and windows styles get recalculated here self.w.wait_visibility() # Need main window to be re-created before returning - theme.active = None # So theme will be re-applied on map + theme.active = None # So theme will be re-applied on map def onmap(self, event=None): if event.widget == self.w: @@ -987,7 +1037,7 @@ if __name__ == "__main__": #abinit = A.B() - Translations.install(config.get('language') or None) # Can generate errors so wait til log set up + Translations.install(config.get('language') or None) # Can generate errors so wait til log set up root = tk.Tk(className=appname.lower()) app = AppWindow(root) From 9face638fe3cbf9adc888a5f5a96f36221552c34 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 13:00:59 +0100 Subject: [PATCH 088/258] Minor cleanups all done. Only remaining are: * TAE001 too few type annotations * Multiple "Cognitive complexity is too high" / "is too complex" --- EDMarketConnector.py | 87 ++++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 9c7b4a14..0d23f6dd 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -422,7 +422,7 @@ class AppWindow(object): except (companion.CredentialsError, companion.ServerError, companion.ServerLagging) as e: self.status['text'] = str(e) except Exception as e: - logger.debug(f'Frontier CAPI Auth', exc_info=e) + logger.debug('Frontier CAPI Auth', exc_info=e) self.status['text'] = str(e) self.cooldown() @@ -810,16 +810,12 @@ class AppWindow(object): # remove decoration if platform == 'win32': self.attributes('-toolwindow', tk.TRUE) - + self.resizable(tk.FALSE, tk.FALSE) frame = ttk.Frame(self) frame.grid(sticky=tk.NSEW) - PADX = 10 - BUTTONX = 12 # indent Checkbuttons and Radiobuttons - PADY = 2 # close spacing - row = 1 ############################################################ # applongname @@ -835,8 +831,8 @@ class AppWindow(object): self.appversion_label = tk.Label(frame, text=appversion) self.appversion_label.grid(row=row, column=0, sticky=tk.E) self.appversion = HyperlinkLabel(frame, compound=tk.RIGHT, text=_('Release Notes'), - url='https://github.com/EDCD/EDMarketConnector/releases/tag/Release/{VERSION}'.format( - VERSION=appversion_nobuild), + url='https://github.com/EDCD/EDMarketConnector/releases/tag/Release/' + f'{appversion_nobuild}', underline=True) self.appversion.grid(row=row, column=2, sticky=tk.W) row += 1 @@ -928,7 +924,9 @@ class AppWindow(object): def drag_continue(self, event): if self.drag_offset: - self.w.geometry('+%d+%d' % (event.x_root - self.drag_offset[0], event.y_root - self.drag_offset[1])) + offset_x = event.x_root - self.drag_offset[0] + offset_y = event.y_root - self.drag_offset[1] + self.w.geometry(f'+{offset_x:d}+{offset_y:d}') def drag_end(self, event): self.drag_offset = None @@ -962,28 +960,28 @@ def enforce_single_instance() -> None: if platform == 'win32': import ctypes from ctypes.wintypes import HWND, LPWSTR, LPCWSTR, INT, BOOL, LPARAM - EnumWindows = ctypes.windll.user32.EnumWindows - GetClassName = ctypes.windll.user32.GetClassNameW - GetClassName.argtypes = [HWND, LPWSTR, ctypes.c_int] - GetWindowText = ctypes.windll.user32.GetWindowTextW - GetWindowText.argtypes = [HWND, LPWSTR, ctypes.c_int] - GetWindowTextLength = ctypes.windll.user32.GetWindowTextLengthW - GetProcessHandleFromHwnd = ctypes.windll.oleacc.GetProcessHandleFromHwnd + EnumWindows = ctypes.windll.user32.EnumWindows # noqa: N806 + GetClassName = ctypes.windll.user32.GetClassNameW # noqa: N806 + GetClassName.argtypes = [HWND, LPWSTR, ctypes.c_int] # noqa: N806 + GetWindowText = ctypes.windll.user32.GetWindowTextW # noqa: N806 + GetWindowText.argtypes = [HWND, LPWSTR, ctypes.c_int] # noqa: N806 + GetWindowTextLength = ctypes.windll.user32.GetWindowTextLengthW # noqa: N806 + GetProcessHandleFromHwnd = ctypes.windll.oleacc.GetProcessHandleFromHwnd # noqa: N806 - SW_RESTORE = 9 - SetForegroundWindow = ctypes.windll.user32.SetForegroundWindow - ShowWindow = ctypes.windll.user32.ShowWindow - ShowWindowAsync = ctypes.windll.user32.ShowWindowAsync + SW_RESTORE = 9 # noqa: N806 + SetForegroundWindow = ctypes.windll.user32.SetForegroundWindow # noqa: N806 + ShowWindow = ctypes.windll.user32.ShowWindow # noqa: N806 + ShowWindowAsync = ctypes.windll.user32.ShowWindowAsync # noqa: N806 - COINIT_MULTITHREADED = 0 - COINIT_APARTMENTTHREADED = 0x2 - COINIT_DISABLE_OLE1DDE = 0x4 - CoInitializeEx = ctypes.windll.ole32.CoInitializeEx + COINIT_MULTITHREADED = 0 # noqa: N806,F841 + COINIT_APARTMENTTHREADED = 0x2 # noqa: N806 + COINIT_DISABLE_OLE1DDE = 0x4 # noqa: N806 + CoInitializeEx = ctypes.windll.ole32.CoInitializeEx # noqa: N806 - ShellExecute = ctypes.windll.shell32.ShellExecuteW - ShellExecute.argtypes = [HWND, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR, INT] + ShellExecute = ctypes.windll.shell32.ShellExecuteW # noqa: N806 + ShellExecute.argtypes = [HWND, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR, INT] - def WindowTitle(h): + def window_title(h): if h: text_length = GetWindowTextLength(h) + 1 buf = ctypes.create_unicode_buffer(text_length) @@ -992,22 +990,23 @@ def enforce_single_instance() -> None: return None @ctypes.WINFUNCTYPE(BOOL, HWND, LPARAM) - def enumwindowsproc(hWnd, lParam): + def enumwindowsproc(window_handle, l_param): # class name limited to 256 - https://msdn.microsoft.com/en-us/library/windows/desktop/ms633576 cls = ctypes.create_unicode_buffer(257) - if GetClassName(hWnd, cls, 257)\ + if GetClassName(window_handle, cls, 257)\ and cls.value == 'TkTopLevel'\ - and WindowTitle(hWnd) == applongname\ - and GetProcessHandleFromHwnd(hWnd): + and window_title(window_handle) == applongname\ + and GetProcessHandleFromHwnd(window_handle): # If GetProcessHandleFromHwnd succeeds then the app is already running as this user if len(sys.argv) > 1 and sys.argv[1].startswith(protocolhandler.redirect): # Browser invoked us directly with auth response. Forward the response to the other app instance. CoInitializeEx(0, COINIT_APARTMENTTHREADED | COINIT_DISABLE_OLE1DDE) - ShowWindow(hWnd, SW_RESTORE) # Wait for it to be responsive to avoid ShellExecute recursing + # Wait for it to be responsive to avoid ShellExecute recursing + ShowWindow(window_handle, SW_RESTORE) ShellExecute(0, None, sys.argv[1], None, None, SW_RESTORE) else: - ShowWindowAsync(hWnd, SW_RESTORE) - SetForegroundWindow(hWnd) + ShowWindowAsync(window_handle, SW_RESTORE) + SetForegroundWindow(window_handle) sys.exit(0) return True @@ -1035,7 +1034,7 @@ if __name__ == "__main__": def __init__(self): logger.debug('A call from A.B.__init__') - #abinit = A.B() + # abinit = A.B() Translations.install(config.get('language') or None) # Can generate errors so wait til log set up @@ -1045,9 +1044,25 @@ if __name__ == "__main__": def messagebox_not_py3(): plugins_not_py3_last = config.getint('plugins_not_py3_last') or 0 if (plugins_not_py3_last + 86400) < int(time()) and len(plug.PLUGINS_not_py3): + # Yes, this is horribly hacky so as to be sure we match the key + # that we told Translators to use. + popup_text = "One or more of your enabled plugins do not yet have support for Python 3.x. Please see the "\ + "list on the '{PLUGINS}' tab of '{FILE}' > '{SETTINGS}'. You should check if there is an "\ + "updated version available, else alert the developer that they need to update the code for "\ + "Python 3.x.\r\n\r\nYou can disable a plugin by renaming its folder to have '{DISABLED}' on "\ + "the end of the name." + popup_text = popup_text.replace('\n', '\\n') + popup_text = popup_text.replace('\r', '\\r') + # Now the string should match, so try translation + popup_text = _(popup_text) + # And substitute in the other words. + popup_text.format(PLUGINS=_('Plugins'), FILE=_('File'), SETTINGS=_('Settings'), DISABLED='.disabled') + # And now we do need these to be actual \r\n + popup_text = popup_text.replace('\\n', '\n') + popup_text = popup_text.replace('\\r', '\r') tk.messagebox.showinfo( _('EDMC: Plugins Without Python 3.x Support'), - _("One or more of your enabled plugins do not yet have support for Python 3.x. Please see the list on the '{PLUGINS}' tab of '{FILE}' > '{SETTINGS}'. You should check if there is an updated version available, else alert the developer that they need to update the code for Python 3.x.\r\n\r\nYou can disable a plugin by renaming its folder to have '{DISABLED}' on the end of the name.".format(PLUGINS=_('Plugins'), FILE=_('File'), SETTINGS=_('Settings'), DISABLED='.disabled')) + popup_text ) config.set('plugins_not_py3_last', int(time())) From 72498b3bbcc434f58a22f43fa7095cf160f9c56e Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 22 Jul 2020 16:26:06 +0200 Subject: [PATCH 089/258] Switched to using a timer for inara updates This adds a new thread that will run in a loop sending inara events once ever 30 seconds. The old method of sending once for "special" events has been removed. --- plugins/inara.py | 56 ++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index bfff9e5d..7cf0919e 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -4,6 +4,7 @@ from collections import OrderedDict import json +from typing import Any, Union import requests import sys import time @@ -26,7 +27,7 @@ FAKE = ['CQC', 'Training', 'Destination'] # Fake systems that shouldn't be sent CREDIT_RATIO = 1.05 # Update credits if they change by 5% over the course of a session -this = sys.modules[__name__] # For holding module globals +this: Any = sys.modules[__name__] # For holding module globals this.session = requests.Session() this.queue = Queue() # Items to be sent to Inara by worker thread this.lastlocation = None # eventData from the last Commander's Flight Log event @@ -50,7 +51,8 @@ this.fleet = None this.shipswap = False # just swapped ship this.last_update_time = time.time() # last time we updated (set to now because we're going to update quickly) -FLOOD_LIMIT_SECONDS = 45 # minimum time between sending non-major cargo triggered messages +FLOOD_LIMIT_SECONDS = 30 # minimum time between sending events +this.timer_run = True # Main window clicks @@ -85,6 +87,10 @@ def plugin_start3(plugin_dir): this.thread = Thread(target = worker, name = 'Inara worker') this.thread.daemon = True this.thread.start() + + this.timer_thread = Thread(target=call_timer, name='Inara timer') + this.timer_thread.daemon = True + this.timer_thread.start() return 'Inara' def plugin_app(parent): @@ -95,12 +101,14 @@ def plugin_app(parent): def plugin_stop(): # Send any unsent events - call() + call(force=True) # Signal thread to close and wait for it this.queue.put(None) this.thread.join() this.thread = None + this.timer_run = False + def plugin_prefs(parent, cmdr, is_beta): PADX = 10 @@ -188,7 +196,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # Send any unsent events when switching accounts if cmdr and cmdr != this.cmdr: - call() + call(force=True) this.cmdr = cmdr this.FID = state['FID'] @@ -245,8 +253,6 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(cmdr): try: - old_events = len(this.events) # Will only send existing events if we add a new event below - # Dump starting state to Inara if (this.newuser or @@ -305,6 +311,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.loadout = make_loadout(state) add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) + + call() # Call here just to be sure that if we can send, we do, otherwise it'll get it in the next tick # Promotions @@ -437,23 +445,17 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # if our cargo differers from last we checked, we're at a station, # and our flood limit isnt covered, queue an update should_poll = this.cargo != cargo and time.time() - this.last_update_time > FLOOD_LIMIT_SECONDS - - # Send event(s) to Inara - if entry['event'] == 'ShutDown' or len(this.events) > old_events or should_poll: - # Send cargo and materials if changed - if this.cargo != cargo: - add_event('setCommanderInventoryCargo', entry['timestamp'], cargo) - this.cargo = cargo - materials = [] - for category in ['Raw', 'Manufactured', 'Encoded']: - materials.extend([ OrderedDict([('itemName', k), ('itemCount', state[category][k])]) for k in sorted(state[category]) ]) - if this.materials != materials: - add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) - this.materials = materials - - # Queue a call to Inara - call() + # Send cargo and materials if changed + if this.cargo != cargo: + add_event('setCommanderInventoryCargo', entry['timestamp'], cargo) + this.cargo = cargo + materials = [] + for category in ['Raw', 'Manufactured', 'Encoded']: + materials.extend([ OrderedDict([('itemName', k), ('itemCount', state[category][k])]) for k in sorted(state[category]) ]) + if this.materials != materials: + add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) + this.materials = materials except Exception as e: logger.debug('Adding events', exc_info=e) @@ -856,12 +858,20 @@ def add_event(name, timestamp, data): ('eventData', data), ])) +def call_timer(wait=FLOOD_LIMIT_SECONDS): + while this.timer_run: + time.sleep(wait) + if this.timer_run: # check again in here just in case we're closing and the stars align + call() # Queue a call to Inara, handled in Worker thread -def call(callback=None): +def call(callback=None, force=False): if not this.events: return + if (time.time() - this.last_update_time) <= FLOOD_LIMIT_SECONDS and not force: + return + this.last_update_time = time.time() data = OrderedDict([ From 34760683e58fd33e15a319060b30ed5666372fd3 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 23 Jul 2020 16:47:29 +0200 Subject: [PATCH 090/258] added some debug logging --- plugins/inara.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index 7cf0919e..4242dcdf 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -861,6 +861,7 @@ def add_event(name, timestamp, data): def call_timer(wait=FLOOD_LIMIT_SECONDS): while this.timer_run: time.sleep(wait) + print("INARA: TICK") if this.timer_run: # check again in here just in case we're closing and the stars align call() @@ -873,7 +874,7 @@ def call(callback=None, force=False): return this.last_update_time = time.time() - + print(f"INARA: {time.asctime()}sending {len(this.events)} entries to inara (call)") data = OrderedDict([ ('header', OrderedDict([ ('appName', applongname), @@ -896,6 +897,8 @@ def worker(): else: (url, data, callback) = item + print(f"INARA: {time.asctime()}sending {len(data['events'])} entries to inara (worker)") + retrying = 0 while retrying < 3: try: From 340f8928b353aca6d381a9d60a419aa112100960 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 23 Jul 2020 18:38:55 +0200 Subject: [PATCH 091/258] switched to using config for last update time --- plugins/inara.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 4242dcdf..a8b3ee8d 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -50,7 +50,9 @@ this.loadout = None this.fleet = None this.shipswap = False # just swapped ship -this.last_update_time = time.time() # last time we updated (set to now because we're going to update quickly) +# last time we updated, if unset in config this is 0, which means an instant update +LAST_UPDATE_CONF_KEY = 'inara_last_update' +# this.last_update_time = config.getint(LAST_UPDATE_CONF_KEY) FLOOD_LIMIT_SECONDS = 30 # minimum time between sending events this.timer_run = True @@ -442,10 +444,6 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): cargo = [OrderedDict([('itemName', k), ('itemCount', state['Cargo'][k])]) for k in sorted(state['Cargo'])] - # if our cargo differers from last we checked, we're at a station, - # and our flood limit isnt covered, queue an update - should_poll = this.cargo != cargo and time.time() - this.last_update_time > FLOOD_LIMIT_SECONDS - # Send cargo and materials if changed if this.cargo != cargo: add_event('setCommanderInventoryCargo', entry['timestamp'], cargo) @@ -861,7 +859,7 @@ def add_event(name, timestamp, data): def call_timer(wait=FLOOD_LIMIT_SECONDS): while this.timer_run: time.sleep(wait) - print("INARA: TICK") + print(f"INARA: {time.asctime()} TICK") if this.timer_run: # check again in here just in case we're closing and the stars align call() @@ -870,11 +868,11 @@ def call(callback=None, force=False): if not this.events: return - if (time.time() - this.last_update_time) <= FLOOD_LIMIT_SECONDS and not force: + if (time.time() - config.getint(LAST_UPDATE_CONF_KEY)) <= FLOOD_LIMIT_SECONDS and not force: return - this.last_update_time = time.time() - print(f"INARA: {time.asctime()}sending {len(this.events)} entries to inara (call)") + config.set(LAST_UPDATE_CONF_KEY, int(time.time())) + print(f"INARA: {time.asctime()} sending {len(this.events)} entries to inara (call)") data = OrderedDict([ ('header', OrderedDict([ ('appName', applongname), @@ -897,7 +895,7 @@ def worker(): else: (url, data, callback) = item - print(f"INARA: {time.asctime()}sending {len(data['events'])} entries to inara (worker)") + print(f"INARA: {time.asctime()} sending {len(data['events'])} entries to inara (worker)") retrying = 0 while retrying < 3: From bb817eee3b83348e21dfbf89f88332471bf0ae7f Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 24 Jul 2020 11:43:40 +0200 Subject: [PATCH 092/258] removed outdated comment --- plugins/inara.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index a8b3ee8d..43fd5880 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -459,10 +459,6 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): logger.debug('Adding events', exc_info=e) return str(e) - # - # Events that don't need to be sent immediately but will be sent on the next mandatory event - # - # Send credits and stats to Inara on startup only - otherwise may be out of date if entry['event'] == 'LoadGame': add_event('setCommanderCredits', entry['timestamp'], From d4299c224f2a783d3380b9ca244c05b1823440e6 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 27 Jul 2020 15:12:32 +0200 Subject: [PATCH 093/258] dont force a send on exit just try and send if we can --- plugins/inara.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index 43fd5880..aa7e7b90 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -103,7 +103,8 @@ def plugin_app(parent): def plugin_stop(): # Send any unsent events - call(force=True) + call() + time.sleep(0.1) # Sleep for 100ms to allow call to go out, and to force a context switch to our other threads # Signal thread to close and wait for it this.queue.put(None) this.thread.join() From ab58aa6355fafe75e188864ef0af51f09843d992 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 19:41:51 +0200 Subject: [PATCH 094/258] autoformatted file --- plugins/eddb.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 17e72bf4..09f84710 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -29,9 +29,9 @@ import requests from config import config -STATION_UNDOCKED: str = u'×' # "Station" name to display when not docked = U+00D7 +STATION_UNDOCKED: str = u'×' # "Station" name to display when not docked = U+00D7 -this = sys.modules[__name__] # For holding module globals +this = sys.modules[__name__] # For holding module globals # Main window clicks this.system_link = None @@ -51,23 +51,27 @@ def system_url(system_name: str) -> str: else: return '' + def station_url(system_name: str, station_name: str) -> str: if this.station_marketid: return requests.utils.requote_uri(f'https://eddb.io/station/market-id/{this.station_marketid}') else: return system_url('') + def plugin_start3(plugin_dir): return 'eddb' + def plugin_app(parent): - this.system_link = parent.children['system'] # system label in main window + this.system_link = parent.children['system'] # system label in main window this.system = None this.system_address = None this.station = None this.station_marketid = None # Frontier MarketID this.station_link = parent.children['station'] # station label in main window - this.station_link.configure(popup_copy = lambda x: x != STATION_UNDOCKED) + this.station_link.configure(popup_copy=lambda x: x != STATION_UNDOCKED) + def prefs_changed(cmdr, is_beta): # Override standard URL functions @@ -103,7 +107,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # But only actually change the URL if we are current station provider. if config.get('station_provider') == 'eddb': - this.station_link['text'] = this.station or (this.system_population and this.system_population > 0 and STATION_UNDOCKED or '') + this.station_link['text'] = this.station or ( + this.system_population and this.system_population > 0 and STATION_UNDOCKED or '') this.station_link['url'] = station_url(this.system, this.station) # Override standard URL function this.station_link.update_idletasks() @@ -131,4 +136,3 @@ def cmdr_data(data, is_beta): this.station_link['url'] = station_url(this.system, this.station) this.station_link.update_idletasks() - From 4eed4404c62b25566fbcbc507c154f0c9e1e70a2 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 19:42:34 +0200 Subject: [PATCH 095/258] Added Any type hint to `this` Resolves most type warnings --- plugins/eddb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 09f84710..7eb134d6 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -24,6 +24,7 @@ import sys +from typing import Any import requests from config import config @@ -31,7 +32,7 @@ from config import config STATION_UNDOCKED: str = u'×' # "Station" name to display when not docked = U+00D7 -this = sys.modules[__name__] # For holding module globals +this: Any = sys.modules[__name__] # For holding module globals # Main window clicks this.system_link = None From 37181264c9a483c3dd9cb77e6465b4fc988bc6f7 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 19:46:27 +0200 Subject: [PATCH 096/258] Added newline after scope changes --- plugins/eddb.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/eddb.py b/plugins/eddb.py index 7eb134d6..db49c66a 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -47,8 +47,10 @@ this.station_marketid = None def system_url(system_name: str) -> str: if this.system_address: return requests.utils.requote_uri(f'https://eddb.io/system/ed-address/{this.system_address}') + elif system_name: return requests.utils.requote_uri(f'https://eddb.io/system/name/{system_name}') + else: return '' @@ -56,6 +58,7 @@ def system_url(system_name: str) -> str: def station_url(system_name: str, station_name: str) -> str: if this.station_marketid: return requests.utils.requote_uri(f'https://eddb.io/station/market-id/{this.station_marketid}') + else: return system_url('') @@ -78,6 +81,7 @@ def prefs_changed(cmdr, is_beta): # Override standard URL functions if config.get('system_provider') == 'eddb': this.system_link['url'] = system_url(this.system) + if config.get('station_provider') == 'eddb': this.station_link['url'] = station_url(this.system, this.station) @@ -118,6 +122,7 @@ def cmdr_data(data, is_beta): # Always store initially, even if we're not the *current* system provider. if not this.station_marketid: this.station_marketid = data['commander']['docked'] and data['lastStarport']['id'] + # Only trust CAPI if these aren't yet set this.system = this.system or data['lastSystem']['name'] this.station = this.station or data['commander']['docked'] and data['lastStarport']['name'] @@ -127,11 +132,14 @@ def cmdr_data(data, is_beta): this.system_link['text'] = this.system this.system_link['url'] = system_url(this.system) this.system_link.update_idletasks() + if config.get('station_provider') == 'eddb': if data['commander']['docked']: this.station_link['text'] = this.station + elif data['lastStarport']['name'] and data['lastStarport']['name'] != "": this.station_link['text'] = STATION_UNDOCKED + else: this.station_link['text'] = '' From e0462d87194c1e4f3831e1e89ce497e7c76777a1 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 20:08:56 +0200 Subject: [PATCH 097/258] Replaced complex oneliner with multiline if --- plugins/eddb.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index db49c66a..48d63d7a 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -112,20 +112,30 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # But only actually change the URL if we are current station provider. if config.get('station_provider') == 'eddb': - this.station_link['text'] = this.station or ( - this.system_population and this.system_population > 0 and STATION_UNDOCKED or '') + text = this.station + if not text: + if this.system_population is not None and this.system_population > 0: + text = STATION_UNDOCKED + + else: + text = '' + + this.station_link['text'] = text this.station_link['url'] = station_url(this.system, this.station) # Override standard URL function this.station_link.update_idletasks() def cmdr_data(data, is_beta): # Always store initially, even if we're not the *current* system provider. - if not this.station_marketid: - this.station_marketid = data['commander']['docked'] and data['lastStarport']['id'] + if not this.station_marketid and data['commander']['docked']: + this.station_marketid = data['lastStarport']['id'] # Only trust CAPI if these aren't yet set - this.system = this.system or data['lastSystem']['name'] - this.station = this.station or data['commander']['docked'] and data['lastStarport']['name'] + if not this.system: + this.system = data['lastSystem']['name'] + + if not this.station and data['commander']['docked']: + this.station = data['lastStarport']['name'] # Override standard URL functions if config.get('system_provider') == 'eddb': From 07c43d30574e74577a9a38381164592c9510ef1e Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 20:12:56 +0200 Subject: [PATCH 098/258] added type annotations to globals --- plugins/eddb.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 48d63d7a..b8c9c9d1 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -24,7 +24,7 @@ import sys -from typing import Any +from typing import Any, Optional import requests from config import config @@ -35,13 +35,13 @@ STATION_UNDOCKED: str = u'×' # "Station" name to display when not docked = U+0 this: Any = sys.modules[__name__] # For holding module globals # Main window clicks -this.system_link = None -this.system = None -this.system_address = None -this.system_population = None -this.station_link = None -this.station = None -this.station_marketid = None +this.system_link: Optional[str] = None +this.system: Optional[str] = None +this.system_address: Optional[str] = None +this.system_population: Optional[int] = None +this.station_link = None # tk thing, not annotated +this.station: Optional[str] = None +this.station_marketid: Optional[int] = None def system_url(system_name: str) -> str: From 7b231fb244429dc5c2dfc720bc2faf4892d63e0e Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 26 Jul 2020 20:14:08 +0200 Subject: [PATCH 099/258] removed unicode specifier from string python3 strings are always unicode --- plugins/eddb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index b8c9c9d1..83f4be59 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -30,7 +30,7 @@ import requests from config import config -STATION_UNDOCKED: str = u'×' # "Station" name to display when not docked = U+00D7 +STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00D7 this: Any = sys.modules[__name__] # For holding module globals @@ -39,7 +39,7 @@ this.system_link: Optional[str] = None this.system: Optional[str] = None this.system_address: Optional[str] = None this.system_population: Optional[int] = None -this.station_link = None # tk thing, not annotated +this.station_link = None # tk thing, not annotated this.station: Optional[str] = None this.station_marketid: Optional[int] = None From cad5f72b0ac4cfbdee09c05aab5962fc2763a777 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 27 Jul 2020 11:26:31 +0200 Subject: [PATCH 100/258] updated type annotation --- plugins/eddb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 83f4be59..43a5b09f 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -24,11 +24,15 @@ import sys -from typing import Any, Optional +import tkinter +from typing import Any, Optional, TYPE_CHECKING import requests from config import config +if TYPE_CHECKING: + from tkinter import Tk + STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00D7 @@ -39,7 +43,7 @@ this.system_link: Optional[str] = None this.system: Optional[str] = None this.system_address: Optional[str] = None this.system_population: Optional[int] = None -this.station_link = None # tk thing, not annotated +this.station_link: 'Optional[tkinter.Tk]' = None this.station: Optional[str] = None this.station_marketid: Optional[int] = None @@ -67,7 +71,7 @@ def plugin_start3(plugin_dir): return 'eddb' -def plugin_app(parent): +def plugin_app(parent: 'tkinter.Tk'): this.system_link = parent.children['system'] # system label in main window this.system = None this.system_address = None From b27e4a362818b9e828ed96e13c13a1c32fbfc4f5 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 27 Jul 2020 15:15:48 +0200 Subject: [PATCH 101/258] fixed tk import and type annotations --- plugins/eddb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 43a5b09f..1613ba5b 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -24,7 +24,6 @@ import sys -import tkinter from typing import Any, Optional, TYPE_CHECKING import requests @@ -43,7 +42,7 @@ this.system_link: Optional[str] = None this.system: Optional[str] = None this.system_address: Optional[str] = None this.system_population: Optional[int] = None -this.station_link: 'Optional[tkinter.Tk]' = None +this.station_link: 'Optional[Tk]' = None this.station: Optional[str] = None this.station_marketid: Optional[int] = None @@ -71,7 +70,7 @@ def plugin_start3(plugin_dir): return 'eddb' -def plugin_app(parent: 'tkinter.Tk'): +def plugin_app(parent: 'Tk'): this.system_link = parent.children['system'] # system label in main window this.system = None this.system_address = None From a5b7bea2ca47d4000facf63f676bdf7fa3ea23a6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 15:16:23 +0100 Subject: [PATCH 102/258] Move comment to line above if --- EDMarketConnector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 0d23f6dd..9bf7496d 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -804,7 +804,8 @@ class AppWindow(object): self.transient(parent) # position over parent - if platform != 'darwin' or parent.winfo_rooty() > 0: # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + # http://core.tcl.tk/tk/tktview/c84f660833546b1b84e7 + if platform != 'darwin' or parent.winfo_rooty() > 0: self.geometry(f'+{parent.winfo_rootx():d}+{parent.winfo_rooty():d}') # remove decoration From 9d2548c70bf2e4490cc13b47508f8dc9db1ea7fb Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 11:24:49 +0200 Subject: [PATCH 103/258] Removed keyring dependency This remove all dependencies on the keyring lib, updates the requirements.txt to reflect that, and ensures that setup.py does not attempt to package it. Any use of the "old" keyring code will now return None and warn about its deprecation. --- config.py | 19 ++++--------------- requirements.txt | 1 - setup.py | 2 -- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/config.py b/config.py index f6ad1acf..b0b887ec 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ import numbers import sys +import warnings from os import getenv, makedirs, mkdir, pardir from os.path import expanduser, dirname, exists, isdir, join, normpath from sys import platform @@ -367,25 +368,13 @@ class Config(object): # Common def get_password(self, account): - try: - import keyring - return keyring.get_password(self.identifier, account) - except ImportError: - return None + warnings.warn("password subsystem is no longer supported", DeprecationWarning) def set_password(self, account, password): - try: - import keyring - keyring.set_password(self.identifier, account, password) - except ImportError: - pass + warnings.warn("password subsystem is no longer supported", DeprecationWarning) def delete_password(self, account): - try: - import keyring - keyring.delete_password(self.identifier, account) - except: - pass # don't care - silently fail + warnings.warn("password subsystem is no longer supported", DeprecationWarning) # singleton config = Config() diff --git a/requirements.txt b/requirements.txt index 5fa59d6a..bf29adc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ certifi==2019.9.11 -keyring==19.2.0 requests>=2.11.1 watchdog>=0.8.3 # argh==0.26.2 watchdog dep diff --git a/setup.py b/setup.py index be40d655..6bf35f8e 100755 --- a/setup.py +++ b/setup.py @@ -73,7 +73,6 @@ if sys.platform=='darwin': 'optimize': 2, 'packages': [ 'requests', - 'keyring.backends', 'sqlite3', # Included for plugins ], 'includes': [ @@ -119,7 +118,6 @@ elif sys.platform=='win32': 'optimize': 2, 'packages': [ 'requests', - 'keyring.backends', 'sqlite3', # Included for plugins ], 'includes': [ From 4da1d9df4ed14909574a69c70d7d0dd3d228e529 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 13:45:55 +0100 Subject: [PATCH 104/258] Change the push-checks back into also PRs. Renamed to suit. --- .github/workflows/{push-checks.yml => push-and-pr-checks.yml} | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename .github/workflows/{push-checks.yml => push-and-pr-checks.yml} (95%) diff --git a/.github/workflows/push-checks.yml b/.github/workflows/push-and-pr-checks.yml similarity index 95% rename from .github/workflows/push-checks.yml rename to .github/workflows/push-and-pr-checks.yml index ba8367c7..dc5ec619 100644 --- a/.github/workflows/push-checks.yml +++ b/.github/workflows/push-and-pr-checks.yml @@ -5,11 +5,13 @@ # # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Push-Checks +name: Push and PR Checks on: push: branches: [ develop ] + pull_request: + branches: [ develop ] jobs: build: From ce4149efd58127adc895a16c5585eed578ff6707 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 12:34:18 +0200 Subject: [PATCH 105/258] Added debug logs --- plugins/inara.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index aa7e7b90..57d8d58e 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -856,7 +856,6 @@ def add_event(name, timestamp, data): def call_timer(wait=FLOOD_LIMIT_SECONDS): while this.timer_run: time.sleep(wait) - print(f"INARA: {time.asctime()} TICK") if this.timer_run: # check again in here just in case we're closing and the stars align call() @@ -869,7 +868,7 @@ def call(callback=None, force=False): return config.set(LAST_UPDATE_CONF_KEY, int(time.time())) - print(f"INARA: {time.asctime()} sending {len(this.events)} entries to inara (call)") + logger.info(f"queuing upload of {len(this.events)} events") data = OrderedDict([ ('header', OrderedDict([ ('appName', applongname), @@ -892,8 +891,6 @@ def worker(): else: (url, data, callback) = item - print(f"INARA: {time.asctime()} sending {len(data['events'])} entries to inara (worker)") - retrying = 0 while retrying < 3: try: @@ -932,7 +929,7 @@ def worker(): break except Exception as e: - logger.debug('Sending events', exc_info=e) + logger.debug('Unable to send events', exc_info=e) retrying += 1 else: if callback: From 755e401df8fbbc2b23e52a9a7e374c29efac3bda Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:02:32 +0100 Subject: [PATCH 106/258] Revert "Change the push-checks back into also PRs." This reverts commit 4da1d9df4ed14909574a69c70d7d0dd3d228e529. --- .github/workflows/{push-and-pr-checks.yml => push-checks.yml} | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) rename .github/workflows/{push-and-pr-checks.yml => push-checks.yml} (95%) diff --git a/.github/workflows/push-and-pr-checks.yml b/.github/workflows/push-checks.yml similarity index 95% rename from .github/workflows/push-and-pr-checks.yml rename to .github/workflows/push-checks.yml index dc5ec619..ba8367c7 100644 --- a/.github/workflows/push-and-pr-checks.yml +++ b/.github/workflows/push-checks.yml @@ -5,13 +5,11 @@ # # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Push and PR Checks +name: Push-Checks on: push: branches: [ develop ] - pull_request: - branches: [ develop ] jobs: build: From 56e33d20d68ab6ec963a1d1d2ab8be181d4b37d3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:03:20 +0100 Subject: [PATCH 107/258] Copy push-checks.yml to pr-checks.yml --- .github/workflows/pr-checks.yml | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/pr-checks.yml diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 00000000..ba8367c7 --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,54 @@ +# This workflow will: +# +# * install Python dependencies +# * lint with a single version of Python +# +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Push-Checks + +on: + push: + branches: [ develop ] + +jobs: + build: + + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + - name: Lint with flake8 + env: + ROOT_SHA: ${{github.base_ref}} + + run: | + DATA=$(python3 < Date: Tue, 28 Jul 2020 15:12:17 +0100 Subject: [PATCH 108/258] pr-checks: Changed to pull_request and fleshed out * Name all steps * After checkout@v2 fetch: 1) checkout base_ref, 2) checkout head_ref * Set and use BASE_REF env var --- .github/workflows/pr-checks.yml | 42 +++++++++++++++------------------ 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index ba8367c7..b7bcd108 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -5,21 +5,30 @@ # # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Push-Checks +name: PR-Checks on: - push: + pull_request: branches: [ develop ] jobs: - build: - + flake8: runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v2 + -name: Checkout commits + uses: actions/checkout@v2 with: fetch-depth: 0 + -name: Checkout base ref + uses: actions/checkout@v2 + with: + ref: ${{github.base_ref}} + -name: Checkout head of PR + uses: actions/checkout@v2 + with: + ref: ${{github.head_ref}} + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: @@ -29,26 +38,13 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + - name: Lint with flake8 env: - ROOT_SHA: ${{github.base_ref}} - + BASE_REF: ${{github.base_ref}} + run: | - DATA=$(python3 < Date: Tue, 28 Jul 2020 15:19:56 +0100 Subject: [PATCH 109/258] Correct "-name" -> "- name" --- .github/workflows/pr-checks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index b7bcd108..a917ea42 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -16,15 +16,15 @@ jobs: runs-on: ubuntu-18.04 steps: - -name: Checkout commits + - name: Checkout commits uses: actions/checkout@v2 with: fetch-depth: 0 - -name: Checkout base ref + - name: Checkout base ref uses: actions/checkout@v2 with: ref: ${{github.base_ref}} - -name: Checkout head of PR + - name: Checkout head of PR uses: actions/checkout@v2 with: ref: ${{github.head_ref}} From 18ff1c0e3f0f5803ae786b76cdcfe55341720e22 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:21:26 +0100 Subject: [PATCH 110/258] Add 'set -x' to get feedback on all bash commands run --- .github/workflows/pr-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index a917ea42..32741e44 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -44,6 +44,8 @@ jobs: BASE_REF: ${{github.base_ref}} run: | + # Show all commands as run + set -x # stop the build if there are Python syntax errors or undefined names, ignore existing git diff "${BASE_REF}" | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide From 3f97588ab676c64b9a8b3192115e5acb4acff7a8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:23:35 +0100 Subject: [PATCH 111/258] Try "fetch-depth: 0" on the extra checkouts --- .github/workflows/pr-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 32741e44..b1d86c13 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -24,10 +24,12 @@ jobs: uses: actions/checkout@v2 with: ref: ${{github.base_ref}} + fetch-depth: 0 - name: Checkout head of PR uses: actions/checkout@v2 with: ref: ${{github.head_ref}} + fetch-depth: 0 - name: Set up Python 3.7 uses: actions/setup-python@v2 From 4a965bb7c945b0cbed49182b54b2edbc985debaa Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:28:20 +0100 Subject: [PATCH 112/258] Extra git commands for diagnosis --- .github/workflows/pr-checks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index b1d86c13..a29e8b00 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -48,6 +48,9 @@ jobs: run: | # Show all commands as run set -x + git branch + git log | head -100 + git diff # stop the build if there are Python syntax errors or undefined names, ignore existing git diff "${BASE_REF}" | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide From f2fa489819a3802febb2b040d0f67f224b5ed5c4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:30:11 +0100 Subject: [PATCH 113/258] Attempt "git diff --" As in "git diff [] [] [--] [...]", in case it's an ambiguity rather than not knowing what 'develop' is. --- .github/workflows/pr-checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index a29e8b00..a73ebeba 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -52,6 +52,6 @@ jobs: git log | head -100 git diff # stop the build if there are Python syntax errors or undefined names, ignore existing - git diff "${BASE_REF}" | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff + git diff "${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - git diff "${BASE_REF}" | flake8 . --count --exit-zero --statistics --diff + git diff "${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff From 648c1af33902abef3e7847965522ec3a57adb01d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:35:34 +0100 Subject: [PATCH 114/258] See if prepending "refs/remotes/origin/" helps --- .github/workflows/pr-checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index a73ebeba..a03fff4a 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -52,6 +52,6 @@ jobs: git log | head -100 git diff # stop the build if there are Python syntax errors or undefined names, ignore existing - git diff "${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff + git diff "refs/remotes/origin/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - git diff "${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff + git diff "refs/remotes/origin/${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff From 629a3461f6028133dd552b3d87ddd74aa79aee25 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 15:39:39 +0100 Subject: [PATCH 115/258] Final clean ups. It works! So, yes, need refs/remotes/origin/ prepended to the base_ref, which will always be a branch for a PR. * Remove debug lines (set -x, etc). * No need for pytest here as yet. * Fall back to requirements.txt if no requirements-dev.txt. --- .github/workflows/pr-checks.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index a03fff4a..7e1c6836 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -38,19 +38,14 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + pip install flake8 + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt ; fi - name: Lint with flake8 env: BASE_REF: ${{github.base_ref}} run: | - # Show all commands as run - set -x - git branch - git log | head -100 - git diff # stop the build if there are Python syntax errors or undefined names, ignore existing git diff "refs/remotes/origin/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide From 3ae0b83ac234538663151600a81b26895c1e6f55 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:02:38 +0100 Subject: [PATCH 116/258] Only run flake8 annotation if there are *.py files in diff --- .github/workflows/pr-annotate-with-flake8.yml | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 2f089a9d..09722b5f 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -11,19 +11,43 @@ on: branches: [ develop ] jobs: - build: - + check_py: runs-on: ubuntu-18.04 - + steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Annotate with Flake8 - uses: "tayfun/flake8-your-pr@master" + + - name: Check for PY files env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BASE_REF: ${{github.base_ref}} + run: | + # Will exit with non-zero if no filenames ending in ".py" are in + # the diff. + git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep -v '.py$' + + flake8_annotate: + - name: Annotate PR with Flake8 + needs: check_py + # Only run if the check_py succeeded + if: ${{ success() }} + + runs-on: ubuntu-18.04 + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Annotate with Flake8 + uses: "tayfun/flake8-your-pr@master" + needs: + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dff1e3bce2e396344d542747642a7adfc9ba5d52 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:06:48 +0100 Subject: [PATCH 117/258] Do *.py check as step in flake8_annotate And make both the 'Annotate with Flake8' and 'Set up Python 3.7' steps dependent on it. --- .github/workflows/pr-annotate-with-flake8.yml | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 09722b5f..81520373 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -11,42 +11,31 @@ on: branches: [ develop ] jobs: - check_py: - runs-on: ubuntu-18.04 - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Check for PY files - env: - BASE_REF: ${{github.base_ref}} - run: | - # Will exit with non-zero if no filenames ending in ".py" are in - # the diff. - git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep -v '.py$' - flake8_annotate: - - name: Annotate PR with Flake8 - needs: check_py - # Only run if the check_py succeeded - if: ${{ success() }} + runs-on: ubuntu-18.04 - runs-on: ubuntu-18.04 - - steps: - - uses: actions/checkout@v2 + steps: + - name: Checkout + uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Check for PY files + env: + BASE_REF: ${{github.base_ref}} + run: | + # Will exit with non-zero if no filenames ending in ".py" are in + # the diff. + git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep -v '.py$' + - name: Set up Python 3.7 + if: ${{ success() }} uses: actions/setup-python@v2 with: python-version: 3.7 - name: Annotate with Flake8 + if: ${{ success() }} uses: "tayfun/flake8-your-pr@master" needs: env: From 9c413a8ba9740f22ebfe89686f5d20c1b8621bb5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:08:15 +0100 Subject: [PATCH 118/258] Remove extraneous 'needs:' --- .github/workflows/pr-annotate-with-flake8.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 81520373..a951c274 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -37,6 +37,5 @@ jobs: - name: Annotate with Flake8 if: ${{ success() }} uses: "tayfun/flake8-your-pr@master" - needs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 792583bfe1f29aed88c07a45b54a226fd5009519 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:10:04 +0100 Subject: [PATCH 119/258] *.py exit status was inverted That'll teach me for testing with a commit only affecting *.yml files. --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index a951c274..05d914cb 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -26,7 +26,7 @@ jobs: run: | # Will exit with non-zero if no filenames ending in ".py" are in # the diff. - git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep -v '.py$' + git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$' - name: Set up Python 3.7 if: ${{ success() }} From a93cfb5ed8bf73ba3e5afb7629f55511d884a0ba Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:25:27 +0100 Subject: [PATCH 120/258] Try ENV var for passing presence of *.py files This uses to set PYFILES env var globally in the job, so should be testable in the following steps. --- .github/workflows/pr-annotate-with-flake8.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 05d914cb..32bbdf5b 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -26,16 +26,17 @@ jobs: run: | # Will exit with non-zero if no filenames ending in ".py" are in # the diff. - git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$' + PYFILES=$(git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$') + echo '::set-env name=PYFILES::${PYFILES}' - name: Set up Python 3.7 - if: ${{ success() }} + if: ${{ env.PYFILES != "" }} uses: actions/setup-python@v2 with: python-version: 3.7 - name: Annotate with Flake8 - if: ${{ success() }} + if: ${{ env.PYFILES != "" }} uses: "tayfun/flake8-your-pr@master" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From a548d383a30ca625f1a0564e7da44895244dff8c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:29:42 +0100 Subject: [PATCH 121/258] Single quotes then ? --- .github/workflows/pr-annotate-with-flake8.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 32bbdf5b..f65d7452 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -30,13 +30,13 @@ jobs: echo '::set-env name=PYFILES::${PYFILES}' - name: Set up Python 3.7 - if: ${{ env.PYFILES != "" }} + if: ${{ env.PYFILES != '' }} uses: actions/setup-python@v2 with: python-version: 3.7 - name: Annotate with Flake8 - if: ${{ env.PYFILES != "" }} + if: ${{ env.PYFILES != '' }} uses: "tayfun/flake8-your-pr@master" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 40dc913367053deffcbb453a4650ac4f05bc3ed2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:32:06 +0100 Subject: [PATCH 122/258] *.py check should always exit 0 --- .github/workflows/pr-annotate-with-flake8.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index f65d7452..41728432 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -28,6 +28,7 @@ jobs: # the diff. PYFILES=$(git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$') echo '::set-env name=PYFILES::${PYFILES}' + exit 0 - name: Set up Python 3.7 if: ${{ env.PYFILES != '' }} From 24cf445254031ac669b61e24544f8d8d8b6b10cf Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:42:57 +0100 Subject: [PATCH 123/258] Gurantee exit 0 AND fix the echo * GH Workflows run under "/bin/bash -e", which means if anything in it fails it all exits immediately. So append " || true" to the egrep. * The echo to magically set the env var back in the workflow needs to use "" not '' else the in-shell variable won't interpolate. --- .github/workflows/pr-annotate-with-flake8.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 41728432..0783eecd 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -26,9 +26,8 @@ jobs: run: | # Will exit with non-zero if no filenames ending in ".py" are in # the diff. - PYFILES=$(git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$') - echo '::set-env name=PYFILES::${PYFILES}' - exit 0 + PYFILES=$(git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$' || true) + echo "::set-env name=PYFILES::${PYFILES}" - name: Set up Python 3.7 if: ${{ env.PYFILES != '' }} From 9a5884b57d6fc8ea7c7e7877669e865f967d3082 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:45:52 +0100 Subject: [PATCH 124/258] Cause EDMC.py to diff to check the detection --- EDMC.py | 1 + 1 file changed, 1 insertion(+) diff --git a/EDMC.py b/EDMC.py index 2d2eb476..911c6c1d 100755 --- a/EDMC.py +++ b/EDMC.py @@ -3,6 +3,7 @@ # Command-line interface. Requires prior setup through the GUI. # +# ping! import argparse import json import sys From ed9a4cfe2905d77bb7641406ae4b0deb43080750 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 16:48:57 +0100 Subject: [PATCH 125/258] Revert "Cause EDMC.py to diff to check the detection" This reverts commit 3b55a7a5deb009fac48b021602a074e3345afed2. --- EDMC.py | 1 - 1 file changed, 1 deletion(-) diff --git a/EDMC.py b/EDMC.py index 911c6c1d..2d2eb476 100755 --- a/EDMC.py +++ b/EDMC.py @@ -3,7 +3,6 @@ # Command-line interface. Requires prior setup through the GUI. # -# ping! import argparse import json import sys From 09e39a47ef29c5ae0161568e583187cf22a097f8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:01:24 +0100 Subject: [PATCH 126/258] Attempt to limit flake8 annotations to Athanasius --- .github/workflows/pr-annotate-with-flake8.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 0783eecd..ab7f86c4 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,11 +16,13 @@ jobs: steps: - name: Checkout + if: ${{ github.actor == "Athanasius" }} uses: actions/checkout@v2 with: fetch-depth: 0 - name: Check for PY files + if: ${{ github.actor == "Athanasius" }} env: BASE_REF: ${{github.base_ref}} run: | @@ -30,12 +32,14 @@ jobs: echo "::set-env name=PYFILES::${PYFILES}" - name: Set up Python 3.7 + if: ${{ github.actor == "Athanasius" }} if: ${{ env.PYFILES != '' }} uses: actions/setup-python@v2 with: python-version: 3.7 - name: Annotate with Flake8 + if: ${{ github.actor == "Athanasius" }} if: ${{ env.PYFILES != '' }} uses: "tayfun/flake8-your-pr@master" env: From dc58d0f3af673a0a494009b02d92e20d288fb739 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:02:32 +0100 Subject: [PATCH 127/258] S I N G L E Q U O T E S --- .github/workflows/pr-annotate-with-flake8.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index ab7f86c4..6b5e0ed2 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,13 +16,13 @@ jobs: steps: - name: Checkout - if: ${{ github.actor == "Athanasius" }} + if: ${{ github.actor == 'Athanasius' }} uses: actions/checkout@v2 with: fetch-depth: 0 - name: Check for PY files - if: ${{ github.actor == "Athanasius" }} + if: ${{ github.actor == 'Athanasius' }} env: BASE_REF: ${{github.base_ref}} run: | @@ -32,14 +32,14 @@ jobs: echo "::set-env name=PYFILES::${PYFILES}" - name: Set up Python 3.7 - if: ${{ github.actor == "Athanasius" }} + if: ${{ github.actor == 'Athanasius' }} if: ${{ env.PYFILES != '' }} uses: actions/setup-python@v2 with: python-version: 3.7 - name: Annotate with Flake8 - if: ${{ github.actor == "Athanasius" }} + if: ${{ github.actor == 'Athanasius' }} if: ${{ env.PYFILES != '' }} uses: "tayfun/flake8-your-pr@master" env: From 6c9bffcaa05bf4abc4d4d9d2848a877308c198b4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:11:16 +0100 Subject: [PATCH 128/258] Don't run flake8_annotate if no access This is using github.actor and checking who it is. If they're not someone who should have access then use a run with 'exit 1' to fail the check job. Then flake8_annotate job needs check_access to have succeeded. This *will* flag the workflow action as failed if no access, hopefully it will be obvious why. Using 'AthanasiusTEST' so it fails for me on this commit/push. --- .github/workflows/pr-annotate-with-flake8.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 6b5e0ed2..58d2fc4d 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -11,18 +11,23 @@ on: branches: [ develop ] jobs: + check_access: + steps: + -name: Check annotation access + if: ${{ github.actor != 'AthanasiusTEST' }} + run: exit 1 + flake8_annotate: + needs: check_access runs-on: ubuntu-18.04 steps: - name: Checkout - if: ${{ github.actor == 'Athanasius' }} uses: actions/checkout@v2 with: fetch-depth: 0 - name: Check for PY files - if: ${{ github.actor == 'Athanasius' }} env: BASE_REF: ${{github.base_ref}} run: | @@ -32,14 +37,12 @@ jobs: echo "::set-env name=PYFILES::${PYFILES}" - name: Set up Python 3.7 - if: ${{ github.actor == 'Athanasius' }} if: ${{ env.PYFILES != '' }} uses: actions/setup-python@v2 with: python-version: 3.7 - name: Annotate with Flake8 - if: ${{ github.actor == 'Athanasius' }} if: ${{ env.PYFILES != '' }} uses: "tayfun/flake8-your-pr@master" env: From b40ee967558e11cf50ceb44d5f8926a08cc7da68 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:14:27 +0100 Subject: [PATCH 129/258] Correct syntax --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 58d2fc4d..e7023269 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -13,7 +13,7 @@ on: jobs: check_access: steps: - -name: Check annotation access + - name: Check annotation access if: ${{ github.actor != 'AthanasiusTEST' }} run: exit 1 From 912f82ce386675026a1f23fa74929cc9d37a0e0a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:15:14 +0100 Subject: [PATCH 130/258] Needs a runs-on for the check_access --- .github/workflows/pr-annotate-with-flake8.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index e7023269..ddfa09c8 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -12,6 +12,7 @@ on: jobs: check_access: + runs-on: ubuntu-18.04 steps: - name: Check annotation access if: ${{ github.actor != 'AthanasiusTEST' }} From 9f860a5990bf58562bddd3c21158b545651e421d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:27:59 +0100 Subject: [PATCH 131/258] Check if a conditional is needed on all steps Basically, if it's only on the first and fails, then do the others still run ? --- .github/workflows/pr-annotate-with-flake8.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index ddfa09c8..06ee9af2 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -11,19 +11,13 @@ on: branches: [ develop ] jobs: - check_access: - runs-on: ubuntu-18.04 - steps: - - name: Check annotation access - if: ${{ github.actor != 'AthanasiusTEST' }} - run: exit 1 - flake8_annotate: needs: check_access runs-on: ubuntu-18.04 steps: - name: Checkout + if: ${{ github.actor != 'AthanasiusTEST' }} uses: actions/checkout@v2 with: fetch-depth: 0 From e3b01f6c58094ff8d53adc9117e64bc9315c0cc0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:28:55 +0100 Subject: [PATCH 132/258] Remove non-existent dependency --- .github/workflows/pr-annotate-with-flake8.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 06ee9af2..3da4742a 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -12,7 +12,6 @@ on: jobs: flake8_annotate: - needs: check_access runs-on: ubuntu-18.04 steps: From a1e234426334bc8ab56c92c9a0c8082ccaf92ada Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:32:10 +0100 Subject: [PATCH 133/258] Invert the actor test --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 3da4742a..9bb28cbf 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - if: ${{ github.actor != 'AthanasiusTEST' }} + if: ${{ github.actor == 'AthanasiusTEST' }} uses: actions/checkout@v2 with: fetch-depth: 0 From cea7d14214a99ee29c22072be6822e50ef39bcb5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:37:57 +0100 Subject: [PATCH 134/258] Check for .git in PY files check. NB: This might cause env.PYFILES checks to fail if they need the var to at least exist. --- .github/workflows/pr-annotate-with-flake8.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 9bb28cbf..779b61ba 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,8 +25,11 @@ jobs: env: BASE_REF: ${{github.base_ref}} run: | - # Will exit with non-zero if no filenames ending in ".py" are in - # the diff. + # Checkout might not have happened. + if [ ! -f ".git" ]; then exit 0 ; fi + # Get a list of files ending with ".py", stuff in environment. + # We don't rely on exit status because Workflows run with + # "set -e" so any failure in a pipe is total failure. PYFILES=$(git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$' || true) echo "::set-env name=PYFILES::${PYFILES}" From 2375bf19955551be7535b82e889f232c06f85bf6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:45:34 +0100 Subject: [PATCH 135/258] Try contains(['Athanasius'], github.actor) --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 779b61ba..df8926c2 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - if: ${{ github.actor == 'AthanasiusTEST' }} + if: ${{ contains(['Athanasius'], github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 5dd17cacaec515653792c03d72d7887dc9a7ed11 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:57:07 +0100 Subject: [PATCH 136/258] Set array in earlier job using JSON --- .github/workflows/pr-annotate-with-flake8.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index df8926c2..3028154e 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -11,12 +11,21 @@ on: branches: [ develop ] jobs: + set_allowed: + runs-on: ubuntu-18.04 + outputs: + matrix: ${{ steps.set-allowed.outputs.allowed }} + steps: + - id: set-allowed + run: echo "::set-output name=allowed::{['Athanasius']}" + flake8_annotate: runs-on: ubuntu-18.04 + needs: set_allowed steps: - name: Checkout - if: ${{ contains(['Athanasius'], github.actor) }} + if: ${{ contains(${{fromJson(needs.set_allowed.outputs.allowed)}}, github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 52306b61b7eb3d305ec208f710983e209f96cbf3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 17:58:23 +0100 Subject: [PATCH 137/258] No nesting ${{..}} --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 3028154e..d0077aa1 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(${{fromJson(needs.set_allowed.outputs.allowed)}}, github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From d38c74d36ab2a047eb39e58de0d3737487b5e360 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:01:16 +0100 Subject: [PATCH 138/258] JSON must have double quotes --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index d0077aa1..f9798cc6 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -17,7 +17,7 @@ jobs: matrix: ${{ steps.set-allowed.outputs.allowed }} steps: - id: set-allowed - run: echo "::set-output name=allowed::{['Athanasius']}" + run: echo '::set-output name=allowed::{["Athanasius"]}' flake8_annotate: runs-on: ubuntu-18.04 From 01354dc1eccc2911112f9cdf3b6e2346492d4390 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:03:24 +0100 Subject: [PATCH 139/258] Wildcard on the end ? --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index f9798cc6..c3eb3beb 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 32190a6ebc15c61ee90bacbf10682f9581b0da25 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:06:01 +0100 Subject: [PATCH 140/258] Need at least one JSON key ? --- .github/workflows/pr-annotate-with-flake8.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index c3eb3beb..354c763f 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -17,7 +17,7 @@ jobs: matrix: ${{ steps.set-allowed.outputs.allowed }} steps: - id: set-allowed - run: echo '::set-output name=allowed::{["Athanasius"]}' + run: echo '::set-output name=allowed::{"allowed":["Athanasius"]}' flake8_annotate: runs-on: ubuntu-18.04 @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.allowed.*), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 75947cb41bb22b383fd782547044417f5643e8d2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:08:38 +0100 Subject: [PATCH 141/258] We don't want 'matrix', just 'allowed'. --- .github/workflows/pr-annotate-with-flake8.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 354c763f..3a27e973 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -14,10 +14,10 @@ jobs: set_allowed: runs-on: ubuntu-18.04 outputs: - matrix: ${{ steps.set-allowed.outputs.allowed }} + allowed: ${{ steps.set-allowed.outputs.allowed }} steps: - id: set-allowed - run: echo '::set-output name=allowed::{"allowed":["Athanasius"]}' + run: echo '::set-output name=allowed::{["Athanasius"]}' flake8_annotate: runs-on: ubuntu-18.04 @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.allowed.*), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From dc8497cd9b30d53cac8072da85953ca7b7a7e7f7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:10:26 +0100 Subject: [PATCH 142/258] Maybe no wildcard now? --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 3a27e973..f5286a65 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From c71d718c5550ea37808fad42a54cad71bac11b06 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:48 +0100 Subject: [PATCH 143/258] Revert "Maybe no wildcard now?" This reverts commit 61898d1f91fe8f59eb988b79ffce93f24552b92c. --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index f5286a65..3a27e973 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From ba437aecd3e280885cb257b2840f0fd5e8980c0d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:49 +0100 Subject: [PATCH 144/258] Revert "We don't want 'matrix', just 'allowed'." This reverts commit 9938370e91e3144ce0f29f3da7057e402f9fe3fb. --- .github/workflows/pr-annotate-with-flake8.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 3a27e973..354c763f 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -14,10 +14,10 @@ jobs: set_allowed: runs-on: ubuntu-18.04 outputs: - allowed: ${{ steps.set-allowed.outputs.allowed }} + matrix: ${{ steps.set-allowed.outputs.allowed }} steps: - id: set-allowed - run: echo '::set-output name=allowed::{["Athanasius"]}' + run: echo '::set-output name=allowed::{"allowed":["Athanasius"]}' flake8_annotate: runs-on: ubuntu-18.04 @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.allowed.*), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 9b2db0c2e21839207f6a20bbd736c7406950f8f5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:50 +0100 Subject: [PATCH 145/258] Revert "Need at least one JSON key ?" This reverts commit f509d974c1eb926bc1640ba3c2703518ac972136. --- .github/workflows/pr-annotate-with-flake8.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 354c763f..c3eb3beb 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -17,7 +17,7 @@ jobs: matrix: ${{ steps.set-allowed.outputs.allowed }} steps: - id: set-allowed - run: echo '::set-output name=allowed::{"allowed":["Athanasius"]}' + run: echo '::set-output name=allowed::{["Athanasius"]}' flake8_annotate: runs-on: ubuntu-18.04 @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.allowed.*), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 069e7ce9e541b987bcd46035faed4f236975fde9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:51 +0100 Subject: [PATCH 146/258] Revert "Wildcard on the end ?" This reverts commit 511e7a48d6e4dfb4b61cfc43517e7ac9a5f5d7ec. --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index c3eb3beb..f9798cc6 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed.*), github.actor) }} + if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed), github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 0736bbf97ba3a09a2978d092af84b12b1c466314 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:52 +0100 Subject: [PATCH 147/258] Revert "JSON must have double quotes" This reverts commit 4db2c907dfd384bb04d99492b45283af1d2e2cc7. --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index f9798cc6..d0077aa1 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -17,7 +17,7 @@ jobs: matrix: ${{ steps.set-allowed.outputs.allowed }} steps: - id: set-allowed - run: echo '::set-output name=allowed::{["Athanasius"]}' + run: echo "::set-output name=allowed::{['Athanasius']}" flake8_annotate: runs-on: ubuntu-18.04 From 51f92b37931fef4b57ae13552b91e9db709ed1d9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:53 +0100 Subject: [PATCH 148/258] Revert "No nesting ${{..}}" This reverts commit 9ed541f0e00a5041e8308228fb6a8bff32ceb01c. --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index d0077aa1..3028154e 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(fromJson(needs.set_allowed.outputs.allowed), github.actor) }} + if: ${{ contains(${{fromJson(needs.set_allowed.outputs.allowed)}}, github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From e22fadd5f5a7877b27d85e02ae2fdf5842b8d3c0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:12:54 +0100 Subject: [PATCH 149/258] Revert "Set array in earlier job using JSON" This reverts commit 8b863c5efdb1414c3fbdf4817c14129d0ccd392a. --- .github/workflows/pr-annotate-with-flake8.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 3028154e..df8926c2 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -11,21 +11,12 @@ on: branches: [ develop ] jobs: - set_allowed: - runs-on: ubuntu-18.04 - outputs: - matrix: ${{ steps.set-allowed.outputs.allowed }} - steps: - - id: set-allowed - run: echo "::set-output name=allowed::{['Athanasius']}" - flake8_annotate: runs-on: ubuntu-18.04 - needs: set_allowed steps: - name: Checkout - if: ${{ contains(${{fromJson(needs.set_allowed.outputs.allowed)}}, github.actor) }} + if: ${{ contains(['Athanasius'], github.actor) }} uses: actions/checkout@v2 with: fetch-depth: 0 From 7a44c28b955ae868a6c9199e82df51ffd19e9cba Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:13:15 +0100 Subject: [PATCH 150/258] Revert "Try contains(['Athanasius'], github.actor)" This reverts commit 72c201c5a1336bb7face0a5c686d114194efbdb1. --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index df8926c2..779b61ba 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - if: ${{ contains(['Athanasius'], github.actor) }} + if: ${{ github.actor == 'AthanasiusTEST' }} uses: actions/checkout@v2 with: fetch-depth: 0 From 59babd113bce1c4af3330b55e5ab446bd6d5fec1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:13:34 +0100 Subject: [PATCH 151/258] Just use the single allowed actor for now. And it's now set to "Athanasius". --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 779b61ba..db5c0114 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - if: ${{ github.actor == 'AthanasiusTEST' }} + if: ${{ github.actor == 'Athanasius' }} uses: actions/checkout@v2 with: fetch-depth: 0 From c18f183db918a02d44e3261ef70669dde9a26e34 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 18:22:58 +0100 Subject: [PATCH 152/258] .git would be a directory if present, fix test --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index db5c0114..54e91393 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -26,7 +26,7 @@ jobs: BASE_REF: ${{github.base_ref}} run: | # Checkout might not have happened. - if [ ! -f ".git" ]; then exit 0 ; fi + if [ ! -d ".git" ]; then exit 0 ; fi # Get a list of files ending with ".py", stuff in environment. # We don't rely on exit status because Workflows run with # "set -e" so any failure in a pipe is total failure. From a3b7dcbedc8a02580516f69ec7734ae8ffabd567 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 27 Jul 2020 21:30:52 +0100 Subject: [PATCH 153/258] Logging: Handle 'bare function' caller for class/qualname --- EDMCLogging.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/EDMCLogging.py b/EDMCLogging.py index acae327d..a4906a08 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -133,6 +133,22 @@ class EDMCContextFilter(logging.Filter): # Paranoia checks if frame_class and frame_class.__qualname__: caller_class_names = frame_class.__qualname__ + # If the frame caller is a bare function then there's no 'self' + elif frame.f_code.co_name and frame.f_code.co_name in frame.f_globals: + fn = frame.f_globals[frame.f_code.co_name] + if fn and fn.__qualname__: + caller_qualname = fn.__qualname__ + + frame_class = getattr(fn, '__class__', None) + if frame_class and frame_class.__qualname__: + caller_class_names = frame_class.__qualname__ + + # 'class' __qualname__ of 'function' means it's a bare + # function for sure. You *can* have a class called + # 'function', so let's make this 100% obvious what it is. + if caller_class_names == 'function': + # In case the condition above tests a tuple of values + caller_class_names = f'<{caller_class_names}>' if caller_qualname == '': print('ALERT! Something went wrong with finding caller qualname for logging!') From 7a8f29edcf119f2da6ac9cbbe54a2698205910f7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 07:32:39 +0100 Subject: [PATCH 154/258] Add comments about future unit tests to implement. Some of these won't even have been manually tested yet, so might require more updates to the frame-walking function. --- EDMCLogging.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/EDMCLogging.py b/EDMCLogging.py index a4906a08..0b81f1d2 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -10,6 +10,30 @@ import logging from typing import Tuple +# TODO: Tests: +# +# 1. Call from bare function in file. +# 2. Call from `if __name__ == "__main__":` section +# +# 3. Call from 1st level function in 1st level Class in file +# 4. Call from 2nd level function in 1st level Class in file +# 5. Call from 3rd level function in 1st level Class in file +# +# 6. Call from 1st level function in 2nd level Class in file +# 7. Call from 2nd level function in 2nd level Class in file +# 8. Call from 3rd level function in 2nd level Class in file +# +# 9. Call from 1st level function in 3rd level Class in file +# 10. Call from 2nd level function in 3rd level Class in file +# 11. Call from 3rd level function in 3rd level Class in file +# +# 12. Call from 2nd level file, all as above. +# +# 13. Call from *module* +# +# 14. Call from *package* + + class Logger: """ Wrapper class for all logging configuration and code. From 1989b272eb6ad0938efb2599ce626132a350e601 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 08:17:50 +0100 Subject: [PATCH 155/258] Use `inspect` to get frame information for qualname Also includes some minor tweaks to make PyCharm happier: * @classmethod caller_class_and_qualname(). * No need to () when "returning a tuple". --- EDMCLogging.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 0b81f1d2..00cf504d 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -5,11 +5,14 @@ members on the logging.LogRecord instance for use in logging.Formatter() strings. """ -import sys +from sys import _getframe as getframe +import inspect import logging -from typing import Tuple +from typing import TYPE_CHECKING, Tuple +# if TYPE_CHECKING: + # TODO: Tests: # # 1. Call from bare function in file. @@ -115,7 +118,8 @@ class EDMCContextFilter(logging.Filter): return True - def caller_class_and_qualname(self) -> Tuple[str, str]: + @classmethod + def caller_class_and_qualname(cls) -> Tuple[str, str]: """ Figure out our caller's class name(s) and qualname @@ -126,7 +130,7 @@ class EDMCContextFilter(logging.Filter): # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start # of the frames internal to logging. - frame = sys._getframe(0) + frame: 'frameobject' = getframe(0) while frame: if isinstance(frame.f_locals.get('self'), logging.Logger): frame = frame.f_back # Want to start on the next frame below @@ -143,20 +147,21 @@ class EDMCContextFilter(logging.Filter): caller_qualname = caller_class_names = '' if frame: - if frame.f_locals and 'self' in frame.f_locals: + # + frame_info = inspect.getframeinfo(frame) + args, _, _, value_dict = inspect.getargvalues(frame) + if len(args) and args[0] == 'self': + frame_class = value_dict['self'] # Find __qualname__ of the caller - # Paranoia checks - if frame.f_code and frame.f_code.co_name: - fn = getattr(frame.f_locals['self'], frame.f_code.co_name) + fn = getattr(frame_class, frame_info.function) - if fn and fn.__qualname__: - caller_qualname = fn.__qualname__ + if fn and fn.__qualname__: + caller_qualname = fn.__qualname__ # Find containing class name(s) of caller, if any - frame_class = frame.f_locals['self'].__class__ - # Paranoia checks if frame_class and frame_class.__qualname__: caller_class_names = frame_class.__qualname__ + # If the frame caller is a bare function then there's no 'self' elif frame.f_code.co_name and frame.f_code.co_name in frame.f_globals: fn = frame.f_globals[frame.f_code.co_name] @@ -174,6 +179,9 @@ class EDMCContextFilter(logging.Filter): # In case the condition above tests a tuple of values caller_class_names = f'<{caller_class_names}>' + # https://docs.python.org/3.7/library/inspect.html#the-interpreter-stack + del frame + if caller_qualname == '': print('ALERT! Something went wrong with finding caller qualname for logging!') caller_qualname = '' @@ -182,4 +190,4 @@ class EDMCContextFilter(logging.Filter): print('ALERT! Something went wrong with finding caller class name(s) for logging!') caller_class_names = '' - return (caller_class_names, caller_qualname) + return caller_class_names, caller_qualname From 589bc0b5f12f65358dcde01379ebbbb2ea39bec7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 08:51:18 +0100 Subject: [PATCH 156/258] Cover all cases with `inspect` * works. * top-level function in works, presumably also any other file. * Call from within classes works. * Extra, commented out, test cases in EDMarketConnector.py --- EDMCLogging.py | 37 +++++++++++++++---------------------- EDMarketConnector.py | 13 +++++++++---- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 00cf504d..a8fb45df 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -152,32 +152,25 @@ class EDMCContextFilter(logging.Filter): args, _, _, value_dict = inspect.getargvalues(frame) if len(args) and args[0] == 'self': frame_class = value_dict['self'] - # Find __qualname__ of the caller - fn = getattr(frame_class, frame_info.function) - if fn and fn.__qualname__: - caller_qualname = fn.__qualname__ + if frame_class: + # Find __qualname__ of the caller + fn = getattr(frame_class, frame_info.function) + if fn and fn.__qualname__: + caller_qualname = fn.__qualname__ - # Find containing class name(s) of caller, if any - if frame_class and frame_class.__qualname__: - caller_class_names = frame_class.__qualname__ + # Find containing class name(s) of caller, if any + if frame_class.__class__ and frame_class.__class__.__qualname__: + caller_class_names = frame_class.__class__.__qualname__ - # If the frame caller is a bare function then there's no 'self' - elif frame.f_code.co_name and frame.f_code.co_name in frame.f_globals: - fn = frame.f_globals[frame.f_code.co_name] - if fn and fn.__qualname__: - caller_qualname = fn.__qualname__ + # It's a call from the top level module file + elif frame_info.function == '': + caller_class_names = '' + caller_qualname = value_dict['__name__'] - frame_class = getattr(fn, '__class__', None) - if frame_class and frame_class.__qualname__: - caller_class_names = frame_class.__qualname__ - - # 'class' __qualname__ of 'function' means it's a bare - # function for sure. You *can* have a class called - # 'function', so let's make this 100% obvious what it is. - if caller_class_names == 'function': - # In case the condition above tests a tuple of values - caller_class_names = f'<{caller_class_names}>' + elif frame_info.function != '': + caller_class_names = '' + caller_qualname = frame_info.function # https://docs.python.org/3.7/library/inspect.html#the-interpreter-stack del frame diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 9bf7496d..d953e819 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1014,6 +1014,9 @@ def enforce_single_instance() -> None: EnumWindows(enumwindowsproc, 0) +def test_logging(): + logger.debug('Test from EDMarketConnector.py top-level test_logging()') + # Run the app if __name__ == "__main__": @@ -1026,10 +1029,9 @@ if __name__ == "__main__": logger = EDMCLogging.Logger(appname).get_logger() - # Plain, not via `logger` - print(f'{applongname} {appversion}') - - # TODO: unittest in place of this + # TODO: unittests in place of these + # logger.debug('Test from __main__') + # test_logging() class A(object): class B(object): def __init__(self): @@ -1037,6 +1039,9 @@ if __name__ == "__main__": # abinit = A.B() + # Plain, not via `logger` + print(f'{applongname} {appversion}') + Translations.install(config.get('language') or None) # Can generate errors so wait til log set up root = tk.Tk(className=appname.lower()) From 45ef87bcb43be51383fdb43b4595a00cdc61f00d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 09:19:34 +0100 Subject: [PATCH 157/258] Correct frame typing, and import clean up. Also explains why we're doing that _getframe import that way. --- EDMCLogging.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index a8fb45df..15a29286 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -5,14 +5,13 @@ members on the logging.LogRecord instance for use in logging.Formatter() strings. """ +# So that any warning about accessing a protected member is only in one place. from sys import _getframe as getframe import inspect import logging -from typing import TYPE_CHECKING, Tuple +from typing import Tuple -# if TYPE_CHECKING: - # TODO: Tests: # # 1. Call from bare function in file. @@ -130,7 +129,7 @@ class EDMCContextFilter(logging.Filter): # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start # of the frames internal to logging. - frame: 'frameobject' = getframe(0) + frame: 'frame' = getframe(0) while frame: if isinstance(frame.f_locals.get('self'), logging.Logger): frame = frame.f_back # Want to start on the next frame below From 4ecb4f573a00af788f5c680c9767f4be49edbcf2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 11:20:30 +0100 Subject: [PATCH 158/258] 2 lines after def test_logging() --- EDMarketConnector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index d953e819..316add60 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1017,6 +1017,7 @@ def enforce_single_instance() -> None: def test_logging(): logger.debug('Test from EDMarketConnector.py top-level test_logging()') + # Run the app if __name__ == "__main__": From 3b87df17afc754182601d92dc72859a0f402b7df Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 28 Jul 2020 11:46:55 +0100 Subject: [PATCH 159/258] Tweak EDMarketConnector.py startup so the redirect is first * Don't want any output until the redirect is done when running frozen. * Make the line buffering in the redirect more obvious. --- EDMarketConnector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 316add60..2d1b81b9 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1020,13 +1020,15 @@ def test_logging(): # Run the app if __name__ == "__main__": - - enforce_single_instance() + # Keep this as the very first code run to be as sure as possible of no + # output until after this redirect is done, if needed. if getattr(sys, 'frozen', False): # By default py2exe tries to write log to dirname(sys.executable) which fails when installed import tempfile - # unbuffered not allowed for text in python3, so use line buffering - sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), f'{appname}.log'), 'wt', 1) + # unbuffered not allowed for text in python3, so use `1 for line buffering + sys.stdout = sys.stderr = open(join(tempfile.gettempdir(), f'{appname}.log'), mode='wt', buffering=1) + + enforce_single_instance() logger = EDMCLogging.Logger(appname).get_logger() From 0115e5c6ba30a94f70a4671e3c71cc3fd6aefd5f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 14:32:12 +0100 Subject: [PATCH 160/258] PLUGINS.md: except not catch --- PLUGINS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PLUGINS.md b/PLUGINS.md index d574620b..8a43bf6f 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -110,14 +110,14 @@ Then replace `print(...)` statements with one of the following: try: ... - catch Exception: + except Exception: # This logs at 'ERROR' level. # Also automatically includes exception information. logger.exception('An exception occurred') try: ... - catch Exception as e: + except Exception as e: logger.debug('Exception we only note in debug output', exc_info=e) ``` From 9259afd344bcda4e2c5021de8aaae75c0edc1dfa Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 11:58:13 +0100 Subject: [PATCH 161/258] Add display of entire github context --- .github/workflows/pr-checks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 7e1c6836..c396783c 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -16,6 +16,7 @@ jobs: runs-on: ubuntu-18.04 steps: + # https://github.com/actions/checkout - name: Checkout commits uses: actions/checkout@v2 with: @@ -31,6 +32,9 @@ jobs: ref: ${{github.head_ref}} fetch-depth: 0 + -name: Show github context + -run: cat $GITHUB_EVENT_PATH + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: From e430246d5b8a17b78de5ce049adc4c69a537cd7e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:07:52 +0100 Subject: [PATCH 162/258] Add comment to trigger change --- EDMCLogging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/EDMCLogging.py b/EDMCLogging.py index 15a29286..72e7c81c 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -5,6 +5,7 @@ members on the logging.LogRecord instance for use in logging.Formatter() strings. """ +# A small comment # So that any warning about accessing a protected member is only in one place. from sys import _getframe as getframe import inspect From 5d7612b0db0166137d4af07684e83ecc5bdf9f20 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:09:05 +0100 Subject: [PATCH 163/258] Move comment, was it stopping any steps from being defined? --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index c396783c..d85fddff 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -16,8 +16,8 @@ jobs: runs-on: ubuntu-18.04 steps: - # https://github.com/actions/checkout - name: Checkout commits + # https://github.com/actions/checkout uses: actions/checkout@v2 with: fetch-depth: 0 From 91e07a62eb6c0787885bf876963810a4095c5a3e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:12:53 +0100 Subject: [PATCH 164/258] Revert pr-checks.yml to see if it runs again --- .github/workflows/pr-checks.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index d85fddff..7e1c6836 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -17,7 +17,6 @@ jobs: steps: - name: Checkout commits - # https://github.com/actions/checkout uses: actions/checkout@v2 with: fetch-depth: 0 @@ -32,9 +31,6 @@ jobs: ref: ${{github.head_ref}} fetch-depth: 0 - -name: Show github context - -run: cat $GITHUB_EVENT_PATH - - name: Set up Python 3.7 uses: actions/setup-python@v2 with: From 43ae6dde61301bda2daa021b947d16ba1e16481b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:15:15 +0100 Subject: [PATCH 165/258] Restore changes, draft status was to blame ? --- .github/workflows/pr-checks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 7e1c6836..d85fddff 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -17,6 +17,7 @@ jobs: steps: - name: Checkout commits + # https://github.com/actions/checkout uses: actions/checkout@v2 with: fetch-depth: 0 @@ -31,6 +32,9 @@ jobs: ref: ${{github.head_ref}} fetch-depth: 0 + -name: Show github context + -run: cat $GITHUB_EVENT_PATH + - name: Set up Python 3.7 uses: actions/setup-python@v2 with: From 015a50d53f35838e85af5906d470af55bf6bbd0d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:18:05 +0100 Subject: [PATCH 166/258] Fix small typo "-name" -> "- name" --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index d85fddff..19c79ec0 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -32,7 +32,7 @@ jobs: ref: ${{github.head_ref}} fetch-depth: 0 - -name: Show github context + - name: Show github context -run: cat $GITHUB_EVENT_PATH - name: Set up Python 3.7 From a01c36f660e4502218e245c0153c8b37588cdd28 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:20:12 +0100 Subject: [PATCH 167/258] Correct "-run:" to "run:" --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 19c79ec0..a3556326 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 0 - name: Show github context - -run: cat $GITHUB_EVENT_PATH + run: cat $GITHUB_EVENT_PATH - name: Set up Python 3.7 uses: actions/setup-python@v2 From 92ea8943dfddc5866cc5eb2a3da80f612c2dc7d1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:21:32 +0100 Subject: [PATCH 168/258] Run 'Show github context' as first step --- .github/workflows/pr-checks.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index a3556326..c3de1a0d 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -16,25 +16,27 @@ jobs: runs-on: ubuntu-18.04 steps: + - name: Show github context + run: cat $GITHUB_EVENT_PATH + - name: Checkout commits # https://github.com/actions/checkout uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Checkout base ref uses: actions/checkout@v2 with: ref: ${{github.base_ref}} fetch-depth: 0 + - name: Checkout head of PR uses: actions/checkout@v2 with: ref: ${{github.head_ref}} fetch-depth: 0 - - name: Show github context - run: cat $GITHUB_EVENT_PATH - - name: Set up Python 3.7 uses: actions/setup-python@v2 with: From c5d43a4adaac31b2e9fe60d80b9993ccc43be106 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:34:12 +0100 Subject: [PATCH 169/258] Try a whole new method of referring to 'base', and checkout 'head' --- .github/workflows/pr-checks.yml | 36 ++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index c3de1a0d..3f75da38 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -15,28 +15,30 @@ jobs: flake8: runs-on: ubuntu-18.04 + #################################################################### + # Show full github context to aid debugging. + #################################################################### steps: - name: Show github context run: cat $GITHUB_EVENT_PATH + #################################################################### + #################################################################### + # Checkout the necessary commits + #################################################################### + # We need the commits from the 'head' of the PR, not what it's + # based on. - name: Checkout commits # https://github.com/actions/checkout uses: actions/checkout@v2 with: + repository: ${{github.head.repo.full_name}} fetch-depth: 0 + #################################################################### - - name: Checkout base ref - uses: actions/checkout@v2 - with: - ref: ${{github.base_ref}} - fetch-depth: 0 - - - name: Checkout head of PR - uses: actions/checkout@v2 - with: - ref: ${{github.head_ref}} - fetch-depth: 0 - + #################################################################### + # Get Python set up + #################################################################### - name: Set up Python 3.7 uses: actions/setup-python@v2 with: @@ -46,13 +48,19 @@ jobs: python -m pip install --upgrade pip pip install flake8 if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt ; fi + #################################################################### + #################################################################### + # Lint with flake8 + #################################################################### - name: Lint with flake8 env: + BASE_REPO: ${{github.base.full_name}} BASE_REF: ${{github.base_ref}} run: | # stop the build if there are Python syntax errors or undefined names, ignore existing - git diff "refs/remotes/origin/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff + git diff "refs/remotes/${BASE_REPO}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - git diff "refs/remotes/origin/${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff + git diff "refs/remotes/${BASE_REPO}/${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff + #################################################################### From 2fa10b4a9eb09a2a65dab277aa4b90493be3366e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:35:54 +0100 Subject: [PATCH 170/258] Correct setting of BASE_REPO --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 3f75da38..a71f3ee2 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -55,7 +55,7 @@ jobs: #################################################################### - name: Lint with flake8 env: - BASE_REPO: ${{github.base.full_name}} + BASE_REPO: ${{github.base.repo.full_name}} BASE_REF: ${{github.base_ref}} run: | From 7c00ddcebfe0e8335199545a08eb8f3498544112 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:40:38 +0100 Subject: [PATCH 171/258] We want github.context for the full info --- .github/workflows/pr-checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index a71f3ee2..1a3d672e 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -32,7 +32,7 @@ jobs: # https://github.com/actions/checkout uses: actions/checkout@v2 with: - repository: ${{github.head.repo.full_name}} + repository: ${{github.event.head.repo.full_name}} fetch-depth: 0 #################################################################### @@ -55,7 +55,7 @@ jobs: #################################################################### - name: Lint with flake8 env: - BASE_REPO: ${{github.base.repo.full_name}} + BASE_REPO: ${{github.event.base.repo.full_name}} BASE_REF: ${{github.base_ref}} run: | From e1a5a1f811f82cb9d547f42bf89a522b5f87812c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:51:07 +0100 Subject: [PATCH 172/258] Top-level event key needs to be pull_request --- .github/workflows/pr-checks.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 1a3d672e..01decfc1 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -20,7 +20,8 @@ jobs: #################################################################### steps: - name: Show github context - run: cat $GITHUB_EVENT_PATH + run: | + cat $GITHUB_EVENT_PATH #################################################################### #################################################################### @@ -32,7 +33,7 @@ jobs: # https://github.com/actions/checkout uses: actions/checkout@v2 with: - repository: ${{github.event.head.repo.full_name}} + repository: ${{github.event.pull_request.head.repo.full_name}} fetch-depth: 0 #################################################################### @@ -55,7 +56,7 @@ jobs: #################################################################### - name: Lint with flake8 env: - BASE_REPO: ${{github.event.base.repo.full_name}} + BASE_REPO: ${{github.event.pull_request.base.repo.full_name}} BASE_REF: ${{github.base_ref}} run: | From 62fe9deae7ce8839cfe03b4d1010c22e12777605 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 12:57:44 +0100 Subject: [PATCH 173/258] repo.owner.login not repo.full_name --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 01decfc1..179cc174 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -56,7 +56,7 @@ jobs: #################################################################### - name: Lint with flake8 env: - BASE_REPO: ${{github.event.pull_request.base.repo.full_name}} + BASE_REPO: ${{github.event.pull_request.base.repo.owner.login}} BASE_REF: ${{github.base_ref}} run: | From 895376c0392c033169880e57428ab18d846969a8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:05:20 +0100 Subject: [PATCH 174/258] Try manually adding remote and fetching to get the refs Also, add 'end' to 'Show github context' --- .github/workflows/pr-checks.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 179cc174..e908063b 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -21,6 +21,7 @@ jobs: steps: - name: Show github context run: | + env cat $GITHUB_EVENT_PATH #################################################################### @@ -56,12 +57,15 @@ jobs: #################################################################### - name: Lint with flake8 env: - BASE_REPO: ${{github.event.pull_request.base.repo.owner.login}} + BASE_REPO_URL: ${{github.event.pull_request.base.repo.url}} + BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} BASE_REF: ${{github.base_ref}} run: | + git remote add ${BASE_REPO_OWNER} ${BASE_REPO_URL} + git fetch ${BASE_REPO_OWNER} # stop the build if there are Python syntax errors or undefined names, ignore existing - git diff "refs/remotes/${BASE_REPO}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff + git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - git diff "refs/remotes/${BASE_REPO}/${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff + git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff #################################################################### From 3d46f23fc715236bf21d17ac5284e3d626b94f39 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:07:40 +0100 Subject: [PATCH 175/258] Need svn_url, not url 'url' is api.github.com --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index e908063b..78195036 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -57,7 +57,7 @@ jobs: #################################################################### - name: Lint with flake8 env: - BASE_REPO_URL: ${{github.event.pull_request.base.repo.url}} + BASE_REPO_URL: ${{github.event.pull_request.base.repo.svn_url}} BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} BASE_REF: ${{github.base_ref}} From 71fa1dbcf6315e27a077f94821e1616d99e61ca7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:15:52 +0100 Subject: [PATCH 176/258] Comment the 'add base remote and fetch it' --- .github/workflows/pr-checks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 78195036..8f6937e3 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -62,7 +62,11 @@ jobs: BASE_REF: ${{github.base_ref}} run: | + # We have checked out the 'head' repo, so we need the references + # for the 'base' repo in order to do the 'git diff' below + # So, add the 'base' repo as a new remote git remote add ${BASE_REPO_OWNER} ${BASE_REPO_URL} + # And then fetch its references git fetch ${BASE_REPO_OWNER} # stop the build if there are Python syntax errors or undefined names, ignore existing git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff From 063a83b8e85a5bdaf8c5016a622e818b39447d4a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:17:56 +0100 Subject: [PATCH 177/258] Move 'Show github context' to just before flake8 $GITHUB_EVENT_PATH wasn't set in the shell with it first. --- .github/workflows/pr-checks.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 8f6937e3..18af3222 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -15,16 +15,6 @@ jobs: flake8: runs-on: ubuntu-18.04 - #################################################################### - # Show full github context to aid debugging. - #################################################################### - steps: - - name: Show github context - run: | - env - cat $GITHUB_EVENT_PATH - #################################################################### - #################################################################### # Checkout the necessary commits #################################################################### @@ -52,6 +42,16 @@ jobs: if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt ; fi #################################################################### + #################################################################### + # Show full github context to aid debugging. + #################################################################### + steps: + - name: Show github context + run: | + env + cat $GITHUB_EVENT_PATH + #################################################################### + #################################################################### # Lint with flake8 #################################################################### From 8d7f91248d099d6b15aaa320bae4f037cc7be5c6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:18:46 +0100 Subject: [PATCH 178/258] Oops, moved the 'steps:' line too --- .github/workflows/pr-checks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 18af3222..8d66a6b9 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -15,6 +15,8 @@ jobs: flake8: runs-on: ubuntu-18.04 + steps: + #################################################################### # Checkout the necessary commits #################################################################### @@ -45,7 +47,6 @@ jobs: #################################################################### # Show full github context to aid debugging. #################################################################### - steps: - name: Show github context run: | env From 64e6ad439c0e338328248c27377a2d3518773273 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:28:28 +0100 Subject: [PATCH 179/258] Try remote-add/fetch in separate step --- .github/workflows/pr-checks.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 8d66a6b9..4e0a0a79 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -20,16 +20,28 @@ jobs: #################################################################### # Checkout the necessary commits #################################################################### - # We need the commits from the 'head' of the PR, not what it's + # We need the repo from the 'head' of the PR, not what it's # based on. - - name: Checkout commits + - name: Checkout head commits # https://github.com/actions/checkout uses: actions/checkout@v2 with: repository: ${{github.event.pull_request.head.repo.full_name}} fetch-depth: 0 - #################################################################### + # But we do need the base references + - name: Fetch base commits + env: + BASE_REPO_URL: ${{github.event.pull_request.base.repo.svn_url}} + BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} + + run: | + # Add the 'base' repo as a new remote + git remote add ${BASE_REPO_OWNER} ${BASE_REPO_URL} + # And then fetch its references + git fetch ${BASE_REPO_OWNER} + #################################################################### + #################################################################### # Get Python set up #################################################################### @@ -63,12 +75,6 @@ jobs: BASE_REF: ${{github.base_ref}} run: | - # We have checked out the 'head' repo, so we need the references - # for the 'base' repo in order to do the 'git diff' below - # So, add the 'base' repo as a new remote - git remote add ${BASE_REPO_OWNER} ${BASE_REPO_URL} - # And then fetch its references - git fetch ${BASE_REPO_OWNER} # stop the build if there are Python syntax errors or undefined names, ignore existing git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide From b75e2ec2b1344d3f3a7bc1c7bf5da0fc82378277 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:36:49 +0100 Subject: [PATCH 180/258] Comment about the check being wrong This gets the branch rolling for a PR --- .github/workflows/pr-annotate-with-flake8.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 54e91393..fe6f60ef 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout + # This check isn't correct, it only worked for branches within the + # repo. if: ${{ github.actor == 'Athanasius' }} uses: actions/checkout@v2 with: From 3972abbaceefcaa81952122fd72581eaf6035ae4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:42:00 +0100 Subject: [PATCH 181/258] Comment out the current steps --- .github/workflows/pr-annotate-with-flake8.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index fe6f60ef..775cd560 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -15,6 +15,8 @@ jobs: runs-on: ubuntu-18.04 steps: + ################################################################## + ################################################################## - name: Checkout # This check isn't correct, it only worked for branches within the # repo. @@ -22,7 +24,12 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + ################################################################## + ################################################################## + # flake8-your-pr 'fails' if no *.py files changed + # So we check if any were affected and store that in env + ################################################################## - name: Check for PY files env: BASE_REF: ${{github.base_ref}} @@ -33,16 +40,27 @@ jobs: # We don't rely on exit status because Workflows run with # "set -e" so any failure in a pipe is total failure. PYFILES=$(git diff --name-only "refs/remotes/origin/${BASE_REF}" -- | egrep '.py$' || true) + # Use magic output to store in env for rest of workflow echo "::set-env name=PYFILES::${PYFILES}" + ################################################################## + ################################################################## + # Get Python set up + ################################################################## - name: Set up Python 3.7 if: ${{ env.PYFILES != '' }} uses: actions/setup-python@v2 with: python-version: 3.7 + ################################################################## + ################################################################## + # Perform the annotation + ################################################################## - name: Annotate with Flake8 + # Only if at least one *.py file was affected if: ${{ env.PYFILES != '' }} uses: "tayfun/flake8-your-pr@master" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ################################################################## From 937dacd7cac2a6fe484b48522dd5fcd7905be125 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:47:40 +0100 Subject: [PATCH 182/258] Implement "same owner?" check for base/head * Also updates top comment. --- .github/workflows/pr-annotate-with-flake8.yml | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 775cd560..54f443b3 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -1,7 +1,9 @@ # This workflow will: # -# install Python dependencies -# Run flake8 to add annotations to the PR +# 1. Store some github context in env vars. +# 2. Only checkout if base and head ref owners are the same. +# 3. Check if any *.py files are in the diff. +# 4. Only then install python and perform the annotation with flake8. # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: PR-annotate-flake8 @@ -16,11 +18,21 @@ jobs: steps: ################################################################## + # Check if the base and head repos are the same, if not we won't + # be able to add annotations. + ################################################################## + - name: Check head and base repo same + env: + BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} + HEAD_REPO_OWNER: ${{github.event.pull_request.head.repo.owner.login}} + BASE_REF: ${{github.base_ref}} + ################################################################## + + ################################################################## + # Perform the checkout ################################################################## - name: Checkout - # This check isn't correct, it only worked for branches within the - # repo. - if: ${{ github.actor == 'Athanasius' }} + if: ${{ env.BASE_REPO_OWNER == env.HEAD_REPO_OWNER }} uses: actions/checkout@v2 with: fetch-depth: 0 @@ -31,8 +43,6 @@ jobs: # So we check if any were affected and store that in env ################################################################## - name: Check for PY files - env: - BASE_REF: ${{github.base_ref}} run: | # Checkout might not have happened. if [ ! -d ".git" ]; then exit 0 ; fi From 13529314659ae4689321abec8978ed27d5f18e11 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:50:27 +0100 Subject: [PATCH 183/258] Remove EDMCLogging.py comment to trigger diff --- EDMCLogging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 72e7c81c..15a29286 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -5,7 +5,6 @@ members on the logging.LogRecord instance for use in logging.Formatter() strings. """ -# A small comment # So that any warning about accessing a protected member is only in one place. from sys import _getframe as getframe import inspect From ecf4732e448895eea09398096bb4b2f7d1003474 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:51:50 +0100 Subject: [PATCH 184/258] Check step needs a 'uses' or 'run', so add latter with 'env' for check --- .github/workflows/pr-annotate-with-flake8.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 54f443b3..e9dbf478 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -26,6 +26,8 @@ jobs: BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} HEAD_REPO_OWNER: ${{github.event.pull_request.head.repo.owner.login}} BASE_REF: ${{github.base_ref}} + run: | + env ################################################################## ################################################################## From e087cd7f85b6014e7931703e5255e2021df884db Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:53:23 +0100 Subject: [PATCH 185/258] Comment that 'env' for another commit --- .github/workflows/pr-annotate-with-flake8.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index e9dbf478..dd1f5e51 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -27,6 +27,7 @@ jobs: HEAD_REPO_OWNER: ${{github.event.pull_request.head.repo.owner.login}} BASE_REF: ${{github.base_ref}} run: | + # Just to be running something env ################################################################## From 67a13d3869db33887fe705b8fadb77b11ad9e324 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:56:18 +0100 Subject: [PATCH 186/258] Change Checkout conditional to direct context, not env vars --- .github/workflows/pr-annotate-with-flake8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index dd1f5e51..732d19b5 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -35,7 +35,7 @@ jobs: # Perform the checkout ################################################################## - name: Checkout - if: ${{ env.BASE_REPO_OWNER == env.HEAD_REPO_OWNER }} + if: ${{ github.event.pull_request.base.repo.owner.login == github.event.pull_request.head.repo.owner.login }} uses: actions/checkout@v2 with: fetch-depth: 0 From 4aa841c6e211c4c7f84df4fe641a0b3454713965 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 13:59:52 +0100 Subject: [PATCH 187/258] Will PY file check work if env.PYFILES not set at all ? --- .github/workflows/pr-annotate-with-flake8.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-annotate-with-flake8.yml b/.github/workflows/pr-annotate-with-flake8.yml index 732d19b5..f293a9fc 100644 --- a/.github/workflows/pr-annotate-with-flake8.yml +++ b/.github/workflows/pr-annotate-with-flake8.yml @@ -46,6 +46,7 @@ jobs: # So we check if any were affected and store that in env ################################################################## - name: Check for PY files + if: ${{ github.event.pull_request.base.repo.owner.login == github.event.pull_request.head.repo.owner.login }} run: | # Checkout might not have happened. if [ ! -d ".git" ]; then exit 0 ; fi From 3dab0e9d914d70879d2ab8864c97a4721ffcf96e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 14:44:44 +0100 Subject: [PATCH 188/258] Comment pr-check flake8, and don't --exit-zero the full check --- .github/workflows/pr-checks.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 4e0a0a79..f39bb335 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -75,8 +75,13 @@ jobs: BASE_REF: ${{github.base_ref}} run: | - # stop the build if there are Python syntax errors or undefined names, ignore existing + # Explicitly check for some errors + # E9 - Runtime (syntax and the like) + # F63 - 'tests' checking + # F7 - syntax errors + # F82 - undefined checking git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --exit-zero --statistics --diff + # Can optionally add `--exit-zero` to the flake8 arguments so that + # this doesn't fail the build. + git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --statistics --diff #################################################################### From 898ff9fbb2548bb4aa55aa446ba911e20a451ac5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 15:33:55 +0100 Subject: [PATCH 189/258] Logging: Make correct loggers for 'found' plugins * Log messages propagate up Parent.Child chains, so we don't need a channel on the plugin logger. * But it still needs the filter to define qualname and class for formatting. --- EDMCLogging.py | 31 ++++++++++++++++++++++++++++++- plug.py | 16 ++++++++++++---- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 15a29286..7ad069a1 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -35,6 +35,8 @@ from typing import Tuple # # 14. Call from *package* +_default_loglevel = logging.DEBUG + class Logger: """ @@ -47,7 +49,7 @@ class Logger: Users of this class should then call getLogger() to get the logging.Logger instance. """ - def __init__(self, logger_name: str, loglevel: int = logging.DEBUG): + def __init__(self, logger_name: str, loglevel: int = _default_loglevel): """ Set up a `logging.Logger` with our preferred configuration. This includes using an EDMCContextFilter to add 'class' and 'qualname' @@ -78,6 +80,33 @@ class Logger: return self.logger +def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.Logger: + """ + 'Found' plugins need their own logger to call out where the logging is + coming from, but we don't need to set up *everything* for them. + + The name will be '{config.appname}.{plugin.name}', e.g. + 'EDMarketConnector.miggytest'. This means that any logging sent through + there *also* goes to the channels defined in the 'EDMarketConnector' + logger, so we can let that take care of the formatting. + + If we add our own channel then the output gets duplicated (assuming same + logLevel set). + + However we do need to attach our filter to this still. That's not at + the channel level. + :param name: Name of this Logger. + :param loglevel: Optional logLevel for this Logger. + :return: logging.Logger instance, all set up. + """ + plugin_logger = logging.getLogger(name) + plugin_logger.setLevel(loglevel) + + plugin_logger.addFilter(EDMCContextFilter()) + + return plugin_logger + + class EDMCContextFilter(logging.Filter): """ logging.Filter sub-class to place extra attributes of the calling site diff --git a/plug.py b/plug.py index f836d7cd..0a0a3f8a 100644 --- a/plug.py +++ b/plug.py @@ -8,12 +8,14 @@ import importlib import sys import operator import threading # noqa: F401 - We don't use it, but plugins might +from typing import Optional import logging import tkinter as tk import myNotebook as nb # noqa: N813 from config import config, appname +import EDMCLogging logger = logging.getLogger(appname) @@ -79,7 +81,7 @@ last_error = { class Plugin(object): - def __init__(self, name, loadfile): + def __init__(self, name: str, loadfile: str, plugin_logger: Optional[logging.Logger]): """ Load a single plugin :param name: module name @@ -90,6 +92,7 @@ class Plugin(object): self.name = name # Display name. self.folder = name # basename of plugin folder. None for internal plugins. self.module = None # None for disabled plugins. + self.logger = plugin_logger if loadfile: logger.info(f'loading plugin "{name.replace(".", "_")}" from "{loadfile}"') @@ -173,7 +176,7 @@ def load_plugins(master): for name in sorted(os.listdir(config.internal_plugin_dir)): if name.endswith('.py') and not name[0] in ['.', '_']: try: - plugin = Plugin(name[:-3], os.path.join(config.internal_plugin_dir, name)) + plugin = Plugin(name[:-3], os.path.join(config.internal_plugin_dir, name), logger) plugin.folder = None # Suppress listing in Plugins prefs tab internal.append(plugin) except Exception as e: @@ -191,12 +194,17 @@ def load_plugins(master): pass elif name.endswith('.disabled'): name, discard = name.rsplit('.', 1) - found.append(Plugin(name, None)) + found.append(Plugin(name, None, logger)) else: try: # Add plugin's folder to load path in case plugin has internal package dependencies sys.path.append(os.path.join(config.plugin_dir, name)) - found.append(Plugin(name, os.path.join(config.plugin_dir, name, 'load.py'))) + + # Create a logger for this 'found' plugin. Must be before the + # load.py is loaded. + plugin_logger = EDMCLogging.get_plugin_logger(f'{appname}.{name}') + + found.append(Plugin(name, os.path.join(config.plugin_dir, name, 'load.py'), plugin_logger)) except Exception as e: logger.exception(f'Failure loading found Plugin "{name}"') pass From 04c4f5e6834907811e242c682119e4479eafdad7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 16:29:43 +0100 Subject: [PATCH 190/258] Using a LoggerAdapter to prepend a string. 1. This makes setting up logging everywhere slightly more involved. 2. If I then want to change, say, %(module)s value I'll end up needing to stack walk again. So this might be better done in a filter. But these commits for the record, and to come back to if needs be. --- EDMCLogging.py | 24 ++++++++++++++++-------- EDMarketConnector.py | 2 +- plug.py | 2 +- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 7ad069a1..ecb96050 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -9,7 +9,7 @@ strings. from sys import _getframe as getframe import inspect import logging -from typing import Tuple +from typing import Any, MutableMapping, Tuple # TODO: Tests: @@ -55,13 +55,14 @@ class Logger: This includes using an EDMCContextFilter to add 'class' and 'qualname' expansions for logging.Formatter(). """ - self.logger = logging.getLogger(logger_name) + # Using a LoggerAdapter, so make the actual Logger internal + self._logger = logging.getLogger(logger_name) # Configure the logging.Logger - self.logger.setLevel(loglevel) + self._logger.setLevel(loglevel) # Set up filter for adding class name self.logger_filter = EDMCContextFilter() - self.logger.addFilter(self.logger_filter) + self._logger.addFilter(self.logger_filter) self.logger_channel = logging.StreamHandler() self.logger_channel.setLevel(loglevel) @@ -71,16 +72,18 @@ class Logger: self.logger_formatter.default_msec_format = '%s.%03d' self.logger_channel.setFormatter(self.logger_formatter) - self.logger.addHandler(self.logger_channel) + self._logger.addHandler(self.logger_channel) - def get_logger(self) -> logging.Logger: + self.logger = EDMCLoggerAdapter(self._logger, {'from': self.__class__.__qualname__}) + + def get_logger(self) -> logging.LoggerAdapter: """ :return: The logging.Logger instance. """ return self.logger -def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.Logger: +def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.LoggerAdapter: """ 'Found' plugins need their own logger to call out where the logging is coming from, but we don't need to set up *everything* for them. @@ -104,7 +107,7 @@ def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.L plugin_logger.addFilter(EDMCContextFilter()) - return plugin_logger + return EDMCLoggerAdapter(plugin_logger, {'from': __name__}) class EDMCContextFilter(logging.Filter): @@ -212,3 +215,8 @@ class EDMCContextFilter(logging.Filter): caller_class_names = '' return caller_class_names, caller_qualname + + +class EDMCLoggerAdapter(logging.LoggerAdapter): + def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> Tuple[Any, MutableMapping[str, Any]]: + return f'ADAPTED {msg}', kwargs \ No newline at end of file diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 2d1b81b9..c1005a80 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1033,7 +1033,7 @@ if __name__ == "__main__": logger = EDMCLogging.Logger(appname).get_logger() # TODO: unittests in place of these - # logger.debug('Test from __main__') + logger.debug('Test from __main__') # test_logging() class A(object): class B(object): diff --git a/plug.py b/plug.py index 0a0a3f8a..8731a173 100644 --- a/plug.py +++ b/plug.py @@ -17,7 +17,7 @@ import myNotebook as nb # noqa: N813 from config import config, appname import EDMCLogging -logger = logging.getLogger(appname) +logger = EDMCLogging.EDMCLoggerAdapter(logging.getLogger(appname), {'from': __name__}) # Dashboard Flags constants FlagsDocked = 1 << 0 # on a landing pad From e5723957189a8c0fa8c987c5db459533bad62d91 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 16:34:02 +0100 Subject: [PATCH 191/258] LoggerAdapter can't change anything but %(message)s --- EDMCLogging.py | 24 ++++++++---------------- EDMarketConnector.py | 2 +- plug.py | 2 +- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index ecb96050..7ad069a1 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -9,7 +9,7 @@ strings. from sys import _getframe as getframe import inspect import logging -from typing import Any, MutableMapping, Tuple +from typing import Tuple # TODO: Tests: @@ -55,14 +55,13 @@ class Logger: This includes using an EDMCContextFilter to add 'class' and 'qualname' expansions for logging.Formatter(). """ - # Using a LoggerAdapter, so make the actual Logger internal - self._logger = logging.getLogger(logger_name) + self.logger = logging.getLogger(logger_name) # Configure the logging.Logger - self._logger.setLevel(loglevel) + self.logger.setLevel(loglevel) # Set up filter for adding class name self.logger_filter = EDMCContextFilter() - self._logger.addFilter(self.logger_filter) + self.logger.addFilter(self.logger_filter) self.logger_channel = logging.StreamHandler() self.logger_channel.setLevel(loglevel) @@ -72,18 +71,16 @@ class Logger: self.logger_formatter.default_msec_format = '%s.%03d' self.logger_channel.setFormatter(self.logger_formatter) - self._logger.addHandler(self.logger_channel) + self.logger.addHandler(self.logger_channel) - self.logger = EDMCLoggerAdapter(self._logger, {'from': self.__class__.__qualname__}) - - def get_logger(self) -> logging.LoggerAdapter: + def get_logger(self) -> logging.Logger: """ :return: The logging.Logger instance. """ return self.logger -def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.LoggerAdapter: +def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.Logger: """ 'Found' plugins need their own logger to call out where the logging is coming from, but we don't need to set up *everything* for them. @@ -107,7 +104,7 @@ def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.L plugin_logger.addFilter(EDMCContextFilter()) - return EDMCLoggerAdapter(plugin_logger, {'from': __name__}) + return plugin_logger class EDMCContextFilter(logging.Filter): @@ -215,8 +212,3 @@ class EDMCContextFilter(logging.Filter): caller_class_names = '' return caller_class_names, caller_qualname - - -class EDMCLoggerAdapter(logging.LoggerAdapter): - def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> Tuple[Any, MutableMapping[str, Any]]: - return f'ADAPTED {msg}', kwargs \ No newline at end of file diff --git a/EDMarketConnector.py b/EDMarketConnector.py index c1005a80..2d1b81b9 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1033,7 +1033,7 @@ if __name__ == "__main__": logger = EDMCLogging.Logger(appname).get_logger() # TODO: unittests in place of these - logger.debug('Test from __main__') + # logger.debug('Test from __main__') # test_logging() class A(object): class B(object): diff --git a/plug.py b/plug.py index 8731a173..0a0a3f8a 100644 --- a/plug.py +++ b/plug.py @@ -17,7 +17,7 @@ import myNotebook as nb # noqa: N813 from config import config, appname import EDMCLogging -logger = EDMCLogging.EDMCLoggerAdapter(logging.getLogger(appname), {'from': __name__}) +logger = logging.getLogger(appname) # Dashboard Flags constants FlagsDocked = 1 << 0 # on a landing pad From e3b3f1e5b5b5449da93415f6e5c46ce3fdac9fb2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 17:37:31 +0100 Subject: [PATCH 192/258] Prepend some useful information to %(module)s if caller is a plugin NB: This assumes only one level within the plugin folder, TODO to fix that. --- EDMCLogging.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 7ad069a1..0ecdbe96 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -9,8 +9,10 @@ strings. from sys import _getframe as getframe import inspect import logging +import pathlib from typing import Tuple +from config import config # TODO: Tests: # @@ -131,10 +133,11 @@ class EDMCContextFilter(logging.Filter): :param record: The LogRecord we're "filtering" :return: bool - Always true in order for this record to be logged. """ - class_name = qualname = '' - # Don't even call in if both already set. - if not getattr(record, 'class', None) or not getattr(record, 'qualname', None): - (class_name, qualname) = self.caller_class_and_qualname() + (class_name, qualname, module_name) = self.caller_attributes(module_name=getattr(record, 'module')) + + # Only set if we got a useful value + if module_name: + setattr(record, 'module', module_name) # Only set if not already provided by logging itself if getattr(record, 'class', None) is None: @@ -147,13 +150,13 @@ class EDMCContextFilter(logging.Filter): return True @classmethod - def caller_class_and_qualname(cls) -> Tuple[str, str]: + def caller_attributes(cls, module_name: str = ''): """ Figure out our caller's class name(s) and qualname Ref: - :return: Tuple[str, str]: The caller's class name(s) and qualname + :return: Tuple[str, str, str]: The caller's class name(s), qualname and module_name """ # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start @@ -200,6 +203,15 @@ class EDMCContextFilter(logging.Filter): caller_class_names = '' caller_qualname = frame_info.function + # TODO: This assumes caller is only one level into the plugin folder + # Need to walk back up checking. + # Is this a 'found' plugin calling us? + file_name = pathlib.Path(frame_info.filename).expanduser() + plugin_dir = pathlib.Path(config.plugin_dir).expanduser() + if file_name.parent.parent == plugin_dir: + # Pre-pend 'plugins..' to module + module_name = f'.{file_name.parent.name}.{module_name}' + # https://docs.python.org/3.7/library/inspect.html#the-interpreter-stack del frame @@ -211,4 +223,4 @@ class EDMCContextFilter(logging.Filter): print('ALERT! Something went wrong with finding caller class name(s) for logging!') caller_class_names = '' - return caller_class_names, caller_qualname + return caller_class_names, caller_qualname, module_name From beea4ef39b7d06af43f7a0167b547bea6cabd0ec Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 17:38:09 +0100 Subject: [PATCH 193/258] Remove %(name)s from logging format as un-necessary. --- EDMCLogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 0ecdbe96..4143edff 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -68,7 +68,7 @@ class Logger: self.logger_channel = logging.StreamHandler() self.logger_channel.setLevel(loglevel) - self.logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(class)s: %(message)s') # noqa: E501 + self.logger_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(class)s: %(message)s') # noqa: E501 self.logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' self.logger_formatter.default_msec_format = '%s.%03d' From ad021e0765d8591ea09f037fc0486ad87523f2c7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 17:39:16 +0100 Subject: [PATCH 194/258] Remove %(class)s from logging format as un-necessary. It was only there to test the code populating it. --- EDMCLogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 4143edff..8bba3f13 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -68,7 +68,7 @@ class Logger: self.logger_channel = logging.StreamHandler() self.logger_channel.setLevel(loglevel) - self.logger_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(class)s: %(message)s') # noqa: E501 + self.logger_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(module)s.%(qualname)s:%(lineno)d: %(message)s') # noqa: E501 self.logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' self.logger_formatter.default_msec_format = '%s.%03d' From 89f2726e48dbc44bfa857c072a6dd5098d3d7281 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 18:04:03 +0100 Subject: [PATCH 195/258] Set internal and found plugin module_name separately. Settled on `plugins..found` as the format. --- EDMCLogging.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/EDMCLogging.py b/EDMCLogging.py index 8bba3f13..41f06044 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -208,10 +208,15 @@ class EDMCContextFilter(logging.Filter): # Is this a 'found' plugin calling us? file_name = pathlib.Path(frame_info.filename).expanduser() plugin_dir = pathlib.Path(config.plugin_dir).expanduser() + internal_plugin_dir = pathlib.Path(config.internal_plugin_dir).expanduser() + if file_name.parent.parent == plugin_dir: # Pre-pend 'plugins..' to module module_name = f'.{file_name.parent.name}.{module_name}' + elif file_name.parent == internal_plugin_dir: + module_name = f'plugins.{module_name}' + # https://docs.python.org/3.7/library/inspect.html#the-interpreter-stack del frame From 657253b3e35aa5bfd8abfd141a5e31d38c513d0c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 19:29:12 +0100 Subject: [PATCH 196/258] Set internal and found plugin module_name separately. * Settled on `plugins.internal` and `.found` as the format. * A PyCharm recommendation was to use 'cls' instead of 'self' on class methods, so the class detection code needs to cater for that. Technically a developer could use any string for the "myself" member name, but we'll assume just these two. * Found will always have at least one folder level within plugin_dir * Internal should always have *no* folder within internal_plugin_dir, but cater for it just in case in future. --- EDMCLogging.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index 41f06044..b6cdefe8 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -181,8 +181,8 @@ class EDMCContextFilter(logging.Filter): # frame_info = inspect.getframeinfo(frame) args, _, _, value_dict = inspect.getargvalues(frame) - if len(args) and args[0] == 'self': - frame_class = value_dict['self'] + if len(args) and args[0] in ('self', 'cls'): + frame_class = value_dict[args[0]] if frame_class: # Find __qualname__ of the caller @@ -203,6 +203,9 @@ class EDMCContextFilter(logging.Filter): caller_class_names = '' caller_qualname = frame_info.function + ################################################################### + # Fixups for EDMC plugins + ################################################################### # TODO: This assumes caller is only one level into the plugin folder # Need to walk back up checking. # Is this a 'found' plugin calling us? @@ -210,12 +213,36 @@ class EDMCContextFilter(logging.Filter): plugin_dir = pathlib.Path(config.plugin_dir).expanduser() internal_plugin_dir = pathlib.Path(config.internal_plugin_dir).expanduser() - if file_name.parent.parent == plugin_dir: - # Pre-pend 'plugins..' to module - module_name = f'.{file_name.parent.name}.{module_name}' + # Find the first parent called 'plugins' + plugin_top = file_name + while plugin_top and plugin_top.name != '': + if plugin_top.parent.name == 'plugins': + break - elif file_name.parent == internal_plugin_dir: - module_name = f'plugins.{module_name}' + plugin_top = plugin_top.parent + + # Check we didn't walk up to the root/anchor + if plugin_top.name != '': + # And this check means we must still be inside config.app_dir + if plugin_top.parent == plugin_dir: + # In case of deeper callers we need a range of the file_name + pt_len = len(plugin_top.parts) + name_path = '.'.join(file_name.parts[(pt_len - 1):-1]) + module_name = f'.{name_path}.{module_name}' + + # Or in this case the installation folder + elif file_name.parent == internal_plugin_dir: + # Is this a deeper caller ? + pt_len = len(plugin_top.parts) + name_path = '.'.join(file_name.parts[(pt_len - 1):-1]) + # Pre-pend 'plugins..' to module + if name_path == '': + # No sub-folder involved so module_name is sufficient + module_name = f'plugins.{module_name}' + else: + # Sub-folder(s) involved, so include them + module_name = f'plugins.{name_path}.{module_name}' + ################################################################### # https://docs.python.org/3.7/library/inspect.html#the-interpreter-stack del frame From 282e3ddbc52b6b0bac608046c5c5c8e1de451fd3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 30 Jul 2020 23:59:10 +0100 Subject: [PATCH 197/258] Fix up and expand on docstrings. --- EDMCLogging.py | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index b6cdefe8..c2445c09 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -1,4 +1,6 @@ """ +Set up required logging for the application. + This module provides for a common logging-powered log facility. Mostly it implements a logging.Filter() in order to get two extra members on the logging.LogRecord instance for use in logging.Formatter() @@ -51,9 +53,11 @@ class Logger: Users of this class should then call getLogger() to get the logging.Logger instance. """ + def __init__(self, logger_name: str, loglevel: int = _default_loglevel): """ Set up a `logging.Logger` with our preferred configuration. + This includes using an EDMCContextFilter to add 'class' and 'qualname' expansions for logging.Formatter(). """ @@ -77,13 +81,17 @@ class Logger: def get_logger(self) -> logging.Logger: """ - :return: The logging.Logger instance. + Obtain the self.logger of the class instance. + + Not to be confused with logging.getLogger(). """ return self.logger def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.Logger: """ + Return a logger suitable for a plugin. + 'Found' plugins need their own logger to call out where the logging is coming from, but we don't need to set up *everything* for them. @@ -111,24 +119,27 @@ def get_plugin_logger(name: str, loglevel: int = _default_loglevel) -> logging.L class EDMCContextFilter(logging.Filter): """ + Implements filtering to add extra format specifiers, and tweak others. + logging.Filter sub-class to place extra attributes of the calling site into the record. """ + def filter(self, record: logging.LogRecord) -> bool: """ - Attempt to set the following in the LogRecord: + Attempt to set/change fields in the LogRecord. - 1. class = class name(s) of the call site, if applicable - 2. qualname = __qualname__ of the call site. This simplifies - logging.Formatter() as you can use just this no matter if there is - a class involved or not, so you get a nice clean: - .[.classB....]. + 1. class = class name(s) of the call site, if applicable + 2. qualname = __qualname__ of the call site. This simplifies + logging.Formatter() as you can use just this no matter if there is + a class involved or not, so you get a nice clean: + .[.classB....]. If we fail to be able to properly set either then: - 1. Use print() to alert, to be SURE a message is seen. - 2. But also return strings noting the error, so there'll be - something in the log output if it happens. + 1. Use print() to alert, to be SURE a message is seen. + 2. But also return strings noting the error, so there'll be + something in the log output if it happens. :param record: The LogRecord we're "filtering" :return: bool - Always true in order for this record to be logged. @@ -150,13 +161,15 @@ class EDMCContextFilter(logging.Filter): return True @classmethod - def caller_attributes(cls, module_name: str = ''): + def caller_attributes(cls, module_name: str = '') -> Tuple[str, str, str]: """ - Figure out our caller's class name(s) and qualname + Determine extra or changed fields for the caller. - Ref: - - :return: Tuple[str, str, str]: The caller's class name(s), qualname and module_name + 1. qualname finds the relevant object and its __qualname__ + 2. caller_class_names is just the full class names of the calling + class if relevant. + 3. module is munged if we detect the caller is an EDMC plugin, + whether internal or found. """ # Go up through stack frames until we find the first with a # type(f_locals.self) of logging.Logger. This should be the start @@ -206,8 +219,7 @@ class EDMCContextFilter(logging.Filter): ################################################################### # Fixups for EDMC plugins ################################################################### - # TODO: This assumes caller is only one level into the plugin folder - # Need to walk back up checking. + # TODO: Refactor this bit out into another function # Is this a 'found' plugin calling us? file_name = pathlib.Path(frame_info.filename).expanduser() plugin_dir = pathlib.Path(config.plugin_dir).expanduser() From efe63ceac97a1e0726a76e32575c9c235ec489bb Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 11:20:14 +0100 Subject: [PATCH 198/258] Refactor caller_attributes() code into separate methods * find_caller_frame() to do the frame walk. * munge_module_name() to fix up for plugins. NB: caller_attributes() now has a noqa on CCR001 as I don't think it can sensibly be made any less complex. Pulling out the 'if frame:' section just results in *that* new method then being labelled as too complex.:244 --- EDMCLogging.py | 137 +++++++++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 57 deletions(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index c2445c09..c8710cd0 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -160,7 +160,7 @@ class EDMCContextFilter(logging.Filter): return True - @classmethod + @classmethod # noqa: CCR001 - this is as refactored as is sensible def caller_attributes(cls, module_name: str = '') -> Tuple[str, str, str]: """ Determine extra or changed fields for the caller. @@ -171,23 +171,7 @@ class EDMCContextFilter(logging.Filter): 3. module is munged if we detect the caller is an EDMC plugin, whether internal or found. """ - # Go up through stack frames until we find the first with a - # type(f_locals.self) of logging.Logger. This should be the start - # of the frames internal to logging. - frame: 'frame' = getframe(0) - while frame: - if isinstance(frame.f_locals.get('self'), logging.Logger): - frame = frame.f_back # Want to start on the next frame below - break - frame = frame.f_back - - # Now continue up through frames until we find the next one where - # that is *not* true, as it should be the call site of the logger - # call - while frame: - if not isinstance(frame.f_locals.get('self'), logging.Logger): - break # We've found the frame we want - frame = frame.f_back + frame = cls.find_caller_frame() caller_qualname = caller_class_names = '' if frame: @@ -216,45 +200,7 @@ class EDMCContextFilter(logging.Filter): caller_class_names = '' caller_qualname = frame_info.function - ################################################################### - # Fixups for EDMC plugins - ################################################################### - # TODO: Refactor this bit out into another function - # Is this a 'found' plugin calling us? - file_name = pathlib.Path(frame_info.filename).expanduser() - plugin_dir = pathlib.Path(config.plugin_dir).expanduser() - internal_plugin_dir = pathlib.Path(config.internal_plugin_dir).expanduser() - - # Find the first parent called 'plugins' - plugin_top = file_name - while plugin_top and plugin_top.name != '': - if plugin_top.parent.name == 'plugins': - break - - plugin_top = plugin_top.parent - - # Check we didn't walk up to the root/anchor - if plugin_top.name != '': - # And this check means we must still be inside config.app_dir - if plugin_top.parent == plugin_dir: - # In case of deeper callers we need a range of the file_name - pt_len = len(plugin_top.parts) - name_path = '.'.join(file_name.parts[(pt_len - 1):-1]) - module_name = f'.{name_path}.{module_name}' - - # Or in this case the installation folder - elif file_name.parent == internal_plugin_dir: - # Is this a deeper caller ? - pt_len = len(plugin_top.parts) - name_path = '.'.join(file_name.parts[(pt_len - 1):-1]) - # Pre-pend 'plugins..' to module - if name_path == '': - # No sub-folder involved so module_name is sufficient - module_name = f'plugins.{module_name}' - else: - # Sub-folder(s) involved, so include them - module_name = f'plugins.{name_path}.{module_name}' - ################################################################### + module_name = cls.munge_module_name(frame_info, module_name) # https://docs.python.org/3.7/library/inspect.html#the-interpreter-stack del frame @@ -268,3 +214,80 @@ class EDMCContextFilter(logging.Filter): caller_class_names = '' return caller_class_names, caller_qualname, module_name + + @classmethod + def find_caller_frame(cls): + """ + Find the stack frame of the logging caller. + + :returns: 'frame' object such as from sys._getframe() + """ + # Go up through stack frames until we find the first with a + # type(f_locals.self) of logging.Logger. This should be the start + # of the frames internal to logging. + frame: 'frame' = getframe(0) + while frame: + if isinstance(frame.f_locals.get('self'), logging.Logger): + frame = frame.f_back # Want to start on the next frame below + break + frame = frame.f_back + # Now continue up through frames until we find the next one where + # that is *not* true, as it should be the call site of the logger + # call + while frame: + if not isinstance(frame.f_locals.get('self'), logging.Logger): + break # We've found the frame we want + frame = frame.f_back + return frame + + @classmethod + def munge_module_name(cls, frame_info: inspect.FrameInfo, module_name: str) -> str: + """ + Adjust module_name based on the file path for the given frame. + + We want to distinguish between other code and both our internal plugins + and the 'found' ones. + + For internal plugins we want "plugins.". + For 'found' plugins we want "....". + + :param frame_info: The frame_info of the caller. + :param module_name: The module_name string to munge. + :return: The munged module_name. + """ + file_name = pathlib.Path(frame_info.filename).expanduser() + plugin_dir = pathlib.Path(config.plugin_dir).expanduser() + internal_plugin_dir = pathlib.Path(config.internal_plugin_dir).expanduser() + # Find the first parent called 'plugins' + plugin_top = file_name + while plugin_top and plugin_top.name != '': + if plugin_top.parent.name == 'plugins': + break + + plugin_top = plugin_top.parent + + # Check we didn't walk up to the root/anchor + if plugin_top.name != '': + # Check we're still inside config.plugin_dir + if plugin_top.parent == plugin_dir: + # In case of deeper callers we need a range of the file_name + pt_len = len(plugin_top.parts) + name_path = '.'.join(file_name.parts[(pt_len - 1):-1]) + module_name = f'.{name_path}.{module_name}' + + # Check we're still inside the installation folder. + elif file_name.parent == internal_plugin_dir: + # Is this a deeper caller ? + pt_len = len(plugin_top.parts) + name_path = '.'.join(file_name.parts[(pt_len - 1):-1]) + + # Pre-pend 'plugins..' to module + if name_path == '': + # No sub-folder involved so module_name is sufficient + module_name = f'plugins.{module_name}' + + else: + # Sub-folder(s) involved, so include them + module_name = f'plugins.{name_path}.{module_name}' + + return module_name From 3a31139cbdad517be554437cf5daf8bb19a11838 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 31 Jul 2020 11:41:23 +0100 Subject: [PATCH 199/258] Correct frame_info type --- EDMCLogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMCLogging.py b/EDMCLogging.py index c8710cd0..81487d8a 100644 --- a/EDMCLogging.py +++ b/EDMCLogging.py @@ -241,7 +241,7 @@ class EDMCContextFilter(logging.Filter): return frame @classmethod - def munge_module_name(cls, frame_info: inspect.FrameInfo, module_name: str) -> str: + def munge_module_name(cls, frame_info: inspect.Traceback, module_name: str) -> str: """ Adjust module_name based on the file path for the given frame. From eec4c2ebd6f47ce04d9f2b1d13ee9405f703a4df Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 12:51:00 +0200 Subject: [PATCH 200/258] autoformat with autopep8 --- plugins/inara.py | 177 ++++++++++++++++++++++++++--------------------- 1 file changed, 99 insertions(+), 78 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 57d8d58e..0ad4f73f 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -23,32 +23,32 @@ logger = logging.getLogger(appname) _TIMEOUT = 20 -FAKE = ['CQC', 'Training', 'Destination'] # Fake systems that shouldn't be sent to Inara +FAKE = ['CQC', 'Training', 'Destination'] # Fake systems that shouldn't be sent to Inara CREDIT_RATIO = 1.05 # Update credits if they change by 5% over the course of a session -this: Any = sys.modules[__name__] # For holding module globals +this: Any = sys.modules[__name__] # For holding module globals this.session = requests.Session() -this.queue = Queue() # Items to be sent to Inara by worker thread -this.lastlocation = None # eventData from the last Commander's Flight Log event -this.lastship = None # eventData from the last addCommanderShip or setCommanderShip event +this.queue = Queue() # Items to be sent to Inara by worker thread +this.lastlocation = None # eventData from the last Commander's Flight Log event +this.lastship = None # eventData from the last addCommanderShip or setCommanderShip event # Cached Cmdr state -this.events = [] # Unsent events +this.events = [] # Unsent events this.cmdr = None this.FID = None # Frontier ID -this.multicrew = False # don't send captain's ship info to Inara while on a crew -this.newuser = False # just entered API Key - send state immediately -this.newsession = True # starting a new session - wait for Cargo event -this.undocked = False # just undocked -this.suppress_docked = False # Skip initial Docked event if started docked +this.multicrew = False # don't send captain's ship info to Inara while on a crew +this.newuser = False # just entered API Key - send state immediately +this.newsession = True # starting a new session - wait for Cargo event +this.undocked = False # just undocked +this.suppress_docked = False # Skip initial Docked event if started docked this.cargo = None this.materials = None -this.lastcredits = 0 # Send credit update soon after Startup / new game +this.lastcredits = 0 # Send credit update soon after Startup / new game this.storedmodules = None this.loadout = None this.fleet = None -this.shipswap = False # just swapped ship +this.shipswap = False # just swapped ship # last time we updated, if unset in config this is 0, which means an instant update LAST_UPDATE_CONF_KEY = 'inara_last_update' @@ -67,6 +67,7 @@ this.station = None this.station_marketid = None STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00D7 + def system_url(system_name): if this.system_address: return requests.utils.requote_uri(f'https://inara.cz/galaxy-starsystem/?search={this.system_address}') @@ -76,6 +77,7 @@ def system_url(system_name): return this.system + def station_url(system_name, station_name): if system_name: if station_name: @@ -86,7 +88,7 @@ def station_url(system_name, station_name): def plugin_start3(plugin_dir): - this.thread = Thread(target = worker, name = 'Inara worker') + this.thread = Thread(target=worker, name='Inara worker') this.thread.daemon = True this.thread.start() @@ -95,12 +97,14 @@ def plugin_start3(plugin_dir): this.timer_thread.start() return 'Inara' + def plugin_app(parent): - this.system_link = parent.children['system'] # system label in main window - this.station_link = parent.children['station'] # station label in main window + this.system_link = parent.children['system'] # system label in main window + this.station_link = parent.children['station'] # station label in main window this.system_link.bind_all('<>', update_location) this.system_link.bind_all('<>', update_ship) + def plugin_stop(): # Send any unsent events call() @@ -112,25 +116,25 @@ def plugin_stop(): this.timer_run = False -def plugin_prefs(parent, cmdr, is_beta): +def plugin_prefs(parent, cmdr, is_beta): PADX = 10 - BUTTONX = 12 # indent Checkbuttons and Radiobuttons + BUTTONX = 12 # indent Checkbuttons and Radiobuttons PADY = 2 # close spacing frame = nb.Frame(parent) frame.columnconfigure(1, weight=1) - HyperlinkLabel(frame, text='Inara', background=nb.Label().cget('background'), url='https://inara.cz/', underline=True).grid(columnspan=2, padx=PADX, sticky=tk.W) # Don't translate + HyperlinkLabel(frame, text='Inara', background=nb.Label().cget('background'), url='https://inara.cz/', underline=True).grid(columnspan=2, padx=PADX, sticky=tk.W) # Don't translate this.log = tk.IntVar(value = config.getint('inara_out') and 1) this.log_button = nb.Checkbutton(frame, text=_('Send flight log and Cmdr status to Inara'), variable=this.log, command=prefsvarchanged) this.log_button.grid(columnspan=2, padx=BUTTONX, pady=(5,0), sticky=tk.W) nb.Label(frame).grid(sticky=tk.W) # big spacer - this.label = HyperlinkLabel(frame, text=_('Inara credentials'), background=nb.Label().cget('background'), url='https://inara.cz/settings-api', underline=True) # Section heading in settings + this.label = HyperlinkLabel(frame, text=_('Inara credentials'), background=nb.Label().cget('background'), url='https://inara.cz/settings-api', underline=True) # Section heading in settings this.label.grid(columnspan=2, padx=PADX, sticky=tk.W) - this.apikey_label = nb.Label(frame, text=_('API Key')) # EDSM setting + this.apikey_label = nb.Label(frame, text=_('API Key')) # EDSM setting this.apikey_label.grid(row=12, padx=PADX, sticky=tk.W) this.apikey = nb.Entry(frame) this.apikey.grid(row=12, column=1, padx=PADX, pady=PADY, sticky=tk.EW) @@ -139,6 +143,7 @@ def plugin_prefs(parent, cmdr, is_beta): return frame + def prefs_cmdr_changed(cmdr, is_beta): this.log_button['state'] = cmdr and not is_beta and tk.NORMAL or tk.DISABLED this.apikey['state'] = tk.NORMAL @@ -149,9 +154,11 @@ def prefs_cmdr_changed(cmdr, is_beta): this.apikey.insert(0, cred) this.label['state'] = this.apikey_label['state'] = this.apikey['state'] = cmdr and not is_beta and this.log.get() and tk.NORMAL or tk.DISABLED + def prefsvarchanged(): this.label['state'] = this.apikey_label['state'] = this.apikey['state'] = this.log.get() and this.log_button['state'] or tk.DISABLED + def prefs_changed(cmdr, is_beta): changed = config.getint('inara_out') != this.log.get() config.set('inara_out', this.log.get()) @@ -179,10 +186,11 @@ def prefs_changed(cmdr, is_beta): config.set('inara_apikeys', apikeys) if this.log.get() and changed: - this.newuser = True # Send basic info at next Journal event - add_event('getCommanderProfile', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), { 'searchName': cmdr }) + this.newuser = True # Send basic info at next Journal event + add_event('getCommanderProfile', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), {'searchName': cmdr}) call() + def credentials(cmdr): # Credentials for cmdr if not cmdr: @@ -260,7 +268,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): if (this.newuser or entry['event'] == 'StartUp' or - (this.newsession and entry['event'] == 'Cargo')): + (this.newsession and entry['event'] == 'Cargo')): this.newuser = False this.newsession = False @@ -271,23 +279,23 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ('rankName', k.lower()), ('rankValue', v[0]), ('rankProgress', v[1] / 100.0), - ]) for k,v in state['Rank'].items() if v is not None - ]) + ]) for k, v in state['Rank'].items() if v is not None + ]) add_event('setCommanderReputationMajorFaction', entry['timestamp'], [ OrderedDict([ ('majorfactionName', k.lower()), ('majorfactionReputation', v / 100.0), - ]) for k,v in state['Reputation'].items() if v is not None - ]) - if state['Engineers']: # Not populated < 3.3 + ]) for k, v in state['Reputation'].items() if v is not None + ]) + if state['Engineers']: # Not populated < 3.3 add_event('setCommanderRankEngineer', entry['timestamp'], [ OrderedDict([ ('engineerName', k), type(v) is tuple and ('rankValue', v[0]) or ('rankStage', v), - ]) for k,v in state['Engineers'].items() - ]) + ]) for k, v in state['Engineers'].items() + ]) # Update location add_event('setCommanderTravelLocation', entry['timestamp'], @@ -297,12 +305,12 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ])) # Update ship - if state['ShipID']: # Unknown if started in Fighter or SRV + if state['ShipID']: # Unknown if started in Fighter or SRV data = OrderedDict([ ('shipType', state['ShipType']), ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None + ('shipName', state['ShipName']), # Can be None + ('shipIdent', state['ShipIdent']), # Can be None ('isCurrentShip', True), ]) if state['HullValue']: @@ -314,13 +322,12 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.loadout = make_loadout(state) add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) - - call() # Call here just to be sure that if we can send, we do, otherwise it'll get it in the next tick + call() # Call here just to be sure that if we can send, we do, otherwise it'll get it in the next tick # Promotions elif entry['event'] == 'Promotion': - for k,v in state['Rank'].items(): + for k, v in state['Rank'].items(): if k in entry: add_event('setCommanderRankPilot', entry['timestamp'], OrderedDict([ @@ -360,8 +367,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): data = OrderedDict([ ('shipType', state['ShipType']), ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None + ('shipName', state['ShipName']), # Can be None + ('shipIdent', state['ShipIdent']), # Can be None ('isCurrentShip', True), ]) if state['HullValue']: @@ -421,7 +428,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ('minorfactionName', f['Name']), ('minorfactionReputation', f['MyReputation']/100.0), ]) for f in entry['Factions'] - ]) + ]) elif entry['event'] == 'CarrierJump': add_event('addCommanderTravelCarrierJump', entry['timestamp'], OrderedDict([ @@ -438,11 +445,10 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ('minorfactionName', f['Name']), ('minorfactionReputation', f['MyReputation']/100.0), ]) for f in entry['Factions'] - ]) + ]) # Ignore the following 'Docked' event this.suppress_docked = True - cargo = [OrderedDict([('itemName', k), ('itemCount', state['Cargo'][k])]) for k in sorted(state['Cargo'])] # Send cargo and materials if changed @@ -451,7 +457,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.cargo = cargo materials = [] for category in ['Raw', 'Manufactured', 'Encoded']: - materials.extend([ OrderedDict([('itemName', k), ('itemCount', state[category][k])]) for k in sorted(state[category]) ]) + materials.extend([OrderedDict([('itemName', k), ('itemCount', state[category][k])]) + for k in sorted(state[category])]) if this.materials != materials: add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) this.materials = materials @@ -469,7 +476,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ])) this.lastcredits = state['Credits'] elif entry['event'] == 'Statistics': - add_event('setCommanderGameStatistics', entry['timestamp'], state['Statistics']) # may be out of date + add_event('setCommanderGameStatistics', entry['timestamp'], state['Statistics']) # may be out of date # Selling / swapping ships if entry['event'] == 'ShipyardNew': @@ -478,11 +485,11 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ('shipType', entry['ShipType']), ('shipGameID', entry['NewShipID']), ])) - this.shipswap = True # Want subsequent Loadout event to be sent immediately + this.shipswap = True # Want subsequent Loadout event to be sent immediately elif entry['event'] in ['ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy', 'ShipyardSwap']: if entry['event'] == 'ShipyardSwap': - this.shipswap = True # Don't know new ship name and ident 'til the following Loadout event + this.shipswap = True # Don't know new ship name and ident 'til the following Loadout event if 'StoreShipID' in entry: add_event('setCommanderShip', entry['timestamp'], OrderedDict([ @@ -503,8 +510,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): OrderedDict([ ('shipType', state['ShipType']), ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None + ('shipName', state['ShipName']), # Can be None + ('shipIdent', state['ShipIdent']), # Can be None ('isCurrentShip', True), ])) @@ -535,14 +542,14 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): 'shipGameID': x['ShipID'], 'shipName': x.get('Name'), 'isHot': x['Hot'], - 'starsystemName': x.get('StarSystem'), # Not present for ships in transit - 'marketID': x.get('ShipMarketID'), # " + 'starsystemName': x.get('StarSystem'), # Not present for ships in transit + 'marketID': x.get('ShipMarketID'), # " } for x in entry['ShipsRemote']], - key = itemgetter('shipGameID') + key=itemgetter('shipGameID') ) if this.fleet != fleet: this.fleet = fleet - this.events = [x for x in this.events if x['eventName'] != 'setCommanderShip'] # Remove any unsent + this.events = [x for x in this.events if x['eventName'] != 'setCommanderShip'] # Remove any unsent for ship in this.fleet: add_event('setCommanderShip', entry['timestamp'], ship) @@ -551,12 +558,12 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): loadout = make_loadout(state) if this.loadout != loadout: this.loadout = loadout - this.events = [x for x in this.events if x['eventName'] != 'setCommanderShipLoadout' or x['shipGameID'] != this.loadout['shipGameID']] # Remove any unsent for this ship + this.events = [x for x in this.events if x['eventName'] != 'setCommanderShipLoadout' or x['shipGameID'] != this.loadout['shipGameID']] # Remove any unsent for this ship add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) # Stored modules if entry['event'] == 'StoredModules': - items = dict([(x['StorageSlot'], x) for x in entry['Items']]) # Impose an order + items = dict([(x['StorageSlot'], x) for x in entry['Items']]) # Impose an order modules = [] for slot in sorted(items): item = items[slot] @@ -584,7 +591,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): if this.storedmodules != modules: # Only send on change this.storedmodules = modules - this.events = [x for x in this.events if x['eventName'] != 'setCommanderStorageModules'] # Remove any unsent + this.events = [x for x in this.events if x['eventName'] + != 'setCommanderStorageModules'] # Remove any unsent add_event('setCommanderStorageModules', entry['timestamp'], this.storedmodules) # Missions @@ -600,7 +608,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ]) # optional mission-specific properties for (iprop, prop) in [ - ('missionExpiry', 'Expiry'), # Listed as optional in the docs, but always seems to be present + ('missionExpiry', 'Expiry'), # Listed as optional in the docs, but always seems to be present ('starsystemNameTarget', 'DestinationSystem'), ('stationNameTarget', 'DestinationStation'), ('minorfactionNameTarget', 'TargetFaction'), @@ -619,29 +627,31 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): add_event('addCommanderMission', entry['timestamp'], data) elif entry['event'] == 'MissionAbandoned': - add_event('setCommanderMissionAbandoned', entry['timestamp'], { 'missionGameID': entry['MissionID'] }) + add_event('setCommanderMissionAbandoned', entry['timestamp'], {'missionGameID': entry['MissionID']}) elif entry['event'] == 'MissionCompleted': for x in entry.get('PermitsAwarded', []): - add_event('addCommanderPermit', entry['timestamp'], { 'starsystemName': x }) + add_event('addCommanderPermit', entry['timestamp'], {'starsystemName': x}) - data = OrderedDict([ ('missionGameID', entry['MissionID']) ]) + data = OrderedDict([('missionGameID', entry['MissionID'])]) if 'Donation' in entry: data['donationCredits'] = entry['Donation'] if 'Reward' in entry: data['rewardCredits'] = entry['Reward'] if 'PermitsAwarded' in entry: - data['rewardPermits'] = [{ 'starsystemName': x } for x in entry['PermitsAwarded']] + data['rewardPermits'] = [{'starsystemName': x} for x in entry['PermitsAwarded']] if 'CommodityReward' in entry: - data['rewardCommodities'] = [{ 'itemName': x['Name'], 'itemCount': x['Count'] } for x in entry['CommodityReward']] + data['rewardCommodities'] = [{'itemName': x['Name'], 'itemCount': x['Count']} + for x in entry['CommodityReward']] if 'MaterialsReward' in entry: - data['rewardMaterials'] = [{ 'itemName': x['Name'], 'itemCount': x['Count'] } for x in entry['MaterialsReward']] + data['rewardMaterials'] = [{'itemName': x['Name'], 'itemCount': x['Count']} + for x in entry['MaterialsReward']] factioneffects = [] for faction in entry.get('FactionEffects', []): - effect = OrderedDict([ ('minorfactionName', faction['Faction']) ]) + effect = OrderedDict([('minorfactionName', faction['Faction'])]) for influence in faction.get('Influence', []): if 'Influence' in influence: - effect['influenceGain'] = len(effect.get('influenceGain', '')) > len(influence['Influence']) and effect['influenceGain'] or influence['Influence'] # pick highest + effect['influenceGain'] = len(effect.get('influenceGain', '')) > len(influence['Influence']) and effect['influenceGain'] or influence['Influence'] # pick highest if 'Reputation' in faction: effect['reputationGain'] = faction['Reputation'] factioneffects.append(effect) @@ -650,11 +660,11 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): add_event('setCommanderMissionCompleted', entry['timestamp'], data) elif entry['event'] == 'MissionFailed': - add_event('setCommanderMissionFailed', entry['timestamp'], { 'missionGameID': entry['MissionID'] }) + add_event('setCommanderMissionFailed', entry['timestamp'], {'missionGameID': entry['MissionID']}) # Combat if entry['event'] == 'Died': - data = OrderedDict([ ('starsystemName', system) ]) + data = OrderedDict([('starsystemName', system)]) if 'Killers' in entry: data['wingOpponentNames'] = [x['Name'] for x in entry['Killers']] elif 'KillerName' in entry: @@ -665,7 +675,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): data = OrderedDict([('starsystemName', system), ('isPlayer', entry['IsPlayer']), ('isSubmit', entry['Submitted']), - ]) + ]) if 'Interdictor' in entry: data['opponentName'] = entry['Interdictor'] elif 'Faction' in entry: @@ -678,7 +688,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): data = OrderedDict([('starsystemName', system), ('isPlayer', entry['IsPlayer']), ('isSuccess', entry['Success']), - ]) + ]) if 'Interdicted' in entry: data['opponentName'] = entry['Interdicted'] elif 'Faction' in entry: @@ -692,17 +702,18 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): OrderedDict([('starsystemName', system), ('opponentName', entry['Interdictor']), ('isPlayer', entry['IsPlayer']), - ])) + ])) elif entry['event'] == 'PVPKill': add_event('addCommanderCombatKill', entry['timestamp'], OrderedDict([('starsystemName', system), ('opponentName', entry['Victim']), - ])) + ])) # Community Goals if entry['event'] == 'CommunityGoal': - this.events = [x for x in this.events if x['eventName'] not in ['setCommunityGoal', 'setCommanderCommunityGoalProgress']] # Remove any unsent + this.events = [x for x in this.events if x['eventName'] not in [ + 'setCommunityGoal', 'setCommanderCommunityGoalProgress']] # Remove any unsent for goal in entry['CurrentGoals']: data = OrderedDict([ @@ -741,12 +752,12 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): add_event('addCommanderFriend', entry['timestamp'], OrderedDict([('commanderName', entry['Name']), ('gamePlatform', 'pc'), - ])) + ])) elif entry['Status'] in ['Declined', 'Lost']: add_event('delCommanderFriend', entry['timestamp'], OrderedDict([('commanderName', entry['Name']), ('gamePlatform', 'pc'), - ])) + ])) this.newuser = False @@ -761,6 +772,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.station_link['url'] = station_url(this.system, this.station) this.station_link.update_idletasks() + def cmdr_data(data, is_beta): this.cmdr = data['commander']['name'] @@ -789,7 +801,7 @@ def cmdr_data(data, is_beta): if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(this.cmdr): if not (CREDIT_RATIO > this.lastcredits / data['commander']['credits'] > 1/CREDIT_RATIO): - this.events = [x for x in this.events if x['eventName'] != 'setCommanderCredits'] # Remove any unsent + this.events = [x for x in this.events if x['eventName'] != 'setCommanderCredits'] # Remove any unsent add_event('setCommanderCredits', data['timestamp'], OrderedDict([ ('commanderCredits', data['commander']['credits']), @@ -846,6 +858,7 @@ def make_loadout(state): ('shipLoadout', modules), ]) + def add_event(name, timestamp, data): this.events.append(OrderedDict([ ('eventName', name), @@ -853,6 +866,7 @@ def add_event(name, timestamp, data): ('eventData', data), ])) + def call_timer(wait=FLOOD_LIMIT_SECONDS): while this.timer_run: time.sleep(wait) @@ -860,11 +874,13 @@ def call_timer(wait=FLOOD_LIMIT_SECONDS): call() # Queue a call to Inara, handled in Worker thread + + def call(callback=None, force=False): if not this.events: return - if (time.time() - config.getint(LAST_UPDATE_CONF_KEY)) <= FLOOD_LIMIT_SECONDS and not force: + if (time.time() - config.getint(LAST_UPDATE_CONF_KEY)) <= FLOOD_LIMIT_SECONDS and not force: return config.set(LAST_UPDATE_CONF_KEY, int(time.time())) @@ -877,24 +893,26 @@ def call(callback=None, force=False): ('commanderName', this.cmdr), ('commanderFrontierID', this.FID), ])), - ('events', list(this.events)), # shallow copy + ('events', list(this.events)), # shallow copy ]) this.events = [] this.queue.put(('https://inara.cz/inapi/v1/', data, None)) # Worker thread + + def worker(): while True: item = this.queue.get() if not item: - return # Closing + return # Closing else: (url, data, callback) = item retrying = 0 while retrying < 3: try: - r = this.session.post(url, data=json.dumps(data, separators = (',', ':')), timeout=_TIMEOUT) + r = this.session.post(url, data=json.dumps(data, separators=(',', ':')), timeout=_TIMEOUT) r.raise_for_status() reply = r.json() status = reply['header']['eventStatus'] @@ -944,10 +962,13 @@ def update_location(event=None): for plugin in plug.provides('inara_notify_location'): plug.invoke(plugin, None, 'inara_notify_location', this.lastlocation) + def inara_notify_location(eventData): pass # Call inara_notify_ship() in interested plugins with Inara's response when changing ship + + def update_ship(event=None): if this.lastship: for plugin in plug.provides('inara_notify_ship'): From 160013e3e52400ba681c472b5f5dafbfed549a7d Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 13:55:03 +0200 Subject: [PATCH 201/258] Added type annotations and docstrings --- plugins/inara.py | 90 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 0ad4f73f..e94ff152 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -4,7 +4,7 @@ from collections import OrderedDict import json -from typing import Any, Union +from typing import Any, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT import requests import sys import time @@ -34,25 +34,24 @@ this.lastlocation = None # eventData from the last Commander's Flight Log event this.lastship = None # eventData from the last addCommanderShip or setCommanderShip event # Cached Cmdr state -this.events = [] # Unsent events -this.cmdr = None -this.FID = None # Frontier ID +this.events: List[OrderedDictT[str, Any]] = [] # Unsent events +this.cmdr: Optional[str] = None +this.FID: Optional[str] = None # Frontier ID this.multicrew = False # don't send captain's ship info to Inara while on a crew this.newuser = False # just entered API Key - send state immediately this.newsession = True # starting a new session - wait for Cargo event this.undocked = False # just undocked this.suppress_docked = False # Skip initial Docked event if started docked -this.cargo = None -this.materials = None +this.cargo: Optional[OrderedDictT[str, Any]] = None +this.materials: Optional[OrderedDictT[str, Any]] = None this.lastcredits = 0 # Send credit update soon after Startup / new game -this.storedmodules = None -this.loadout = None -this.fleet = None +this.storedmodules: Optional[OrderedDictT[str, Any]] = None +this.loadout: Optional[OrderedDictT[str, Any]] = None +this.fleet: Optional[str] = None this.shipswap = False # just swapped ship # last time we updated, if unset in config this is 0, which means an instant update LAST_UPDATE_CONF_KEY = 'inara_last_update' -# this.last_update_time = config.getint(LAST_UPDATE_CONF_KEY) FLOOD_LIMIT_SECONDS = 30 # minimum time between sending events this.timer_run = True @@ -68,7 +67,7 @@ this.station_marketid = None STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00D7 -def system_url(system_name): +def system_url(system_name: str): if this.system_address: return requests.utils.requote_uri(f'https://inara.cz/galaxy-starsystem/?search={this.system_address}') @@ -78,7 +77,7 @@ def system_url(system_name): return this.system -def station_url(system_name, station_name): +def station_url(system_name: str, station_name: str): if system_name: if station_name: return requests.utils.requote_uri(f'https://inara.cz/galaxy-station/?search={system_name}%20[{station_name}]') @@ -98,7 +97,7 @@ def plugin_start3(plugin_dir): return 'Inara' -def plugin_app(parent): +def plugin_app(parent: tk.Tk): this.system_link = parent.children['system'] # system label in main window this.station_link = parent.children['station'] # station label in main window this.system_link.bind_all('<>', update_location) @@ -117,7 +116,7 @@ def plugin_stop(): this.timer_run = False -def plugin_prefs(parent, cmdr, is_beta): +def plugin_prefs(parent: tk.Tk, cmdr: str, is_beta: bool): PADX = 10 BUTTONX = 12 # indent Checkbuttons and Radiobuttons PADY = 2 # close spacing @@ -144,7 +143,7 @@ def plugin_prefs(parent, cmdr, is_beta): return frame -def prefs_cmdr_changed(cmdr, is_beta): +def prefs_cmdr_changed(cmdr: str, is_beta: bool): this.log_button['state'] = cmdr and not is_beta and tk.NORMAL or tk.DISABLED this.apikey['state'] = tk.NORMAL this.apikey.delete(0, tk.END) @@ -159,7 +158,7 @@ def prefsvarchanged(): this.label['state'] = this.apikey_label['state'] = this.apikey['state'] = this.log.get() and this.log_button['state'] or tk.DISABLED -def prefs_changed(cmdr, is_beta): +def prefs_changed(cmdr: str, is_beta: bool): changed = config.getint('inara_out') != this.log.get() config.set('inara_out', this.log.get()) @@ -191,7 +190,13 @@ def prefs_changed(cmdr, is_beta): call() -def credentials(cmdr): +def credentials(cmdr: str) -> Optional[str]: + """ + credentials fetches the credentials for the given commander + + :param cmdr: Commander name to search for credentials + :return: Credentials for the given commander or None + """ # Credentials for cmdr if not cmdr: return None @@ -203,8 +208,7 @@ def credentials(cmdr): return None -def journal_entry(cmdr, is_beta, system, station, entry, state): - +def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Dict[str, Any], state: Dict[str, Any]): # Send any unsent events when switching accounts if cmdr and cmdr != this.cmdr: call(force=True) @@ -810,7 +814,7 @@ def cmdr_data(data, is_beta): this.lastcredits = float(data['commander']['credits']) -def make_loadout(state): +def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: modules = [] for m in state['Modules'].values(): module = OrderedDict([ @@ -859,7 +863,15 @@ def make_loadout(state): ]) -def add_event(name, timestamp, data): +def add_event(name: str, timestamp: str, data: Mapping[str, Any]): + """ + Add an event to the event queue + + :param name: name of the event + :param timestamp: timestamp for the event + :param data: data to be sent in the payload + """ + this.events.append(OrderedDict([ ('eventName', name), ('eventTimestamp', timestamp), @@ -868,6 +880,11 @@ def add_event(name, timestamp, data): def call_timer(wait=FLOOD_LIMIT_SECONDS): + """ + call_timer runs in its own thread polling out to INARA once every FLOOD_LIMIT_SECONDS + + :param wait: time to wait between polls, defaults to FLOOD_LIMIT_SECONDS + """ while this.timer_run: time.sleep(wait) if this.timer_run: # check again in here just in case we're closing and the stars align @@ -877,6 +894,15 @@ def call_timer(wait=FLOOD_LIMIT_SECONDS): def call(callback=None, force=False): + """ + call queues a call out to the inara API + + Note that it will not allow a call more than once every FLOOD_LIMIT_SECONDS + unless the force parameter is True. + + :param callback: Unused and ignored. , defaults to None + :param force: Whether or not to ignore flood limits, defaults to False + """ if not this.events: return @@ -902,6 +928,12 @@ def call(callback=None, force=False): def worker(): + """ + worker is the main thread worker and backbone of the plugin. + + As events are added to `this.queue`, the worker thread will push them to the API + """ + while True: item = this.queue.get() if not item: @@ -956,8 +988,14 @@ def worker(): plug.show_error(_("Error: Can't connect to Inara")) -# Call inara_notify_location() in this and other interested plugins with Inara's response when changing system or station +# TODO(A_D) event is unused def update_location(event=None): + """ + Call inara_notify_location in this and other interested plugins with Inara's response when changing system + or station + + :param event: Unused and ignored, defaults to None + """ if this.lastlocation: for plugin in plug.provides('inara_notify_location'): plug.invoke(plugin, None, 'inara_notify_location', this.lastlocation) @@ -966,10 +1004,14 @@ def update_location(event=None): def inara_notify_location(eventData): pass -# Call inara_notify_ship() in interested plugins with Inara's response when changing ship - +# TODO(A_D) event is unused def update_ship(event=None): + """ + Call inara_notify_ship() in interested plugins with Inara's response when changing ship + + :param event: Unused and ignored, defaults to None + """ if this.lastship: for plugin in plug.provides('inara_notify_ship'): plug.invoke(plugin, None, 'inara_notify_ship', this.lastship) From 13c2679c38485aefec8ebf7d5a25f74784473bea Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 14:12:34 +0200 Subject: [PATCH 202/258] Added spacing around scope changes --- plugins/inara.py | 103 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index e94ff152..16fc3f70 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -4,7 +4,7 @@ from collections import OrderedDict import json -from typing import Any, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT +from typing import Any, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT, TYPE_CHECKING import requests import sys import time @@ -21,6 +21,10 @@ from config import appname, applongname, appversion, config import plug logger = logging.getLogger(appname) +if TYPE_CHECKING: + def _(x): + return x + _TIMEOUT = 20 FAKE = ['CQC', 'Training', 'Destination'] # Fake systems that shouldn't be sent to Inara @@ -47,7 +51,7 @@ this.materials: Optional[OrderedDictT[str, Any]] = None this.lastcredits = 0 # Send credit update soon after Startup / new game this.storedmodules: Optional[OrderedDictT[str, Any]] = None this.loadout: Optional[OrderedDictT[str, Any]] = None -this.fleet: Optional[str] = None +this.fleet: Optional[List[OrderedDictT[str, Any]]] = None this.shipswap = False # just swapped ship # last time we updated, if unset in config this is 0, which means an instant update @@ -125,11 +129,11 @@ def plugin_prefs(parent: tk.Tk, cmdr: str, is_beta: bool): frame.columnconfigure(1, weight=1) HyperlinkLabel(frame, text='Inara', background=nb.Label().cget('background'), url='https://inara.cz/', underline=True).grid(columnspan=2, padx=PADX, sticky=tk.W) # Don't translate - this.log = tk.IntVar(value = config.getint('inara_out') and 1) + this.log = tk.IntVar(value=config.getint('inara_out') and 1) this.log_button = nb.Checkbutton(frame, text=_('Send flight log and Cmdr status to Inara'), variable=this.log, command=prefsvarchanged) - this.log_button.grid(columnspan=2, padx=BUTTONX, pady=(5,0), sticky=tk.W) + this.log_button.grid(columnspan=2, padx=BUTTONX, pady=(5, 0), sticky=tk.W) - nb.Label(frame).grid(sticky=tk.W) # big spacer + nb.Label(frame).grid(sticky=tk.W) # big spacer this.label = HyperlinkLabel(frame, text=_('Inara credentials'), background=nb.Label().cget('background'), url='https://inara.cz/settings-api', underline=True) # Section heading in settings this.label.grid(columnspan=2, padx=PADX, sticky=tk.W) @@ -151,6 +155,7 @@ def prefs_cmdr_changed(cmdr: str, is_beta: bool): cred = credentials(cmdr) if cred: this.apikey.insert(0, cred) + this.label['state'] = this.apikey_label['state'] = this.apikey['state'] = cmdr and not is_beta and this.log.get() and tk.NORMAL or tk.DISABLED @@ -165,6 +170,7 @@ def prefs_changed(cmdr: str, is_beta: bool): # Override standard URL functions if config.get('system_provider') == 'Inara': this.system_link['url'] = system_url(this.system) + if config.get('station_provider') == 'Inara': this.station_link['url'] = station_url(this.system, this.station) @@ -178,10 +184,12 @@ def prefs_changed(cmdr: str, is_beta: bool): apikeys.extend([''] * (1 + idx - len(apikeys))) changed |= (apikeys[idx] != this.apikey.get().strip()) apikeys[idx] = this.apikey.get().strip() + else: config.set('inara_cmdrs', cmdrs + [cmdr]) changed = True apikeys.append(this.apikey.get().strip()) + config.set('inara_apikeys', apikeys) if this.log.get() and changed: @@ -197,13 +205,13 @@ def credentials(cmdr: str) -> Optional[str]: :param cmdr: Commander name to search for credentials :return: Credentials for the given commander or None """ - # Credentials for cmdr if not cmdr: return None cmdrs = config.get('inara_cmdrs') or [] if cmdr in cmdrs and config.get('inara_apikeys'): return config.get('inara_apikeys')[cmdrs.index(cmdr)] + else: return None @@ -223,9 +231,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # User setup Inara API while at the loading screen - proceed as for new session this.newuser = False this.newsession = True + else: this.newuser = True this.newsession = False + this.undocked = False this.suppress_docked = False this.cargo = None @@ -239,9 +249,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.system_address = None this.station = None this.station_marketid = None + elif entry['event'] in ['Resurrect', 'ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy']: # Events that mean a significant change in credits so we should send credits after next "Update" this.lastcredits = 0 + elif entry['event'] in ['ShipyardNew', 'ShipyardSwap'] or (entry['event'] == 'Location' and entry['Docked']): this.suppress_docked = True @@ -269,10 +281,10 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(cmdr): try: # Dump starting state to Inara - if (this.newuser or entry['event'] == 'StartUp' or (this.newsession and entry['event'] == 'Cargo')): + this.newuser = False this.newsession = False @@ -285,6 +297,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('rankProgress', v[1] / 100.0), ]) for k, v in state['Rank'].items() if v is not None ]) + add_event('setCommanderReputationMajorFaction', entry['timestamp'], [ OrderedDict([ @@ -292,6 +305,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('majorfactionReputation', v / 100.0), ]) for k, v in state['Reputation'].items() if v is not None ]) + if state['Engineers']: # Not populated < 3.3 add_event('setCommanderRankEngineer', entry['timestamp'], [ @@ -317,10 +331,13 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipIdent', state['ShipIdent']), # Can be None ('isCurrentShip', True), ]) + if state['HullValue']: data['shipHullValue'] = state['HullValue'] + if state['ModulesValue']: data['shipModulesValue'] = state['ModulesValue'] + data['shipRebuyCost'] = state['Rebuy'] add_event('setCommanderShip', entry['timestamp'], data) @@ -339,6 +356,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('rankValue', v[0]), ('rankProgress', 0), ])) + elif entry['event'] == 'EngineerProgress' and 'Engineer' in entry: add_event('setCommanderRankEngineer', entry['timestamp'], OrderedDict([ @@ -353,12 +371,14 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('powerName', entry['Power']), ('rankValue', 1), ])) + elif entry['event'] == 'PowerplayLeave': add_event('setCommanderRankPower', entry['timestamp'], OrderedDict([ ('powerName', entry['Power']), ('rankValue', 0), ])) + elif entry['event'] == 'PowerplayDefect': add_event('setCommanderRankPower', entry['timestamp'], OrderedDict([ @@ -405,6 +425,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif entry['event'] == 'Undocked': this.undocked = True this.station = None + elif entry['event'] == 'SupercruiseEntry': if this.undocked: # Staying in system after undocking - send any pending events from in-station action @@ -415,6 +436,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipGameID', state['ShipID']), ])) this.undocked = False + elif entry['event'] == 'FSDJump': this.undocked = False add_event('addCommanderTravelFSDJump', entry['timestamp'], @@ -433,6 +455,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('minorfactionReputation', f['MyReputation']/100.0), ]) for f in entry['Factions'] ]) + elif entry['event'] == 'CarrierJump': add_event('addCommanderTravelCarrierJump', entry['timestamp'], OrderedDict([ @@ -442,6 +465,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipType', state['ShipType']), ('shipGameID', state['ShipID']), ])) + if entry.get('Factions'): add_event('setCommanderReputationMinorFaction', entry['timestamp'], [ @@ -450,6 +474,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('minorfactionReputation', f['MyReputation']/100.0), ]) for f in entry['Factions'] ]) + # Ignore the following 'Docked' event this.suppress_docked = True @@ -459,10 +484,12 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if this.cargo != cargo: add_event('setCommanderInventoryCargo', entry['timestamp'], cargo) this.cargo = cargo + materials = [] for category in ['Raw', 'Manufactured', 'Encoded']: materials.extend([OrderedDict([('itemName', k), ('itemCount', state[category][k])]) for k in sorted(state[category])]) + if this.materials != materials: add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) this.materials = materials @@ -478,7 +505,9 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('commanderCredits', state['Credits']), ('commanderLoan', state['Loan']), ])) + this.lastcredits = state['Credits'] + elif entry['event'] == 'Statistics': add_event('setCommanderGameStatistics', entry['timestamp'], state['Statistics']) # may be out of date @@ -489,11 +518,13 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipType', entry['ShipType']), ('shipGameID', entry['NewShipID']), ])) + this.shipswap = True # Want subsequent Loadout event to be sent immediately elif entry['event'] in ['ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy', 'ShipyardSwap']: if entry['event'] == 'ShipyardSwap': this.shipswap = True # Don't know new ship name and ident 'til the following Loadout event + if 'StoreShipID' in entry: add_event('setCommanderShip', entry['timestamp'], OrderedDict([ @@ -502,6 +533,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('starsystemName', system), ('stationName', station), ])) + elif 'SellShipID' in entry: add_event('delCommanderShip', entry['timestamp'], OrderedDict([ @@ -551,6 +583,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di } for x in entry['ShipsRemote']], key=itemgetter('shipGameID') ) + if this.fleet != fleet: this.fleet = fleet this.events = [x for x in this.events if x['eventName'] != 'setCommanderShip'] # Remove any unsent @@ -580,6 +613,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Location can be absent if in transit if 'StarSystem' in item: module['starsystemName'] = item['StarSystem'] + if 'MarketID' in item: module['marketID'] = item['MarketID'] @@ -587,6 +621,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di module['engineering'] = OrderedDict([('blueprintName', item['EngineerModifications'])]) if 'Level' in item: module['engineering']['blueprintLevel'] = item['Level'] + if 'Quality' in item: module['engineering']['blueprintQuality'] = item['Quality'] @@ -610,6 +645,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('stationNameOrigin', station), ('minorfactionNameOrigin', entry['Faction']), ]) + # optional mission-specific properties for (iprop, prop) in [ ('missionExpiry', 'Expiry'), # Listed as optional in the docs, but always seems to be present @@ -626,8 +662,10 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('passengerIsVIP', 'PassengerVIPs'), ('passengerIsWanted', 'PassengerWanted'), ]: + if prop in entry: data[iprop] = entry[prop] + add_event('addCommanderMission', entry['timestamp'], data) elif entry['event'] == 'MissionAbandoned': @@ -640,27 +678,36 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di data = OrderedDict([('missionGameID', entry['MissionID'])]) if 'Donation' in entry: data['donationCredits'] = entry['Donation'] + if 'Reward' in entry: data['rewardCredits'] = entry['Reward'] + if 'PermitsAwarded' in entry: data['rewardPermits'] = [{'starsystemName': x} for x in entry['PermitsAwarded']] + if 'CommodityReward' in entry: data['rewardCommodities'] = [{'itemName': x['Name'], 'itemCount': x['Count']} for x in entry['CommodityReward']] + if 'MaterialsReward' in entry: data['rewardMaterials'] = [{'itemName': x['Name'], 'itemCount': x['Count']} for x in entry['MaterialsReward']] + factioneffects = [] for faction in entry.get('FactionEffects', []): effect = OrderedDict([('minorfactionName', faction['Faction'])]) for influence in faction.get('Influence', []): if 'Influence' in influence: effect['influenceGain'] = len(effect.get('influenceGain', '')) > len(influence['Influence']) and effect['influenceGain'] or influence['Influence'] # pick highest + if 'Reputation' in faction: effect['reputationGain'] = faction['Reputation'] + factioneffects.append(effect) + if factioneffects: data['minorfactionEffects'] = factioneffects + add_event('setCommanderMissionCompleted', entry['timestamp'], data) elif entry['event'] == 'MissionFailed': @@ -671,8 +718,10 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di data = OrderedDict([('starsystemName', system)]) if 'Killers' in entry: data['wingOpponentNames'] = [x['Name'] for x in entry['Killers']] + elif 'KillerName' in entry: data['opponentName'] = entry['KillerName'] + add_event('addCommanderCombatDeath', entry['timestamp'], data) elif entry['event'] == 'Interdicted': @@ -680,12 +729,16 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('isPlayer', entry['IsPlayer']), ('isSubmit', entry['Submitted']), ]) + if 'Interdictor' in entry: data['opponentName'] = entry['Interdictor'] + elif 'Faction' in entry: data['opponentName'] = entry['Faction'] + elif 'Power' in entry: data['opponentName'] = entry['Power'] + add_event('addCommanderCombatInterdicted', entry['timestamp'], data) elif entry['event'] == 'Interdiction': @@ -693,12 +746,16 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('isPlayer', entry['IsPlayer']), ('isSuccess', entry['Success']), ]) + if 'Interdicted' in entry: data['opponentName'] = entry['Interdicted'] + elif 'Faction' in entry: data['opponentName'] = entry['Faction'] + elif 'Power' in entry: data['opponentName'] = entry['Power'] + add_event('addCommanderCombatInterdiction', entry['timestamp'], data) elif entry['event'] == 'EscapeInterdiction': @@ -718,8 +775,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if entry['event'] == 'CommunityGoal': this.events = [x for x in this.events if x['eventName'] not in [ 'setCommunityGoal', 'setCommanderCommunityGoalProgress']] # Remove any unsent - for goal in entry['CurrentGoals']: + for goal in entry['CurrentGoals']: data = OrderedDict([ ('communitygoalGameID', goal['CGID']), ('communitygoalName', goal['Title']), @@ -730,13 +787,17 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('contributorsNum', goal['NumContributors']), ('contributionsTotal', goal['CurrentTotal']), ]) + if 'TierReached' in goal: data['tierReached'] = int(goal['TierReached'].split()[-1]) + if 'TopRankSize' in goal: data['topRankSize'] = goal['TopRankSize'] + if 'TopTier' in goal: data['tierMax'] = int(goal['TopTier']['Name'].split()[-1]) data['completionBonus'] = goal['TopTier']['Bonus'] + add_event('setCommunityGoal', entry['timestamp'], data) data = OrderedDict([ @@ -744,10 +805,13 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('contribution', goal['PlayerContribution']), ('percentileBand', goal['PlayerPercentileBand']), ]) + if 'Bonus' in goal: data['percentileBandReward'] = goal['Bonus'] + if 'PlayerInTopRank' in goal: data['isTopRank'] = goal['PlayerInTopRank'] + add_event('setCommanderCommunityGoalProgress', entry['timestamp'], data) # Friends @@ -757,6 +821,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di OrderedDict([('commanderName', entry['Name']), ('gamePlatform', 'pc'), ])) + elif entry['Status'] in ['Declined', 'Lost']: add_event('delCommanderFriend', entry['timestamp'], OrderedDict([('commanderName', entry['Name']), @@ -783,6 +848,7 @@ def cmdr_data(data, is_beta): # Always store initially, even if we're not the *current* system provider. if not this.station_marketid: this.station_marketid = data['commander']['docked'] and data['lastStarport']['id'] + # Only trust CAPI if these aren't yet set this.system = this.system or data['lastSystem']['name'] this.station = this.station or data['commander']['docked'] and data['lastStarport']['name'] @@ -792,11 +858,14 @@ def cmdr_data(data, is_beta): this.system_link['text'] = this.system this.system_link['url'] = system_url(this.system) this.system_link.update_idletasks() + if config.get('station_provider') == 'Inara': if data['commander']['docked']: this.station_link['text'] = this.station + elif data['lastStarport']['name'] and data['lastStarport']['name'] != "": this.station_link['text'] = STATION_UNDOCKED + else: this.station_link['text'] = '' @@ -811,6 +880,7 @@ def cmdr_data(data, is_beta): ('commanderCredits', data['commander']['credits']), ('commanderLoan', data['commander'].get('debt', 0)), ])) + this.lastcredits = float(data['commander']['credits']) @@ -824,34 +894,44 @@ def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: ('isOn', m['On']), ('itemPriority', m['Priority']), ]) + if 'AmmoInClip' in m: module['itemAmmoClip'] = m['AmmoInClip'] + if 'AmmoInHopper' in m: module['itemAmmoHopper'] = m['AmmoInHopper'] + if 'Value' in m: module['itemValue'] = m['Value'] + if 'Hot' in m: module['isHot'] = m['Hot'] + if 'Engineering' in m: engineering = OrderedDict([ ('blueprintName', m['Engineering']['BlueprintName']), ('blueprintLevel', m['Engineering']['Level']), ('blueprintQuality', m['Engineering']['Quality']), ]) + if 'ExperimentalEffect' in m['Engineering']: engineering['experimentalEffect'] = m['Engineering']['ExperimentalEffect'] + engineering['modifiers'] = [] for mod in m['Engineering']['Modifiers']: modifier = OrderedDict([ ('name', mod['Label']), ]) + if 'OriginalValue' in mod: modifier['value'] = mod['Value'] modifier['originalValue'] = mod['OriginalValue'] modifier['lessIsGood'] = mod['LessIsGood'] + else: modifier['value'] = mod['ValueStr'] engineering['modifiers'].append(modifier) + module['engineering'] = engineering modules.append(module) @@ -921,6 +1001,7 @@ def call(callback=None, force=False): ])), ('events', list(this.events)), # shallow copy ]) + this.events = [] this.queue.put(('https://inara.cz/inapi/v1/', data, None)) @@ -950,11 +1031,13 @@ def worker(): status = reply['header']['eventStatus'] if callback: callback(reply) + elif status // 100 != 2: # 2xx == OK (maybe with warnings) # Log fatal errors logger.warning(f'Inara\t{status} {reply["header"].get("eventStatusText", "")}') logger.debug(f'JSON data:\n{json.dumps(data, indent=2, separators = (",", ": "))}') plug.show_error(_('Error: Inara {MSG}').format(MSG=reply['header'].get('eventStatusText', status))) + else: # Log individual errors and warnings for data_event, reply_event in zip(data['events'], reply['events']): @@ -965,6 +1048,7 @@ def worker(): plug.show_error(_('Error: Inara {MSG}').format( MSG=f'{data_event["eventName"]},' f'{reply_event.get("eventStatusText", reply_event["eventStatus"])}')) + if data_event['eventName'] in ('addCommanderTravelCarrierJump', 'addCommanderTravelDock', 'addCommanderTravelFSDJump', @@ -972,18 +1056,21 @@ def worker(): this.lastlocation = reply_event.get('eventData', {}) # calls update_location in main thread this.system_link.event_generate('<>', when="tail") + elif data_event['eventName'] in ['addCommanderShip', 'setCommanderShip']: this.lastship = reply_event.get('eventData', {}) # calls update_ship in main thread this.system_link.event_generate('<>', when="tail") break + except Exception as e: logger.debug('Unable to send events', exc_info=e) retrying += 1 else: if callback: callback(None) + else: plug.show_error(_("Error: Can't connect to Inara")) From 9db9ed4dfcd71159ce81faac82281a94c79a27fb Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 18:02:54 +0200 Subject: [PATCH 203/258] Cleaned up overlong lines --- plugins/inara.py | 64 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 16fc3f70..fd78aafa 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -84,7 +84,10 @@ def system_url(system_name: str): def station_url(system_name: str, station_name: str): if system_name: if station_name: - return requests.utils.requote_uri(f'https://inara.cz/galaxy-station/?search={system_name}%20[{station_name}]') + return requests.utils.requote_uri( + f'https://inara.cz/galaxy-station/?search={system_name}%20[{station_name}]' + ) + return system_url(system_name) return this.station or this.system @@ -128,13 +131,28 @@ def plugin_prefs(parent: tk.Tk, cmdr: str, is_beta: bool): frame = nb.Frame(parent) frame.columnconfigure(1, weight=1) - HyperlinkLabel(frame, text='Inara', background=nb.Label().cget('background'), url='https://inara.cz/', underline=True).grid(columnspan=2, padx=PADX, sticky=tk.W) # Don't translate + HyperlinkLabel( + frame, text='Inara', background=nb.Label().cget('background'), url='https://inara.cz/', underline=True + ).grid(columnspan=2, padx=PADX, sticky=tk.W) # Don't translate + this.log = tk.IntVar(value=config.getint('inara_out') and 1) - this.log_button = nb.Checkbutton(frame, text=_('Send flight log and Cmdr status to Inara'), variable=this.log, command=prefsvarchanged) + this.log_button = nb.Checkbutton( + frame, text=_('Send flight log and Cmdr status to Inara'), variable=this.log, command=prefsvarchanged + ) + this.log_button.grid(columnspan=2, padx=BUTTONX, pady=(5, 0), sticky=tk.W) nb.Label(frame).grid(sticky=tk.W) # big spacer - this.label = HyperlinkLabel(frame, text=_('Inara credentials'), background=nb.Label().cget('background'), url='https://inara.cz/settings-api', underline=True) # Section heading in settings + + # Section heading in settings + this.label = HyperlinkLabel( + frame, + text=_('Inara credentials'), + background=nb.Label().cget('background'), + url='https://inara.cz/settings-api', + underline=True + ) + this.label.grid(columnspan=2, padx=PADX, sticky=tk.W) this.apikey_label = nb.Label(frame, text=_('API Key')) # EDSM setting @@ -156,11 +174,23 @@ def prefs_cmdr_changed(cmdr: str, is_beta: bool): if cred: this.apikey.insert(0, cred) - this.label['state'] = this.apikey_label['state'] = this.apikey['state'] = cmdr and not is_beta and this.log.get() and tk.NORMAL or tk.DISABLED + state = tk.DISABLED + if cmdr and not is_beta and this.log.get(): + state = tk.NORMAL + + this.label['state'] = state + this.apikey_label['state'] = state + this.apikey['state'] = state def prefsvarchanged(): - this.label['state'] = this.apikey_label['state'] = this.apikey['state'] = this.log.get() and this.log_button['state'] or tk.DISABLED + state = tk.DISABLED + if this.log.get(): + state = this.log_button['state'] + + this.label['state'] = state + this.apikey_label['state'] = state + this.apikey['state'] = state def prefs_changed(cmdr: str, is_beta: bool): @@ -595,7 +625,12 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di loadout = make_loadout(state) if this.loadout != loadout: this.loadout = loadout - this.events = [x for x in this.events if x['eventName'] != 'setCommanderShipLoadout' or x['shipGameID'] != this.loadout['shipGameID']] # Remove any unsent for this ship + # Remove any unsent for this ship + this.events = [ + e for e in this.events + if e['eventName'] != 'setCommanderShipLoadout' or e['shipGameID'] != this.loadout['shipGameID'] + ] + add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) # Stored modules @@ -698,7 +733,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di effect = OrderedDict([('minorfactionName', faction['Faction'])]) for influence in faction.get('Influence', []): if 'Influence' in influence: - effect['influenceGain'] = len(effect.get('influenceGain', '')) > len(influence['Influence']) and effect['influenceGain'] or influence['Influence'] # pick highest + highest_gain = influence['Influence'] + if len(effect.get('influenceGain', '')) > len(highest_gain): + highest_gain = effect['influenceGain'] + + effect['influenceGain'] = highest_gain if 'Reputation' in faction: effect['reputationGain'] = faction['Reputation'] @@ -837,7 +876,14 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.system_link.update_idletasks() if config.get('station_provider') == 'Inara': - this.station_link['text'] = this.station or (this.system_population and this.system_population > 0 and STATION_UNDOCKED or '') + to_set = this.station + if not to_set and this.system_population is not None: + if this.system_population > 0: + to_set = STATION_UNDOCKED + else: + to_set = '' + + this.station_link['text'] = to_set this.station_link['url'] = station_url(this.system, this.station) this.station_link.update_idletasks() From 8e49066ca42134fab85a667a107fe8506df7928b Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 18:09:25 +0200 Subject: [PATCH 204/258] Replaced convoluted or-based logic with terneries --- plugins/inara.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index fd78aafa..9f204b34 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -90,7 +90,7 @@ def station_url(system_name: str, station_name: str): return system_url(system_name) - return this.station or this.system + return this.station if this.station else this.system def plugin_start3(plugin_dir): @@ -166,7 +166,7 @@ def plugin_prefs(parent: tk.Tk, cmdr: str, is_beta: bool): def prefs_cmdr_changed(cmdr: str, is_beta: bool): - this.log_button['state'] = cmdr and not is_beta and tk.NORMAL or tk.DISABLED + this.log_button['state'] = tk.NORMAL if cmdr and not is_beta else tk.DISABLED this.apikey['state'] = tk.NORMAL this.apikey.delete(0, tk.END) if cmdr: @@ -896,8 +896,8 @@ def cmdr_data(data, is_beta): this.station_marketid = data['commander']['docked'] and data['lastStarport']['id'] # Only trust CAPI if these aren't yet set - this.system = this.system or data['lastSystem']['name'] - this.station = this.station or data['commander']['docked'] and data['lastStarport']['name'] + this.system = this.system if this.system else data['lastSystem']['name'] + this.station = data['lastStarport']['name'] if data['commander']['docked'] and not this.station else this.station # Override standard URL functions if config.get('system_provider') == 'Inara': From 71ad0f92ac10043c7932c892417518b0ee48a3e0 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 18:13:04 +0200 Subject: [PATCH 205/258] removed outdated comment --- plugins/inara.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 9f204b34..2e9445eb 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -288,8 +288,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.suppress_docked = True # Always update, even if we're not the *current* system or station provider. - this.system_address = entry.get('SystemAddress') or this.system_address - this.system = entry.get('StarSystem') or this.system + this.system_address = entry.get('SystemAddress', this.system_address) + this.system = entry.get('StarSystem', this.system) # We need pop == 0 to set the value so as to clear 'x' in systems with # no stations. @@ -297,17 +297,13 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if pop is not None: this.system_population = pop - this.station = entry.get('StationName') or this.station - this.station_marketid = entry.get('MarketID') or this.station_marketid + this.station = entry.get('StationName', this.station) + this.station_marketid = entry.get('MarketID', this.station_marketid) or this.station_marketid # We might pick up StationName in DockingRequested, make sure we clear it if leaving if entry['event'] in ('Undocked', 'FSDJump', 'SupercruiseEntry'): this.station = None this.station_marketid = None - # Send location and status on new game or StartUp. Assumes Cargo is the last event on a new game (other than Docked). - # Always send an update on Docked, FSDJump, Undocked+SuperCruise, Promotion, EngineerProgress and PowerPlay affiliation. - # Also send material and cargo (if changed) whenever we send an update. - if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(cmdr): try: # Dump starting state to Inara From 96279cb0af4cf6957cf1091e62e0f33fdb200b3d Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 18:16:56 +0200 Subject: [PATCH 206/258] Replaced repeated entry['event'] with variable --- plugins/inara.py | 85 +++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 2e9445eb..951deea6 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -3,6 +3,7 @@ # from collections import OrderedDict +from dataclasses import dataclass import json from typing import Any, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT, TYPE_CHECKING import requests @@ -12,6 +13,7 @@ from operator import itemgetter from queue import Queue from threading import Thread import logging +import dataclasses import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel @@ -27,7 +29,7 @@ if TYPE_CHECKING: _TIMEOUT = 20 -FAKE = ['CQC', 'Training', 'Destination'] # Fake systems that shouldn't be sent to Inara +FAKE = ('CQC', 'Training', 'Destination') # Fake systems that shouldn't be sent to Inara CREDIT_RATIO = 1.05 # Update credits if they change by 5% over the course of a session @@ -251,13 +253,14 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if cmdr and cmdr != this.cmdr: call(force=True) + event_name = entry['event'] this.cmdr = cmdr this.FID = state['FID'] this.multicrew = bool(state['Role']) - if entry['event'] == 'LoadGame' or this.newuser: + if event_name == 'LoadGame' or this.newuser: # clear cached state - if entry['event'] == 'LoadGame': + if event_name == 'LoadGame': # User setup Inara API while at the loading screen - proceed as for new session this.newuser = False this.newsession = True @@ -280,11 +283,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.station = None this.station_marketid = None - elif entry['event'] in ['Resurrect', 'ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy']: + elif event_name in ('Resurrect', 'ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy'): # Events that mean a significant change in credits so we should send credits after next "Update" this.lastcredits = 0 - elif entry['event'] in ['ShipyardNew', 'ShipyardSwap'] or (entry['event'] == 'Location' and entry['Docked']): + elif event_name in ('ShipyardNew', 'ShipyardSwap') or (event_name == 'Location' and entry['Docked']): this.suppress_docked = True # Always update, even if we're not the *current* system or station provider. @@ -300,7 +303,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.station = entry.get('StationName', this.station) this.station_marketid = entry.get('MarketID', this.station_marketid) or this.station_marketid # We might pick up StationName in DockingRequested, make sure we clear it if leaving - if entry['event'] in ('Undocked', 'FSDJump', 'SupercruiseEntry'): + if event_name in ('Undocked', 'FSDJump', 'SupercruiseEntry'): this.station = None this.station_marketid = None @@ -308,8 +311,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di try: # Dump starting state to Inara if (this.newuser or - entry['event'] == 'StartUp' or - (this.newsession and entry['event'] == 'Cargo')): + event_name == 'StartUp' or + (this.newsession and event_name == 'Cargo')): this.newuser = False this.newsession = False @@ -373,7 +376,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di call() # Call here just to be sure that if we can send, we do, otherwise it'll get it in the next tick # Promotions - elif entry['event'] == 'Promotion': + elif event_name == 'Promotion': for k, v in state['Rank'].items(): if k in entry: add_event('setCommanderRankPilot', entry['timestamp'], @@ -383,7 +386,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('rankProgress', 0), ])) - elif entry['event'] == 'EngineerProgress' and 'Engineer' in entry: + elif event_name == 'EngineerProgress' and 'Engineer' in entry: add_event('setCommanderRankEngineer', entry['timestamp'], OrderedDict([ ('engineerName', entry['Engineer']), @@ -391,21 +394,21 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ])) # PowerPlay status change - if entry['event'] == 'PowerplayJoin': + if event_name == 'PowerplayJoin': add_event('setCommanderRankPower', entry['timestamp'], OrderedDict([ ('powerName', entry['Power']), ('rankValue', 1), ])) - elif entry['event'] == 'PowerplayLeave': + elif event_name == 'PowerplayLeave': add_event('setCommanderRankPower', entry['timestamp'], OrderedDict([ ('powerName', entry['Power']), ('rankValue', 0), ])) - elif entry['event'] == 'PowerplayDefect': + elif event_name == 'PowerplayDefect': add_event('setCommanderRankPower', entry['timestamp'], OrderedDict([ ('powerName', entry['ToPower']), @@ -413,7 +416,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ])) # Ship change - if entry['event'] == 'Loadout' and this.shipswap: + if event_name == 'Loadout' and this.shipswap: data = OrderedDict([ ('shipType', state['ShipType']), ('shipGameID', state['ShipID']), @@ -433,7 +436,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.shipswap = False # Location change - elif entry['event'] == 'Docked': + elif event_name == 'Docked': if this.undocked: # Undocked and now docking again. Don't send. this.undocked = False @@ -448,11 +451,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipType', state['ShipType']), ('shipGameID', state['ShipID']), ])) - elif entry['event'] == 'Undocked': + elif event_name == 'Undocked': this.undocked = True this.station = None - elif entry['event'] == 'SupercruiseEntry': + elif event_name == 'SupercruiseEntry': if this.undocked: # Staying in system after undocking - send any pending events from in-station action add_event('setCommanderTravelLocation', entry['timestamp'], @@ -463,7 +466,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ])) this.undocked = False - elif entry['event'] == 'FSDJump': + elif event_name == 'FSDJump': this.undocked = False add_event('addCommanderTravelFSDJump', entry['timestamp'], OrderedDict([ @@ -482,7 +485,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ]) for f in entry['Factions'] ]) - elif entry['event'] == 'CarrierJump': + elif event_name == 'CarrierJump': add_event('addCommanderTravelCarrierJump', entry['timestamp'], OrderedDict([ ('starsystemName', entry['StarSystem']), @@ -512,7 +515,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.cargo = cargo materials = [] - for category in ['Raw', 'Manufactured', 'Encoded']: + for category in ('Raw', 'Manufactured', 'Encoded'): materials.extend([OrderedDict([('itemName', k), ('itemCount', state[category][k])]) for k in sorted(state[category])]) @@ -525,7 +528,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di return str(e) # Send credits and stats to Inara on startup only - otherwise may be out of date - if entry['event'] == 'LoadGame': + if event_name == 'LoadGame': add_event('setCommanderCredits', entry['timestamp'], OrderedDict([ ('commanderCredits', state['Credits']), @@ -534,11 +537,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.lastcredits = state['Credits'] - elif entry['event'] == 'Statistics': + elif event_name == 'Statistics': add_event('setCommanderGameStatistics', entry['timestamp'], state['Statistics']) # may be out of date # Selling / swapping ships - if entry['event'] == 'ShipyardNew': + if event_name == 'ShipyardNew': add_event('addCommanderShip', entry['timestamp'], OrderedDict([ ('shipType', entry['ShipType']), @@ -547,8 +550,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.shipswap = True # Want subsequent Loadout event to be sent immediately - elif entry['event'] in ['ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy', 'ShipyardSwap']: - if entry['event'] == 'ShipyardSwap': + elif event_name in ('ShipyardBuy', 'ShipyardSell', 'SellShipOnRebuy', 'ShipyardSwap'): + if event_name == 'ShipyardSwap': this.shipswap = True # Don't know new ship name and ident 'til the following Loadout event if 'StoreShipID' in entry: @@ -567,7 +570,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipGameID', entry['SellShipID']), ])) - elif entry['event'] == 'SetUserShipName': + elif event_name == 'SetUserShipName': add_event('setCommanderShip', entry['timestamp'], OrderedDict([ ('shipType', state['ShipType']), @@ -577,7 +580,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('isCurrentShip', True), ])) - elif entry['event'] == 'ShipyardTransfer': + elif event_name == 'ShipyardTransfer': add_event('setCommanderShipTransfer', entry['timestamp'], OrderedDict([ ('shipType', entry['ShipType']), @@ -588,7 +591,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ])) # Fleet - if entry['event'] == 'StoredShips': + if event_name == 'StoredShips': fleet = sorted( [{ 'shipType': x['ShipType'], @@ -617,7 +620,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('setCommanderShip', entry['timestamp'], ship) # Loadout - if entry['event'] == 'Loadout' and not this.newsession: + if event_name == 'Loadout' and not this.newsession: loadout = make_loadout(state) if this.loadout != loadout: this.loadout = loadout @@ -630,7 +633,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) # Stored modules - if entry['event'] == 'StoredModules': + if event_name == 'StoredModules': items = dict([(x['StorageSlot'], x) for x in entry['Items']]) # Impose an order modules = [] for slot in sorted(items): @@ -666,7 +669,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('setCommanderStorageModules', entry['timestamp'], this.storedmodules) # Missions - if entry['event'] == 'MissionAccepted': + if event_name == 'MissionAccepted': data = OrderedDict([ ('missionName', entry['Name']), ('missionGameID', entry['MissionID']), @@ -699,10 +702,10 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderMission', entry['timestamp'], data) - elif entry['event'] == 'MissionAbandoned': + elif event_name == 'MissionAbandoned': add_event('setCommanderMissionAbandoned', entry['timestamp'], {'missionGameID': entry['MissionID']}) - elif entry['event'] == 'MissionCompleted': + elif event_name == 'MissionCompleted': for x in entry.get('PermitsAwarded', []): add_event('addCommanderPermit', entry['timestamp'], {'starsystemName': x}) @@ -745,11 +748,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('setCommanderMissionCompleted', entry['timestamp'], data) - elif entry['event'] == 'MissionFailed': + elif event_name == 'MissionFailed': add_event('setCommanderMissionFailed', entry['timestamp'], {'missionGameID': entry['MissionID']}) # Combat - if entry['event'] == 'Died': + if event_name == 'Died': data = OrderedDict([('starsystemName', system)]) if 'Killers' in entry: data['wingOpponentNames'] = [x['Name'] for x in entry['Killers']] @@ -759,7 +762,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderCombatDeath', entry['timestamp'], data) - elif entry['event'] == 'Interdicted': + elif event_name == 'Interdicted': data = OrderedDict([('starsystemName', system), ('isPlayer', entry['IsPlayer']), ('isSubmit', entry['Submitted']), @@ -776,7 +779,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderCombatInterdicted', entry['timestamp'], data) - elif entry['event'] == 'Interdiction': + elif event_name == 'Interdiction': data = OrderedDict([('starsystemName', system), ('isPlayer', entry['IsPlayer']), ('isSuccess', entry['Success']), @@ -793,21 +796,21 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderCombatInterdiction', entry['timestamp'], data) - elif entry['event'] == 'EscapeInterdiction': + elif event_name == 'EscapeInterdiction': add_event('addCommanderCombatInterdictionEscape', entry['timestamp'], OrderedDict([('starsystemName', system), ('opponentName', entry['Interdictor']), ('isPlayer', entry['IsPlayer']), ])) - elif entry['event'] == 'PVPKill': + elif event_name == 'PVPKill': add_event('addCommanderCombatKill', entry['timestamp'], OrderedDict([('starsystemName', system), ('opponentName', entry['Victim']), ])) # Community Goals - if entry['event'] == 'CommunityGoal': + if event_name == 'CommunityGoal': this.events = [x for x in this.events if x['eventName'] not in [ 'setCommunityGoal', 'setCommanderCommunityGoalProgress']] # Remove any unsent @@ -850,7 +853,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('setCommanderCommunityGoalProgress', entry['timestamp'], data) # Friends - if entry['event'] == 'Friends': + if event_name == 'Friends': if entry['Status'] in ['Added', 'Online']: add_event('addCommanderFriend', entry['timestamp'], OrderedDict([('commanderName', entry['Name']), From 539580e38dc9449d1897fe06151afef0bb5904bc Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 20:55:23 +0200 Subject: [PATCH 207/258] Consistently formatted multi-line add_event calls --- plugins/inara.py | 409 ++++++++++++++++++++++++++++------------------- 1 file changed, 246 insertions(+), 163 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 951deea6..8584a1aa 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -13,7 +13,6 @@ from operator import itemgetter from queue import Queue from threading import Thread import logging -import dataclasses import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel @@ -145,7 +144,7 @@ def plugin_prefs(parent: tk.Tk, cmdr: str, is_beta: bool): this.log_button.grid(columnspan=2, padx=BUTTONX, pady=(5, 0), sticky=tk.W) nb.Label(frame).grid(sticky=tk.W) # big spacer - + # Section heading in settings this.label = HyperlinkLabel( frame, @@ -318,38 +317,48 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.newsession = False # Send rank info to Inara on startup - add_event('setCommanderRankPilot', entry['timestamp'], - [ - OrderedDict([ - ('rankName', k.lower()), - ('rankValue', v[0]), - ('rankProgress', v[1] / 100.0), - ]) for k, v in state['Rank'].items() if v is not None - ]) + add_event( + 'setCommanderRankPilot', + entry['timestamp'], + [ + OrderedDict([ + ('rankName', k.lower()), + ('rankValue', v[0]), + ('rankProgress', v[1] / 100.0), + ]) for k, v in state['Rank'].items() if v is not None + ] + ) - add_event('setCommanderReputationMajorFaction', entry['timestamp'], - [ - OrderedDict([ - ('majorfactionName', k.lower()), - ('majorfactionReputation', v / 100.0), - ]) for k, v in state['Reputation'].items() if v is not None - ]) + add_event( + 'setCommanderReputationMajorFaction', + entry['timestamp'], + [ + OrderedDict([('majorfactionName', k.lower()), ('majorfactionReputation', v / 100.0)]) + for k, v in state['Reputation'].items() if v is not None + ] + ) if state['Engineers']: # Not populated < 3.3 - add_event('setCommanderRankEngineer', entry['timestamp'], - [ - OrderedDict([ - ('engineerName', k), - type(v) is tuple and ('rankValue', v[0]) or ('rankStage', v), - ]) for k, v in state['Engineers'].items() - ]) + add_event( + 'setCommanderRankEngineer', + entry['timestamp'], + [ + OrderedDict( + [('engineerName', k), isinstance(v, tuple) and ('rankValue', v[0]) or ('rankStage', v)] + ) + for k, v in state['Engineers'].items() + ] + ) # Update location - add_event('setCommanderTravelLocation', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('stationName', station), # Can be None - ])) + add_event( + 'setCommanderTravelLocation', + entry['timestamp'], + OrderedDict([ + ('starsystemName', system), + ('stationName', station), # Can be None + ]) + ) # Update ship if state['ShipID']: # Unknown if started in Fighter or SRV @@ -379,41 +388,56 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif event_name == 'Promotion': for k, v in state['Rank'].items(): if k in entry: - add_event('setCommanderRankPilot', entry['timestamp'], - OrderedDict([ - ('rankName', k.lower()), - ('rankValue', v[0]), - ('rankProgress', 0), - ])) + add_event( + 'setCommanderRankPilot', + entry['timestamp'], + OrderedDict([ + ('rankName', k.lower()), + ('rankValue', v[0]), + ('rankProgress', 0), + ]) + ) elif event_name == 'EngineerProgress' and 'Engineer' in entry: - add_event('setCommanderRankEngineer', entry['timestamp'], - OrderedDict([ - ('engineerName', entry['Engineer']), - 'Rank' in entry and ('rankValue', entry['Rank']) or ('rankStage', entry['Progress']), - ])) + add_event( + 'setCommanderRankEngineer', + entry['timestamp'], + OrderedDict([ + ('engineerName', entry['Engineer']), + ('rankValue', entry['Rank']) if 'Rank' in entry else ('rankStage', entry['Progress']), + ]) + ) # PowerPlay status change if event_name == 'PowerplayJoin': - add_event('setCommanderRankPower', entry['timestamp'], - OrderedDict([ - ('powerName', entry['Power']), - ('rankValue', 1), - ])) + add_event( + 'setCommanderRankPower', + entry['timestamp'], + OrderedDict([ + ('powerName', entry['Power']), + ('rankValue', 1), + ]) + ) elif event_name == 'PowerplayLeave': - add_event('setCommanderRankPower', entry['timestamp'], - OrderedDict([ - ('powerName', entry['Power']), - ('rankValue', 0), - ])) + add_event( + 'setCommanderRankPower', + entry['timestamp'], + OrderedDict([ + ('powerName', entry['Power']), + ('rankValue', 0), + ]) + ) elif event_name == 'PowerplayDefect': - add_event('setCommanderRankPower', entry['timestamp'], - OrderedDict([ - ('powerName', entry['ToPower']), - ('rankValue', 1), - ])) + add_event( + 'setCommanderRankPower', + entry['timestamp'], + OrderedDict([ + ('powerName', entry['ToPower']), + ('rankValue', 1), + ]) + ) # Ship change if event_name == 'Loadout' and this.shipswap: @@ -424,10 +448,13 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ('shipIdent', state['ShipIdent']), # Can be None ('isCurrentShip', True), ]) + if state['HullValue']: data['shipHullValue'] = state['HullValue'] + if state['ModulesValue']: data['shipModulesValue'] = state['ModulesValue'] + data['shipRebuyCost'] = state['Rebuy'] add_event('setCommanderShip', entry['timestamp'], data) @@ -440,17 +467,23 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if this.undocked: # Undocked and now docking again. Don't send. this.undocked = False + elif this.suppress_docked: # Don't send initial Docked event on new game this.suppress_docked = False + else: - add_event('addCommanderTravelDock', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('stationName', station), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ])) + add_event( + 'addCommanderTravelDock', + entry['timestamp'], + OrderedDict([ + ('starsystemName', system), + ('stationName', station), + ('shipType', state['ShipType']), + ('shipGameID', state['ShipID']), + ]) + ) + elif event_name == 'Undocked': this.undocked = True this.station = None @@ -458,51 +491,67 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif event_name == 'SupercruiseEntry': if this.undocked: # Staying in system after undocking - send any pending events from in-station action - add_event('setCommanderTravelLocation', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ])) + add_event( + 'setCommanderTravelLocation', + entry['timestamp'], + OrderedDict([ + ('starsystemName', system), + ('shipType', state['ShipType']), + ('shipGameID', state['ShipID']), + ]) + ) + this.undocked = False elif event_name == 'FSDJump': this.undocked = False - add_event('addCommanderTravelFSDJump', entry['timestamp'], - OrderedDict([ - ('starsystemName', entry['StarSystem']), - ('jumpDistance', entry['JumpDist']), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ])) + add_event( + 'addCommanderTravelFSDJump', + entry['timestamp'], + OrderedDict([ + ('starsystemName', entry['StarSystem']), + ('jumpDistance', entry['JumpDist']), + ('shipType', state['ShipType']), + ('shipGameID', state['ShipID']), + ]) + ) if entry.get('Factions'): - add_event('setCommanderReputationMinorFaction', entry['timestamp'], - [ - OrderedDict([ - ('minorfactionName', f['Name']), - ('minorfactionReputation', f['MyReputation']/100.0), - ]) for f in entry['Factions'] - ]) + add_event( + 'setCommanderReputationMinorFaction', + entry['timestamp'], + [ + OrderedDict( + [('minorfactionName', f['Name']), ('minorfactionReputation', f['MyReputation']/100.0)] + ) + for f in entry['Factions'] + ] + ) elif event_name == 'CarrierJump': - add_event('addCommanderTravelCarrierJump', entry['timestamp'], - OrderedDict([ - ('starsystemName', entry['StarSystem']), - ('stationName', entry['StationName']), - ('marketID', entry['MarketID']), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ])) + add_event( + 'addCommanderTravelCarrierJump', + entry['timestamp'], + OrderedDict([ + ('starsystemName', entry['StarSystem']), + ('stationName', entry['StationName']), + ('marketID', entry['MarketID']), + ('shipType', state['ShipType']), + ('shipGameID', state['ShipID']), + ]) + ) if entry.get('Factions'): - add_event('setCommanderReputationMinorFaction', entry['timestamp'], - [ - OrderedDict([ - ('minorfactionName', f['Name']), - ('minorfactionReputation', f['MyReputation']/100.0), - ]) for f in entry['Factions'] - ]) + add_event( + 'setCommanderReputationMinorFaction', + entry['timestamp'], + [ + OrderedDict( + [('minorfactionName', f['Name']), ('minorfactionReputation', f['MyReputation']/100.0)] + ) + for f in entry['Factions'] + ] + ) # Ignore the following 'Docked' event this.suppress_docked = True @@ -516,8 +565,9 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di materials = [] for category in ('Raw', 'Manufactured', 'Encoded'): - materials.extend([OrderedDict([('itemName', k), ('itemCount', state[category][k])]) - for k in sorted(state[category])]) + materials.extend( + [OrderedDict([('itemName', k), ('itemCount', state[category][k])]) for k in sorted(state[category])] + ) if this.materials != materials: add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) @@ -529,11 +579,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Send credits and stats to Inara on startup only - otherwise may be out of date if event_name == 'LoadGame': - add_event('setCommanderCredits', entry['timestamp'], - OrderedDict([ - ('commanderCredits', state['Credits']), - ('commanderLoan', state['Loan']), - ])) + add_event( + 'setCommanderCredits', + entry['timestamp'], + OrderedDict([('commanderCredits', state['Credits']), ('commanderLoan', state['Loan'])]) + ) this.lastcredits = state['Credits'] @@ -542,11 +592,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Selling / swapping ships if event_name == 'ShipyardNew': - add_event('addCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', entry['ShipType']), - ('shipGameID', entry['NewShipID']), - ])) + add_event( + 'addCommanderShip', + entry['timestamp'], + OrderedDict([('shipType', entry['ShipType']), ('shipGameID', entry['NewShipID'])]) + ) this.shipswap = True # Want subsequent Loadout event to be sent immediately @@ -555,40 +605,52 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.shipswap = True # Don't know new ship name and ident 'til the following Loadout event if 'StoreShipID' in entry: - add_event('setCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', entry['StoreOldShip']), - ('shipGameID', entry['StoreShipID']), - ('starsystemName', system), - ('stationName', station), - ])) + add_event( + 'setCommanderShip', + entry['timestamp'], + OrderedDict([ + ('shipType', entry['StoreOldShip']), + ('shipGameID', entry['StoreShipID']), + ('starsystemName', system), + ('stationName', station), + ]) + ) elif 'SellShipID' in entry: - add_event('delCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', entry.get('SellOldShip', entry['ShipType'])), - ('shipGameID', entry['SellShipID']), - ])) + add_event( + 'delCommanderShip', + entry['timestamp'], + OrderedDict([ + ('shipType', entry.get('SellOldShip', entry['ShipType'])), + ('shipGameID', entry['SellShipID']), + ]) + ) elif event_name == 'SetUserShipName': - add_event('setCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None - ('isCurrentShip', True), - ])) + add_event( + 'setCommanderShip', + entry['timestamp'], + OrderedDict([ + ('shipType', state['ShipType']), + ('shipGameID', state['ShipID']), + ('shipName', state['ShipName']), # Can be None + ('shipIdent', state['ShipIdent']), # Can be None + ('isCurrentShip', True), + ]) + ) elif event_name == 'ShipyardTransfer': - add_event('setCommanderShipTransfer', entry['timestamp'], - OrderedDict([ - ('shipType', entry['ShipType']), - ('shipGameID', entry['ShipID']), - ('starsystemName', system), - ('stationName', station), - ('transferTime', entry['TransferTime']), - ])) + add_event( + 'setCommanderShipTransfer', + entry['timestamp'], + OrderedDict([ + ('shipType', entry['ShipType']), + ('shipGameID', entry['ShipID']), + ('starsystemName', system), + ('stationName', station), + ('transferTime', entry['TransferTime']), + ]) + ) # Fleet if event_name == 'StoredShips': @@ -780,10 +842,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderCombatInterdicted', entry['timestamp'], data) elif event_name == 'Interdiction': - data = OrderedDict([('starsystemName', system), - ('isPlayer', entry['IsPlayer']), - ('isSuccess', entry['Success']), - ]) + data = OrderedDict([ + ('starsystemName', system), + ('isPlayer', entry['IsPlayer']), + ('isSuccess', entry['Success']), + ]) if 'Interdicted' in entry: data['opponentName'] = entry['Interdicted'] @@ -797,17 +860,25 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderCombatInterdiction', entry['timestamp'], data) elif event_name == 'EscapeInterdiction': - add_event('addCommanderCombatInterdictionEscape', entry['timestamp'], - OrderedDict([('starsystemName', system), - ('opponentName', entry['Interdictor']), - ('isPlayer', entry['IsPlayer']), - ])) + add_event( + 'addCommanderCombatInterdictionEscape', + entry['timestamp'], + OrderedDict([ + ('starsystemName', system), + ('opponentName', entry['Interdictor']), + ('isPlayer', entry['IsPlayer']), + ]) + ) elif event_name == 'PVPKill': - add_event('addCommanderCombatKill', entry['timestamp'], - OrderedDict([('starsystemName', system), - ('opponentName', entry['Victim']), - ])) + add_event( + 'addCommanderCombatKill', + entry['timestamp'], + OrderedDict([ + ('starsystemName', system), + ('opponentName', entry['Victim']), + ]) + ) # Community Goals if event_name == 'CommunityGoal': @@ -855,16 +926,24 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Friends if event_name == 'Friends': if entry['Status'] in ['Added', 'Online']: - add_event('addCommanderFriend', entry['timestamp'], - OrderedDict([('commanderName', entry['Name']), - ('gamePlatform', 'pc'), - ])) + add_event( + 'addCommanderFriend', + entry['timestamp'], + OrderedDict([ + ('commanderName', entry['Name']), + ('gamePlatform', 'pc'), + ]) + ) elif entry['Status'] in ['Declined', 'Lost']: - add_event('delCommanderFriend', entry['timestamp'], - OrderedDict([('commanderName', entry['Name']), - ('gamePlatform', 'pc'), - ])) + add_event( + 'delCommanderFriend', + entry['timestamp'], + OrderedDict([ + ('commanderName', entry['Name']), + ('gamePlatform', 'pc'), + ]) + ) this.newuser = False @@ -920,11 +999,14 @@ def cmdr_data(data, is_beta): if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(this.cmdr): if not (CREDIT_RATIO > this.lastcredits / data['commander']['credits'] > 1/CREDIT_RATIO): this.events = [x for x in this.events if x['eventName'] != 'setCommanderCredits'] # Remove any unsent - add_event('setCommanderCredits', data['timestamp'], - OrderedDict([ - ('commanderCredits', data['commander']['credits']), - ('commanderLoan', data['commander'].get('debt', 0)), - ])) + add_event( + 'setCommanderCredits', + data['timestamp'], + OrderedDict([ + ('commanderCredits', data['commander']['credits']), + ('commanderLoan', data['commander'].get('debt', 0)), + ]) + ) this.lastcredits = float(data['commander']['credits']) @@ -975,6 +1057,7 @@ def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: else: modifier['value'] = mod['ValueStr'] + engineering['modifiers'].append(modifier) module['engineering'] = engineering From 820b29f7dda2c5fda2cb82a5c0c437031d58310c Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 28 Jul 2020 21:02:16 +0200 Subject: [PATCH 208/258] replace dict(list-comp) with dict comp --- plugins/inara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index 8584a1aa..73416223 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -696,7 +696,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Stored modules if event_name == 'StoredModules': - items = dict([(x['StorageSlot'], x) for x in entry['Items']]) # Impose an order + items = {mod['StorageSlot']: mod for mod in entry['Items']} # Impose an order modules = [] for slot in sorted(items): item = items[slot] From e74c5c8ceb6d63c527bb401a122d3865fc908ca3 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 17:14:57 +0200 Subject: [PATCH 209/258] replaced list comp with filter --- plugins/inara.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 73416223..74d68b7a 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -882,8 +882,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Community Goals if event_name == 'CommunityGoal': - this.events = [x for x in this.events if x['eventName'] not in [ - 'setCommunityGoal', 'setCommanderCommunityGoalProgress']] # Remove any unsent + # Remove any unsent + this.events = list(filter( + lambda e: e['eventName'] not in ('setCommunityGoal', 'setCommanderCommunityGoalProgress'), + this.events + )) for goal in entry['CurrentGoals']: data = OrderedDict([ From a699ab062d9e7cc802f68d65a091ad1886ad8df6 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 17:15:17 +0200 Subject: [PATCH 210/258] fixed station link logic --- plugins/inara.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 74d68b7a..626c7ca4 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -958,8 +958,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if config.get('station_provider') == 'Inara': to_set = this.station - if not to_set and this.system_population is not None: - if this.system_population > 0: + if not to_set: + if this.system_population is not None and this.system_population > 0: to_set = STATION_UNDOCKED else: to_set = '' From bb1def48cd44666f06cf701613e41b4123ec651b Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 17:16:38 +0200 Subject: [PATCH 211/258] replaced list comp with filter --- plugins/inara.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 626c7ca4..6ec9bbc0 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -726,8 +726,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if this.storedmodules != modules: # Only send on change this.storedmodules = modules - this.events = [x for x in this.events if x['eventName'] - != 'setCommanderStorageModules'] # Remove any unsent + # Remove any unsent + this.events = list(filter(lambda e: e['eventName'] != 'setCommanderStorageModules', this.events)) add_event('setCommanderStorageModules', entry['timestamp'], this.storedmodules) # Missions From 90057988620e63b63eff6adfd60c40c28c9d0e0e Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 17:17:00 +0200 Subject: [PATCH 212/258] fixed incorrect comment --- plugins/inara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index 6ec9bbc0..c75f3004 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -156,7 +156,7 @@ def plugin_prefs(parent: tk.Tk, cmdr: str, is_beta: bool): this.label.grid(columnspan=2, padx=PADX, sticky=tk.W) - this.apikey_label = nb.Label(frame, text=_('API Key')) # EDSM setting + this.apikey_label = nb.Label(frame, text=_('API Key')) # Inara setting this.apikey_label.grid(row=12, padx=PADX, sticky=tk.W) this.apikey = nb.Entry(frame) this.apikey.grid(row=12, column=1, padx=PADX, pady=PADY, sticky=tk.EW) From 67a260c07f801046e8f428f1947b6087481ec4a0 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 17:18:38 +0200 Subject: [PATCH 213/258] added type hints to literals --- plugins/inara.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index c75f3004..d5a6cf66 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -42,18 +42,18 @@ this.lastship = None # eventData from the last addCommanderShip or setCommander this.events: List[OrderedDictT[str, Any]] = [] # Unsent events this.cmdr: Optional[str] = None this.FID: Optional[str] = None # Frontier ID -this.multicrew = False # don't send captain's ship info to Inara while on a crew -this.newuser = False # just entered API Key - send state immediately -this.newsession = True # starting a new session - wait for Cargo event -this.undocked = False # just undocked +this.multicrew: bool = False # don't send captain's ship info to Inara while on a crew +this.newuser: bool = False # just entered API Key - send state immediately +this.newsession: bool = True # starting a new session - wait for Cargo event +this.undocked: bool = False # just undocked this.suppress_docked = False # Skip initial Docked event if started docked this.cargo: Optional[OrderedDictT[str, Any]] = None this.materials: Optional[OrderedDictT[str, Any]] = None -this.lastcredits = 0 # Send credit update soon after Startup / new game +this.lastcredits: int = 0 # Send credit update soon after Startup / new game this.storedmodules: Optional[OrderedDictT[str, Any]] = None this.loadout: Optional[OrderedDictT[str, Any]] = None this.fleet: Optional[List[OrderedDictT[str, Any]]] = None -this.shipswap = False # just swapped ship +this.shipswap: bool = False # just swapped ship # last time we updated, if unset in config this is 0, which means an instant update LAST_UPDATE_CONF_KEY = 'inara_last_update' @@ -1090,7 +1090,7 @@ def add_event(name: str, timestamp: str, data: Mapping[str, Any]): ])) -def call_timer(wait=FLOOD_LIMIT_SECONDS): +def call_timer(wait: int = FLOOD_LIMIT_SECONDS): """ call_timer runs in its own thread polling out to INARA once every FLOOD_LIMIT_SECONDS @@ -1101,8 +1101,6 @@ def call_timer(wait=FLOOD_LIMIT_SECONDS): if this.timer_run: # check again in here just in case we're closing and the stars align call() -# Queue a call to Inara, handled in Worker thread - def call(callback=None, force=False): """ From 446c81248584034ab3392186d5c79e5d5b1975b5 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 17:30:56 +0200 Subject: [PATCH 214/258] replaced oneline logic with multiline --- plugins/inara.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index d5a6cf66..82000157 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -978,7 +978,9 @@ def cmdr_data(data, is_beta): # Only trust CAPI if these aren't yet set this.system = this.system if this.system else data['lastSystem']['name'] - this.station = data['lastStarport']['name'] if data['commander']['docked'] and not this.station else this.station + + if not this.station and data['commander'['docked']]: + this.station = data['lastStarport']['name'] # Override standard URL functions if config.get('system_provider') == 'Inara': From eba318430a48d1399ee5915ff6738b6794bb3e71 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 29 Jul 2020 22:28:26 +0200 Subject: [PATCH 215/258] Removed todos about unused args --- plugins/inara.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 82000157..dc6b8fe4 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -1206,7 +1206,6 @@ def worker(): plug.show_error(_("Error: Can't connect to Inara")) -# TODO(A_D) event is unused def update_location(event=None): """ Call inara_notify_location in this and other interested plugins with Inara's response when changing system @@ -1223,7 +1222,6 @@ def inara_notify_location(eventData): pass -# TODO(A_D) event is unused def update_ship(event=None): """ Call inara_notify_ship() in interested plugins with Inara's response when changing ship From f8d4731472a1122c79590b2d77d34ec232a1fcdb Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 30 Jul 2020 14:32:43 +0200 Subject: [PATCH 216/258] fixed incorrect indexing --- plugins/inara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index dc6b8fe4..a8f5eda4 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -979,7 +979,7 @@ def cmdr_data(data, is_beta): # Only trust CAPI if these aren't yet set this.system = this.system if this.system else data['lastSystem']['name'] - if not this.station and data['commander'['docked']]: + if not this.station and data['commander']['docked']: this.station = data['lastStarport']['name'] # Override standard URL functions From 21a285e6e943e0b113eb576b85954df522fc7891 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 30 Jul 2020 14:38:19 +0200 Subject: [PATCH 217/258] added some more type hints --- plugins/inara.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index a8f5eda4..5a253bb5 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -3,9 +3,10 @@ # from collections import OrderedDict -from dataclasses import dataclass import json -from typing import Any, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT, TYPE_CHECKING +from typing import Any, AnyStr, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT, \ + Sequence, TYPE_CHECKING, Union + import requests import sys import time @@ -82,7 +83,7 @@ def system_url(system_name: str): return this.system -def station_url(system_name: str, station_name: str): +def station_url(system_name: Optional[str], station_name: Optional[str]): if system_name: if station_name: return requests.utils.requote_uri( @@ -700,7 +701,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di modules = [] for slot in sorted(items): item = items[slot] - module = OrderedDict([ + module: OrderedDictT[str, Any] = OrderedDict([ ('itemName', item['Name']), ('itemValue', item['BuyPrice']), ('isHot', item['Hot']), @@ -791,7 +792,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di factioneffects = [] for faction in entry.get('FactionEffects', []): - effect = OrderedDict([('minorfactionName', faction['Faction'])]) + effect: OrderedDictT[str, Any] = OrderedDict([('minorfactionName', faction['Faction'])]) for influence in faction.get('Influence', []): if 'Influence' in influence: highest_gain = influence['Influence'] @@ -842,7 +843,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di add_event('addCommanderCombatInterdicted', entry['timestamp'], data) elif event_name == 'Interdiction': - data = OrderedDict([ + data: OrderedDictT[str, Any] = OrderedDict([ ('starsystemName', system), ('isPlayer', entry['IsPlayer']), ('isSuccess', entry['Success']), @@ -1019,7 +1020,7 @@ def cmdr_data(data, is_beta): def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: modules = [] for m in state['Modules'].values(): - module = OrderedDict([ + module: OrderedDictT[str, Any] = OrderedDict([ ('slotName', m['Slot']), ('itemName', m['Item']), ('itemHealth', m['Health']), @@ -1040,7 +1041,7 @@ def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: module['isHot'] = m['Hot'] if 'Engineering' in m: - engineering = OrderedDict([ + engineering: OrderedDictT[str, Any] = OrderedDict([ ('blueprintName', m['Engineering']['BlueprintName']), ('blueprintLevel', m['Engineering']['Level']), ('blueprintQuality', m['Engineering']['Quality']), @@ -1051,7 +1052,7 @@ def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: engineering['modifiers'] = [] for mod in m['Engineering']['Modifiers']: - modifier = OrderedDict([ + modifier: OrderedDictT[str, Any] = OrderedDict([ ('name', mod['Label']), ]) @@ -1076,7 +1077,10 @@ def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: ]) -def add_event(name: str, timestamp: str, data: Mapping[str, Any]): +EVENT_DATA = Mapping[AnyStr, Any] + + +def add_event(name: str, timestamp: str, data: Union[EVENT_DATA, Sequence[EVENT_DATA]]): """ Add an event to the event queue From 81dfbdfb8e1544d2d6af985b443184a781778889 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 30 Jul 2020 19:49:54 +0200 Subject: [PATCH 218/258] Added timeout_session, made inara use it timeout_session provides two things, TimeoutAdapter, a HTTP adapter subclass that automatically adds timeouts to all requests, and new_session, which automatically creates a request.Session with the adapter in the correct place. --- PLUGINS.md | 3 +++ plugins/inara.py | 3 ++- timeout_session.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 timeout_session.py diff --git a/PLUGINS.md b/PLUGINS.md index 8a43bf6f..8e1a8d77 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -47,6 +47,9 @@ breaking with future code changes.** `from monitor import gamerunning` - in case a plugin needs to know if we think the game is running. + +`import timeout_session` - provides a method called `new_session` that creates a requests.session with a default timeout +on all requests. Recommended to reduce noise in HTTP requests ```python diff --git a/plugins/inara.py b/plugins/inara.py index 5a253bb5..5cd381c0 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -21,6 +21,7 @@ import myNotebook as nb from config import appname, applongname, appversion, config import plug +import timeout_session logger = logging.getLogger(appname) if TYPE_CHECKING: @@ -34,7 +35,7 @@ CREDIT_RATIO = 1.05 # Update credits if they change by 5% over the course of a this: Any = sys.modules[__name__] # For holding module globals -this.session = requests.Session() +this.session = timeout_session.new_session() this.queue = Queue() # Items to be sent to Inara by worker thread this.lastlocation = None # eventData from the last Commander's Flight Log event this.lastship = None # eventData from the last addCommanderShip or setCommanderShip event diff --git a/timeout_session.py b/timeout_session.py new file mode 100644 index 00000000..dea65dfa --- /dev/null +++ b/timeout_session.py @@ -0,0 +1,40 @@ + +import requests +from requests.adapters import HTTPAdapter + +REQUEST_TIMEOUT = 10 # reasonable timeout that all HTTP requests should use + + +class TimeoutAdapter(HTTPAdapter): + """ + TimeoutAdapter is an HTTP Adapter that enforces an overridable default timeout on HTTP requests. + """ + def __init__(self, timeout, *args, **kwargs): + self.default_timeout = timeout + if kwargs.get("timeout") is not None: + del kwargs["timeout"] + + super().__init__(*args, **kwargs) + + def send(self, *args, **kwargs): + if kwargs["timeout"] is None: + kwargs["timeout"] = self.default_timeout + + super().send(*args, **kwargs) + + +def new_session(timeout: int = REQUEST_TIMEOUT, session: requests.Session = None) -> requests.Session: + """ + new_session creates a new requests.Session and overrides the default HTTPAdapter with a TimeoutAdapter. + + :param timeout: the timeout to set the TimeoutAdapter to, defaults to REQUEST_TIMEOUT + :param session: the Session object to attach the Adapter to, defaults to a new session + :return: The created Session + """ + if session is None: + session = requests.Session() + + adapter = TimeoutAdapter(timeout) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session From faac29fe844150731f6d9f242d290aa54705c41b Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 31 Jul 2020 15:52:05 +0200 Subject: [PATCH 219/258] Made sure to return super().send() --- timeout_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timeout_session.py b/timeout_session.py index dea65dfa..3da5e3ab 100644 --- a/timeout_session.py +++ b/timeout_session.py @@ -20,7 +20,7 @@ class TimeoutAdapter(HTTPAdapter): if kwargs["timeout"] is None: kwargs["timeout"] = self.default_timeout - super().send(*args, **kwargs) + return super().send(*args, **kwargs) def new_session(timeout: int = REQUEST_TIMEOUT, session: requests.Session = None) -> requests.Session: From 0be9bb7a6d52c2ddcec530161a66bde6b6969a3c Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 6 Aug 2020 02:21:27 +0200 Subject: [PATCH 220/258] renamed pyproject.toml, added flake8-isort --- .pyproject.toml | 2 -- pyproject.toml | 6 ++++++ requirements-dev.txt | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 .pyproject.toml create mode 100644 pyproject.toml diff --git a/.pyproject.toml b/.pyproject.toml deleted file mode 100644 index 7ff18c21..00000000 --- a/.pyproject.toml +++ /dev/null @@ -1,2 +0,0 @@ -[tool.autopep8] -max_line_length = 120 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..5b2d299b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[tool.autopep8] +max_line_length = 120 + +[tool.isort] +multi_line_output = 5 +line_length = 119 diff --git a/requirements-dev.txt b/requirements-dev.txt index 433b140d..cdd511d1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,7 @@ flake8-comprehensions==3.2.3 flake8-pep3101==1.3.0 flake8-polyfill==1.0.2 flake8-json +flake8-isort==3.0.1 pep8-naming==0.11.1 # Code formatting tools From b186e977477f63ab6fc13d2c9ef2a1e317591c2c Mon Sep 17 00:00:00 2001 From: A_D Date: Sat, 1 Aug 2020 21:36:50 +0200 Subject: [PATCH 221/258] autoformatted code --- plugins/eddn.py | 118 ++++++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 54 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index b65ac034..68e1dbb7 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -38,19 +38,19 @@ this.commodities = this.outfitting = this.shipyard = None class EDDN(object): - ### SERVER = 'http://localhost:8081' # testing + # SERVER = 'http://localhost:8081' # testing SERVER = 'https://eddn.edcd.io:4430' UPLOAD = '%s/upload/' % SERVER - REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] - REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds - TIMEOUT= 10 # requests timeout + REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] + REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds + TIMEOUT = 10 # requests timeout MODULE_RE = re.compile('^Hpt_|^Int_|Armour_', re.IGNORECASE) CANONICALISE_RE = re.compile(r'\$(.+)_name;') def __init__(self, parent): self.parent = parent self.session = requests.Session() - self.replayfile = None # For delayed messages + self.replayfile = None # For delayed messages self.replaylog = [] def load(self): @@ -94,7 +94,7 @@ class EDDN(object): msg = OrderedDict([ ('$schemaRef', msg['$schemaRef']), ('header', OrderedDict([ - ('softwareName', '%s [%s]' % (applongname, sys.platform=='darwin' and "Mac OS" or system())), + ('softwareName', '%s [%s]' % (applongname, sys.platform == 'darwin' and "Mac OS" or system())), ('softwareVersion', appversion), ('uploaderID', uploaderID), ])), @@ -119,7 +119,7 @@ class EDDN(object): if len(self.replaylog) == 1: status['text'] = _('Sending data to EDDN...') else: - status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...',''), len(self.replaylog)) + status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...', ''), len(self.replaylog)) self.parent.update_idletasks() try: cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict) @@ -166,9 +166,9 @@ class EDDN(object): ])) if commodity['statusFlags']: commodities[-1]['statusFlags'] = commodity['statusFlags'] - commodities.sort(key = lambda c: c['name']) + commodities.sort(key=lambda c: c['name']) - if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it + if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it message = OrderedDict([ ('timestamp', data['timestamp']), ('systemName', data['lastSystem']['name']), @@ -181,24 +181,25 @@ class EDDN(object): if 'prohibited' in data['lastStarport']: message['prohibited'] = sorted(list([x for x in (data['lastStarport']['prohibited'] or {}).values()])) self.send(data['commander']['name'], { - '$schemaRef' : 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), - 'message' : message, + '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), + 'message': message, }) this.commodities = commodities def export_outfitting(self, data, is_beta): economies = data['lastStarport'].get('economies') or {} modules = data['lastStarport'].get('modules') or {} - ships = data['lastStarport'].get('ships') or { 'shipyard_list': {}, 'unavailable_list': [] } + ships = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} # Horizons flag - will hit at least Int_PlanetApproachSuite other than at engineer bases ("Colony"), prison or rescue Megaships, or under Pirate Attack etc horizons = (any(economy['name'] == 'Colony' for economy in economies.values()) or any(module.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for module in modules.values()) or any(ship.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for ship in list((ships['shipyard_list'] or {}).values()))) outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['name'].lower()) for module in modules.values() if self.MODULE_RE.search(module['name']) and module.get('sku') in [None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'] and module['name'] != 'Int_PlanetApproachSuite']) - if outfitting and this.outfitting != (horizons, outfitting): # Don't send empty modules list - schema won't allow it + # Don't send empty modules list - schema won't allow it + if outfitting and this.outfitting != (horizons, outfitting): self.send(data['commander']['name'], { - '$schemaRef' : 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''), - 'message' : OrderedDict([ + '$schemaRef': 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''), + 'message': OrderedDict([ ('timestamp', data['timestamp']), ('systemName', data['lastSystem']['name']), ('stationName', data['lastStarport']['name']), @@ -212,15 +213,16 @@ class EDDN(object): def export_shipyard(self, data, is_beta): economies = data['lastStarport'].get('economies') or {} modules = data['lastStarport'].get('modules') or {} - ships = data['lastStarport'].get('ships') or { 'shipyard_list': {}, 'unavailable_list': [] } + ships = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} horizons = (any(economy['name'] == 'Colony' for economy in economies.values()) or any(module.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for module in modules.values()) or any(ship.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for ship in list((ships['shipyard_list'] or {}).values()))) shipyard = sorted([ship['name'].lower() for ship in list((ships['shipyard_list'] or {}).values()) + ships['unavailable_list']]) - if shipyard and this.shipyard != (horizons, shipyard): # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. + # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. + if shipyard and this.shipyard != (horizons, shipyard): self.send(data['commander']['name'], { - '$schemaRef' : 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''), - 'message' : OrderedDict([ + '$schemaRef': 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''), + 'message': OrderedDict([ ('timestamp', data['timestamp']), ('systemName', data['lastSystem']['name']), ('stationName', data['lastStarport']['name']), @@ -242,12 +244,12 @@ class EDDN(object): ('sellPrice', commodity['SellPrice']), ('demand', commodity['Demand']), ('demandBracket', commodity['DemandBracket']), - ]) for commodity in items], key = lambda c: c['name']) + ]) for commodity in items], key=lambda c: c['name']) - if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it + if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it self.send(cmdr, { - '$schemaRef' : 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), - 'message' : OrderedDict([ + '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), + 'message': OrderedDict([ ('timestamp', entry['timestamp']), ('systemName', entry['StarSystem']), ('stationName', entry['StationName']), @@ -261,10 +263,11 @@ class EDDN(object): modules = entry.get('Items') or [] horizons = entry.get('Horizons', False) outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['Name']) for module in modules if module['Name'] != 'int_planetapproachsuite']) - if outfitting and this.outfitting != (horizons, outfitting): # Don't send empty modules list - schema won't allow it + # Don't send empty modules list - schema won't allow it + if outfitting and this.outfitting != (horizons, outfitting): self.send(cmdr, { - '$schemaRef' : 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''), - 'message' : OrderedDict([ + '$schemaRef': 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''), + 'message': OrderedDict([ ('timestamp', entry['timestamp']), ('systemName', entry['StarSystem']), ('stationName', entry['StationName']), @@ -279,10 +282,11 @@ class EDDN(object): ships = entry.get('PriceList') or [] horizons = entry.get('Horizons', False) shipyard = sorted([ship['ShipType'] for ship in ships]) - if shipyard and this.shipyard != (horizons, shipyard): # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. + # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. + if shipyard and this.shipyard != (horizons, shipyard): self.send(cmdr, { - '$schemaRef' : 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''), - 'message' : OrderedDict([ + '$schemaRef': 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''), + 'message': OrderedDict([ ('timestamp', entry['timestamp']), ('systemName', entry['StarSystem']), ('stationName', entry['StationName']), @@ -295,8 +299,8 @@ class EDDN(object): def export_journal_entry(self, cmdr, is_beta, entry): msg = { - '$schemaRef' : 'https://eddn.edcd.io/schemas/journal/1' + (is_beta and '/test' or ''), - 'message' : entry + '$schemaRef': 'https://eddn.edcd.io/schemas/journal/1' + (is_beta and '/test' or ''), + 'message': entry } if self.replayfile or self.load(): # Store the entry @@ -304,9 +308,8 @@ class EDDN(object): self.replayfile.write('%s\n' % self.replaylog[-1]) if (entry['event'] == 'Docked' or - (entry['event'] == 'Location' and entry['Docked']) or - not (config.getint('output') & config.OUT_SYS_DELAY)): - self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries + (entry['event'] == 'Location' and entry['Docked']) or not (config.getint('output') & config.OUT_SYS_DELAY)): + self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries else: # Can't access replay file! Send immediately. status = self.parent.children['status'] @@ -325,54 +328,61 @@ class EDDN(object): def plugin_start3(plugin_dir): return 'EDDN' + def plugin_app(parent): this.parent = parent this.eddn = EDDN(parent) # Try to obtain exclusive lock on journal cache, even if we don't need it yet if not this.eddn.load(): - this.status['text'] = 'Error: Is another copy of this app already running?' # Shouldn't happen - don't bother localizing + # Shouldn't happen - don't bother localizing + this.status['text'] = 'Error: Is another copy of this app already running?' + def plugin_prefs(parent, cmdr, is_beta): PADX = 10 - BUTTONX = 12 # indent Checkbuttons and Radiobuttons + BUTTONX = 12 # indent Checkbuttons and Radiobuttons PADY = 2 # close spacing if prefsVersion.shouldSetDefaults('0.0.0.0', not bool(config.getint('output'))): - output = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings + output = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings else: output = config.getint('output') eddnframe = nb.Frame(parent) - HyperlinkLabel(eddnframe, text='Elite Dangerous Data Network', background=nb.Label().cget('background'), url='https://github.com/EDSM-NET/EDDN/wiki', underline=True).grid(padx=PADX, sticky=tk.W) # Don't translate - this.eddn_station= tk.IntVar(value = (output & config.OUT_MKT_EDDN) and 1) - this.eddn_station_button = nb.Checkbutton(eddnframe, text=_('Send station data to the Elite Dangerous Data Network'), variable=this.eddn_station, command=prefsvarchanged) # Output setting - this.eddn_station_button.grid(padx=BUTTONX, pady=(5,0), sticky=tk.W) - this.eddn_system = tk.IntVar(value = (output & config.OUT_SYS_EDDN) and 1) - this.eddn_system_button = nb.Checkbutton(eddnframe, text=_('Send system and scan data to the Elite Dangerous Data Network'), variable=this.eddn_system, command=prefsvarchanged) # Output setting new in E:D 2.2 - this.eddn_system_button.grid(padx=BUTTONX, pady=(5,0), sticky=tk.W) - this.eddn_delay= tk.IntVar(value = (output & config.OUT_SYS_DELAY) and 1) - this.eddn_delay_button = nb.Checkbutton(eddnframe, text=_('Delay sending until docked'), variable=this.eddn_delay) # Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2 - this.eddn_delay_button.grid(padx=BUTTONX, sticky=tk.W) + HyperlinkLabel(eddnframe, text='Elite Dangerous Data Network', background=nb.Label().cget('background'), url='https://github.com/EDSM-NET/EDDN/wiki', underline=True).grid(padx=PADX, sticky=tk.W) # Don't translate + this.eddn_station = tk.IntVar(value=(output & config.OUT_MKT_EDDN) and 1) + this.eddn_station_button = nb.Checkbutton(eddnframe, text=_('Send station data to the Elite Dangerous Data Network'), variable=this.eddn_station, command=prefsvarchanged) # Output setting + this.eddn_station_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) + this.eddn_system = tk.IntVar(value=(output & config.OUT_SYS_EDDN) and 1) + this.eddn_system_button = nb.Checkbutton(eddnframe, text=_('Send system and scan data to the Elite Dangerous Data Network'), variable=this.eddn_system, command=prefsvarchanged) # Output setting new in E:D 2.2 + this.eddn_system_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) + this.eddn_delay = tk.IntVar(value=(output & config.OUT_SYS_DELAY) and 1) + # Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2 + this.eddn_delay_button = nb.Checkbutton(eddnframe, text=_('Delay sending until docked'), variable=this.eddn_delay) this.eddn_delay_button.grid(padx=BUTTONX, sticky=tk.W) return eddnframe + def prefsvarchanged(event=None): this.eddn_station_button['state'] = tk.NORMAL - this.eddn_system_button['state']= tk.NORMAL + this.eddn_system_button['state'] = tk.NORMAL this.eddn_delay_button['state'] = this.eddn.replayfile and this.eddn_system.get() and tk.NORMAL or tk.DISABLED + def prefs_changed(cmdr, is_beta): config.set('output', - (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP |config.OUT_MKT_MANUAL)) + + (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + (this.eddn_station.get() and config.OUT_MKT_EDDN) + (this.eddn_system.get() and config.OUT_SYS_EDDN) + (this.eddn_delay.get() and config.OUT_SYS_DELAY)) + def plugin_stop(): this.eddn.close() + def journal_entry(cmdr, is_beta, system, station, entry, state): # Recursively filter '*_Localised' keys from dict @@ -381,9 +391,9 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): for k, v in d.items(): if k.endswith('_Localised'): pass - elif hasattr(v, 'items'): # dict -> recurse + elif hasattr(v, 'items'): # dict -> recurse filtered[k] = filter_localised(v) - elif isinstance(v, list): # list of dicts -> recurse + elif isinstance(v, list): # list of dicts -> recurse filtered[k] = [filter_localised(x) if hasattr(x, 'items') else x for x in v] else: filtered[k] = v @@ -398,7 +408,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): if 'StarPos' in entry: this.coordinates = tuple(entry['StarPos']) elif this.systemaddress != entry.get('SystemAddress'): - this.coordinates = None # Docked event doesn't include coordinates + this.coordinates = None # Docked event doesn't include coordinates this.systemaddress = entry.get('SystemAddress') elif entry['event'] == 'ApproachBody': this.planet = entry['Body'] @@ -408,13 +418,13 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # Send interesting events to EDDN, but not when on a crew if (config.getint('output') & config.OUT_SYS_EDDN and not state['Captain'] and (entry['event'] in ('Location', 'FSDJump', 'Docked', 'Scan', 'SAASignalsFound', 'CarrierJump')) and - ('StarPos' in entry or this.coordinates)): + ('StarPos' in entry or this.coordinates)): # strip out properties disallowed by the schema for thing in ['ActiveFine', 'CockpitBreach', 'BoostUsed', 'FuelLevel', 'FuelUsed', 'JumpDist', 'Latitude', 'Longitude', 'Wanted']: entry.pop(thing, None) if 'Factions' in entry: # Filter faction state. `entry` is a shallow copy so replace 'Factions' value rather than modify in-place. - entry['Factions'] = [ {k: v for k, v in f.items() if k not in ['HappiestSystem', 'HomeSystem', 'MyReputation', 'SquadronFaction']} for f in entry['Factions']] + entry['Factions'] = [{k: v for k, v in f.items() if k not in ['HappiestSystem', 'HomeSystem', 'MyReputation', 'SquadronFaction']} for f in entry['Factions']] # add planet to Docked event for planetary stations if known if entry['event'] == 'Docked' and this.planet: From 4d0cf4e335231fccfe7bb9b34300ed6713821bda Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 02:24:37 +0200 Subject: [PATCH 222/258] Added scope change newlines --- plugins/eddn.py | 65 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 68e1dbb7..fced18e7 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -6,6 +6,7 @@ from os import SEEK_SET, SEEK_CUR, SEEK_END from os.path import exists, join from platform import system import re +from typing import Any import requests import sys import logging @@ -24,7 +25,7 @@ from companion import category_map logger = logging.getLogger(appname) -this = sys.modules[__name__] # For holding module globals +this: Any = sys.modules[__name__] # For holding module globals # Track location to add to Journal events this.systemaddress = None @@ -60,19 +61,25 @@ class EDDN(object): try: # Try to open existing file self.replayfile = open(filename, 'r+', buffering=1) + except Exception: if exists(filename): raise # Couldn't open existing file + else: self.replayfile = open(filename, 'w+', buffering=1) # Create file + if sys.platform != 'win32': # open for writing is automatically exclusive on Windows lockf(self.replayfile, LOCK_EX | LOCK_NB) + except Exception as e: logger.debug('Failed opening "replay.jsonl"', exc_info=e) if self.replayfile: self.replayfile.close() + self.replayfile = None return False + self.replaylog = [line.strip() for line in self.replayfile] return True @@ -104,6 +111,7 @@ class EDDN(object): r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=self.TIMEOUT) if r.status_code != requests.codes.ok: logger.debug(f':\nStatus\t{r.status_code}URL\t{r.url}Headers\t{r.headers}Content:\n{r.text}') + r.raise_for_status() def sendreplay(self): @@ -118,9 +126,12 @@ class EDDN(object): if len(self.replaylog) == 1: status['text'] = _('Sending data to EDDN...') + else: status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...', ''), len(self.replaylog)) + self.parent.update_idletasks() + try: cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict) except json.JSONDecodeError as e: @@ -128,19 +139,23 @@ class EDDN(object): logger.debug(f'\n{self.replaylog[0]}\n', exc_info=e) # Discard and continue self.replaylog.pop(0) + else: # Rewrite old schema name if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'): msg['$schemaRef'] = 'https://eddn.edcd.io/schemas/' + msg['$schemaRef'][38:] + try: self.send(cmdr, msg) self.replaylog.pop(0) if not len(self.replaylog) % self.REPLAYFLUSH: self.flush() + except requests.exceptions.RequestException as e: logger.debug('Failed sending', exc_info=e) status['text'] = _("Error: Can't connect to EDDN") return # stop sending + except Exception as e: logger.debug('Failed sending', exc_info=e) status['text'] = str(e) @@ -164,8 +179,10 @@ class EDDN(object): ('demand', int(commodity['demand'])), ('demandBracket', commodity['demandBracket']), ])) + if commodity['statusFlags']: commodities[-1]['statusFlags'] = commodity['statusFlags'] + commodities.sort(key=lambda c: c['name']) if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it @@ -176,14 +193,18 @@ class EDDN(object): ('marketId', data['lastStarport']['id']), ('commodities', commodities), ]) + if 'economies' in data['lastStarport']: message['economies'] = sorted(list([x for x in (data['lastStarport']['economies'] or {}).values()]), key=lambda x: x['name']) + if 'prohibited' in data['lastStarport']: message['prohibited'] = sorted(list([x for x in (data['lastStarport']['prohibited'] or {}).values()])) + self.send(data['commander']['name'], { '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), 'message': message, }) + this.commodities = commodities def export_outfitting(self, data, is_beta): @@ -194,6 +215,7 @@ class EDDN(object): horizons = (any(economy['name'] == 'Colony' for economy in economies.values()) or any(module.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for module in modules.values()) or any(ship.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for ship in list((ships['shipyard_list'] or {}).values()))) + outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['name'].lower()) for module in modules.values() if self.MODULE_RE.search(module['name']) and module.get('sku') in [None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'] and module['name'] != 'Int_PlanetApproachSuite']) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): @@ -208,6 +230,7 @@ class EDDN(object): ('modules', outfitting), ]), }) + this.outfitting = (horizons, outfitting) def export_shipyard(self, data, is_beta): @@ -217,6 +240,7 @@ class EDDN(object): horizons = (any(economy['name'] == 'Colony' for economy in economies.values()) or any(module.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for module in modules.values()) or any(ship.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for ship in list((ships['shipyard_list'] or {}).values()))) + shipyard = sorted([ship['name'].lower() for ship in list((ships['shipyard_list'] or {}).values()) + ships['unavailable_list']]) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): @@ -231,6 +255,7 @@ class EDDN(object): ('ships', shipyard), ]), }) + this.shipyard = (horizons, shipyard) def export_journal_commodities(self, cmdr, is_beta, entry): @@ -257,6 +282,7 @@ class EDDN(object): ('commodities', commodities), ]), }) + this.commodities = commodities def export_journal_outfitting(self, cmdr, is_beta, entry): @@ -276,6 +302,7 @@ class EDDN(object): ('modules', outfitting), ]), }) + this.outfitting = (horizons, outfitting) def export_journal_shipyard(self, cmdr, is_beta, entry): @@ -295,6 +322,7 @@ class EDDN(object): ('ships', shipyard), ]), }) + this.shipyard = (horizons, shipyard) def export_journal_entry(self, cmdr, is_beta, entry): @@ -302,14 +330,16 @@ class EDDN(object): '$schemaRef': 'https://eddn.edcd.io/schemas/journal/1' + (is_beta and '/test' or ''), 'message': entry } + if self.replayfile or self.load(): # Store the entry self.replaylog.append(json.dumps([cmdr, msg])) self.replayfile.write('%s\n' % self.replaylog[-1]) - if (entry['event'] == 'Docked' or - (entry['event'] == 'Location' and entry['Docked']) or not (config.getint('output') & config.OUT_SYS_DELAY)): + if (entry['event'] == 'Docked' or (entry['event'] == 'Location' and entry['Docked']) or not (config.getint('output') & config.OUT_SYS_DELAY)): + self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries + else: # Can't access replay file! Send immediately. status = self.parent.children['status'] @@ -339,13 +369,13 @@ def plugin_app(parent): def plugin_prefs(parent, cmdr, is_beta): - PADX = 10 BUTTONX = 12 # indent Checkbuttons and Radiobuttons PADY = 2 # close spacing if prefsVersion.shouldSetDefaults('0.0.0.0', not bool(config.getint('output'))): output = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings + else: output = config.getint('output') @@ -384,34 +414,43 @@ def plugin_stop(): def journal_entry(cmdr, is_beta, system, station, entry, state): - # Recursively filter '*_Localised' keys from dict def filter_localised(d): filtered = OrderedDict() for k, v in d.items(): if k.endswith('_Localised'): pass + elif hasattr(v, 'items'): # dict -> recurse filtered[k] = filter_localised(v) + elif isinstance(v, list): # list of dicts -> recurse filtered[k] = [filter_localised(x) if hasattr(x, 'items') else x for x in v] + else: filtered[k] = v + return filtered # Track location if entry['event'] in ['Location', 'FSDJump', 'Docked', 'CarrierJump']: if entry['event'] in ('Location', 'CarrierJump'): this.planet = entry.get('Body') if entry.get('BodyType') == 'Planet' else None + elif entry['event'] == 'FSDJump': this.planet = None + if 'StarPos' in entry: this.coordinates = tuple(entry['StarPos']) + elif this.systemaddress != entry.get('SystemAddress'): this.coordinates = None # Docked event doesn't include coordinates + this.systemaddress = entry.get('SystemAddress') + elif entry['event'] == 'ApproachBody': this.planet = entry['Body'] + elif entry['event'] in ['LeaveBody', 'SupercruiseEntry']: this.planet = None @@ -419,9 +458,11 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): if (config.getint('output') & config.OUT_SYS_EDDN and not state['Captain'] and (entry['event'] in ('Location', 'FSDJump', 'Docked', 'Scan', 'SAASignalsFound', 'CarrierJump')) and ('StarPos' in entry or this.coordinates)): + # strip out properties disallowed by the schema for thing in ['ActiveFine', 'CockpitBreach', 'BoostUsed', 'FuelLevel', 'FuelUsed', 'JumpDist', 'Latitude', 'Longitude', 'Wanted']: entry.pop(thing, None) + if 'Factions' in entry: # Filter faction state. `entry` is a shallow copy so replace 'Factions' value rather than modify in-place. entry['Factions'] = [{k: v for k, v in f.items() if k not in ['HappiestSystem', 'HomeSystem', 'MyReputation', 'SquadronFaction']} for f in entry['Factions']] @@ -435,27 +476,35 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): if 'StarSystem' not in entry: if not system: return("system is None, can't add StarSystem") + entry['StarSystem'] = system + if 'StarPos' not in entry: if not this.coordinates: return("this.coordinates is None, can't add StarPos") + entry['StarPos'] = list(this.coordinates) + if 'SystemAddress' not in entry: if not this.systemaddress: return("this.systemaddress is None, can't add SystemAddress") + entry['SystemAddress'] = this.systemaddress try: this.eddn.export_journal_entry(cmdr, is_beta, filter_localised(entry)) + except requests.exceptions.RequestException as e: logger.debug('Failed in export_journal_entry', exc_info=e) return _("Error: Can't connect to EDDN") + except Exception as e: logger.debug('Failed in export_journal_entry', exc_info=e) return str(e) elif (config.getint('output') & config.OUT_MKT_EDDN and not state['Captain'] and - entry['event'] in ['Market', 'Outfitting', 'Shipyard']): + entry['event'] in ['Market', 'Outfitting', 'Shipyard']): + try: if this.marketId != entry['MarketID']: this.commodities = this.outfitting = this.shipyard = None @@ -465,14 +514,17 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): entry = json.load(h) if entry['event'] == 'Market': this.eddn.export_journal_commodities(cmdr, is_beta, entry) + elif entry['event'] == 'Outfitting': this.eddn.export_journal_outfitting(cmdr, is_beta, entry) + elif entry['event'] == 'Shipyard': this.eddn.export_journal_shipyard(cmdr, is_beta, entry) except requests.exceptions.RequestException as e: logger.debug(f'Failed exporting {entry["event"]}', exc_info=e) return _("Error: Can't connect to EDDN") + except Exception as e: logger.debug(f'Failed exporting {entry["event"]}', exc_info=e) return str(e) @@ -490,6 +542,7 @@ def cmdr_data(data, is_beta): if not old_status: status['text'] = _('Sending data to EDDN...') status.update_idletasks() + this.eddn.export_commodities(data, is_beta) this.eddn.export_outfitting(data, is_beta) this.eddn.export_shipyard(data, is_beta) From 4105662fb572c5f14748880c75db5992c514f041 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:23:40 +0200 Subject: [PATCH 223/258] Cleaned up logic and removed overlong lines Generally for the logic cleanups it was replacing giant list comprehensions with slightly smaller filter calls. filter() is just plain cleaner when all you're doing in a list comp is [x for x in y if somecondition]. --- plugins/eddn.py | 142 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 27 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index fced18e7..9f0b08f2 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1,12 +1,13 @@ # Export to EDDN from collections import OrderedDict +import itertools import json from os import SEEK_SET, SEEK_CUR, SEEK_END from os.path import exists, join from platform import system import re -from typing import Any +from typing import Any, Mapping, TYPE_CHECKING import requests import sys import logging @@ -23,6 +24,9 @@ if sys.platform != 'win32': from config import appname, applongname, appversion, config from companion import category_map +if TYPE_CHECKING: + def _(x): return x + logger = logging.getLogger(appname) this: Any = sys.modules[__name__] # For holding module globals @@ -36,6 +40,15 @@ this.planet = None this.marketId = None this.commodities = this.outfitting = this.shipyard = None +HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' + + +# TODO: +# 1, this does a bunch of magic checks for whether or not the given commander has horizons. We can get that directly +# from CAPI via commander.capabilities.Horizons -- Should this be changed to use that? did such a thing exist when +# this was written? +# +# 2, a good few of these methods are static or could be classmethods. they should be created as such. class EDDN(object): @@ -88,11 +101,13 @@ class EDDN(object): self.replayfile.truncate() for line in self.replaylog: self.replayfile.write('%s\n' % line) + self.replayfile.flush() def close(self): if self.replayfile: self.replayfile.close() + self.replayfile = None def send(self, cmdr, msg): @@ -195,7 +210,9 @@ class EDDN(object): ]) if 'economies' in data['lastStarport']: - message['economies'] = sorted(list([x for x in (data['lastStarport']['economies'] or {}).values()]), key=lambda x: x['name']) + message['economies'] = sorted( + list([x for x in (data['lastStarport']['economies'] or {}).values()]), key=lambda x: x['name'] + ) if 'prohibited' in data['lastStarport']: message['prohibited'] = sorted(list([x for x in (data['lastStarport']['prohibited'] or {}).values()])) @@ -209,14 +226,25 @@ class EDDN(object): def export_outfitting(self, data, is_beta): economies = data['lastStarport'].get('economies') or {} - modules = data['lastStarport'].get('modules') or {} + modules: Mapping[str, Any] = data['lastStarport'].get('modules') or {} ships = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} - # Horizons flag - will hit at least Int_PlanetApproachSuite other than at engineer bases ("Colony"), prison or rescue Megaships, or under Pirate Attack etc - horizons = (any(economy['name'] == 'Colony' for economy in economies.values()) or - any(module.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for module in modules.values()) or - any(ship.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for ship in list((ships['shipyard_list'] or {}).values()))) + # Horizons flag - will hit at least Int_PlanetApproachSuite other than at engineer bases ("Colony"), + # prison or rescue Megaships, or under Pirate Attack etc + horizons = ( + any(economy['name'] == 'Colony' for economy in economies.values()) or + any(module.get('sku') == HORIZ_SKU for module in modules.values()) or + any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) + ) - outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['name'].lower()) for module in modules.values() if self.MODULE_RE.search(module['name']) and module.get('sku') in [None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'] and module['name'] != 'Int_PlanetApproachSuite']) + to_search = filter( + lambda m: self.MODULE_RE.search(m['name']) and m.get('sku') in (None, HORIZ_SKU) and + m['name'] != 'Int_PlanetApproachSuite', + modules.values() + ) + + outfitting = sorted( + self.MODULE_RE.sub(lambda match: match.group(0).capitalize(), mod['name'].lower()) for mod in to_search + ) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): self.send(data['commander']['name'], { @@ -237,11 +265,18 @@ class EDDN(object): economies = data['lastStarport'].get('economies') or {} modules = data['lastStarport'].get('modules') or {} ships = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} - horizons = (any(economy['name'] == 'Colony' for economy in economies.values()) or - any(module.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for module in modules.values()) or - any(ship.get('sku') == 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' for ship in list((ships['shipyard_list'] or {}).values()))) + horizons = ( + any(economy['name'] == 'Colony' for economy in economies.values()) or + any(module.get('sku') == HORIZ_SKU for module in modules.values()) or + any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) + ) - shipyard = sorted([ship['name'].lower() for ship in list((ships['shipyard_list'] or {}).values()) + ships['unavailable_list']]) + shipyard = sorted( + itertools.chain( + (ship['name'].lower() for ship in (ships['shipyard_list'] or {}).values()), + ships['unavailable_list'] + ) + ) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): self.send(data['commander']['name'], { @@ -288,7 +323,12 @@ class EDDN(object): def export_journal_outfitting(self, cmdr, is_beta, entry): modules = entry.get('Items') or [] horizons = entry.get('Horizons', False) - outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['Name']) for module in modules if module['Name'] != 'int_planetapproachsuite']) + # outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['Name']) + # for module in modules if module['Name'] != 'int_planetapproachsuite']) + outfitting = sorted( + self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), mod['Name']) for mod in + filter(lambda m: m['Name'] != 'int_planetapproachsuite', modules) + ) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): self.send(cmdr, { @@ -336,8 +376,10 @@ class EDDN(object): self.replaylog.append(json.dumps([cmdr, msg])) self.replayfile.write('%s\n' % self.replaylog[-1]) - if (entry['event'] == 'Docked' or (entry['event'] == 'Location' and entry['Docked']) or not (config.getint('output') & config.OUT_SYS_DELAY)): - + if ( + entry['event'] == 'Docked' or (entry['event'] == 'Location' and entry['Docked']) or not + (config.getint('output') & config.OUT_SYS_DELAY) + ): self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries else: @@ -381,16 +423,41 @@ def plugin_prefs(parent, cmdr, is_beta): eddnframe = nb.Frame(parent) - HyperlinkLabel(eddnframe, text='Elite Dangerous Data Network', background=nb.Label().cget('background'), url='https://github.com/EDSM-NET/EDDN/wiki', underline=True).grid(padx=PADX, sticky=tk.W) # Don't translate + HyperlinkLabel( + eddnframe, + text='Elite Dangerous Data Network', + background=nb.Label().cget('background'), + url='https://github.com/EDSM-NET/EDDN/wiki', + underline=True + ).grid(padx=PADX, sticky=tk.W) # Don't translate + this.eddn_station = tk.IntVar(value=(output & config.OUT_MKT_EDDN) and 1) - this.eddn_station_button = nb.Checkbutton(eddnframe, text=_('Send station data to the Elite Dangerous Data Network'), variable=this.eddn_station, command=prefsvarchanged) # Output setting + this.eddn_station_button = nb.Checkbutton( + eddnframe, + text=_('Send station data to the Elite Dangerous Data Network'), + variable=this.eddn_station, + command=prefsvarchanged + ) # Output setting + this.eddn_station_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) this.eddn_system = tk.IntVar(value=(output & config.OUT_SYS_EDDN) and 1) - this.eddn_system_button = nb.Checkbutton(eddnframe, text=_('Send system and scan data to the Elite Dangerous Data Network'), variable=this.eddn_system, command=prefsvarchanged) # Output setting new in E:D 2.2 + # Output setting new in E:D 2.2 + this.eddn_system_button = nb.Checkbutton( + eddnframe, + text=_('Send system and scan data to the Elite Dangerous Data Network'), + variable=this.eddn_system, + command=prefsvarchanged + ) + this.eddn_system_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) this.eddn_delay = tk.IntVar(value=(output & config.OUT_SYS_DELAY) and 1) # Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2 - this.eddn_delay_button = nb.Checkbutton(eddnframe, text=_('Delay sending until docked'), variable=this.eddn_delay) this.eddn_delay_button.grid(padx=BUTTONX, sticky=tk.W) + this.eddn_delay_button = nb.Checkbutton( + eddnframe, + text=_('Delay sending until docked'), + variable=this.eddn_delay + ) + this.eddn_delay_button.grid(padx=BUTTONX, sticky=tk.W) return eddnframe @@ -402,11 +469,13 @@ def prefsvarchanged(event=None): def prefs_changed(cmdr, is_beta): - config.set('output', - (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + - (this.eddn_station.get() and config.OUT_MKT_EDDN) + - (this.eddn_system.get() and config.OUT_SYS_EDDN) + - (this.eddn_delay.get() and config.OUT_SYS_DELAY)) + config.set( + 'output', + (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + + (this.eddn_station.get() and config.OUT_MKT_EDDN) + + (this.eddn_system.get() and config.OUT_SYS_EDDN) + + (this.eddn_delay.get() and config.OUT_SYS_DELAY) + ) def plugin_stop(): @@ -460,12 +529,29 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ('StarPos' in entry or this.coordinates)): # strip out properties disallowed by the schema - for thing in ['ActiveFine', 'CockpitBreach', 'BoostUsed', 'FuelLevel', 'FuelUsed', 'JumpDist', 'Latitude', 'Longitude', 'Wanted']: + for thing in [ + 'ActiveFine', + 'CockpitBreach', + 'BoostUsed', + 'FuelLevel', + 'FuelUsed', + 'JumpDist', + 'Latitude', + 'Longitude', + 'Wanted' + ]: entry.pop(thing, None) if 'Factions' in entry: # Filter faction state. `entry` is a shallow copy so replace 'Factions' value rather than modify in-place. - entry['Factions'] = [{k: v for k, v in f.items() if k not in ['HappiestSystem', 'HomeSystem', 'MyReputation', 'SquadronFaction']} for f in entry['Factions']] + entry['Factions'] = [ + { + k: v for k, v in f.items() if k not in [ + 'HappiestSystem', 'HomeSystem', 'MyReputation', 'SquadronFaction' + ] + } + for f in entry['Factions'] + ] # add planet to Docked event for planetary stations if known if entry['event'] == 'Docked' and this.planet: @@ -510,7 +596,9 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.commodities = this.outfitting = this.shipyard = None this.marketId = entry['MarketID'] - with open(join(config.get('journaldir') or config.default_journal_dir, '%s.json' % entry['event']), 'rb') as h: + with open( + join(config.get('journaldir') or config.default_journal_dir, '%s.json' % entry['event']), 'rb' + ) as h: entry = json.load(h) if entry['event'] == 'Market': this.eddn.export_journal_commodities(cmdr, is_beta, entry) From c012ebc54e6573ee6d5f87ed89893e8399c6cce2 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:35:51 +0200 Subject: [PATCH 224/258] Replaced modulo-formatting with fstrings Specifically did not replace `somefile.write('%s\n' % some_data)` as print may change the line endings on some OSes which may not be wanted --- plugins/eddn.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 9f0b08f2..5f679f2a 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -54,7 +54,7 @@ class EDDN(object): # SERVER = 'http://localhost:8081' # testing SERVER = 'https://eddn.edcd.io:4430' - UPLOAD = '%s/upload/' % SERVER + UPLOAD = f'{SERVER}/upload/' REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds TIMEOUT = 10 # requests timeout @@ -100,7 +100,7 @@ class EDDN(object): self.replayfile.seek(0, SEEK_SET) self.replayfile.truncate() for line in self.replaylog: - self.replayfile.write('%s\n' % line) + self.replayfile.write(f'{line}\n') self.replayfile.flush() @@ -115,12 +115,12 @@ class EDDN(object): msg = OrderedDict([ ('$schemaRef', msg['$schemaRef']), - ('header', OrderedDict([ - ('softwareName', '%s [%s]' % (applongname, sys.platform == 'darwin' and "Mac OS" or system())), + ('header', OrderedDict([ + ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), ('softwareVersion', appversion), ('uploaderID', uploaderID), ])), - ('message', msg['message']), + ('message', msg['message']), ]) r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=self.TIMEOUT) @@ -138,12 +138,12 @@ class EDDN(object): if not self.replaylog: status['text'] = '' return - + localized = _('Sending data to EDDN...') if len(self.replaylog) == 1: - status['text'] = _('Sending data to EDDN...') + status['text'] = localized else: - status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...', ''), len(self.replaylog)) + status['text'] = f'{localized.replace("...", "")} [{len(self.replaylog)}]' self.parent.update_idletasks() @@ -374,12 +374,12 @@ class EDDN(object): if self.replayfile or self.load(): # Store the entry self.replaylog.append(json.dumps([cmdr, msg])) - self.replayfile.write('%s\n' % self.replaylog[-1]) + self.replayfile.write(f'{self.replaylog[-1]}\n') if ( entry['event'] == 'Docked' or (entry['event'] == 'Location' and entry['Docked']) or not (config.getint('output') & config.OUT_SYS_DELAY) - ): + ): self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries else: @@ -597,7 +597,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.marketId = entry['MarketID'] with open( - join(config.get('journaldir') or config.default_journal_dir, '%s.json' % entry['event']), 'rb' + join(config.get('journaldir') or config.default_journal_dir, f'{entry["event"]}.json'), 'rb' ) as h: entry = json.load(h) if entry['event'] == 'Market': From c0026cea617b42c5be7653d1eebbb59fd19f0551 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:39:21 +0200 Subject: [PATCH 225/258] replaced list comps with generators --- plugins/eddn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 5f679f2a..b58b39fe 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -211,11 +211,11 @@ class EDDN(object): if 'economies' in data['lastStarport']: message['economies'] = sorted( - list([x for x in (data['lastStarport']['economies'] or {}).values()]), key=lambda x: x['name'] + (x for x in (data['lastStarport']['economies'] or {}).values()), key=lambda x: x['name'] ) if 'prohibited' in data['lastStarport']: - message['prohibited'] = sorted(list([x for x in (data['lastStarport']['prohibited'] or {}).values()])) + message['prohibited'] = sorted(x for x in (data['lastStarport']['prohibited'] or {}).values()) self.send(data['commander']['name'], { '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), @@ -295,7 +295,7 @@ class EDDN(object): def export_journal_commodities(self, cmdr, is_beta, entry): items = entry.get('Items') or [] - commodities = sorted([OrderedDict([ + commodities = sorted((OrderedDict([ ('name', self.canonicalise(commodity['Name'])), ('meanPrice', commodity['MeanPrice']), ('buyPrice', commodity['BuyPrice']), @@ -304,7 +304,7 @@ class EDDN(object): ('sellPrice', commodity['SellPrice']), ('demand', commodity['Demand']), ('demandBracket', commodity['DemandBracket']), - ]) for commodity in items], key=lambda c: c['name']) + ]) for commodity in items), key=lambda c: c['name']) if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it self.send(cmdr, { @@ -348,7 +348,7 @@ class EDDN(object): def export_journal_shipyard(self, cmdr, is_beta, entry): ships = entry.get('PriceList') or [] horizons = entry.get('Horizons', False) - shipyard = sorted([ship['ShipType'] for ship in ships]) + shipyard = sorted(ship['ShipType'] for ship in ships) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): self.send(cmdr, { From 6772c84a5fc9828f2f6648c9c6a191cb9da9c18d Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:41:43 +0200 Subject: [PATCH 226/258] fixed names where possible --- plugins/eddn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index b58b39fe..ea13ddad 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -14,7 +14,7 @@ import logging import tkinter as tk from ttkHyperlinkLabel import HyperlinkLabel -import myNotebook as nb +import myNotebook as nb # noqa: N813 from prefs import prefsVersion @@ -111,14 +111,14 @@ class EDDN(object): self.replayfile = None def send(self, cmdr, msg): - uploaderID = cmdr + uploader_id = cmdr msg = OrderedDict([ ('$schemaRef', msg['$schemaRef']), ('header', OrderedDict([ ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), ('softwareVersion', appversion), - ('uploaderID', uploaderID), + ('uploaderID', uploader_id), ])), ('message', msg['message']), ]) From 3c66ab7246ceac23e71f552a8a3313e072f0be85 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:42:19 +0200 Subject: [PATCH 227/258] removed unused imports --- plugins/eddn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index ea13ddad..75a4714d 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -3,7 +3,7 @@ from collections import OrderedDict import itertools import json -from os import SEEK_SET, SEEK_CUR, SEEK_END +from os import SEEK_SET from os.path import exists, join from platform import system import re From e477f896640fea4dad8352055000c5547a1af530 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:42:44 +0200 Subject: [PATCH 228/258] removed un-needed object subclass --- plugins/eddn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 75a4714d..c6a109a8 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -50,8 +50,7 @@ HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' # # 2, a good few of these methods are static or could be classmethods. they should be created as such. -class EDDN(object): - +class EDDN: # SERVER = 'http://localhost:8081' # testing SERVER = 'https://eddn.edcd.io:4430' UPLOAD = f'{SERVER}/upload/' From cea8ac49caa5418aa960da6b3616efef37fa0751 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 13:43:20 +0200 Subject: [PATCH 229/258] Ensured that regexps are raw strings Ensures that there are no weird excaping issues --- plugins/eddn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index c6a109a8..0875c007 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -57,7 +57,7 @@ class EDDN: REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds TIMEOUT = 10 # requests timeout - MODULE_RE = re.compile('^Hpt_|^Int_|Armour_', re.IGNORECASE) + MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) CANONICALISE_RE = re.compile(r'\$(.+)_name;') def __init__(self, parent): From 94c4a7b7d0154e185493128aa171ca55b56e112f Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 14:54:56 +0200 Subject: [PATCH 230/258] Replaced static lists with tuples --- plugins/eddn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 0875c007..52a08e61 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -528,7 +528,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): ('StarPos' in entry or this.coordinates)): # strip out properties disallowed by the schema - for thing in [ + for thing in ( 'ActiveFine', 'CockpitBreach', 'BoostUsed', @@ -538,16 +538,16 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): 'Latitude', 'Longitude', 'Wanted' - ]: + ): entry.pop(thing, None) if 'Factions' in entry: # Filter faction state. `entry` is a shallow copy so replace 'Factions' value rather than modify in-place. entry['Factions'] = [ { - k: v for k, v in f.items() if k not in [ + k: v for k, v in f.items() if k not in ( 'HappiestSystem', 'HomeSystem', 'MyReputation', 'SquadronFaction' - ] + ) } for f in entry['Factions'] ] @@ -588,7 +588,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): return str(e) elif (config.getint('output') & config.OUT_MKT_EDDN and not state['Captain'] and - entry['event'] in ['Market', 'Outfitting', 'Shipyard']): + entry['event'] in ('Market', 'Outfitting', 'Shipyard')): try: if this.marketId != entry['MarketID']: From 5aa67957739a6fb65960ae1ac80261de40775042 Mon Sep 17 00:00:00 2001 From: A_D Date: Sun, 2 Aug 2020 14:56:43 +0200 Subject: [PATCH 231/258] Type annotate all the things! --- plugins/eddn.py | 156 ++++++++++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 72 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 52a08e61..ecbabe1f 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -7,12 +7,17 @@ from os import SEEK_SET from os.path import exists, join from platform import system import re -from typing import Any, Mapping, TYPE_CHECKING +from typing import ( + Any, AnyStr, Dict, Iterator, List, Mapping, MutableMapping, Optional, + Sequence, TYPE_CHECKING, OrderedDict as OrderedDictT, TextIO, Tuple, Union +) + import requests import sys import logging import tkinter as tk +from myNotebook import Frame from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb # noqa: N813 @@ -25,7 +30,8 @@ from config import appname, applongname, appversion, config from companion import category_map if TYPE_CHECKING: - def _(x): return x + def _(x: str) -> str: + return x logger = logging.getLogger(appname) @@ -60,13 +66,13 @@ class EDDN: MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) CANONICALISE_RE = re.compile(r'\$(.+)_name;') - def __init__(self, parent): - self.parent = parent + def __init__(self, parent: tk.Tk): + self.parent: tk.Tk = parent self.session = requests.Session() - self.replayfile = None # For delayed messages - self.replaylog = [] + self.replayfile: Optional[TextIO] = None # For delayed messages + self.replaylog: List[str] = [] - def load(self): + def load(self) -> bool: # Try to obtain exclusive access to the journal cache filename = join(config.app_dir, 'replay.jsonl') try: @@ -109,10 +115,10 @@ class EDDN: self.replayfile = None - def send(self, cmdr, msg): + def send(self, cmdr: str, msg: Mapping[str, Any]): uploader_id = cmdr - msg = OrderedDict([ + to_send: OrderedDictT[str, str] = OrderedDict([ ('$schemaRef', msg['$schemaRef']), ('header', OrderedDict([ ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), @@ -122,7 +128,7 @@ class EDDN: ('message', msg['message']), ]) - r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=self.TIMEOUT) + r = self.session.post(self.UPLOAD, data=json.dumps(to_send), timeout=self.TIMEOUT) if r.status_code != requests.codes.ok: logger.debug(f':\nStatus\t{r.status_code}URL\t{r.url}Headers\t{r.headers}Content:\n{r.text}') @@ -132,12 +138,13 @@ class EDDN: if not self.replayfile: return # Probably closing app - status = self.parent.children['status'] + status: Dict[str, Any] = self.parent.children['status'] if not self.replaylog: status['text'] = '' return - localized = _('Sending data to EDDN...') + + localized: str = _('Sending data to EDDN...') if len(self.replaylog) == 1: status['text'] = localized @@ -148,6 +155,7 @@ class EDDN: try: cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict) + except json.JSONDecodeError as e: # Couldn't decode - shouldn't happen! logger.debug(f'\n{self.replaylog[0]}\n', exc_info=e) @@ -157,7 +165,7 @@ class EDDN: else: # Rewrite old schema name if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'): - msg['$schemaRef'] = 'https://eddn.edcd.io/schemas/' + msg['$schemaRef'][38:] + msg['$schemaRef'] = 'https://eddn.edcd.io/schemas/' + msg['$schemaRef'][38:] # TODO: 38?! try: self.send(cmdr, msg) @@ -177,8 +185,8 @@ class EDDN: self.parent.after(self.REPLAYPERIOD, self.sendreplay) - def export_commodities(self, data, is_beta): - commodities = [] + def export_commodities(self, data: Mapping[str, Any], is_beta: bool): + commodities: List[OrderedDictT[str, Any]] = [] for commodity in data['lastStarport'].get('commodities') or []: # Check 'marketable' and 'not prohibited' if (category_map.get(commodity['categoryname'], True) @@ -200,7 +208,7 @@ class EDDN: commodities.sort(key=lambda c: c['name']) if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it - message = OrderedDict([ + message: OrderedDictT[str, Any] = OrderedDict([ ('timestamp', data['timestamp']), ('systemName', data['lastSystem']['name']), ('stationName', data['lastStarport']['name']), @@ -217,16 +225,19 @@ class EDDN: message['prohibited'] = sorted(x for x in (data['lastStarport']['prohibited'] or {}).values()) self.send(data['commander']['name'], { - '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': message, }) this.commodities = commodities - def export_outfitting(self, data, is_beta): - economies = data['lastStarport'].get('economies') or {} - modules: Mapping[str, Any] = data['lastStarport'].get('modules') or {} - ships = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} + def export_outfitting(self, data: Mapping[str, Any], is_beta: bool): + economies: Dict[str, Any] = data['lastStarport'].get('economies') or {} + modules: Dict[str, Any] = data['lastStarport'].get('modules') or {} + ships: Dict[str, Union[Dict[str, Any], List]] = data['lastStarport'].get('ships') or { + 'shipyard_list': {}, 'unavailable_list': [] + } + # Horizons flag - will hit at least Int_PlanetApproachSuite other than at engineer bases ("Colony"), # prison or rescue Megaships, or under Pirate Attack etc horizons = ( @@ -235,19 +246,19 @@ class EDDN: any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) ) - to_search = filter( + to_search: Iterator[Mapping[str, Any]] = filter( lambda m: self.MODULE_RE.search(m['name']) and m.get('sku') in (None, HORIZ_SKU) and m['name'] != 'Int_PlanetApproachSuite', modules.values() ) - outfitting = sorted( + outfitting: List[str] = sorted( self.MODULE_RE.sub(lambda match: match.group(0).capitalize(), mod['name'].lower()) for mod in to_search ) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): self.send(data['commander']['name'], { - '$schemaRef': 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/outfitting/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', data['timestamp']), ('systemName', data['lastSystem']['name']), @@ -260,17 +271,17 @@ class EDDN: this.outfitting = (horizons, outfitting) - def export_shipyard(self, data, is_beta): - economies = data['lastStarport'].get('economies') or {} - modules = data['lastStarport'].get('modules') or {} - ships = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} - horizons = ( + def export_shipyard(self, data: Dict[str, Any], is_beta: bool): + economies: Dict[str, Any] = data['lastStarport'].get('economies') or {} + modules: Dict[str, Any] = data['lastStarport'].get('modules') or {} + ships: Dict[str, Any] = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} + horizons: bool = ( any(economy['name'] == 'Colony' for economy in economies.values()) or any(module.get('sku') == HORIZ_SKU for module in modules.values()) or any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) ) - shipyard = sorted( + shipyard: List[Mapping[str, Any]] = sorted( itertools.chain( (ship['name'].lower() for ship in (ships['shipyard_list'] or {}).values()), ships['unavailable_list'] @@ -279,7 +290,7 @@ class EDDN: # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): self.send(data['commander']['name'], { - '$schemaRef': 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/shipyard/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', data['timestamp']), ('systemName', data['lastSystem']['name']), @@ -292,9 +303,9 @@ class EDDN: this.shipyard = (horizons, shipyard) - def export_journal_commodities(self, cmdr, is_beta, entry): - items = entry.get('Items') or [] - commodities = sorted((OrderedDict([ + def export_journal_commodities(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + items: List[Mapping[str, Any]] = entry.get('Items') or [] + commodities: Sequence[OrderedDictT[AnyStr, Any]] = sorted((OrderedDict([ ('name', self.canonicalise(commodity['Name'])), ('meanPrice', commodity['MeanPrice']), ('buyPrice', commodity['BuyPrice']), @@ -307,7 +318,7 @@ class EDDN: if commodities and this.commodities != commodities: # Don't send empty commodities list - schema won't allow it self.send(cmdr, { - '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), ('systemName', entry['StarSystem']), @@ -317,21 +328,21 @@ class EDDN: ]), }) - this.commodities = commodities + this.commodities: OrderedDictT[str, Any] = commodities - def export_journal_outfitting(self, cmdr, is_beta, entry): - modules = entry.get('Items') or [] - horizons = entry.get('Horizons', False) + def export_journal_outfitting(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + modules: List[Mapping[str, Any]] = entry.get('Items', []) + horizons: bool = entry.get('Horizons', False) # outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['Name']) # for module in modules if module['Name'] != 'int_planetapproachsuite']) - outfitting = sorted( + outfitting: List[str] = sorted( self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), mod['Name']) for mod in filter(lambda m: m['Name'] != 'int_planetapproachsuite', modules) ) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): self.send(cmdr, { - '$schemaRef': 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/outfitting/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), ('systemName', entry['StarSystem']), @@ -344,14 +355,14 @@ class EDDN: this.outfitting = (horizons, outfitting) - def export_journal_shipyard(self, cmdr, is_beta, entry): - ships = entry.get('PriceList') or [] - horizons = entry.get('Horizons', False) + def export_journal_shipyard(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + ships: List[Mapping[str, Any]] = entry.get('PriceList') or [] + horizons: bool = entry.get('Horizons', False) shipyard = sorted(ship['ShipType'] for ship in ships) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): self.send(cmdr, { - '$schemaRef': 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/shipyard/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), ('systemName', entry['StarSystem']), @@ -364,9 +375,9 @@ class EDDN: this.shipyard = (horizons, shipyard) - def export_journal_entry(self, cmdr, is_beta, entry): + def export_journal_entry(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): msg = { - '$schemaRef': 'https://eddn.edcd.io/schemas/journal/1' + (is_beta and '/test' or ''), + '$schemaRef': f'https://eddn.edcd.io/schemas/journal/1{"/test" if is_beta else ""}', 'message': entry } @@ -383,13 +394,13 @@ class EDDN: else: # Can't access replay file! Send immediately. - status = self.parent.children['status'] + status: MutableMapping[str, str] = self.parent.children['status'] status['text'] = _('Sending data to EDDN...') self.parent.update_idletasks() self.send(cmdr, msg) status['text'] = '' - def canonicalise(self, item): + def canonicalise(self, item: str) -> str: match = self.CANONICALISE_RE.match(item) return match and match.group(1) or item @@ -400,7 +411,7 @@ def plugin_start3(plugin_dir): return 'EDDN' -def plugin_app(parent): +def plugin_app(parent: tk.Tk): this.parent = parent this.eddn = EDDN(parent) # Try to obtain exclusive lock on journal cache, even if we don't need it yet @@ -409,16 +420,15 @@ def plugin_app(parent): this.status['text'] = 'Error: Is another copy of this app already running?' -def plugin_prefs(parent, cmdr, is_beta): - PADX = 10 - BUTTONX = 12 # indent Checkbuttons and Radiobuttons - PADY = 2 # close spacing +def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: + PADX = 10 # noqa: N806 + BUTTONX = 12 # noqa: N806 # indent Checkbuttons and Radiobuttons if prefsVersion.shouldSetDefaults('0.0.0.0', not bool(config.getint('output'))): - output = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings + output: int = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings else: - output = config.getint('output') + output: int = config.getint('output') eddnframe = nb.Frame(parent) @@ -467,7 +477,7 @@ def prefsvarchanged(event=None): this.eddn_delay_button['state'] = this.eddn.replayfile and this.eddn_system.get() and tk.NORMAL or tk.DISABLED -def prefs_changed(cmdr, is_beta): +def prefs_changed(cmdr: str, is_beta: bool): config.set( 'output', (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + @@ -481,10 +491,12 @@ def plugin_stop(): this.eddn.close() -def journal_entry(cmdr, is_beta, system, station, entry, state): +def journal_entry( + cmdr: str, is_beta: bool, system: str, station: str, entry: MutableMapping[str, Any], state: Mapping[str, Any] +) -> Optional[str]: # Recursively filter '*_Localised' keys from dict - def filter_localised(d): - filtered = OrderedDict() + def filter_localised(d: Mapping[str, Any]) -> OrderedDictT[str, Any]: + filtered: OrderedDictT[str, Any] = OrderedDict() for k, v in d.items(): if k.endswith('_Localised'): pass @@ -501,25 +513,25 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): return filtered # Track location - if entry['event'] in ['Location', 'FSDJump', 'Docked', 'CarrierJump']: + if entry['event'] in ('Location', 'FSDJump', 'Docked', 'CarrierJump'): if entry['event'] in ('Location', 'CarrierJump'): - this.planet = entry.get('Body') if entry.get('BodyType') == 'Planet' else None + this.planet: Optional[str] = entry.get('Body') if entry.get('BodyType') == 'Planet' else None elif entry['event'] == 'FSDJump': - this.planet = None + this.planet: Optional[str] = None if 'StarPos' in entry: - this.coordinates = tuple(entry['StarPos']) + this.coordinates: Optional[Tuple[int, int, int]] = tuple(entry['StarPos']) elif this.systemaddress != entry.get('SystemAddress'): - this.coordinates = None # Docked event doesn't include coordinates + this.coordinates: Optional[Tuple[int, int, int]] = None # Docked event doesn't include coordinates - this.systemaddress = entry.get('SystemAddress') + this.systemaddress: Optional[str] = entry.get('SystemAddress') elif entry['event'] == 'ApproachBody': this.planet = entry['Body'] - elif entry['event'] in ['LeaveBody', 'SupercruiseEntry']: + elif entry['event'] in ('LeaveBody', 'SupercruiseEntry'): this.planet = None # Send interesting events to EDDN, but not when on a crew @@ -560,19 +572,19 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # add mandatory StarSystem, StarPos and SystemAddress properties to Scan events if 'StarSystem' not in entry: if not system: - return("system is None, can't add StarSystem") + return "system is None, can't add StarSystem" entry['StarSystem'] = system if 'StarPos' not in entry: if not this.coordinates: - return("this.coordinates is None, can't add StarPos") + return "this.coordinates is None, can't add StarPos" entry['StarPos'] = list(this.coordinates) if 'SystemAddress' not in entry: if not this.systemaddress: - return("this.systemaddress is None, can't add SystemAddress") + return "this.systemaddress is None, can't add SystemAddress" entry['SystemAddress'] = this.systemaddress @@ -596,7 +608,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): this.marketId = entry['MarketID'] with open( - join(config.get('journaldir') or config.default_journal_dir, f'{entry["event"]}.json'), 'rb' + join(str(config.get('journaldir') or config.default_journal_dir), f'{entry["event"]}.json'), 'rb' ) as h: entry = json.load(h) if entry['event'] == 'Market': @@ -617,7 +629,7 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): return str(e) -def cmdr_data(data, is_beta): +def cmdr_data(data: Mapping[str, Any], is_beta: bool) -> str: if data['commander'].get('docked') and config.getint('output') & config.OUT_MKT_EDDN: try: if this.marketId != data['lastStarport']['id']: From a21280ed3b4fb8880ce297edec32d0d8512e99f4 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 13:32:01 +0200 Subject: [PATCH 232/258] reordered imports --- plugins/eddn.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index ecbabe1f..e3392dc4 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1,33 +1,31 @@ # Export to EDDN -from collections import OrderedDict import itertools import json +import logging +import re +import sys +import tkinter as tk +from collections import OrderedDict from os import SEEK_SET from os.path import exists, join from platform import system -import re -from typing import ( - Any, AnyStr, Dict, Iterator, List, Mapping, MutableMapping, Optional, - Sequence, TYPE_CHECKING, OrderedDict as OrderedDictT, TextIO, Tuple, Union -) +from typing import TYPE_CHECKING, Any, AnyStr, Dict, Iterator, List, Mapping, MutableMapping, Optional +from typing import OrderedDict as OrderedDictT +from typing import Sequence, TextIO, Tuple, Union import requests -import sys -import logging -import tkinter as tk -from myNotebook import Frame -from ttkHyperlinkLabel import HyperlinkLabel import myNotebook as nb # noqa: N813 - +from companion import category_map +from config import applongname, appname, appversion, config +from myNotebook import Frame from prefs import prefsVersion +from ttkHyperlinkLabel import HyperlinkLabel if sys.platform != 'win32': from fcntl import lockf, LOCK_EX, LOCK_NB -from config import appname, applongname, appversion, config -from companion import category_map if TYPE_CHECKING: def _(x: str) -> str: From dd85fc050469a3719593f2266aeee56b76981bec Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 3 Aug 2020 21:49:52 +0200 Subject: [PATCH 233/258] Rewrote inara queue system This replaces the list+queue system that the inara plugin originally used with a deque based one. The main differences here are that the list the worker thread uses to send to inara and the list that events are added to is the same, with the worker thread making a duplicate and clearing the original each time it sends events (losing events if it fails to upload three times). The format of the data has changed as well, from simple tuples to NamedTuple classes that provide some extra type safety and sanity when accessing fields. The event queue itself is actually multiple queues, one per API/FID/CMDR_name triplicate, thus allowing multiple commander switches while we're running without causing any weird issues --- plugins/inara.py | 695 +++++++++++++++++++++++++---------------------- 1 file changed, 377 insertions(+), 318 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 5cd381c0..1e252ac9 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -2,17 +2,18 @@ # Inara sync # -from collections import OrderedDict +from collections import OrderedDict, defaultdict import json -from typing import Any, AnyStr, Dict, List, Mapping, Optional, OrderedDict as OrderedDictT, \ - Sequence, TYPE_CHECKING, Union +from typing import Any, AnyStr, Callable, Deque, Dict, List, Mapping, NamedTuple, Optional, OrderedDict as OrderedDictT, \ + Sequence, TYPE_CHECKING, Tuple, Union +import dataclasses import requests import sys import time from operator import itemgetter from queue import Queue -from threading import Thread +from threading import Lock, Thread import logging import tkinter as tk @@ -22,6 +23,11 @@ import myNotebook as nb from config import appname, applongname, appversion, config import plug import timeout_session + +# For new impl +from collections import deque + + logger = logging.getLogger(appname) if TYPE_CHECKING: @@ -42,6 +48,7 @@ this.lastship = None # eventData from the last addCommanderShip or setCommander # Cached Cmdr state this.events: List[OrderedDictT[str, Any]] = [] # Unsent events +this.event_lock = Lock this.cmdr: Optional[str] = None this.FID: Optional[str] = None # Frontier ID this.multicrew: bool = False # don't send captain's ship info to Inara while on a crew @@ -59,7 +66,9 @@ this.shipswap: bool = False # just swapped ship # last time we updated, if unset in config this is 0, which means an instant update LAST_UPDATE_CONF_KEY = 'inara_last_update' -FLOOD_LIMIT_SECONDS = 30 # minimum time between sending events +EVENT_COLLECT_TIME = 31 # Minimum time to take collecting events before requesting a send +WORKER_WAIT_TIME = 35 # Minimum time for worker to wait between sends + this.timer_run = True @@ -74,6 +83,50 @@ this.station_marketid = None STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00D7 +class Credentials(NamedTuple): + """ + Credentials holds an inara API payload + """ + cmdr: str + fid: str + api_key: str + + +EVENT_DATA = Union[Mapping[AnyStr, Any], Sequence[Mapping[AnyStr, Any]]] + + +class Event(NamedTuple): + """ + Event represents an event for the Inara API + """ + name: str + timestamp: str + data: EVENT_DATA + + +@dataclasses.dataclass +class NewThis: + events: Dict[Credentials, Deque[Event]] = dataclasses.field(default_factory=lambda: defaultdict(deque)) + event_lock: Lock = dataclasses.field(default_factory=Lock) # protects events, for use when rewriting events + + def filter_events(self, key: Credentials, predicate: Callable[[Event], bool]): + """ + filter_events is the equivalent of running filter() on any event list in the events dict. + it will automatically handle locking, and replacing the event list with the filtered version. + + :param key: the key to filter + :param predicate: the predicate to use while filtering + """ + with self.event_lock: + tmp = self.events[key].copy() + self.events[key].clear() + self.events[key].extend(filter(predicate, tmp)) + + +new_this = NewThis() +TARGET_URL = 'https://inara.cz/inapi/v1/' + + def system_url(system_name: str): if this.system_address: return requests.utils.requote_uri(f'https://inara.cz/galaxy-starsystem/?search={this.system_address}') @@ -97,13 +150,10 @@ def station_url(system_name: Optional[str], station_name: Optional[str]): def plugin_start3(plugin_dir): - this.thread = Thread(target=worker, name='Inara worker') + this.thread = Thread(target=new_worker, name='Inara worker') this.thread.daemon = True this.thread.start() - this.timer_thread = Thread(target=call_timer, name='Inara timer') - this.timer_thread.daemon = True - this.timer_thread.start() return 'Inara' @@ -115,13 +165,10 @@ def plugin_app(parent: tk.Tk): def plugin_stop(): - # Send any unsent events - call() - time.sleep(0.1) # Sleep for 100ms to allow call to go out, and to force a context switch to our other threads # Signal thread to close and wait for it this.queue.put(None) - this.thread.join() - this.thread = None + # this.thread.join() + # this.thread = None this.timer_run = False @@ -227,8 +274,9 @@ def prefs_changed(cmdr: str, is_beta: bool): if this.log.get() and changed: this.newuser = True # Send basic info at next Journal event - add_event('getCommanderProfile', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), {'searchName': cmdr}) - call() + new_add_event('getCommanderProfile', time.strftime( + '%Y-%m-%dT%H:%M:%SZ', time.gmtime()), {'searchName': cmdr}) + # call() def credentials(cmdr: str) -> Optional[str]: @@ -251,8 +299,8 @@ def credentials(cmdr: str) -> Optional[str]: def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Dict[str, Any], state: Dict[str, Any]): # Send any unsent events when switching accounts - if cmdr and cmdr != this.cmdr: - call(force=True) + # if cmdr and cmdr != this.cmdr: + # call(force=True) event_name = entry['event'] this.cmdr = cmdr @@ -309,6 +357,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.station_marketid = None if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(cmdr): + current_creds = Credentials(this.cmdr, this.FID, str(credentials(this.cmdr))) try: # Dump starting state to Inara if (this.newuser or @@ -319,41 +368,44 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.newsession = False # Send rank info to Inara on startup - add_event( + new_add_event( 'setCommanderRankPilot', entry['timestamp'], [ - OrderedDict([ - ('rankName', k.lower()), - ('rankValue', v[0]), - ('rankProgress', v[1] / 100.0), - ]) for k, v in state['Rank'].items() if v is not None + {'rankName': k.lower(), 'rankValue': v[0], 'rankProgress': v[1] / 100.0} + for k, v in state['Rank'].items() if v is not None ] ) - add_event( + new_add_event( 'setCommanderReputationMajorFaction', entry['timestamp'], [ - OrderedDict([('majorfactionName', k.lower()), ('majorfactionReputation', v / 100.0)]) + {'majorfactionName': k.lower(), 'majorfactionReputation': v / 100.0} for k, v in state['Reputation'].items() if v is not None ] ) if state['Engineers']: # Not populated < 3.3 - add_event( + to_send = [] + for k, v in state['Engineers'].items(): + e = {'engineerName': k} + if isinstance(v, tuple): + e['rankValue'] = v[0] + + else: + e['rankStage'] = v + + to_send.append(e) + + new_add_event( 'setCommanderRankEngineer', entry['timestamp'], - [ - OrderedDict( - [('engineerName', k), isinstance(v, tuple) and ('rankValue', v[0]) or ('rankStage', v)] - ) - for k, v in state['Engineers'].items() - ] + to_send, ) # Update location - add_event( + new_add_event( 'setCommanderTravelLocation', entry['timestamp'], OrderedDict([ @@ -364,104 +416,94 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Update ship if state['ShipID']: # Unknown if started in Fighter or SRV - data = OrderedDict([ - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None - ('isCurrentShip', True), - ]) + cur_ship = { + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + 'shipName': state['ShipName'], + 'shipIdent': state['ShipIdent'], + 'isCurrentShip': True, + + } if state['HullValue']: - data['shipHullValue'] = state['HullValue'] + cur_ship['shipHullValue'] = state['HullValue'] if state['ModulesValue']: - data['shipModulesValue'] = state['ModulesValue'] + cur_ship['shipModulesValue'] = state['ModulesValue'] - data['shipRebuyCost'] = state['Rebuy'] - add_event('setCommanderShip', entry['timestamp'], data) + cur_ship['shipRebuyCost'] = state['Rebuy'] + new_add_event('setCommanderShip', entry['timestamp'], cur_ship) this.loadout = make_loadout(state) - add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) - - call() # Call here just to be sure that if we can send, we do, otherwise it'll get it in the next tick + new_add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) # Promotions elif event_name == 'Promotion': for k, v in state['Rank'].items(): if k in entry: - add_event( + new_add_event( 'setCommanderRankPilot', entry['timestamp'], - OrderedDict([ - ('rankName', k.lower()), - ('rankValue', v[0]), - ('rankProgress', 0), - ]) + {'rankName': k.lower(), 'rankValue': v[0], 'rankProgress': 0} ) elif event_name == 'EngineerProgress' and 'Engineer' in entry: - add_event( + to_send = {'engineerName': entry['Engineer']} + if 'Rank' in entry: + to_send['rankValue'] = entry['Rank'] + + else: + to_send['rankStage'] = entry['Progress'] + + new_add_event( 'setCommanderRankEngineer', entry['timestamp'], - OrderedDict([ - ('engineerName', entry['Engineer']), - ('rankValue', entry['Rank']) if 'Rank' in entry else ('rankStage', entry['Progress']), - ]) + to_send ) # PowerPlay status change if event_name == 'PowerplayJoin': - add_event( + new_add_event( 'setCommanderRankPower', entry['timestamp'], - OrderedDict([ - ('powerName', entry['Power']), - ('rankValue', 1), - ]) + {'powerName': entry['Power'], 'rankValue': 1} ) elif event_name == 'PowerplayLeave': - add_event( + new_add_event( 'setCommanderRankPower', entry['timestamp'], - OrderedDict([ - ('powerName', entry['Power']), - ('rankValue', 0), - ]) + {'powerName': entry['Power'], 'rankValue': 0} ) elif event_name == 'PowerplayDefect': - add_event( + new_add_event( 'setCommanderRankPower', entry['timestamp'], - OrderedDict([ - ('powerName', entry['ToPower']), - ('rankValue', 1), - ]) + {'powerName': entry['ToPower'], 'rankValue': 1} ) # Ship change if event_name == 'Loadout' and this.shipswap: - data = OrderedDict([ - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None - ('isCurrentShip', True), - ]) + cur_ship = { + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + 'shipName': state['ShipName'], # Can be None + 'shipIdent': state['ShipIdent'], # Can be None + 'isCurrentShip': True, + } if state['HullValue']: - data['shipHullValue'] = state['HullValue'] + cur_ship['shipHullValue'] = state['HullValue'] if state['ModulesValue']: - data['shipModulesValue'] = state['ModulesValue'] + cur_ship['shipModulesValue'] = state['ModulesValue'] - data['shipRebuyCost'] = state['Rebuy'] - add_event('setCommanderShip', entry['timestamp'], data) + cur_ship['shipRebuyCost'] = state['Rebuy'] + new_add_event('setCommanderShip', entry['timestamp'], cur_ship) this.loadout = make_loadout(state) - add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) + new_add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) this.shipswap = False # Location change @@ -475,15 +517,15 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.suppress_docked = False else: - add_event( + new_add_event( 'addCommanderTravelDock', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('stationName', station), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ]) + { + 'starsystemName': system, + 'stationName': station, + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + } ) elif event_name == 'Undocked': @@ -493,64 +535,60 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif event_name == 'SupercruiseEntry': if this.undocked: # Staying in system after undocking - send any pending events from in-station action - add_event( + new_add_event( 'setCommanderTravelLocation', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ]) + { + 'starsystemName': system, + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + } ) this.undocked = False elif event_name == 'FSDJump': this.undocked = False - add_event( + new_add_event( 'addCommanderTravelFSDJump', entry['timestamp'], - OrderedDict([ - ('starsystemName', entry['StarSystem']), - ('jumpDistance', entry['JumpDist']), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ]) + { + 'starsystemName': entry['StarSystem'], + 'jumpDistance': entry['JumpDist'], + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + } ) if entry.get('Factions'): - add_event( + new_add_event( 'setCommanderReputationMinorFaction', entry['timestamp'], [ - OrderedDict( - [('minorfactionName', f['Name']), ('minorfactionReputation', f['MyReputation']/100.0)] - ) + {'minorfactionName': f['Name'], 'minorfactionReputation': f['MyReputation'] / 100.0} for f in entry['Factions'] ] ) elif event_name == 'CarrierJump': - add_event( + new_add_event( 'addCommanderTravelCarrierJump', entry['timestamp'], - OrderedDict([ - ('starsystemName', entry['StarSystem']), - ('stationName', entry['StationName']), - ('marketID', entry['MarketID']), - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ]) + { + 'starsystemName': entry['StarSystem'], + 'stationName': entry['StationName'], + 'marketID': entry['MarketID'], + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + } ) if entry.get('Factions'): - add_event( + new_add_event( 'setCommanderReputationMinorFaction', entry['timestamp'], [ - OrderedDict( - [('minorfactionName', f['Name']), ('minorfactionReputation', f['MyReputation']/100.0)] - ) + {'minorfactionName': f['Name'], 'minorfactionReputation': f['MyReputation'] / 100.0} for f in entry['Factions'] ] ) @@ -558,11 +596,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Ignore the following 'Docked' event this.suppress_docked = True - cargo = [OrderedDict([('itemName', k), ('itemCount', state['Cargo'][k])]) for k in sorted(state['Cargo'])] + cargo = [{'itemName': k, 'itemCount': state['Cargo'][k]} for k in sorted(state['Cargo'])] # Send cargo and materials if changed if this.cargo != cargo: - add_event('setCommanderInventoryCargo', entry['timestamp'], cargo) + new_add_event('setCommanderInventoryCargo', entry['timestamp'], cargo) this.cargo = cargo materials = [] @@ -572,7 +610,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di ) if this.materials != materials: - add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) + new_add_event('setCommanderInventoryMaterials', entry['timestamp'], materials) this.materials = materials except Exception as e: @@ -581,23 +619,23 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Send credits and stats to Inara on startup only - otherwise may be out of date if event_name == 'LoadGame': - add_event( + new_add_event( 'setCommanderCredits', entry['timestamp'], - OrderedDict([('commanderCredits', state['Credits']), ('commanderLoan', state['Loan'])]) + {'commanderCredits': state['Credits'], 'commanderLoan': state['Loan']} ) this.lastcredits = state['Credits'] elif event_name == 'Statistics': - add_event('setCommanderGameStatistics', entry['timestamp'], state['Statistics']) # may be out of date + new_add_event('setCommanderGameStatistics', entry['timestamp'], state['Statistics']) # may be out of date # Selling / swapping ships if event_name == 'ShipyardNew': add_event( 'addCommanderShip', entry['timestamp'], - OrderedDict([('shipType', entry['ShipType']), ('shipGameID', entry['NewShipID'])]) + {'shipType': entry['ShipType'], 'shipGameID': entry['NewShipID']} ) this.shipswap = True # Want subsequent Loadout event to be sent immediately @@ -607,51 +645,51 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di this.shipswap = True # Don't know new ship name and ident 'til the following Loadout event if 'StoreShipID' in entry: - add_event( + new_add_event( 'setCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', entry['StoreOldShip']), - ('shipGameID', entry['StoreShipID']), - ('starsystemName', system), - ('stationName', station), - ]) + { + 'shipType': entry['StoreOldShip'], + 'shipGameID': entry['StoreShipID'], + 'starsystemName': system, + 'stationName': station, + } ) elif 'SellShipID' in entry: - add_event( + new_add_event( 'delCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', entry.get('SellOldShip', entry['ShipType'])), - ('shipGameID', entry['SellShipID']), - ]) + { + 'shipType': entry.get('SellOldShip', entry['ShipType']), + 'shipGameID': entry['SellShipID'], + } ) elif event_name == 'SetUserShipName': - add_event( + new_add_event( 'setCommanderShip', entry['timestamp'], - OrderedDict([ - ('shipType', state['ShipType']), - ('shipGameID', state['ShipID']), - ('shipName', state['ShipName']), # Can be None - ('shipIdent', state['ShipIdent']), # Can be None - ('isCurrentShip', True), - ]) + { + 'shipType': state['ShipType'], + 'shipGameID': state['ShipID'], + 'shipName': state['ShipName'], # Can be None + 'shipIdent': state['ShipIdent'], # Can be None + 'isCurrentShip': True, + } ) elif event_name == 'ShipyardTransfer': - add_event( + new_add_event( 'setCommanderShipTransfer', entry['timestamp'], - OrderedDict([ - ('shipType', entry['ShipType']), - ('shipGameID', entry['ShipID']), - ('starsystemName', system), - ('stationName', station), - ('transferTime', entry['TransferTime']), - ]) + { + 'shipType': entry['ShipType'], + 'shipGameID': entry['ShipID'], + 'starsystemName': system, + 'stationName': station, + 'transferTime': entry['TransferTime'], + } ) # Fleet @@ -679,22 +717,24 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if this.fleet != fleet: this.fleet = fleet - this.events = [x for x in this.events if x['eventName'] != 'setCommanderShip'] # Remove any unsent + new_this.filter_events(current_creds, lambda e: e.name != 'setCommanderShip') + + # this.events = [x for x in this.events if x['eventName'] != 'setCommanderShip'] # Remove any unsent for ship in this.fleet: - add_event('setCommanderShip', entry['timestamp'], ship) + new_add_event('setCommanderShip', entry['timestamp'], ship) # Loadout if event_name == 'Loadout' and not this.newsession: loadout = make_loadout(state) if this.loadout != loadout: this.loadout = loadout - # Remove any unsent for this ship - this.events = [ - e for e in this.events - if e['eventName'] != 'setCommanderShipLoadout' or e['shipGameID'] != this.loadout['shipGameID'] - ] - add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) + new_this.filter_events( + current_creds, + lambda e: e.name != 'setCommanderShipLoadout' or e.data['shipGameID'] != this.loadout['shipGameID'] + ) + + new_add_event('setCommanderShipLoadout', entry['timestamp'], this.loadout) # Stored modules if event_name == 'StoredModules': @@ -729,8 +769,10 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Only send on change this.storedmodules = modules # Remove any unsent - this.events = list(filter(lambda e: e['eventName'] != 'setCommanderStorageModules', this.events)) - add_event('setCommanderStorageModules', entry['timestamp'], this.storedmodules) + new_this.filter_events(current_creds, lambda e: e.name != 'setCommanderStorageModules') + + # this.events = list(filter(lambda e: e['eventName'] != 'setCommanderStorageModules', this.events)) + new_add_event('setCommanderStorageModules', entry['timestamp'], this.storedmodules) # Missions if event_name == 'MissionAccepted': @@ -764,14 +806,14 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if prop in entry: data[iprop] = entry[prop] - add_event('addCommanderMission', entry['timestamp'], data) + new_add_event('addCommanderMission', entry['timestamp'], data) elif event_name == 'MissionAbandoned': - add_event('setCommanderMissionAbandoned', entry['timestamp'], {'missionGameID': entry['MissionID']}) + new_add_event('setCommanderMissionAbandoned', entry['timestamp'], {'missionGameID': entry['MissionID']}) elif event_name == 'MissionCompleted': for x in entry.get('PermitsAwarded', []): - add_event('addCommanderPermit', entry['timestamp'], {'starsystemName': x}) + new_add_event('addCommanderPermit', entry['timestamp'], {'starsystemName': x}) data = OrderedDict([('missionGameID', entry['MissionID'])]) if 'Donation' in entry: @@ -810,10 +852,10 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if factioneffects: data['minorfactionEffects'] = factioneffects - add_event('setCommanderMissionCompleted', entry['timestamp'], data) + new_add_event('setCommanderMissionCompleted', entry['timestamp'], data) elif event_name == 'MissionFailed': - add_event('setCommanderMissionFailed', entry['timestamp'], {'missionGameID': entry['MissionID']}) + new_add_event('setCommanderMissionFailed', entry['timestamp'], {'missionGameID': entry['MissionID']}) # Combat if event_name == 'Died': @@ -824,7 +866,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif 'KillerName' in entry: data['opponentName'] = entry['KillerName'] - add_event('addCommanderCombatDeath', entry['timestamp'], data) + new_add_event('addCommanderCombatDeath', entry['timestamp'], data) elif event_name == 'Interdicted': data = OrderedDict([('starsystemName', system), @@ -841,7 +883,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif 'Power' in entry: data['opponentName'] = entry['Power'] - add_event('addCommanderCombatInterdicted', entry['timestamp'], data) + new_add_event('addCommanderCombatInterdicted', entry['timestamp'], data) elif event_name == 'Interdiction': data: OrderedDictT[str, Any] = OrderedDict([ @@ -859,36 +901,40 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif 'Power' in entry: data['opponentName'] = entry['Power'] - add_event('addCommanderCombatInterdiction', entry['timestamp'], data) + new_add_event('addCommanderCombatInterdiction', entry['timestamp'], data) elif event_name == 'EscapeInterdiction': - add_event( + new_add_event( 'addCommanderCombatInterdictionEscape', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('opponentName', entry['Interdictor']), - ('isPlayer', entry['IsPlayer']), - ]) + { + 'starsystemName': system, + 'opponentName': entry['Interdictor'], + 'isPlayer': entry['IsPlayer'], + } ) elif event_name == 'PVPKill': - add_event( + new_add_event( 'addCommanderCombatKill', entry['timestamp'], - OrderedDict([ - ('starsystemName', system), - ('opponentName', entry['Victim']), - ]) + { + 'starsystemName': system, + 'opponentName': entry['Victim'], + } ) # Community Goals if event_name == 'CommunityGoal': # Remove any unsent - this.events = list(filter( - lambda e: e['eventName'] not in ('setCommunityGoal', 'setCommanderCommunityGoalProgress'), - this.events - )) + new_this.filter_events( + current_creds, lambda e: e.name not in ('setCommunityGoal', 'setCommanderCommunityGoalProgress') + ) + + # this.events = list(filter( + # lambda e: e['eventName'] not in ('setCommunityGoal', 'setCommanderCommunityGoalProgress'), + # this.events + # )) for goal in entry['CurrentGoals']: data = OrderedDict([ @@ -912,7 +958,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di data['tierMax'] = int(goal['TopTier']['Name'].split()[-1]) data['completionBonus'] = goal['TopTier']['Bonus'] - add_event('setCommunityGoal', entry['timestamp'], data) + new_add_event('setCommunityGoal', entry['timestamp'], data) data = OrderedDict([ ('communitygoalGameID', goal['CGID']), @@ -926,28 +972,28 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di if 'PlayerInTopRank' in goal: data['isTopRank'] = goal['PlayerInTopRank'] - add_event('setCommanderCommunityGoalProgress', entry['timestamp'], data) + new_add_event('setCommanderCommunityGoalProgress', entry['timestamp'], data) # Friends if event_name == 'Friends': if entry['Status'] in ['Added', 'Online']: - add_event( + new_add_event( 'addCommanderFriend', entry['timestamp'], - OrderedDict([ - ('commanderName', entry['Name']), - ('gamePlatform', 'pc'), - ]) + { + 'commanderName': entry['Name'], + 'gamePlatform': 'pc', + } ) elif entry['Status'] in ['Declined', 'Lost']: - add_event( + new_add_event( 'delCommanderFriend', entry['timestamp'], - OrderedDict([ - ('commanderName', entry['Name']), - ('gamePlatform', 'pc'), - ]) + { + 'commanderName': entry['Name'], + 'gamePlatform': 'pc', + } ) this.newuser = False @@ -1005,14 +1051,18 @@ def cmdr_data(data, is_beta): if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(this.cmdr): if not (CREDIT_RATIO > this.lastcredits / data['commander']['credits'] > 1/CREDIT_RATIO): - this.events = [x for x in this.events if x['eventName'] != 'setCommanderCredits'] # Remove any unsent - add_event( + new_this.filter_events( + Credentials(this.cmdr, this.FID, str(credentials(this.cmdr))), + lambda e: e.name != 'setCommanderCredits') + + # this.events = [x for x in this.events if x['eventName'] != 'setCommanderCredits'] # Remove any unsent + new_add_event( 'setCommanderCredits', data['timestamp'], - OrderedDict([ - ('commanderCredits', data['commander']['credits']), - ('commanderLoan', data['commander'].get('debt', 0)), - ]) + { + 'commanderCredits': data['commander']['credits'], + 'commanderLoan': data['commander'].get('debt', 0), + } ) this.lastcredits = float(data['commander']['credits']) @@ -1078,137 +1128,146 @@ def make_loadout(state: Dict[str, Any]) -> OrderedDictT[str, Any]: ]) -EVENT_DATA = Mapping[AnyStr, Any] - - -def add_event(name: str, timestamp: str, data: Union[EVENT_DATA, Sequence[EVENT_DATA]]): +def new_add_event( + name: str, + timestamp: str, + data: EVENT_DATA, + cmdr: Optional[str] = None, + fid: Optional[str] = None +): """ - Add an event to the event queue + add a journal event to the queue, to be sent to inara at the next opportunity. If provided, use the given cmdr + name over the current one :param name: name of the event - :param timestamp: timestamp for the event - :param data: data to be sent in the payload + :param timestamp: timestamp of the event + :param data: payload for the event + :param cmdr: the commander to send as, defaults to the current commander """ + if cmdr is None: + cmdr = this.cmdr - this.events.append(OrderedDict([ - ('eventName', name), - ('eventTimestamp', timestamp), - ('eventData', data), - ])) + if fid is None: + fid = this.FID - -def call_timer(wait: int = FLOOD_LIMIT_SECONDS): - """ - call_timer runs in its own thread polling out to INARA once every FLOOD_LIMIT_SECONDS - - :param wait: time to wait between polls, defaults to FLOOD_LIMIT_SECONDS - """ - while this.timer_run: - time.sleep(wait) - if this.timer_run: # check again in here just in case we're closing and the stars align - call() - - -def call(callback=None, force=False): - """ - call queues a call out to the inara API - - Note that it will not allow a call more than once every FLOOD_LIMIT_SECONDS - unless the force parameter is True. - - :param callback: Unused and ignored. , defaults to None - :param force: Whether or not to ignore flood limits, defaults to False - """ - if not this.events: + api_key = credentials(this.cmdr) + if api_key is None: + logger.warn(f"cannot find an API key for cmdr {this.cmdr!r}") return - if (time.time() - config.getint(LAST_UPDATE_CONF_KEY)) <= FLOOD_LIMIT_SECONDS and not force: - return + key = Credentials(str(cmdr), str(fid), api_key) # this fails type checking due to `this` weirdness, hence str() - config.set(LAST_UPDATE_CONF_KEY, int(time.time())) - logger.info(f"queuing upload of {len(this.events)} events") - data = OrderedDict([ - ('header', OrderedDict([ - ('appName', applongname), - ('appVersion', appversion), - ('APIkey', credentials(this.cmdr)), - ('commanderName', this.cmdr), - ('commanderFrontierID', this.FID), - ])), - ('events', list(this.events)), # shallow copy - ]) - - this.events = [] - this.queue.put(('https://inara.cz/inapi/v1/', data, None)) - -# Worker thread + with new_this.event_lock: + new_this.events[key].append(Event(name, timestamp, data)) -def worker(): - """ - worker is the main thread worker and backbone of the plugin. - - As events are added to `this.queue`, the worker thread will push them to the API - """ - +def new_worker(): while True: - item = this.queue.get() - if not item: - return # Closing - else: - (url, data, callback) = item + events = get_events() + for creds, event_list in events.items(): + if not event_list: + continue - retrying = 0 - while retrying < 3: - try: - r = this.session.post(url, data=json.dumps(data, separators=(',', ':')), timeout=_TIMEOUT) - r.raise_for_status() - reply = r.json() - status = reply['header']['eventStatus'] - if callback: - callback(reply) + data = { + 'header': { + 'appName': applongname, + 'appVersion': appversion, + 'APIkey': creds.api_key, + 'commanderName': creds.cmdr, + 'commanderFrontierID': creds.fid + }, + 'events': [ + {'eventName': e.name, 'eventTimestamp': e.timestamp, 'eventData': e.data} for e in event_list + ] + } + logger.info(f'sending {len(data["events"])} events for {creds.cmdr}') + try_send_data(TARGET_URL, data) - elif status // 100 != 2: # 2xx == OK (maybe with warnings) - # Log fatal errors - logger.warning(f'Inara\t{status} {reply["header"].get("eventStatusText", "")}') - logger.debug(f'JSON data:\n{json.dumps(data, indent=2, separators = (",", ": "))}') - plug.show_error(_('Error: Inara {MSG}').format(MSG=reply['header'].get('eventStatusText', status))) + time.sleep(WORKER_WAIT_TIME) - else: - # Log individual errors and warnings - for data_event, reply_event in zip(data['events'], reply['events']): - if reply_event['eventStatus'] != 200: - logger.warning(f'Inara\t{status} {reply_event.get("eventStatusText", "")}') - logger.debug(f'JSON data:\n{json.dumps(data_event)}') - if reply_event['eventStatus'] // 100 != 2: - plug.show_error(_('Error: Inara {MSG}').format( - MSG=f'{data_event["eventName"]},' - f'{reply_event.get("eventStatusText", reply_event["eventStatus"])}')) - if data_event['eventName'] in ('addCommanderTravelCarrierJump', - 'addCommanderTravelDock', - 'addCommanderTravelFSDJump', - 'setCommanderTravelLocation'): - this.lastlocation = reply_event.get('eventData', {}) - # calls update_location in main thread - this.system_link.event_generate('<>', when="tail") +def get_events(clear=True) -> Dict[Credentials, List[Event]]: + """ + get_events fetches all events from the current queue and returns a frozen version of them - elif data_event['eventName'] in ['addCommanderShip', 'setCommanderShip']: - this.lastship = reply_event.get('eventData', {}) - # calls update_ship in main thread - this.system_link.event_generate('<>', when="tail") + :param clear: whether or not to clear the queues as we go, defaults to True + :return: the frozen event list + """ + out: Dict[Credentials, List[Event]] = {} + with new_this.event_lock: + for key, events in new_this.events.items(): + out[key] = list(events) + if clear: + events.clear() + return out + + +def try_send_data(url: str, data: Mapping[str, Any]): + """ + attempt repeatedly to send the payload forward + + :param url: target URL for the payload + :param data: the payload + """ + for i in range(3): + logger.debug(f"sending data to API, retry #{i}") + try: + if send_data(url, data): break - except Exception as e: - logger.debug('Unable to send events', exc_info=e) - retrying += 1 - else: - if callback: - callback(None) + except Exception as e: + logger.debug('unable to send events', exc_info=e) + return - else: - plug.show_error(_("Error: Can't connect to Inara")) + +def send_data(url: str, data: Mapping[str, Any]) -> bool: + """ + write a set of events to the inara API + + :param url: the target URL to post to + :param data: the data to POST + :return: success state + """ + r = this.session.post(url, data=json.dumps(data, separators=(',', ':')), timeout=_TIMEOUT) + r.raise_for_status() + reply = r.json() + status = reply['header']['eventStatus'] + + if status // 100 != 2: # 2xx == OK (maybe with warnings) + # Log fatal errors + logger.warning(f'Inara\t{status} {reply["header"].get("eventStatusText", "")}') + logger.debug(f'JSON data:\n{json.dumps(data, indent=2, separators = (",", ": "))}') + plug.show_error(_('Error: Inara {MSG}').format(MSG=reply['header'].get('eventStatusText', status))) + + else: + # Log individual errors and warnings + for data_event, reply_event in zip(data['events'], reply['events']): + if reply_event['eventStatus'] != 200: + logger.warning(f'Inara\t{status} {reply_event.get("eventStatusText", "")}') + logger.debug(f'JSON data:\n{json.dumps(data_event)}') + if reply_event['eventStatus'] // 100 != 2: + plug.show_error(_('Error: Inara {MSG}').format( + MSG=f'{data_event["eventName"]},' + f'{reply_event.get("eventStatusText", reply_event["eventStatus"])}' + )) + + if data_event['eventName'] in ( + 'addCommanderTravelCarrierJump', + 'addCommanderTravelDock', + 'addCommanderTravelFSDJump', + 'setCommanderTravelLocation' + ): + this.lastlocation = reply_event.get('eventData', {}) + # calls update_location in main thread + this.system_link.event_generate('<>', when="tail") + + elif data_event['eventName'] in ['addCommanderShip', 'setCommanderShip']: + this.lastship = reply_event.get('eventData', {}) + # calls update_ship in main thread + this.system_link.event_generate('<>', when="tail") + + return True # regardless of errors above, we DID manage to send it, therefore inform our caller as such def update_location(event=None): From d5dd23ce3848dbb882dbfc9d0e8681c0483ea5b3 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 4 Aug 2020 19:25:44 +0200 Subject: [PATCH 234/258] Missed an add_event call --- plugins/inara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index 1e252ac9..f727cfec 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -632,7 +632,7 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Selling / swapping ships if event_name == 'ShipyardNew': - add_event( + new_add_event( 'addCommanderShip', entry['timestamp'], {'shipType': entry['ShipType'], 'shipGameID': entry['NewShipID']} From 3155b929fa4ae93a47a39b7ef66c933f70306ad3 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 13:37:19 +0200 Subject: [PATCH 235/258] Fixed docstring on Credentials, log wording --- plugins/inara.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index f727cfec..52813104 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -85,7 +85,7 @@ STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00 class Credentials(NamedTuple): """ - Credentials holds an inara API payload + Credentials holds the set of credentials required to identify an inara API payload to inara """ cmdr: str fid: str @@ -1211,7 +1211,7 @@ def try_send_data(url: str, data: Mapping[str, Any]): :param data: the payload """ for i in range(3): - logger.debug(f"sending data to API, retry #{i}") + logger.debug(f"sending data to API, attempt #{i}") try: if send_data(url, data): break From 265faf3cdd9dc02da3276601f202c75677ee1deb Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 13:45:06 +0200 Subject: [PATCH 236/258] Reordered imports --- plugins/inara.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/plugins/inara.py b/plugins/inara.py index 52813104..608b8d61 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -2,31 +2,28 @@ # Inara sync # -from collections import OrderedDict, defaultdict -import json -from typing import Any, AnyStr, Callable, Deque, Dict, List, Mapping, NamedTuple, Optional, OrderedDict as OrderedDictT, \ - Sequence, TYPE_CHECKING, Tuple, Union import dataclasses - -import requests +import json +import logging import sys import time +import tkinter as tk +# For new impl +from collections import OrderedDict, defaultdict, deque from operator import itemgetter from queue import Queue from threading import Lock, Thread -import logging +from typing import TYPE_CHECKING, Any, AnyStr, Callable, Deque, Dict, List, Mapping, NamedTuple, Optional +from typing import OrderedDict as OrderedDictT +from typing import Sequence, Union -import tkinter as tk -from ttkHyperlinkLabel import HyperlinkLabel -import myNotebook as nb +import requests -from config import appname, applongname, appversion, config +import myNotebook as nb # noqa: N813 import plug import timeout_session - -# For new impl -from collections import deque - +from config import applongname, appname, appversion, config +from ttkHyperlinkLabel import HyperlinkLabel logger = logging.getLogger(appname) From b5d3b89a3cbf44ab05416d1fa34b2b9b2c571fe2 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 14:39:12 +0200 Subject: [PATCH 237/258] Added docstrings and further type annotations --- plugins/eddn.py | 111 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 18 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index e3392dc4..ecc90d5f 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -42,7 +42,9 @@ this.planet = None # Avoid duplicates this.marketId = None -this.commodities = this.outfitting = this.shipyard = None +this.commodities = None +this.outfitting: Optional[Tuple[bool, MutableMapping[str, Any]]] = None +this.shipyard = None HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' @@ -70,7 +72,12 @@ class EDDN: self.replayfile: Optional[TextIO] = None # For delayed messages self.replaylog: List[str] = [] - def load(self) -> bool: + def load_journal_replay(self) -> bool: + """ + Load cached journal entries from disk + + :return: a bool indicating success + """ # Try to obtain exclusive access to the journal cache filename = join(config.app_dir, 'replay.jsonl') try: @@ -100,6 +107,9 @@ class EDDN: return True def flush(self): + """ + flush flushes the replay file, clearing any data currently there that is not in the replaylog list + """ self.replayfile.seek(0, SEEK_SET) self.replayfile.truncate() for line in self.replaylog: @@ -108,12 +118,21 @@ class EDDN: self.replayfile.flush() def close(self): + """ + close closes the replay file + """ if self.replayfile: self.replayfile.close() self.replayfile = None - def send(self, cmdr: str, msg: Mapping[str, Any]): + def send(self, cmdr: str, msg: Mapping[str, Any]) -> None: + """ + Send sends an update to EDDN + + :param cmdr: the CMDR to use as the uploader ID + :param msg: the payload to send + """ uploader_id = cmdr to_send: OrderedDictT[str, str] = OrderedDict([ @@ -132,7 +151,10 @@ class EDDN: r.raise_for_status() - def sendreplay(self): + def sendreplay(self) -> None: + """ + sendreplay updates EDDN with cached journal lines + """ if not self.replayfile: return # Probably closing app @@ -183,7 +205,14 @@ class EDDN: self.parent.after(self.REPLAYPERIOD, self.sendreplay) - def export_commodities(self, data: Mapping[str, Any], is_beta: bool): + def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: + """ + export_commodities updates EDDN with the commodities on the current (lastStarport) station. + Once the send is complete, this.commodities is updated with the new data. + + :param data: a dict containing the starport data + :param is_beta: whether or not we're currently in beta mode + """ commodities: List[OrderedDictT[str, Any]] = [] for commodity in data['lastStarport'].get('commodities') or []: # Check 'marketable' and 'not prohibited' @@ -229,7 +258,14 @@ class EDDN: this.commodities = commodities - def export_outfitting(self, data: Mapping[str, Any], is_beta: bool): + def export_outfitting(self, data: Mapping[str, Any], is_beta: bool) -> None: + """ + export_outfitting updates EDDN with the current (lastStarport) station's outfitting options, if any. + Once the send is complete, this.outfitting is updated with the given data. + + :param data: dict containing the outfitting data + :param is_beta: whether or not we're currently in beta mode + """ economies: Dict[str, Any] = data['lastStarport'].get('economies') or {} modules: Dict[str, Any] = data['lastStarport'].get('modules') or {} ships: Dict[str, Union[Dict[str, Any], List]] = data['lastStarport'].get('ships') or { @@ -269,7 +305,14 @@ class EDDN: this.outfitting = (horizons, outfitting) - def export_shipyard(self, data: Dict[str, Any], is_beta: bool): + def export_shipyard(self, data: Dict[str, Any], is_beta: bool) -> None: + """ + export_shipyard updates EDDN with the current (lastStarport) station's outfitting options, if any. + once the send is complete, this.shipyard is updated to the new data. + + :param data: dict containing the shipyard data + :param is_beta: whether or not we are in beta mode + """ economies: Dict[str, Any] = data['lastStarport'].get('economies') or {} modules: Dict[str, Any] = data['lastStarport'].get('modules') or {} ships: Dict[str, Any] = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} @@ -301,7 +344,15 @@ class EDDN: this.shipyard = (horizons, shipyard) - def export_journal_commodities(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + def export_journal_commodities(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: + """ + export_journal_commodities updates EDDN with the commodities list on the current station (lastStarport) from + data in the journal. As a side effect, it also updates this.commodities with the data + + :param cmdr: The commander to send data under + :param is_beta: whether or not we're in beta mode + :param entry: the journal entry containing the commodities data + """ items: List[Mapping[str, Any]] = entry.get('Items') or [] commodities: Sequence[OrderedDictT[AnyStr, Any]] = sorted((OrderedDict([ ('name', self.canonicalise(commodity['Name'])), @@ -328,7 +379,15 @@ class EDDN: this.commodities: OrderedDictT[str, Any] = commodities - def export_journal_outfitting(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + def export_journal_outfitting(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: + """ + export_journal_outfitting updates EDDN with station outfitting based on a journal entry. As a side effect, + it also updates this.outfitting with the data + + :param cmdr: The commander to send data under + :param is_beta: Whether or not we're in beta mode + :param entry: The relevant journal entry + """ modules: List[Mapping[str, Any]] = entry.get('Items', []) horizons: bool = entry.get('Horizons', False) # outfitting = sorted([self.MODULE_RE.sub(lambda m: m.group(0).capitalize(), module['Name']) @@ -353,7 +412,15 @@ class EDDN: this.outfitting = (horizons, outfitting) - def export_journal_shipyard(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + def export_journal_shipyard(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: + """ + export_journal_shipyard updates EDDN with station shipyard data based on a journal entry. As a side effect, + this.shipyard is updated with the data. + + :param cmdr: the commander to send this update under + :param is_beta: Whether or not we're in beta mode + :param entry: the relevant journal entry + """ ships: List[Mapping[str, Any]] = entry.get('PriceList') or [] horizons: bool = entry.get('Horizons', False) shipyard = sorted(ship['ShipType'] for ship in ships) @@ -373,13 +440,21 @@ class EDDN: this.shipyard = (horizons, shipyard) - def export_journal_entry(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]): + def export_journal_entry(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: + """ + export_journal_entry updates EDDN with a line from the journal. Additionally if additional lines are cached, + it may send those as well. + + :param cmdr: the commander under which this upload is made + :param is_beta: whether or not we are in beta mode + :param entry: the journal entry to send + """ msg = { '$schemaRef': f'https://eddn.edcd.io/schemas/journal/1{"/test" if is_beta else ""}', 'message': entry } - if self.replayfile or self.load(): + if self.replayfile or self.load_journal_replay(): # Store the entry self.replaylog.append(json.dumps([cmdr, msg])) self.replayfile.write(f'{self.replaylog[-1]}\n') @@ -405,15 +480,15 @@ class EDDN: # Plugin callbacks -def plugin_start3(plugin_dir): +def plugin_start3(plugin_dir: str) -> str: return 'EDDN' -def plugin_app(parent: tk.Tk): +def plugin_app(parent: tk.Tk) -> None: this.parent = parent this.eddn = EDDN(parent) # Try to obtain exclusive lock on journal cache, even if we don't need it yet - if not this.eddn.load(): + if not this.eddn.load_journal_replay(): # Shouldn't happen - don't bother localizing this.status['text'] = 'Error: Is another copy of this app already running?' @@ -469,13 +544,13 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: return eddnframe -def prefsvarchanged(event=None): +def prefsvarchanged(event=None) -> None: this.eddn_station_button['state'] = tk.NORMAL this.eddn_system_button['state'] = tk.NORMAL this.eddn_delay_button['state'] = this.eddn.replayfile and this.eddn_system.get() and tk.NORMAL or tk.DISABLED -def prefs_changed(cmdr: str, is_beta: bool): +def prefs_changed(cmdr: str, is_beta: bool) -> None: config.set( 'output', (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + @@ -485,7 +560,7 @@ def prefs_changed(cmdr: str, is_beta: bool): ) -def plugin_stop(): +def plugin_stop() -> None: this.eddn.close() From 237e5ce52d2e5df890ab367d4d83d6e146d11a2f Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 14:43:53 +0200 Subject: [PATCH 238/258] Replaced subscript and concat with replace() call --- plugins/eddn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index ecc90d5f..7d1fc698 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -185,7 +185,10 @@ class EDDN: else: # Rewrite old schema name if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'): - msg['$schemaRef'] = 'https://eddn.edcd.io/schemas/' + msg['$schemaRef'][38:] # TODO: 38?! + msg['$schemaRef'] = str(msg['$schemaRef']).replace( + 'http://schemas.elite-markets.net/eddn/', + 'https://eddn.edcd.io/schemas/' + ) try: self.send(cmdr, msg) From 17f8433a6a7a498126590509d95fde5c3a1d7280 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 14:45:48 +0200 Subject: [PATCH 239/258] clarify comment --- plugins/eddn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 7d1fc698..8786d5a7 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -630,7 +630,8 @@ def journal_entry( entry.pop(thing, None) if 'Factions' in entry: - # Filter faction state. `entry` is a shallow copy so replace 'Factions' value rather than modify in-place. + # Filter faction state to comply with schema restrictions regarding personal data. `entry` is a shallow copy + # so replace 'Factions' value rather than modify in-place. entry['Factions'] = [ { k: v for k, v in f.items() if k not in ( From 94418dc4fa319aeaa53936c93bef62fb92ab748f Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 14:54:04 +0200 Subject: [PATCH 240/258] Switched to using pathlib --- plugins/eddn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 8786d5a7..db648a31 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -3,6 +3,7 @@ import itertools import json import logging +import pathlib import re import sys import tkinter as tk @@ -684,10 +685,9 @@ def journal_entry( this.commodities = this.outfitting = this.shipyard = None this.marketId = entry['MarketID'] - with open( - join(str(config.get('journaldir') or config.default_journal_dir), f'{entry["event"]}.json'), 'rb' - ) as h: - entry = json.load(h) + path = pathlib.Path(str(config.get('journaldir') or config.default_journal_dir)) / f'{entry["event"]}.json' + with path.open('rb') as f: + entry = json.load(f) if entry['event'] == 'Market': this.eddn.export_journal_commodities(cmdr, is_beta, entry) From a88cb454da9c0dddb695673f67c4369caf9f3222 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 14:54:43 +0200 Subject: [PATCH 241/258] Removed todo related to Horizons --- plugins/eddn.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index db648a31..cdd64c45 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -50,12 +50,7 @@ this.shipyard = None HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' -# TODO: -# 1, this does a bunch of magic checks for whether or not the given commander has horizons. We can get that directly -# from CAPI via commander.capabilities.Horizons -- Should this be changed to use that? did such a thing exist when -# this was written? -# -# 2, a good few of these methods are static or could be classmethods. they should be created as such. +# TODO: a good few of these methods are static or could be classmethods. they should be created as such. class EDDN: # SERVER = 'http://localhost:8081' # testing From 7adf522de9d8b5a332f674fbc2b2a498fe1d4e14 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 15:18:33 +0200 Subject: [PATCH 242/258] Replaced repeated code with helper function --- plugins/eddn.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index cdd64c45..877573e2 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -265,19 +265,15 @@ class EDDN: :param data: dict containing the outfitting data :param is_beta: whether or not we're currently in beta mode """ - economies: Dict[str, Any] = data['lastStarport'].get('economies') or {} modules: Dict[str, Any] = data['lastStarport'].get('modules') or {} - ships: Dict[str, Union[Dict[str, Any], List]] = data['lastStarport'].get('ships') or { - 'shipyard_list': {}, 'unavailable_list': [] - } # Horizons flag - will hit at least Int_PlanetApproachSuite other than at engineer bases ("Colony"), # prison or rescue Megaships, or under Pirate Attack etc - horizons = ( - any(economy['name'] == 'Colony' for economy in economies.values()) or - any(module.get('sku') == HORIZ_SKU for module in modules.values()) or - any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) - ) + horizons: bool = is_horizons( + data['lastStarport'].get('economies', {}), + modules, + data['lastStarport'].get('ships', {'shipyard_list': {}, 'unavailable_list': []}) + ) to_search: Iterator[Mapping[str, Any]] = filter( lambda m: self.MODULE_RE.search(m['name']) and m.get('sku') in (None, HORIZ_SKU) and @@ -312,14 +308,12 @@ class EDDN: :param data: dict containing the shipyard data :param is_beta: whether or not we are in beta mode """ - economies: Dict[str, Any] = data['lastStarport'].get('economies') or {} - modules: Dict[str, Any] = data['lastStarport'].get('modules') or {} - ships: Dict[str, Any] = data['lastStarport'].get('ships') or {'shipyard_list': {}, 'unavailable_list': []} - horizons: bool = ( - any(economy['name'] == 'Colony' for economy in economies.values()) or - any(module.get('sku') == HORIZ_SKU for module in modules.values()) or - any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) - ) + ships: Dict[str, Any] = data['lastStarport'].get('ships', {'shipyard_list': {}, 'unavailable_list': []}) + horizons: bool = is_horizons( + data['lastStarport'].get('economies', {}), + data['lastStarport'].get('modules', {}), + ships + ) shipyard: List[Mapping[str, Any]] = sorted( itertools.chain( @@ -728,3 +722,14 @@ def cmdr_data(data: Mapping[str, Any], is_beta: bool) -> str: except Exception as e: logger.debug('Failed exporting data', exc_info=e) return str(e) + + +MAP_STR_ANY = Mapping[str, Any] + + +def is_horizons(economies: MAP_STR_ANY, modules: MAP_STR_ANY, ships: MAP_STR_ANY) -> bool: + return ( + any(economy['name'] == 'Colony' for economy in economies.values()) or + any(module.get('sku') == HORIZ_SKU for module in modules.values()) or + any(ship.get('sku') == HORIZ_SKU for ship in (ships['shipyard_list'] or {}).values()) + ) From de872cdfa665b21b4683430956ed17cb441edb0a Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 7 Aug 2020 15:46:46 +0200 Subject: [PATCH 243/258] Added warning log messages --- plugins/eddn.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/eddn.py b/plugins/eddn.py index 877573e2..b16fc98b 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -639,18 +639,21 @@ def journal_entry( # add mandatory StarSystem, StarPos and SystemAddress properties to Scan events if 'StarSystem' not in entry: if not system: + logger.warn("system is None, can't add StarSystem") return "system is None, can't add StarSystem" entry['StarSystem'] = system if 'StarPos' not in entry: if not this.coordinates: + logger.warn("this.coordinates is None, can't add StarPos") return "this.coordinates is None, can't add StarPos" entry['StarPos'] = list(this.coordinates) if 'SystemAddress' not in entry: if not this.systemaddress: + logger.warn("this.systemaddress is None, can't add SystemAddress") return "this.systemaddress is None, can't add SystemAddress" entry['SystemAddress'] = this.systemaddress From f75d8c9c9c9a9e2bf53d68ab678a337d322aba15 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 25 Aug 2020 13:37:26 +0100 Subject: [PATCH 244/258] plugins/edsm: Set system_link based on system_provider, not station_provider --- plugins/edsm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/edsm.py b/plugins/edsm.py index 790f9289..9c9d0070 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -295,7 +295,7 @@ def cmdr_data(data, is_beta): this.station = this.station or data['commander']['docked'] and data['lastStarport']['name'] # TODO: Fire off the EDSM API call to trigger the callback for the icons - if config.get('station_provider') == 'EDSM': + if config.get('system_provider') == 'EDSM': this.system_link['text'] = this.system this.system_link['url'] = system_url(this.system) this.system_link.update_idletasks() From 7617ce7e9ccec98129e74831400f4f9c82acd1c1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 25 Aug 2020 21:38:22 +0100 Subject: [PATCH 245/258] system/station plugin providers: Don't override 'url' By default the ttkHyperlinkLabels for 'system' and 'station' names have their 'url' members set to the functions in EDMarketConnector.App. The EDDB one used to override the function as it had to do that special name -> EDDB ID lookup from systems.p. When I changed the code to not need that any more I didn't fully understand what these overrides were. After updating the EDDB code I then made sure the same logic was also in the other plugins which meant they *also* set static strings, overriding the call to the EDMarketConnector.App functions (which chain through to the current plugin providers). Unfortunately I didn't quite update the EDSM code enough causing journal_entry() code to *not* set a new system 'url' despite changing the 'text'. This meant that only CAPI updates (so docking and login) caused the URL to change, despite updating the 'text' to the correct system name. Rather than have everything setting static strings just do away with the overrides as they're not needed! --- plugins/eddb.py | 22 +++++++++++----------- plugins/edsm.py | 6 ++++-- plugins/inara.py | 19 ++++++++----------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 1613ba5b..683b1b9a 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -81,13 +81,9 @@ def plugin_app(parent: 'Tk'): def prefs_changed(cmdr, is_beta): - # Override standard URL functions - if config.get('system_provider') == 'eddb': - this.system_link['url'] = system_url(this.system) - - if config.get('station_provider') == 'eddb': - this.station_link['url'] = station_url(this.system, this.station) - + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. + pass def journal_entry(cmdr, is_beta, system, station, entry, state): # Always update, even if we're not the *current* system or station provider. @@ -110,7 +106,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): # Only actually change URLs if we are current provider. if config.get('system_provider') == 'eddb': this.system_link['text'] = this.system - this.system_link['url'] = system_url(this.system) # Override standard URL function + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.system_link.update_idletasks() # But only actually change the URL if we are current station provider. @@ -124,7 +121,8 @@ def journal_entry(cmdr, is_beta, system, station, entry, state): text = '' this.station_link['text'] = text - this.station_link['url'] = station_url(this.system, this.station) # Override standard URL function + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.station_link.update_idletasks() @@ -143,7 +141,8 @@ def cmdr_data(data, is_beta): # Override standard URL functions if config.get('system_provider') == 'eddb': this.system_link['text'] = this.system - this.system_link['url'] = system_url(this.system) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.system_link.update_idletasks() if config.get('station_provider') == 'eddb': @@ -156,5 +155,6 @@ def cmdr_data(data, is_beta): else: this.station_link['text'] = '' - this.station_link['url'] = station_url(this.system, this.station) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.station_link.update_idletasks() diff --git a/plugins/edsm.py b/plugins/edsm.py index 9c9d0070..27cb6c41 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -297,7 +297,8 @@ def cmdr_data(data, is_beta): if config.get('system_provider') == 'EDSM': this.system_link['text'] = this.system - this.system_link['url'] = system_url(this.system) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.system_link.update_idletasks() if config.get('station_provider') == 'EDSM': if data['commander']['docked']: @@ -307,7 +308,8 @@ def cmdr_data(data, is_beta): else: this.station_link['text'] = '' - this.station_link['url'] = station_url(this.system, this.station) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.station_link.update_idletasks() diff --git a/plugins/inara.py b/plugins/inara.py index 608b8d61..ebba39c0 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -244,13 +244,6 @@ def prefs_changed(cmdr: str, is_beta: bool): changed = config.getint('inara_out') != this.log.get() config.set('inara_out', this.log.get()) - # Override standard URL functions - if config.get('system_provider') == 'Inara': - this.system_link['url'] = system_url(this.system) - - if config.get('station_provider') == 'Inara': - this.station_link['url'] = station_url(this.system, this.station) - if cmdr and not is_beta: this.cmdr = cmdr this.FID = None @@ -998,7 +991,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di # Only actually change URLs if we are current provider. if config.get('system_provider') == 'Inara': this.system_link['text'] = this.system - this.system_link['url'] = system_url(this.system) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.system_link.update_idletasks() if config.get('station_provider') == 'Inara': @@ -1010,7 +1004,8 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di to_set = '' this.station_link['text'] = to_set - this.station_link['url'] = station_url(this.system, this.station) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.station_link.update_idletasks() @@ -1030,7 +1025,8 @@ def cmdr_data(data, is_beta): # Override standard URL functions if config.get('system_provider') == 'Inara': this.system_link['text'] = this.system - this.system_link['url'] = system_url(this.system) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.system_link.update_idletasks() if config.get('station_provider') == 'Inara': @@ -1043,7 +1039,8 @@ def cmdr_data(data, is_beta): else: this.station_link['text'] = '' - this.station_link['url'] = station_url(this.system, this.station) + # Do *NOT* set 'url' here, as it's set to a function that will call + # through correctly. We don't want a static string. this.station_link.update_idletasks() if config.getint('inara_out') and not is_beta and not this.multicrew and credentials(this.cmdr): From daed08d20659e1726a141904f53069100a57d96e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 10:18:10 +0100 Subject: [PATCH 246/258] system/station providers: Sanitise {system,station}_url logic * Make all plugins use `requests.utils.requote_uri()` * Make all plugins use roughly the same logic, without if/else trees (as the bodies do a `return ...`), ending with `return ''` if input parameters are None. This throws away the inara fallback to `this.station or this.system` as it's unlikely the in-plugin tracking did a better job than the monitor.py code. --- plugins/eddb.py | 9 +++------ plugins/edsm.py | 18 ++++++++++-------- plugins/inara.py | 12 ++++-------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index 683b1b9a..b9dc6b78 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -51,20 +51,17 @@ def system_url(system_name: str) -> str: if this.system_address: return requests.utils.requote_uri(f'https://eddb.io/system/ed-address/{this.system_address}') - elif system_name: + if system_name: return requests.utils.requote_uri(f'https://eddb.io/system/name/{system_name}') - else: - return '' + return '' def station_url(system_name: str, station_name: str) -> str: if this.station_marketid: return requests.utils.requote_uri(f'https://eddb.io/station/market-id/{this.station_marketid}') - else: - return system_url('') - + return system_url(system_name) def plugin_start3(plugin_dir): return 'eddb' diff --git a/plugins/edsm.py b/plugins/edsm.py index 27cb6c41..90066ee4 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -14,9 +14,6 @@ import json import requests import sys -import urllib.request -import urllib.error -import urllib.parse from queue import Queue from threading import Thread import logging @@ -60,14 +57,19 @@ STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00 # Main window clicks def system_url(system_name): - return 'https://www.edsm.net/en/system?systemName=%s' % urllib.parse.quote(system_name) + if system_name: + return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={system_name}') + + return '' def station_url(system_name, station_name): - if station_name: - return 'https://www.edsm.net/en/system?systemName=%s&stationName=%s' % (urllib.parse.quote(system_name), urllib.parse.quote(station_name)) - else: - return 'https://www.edsm.net/en/system?systemName=%s&stationName=ALL' % urllib.parse.quote(system_name) + if system_name and station_name: + return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={system_name}&stationName={station_name}') + if system_name: + return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={system_name}&stationName=ALL') + + return '' def plugin_start3(plugin_dir): # Can't be earlier since can only call PhotoImage after window is created diff --git a/plugins/inara.py b/plugins/inara.py index ebba39c0..529596f9 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -133,18 +133,14 @@ def system_url(system_name: str): return this.system +def station_url(system_name, station_name): + if system_name and station_name: + return requests.utils.requote_uri(f'https://inara.cz/galaxy-station/?search={system_name}%20[{station_name}]') -def station_url(system_name: Optional[str], station_name: Optional[str]): if system_name: - if station_name: - return requests.utils.requote_uri( - f'https://inara.cz/galaxy-station/?search={system_name}%20[{station_name}]' - ) - return system_url(system_name) - return this.station if this.station else this.system - + return '' def plugin_start3(plugin_dir): this.thread = Thread(target=new_worker, name='Inara worker') From 113b6c427c3972d12ae20d569cf3cac79007b791 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 11:55:23 +0100 Subject: [PATCH 247/258] station providers: Ensure the 'early station' functionality for all * If you request docking successfully then show the station namd and have the link work. * This is then only undone if you: 1) Dock and undock 2) Supercruise away 3) Jump away It is *not* undone if you simply cancel the docking request. Tested only with same provider for system and station for each of the three, not the other 6 combinations. --- plugins/edsm.py | 7 +++++++ plugins/inara.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/plugins/edsm.py b/plugins/edsm.py index 90066ee4..1f25f76d 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -57,6 +57,9 @@ STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00 # Main window clicks def system_url(system_name): + if this.system_address: + return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemID64={this.system_address}') + if system_name: return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={system_name}') @@ -66,6 +69,10 @@ def station_url(system_name, station_name): if system_name and station_name: return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={system_name}&stationName={station_name}') + # monitor state might think these are gone, but we don't yet + if this.system and this.station: + return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={this.system}&stationName={this.station}') + if system_name: return requests.utils.requote_uri(f'https://www.edsm.net/en/system?systemName={system_name}&stationName=ALL') diff --git a/plugins/inara.py b/plugins/inara.py index 529596f9..df03207a 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -137,6 +137,10 @@ def station_url(system_name, station_name): if system_name and station_name: return requests.utils.requote_uri(f'https://inara.cz/galaxy-station/?search={system_name}%20[{station_name}]') + # monitor state might think these are gone, but we don't yet + if this.system and this.station: + return requests.utils.requote_uri(f'https://inara.cz/galaxy-station/?search={this.system}%20[{this.station}]') + if system_name: return system_url(system_name) From bd7a3eca04598ed81073d107771ea1bb18656c95 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 12:01:45 +0100 Subject: [PATCH 248/258] plugins/inara: `return ''` if can't otherwise make system_url --- plugins/inara.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inara.py b/plugins/inara.py index df03207a..d1d5903d 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -131,7 +131,7 @@ def system_url(system_name: str): elif system_name: return requests.utils.requote_uri(f'https://inara.cz/galaxy-starsystem/?search={system_name}') - return this.system + return '' def station_url(system_name, station_name): if system_name and station_name: From 98b6d4db3838440d953c1332e163e30ab4be200f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 14:13:06 +0100 Subject: [PATCH 249/258] Plugins "Not Python 3.x": Hacky escaping fix so translations work --- EDMarketConnector.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 2d1b81b9..2e76163f 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1055,10 +1055,10 @@ if __name__ == "__main__": if (plugins_not_py3_last + 86400) < int(time()) and len(plug.PLUGINS_not_py3): # Yes, this is horribly hacky so as to be sure we match the key # that we told Translators to use. - popup_text = "One or more of your enabled plugins do not yet have support for Python 3.x. Please see the "\ - "list on the '{PLUGINS}' tab of '{FILE}' > '{SETTINGS}'. You should check if there is an "\ - "updated version available, else alert the developer that they need to update the code for "\ - "Python 3.x.\r\n\r\nYou can disable a plugin by renaming its folder to have '{DISABLED}' on "\ + popup_text = "One or more of your enabled plugins do not yet have support for Python 3.x. Please see the " \ + "list on the '{PLUGINS}' tab of '{FILE}' > '{SETTINGS}'. You should check if there is an " \ + "updated version available, else alert the developer that they need to update the code for " \ + "Python 3.x.\r\n\r\nYou can disable a plugin by renaming its folder to have '{DISABLED}' on " \ "the end of the name." popup_text = popup_text.replace('\n', '\\n') popup_text = popup_text.replace('\r', '\\r') @@ -1069,6 +1069,7 @@ if __name__ == "__main__": # And now we do need these to be actual \r\n popup_text = popup_text.replace('\\n', '\n') popup_text = popup_text.replace('\\r', '\r') + tk.messagebox.showinfo( _('EDMC: Plugins Without Python 3.x Support'), popup_text From c12625124641907ecc133bab661b853b9bd8c1a5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 14:21:25 +0100 Subject: [PATCH 250/258] "Not Python 3.x" popup message sub-substitutions fixed. string.format() doesn't assign to string, so actually need to do that. --- EDMarketConnector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 2e76163f..cfd92b1c 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -1065,7 +1065,7 @@ if __name__ == "__main__": # Now the string should match, so try translation popup_text = _(popup_text) # And substitute in the other words. - popup_text.format(PLUGINS=_('Plugins'), FILE=_('File'), SETTINGS=_('Settings'), DISABLED='.disabled') + popup_text = popup_text.format(PLUGINS=_('Plugins'), FILE=_('File'), SETTINGS=_('Settings'), DISABLED='.disabled') # And now we do need these to be actual \r\n popup_text = popup_text.replace('\\n', '\n') popup_text = popup_text.replace('\\r', '\r') From c7cdb6ad665660a6675bd012b03f9d3b560a4001 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 14:29:42 +0100 Subject: [PATCH 251/258] translations: pt-PT: "Use alternate URL method" now translated --- L10n/pt-PT.strings | 3 +++ 1 file changed, 3 insertions(+) diff --git a/L10n/pt-PT.strings b/L10n/pt-PT.strings index 50a86c34..4049d8dd 100644 --- a/L10n/pt-PT.strings +++ b/L10n/pt-PT.strings @@ -490,6 +490,9 @@ /* Update button in main window. [EDMarketConnector.py] */ "Update" = "Actualizar"; +/* Option to use alternate URL method on shipyard links [prefs.py] */ +"Use alternate URL method" = "Usar método URL alternativo."; + /* Status dialog subtitle - CR value of ship. [stats.py] */ "Value" = "Valor"; From cea26ad0cdedd7a442f2ba69d6ca4988c738e820 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 14:31:23 +0100 Subject: [PATCH 252/258] Releasing.md: Now using Python 3.7.9 --- docs/Releasing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Releasing.md b/docs/Releasing.md index 0e704d08..26246803 100644 --- a/docs/Releasing.md +++ b/docs/Releasing.md @@ -32,7 +32,7 @@ You will need several pieces of software installed, or the files from their "Windows Software Development Kit - Windows 10.0.18362.1" in "Apps & Features", *not* "Windows SDK AddOn". 1. [Python](https://python.org): 32-bit version of Python 3.7 for Windows. - [v3.7.7](https://www.python.org/downloads/release/python-377/) is the most + [v3.7.9](https://www.python.org/downloads/release/python-379/) is the most recently tested version. You need the `Windows x86 executable installer` file, for the 32-bit version. 1. [py2exe](https://github.com/albertosottile/py2exe): From af12816efbbc4fb6b430e69300332d2de2504be6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 14:34:56 +0100 Subject: [PATCH 253/258] Release 4.0.5: version, changelog, appcast --- ChangeLog.md | 30 ++++++++++++++++++++++++++++++ config.py | 2 +- edmarketconnector.xml | 42 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 1c5c7701..250a545f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,36 @@ This is the master changelog for Elite Dangerous Market Connector. Entries are in reverse chronological order (latest first). --- +Release 4.0.5 +=== + + * Built using Python 3.7.9. + * Fix EDSM plugin so the System provider actually updates the URLs for + jumps to new systems. + + In general this cleans up the code for all three System and Station Providers; + EDDB, EDSM, Inara. + +Release 4.0.4 +=== + + * Built using Python 3.7.8. Prior 4.0.x releases used 3.7.7. + * Don't crash if no non-default Journal Directory has been set. + * Only send to Inara API at most once every 30 seconds. This should avoid + the "Inara 400 Too much requests, slow down, cowboy. ;) ..." message and + being locked out from the API for an hour as a result. Any events that + require data to be sent during the 30s cooldown will be queued and sent when + that timer expires. + + This was caused by previous changes in an attempt to send cargo events + to Inara more often. This fix retains that enhancement. + + Note that if you log out and stop EDMC within 30 seconds you might have + some events not sent. If we tried to force a send then it might hit the + limit when you want to log back in and continue playing. As it is you can + re-run EDMC and log back into the game to ensure Inara is synchronised + properly. + Release 4.0.3 === diff --git a/config.py b/config.py index b0b887ec..568178ac 100644 --- a/config.py +++ b/config.py @@ -13,7 +13,7 @@ appcmdname = 'EDMC' # appversion **MUST** follow Semantic Versioning rules: # # Major.Minor.Patch(-prerelease)(+buildmetadata) -appversion = '4.0.3' #-rc1+a872b5f' +appversion = '4.0.5' #-rc1+a872b5f' # For some things we want appversion without (possible) +build metadata appversion_nobuild = str(semantic_version.Version(appversion).truncate('prerelease')) copyright = u'© 2015-2019 Jonathan Harris, 2020 EDCD' diff --git a/edmarketconnector.xml b/edmarketconnector.xml index 1ad38955..e2491386 100644 --- a/edmarketconnector.xml +++ b/edmarketconnector.xml @@ -168,11 +168,45 @@ - Release 4.0.3 + Release 4.0.5 body { font-family:"Segoe UI","Tahoma"; font-size: 75%; } h2 { font-family:"Segoe UI","Tahoma"; font-size: 105%; } +

Release 4.0.5

+
    +
  • Built using Python 3.7.9.
    • +
  • Fix EDSM plugin so the System provider actually updates the URLs for +jumps to new systems.
    • +
+

In general this cleans up the code for all three System and Station Providers; +EDDB, EDSM, Inara.

+ +

Release 4.0.4

+
    +
  • +

    Built using Python 3.7.8. Prior 4.0.x releases used 3.7.7.

    +
    • +
  • +

    Don't crash if no non-default Journal Directory has been set.

    +
    • +
  • +

    Only send to Inara API at most once every 30 seconds. This should avoid +the "Inara 400 Too much requests, slow down, cowboy. ;) ..." message and +being locked out from the API for an hour as a result. Any events that +require data to be sent during the 30s cooldown will be queued and sent when +that timer expires.

    +

    This was caused by previous changes in an attempt to send cargo events +to Inara more often. This fix retains that enhancement.

    +

    Note that if you log out and stop EDMC within 30 seconds you might have +some events not sent. If we tried to force a send then it might hit the +limit when you want to log back in and continue playing. As it is you can +re-run EDMC and log back into the game to ensure Inara is synchronised +properly.

    +
    • +
+ +

Release 4.0.3

NB: Anyone who installed a 4.0.3-rcX release candidate version should first uninstall it before installing this. @@ -578,11 +612,11 @@ If any of your plugins are listed in that section then they will need updating, ]]> From 935c852605fb1bc717fa4da13b1f24928c940146 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 14:38:14 +0100 Subject: [PATCH 254/258] appcast: Update size for 4.0.5 --- edmarketconnector.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edmarketconnector.xml b/edmarketconnector.xml index e2491386..683137b0 100644 --- a/edmarketconnector.xml +++ b/edmarketconnector.xml @@ -616,7 +616,7 @@ If any of your plugins are listed in that section then they will need updating, sparkle:os="windows" sparkle:installerArguments="/passive LAUNCH=yes" sparkle:version="4.0.5" - length="11419648" + length="11317248" type="application/octet-stream" /> From a00d5df92b11d3762970c3cc6cffbf0ab4830706 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 15:13:07 +0100 Subject: [PATCH 255/258] Update version in BOTH places on that URL line --- edmarketconnector.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edmarketconnector.xml b/edmarketconnector.xml index 683137b0..7041e5c3 100644 --- a/edmarketconnector.xml +++ b/edmarketconnector.xml @@ -612,7 +612,7 @@ If any of your plugins are listed in that section then they will need updating, ]]> Date: Wed, 26 Aug 2020 18:48:23 +0200 Subject: [PATCH 256/258] Fixed system link updating on FSDTarget FSDTarget contains the target system under `SystemAddress`, meaning that any time you selected a star other than the current one, plugins' `this.system_address` was updated to that target, rather than the current system. Said updating causes the links provided from system_url to reflect that update (for providers that support ID64s). This changes the journal_entry behaviour to only update `this.system_address` when the event is any of Location, Docked, or FSDJump, all of which contain only the current system. --- plugins/eddb.py | 8 +++++--- plugins/edsm.py | 8 +++++--- plugins/inara.py | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/plugins/eddb.py b/plugins/eddb.py index b9dc6b78..c0ef35fc 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -83,9 +83,11 @@ def prefs_changed(cmdr, is_beta): pass def journal_entry(cmdr, is_beta, system, station, entry, state): - # Always update, even if we're not the *current* system or station provider. - this.system_address = entry.get('SystemAddress') or this.system_address - this.system = entry.get('StarSystem') or this.system + # Always update our system address even if we're not currently the provider for system or station, but dont update + # on events that contain "future" data, such as FSDTarget + if entry['event'] in ('Location', 'Docked', 'CarrierJump', 'FSDJump'): + this.system_address = entry.get('SystemAddress') or this.system_address + this.system = entry.get('StarSystem') or this.system # We need pop == 0 to set the value so as to clear 'x' in systems with # no stations. diff --git a/plugins/edsm.py b/plugins/edsm.py index 1f25f76d..806536cb 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -217,9 +217,11 @@ def credentials(cmdr): def journal_entry(cmdr, is_beta, system, station, entry, state): - # Always update, even if we're not the *current* system or station provider. - this.system_address = entry.get('SystemAddress') or this.system_address - this.system = entry.get('StarSystem') or this.system + # Always update our system address even if we're not currently the provider for system or station, but dont update + # on events that contain "future" data, such as FSDTarget + if entry['event'] in ('Location', 'Docked', 'CarrierJump', 'FSDJump'): + this.system_address = entry.get('SystemAddress') or this.system_address + this.system = entry.get('StarSystem') or this.system # We need pop == 0 to set the value so as to clear 'x' in systems with # no stations. diff --git a/plugins/inara.py b/plugins/inara.py index d1d5903d..e78287d1 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -329,9 +329,11 @@ def journal_entry(cmdr: str, is_beta: bool, system: str, station: str, entry: Di elif event_name in ('ShipyardNew', 'ShipyardSwap') or (event_name == 'Location' and entry['Docked']): this.suppress_docked = True - # Always update, even if we're not the *current* system or station provider. - this.system_address = entry.get('SystemAddress', this.system_address) - this.system = entry.get('StarSystem', this.system) + # Always update our system address even if we're not currently the provider for system or station, but dont update + # on events that contain "future" data, such as FSDTarget + if entry['event'] in ('Location', 'Docked', 'CarrierJump', 'FSDJump'): + this.system_address = entry.get('SystemAddress') or this.system_address + this.system = entry.get('StarSystem') or this.system # We need pop == 0 to set the value so as to clear 'x' in systems with # no stations. From a4e1a42783713756e9b4b6fb2a2d062ac5bae61b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 18:12:31 +0100 Subject: [PATCH 257/258] Version 4.0.6 --- ChangeLog.md | 6 ++++++ config.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 250a545f..c6c6e64b 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,12 @@ This is the master changelog for Elite Dangerous Market Connector. Entries are in reverse chronological order (latest first). --- +Release 4.0.6 +=== + + * Correct the three System Provider plugins to *not* show the *next* system + in a plotted route instead of the current system. + Release 4.0.5 === diff --git a/config.py b/config.py index 568178ac..1e1e9146 100644 --- a/config.py +++ b/config.py @@ -13,7 +13,7 @@ appcmdname = 'EDMC' # appversion **MUST** follow Semantic Versioning rules: # # Major.Minor.Patch(-prerelease)(+buildmetadata) -appversion = '4.0.5' #-rc1+a872b5f' +appversion = '4.0.6' #-rc1+a872b5f' # For some things we want appversion without (possible) +build metadata appversion_nobuild = str(semantic_version.Version(appversion).truncate('prerelease')) copyright = u'© 2015-2019 Jonathan Harris, 2020 EDCD' From 253c6988ca1f627066e713339e0792fe1c45159d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 26 Aug 2020 18:15:44 +0100 Subject: [PATCH 258/258] appcast 4.0.6 --- edmarketconnector.xml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/edmarketconnector.xml b/edmarketconnector.xml index 7041e5c3..3addd416 100644 --- a/edmarketconnector.xml +++ b/edmarketconnector.xml @@ -168,11 +168,17 @@ - Release 4.0.5 + Release 4.0.6 body { font-family:"Segoe UI","Tahoma"; font-size: 75%; } h2 { font-family:"Segoe UI","Tahoma"; font-size: 105%; } +

Release 4.0.6

+
    +
  • Correct the three System Provider plugins to not show the next system +in a plotted route instead of the current system.
    • +
+

Release 4.0.5

  • Built using Python 3.7.9.
    • @@ -612,10 +618,10 @@ If any of your plugins are listed in that section then they will need updating, ]]>