From f790468078334b9acba844d7a65bd8f25963068f Mon Sep 17 00:00:00 2001 From: Jonathan Harris Date: Sat, 19 Nov 2016 16:41:40 +0000 Subject: [PATCH] Move functions that manipulate API output --- EDMC.py | 5 +- EDMarketConnector.py | 8 +-- collate.py | 2 +- companion.py | 155 +++++++++++++++++++++++++++++-------------- edshipyard.py | 60 +---------------- 5 files changed, 115 insertions(+), 115 deletions(-) diff --git a/EDMC.py b/EDMC.py index e8bdb7e3..2195adea 100755 --- a/EDMC.py +++ b/EDMC.py @@ -68,19 +68,18 @@ try: item.find('title').text) for item in feed.findall('channel/item')]) lastversion = sorted(items, key=versioncmp)[-1] if versioncmp(lastversion) > versioncmp(appversion): - latest = ' (%s is available)' % items[lastversion] except: pass # Quietly suppress timeouts etc. print '%.2f%s' % (float(''.join(appversion.split('.')[:3])) / 100, latest) # just first three digits sys.exit(EXIT_SUCCESS) - session = companion.Session() if args.j: # Import and collate from JSON dump data = json.load(open(args.j)) config.set('querytime', getmtime(args.j)) else: + session = companion.Session() session.login(config.get('username'), config.get('password')) querytime = int(time()) data = session.query() @@ -127,7 +126,7 @@ try: sys.exit(EXIT_SUCCESS) # Fixup anomalies in the commodity data - fixed = session.fixup(data) + fixed = companion.fixup(data) if args.j: # Collate from JSON dump diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 0b9d013b..1c4af87e 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -104,7 +104,7 @@ class AppWindow: self.station_label.grid(row=4, column=0, sticky=tk.W) self.cmdr = tk.Label(frame, anchor=tk.W) - self.ship = HyperlinkLabel(frame, url = self.edshipyard_url) + self.ship = HyperlinkLabel(frame, url = self.shipyard_url) self.system = HyperlinkLabel(frame, compound=tk.RIGHT, url = self.system_url, popup_copy = True) self.station = HyperlinkLabel(frame, url = self.station_url, popup_copy = lambda x: x!=self.STATION_UNDOCKED) @@ -435,7 +435,7 @@ class AppWindow: else: if data['lastStarport'].get('commodities') and config.getint('output') & (config.OUT_MKT_CSV|config.OUT_MKT_TD|config.OUT_MKT_BPC): # Fixup anomalies in the commodity data - fixed = self.session.fixup(data) + fixed = companion.fixup(data) if config.getint('output') & config.OUT_MKT_CSV: commodity.export(fixed, COMMODITY_CSV) @@ -655,13 +655,13 @@ class AppWindow: else: self.w.after(int(EDSM_POLL * 1000), self.edsmpoll) - def edshipyard_url(self, shipname=None): + def shipyard_url(self, shipname=None): self.status['text'] = _('Fetching data...') self.w.update_idletasks() try: data = self.session.query() except companion.VerificationRequired: - return prefs.AuthenticationDialog(self.parent, partial(self.verify, self.edshipyard_url)) + return prefs.AuthenticationDialog(self.parent, partial(self.verify, self.shipyard_url)) except companion.ServerError as e: self.status['text'] = str(e) return diff --git a/collate.py b/collate.py index 852f2864..883976de 100755 --- a/collate.py +++ b/collate.py @@ -175,7 +175,7 @@ if __name__ == "__main__": print 'No starport!' else: if data['lastStarport'].get('commodities'): - fixed = session.fixup(data) + fixed = companion.fixup(data) addcommodities(fixed) else: print 'No market' diff --git a/companion.py b/companion.py index fd25961e..bfbec90b 100644 --- a/companion.py +++ b/companion.py @@ -272,58 +272,115 @@ class Session: pass self.session = None - # Returns a shallow copy of the received data with anomalies in the commodity data fixed up - def fixup(self, data): - commodities = [] - for commodity in data['lastStarport'].get('commodities') or []: - - # Check all required numeric fields are present and are numeric - # Catches "demandBracket": "" for some phantom commodites in ED 1.3 - https://github.com/Marginal/EDMarketConnector/issues/2 - # But also see https://github.com/Marginal/EDMarketConnector/issues/32 - for thing in ['buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket']: - if not isinstance(commodity.get(thing), numbers.Number): - if __debug__: print 'Invalid "%s":"%s" (%s) for "%s"' % (thing, commodity.get(thing), type(commodity.get(thing)), commodity.get('name', '')) - break - else: - if not category_map.get(commodity['categoryname'], True): # Check marketable - pass - elif not commodity.get('categoryname'): - if __debug__: print 'Missing "categoryname" for "%s"' % commodity.get('name', '') - elif not commodity.get('name'): - if __debug__: print 'Missing "name" for a commodity in "%s"' % commodity.get('categoryname', '') - elif not commodity['demandBracket'] in range(4): - if __debug__: print 'Invalid "demandBracket":"%s" for "%s"' % (commodity['demandBracket'], commodity['name']) - elif not commodity['stockBracket'] in range(4): - if __debug__: print 'Invalid "stockBracket":"%s" for "%s"' % (commodity['stockBracket'], commodity['name']) - else: - # Rewrite text fields - new = dict(commodity) # shallow copy - new['categoryname'] = category_map.get(commodity['categoryname'], commodity['categoryname']) - fixed = commodity_map.get(commodity['name']) - if type(fixed) == tuple: - (new['categoryname'], new['name']) = fixed - elif fixed: - new['name'] = fixed - - # Force demand and stock to zero if their corresponding bracket is zero - # Fixes spurious "demand": 1 in ED 1.3 - if not commodity['demandBracket']: - new['demand'] = 0 - if not commodity['stockBracket']: - new['stock'] = 0 - - # We're good - commodities.append(new) - - # return a shallow copy - datacopy = dict(data) - datacopy['lastStarport'] = dict(data['lastStarport']) - datacopy['lastStarport']['commodities'] = commodities - return datacopy - def dump(self, r): if __debug__: print 'Status\t%s' % r.status_code print 'URL\t%s' % r.url print 'Headers\t%s' % r.headers print ('Content:\n%s' % r.text).encode('utf-8') + + +# Returns a shallow copy of the received data with anomalies in the commodity data fixed up +def fixup(data): + commodities = [] + for commodity in data['lastStarport'].get('commodities') or []: + + # Check all required numeric fields are present and are numeric + # Catches "demandBracket": "" for some phantom commodites in ED 1.3 - https://github.com/Marginal/EDMarketConnector/issues/2 + # But also see https://github.com/Marginal/EDMarketConnector/issues/32 + for thing in ['buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket']: + if not isinstance(commodity.get(thing), numbers.Number): + if __debug__: print 'Invalid "%s":"%s" (%s) for "%s"' % (thing, commodity.get(thing), type(commodity.get(thing)), commodity.get('name', '')) + break + else: + if not category_map.get(commodity['categoryname'], True): # Check marketable + pass + elif not commodity.get('categoryname'): + if __debug__: print 'Missing "categoryname" for "%s"' % commodity.get('name', '') + elif not commodity.get('name'): + if __debug__: print 'Missing "name" for a commodity in "%s"' % commodity.get('categoryname', '') + elif not commodity['demandBracket'] in range(4): + if __debug__: print 'Invalid "demandBracket":"%s" for "%s"' % (commodity['demandBracket'], commodity['name']) + elif not commodity['stockBracket'] in range(4): + if __debug__: print 'Invalid "stockBracket":"%s" for "%s"' % (commodity['stockBracket'], commodity['name']) + else: + # Rewrite text fields + new = dict(commodity) # shallow copy + new['categoryname'] = category_map.get(commodity['categoryname'], commodity['categoryname']) + fixed = commodity_map.get(commodity['name']) + if type(fixed) == tuple: + (new['categoryname'], new['name']) = fixed + elif fixed: + new['name'] = fixed + + # Force demand and stock to zero if their corresponding bracket is zero + # Fixes spurious "demand": 1 in ED 1.3 + if not commodity['demandBracket']: + new['demand'] = 0 + if not commodity['stockBracket']: + new['stock'] = 0 + + # We're good + commodities.append(new) + + # return a shallow copy + datacopy = dict(data) + datacopy['lastStarport'] = dict(data['lastStarport']) + datacopy['lastStarport']['commodities'] = commodities + return datacopy + + +# Return a subset of the received data describing the current ship +def ship(data): + + # Add a leaf to a dictionary, creating empty dictionaries along the branch if necessary + def addleaf(data, to, props): + + # special handling for completely empty trees + p = props[0] + if p in data and not data[p]: + to[p] = data[p] + return + + # Does the leaf exist ? + tail = data + for p in props: + if not hasattr(data, 'get') or p not in tail: + return + else: + tail = tail[p] + + for p in props[:-1]: + if not hasattr(data, 'get') or p not in data: + return + elif p not in to: + to[p] = {} + elif not hasattr(to, 'get'): + return # intermediate is not a dictionary - inconsistency! + data = data[p] + to = to[p] + p = props[-1] + to[p] = data[p] + + # subset of "ship" that's not noisy + description = {} + for props in [ + ('alive',), + ('cargo', 'capacity'), + ('free',), + ('fuel', 'main', 'capacity'), + ('fuel', 'reserve', 'capacity'), + ('fuel', 'superchargedFSD'), + ('id',), + ('name',), + ('value', 'hull'), + ('value', 'modules'), + ('value', 'unloaned'), + ]: addleaf(data['ship'], description, props) + + description['modules'] = {} + for slot in data['ship'].get('modules', {}): + for prop in ['free', 'id', 'modifiers', 'name', 'on', 'priority', 'recipeLevel', 'recipeName', 'recipeValue', 'unloaned', 'value']: + addleaf(data['ship']['modules'], description['modules'], (slot, 'module', prop)) + + return description diff --git a/edshipyard.py b/edshipyard.py index bfab3818..742ecbfb 100644 --- a/edshipyard.py +++ b/edshipyard.py @@ -15,65 +15,9 @@ import companion from config import config -# Return a description of the current ship as a JSON object -def description(data): - - # Add a leaf to a dictionary, creating empty dictionaries along the branch if necessary - def addleaf(data, to, props): - - # special handling for completely empty trees - p = props[0] - if p in data and not data[p]: - to[p] = data[p] - return - - # Does the leaf exist ? - tail = data - for p in props: - if not hasattr(data, 'get') or p not in tail: - return - else: - tail = tail[p] - - for p in props[:-1]: - if not hasattr(data, 'get') or p not in data: - return - elif p not in to: - to[p] = {} - elif not hasattr(to, 'get'): - return # intermediate is not a dictionary - inconsistency! - data = data[p] - to = to[p] - p = props[-1] - to[p] = data[p] - - # subset of "ship" that's not noisy - ship = {} - for props in [ - ('alive',), - ('cargo', 'capacity'), - ('free',), - ('fuel', 'main', 'capacity'), - ('fuel', 'reserve', 'capacity'), - ('fuel', 'superchargedFSD'), - ('id',), - ('name',), - ('value', 'hull'), - ('value', 'modules'), - ('value', 'unloaned'), - ]: addleaf(data['ship'], ship, props) - - ship['modules'] = {} - for slot in data['ship'].get('modules', {}): - for prop in ['free', 'id', 'modifiers', 'name', 'on', 'priority', 'recipeLevel', 'recipeName', 'recipeValue', 'unloaned', 'value']: - addleaf(data['ship']['modules'], ship['modules'], (slot, 'module', prop)) - - return ship - - def export(data, filename=None): - string = json.dumps(description(data), ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')) # pretty print + string = json.dumps(companion.ship(data), ensure_ascii=False, indent=2, sort_keys=True, separators=(',', ': ')) # pretty print if filename: with open(filename, 'wt') as h: @@ -100,7 +44,7 @@ def export(data, filename=None): # Return a URL for the current ship def url(data): - string = json.dumps(description(data), ensure_ascii=False, sort_keys=True, separators=(',', ':')) # most compact representation + string = json.dumps(companion.ship(data), ensure_ascii=False, sort_keys=True, separators=(',', ':')) # most compact representation out = StringIO.StringIO() with gzip.GzipFile(fileobj=out, mode='w') as f: