From 6c9c4c73a37b00a07461ba8ce959d6d442e3161b Mon Sep 17 00:00:00 2001
From: Jonathan Harris <jonathan@marginal.org.uk>
Date: Sat, 5 Dec 2015 02:50:53 +0000
Subject: [PATCH] Add helper script for collating lists of seen commodities,
modules and ships.
---
collate.py | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++
outfitting.py | 75 +---------------------
2 files changed, 170 insertions(+), 74 deletions(-)
create mode 100755 collate.py
mode change 100755 => 100644 outfitting.py
diff --git a/collate.py b/collate.py
new file mode 100755
index 00000000..54333f6b
--- /dev/null
+++ b/collate.py
@@ -0,0 +1,169 @@
+#!/usr/bin/python
+#
+# Script for collating lists of seen commodities, modules and ships from dumps of the Companion API output
+#
+
+import csv
+import json
+import os
+from os.path import exists, isfile
+import sys
+
+from eddn import ship_map # use EDDN ship names
+from companion import category_map, commodity_map
+import outfitting
+
+
+# keep a summary of commodities found using in-game names
+def addcommodities(data):
+
+ if not data['lastStarport'].get('commodities'): return
+
+ commodityfile = 'commodities.csv'
+ commodities = set()
+
+ # slurp existing
+ if isfile(commodityfile):
+ with open(commodityfile) as csvfile:
+ reader = csv.reader(csvfile)
+ assert(reader.next()) == ['category', 'name']
+ for row in reader:
+ commodities.add(tuple(row))
+ size_pre = len(commodities)
+
+ for commodity in data['lastStarport'].get('commodities'):
+ commodities.add((category_map.get(commodity['categoryname']) or commodity['categoryname'],
+ commodity_map.get(commodity['name']) or commodity['name']))
+
+ if len(commodities) > size_pre:
+
+ if isfile(commodityfile):
+ if isfile(commodityfile+'.bak'):
+ os.unlink(commodityfile+'.bak')
+ os.rename(commodityfile, commodityfile+'.bak')
+
+ with open(commodityfile, 'wb') as csvfile:
+ writer = csv.writer(csvfile)
+ writer.writerow(['category', 'name'])
+ writer.writerows(sorted(commodities))
+
+ print 'Added %d new commodities' % (len(commodities) - size_pre)
+
+# keep a summary of modules found
+def addmodules(data):
+
+ if not data['lastStarport'].get('modules'): return
+
+ outfile = 'outfitting.csv'
+ modules = {}
+
+ # slurp existing
+ if isfile(outfile):
+ with open(outfile) as csvfile:
+ reader = csv.DictReader(csvfile)
+ for row in reader:
+ key = int(row.pop('id')) # index by int for easier lookup and sorting
+ modules[key] = row
+ size_pre = len(modules)
+
+ for key,module in data['lastStarport'].get('modules').iteritems():
+ # sanity check
+ if int(key) != module.get('id'): raise AssertionError('id: %s!=%s' % (key, module['id']))
+ new = outfitting.lookup(module, ship_map)
+ if new:
+ old = modules.get(int(key))
+ if old:
+ # check consistency with existing data
+ for thing in ['category', 'name', 'mount', 'guidance', 'ship', 'class', 'rating']:
+ if new.get(thing,'') != old.get(thing): raise AssertionError('%s: %s "%s"!="%s"' % (key, thing, new.get(thing), old.get(thing)))
+ else:
+ modules[int(key)] = new
+
+ if len(modules) > size_pre:
+
+ if isfile(outfile):
+ if isfile(outfile+'.bak'):
+ os.unlink(outfile+'.bak')
+ os.rename(outfile, outfile+'.bak')
+
+ with open(outfile, 'wb') as csvfile:
+ writer = csv.DictWriter(csvfile, ['id', 'category', 'name', 'mount', 'guidance', 'ship', 'class', 'rating'])
+ writer.writeheader()
+ for key in sorted(modules):
+ row = modules[key]
+ row['id'] = key
+ writer.writerow(row)
+
+ print 'Added %d new modules' % (len(modules) - size_pre)
+
+# keep a summary of ships found
+def addships(data):
+
+ if not data['lastStarport'].get('ships'): return
+
+ shipfile = 'shipyard.csv'
+ ships = {}
+
+ # slurp existing
+ if isfile(shipfile):
+ with open(shipfile) as csvfile:
+ reader = csv.DictReader(csvfile)
+ for row in reader:
+ ships[int(row['id'])] = row['name'] # index by int for easier lookup and sorting
+ size_pre = len(ships)
+
+ for ship in (data['lastStarport']['ships'].get('shipyard_list') or {}).values() + data['lastStarport']['ships'].get('unavailable_list'):
+ # sanity check
+ key = ship['id']
+ new = ship_map.get(ship['name'].lower())
+ if new:
+ old = ships.get(int(key))
+ if old:
+ # check consistency with existing data
+ if new != old: raise AssertionError('%s: "%s"!="%s"' % (key, new, old))
+ else:
+ ships[int(key)] = new
+
+ if len(ships) > size_pre:
+
+ if isfile(shipfile):
+ if isfile(shipfile+'.bak'):
+ os.unlink(shipfile+'.bak')
+ os.rename(shipfile, shipfile+'.bak')
+
+ with open(shipfile, 'wb') as csvfile:
+ writer = csv.DictWriter(csvfile, ['id', 'name'])
+ writer.writeheader()
+ for key in sorted(ships):
+ row = { 'id': key, 'name': ships[key] }
+ writer.writerow(row)
+
+ print 'Added %d new ships' % (len(ships) - size_pre)
+
+
+if __name__ == "__main__":
+ if len(sys.argv) <= 1:
+ print 'Usage: collate.py [dump.json]'
+ else:
+ # read from dumped json file(s)
+ for f in sys.argv[1:]:
+ with open(f) as h:
+ print f
+ data = json.load(h)
+ if not data['commander'].get('docked'):
+ print 'Not docked!'
+ elif not data.get('lastStarport'):
+ print 'No starport!'
+ else:
+ if data['lastStarport'].get('commodities'):
+ addcommodities(data)
+ else:
+ print 'No market'
+ if data['lastStarport'].get('modules'):
+ addmodules(data)
+ else:
+ print 'No outfitting',
+ if data['lastStarport'].get('ships'):
+ addships(data)
+ else:
+ print 'No shipyard'
diff --git a/outfitting.py b/outfitting.py
old mode 100755
new mode 100644
index a2728920..8cd9c27b
--- a/outfitting.py
+++ b/outfitting.py
@@ -1,21 +1,10 @@
-#!/usr/bin/python
-#
-# Script for building table ID->module mapping table from a dump of the Companion API output
-#
-
-import csv
-import json
-import os
-from os.path import exists, isfile
-import sys
import time
import companion
from config import config
-outfile = 'outfitting.csv'
-outfitting = {}
+# Map API module names to in-game names
armour_map = {
'grade1' : 'Lightweight Alloy',
@@ -349,65 +338,3 @@ def export(data, filename):
except:
if __debug__: raise
h.close()
-
-
-# add all the modules
-def addmodules(data):
- if not data.get('lastStarport'):
- print 'No Starport!'
- return
- elif not data['lastStarport'].get('modules'):
- print 'No outfitting here'
- return
-
- # read into outfitting
- if isfile(outfile):
- with open(outfile) as csvfile:
- reader = csv.DictReader(csvfile)
- for row in reader:
- key = int(row.pop('id')) # index by int for easier lookup and sorting
- outfitting[key] = row
- size_pre = len(outfitting)
-
- for key,module in data['lastStarport'].get('modules').iteritems():
- # sanity check
- if int(key) != module.get('id'): raise AssertionError('id: %s!=%s' % (key, module['id']))
- new = lookup(module)
- if new:
- old = outfitting.get(int(key))
- if old:
- # check consistency with existing data
- for thing in ['category', 'name', 'mount', 'guidance', 'ship', 'class', 'rating']:
- if new.get(thing,'') != old.get(thing): raise AssertionError('%s: %s "%s"!="%s"' % (key, thing, new.get(thing), old.get(thing)))
- else:
- outfitting[int(key)] = new
-
- if len(outfitting) > size_pre:
-
- if isfile(outfile):
- if isfile(outfile+'.bak'):
- os.unlink(outfile+'.bak')
- os.rename(outfile, outfile+'.bak')
-
- with open(outfile, 'wb') as csvfile:
- writer = csv.DictWriter(csvfile, ['id', 'category', 'name', 'mount', 'guidance', 'ship', 'class', 'rating'])
- writer.writeheader()
- for key in sorted(outfitting):
- row = outfitting[key]
- row['id'] = key
- writer.writerow(row)
-
- print 'Added %d new modules' % (len(outfitting) - size_pre)
-
- else:
- print
-
-if __name__ == "__main__":
- if len(sys.argv) <= 1:
- print 'Usage: outfitting.py [dump.json]'
- else:
- # read from dumped json file(s)
- for f in sys.argv[1:]:
- with open(f) as h:
- print f,
- addmodules(json.loads(h.read()))