diff --git a/eddb.py b/eddb.py index f4a299d0..d43e4a5f 100755 --- a/eddb.py +++ b/eddb.py @@ -4,6 +4,7 @@ # import cPickle +import csv import os from os.path import dirname, join, normpath import sys @@ -34,7 +35,7 @@ class EDDB: # -# build databases from files systems.json and stations.json from http://eddb.io/api +# build databases from files systems.csv and stations.json from http://eddb.io/api # if __name__ == "__main__": @@ -42,9 +43,9 @@ if __name__ == "__main__": import requests def download(filename): - r = requests.get('https://eddb.io/archive/v4/' + filename) + r = requests.get('https://eddb.io/archive/v4/' + filename, stream=True) print '\n%s\t%dK' % (filename, len(r.content) / 1024) - return json.loads(r.content) # let json do the utf-8 decode + return r # Ellipsoid that encompasses most of the systems in the bubble (but not outliers like Sothis) RX = RZ = 260 @@ -70,49 +71,68 @@ if __name__ == "__main__": def around_outlier(cx, cy, cz, x, y, z): return ((x - ox) * (x - ox) + (y - oy) * (y - oy) + (z - oz) * (z - oz)) <= RO2 - systems = download('systems.json') + systems = [ + { 'id' : int(s['id']), + 'name' : s['name'].decode('utf-8'), + 'x' : float(s['x']), + 'y' : float(s['y']), + 'z' : float(s['z']), + 'is_populated' : bool(s['is_populated']), + } for s in csv.DictReader(download('systems.csv').iter_lines())] print '%d\tsystems' % len(systems) # system_id by system_name (ignoring duplicate names) system_ids = dict([ (str(s['name']), s['id']) - for s in systems if s['is_populated'] or (inbubble(s['x'], s['y'], s['z']) and all(ord(c) < 128 for c in s['name'])) or around_jaques(s['x'], s['y'], s['z'])]) # skip unpopulated systems outside the bubble and those with a bogus name + for s in systems if s['is_populated'] or ((inbubble(s['x'], s['y'], s['z']) or around_jaques(s['x'], s['y'], s['z'])) and all(ord(c) < 128 for c in s['name']))]) # skip unpopulated systems outside the bubble and those with a bogus name - cut = [s for s in systems if s['is_populated'] and not inbubble(s['x'], s['y'], s['z'])] + cut = dict([(s['name'], s) for s in systems if s['is_populated'] and not inbubble(s['x'], s['y'], s['z']) and s['name'] not in ['Colonia', 'Eol Prou RS-T d3-94']]) + + # Temporary hack for new Colonia outposts: https://community.elitedangerous.com/galnet/uid/5800bf2d9657bab47f9912eb + cut.update({ 'Blu Thua AI-A c14-10': { 'id': 64214, 'x': -54.5, 'y': 149.53125, 'z': 2099.21875 }, + 'Lagoon Sector NI-S b4-10': { 'id': 69637, 'x': -469.1875, 'y': -84.84375, 'z': 4456.125 }, + 'Eagle Sector IR-W d1-117': { 'id': 855737, 'x': -2054.09375, 'y': 85.71875, 'z': 6710.875 }, + 'Skaudai CH-B d14-34': { 'id': 1328989, 'x': -5481.84375, 'y': -579.15625, 'z': 10429.9375 }, + 'Gru Hypue KS-T d3-31': { 'id': 3288878, 'x': -4990.84375, 'y': -935.71875, 'z': 13387.15625 }, + 'Boewnst KS-S c20-959': { 'id': 3317609, 'x': -6195.46875, 'y': -140.28125, 'z': 16462.0625 }, + }) print '\n%d populated systems outside bubble calculation:' % len(cut) extra_ids = {} - for o in cut: + for name,o in cut.iteritems(): ox, oy, oz = o['x'], o['y'], o['z'] - print '%-32s%7d %11.5f %11.5f %11.5f' % (o['name'], o['id'], ox, oy, oz) + print '%-32s%7d %11.5f %11.5f %11.5f' % (name, o['id'], ox, oy, oz) extra_ids.update(dict([ (str(s['name']), s['id']) for s in systems if around_outlier(ox, oy, oz, s['x'], s['y'], s['z']) and all(ord(c) < 128 for c in s['name'])])) print '\n%d systems around outliers' % len(extra_ids) system_ids.update(extra_ids) + print '%d systems around Jacques' % len([s for s in systems if around_jaques(s['x'], s['y'], s['z'])]) + cut = [s for s in systems if inbubble(s['x'], s['y'], s['z']) and system_ids.get(s['name']) is None] print '\n%d dropped systems inside bubble calculation:' % len(cut) for s in cut: print '%s%s%7d %11.5f %11.5f %11.5f' % (s['name'].encode('utf-8'), ' '*(32-len(s['name'])), s['id'], s['x'], s['y'], s['z']) - cut = [s for s in systems if (s['is_populated'] or inbubble(s['x'], s['y'], s['z'])) and system_ids.get(s['name']) and system_ids[s['name']] != s['id']] + cut = [s for s in systems if system_ids.get(s['name']) and system_ids[s['name']] != s['id'] and (s['is_populated'] or inbubble(s['x'], s['y'], s['z']))] print '\n%d duplicate systems inside bubble calculation:' % len(cut) for s in cut: print '%-24s%7d %7d %11.5f %11.5f %11.5f' % (s['name'], system_ids[s['name']], s['id'], s['x'], s['y'], s['z']) - print '\n%d systems around Jacques' % len([s for s in systems if around_jaques(s['x'], s['y'], s['z'])]) - # Hack - ensure duplicate system names are pointing at the more interesting system system_ids['Almar'] = 750 system_ids['Arti'] = 60342 system_ids['Kamba'] = 10358 + # point new name for Jaques at old entry (Eol Prou RS-T d3-94) + system_ids['Colonia'] = 633211 + with open('systems.p', 'wb') as h: cPickle.dump(system_ids, h, protocol = cPickle.HIGHEST_PROTOCOL) - print '%d saved systems' % len(system_ids) + print '\n%d saved systems' % len(system_ids) # station_id by (system_id, station_name) - stations = download('stations.json') + stations = json.loads(download('stations.json').content) # let json do the utf-8 decode station_ids = dict([( (x['system_id'], str(x['name'])), (x['id'], diff --git a/stations.p b/stations.p index 22cc2e9f..fd19c9f1 100644 Binary files a/stations.p and b/stations.p differ diff --git a/systems.p b/systems.p index 500c9a8f..f149c50d 100644 Binary files a/systems.p and b/systems.p differ