Switch EDDN integration to a plugin

EDMarketConnector.py

@@ -48,7 +48,6 @@ import companion
 import commodity
 from commodity import COMMODITY_CSV
 import td
-import eddn
 import stats
 import prefs
 import plug
@@ -60,11 +59,6 @@ from theme import theme
 
 SERVER_RETRY = 5	# retry pause for Companion servers [s]
 
-# Limits on local clock drift from EDDN gateway
-DRIFT_THRESHOLD = 3 * 60
-TZ_THRESHOLD = 30 * 60
-CLOCK_THRESHOLD = 11 * 60 * 60 + TZ_THRESHOLD
-
 
 class AppWindow:
 
@@ -76,7 +70,6 @@ class AppWindow:
     def __init__(self, master):
 
         self.holdofftime = config.getint('querytime') + companion.holdoff
-        self.eddn = eddn.EDDN(self)
 
         self.w = master
         self.w.title(applongname)
@@ -312,10 +305,6 @@ class AppWindow:
         if keyring.get_keyring().priority < 1:
             self.status['text'] = 'Warning: Storing passwords as text'	# Shouldn't happen unless no secure storage on Linux
 
-        # Try to obtain exclusive lock on journal cache, even if we don't need it yet
-        if not self.eddn.load():
-            self.status['text'] = 'Error: Is another copy of this app already running?'	# Shouldn't happen - don't bother localizing
-
     # callback after the Preferences dialog is applied
     def postprefs(self, dologin=True):
         self.prefsdialog = None
@@ -471,58 +460,27 @@ class AppWindow:
                 if err:
                     play_bad = True
 
-                # Export ship for backwards compatibility even 'though it's no longer derived from cAPI
+                # Export market data
-                if config.getint('output') & config.OUT_SHIP:
+                if config.getint('output') & (config.OUT_STATION_ANY):
-                    monitor.export_ship()
+                    if not data['commander'].get('docked'):
-
-                if not (config.getint('output') & ~config.OUT_SHIP & config.OUT_STATION_ANY):
-                    # no station data requested - we're done
-                    pass
-
-                elif not data['commander'].get('docked'):
                         if not self.status['text']:
                             # Signal as error because the user might actually be docked but the server hosting the Companion API hasn't caught up
                             self.status['text'] = _("You're not docked at a station!")
                             play_bad = True
-
+                    elif (config.getint('output') & config.OUT_MKT_EDDN) and not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')):	# Ignore possibly missing shipyard info
-                else:
-                    # Finally - the data looks sane and we're docked at a station
-
-                    # No EDDN output?
-                    if (config.getint('output') & config.OUT_MKT_EDDN) and not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')):	# Ignore possibly missing shipyard info
                         if not self.status['text']:
                             self.status['text'] = _("Station doesn't have anything!")
-
+                    elif not data['lastStarport'].get('commodities'):
-                    # No market output?
-                    elif not (config.getint('output') & config.OUT_MKT_EDDN) and not data['lastStarport'].get('commodities'):
                         if not self.status['text']:
                             self.status['text'] = _("Station doesn't have a market!")
-
+                    elif config.getint('output') & (config.OUT_MKT_CSV|config.OUT_MKT_TD):
-                    else:
-                        if data['lastStarport'].get('commodities') and config.getint('output') & (config.OUT_MKT_CSV|config.OUT_MKT_TD):
                         # Fixup anomalies in the commodity data
                         fixed = companion.fixup(data)
-
                         if config.getint('output') & config.OUT_MKT_CSV:
                             commodity.export(fixed, COMMODITY_CSV)
                         if config.getint('output') & config.OUT_MKT_TD:
                             td.export(fixed)
 
-                        if config.getint('output') & config.OUT_MKT_EDDN:
-                            old_status = self.status['text']
-                            if not old_status:
-                                self.status['text'] = _('Sending data to EDDN...')
-                            self.w.update_idletasks()
-                            self.eddn.export_commodities(data, monitor.is_beta)
-                            self.eddn.export_outfitting(data, monitor.is_beta)
-                            if data['lastStarport'].get('ships', {}).get('shipyard_list'):
-                                self.eddn.export_shipyard(data, monitor.is_beta)
-                            elif data['lastStarport'].get('services', {}).get('shipyard'):
-                                # API is flakey about shipyard info - silently retry if missing (<1s is usually sufficient - 5s for margin).
-                                self.w.after(int(SERVER_RETRY * 1000), lambda:self.retry_for_shipyard(2))
-                            if not old_status:
-                                self.status['text'] = ''
-
         except companion.VerificationRequired:
             if not self.authdialog:
                 self.authdialog = prefs.AuthenticationDialog(self.w, partial(self.verify, self.getandsend))
@@ -537,11 +495,6 @@ class AppWindow:
                 self.w.after(int(SERVER_RETRY * 1000), lambda:self.getandsend(event, True))
                 return	# early exit to avoid starting cooldown count
 
-        except requests.RequestException as e:
-            if __debug__: print_exc()
-            self.status['text'] = _("Error: Can't connect to EDDN")
-            play_bad = True
-
         except Exception as e:
             if __debug__: print_exc()
             self.status['text'] = unicode(e)
@@ -630,6 +583,15 @@ class AppWindow:
             if not entry['event'] or not monitor.mode:
                 return	# Startup or in CQC
 
+            if entry['event'] in ['StartUp', 'LoadGame'] and monitor.started:
+                # Can start dashboard monitoring
+                if not dashboard.start(self.w, monitor.started):
+                    print "Can't start Status monitoring"
+
+            # Export loadout
+            if entry['event'] == 'Loadout' and not monitor.state['Captain'] and config.getint('output') & config.OUT_SHIP:
+                monitor.export_ship()
+
             # Plugins
             err = plug.notify_journal_entry(monitor.cmdr, monitor.is_beta, monitor.system, monitor.station, entry, monitor.state)
             if err:
@@ -637,67 +599,10 @@ class AppWindow:
                 if not config.getint('hotkey_mute'):
                     hotkeymgr.play_bad()
 
-            if entry['event'] in ['StartUp', 'LoadGame'] and monitor.started:
-                # Can start dashboard monitoring
-                if not dashboard.start(self.w, monitor.started):
-                    print "Can't start Status monitoring"
-
-            # Don't send to EDDN while on crew
-            if monitor.state['Captain']:
-                return
-
-            # Plugin backwards compatibility
-            if monitor.mode and entry['event'] in ['StartUp', 'Location', 'FSDJump']:
-                plug.notify_system_changed(timegm(strptime(entry['timestamp'], '%Y-%m-%dT%H:%M:%SZ')), monitor.system, monitor.coordinates)
-
-            # Export loadout
-            if monitor.mode and entry['event'] == 'Loadout' and config.getint('output') & config.OUT_SHIP:
-                monitor.export_ship()
-
             # Auto-Update after docking
-            if monitor.mode and monitor.station and entry['event'] in ['StartUp', 'Location', 'Docked'] and not config.getint('output') & config.OUT_MKT_MANUAL and config.getint('output') & config.OUT_STATION_ANY:
+            if entry['event'] in ['StartUp', 'Location', 'Docked'] and monitor.station and not config.getint('output') & config.OUT_MKT_MANUAL and config.getint('output') & config.OUT_STATION_ANY:
                 self.w.after(int(SERVER_RETRY * 1000), self.getandsend)
 
-            # Send interesting events to EDDN
-            try:
-                if (config.getint('output') & config.OUT_SYS_EDDN and monitor.cmdr and monitor.mode and
-                    (entry['event'] == 'Location' or
-                     entry['event'] == 'FSDJump' or
-                     entry['event'] == 'Docked'  or
-                     entry['event'] == 'Scan'    and monitor.system and monitor.coordinates)):
-                    # strip out properties disallowed by the schema
-                    for thing in ['ActiveFine', 'CockpitBreach', 'BoostUsed', 'FuelLevel', 'FuelUsed', 'JumpDist', 'Latitude', 'Longitude', 'Wanted']:
-                        entry.pop(thing, None)
-                    for faction in entry.get('Factions', []):
-                        faction.pop('MyReputation', None)
-
-                    # add planet to Docked event for planetary stations if known
-                    if entry['event'] == 'Docked' and monitor.planet:
-                        entry['Body'] = monitor.planet
-                        entry['BodyType'] = 'Planet'
-
-                    # add mandatory StarSystem, StarPos and SystemAddress properties to Scan events
-                    if 'StarSystem' not in entry:
-                        entry['StarSystem'] = monitor.system
-                    if 'StarPos' not in entry:
-                        entry['StarPos'] = list(monitor.coordinates)
-                    if 'SystemAddress' not in entry and monitor.systemaddress:
-                        entry['SystemAddress'] = monitor.systemaddress
-
-                    self.eddn.export_journal_entry(monitor.cmdr, monitor.is_beta, self.filter_localised(entry))
-
-            except requests.exceptions.RequestException as e:
-                if __debug__: print_exc()
-                self.status['text'] = _("Error: Can't connect to EDDN")
-                if not config.getint('hotkey_mute'):
-                    hotkeymgr.play_bad()
-
-            except Exception as e:
-                if __debug__: print_exc()
-                self.status['text'] = unicode(e)
-                if not config.getint('hotkey_mute'):
-                    hotkeymgr.play_bad()
-
     # Handle Status event
     def dashboard_event(self, event):
         entry = dashboard.status
@@ -726,21 +631,6 @@ class AppWindow:
     def station_url(self, station):
         return plug.invoke(config.get('station_provider'),  'eddb', 'station_url', monitor.system, monitor.station)
 
-
-    # Recursively filter '*_Localised' keys from dict
-    def filter_localised(self, d):
-        filtered = OrderedDict()
-        for k, v in d.iteritems():
-            if k.endswith('_Localised'):
-                pass
-            elif hasattr(v, 'iteritems'):	# dict -> recurse
-                filtered[k] = self.filter_localised(v)
-            elif isinstance(v, list):	# list of dicts -> recurse
-                filtered[k] = [self.filter_localised(x) if hasattr(x, 'iteritems') else x for x in v]
-            else:
-                filtered[k] = v
-        return filtered
-
     def cooldown(self):
         if time() < self.holdofftime:
             self.button['text'] = self.theme_button['text'] = _('cooldown {SS}s').format(SS = int(self.holdofftime - time()))	# Update button in main window
@@ -800,7 +690,6 @@ class AppWindow:
         dashboard.close()
         monitor.close()
         plug.notify_stop()
-        self.eddn.close()
         self.updater.close()
         companion.session.close()
         config.close()

EDMarketConnector.wxs

@@ -246,6 +246,9 @@
 						<Component Guid="*">
 							<File KeyPath="yes" Source="SourceDir\plugins\eddb.py" />
 						</Component>
+						<Component Guid="*">
+							<File KeyPath="yes" Source="SourceDir\plugins\eddn.py" />
+						</Component>
 						<Component Guid="*">
 							<File KeyPath="yes" Source="SourceDir\plugins\edsm.py" />
 						</Component>
@@ -490,6 +493,7 @@
 			<ComponentRef Id="EDMarketConnector.VisualElementsManifest.xml" />
 			<ComponentRef Id="EDMC.exe" />
 			<ComponentRef Id="eddb.py" />
+			<ComponentRef Id="eddn.py" />
 			<ComponentRef Id="edsm.py" />
 			<ComponentRef Id="edsy.py" />
 			<ComponentRef Id="es.strings" />

L10n/en.template

@@ -103,7 +103,7 @@
 /* Appearance theme and language setting. [l10n.py] */
 "Default" = "Default";
 
-/* Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2. [prefs.py] */
+/* Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2. [eddn.py] */
 "Delay sending until docked" = "Delay sending until docked";
 
 /* List of plugins in settings. [prefs.py] */
@@ -142,7 +142,7 @@
 /* Trade rank. [stats.py] */
 "Entrepreneur" = "Entrepreneur";
 
-/* [EDMarketConnector.py] */
+/* [eddn.py] */
 "Error: Can't connect to EDDN" = "Error: Can't connect to EDDN";
 
 /* [edsm.py] */
@@ -436,13 +436,13 @@
 /* [inara.py] */
 "Send flight log and Cmdr status to Inara" = "Send flight log and Cmdr status to Inara";
 
-/* Output setting. [prefs.py] */
+/* Output setting. [eddn.py] */
 "Send station data to the Elite Dangerous Data Network" = "Send station data to the Elite Dangerous Data Network";
 
-/* Output setting new in E:D 2.2. [prefs.py] */
+/* Output setting new in E:D 2.2. [eddn.py] */
 "Send system and scan data to the Elite Dangerous Data Network" = "Send system and scan data to the Elite Dangerous Data Network";
 
-/* [EDMarketConnector.py] */
+/* [eddn.py] */
 "Sending data to EDDN..." = "Sending data to EDDN...";
 
 /* Empire rank. [stats.py] */

eddn.py

@@ -1,231 +0,0 @@
-# Export to EDDN
-
-from collections import OrderedDict
-import json
-import numbers
-from os import SEEK_SET, SEEK_CUR, SEEK_END
-from os.path import exists, join
-from platform import system
-import re
-import requests
-from sys import platform
-import time
-import uuid
-
-if platform != 'win32':
-    from fcntl import lockf, LOCK_EX, LOCK_NB
-
-if __debug__:
-    from traceback import print_exc
-
-from config import applongname, appversion, config
-from companion import category_map
-
-
-timeout= 10	# requests timeout
-module_re = re.compile('^Hpt_|^Int_|_Armour_')
-
-replayfile = None	# For delayed messages
-
-class EDDN:
-
-    ### SERVER = 'http://localhost:8081'	# testing
-    SERVER = 'https://eddn.edcd.io:4430'
-    UPLOAD = '%s/upload/' % SERVER
-    REPLAYPERIOD = 400	# Roughly two messages per second, accounting for send delays [ms]
-    REPLAYFLUSH = 20	# Update log on disk roughly every 10 seconds
-
-    def __init__(self, parent):
-        self.parent = parent
-        self.session = requests.Session()
-        self.replaylog = []
-
-    def load(self):
-        # Try to obtain exclusive access to the journal cache
-        global replayfile
-        filename = join(config.app_dir, 'replay.jsonl')
-        try:
-            try:
-                # Try to open existing file
-                replayfile = open(filename, 'r+')
-            except:
-                if exists(filename):
-                    raise	# Couldn't open existing file
-                else:
-                    replayfile = open(filename, 'w+')	# Create file
-            if platform != 'win32':	# open for writing is automatically exclusive on Windows
-                lockf(replayfile, LOCK_EX|LOCK_NB)
-        except:
-            if __debug__: print_exc()
-            if replayfile:
-                replayfile.close()
-            replayfile = None
-            return False
-        self.replaylog = [line.strip() for line in replayfile]
-        return True
-
-    def flush(self):
-        replayfile.seek(0, SEEK_SET)
-        replayfile.truncate()
-        for line in self.replaylog:
-            replayfile.write('%s\n' % line)
-        replayfile.flush()
-
-    def close(self):
-        global replayfile
-        if replayfile:
-            replayfile.close()
-        replayfile = None
-
-    def send(self, cmdr, msg):
-        if config.getint('anonymous'):
-            uploaderID = config.get('uploaderID')
-            if not uploaderID:
-                uploaderID = uuid.uuid4().hex
-                config.set('uploaderID', uploaderID)
-        else:
-            uploaderID = cmdr.encode('utf-8')
-
-        msg = OrderedDict([
-            ('$schemaRef', msg['$schemaRef']),
-            ('header',     OrderedDict([
-                ('softwareName',    '%s [%s]' % (applongname, platform=='darwin' and "Mac OS" or system())),
-                ('softwareVersion', appversion),
-                ('uploaderID',      uploaderID),
-            ])),
-            ('message',    msg['message']),
-        ])
-
-        r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=timeout)
-        if __debug__ and r.status_code != requests.codes.ok:
-            print 'Status\t%s'  % r.status_code
-            print 'URL\t%s'  % r.url
-            print 'Headers\t%s' % r.headers
-            print ('Content:\n%s' % r.text).encode('utf-8')
-        r.raise_for_status()
-
-    def sendreplay(self):
-        if not replayfile:
-            return	# Probably closing app
-
-        if not self.replaylog:
-            self.parent.status['text'] = ''
-            return
-
-        if len(self.replaylog) == 1:
-            self.parent.status['text'] = _('Sending data to EDDN...')
-        else:
-            self.parent.status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...',''), len(self.replaylog))
-        self.parent.w.update_idletasks()
-        try:
-            cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict)
-        except:
-            # Couldn't decode - shouldn't happen!
-            if __debug__:
-                print self.replaylog[0]
-                print_exc()
-            self.replaylog.pop(0)	# Discard and continue
-        else:
-            # Rewrite old schema name
-            if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'):
-                msg['$schemaRef'] = 'https://eddn.edcd.io/schemas/' + msg['$schemaRef'][38:]
-            try:
-                self.send(cmdr, msg)
-                self.replaylog.pop(0)
-                if not len(self.replaylog) % self.REPLAYFLUSH:
-                    self.flush()
-            except requests.exceptions.RequestException as e:
-                if __debug__: print_exc()
-                self.parent.status['text'] = _("Error: Can't connect to EDDN")
-                return	# stop sending
-            except Exception as e:
-                if __debug__: print_exc()
-                self.parent.status['text'] = unicode(e)
-                return	# stop sending
-
-        self.parent.w.after(self.REPLAYPERIOD, self.sendreplay)
-
-    def export_commodities(self, data, is_beta):
-        commodities = []
-        for commodity in data['lastStarport'].get('commodities') or []:
-            if (category_map.get(commodity['categoryname'], True) and	# Check marketable
-                not commodity.get('legality')):	# check not prohibited
-                commodities.append(OrderedDict([
-                    ('name',          commodity['name']),
-                    ('meanPrice',     int(commodity['meanPrice'])),
-                    ('buyPrice',      int(commodity['buyPrice'])),
-                    ('stock',         int(commodity['stock'])),
-                    ('stockBracket',  commodity['stockBracket']),
-                    ('sellPrice',     int(commodity['sellPrice'])),
-                    ('demand',        int(commodity['demand'])),
-                    ('demandBracket', commodity['demandBracket']),
-                ]))
-                if commodity['statusFlags']:
-                    commodities[-1]['statusFlags'] = commodity['statusFlags']
-
-        # Don't send empty commodities list - schema won't allow it
-        if commodities:
-            message = OrderedDict([
-                ('timestamp',   data['timestamp']),
-                ('systemName',  data['lastSystem']['name']),
-                ('stationName', data['lastStarport']['name']),
-                ('marketId',    data['lastStarport']['id']),
-                ('commodities', commodities),
-            ])
-            if 'economies' in data['lastStarport']:
-                message['economies']  = sorted([x for x in (data['lastStarport']['economies']  or {}).itervalues()])
-            if 'prohibited' in data['lastStarport']:
-                message['prohibited'] = sorted([x for x in (data['lastStarport']['prohibited'] or {}).itervalues()])
-            self.send(data['commander']['name'], {
-                '$schemaRef' : 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''),
-                'message'    : message,
-            })
-
-    def export_outfitting(self, data, is_beta):
-        # Don't send empty modules list - schema won't allow it
-        if data['lastStarport'].get('modules'):
-            self.send(data['commander']['name'], {
-                '$schemaRef' : 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''),
-                'message'    : OrderedDict([
-                    ('timestamp',   data['timestamp']),
-                    ('systemName',  data['lastSystem']['name']),
-                    ('stationName', data['lastStarport']['name']),
-                    ('marketId',    data['lastStarport']['id']),
-                    ('modules',     sorted([module['name'] for module in data['lastStarport']['modules'].itervalues() if module_re.search(module['name']) and module.get('sku') in [None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'] and module['name'] != 'Int_PlanetApproachSuite'])),
-                ]),
-            })
-
-    def export_shipyard(self, data, is_beta):
-        # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard.
-        if data['lastStarport'].get('ships', {}).get('shipyard_list'):
-            self.send(data['commander']['name'], {
-                '$schemaRef' : 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''),
-                'message'    : OrderedDict([
-                    ('timestamp',   data['timestamp']),
-                    ('systemName',  data['lastSystem']['name']),
-                    ('stationName', data['lastStarport']['name']),
-                    ('marketId',    data['lastStarport']['id']),
-                    ('ships',       sorted([ship['name'] for ship in data['lastStarport']['ships']['shipyard_list'].values() + data['lastStarport']['ships']['unavailable_list']])),
-                ]),
-            })
-
-    def export_journal_entry(self, cmdr, is_beta, entry):
-        msg = {
-            '$schemaRef' : 'https://eddn.edcd.io/schemas/journal/1' + (is_beta and '/test' or ''),
-            'message'    : entry
-        }
-        if replayfile or self.load():
-            # Store the entry
-            self.replaylog.append(json.dumps([cmdr.encode('utf-8'), msg]))
-            replayfile.write('%s\n' % self.replaylog[-1])
-
-            if (entry['event'] == 'Docked' or
-                (entry['event'] == 'Location' and entry['Docked']) or
-                not (config.getint('output') & config.OUT_SYS_DELAY)):
-                self.parent.w.after(self.REPLAYPERIOD, self.sendreplay)	# Try to send this and previous entries
-        else:
-            # Can't access replay file! Send immediately.
-            self.parent.status['text'] = _('Sending data to EDDN...')
-            self.parent.w.update_idletasks()
-            self.send(cmdr, msg)
-            self.parent.status['text'] = ''

plugins/eddn.py

@@ -0,0 +1,382 @@
+# Export to EDDN
+
+from collections import OrderedDict
+import json
+import numbers
+from os import SEEK_SET, SEEK_CUR, SEEK_END
+from os.path import exists, join
+from platform import system
+import re
+import requests
+import sys
+import time
+import uuid
+
+import Tkinter as tk
+from ttkHyperlinkLabel import HyperlinkLabel
+import myNotebook as nb
+
+if sys.platform != 'win32':
+    from fcntl import lockf, LOCK_EX, LOCK_NB
+
+if __debug__:
+    from traceback import print_exc
+
+from config import applongname, appversion, config
+from companion import category_map
+
+
+this = sys.modules[__name__]	# For holding module globals
+
+# Track location to add to Journal events
+this.systemaddress = None
+this.coordinates = None
+this.planet = None
+
+
+class EDDN:
+
+    ### SERVER = 'http://localhost:8081'	# testing
+    SERVER = 'https://eddn.edcd.io:4430'
+    UPLOAD = '%s/upload/' % SERVER
+    REPLAYPERIOD = 400	# Roughly two messages per second, accounting for send delays [ms]
+    REPLAYFLUSH = 20	# Update log on disk roughly every 10 seconds
+    TIMEOUT= 10	# requests timeout
+    MODULE_RE = re.compile('^Hpt_|^Int_|_Armour_')
+
+    def __init__(self, parent):
+        self.parent = parent
+        self.session = requests.Session()
+        self.replayfile = None	# For delayed messages
+        self.replaylog = []
+
+    def load(self):
+        # Try to obtain exclusive access to the journal cache
+        filename = join(config.app_dir, 'replay.jsonl')
+        try:
+            try:
+                # Try to open existing file
+                self.replayfile = open(filename, 'r+')
+            except:
+                if exists(filename):
+                    raise	# Couldn't open existing file
+                else:
+                    self.replayfile = open(filename, 'w+')	# Create file
+            if sys.platform != 'win32':	# open for writing is automatically exclusive on Windows
+                lockf(self.replayfile, LOCK_EX|LOCK_NB)
+        except:
+            if __debug__: print_exc()
+            if self.replayfile:
+                self.replayfile.close()
+            self.replayfile = None
+            return False
+        self.replaylog = [line.strip() for line in self.replayfile]
+        return True
+
+    def flush(self):
+        self.replayfile.seek(0, SEEK_SET)
+        self.replayfile.truncate()
+        for line in self.replaylog:
+            self.replayfile.write('%s\n' % line)
+        self.replayfile.flush()
+
+    def close(self):
+        if self.replayfile:
+            self.replayfile.close()
+        self.replayfile = None
+
+    def send(self, cmdr, msg):
+        if config.getint('anonymous'):
+            uploaderID = config.get('uploaderID')
+            if not uploaderID:
+                uploaderID = uuid.uuid4().hex
+                config.set('uploaderID', uploaderID)
+        else:
+            uploaderID = cmdr.encode('utf-8')
+
+        msg = OrderedDict([
+            ('$schemaRef', msg['$schemaRef']),
+            ('header',     OrderedDict([
+                ('softwareName',    '%s [%s]' % (applongname, sys.platform=='darwin' and "Mac OS" or system())),
+                ('softwareVersion', appversion),
+                ('uploaderID',      uploaderID),
+            ])),
+            ('message',    msg['message']),
+        ])
+
+        r = self.session.post(self.UPLOAD, data=json.dumps(msg), timeout=self.TIMEOUT)
+        if __debug__ and r.status_code != requests.codes.ok:
+            print 'Status\t%s'  % r.status_code
+            print 'URL\t%s'  % r.url
+            print 'Headers\t%s' % r.headers
+            print ('Content:\n%s' % r.text).encode('utf-8')
+        r.raise_for_status()
+
+    def sendreplay(self):
+        if not self.replayfile:
+            return	# Probably closing app
+
+        status = self.parent.children['status']
+
+        if not self.replaylog:
+            status['text'] = ''
+            return
+
+        if len(self.replaylog) == 1:
+            status['text'] = _('Sending data to EDDN...')
+        else:
+            status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...',''), len(self.replaylog))
+        self.parent.update_idletasks()
+        try:
+            cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict)
+        except:
+            # Couldn't decode - shouldn't happen!
+            if __debug__:
+                print self.replaylog[0]
+                print_exc()
+            self.replaylog.pop(0)	# Discard and continue
+        else:
+            # Rewrite old schema name
+            if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'):
+                msg['$schemaRef'] = 'https://eddn.edcd.io/schemas/' + msg['$schemaRef'][38:]
+            try:
+                self.send(cmdr, msg)
+                self.replaylog.pop(0)
+                if not len(self.replaylog) % self.REPLAYFLUSH:
+                    self.flush()
+            except requests.exceptions.RequestException as e:
+                if __debug__: print_exc()
+                status['text'] = _("Error: Can't connect to EDDN")
+                return	# stop sending
+            except Exception as e:
+                if __debug__: print_exc()
+                status['text'] = unicode(e)
+                return	# stop sending
+
+        self.parent.after(self.REPLAYPERIOD, self.sendreplay)
+
+    def export_commodities(self, data, is_beta):
+        commodities = []
+        for commodity in data['lastStarport'].get('commodities') or []:
+            if (category_map.get(commodity['categoryname'], True) and	# Check marketable
+                not commodity.get('legality')):	# check not prohibited
+                commodities.append(OrderedDict([
+                    ('name',          commodity['name']),
+                    ('meanPrice',     int(commodity['meanPrice'])),
+                    ('buyPrice',      int(commodity['buyPrice'])),
+                    ('stock',         int(commodity['stock'])),
+                    ('stockBracket',  commodity['stockBracket']),
+                    ('sellPrice',     int(commodity['sellPrice'])),
+                    ('demand',        int(commodity['demand'])),
+                    ('demandBracket', commodity['demandBracket']),
+                ]))
+                if commodity['statusFlags']:
+                    commodities[-1]['statusFlags'] = commodity['statusFlags']
+
+        # Don't send empty commodities list - schema won't allow it
+        if commodities:
+            message = OrderedDict([
+                ('timestamp',   data['timestamp']),
+                ('systemName',  data['lastSystem']['name']),
+                ('stationName', data['lastStarport']['name']),
+                ('marketId',    data['lastStarport']['id']),
+                ('commodities', commodities),
+            ])
+            if 'economies' in data['lastStarport']:
+                message['economies']  = sorted([x for x in (data['lastStarport']['economies']  or {}).itervalues()])
+            if 'prohibited' in data['lastStarport']:
+                message['prohibited'] = sorted([x for x in (data['lastStarport']['prohibited'] or {}).itervalues()])
+            self.send(data['commander']['name'], {
+                '$schemaRef' : 'https://eddn.edcd.io/schemas/commodity/3' + (is_beta and '/test' or ''),
+                'message'    : message,
+            })
+
+    def export_outfitting(self, data, is_beta):
+        # Don't send empty modules list - schema won't allow it
+        if data['lastStarport'].get('modules'):
+            self.send(data['commander']['name'], {
+                '$schemaRef' : 'https://eddn.edcd.io/schemas/outfitting/2' + (is_beta and '/test' or ''),
+                'message'    : OrderedDict([
+                    ('timestamp',   data['timestamp']),
+                    ('systemName',  data['lastSystem']['name']),
+                    ('stationName', data['lastStarport']['name']),
+                    ('marketId',    data['lastStarport']['id']),
+                    ('modules',     sorted([module['name'] for module in data['lastStarport']['modules'].itervalues() if self.MODULE_RE.search(module['name']) and module.get('sku') in [None, 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'] and module['name'] != 'Int_PlanetApproachSuite'])),
+                ]),
+            })
+
+    def export_shipyard(self, data, is_beta):
+        # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard.
+        if data['lastStarport'].get('ships', {}).get('shipyard_list'):
+            self.send(data['commander']['name'], {
+                '$schemaRef' : 'https://eddn.edcd.io/schemas/shipyard/2' + (is_beta and '/test' or ''),
+                'message'    : OrderedDict([
+                    ('timestamp',   data['timestamp']),
+                    ('systemName',  data['lastSystem']['name']),
+                    ('stationName', data['lastStarport']['name']),
+                    ('marketId',    data['lastStarport']['id']),
+                    ('ships',       sorted([ship['name'] for ship in data['lastStarport']['ships']['shipyard_list'].values() + data['lastStarport']['ships']['unavailable_list']])),
+                ]),
+            })
+
+    def export_journal_entry(self, cmdr, is_beta, entry):
+        msg = {
+            '$schemaRef' : 'https://eddn.edcd.io/schemas/journal/1' + (is_beta and '/test' or ''),
+            'message'    : entry
+        }
+        if self.replayfile or self.load():
+            # Store the entry
+            self.replaylog.append(json.dumps([cmdr.encode('utf-8'), msg]))
+            self.replayfile.write('%s\n' % self.replaylog[-1])
+
+            if (entry['event'] == 'Docked' or
+                (entry['event'] == 'Location' and entry['Docked']) or
+                not (config.getint('output') & config.OUT_SYS_DELAY)):
+                self.parent.after(self.REPLAYPERIOD, self.sendreplay)	# Try to send this and previous entries
+        else:
+            # Can't access replay file! Send immediately.
+            status = self.parent.children['status']
+            status['text'] = _('Sending data to EDDN...')
+            self.parent.update_idletasks()
+            self.send(cmdr, msg)
+            status['text'] = ''
+
+
+# Plugin callbacks
+
+def plugin_start():
+    return 'EDDN'
+
+def plugin_app(parent):
+    this.parent = parent
+    this.eddn = EDDN(parent)
+    # Try to obtain exclusive lock on journal cache, even if we don't need it yet
+    if not this.eddn.load():
+        this.status['text'] = 'Error: Is another copy of this app already running?'	# Shouldn't happen - don't bother localizing
+
+def plugin_prefs(parent, cmdr, is_beta):
+
+    PADX = 10
+    BUTTONX = 12	# indent Checkbuttons and Radiobuttons
+    PADY = 2		# close spacing
+
+    output = config.getint('output') or (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN)	# default settings
+
+    eddnframe = nb.Frame(parent)
+
+    HyperlinkLabel(eddnframe, text='Elite Dangerous Data Network', background=nb.Label().cget('background'), url='https://github.com/EDSM-NET/EDDN/wiki', underline=True).grid(padx=PADX, sticky=tk.W)	# Don't translate
+    this.eddn_station= tk.IntVar(value = (output & config.OUT_MKT_EDDN) and 1)
+    this.eddn_station_button = nb.Checkbutton(eddnframe, text=_('Send station data to the Elite Dangerous Data Network'), variable=this.eddn_station, command=prefsvarchanged)	# Output setting
+    this.eddn_station_button.grid(padx=BUTTONX, pady=(5,0), sticky=tk.W)
+    this.eddn_system = tk.IntVar(value = (output & config.OUT_SYS_EDDN) and 1)
+    this.eddn_system_button = nb.Checkbutton(eddnframe, text=_('Send system and scan data to the Elite Dangerous Data Network'), variable=this.eddn_system, command=prefsvarchanged)	# Output setting new in E:D 2.2
+    this.eddn_system_button.grid(padx=BUTTONX, pady=(5,0), sticky=tk.W)
+    this.eddn_delay= tk.IntVar(value = (output & config.OUT_SYS_DELAY) and 1)
+    this.eddn_delay_button = nb.Checkbutton(eddnframe, text=_('Delay sending until docked'), variable=this.eddn_delay)	# Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2
+    this.eddn_delay_button.grid(padx=BUTTONX, sticky=tk.W)
+
+    return eddnframe
+
+def prefsvarchanged(event=None):
+    this.eddn_station_button['state'] = tk.NORMAL
+    this.eddn_system_button['state']= tk.NORMAL
+    this.eddn_delay_button['state'] = this.eddn.replayfile and this.eddn_system.get() and tk.NORMAL or tk.DISABLED
+
+def prefs_changed(cmdr, is_beta):
+    config.set('output',
+               (config.getint('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP)) +
+               (this.eddn_station.get() and config.OUT_MKT_EDDN) +
+               (this.eddn_system.get() and config.OUT_SYS_EDDN) +
+               (this.eddn_delay.get() and config.OUT_SYS_DELAY))
+
+def plugin_stop():
+    this.eddn.close()
+
+def journal_entry(cmdr, is_beta, system, station, entry, state):
+
+    # Recursively filter '*_Localised' keys from dict
+    def filter_localised(d):
+        filtered = OrderedDict()
+        for k, v in d.iteritems():
+            if k.endswith('_Localised'):
+                pass
+            elif hasattr(v, 'iteritems'):	# dict -> recurse
+                filtered[k] = filter_localised(v)
+            elif isinstance(v, list):	# list of dicts -> recurse
+                filtered[k] = [filter_localised(x) if hasattr(x, 'iteritems') else x for x in v]
+            else:
+                filtered[k] = v
+        return filtered
+
+    # Track location
+    if entry['event'] in ['Location', 'FSDJump', 'Docked']:
+        if entry['event'] == 'Location':
+            this.planet = entry.get('Body') if entry.get('BodyType') == 'Planet' else None
+        elif entry['event'] == 'FSDJump':
+            this.planet = None
+        if 'StarPos' in entry:
+            this.coordinates = tuple(entry['StarPos'])
+        elif this.systemaddress != entry.get('SystemAddress'):
+            this.coordinates = None	# Docked event doesn't include coordinates
+        this.systemaddress = entry.get('SystemAddress')
+    elif entry['event'] == 'ApproachBody':
+        this.planet = entry['Body']
+    elif entry['event'] in ['LeaveBody', 'SupercruiseEntry']:
+        this.planet = None
+
+    # Send interesting events to EDDN, but not when on a crew
+    if (config.getint('output') & config.OUT_SYS_EDDN and not state['Captain'] and
+        (entry['event'] == 'Location' or
+         entry['event'] == 'FSDJump' or
+         entry['event'] == 'Docked'  or
+         entry['event'] == 'Scan'    and this.coordinates)):
+        # strip out properties disallowed by the schema
+        for thing in ['ActiveFine', 'CockpitBreach', 'BoostUsed', 'FuelLevel', 'FuelUsed', 'JumpDist', 'Latitude', 'Longitude', 'Wanted']:
+            entry.pop(thing, None)
+        for faction in entry.get('Factions', []):
+            faction.pop('MyReputation', None)
+
+        # add planet to Docked event for planetary stations if known
+        if entry['event'] == 'Docked' and this.planet:
+            entry['Body'] = this.planet
+            entry['BodyType'] = 'Planet'
+
+        # add mandatory StarSystem, StarPos and SystemAddress properties to Scan events
+        if 'StarSystem' not in entry:
+            entry['StarSystem'] = system
+        if 'StarPos' not in entry:
+            entry['StarPos'] = list(this.coordinates)
+        if 'SystemAddress' not in entry and this.systemaddress:
+            entry['SystemAddress'] = this.systemaddress
+
+        try:
+            this.eddn.export_journal_entry(cmdr, is_beta, filter_localised(entry))
+        except requests.exceptions.RequestException as e:
+            if __debug__: print_exc()
+            return _("Error: Can't connect to EDDN")
+        except Exception as e:
+            if __debug__: print_exc()
+            return unicode(e)
+
+def cmdr_data(data, is_beta):
+    if data['commander'].get('docked') & config.OUT_MKT_EDDN:
+        try:
+            status = this.parent.children['status']
+            old_status = status['text']
+            if not old_status:
+                status['text'] = _('Sending data to EDDN...')
+                status.update_idletasks()
+            this.eddn.export_commodities(data, is_beta)
+            this.eddn.export_outfitting(data, is_beta)
+            this.eddn.export_shipyard(data, is_beta)
+            if not old_status:
+                status['text'] = ''
+                status.update_idletasks()
+
+        except requests.RequestException as e:
+            if __debug__: print_exc()
+            return _("Error: Can't connect to EDDN")
+
+        except Exception as e:
+            if __debug__: print_exc()
+            return unicode(e)

prefs.py

@@ -12,7 +12,6 @@ from ttkHyperlinkLabel import HyperlinkLabel
 import myNotebook as nb
 
 from config import applongname, config
-import eddn
 from hotkey import hotkeymgr
 from l10n import Translations
 from monitor import monitor
@@ -137,7 +136,7 @@ class PreferencesDialog(tk.Toplevel):
         outframe = nb.Frame(notebook)
         outframe.columnconfigure(0, weight=1)
 
-        output = config.getint('output') or (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN | config.OUT_SHIP)	# default settings
+        output = config.getint('output') or config.OUT_SHIP	# default settings
 
         self.out_label = nb.Label(outframe, text=_('Please choose what data to save'))
         self.out_label.grid(columnspan=2, padx=PADX, sticky=tk.W)
@@ -168,24 +167,6 @@ class PreferencesDialog(tk.Toplevel):
 
         notebook.add(outframe, text=_('Output'))		# Tab heading in settings
 
-
-        eddnframe = nb.Frame(notebook)
-
-        HyperlinkLabel(eddnframe, text='Elite Dangerous Data Network', background=nb.Label().cget('background'), url='https://github.com/EDSM-NET/EDDN/wiki', underline=True).grid(padx=PADX, sticky=tk.W)	# Don't translate
-        self.eddn_station= tk.IntVar(value = (output & config.OUT_MKT_EDDN) and 1)
-        self.eddn_station_button = nb.Checkbutton(eddnframe, text=_('Send station data to the Elite Dangerous Data Network'), variable=self.eddn_station, command=self.outvarchanged)	# Output setting
-        self.eddn_station_button.grid(padx=BUTTONX, pady=(5,0), sticky=tk.W)
-        self.eddn_auto_button = nb.Checkbutton(eddnframe, text=_('Automatically update on docking'), variable=self.out_auto, command=self.outvarchanged)	# Output setting
-        self.eddn_auto_button.grid(padx=BUTTONX, sticky=tk.W)
-        self.eddn_system = tk.IntVar(value = (output & config.OUT_SYS_EDDN) and 1)
-        self.eddn_system_button = nb.Checkbutton(eddnframe, text=_('Send system and scan data to the Elite Dangerous Data Network'), variable=self.eddn_system, command=self.outvarchanged)	# Output setting new in E:D 2.2
-        self.eddn_system_button.grid(padx=BUTTONX, pady=(5,0), sticky=tk.W)
-        self.eddn_delay= tk.IntVar(value = (output & config.OUT_SYS_DELAY) and 1)
-        self.eddn_delay_button = nb.Checkbutton(eddnframe, text=_('Delay sending until docked'), variable=self.eddn_delay)	# Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2
-        self.eddn_delay_button.grid(padx=BUTTONX, sticky=tk.W)
-
-        notebook.add(eddnframe, text='EDDN')		# Not translated
-
         # build plugin prefs tabs
         for plugin in plug.PLUGINS:
             plugframe = plugin.get_prefs(notebook, monitor.cmdr, monitor.is_beta)
@@ -413,11 +394,6 @@ class PreferencesDialog(tk.Toplevel):
         self.outbutton['state']         = local and tk.NORMAL  or tk.DISABLED
         self.outdir_entry['state']      = local and 'readonly' or tk.DISABLED
 
-        self.eddn_station_button['state'] = tk.NORMAL or tk.DISABLED
-        self.eddn_auto_button['state']  = self.eddn_station.get() and logvalid and tk.NORMAL or tk.DISABLED
-        self.eddn_system_button['state']= logvalid and tk.NORMAL or tk.DISABLED
-        self.eddn_delay_button['state'] = logvalid and eddn.replayfile and self.eddn_system.get() and tk.NORMAL or tk.DISABLED
-
     def filebrowse(self, title, pathvar):
         if platform != 'win32':
             import tkFileDialog
@@ -564,9 +540,7 @@ class PreferencesDialog(tk.Toplevel):
                    (self.out_csv.get()  and config.OUT_MKT_CSV) +
                    (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) +
                    (self.out_ship.get() and config.OUT_SHIP) +
-                   (self.eddn_station.get()      and config.OUT_MKT_EDDN) +
+                   (config.getint('output') & (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN | config.OUT_SYS_DELAY)))
-                   (self.eddn_system.get()       and config.OUT_SYS_EDDN) +
-                   (self.eddn_delay.get()        and config.OUT_SYS_DELAY))
         config.set('outdir', self.outdir.get().startswith('~') and join(config.home, self.outdir.get()[2:]) or self.outdir.get())
 
         logdir = self.logdir.get()

setup.py

@@ -66,7 +66,7 @@ if sys.platform=='darwin':
 APP = 'EDMarketConnector.py'
 APPCMD = 'EDMC.py'
 SHORTVERSION = ''.join(VERSION.split('.')[:3])
-PLUGINS = [ 'plugins/coriolis.py', 'plugins/eddb.py', 'plugins/edsm.py', 'plugins/edsy.py', 'plugins/inara.py' ]
+PLUGINS = [ 'plugins/coriolis.py', 'plugins/eddb.py', 'plugins/eddn.py', 'plugins/edsm.py', 'plugins/edsy.py', 'plugins/inara.py' ]
 
 if sys.platform=='darwin':
     OPTIONS =  { 'py2app':