Throttle EDDN output when sending replay

EDMarketConnector.py

@@ -43,7 +43,7 @@ import companion
 import commodity
 from commodity import COMMODITY_CSV
 import td
-from eddn import eddn
+import eddn
 import edsm
 import coriolis
 import eddb
@@ -76,6 +76,7 @@ class AppWindow:
         self.holdofftime = config.getint('querytime') + companion.holdoff
         self.session = companion.Session()
         self.edsm = edsm.EDSM()
+        self.eddn = eddn.EDDN(self)
 
         self.w = master
         self.w.title(applongname)
@@ -304,7 +305,7 @@ class AppWindow:
             self.status['text'] = 'Warning: Storing passwords as text'	# Shouldn't happen unless no secure storage on Linux
 
         # Try to obtain exclusive lock on journal cache, even if we don't need it yet
-        if not eddn.load():
+        if not self.eddn.load():
             self.status['text'] = 'Error: Is another copy of this app already running?'	# Shouldn't happen - don't bother localizing
 
     # callback after the Preferences dialog is applied
@@ -496,13 +497,13 @@ class AppWindow:
                             if not old_status:
                                 self.status['text'] = _('Sending data to EDDN...')
                             self.w.update_idletasks()
-                            eddn.export_commodities(data)
+                            self.eddn.export_commodities(data)
-                            eddn.export_outfitting(data)
+                            self.eddn.export_outfitting(data)
                             if has_shipyard and not data['lastStarport'].get('ships'):
                                 # API is flakey about shipyard info - silently retry if missing (<1s is usually sufficient - 5s for margin).
                                 self.w.after(int(SERVER_RETRY * 1000), self.retry_for_shipyard)
                             else:
-                                eddn.export_shipyard(data)
+                                self.eddn.export_shipyard(data)
                             if not old_status:
                                 self.status['text'] = ''
 
@@ -578,7 +579,7 @@ class AppWindow:
             if __debug__:
                 print 'Retry for shipyard - ' + (data['commander'].get('docked') and (data['lastStarport'].get('ships') and 'Success' or 'Failure') or 'Undocked!')
             if data['commander'].get('docked'):	# might have undocked while we were waiting for retry in which case station data is unreliable
-                eddn.export_shipyard(data)
+                self.eddn.export_shipyard(data)
         except:
             pass
 
@@ -704,10 +705,7 @@ class AppWindow:
                     if 'StarPos' not in entry:
                         entry['StarPos'] = list(monitor.coordinates)
 
-                    self.status['text'] = _('Sending data to EDDN...')
+                    self.eddn.export_journal_entry(monitor.cmdr, monitor.is_beta, entry)
-                    self.w.update_idletasks()
-                    eddn.export_journal_entry(monitor.cmdr, monitor.is_beta, entry)
-                    self.status['text'] = ''
 
                 elif (config.getint('output') & config.OUT_MKT_EDDN and monitor.cmdr and
                       entry['event'] == 'MarketSell' and entry.get('BlackMarket')):
@@ -723,7 +721,7 @@ class AppWindow:
 
                     self.status['text'] = _('Sending data to EDDN...')
                     self.w.update_idletasks()
-                    eddn.export_blackmarket(monitor.cmdr, monitor.is_beta, msg)
+                    self.eddn.export_blackmarket(monitor.cmdr, monitor.is_beta, msg)
                     self.status['text'] = ''
 
             except requests.exceptions.RequestException as e:
@@ -756,7 +754,7 @@ class AppWindow:
         try:
             data = self.session.query()
         except companion.VerificationRequired:
-            return prefs.AuthenticationDialog(self.parent, partial(self.verify, self.shipyard_url))
+            return prefs.AuthenticationDialog(self.w, partial(self.verify, self.shipyard_url))
         except companion.ServerError as e:
             self.status['text'] = str(e)
             return
@@ -850,7 +848,7 @@ class AppWindow:
         self.w.withdraw()	# Following items can take a few seconds, so hide the main window while they happen
         hotkeymgr.unregister()
         monitor.close()
-        eddn.close()
+        self.eddn.close()
         self.updater.close()
         self.session.close()
         config.close()

eddn.py

@@ -25,48 +25,56 @@ from companion import category_map
 timeout= 10	# requests timeout
 module_re = re.compile('^Hpt_|^Int_|_Armour_')
 
+replayfile = None	# For delayed messages
 
-class _EDDN:
+class EDDN:
 
     ### UPLOAD = 'http://localhost:8081/upload/'	# testing
     UPLOAD = 'http://eddn-gateway.elite-markets.net:8080/upload/'
+    REPLAYPERIOD = 400	# Roughly two messages per second, accounting for send delays [ms]
+    REPLAYFLUSH = 20	# Update log on disk roughly every 10 seconds
 
-    def __init__(self):
+    def __init__(self, parent):
+        self.parent = parent
         self.session = requests.Session()
-        self.replayfile = None	# For delayed messages
+        self.replaylog = []
 
     def load(self):
         # Try to obtain exclusive access to the journal cache
+        global replayfile
         filename = join(config.app_dir, 'replay.jsonl')
         try:
             try:
                 # Try to open existing file
-                self.replayfile = open(filename, 'r+')
+                replayfile = open(filename, 'r+')
             except:
                 if exists(filename):
                     raise	# Couldn't open existing file
                 else:
-                    self.replayfile = open(filename, 'w+')	# Create file
+                    replayfile = open(filename, 'w+')	# Create file
             if platform != 'win32':	# open for writing is automatically exclusive on Windows
-                lockf(self.replayfile, LOCK_EX|LOCK_NB)
+                lockf(replayfile, LOCK_EX|LOCK_NB)
         except:
             if __debug__: print_exc()
-            if self.replayfile:
+            if replayfile:
-                self.replayfile.close()
+                replayfile.close()
-            self.replayfile = None
+            replayfile = None
             return False
+        self.replaylog = [line.strip() for line in replayfile]
         return True
 
     def flush(self):
-        if self.replayfile or self.load():
+        replayfile.seek(0, SEEK_SET)
-            self.replayfile.seek(0, SEEK_SET)
+        replayfile.truncate()
-            for line in self.replayfile:
+        for line in self.replaylog:
-                self.send(*json.loads(line, object_pairs_hook=OrderedDict))
+            replayfile.write('%s\n' % line)
-            self.replayfile.truncate(0)
+        replayfile.flush()
 
     def close(self):
-        if self.replayfile:
+        global replayfile
-            self.replayfile.close()
+        if replayfile:
+            replayfile.close()
+        replayfile = None
 
     def send(self, cmdr, msg):
         msg['header'] = {
@@ -85,6 +93,38 @@ class _EDDN:
             print ('Content:\n%s' % r.text).encode('utf-8')
         r.raise_for_status()
 
+    def sendreplay(self):
+        if not replayfile:
+            return	# Probably closing app
+
+        if not self.replaylog:
+            self.parent.status['text'] = ''
+            return
+
+        if len(self.replaylog) == 1:
+            self.parent.status['text'] = _('Sending data to EDDN...')
+        else:
+            self.parent.status['text'] = '%s [%d]' % (_('Sending data to EDDN...').replace('...',''), len(self.replaylog))
+        self.parent.w.update_idletasks()
+        try:
+            self.send(*json.loads(self.replaylog[0], object_pairs_hook=OrderedDict))
+            self.replaylog.pop(0)
+            if not len(self.replaylog) % self.REPLAYFLUSH:
+                self.flush()
+        except EnvironmentError as e:
+            if __debug__: print_exc()
+            self.parent.status['text'] = unicode(e)
+        except requests.exceptions.RequestException as e:
+            if __debug__: print_exc()
+            self.parent.status['text'] = _("Error: Can't connect to EDDN")
+            return	# stop sending
+        except Exception as e:
+            if __debug__: print_exc()
+            self.parent.status['text'] = unicode(e)
+            return	# stop sending
+
+        self.parent.w.after(self.REPLAYPERIOD, self.sendreplay)
+
     def export_commodities(self, data):
         commodities = []
         for commodity in data['lastStarport'].get('commodities') or []:
@@ -142,25 +182,23 @@ class _EDDN:
             '$schemaRef' : 'http://schemas.elite-markets.net/eddn/journal/1' + (is_beta and '/test' or ''),
             'message'    : entry
         }
-        if self.replayfile or self.load():
+        if replayfile or self.load():
             # Store the entry
-            self.replayfile.seek(0, SEEK_END)
+            self.replaylog.append(json.dumps([cmdr.encode('utf-8'), msg]))
-            self.replayfile.write('%s\n' % json.dumps([cmdr.encode('utf-8'), msg]))
+            replayfile.write('%s\n' % self.replaylog[-1])
-            self.replayfile.flush()
 
             if entry['event'] == 'Docked' or not (config.getint('output') & config.OUT_SYS_DELAY):
                 # Try to send this and previous entries
-                self.flush()
+                self.sendreplay()
         else:
             # Can't access replay file! Send immediately.
+            self.parent.status['text'] = _('Sending data to EDDN...')
+            self.parent.w.update_idletasks()
             self.send(cmdr, msg)
+            self.parent.status['text'] = ''
 
     def export_blackmarket(self, cmdr, is_beta, msg):
         self.send(cmdr, {
             '$schemaRef' : 'http://schemas.elite-markets.net/eddn/blackmarket/1' + (is_beta and '/test' or ''),
             'message'    : msg
         })
-
-
-# singleton
-eddn = _EDDN()

prefs.py

@@ -11,7 +11,7 @@ from ttkHyperlinkLabel import HyperlinkLabel
 import myNotebook as nb
 
 from config import applongname, config
-from eddn import eddn
+import eddn
 from hotkey import hotkeymgr
 from l10n import Translations
 from monitor import monitor