From 32ed86da91eb5f8c3a91ee12d0e2f075b9ab0295 Mon Sep 17 00:00:00 2001 From: AnthorNet Date: Fri, 5 Jan 2018 16:35:57 +0100 Subject: [PATCH] Fix duplicate message not copied --- src/eddn/Gateway.py | 2 +- src/eddn/Monitor.py | 46 ++++++++++---------- src/eddn/Relay.py | 18 ++++---- src/eddn/conf/Settings.py | 6 +-- src/eddn/conf/tests/TestSettings.py | 11 ----- src/eddn/conf/tests/__init__.py | 0 src/eddn/conf/tests/testLoadSettings.json | 3 -- src/eddn/core/DuplicateMessages.py | 51 ++++++++++++----------- src/eddn/core/tests/TestStatsCollector.py | 17 -------- src/eddn/core/tests/__init__.py | 0 10 files changed, 63 insertions(+), 91 deletions(-) delete mode 100644 src/eddn/conf/tests/TestSettings.py delete mode 100644 src/eddn/conf/tests/__init__.py delete mode 100644 src/eddn/conf/tests/testLoadSettings.json delete mode 100644 src/eddn/core/tests/TestStatsCollector.py delete mode 100644 src/eddn/core/tests/__init__.py diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 8cc698a..7e7739b 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -144,7 +144,7 @@ def parse_and_error_handle(data): if validationResults.severity <= ValidationSeverity.WARN: parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z' - #parsed_message['header']['uploaderIP'] = get_remote_address() + #parsed_message['header']['uploaderIP'] = get_remote_address() # Sends the parsed to the Relay/Monitor as compressed JSON. gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 4a0bd0a..8848175 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -11,6 +11,7 @@ import sqlite3 import datetime import collections import zmq.green as zmq +import re from bottle import get, request, response, run as bottle_run from eddn.conf.Settings import Settings, loadConfig @@ -206,55 +207,58 @@ class Monitor(Thread): else: message = message[0] + message = zlib.decompress(message) + json = simplejson.loads(message) + + # Default variables + schemaID = json['$schemaRef'] + softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8') + + uploaderID = json['header']['uploaderID'].encode('utf8') + uploaderIP = None + if 'uploaderIP' in json['header']: + uploaderIP = json['header']['uploaderIP'].encode('utf8') + + # Duplicates? if Settings.RELAY_DUPLICATE_MAX_MINUTES: - if duplicateMessages.isDuplicated(message): + if duplicateMessages.isDuplicated(json): schemaID = 'DUPLICATE MESSAGE' c = db.cursor() c.execute('UPDATE schemas SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (schemaID, )) c.execute('INSERT OR IGNORE INTO schemas (name, dateStats) VALUES (?, DATE("now", "utc"))', (schemaID, )) db.commit() + db.close() + analytics.hit('DUPLICATE', uploaderID, uploaderIP) return - - if Settings.MONITOR_DECOMPRESS_MESSAGES: - message = zlib.decompress(message) - - json = simplejson.loads(message) - + # Update software count - softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8') - c = db.cursor() c.execute('UPDATE softwares SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (softwareID, )) c.execute('INSERT OR IGNORE INTO softwares (name, dateStats) VALUES (?, DATE("now", "utc"))', (softwareID, )) db.commit() - + # Update uploader count - uploaderID = json['header']['uploaderID'].encode('utf8') - if uploaderID: # Don't get empty uploaderID c = db.cursor() c.execute('UPDATE uploaders SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (uploaderID, )) c.execute('INSERT OR IGNORE INTO uploaders (name, dateStats) VALUES (?, DATE("now", "utc"))', (uploaderID, )) db.commit() - + # Update schemas count - schemaID = json['$schemaRef'] - c = db.cursor() c.execute('UPDATE schemas SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (schemaID, )) c.execute('INSERT OR IGNORE INTO schemas (name, dateStats) VALUES (?, DATE("now", "utc"))', (schemaID, )) db.commit() - + db.close() - uploaderIP = None - if 'uploaderIP' in json['header']: - uploaderIP = json['header']['uploaderIP'].encode('utf8') - - analytics.hit(schemaID, uploaderID, uploaderIP) + if re.search('test', schemaID, re.I): + analytics.hit(Settings.GATEWAY_JSON_SCHEMAS[schemaID] + '#test', uploaderID, uploaderIP) + else: + analytics.hit(Settings.GATEWAY_JSON_SCHEMAS[schemaID], uploaderID, uploaderIP) while True: inboundMessage = receiver.recv() diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index f240e27..6370e02 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -86,26 +86,26 @@ class Relay(Thread): message = message[0] message = zlib.decompress(message) - message = simplejson.loads(message) + json = simplejson.loads(message) # Handle duplicate message if Settings.RELAY_DUPLICATE_MAX_MINUTES: - if duplicateMessages.isDuplicated(message): + if duplicateMessages.isDuplicated(json): # We've already seen this message recently. Discard it. statsCollector.tally("duplicate") return # Remove IP to end consumer - #if 'uploaderIP' in message['header']: - # del message['header']['uploaderIP'] + if 'uploaderIP' in json['header']: + del json['header']['uploaderIP'] - # Convert messgae back to JSON - message = simplejson.dumps(message, sort_keys=True) + # Convert message back to JSON + message = simplejson.dumps(json, sort_keys=True) - # Recompress message when needed - if not Settings.RELAY_DECOMPRESS_MESSAGES: - message = zlib.compress(message) + # Recompress message + message = zlib.compress(message) + # Send message sender.send(message) statsCollector.tally("outbound") diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index e905983..8b3f567 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -27,8 +27,6 @@ class _Settings(object): RELAY_SENDER_BINDINGS = ["tcp://*:9500"] - RELAY_DECOMPRESS_MESSAGES = False - # If set to False, no deduplicate is made RELAY_DUPLICATE_MAX_MINUTES = 15 @@ -93,9 +91,7 @@ class _Settings(object): ] MONITOR_DB = "/home/EDDN_Monitor.s3db" - MONITOR_DB = "D:/EDDN_Monitor.s3db" #DEBUG - - MONITOR_DECOMPRESS_MESSAGES = True + #MONITOR_DB = "D:/EDDN_Monitor.s3db" #DEBUG MONITOR_UA = "UA-496332-23" diff --git a/src/eddn/conf/tests/TestSettings.py b/src/eddn/conf/tests/TestSettings.py deleted file mode 100644 index e626b40..0000000 --- a/src/eddn/conf/tests/TestSettings.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import unittest - - -class TestSettings(unittest.TestCase): - - def testLoadSettings(self): - from eddn.conf.Settings import Settings - self.assertEqual(Settings.RELAY_DECOMPRESS_MESSAGES, False) - Settings.loadFrom(os.path.join(os.path.dirname(__file__), "testLoadSettings.json")) - self.assertEqual(Settings.RELAY_DECOMPRESS_MESSAGES, True) diff --git a/src/eddn/conf/tests/__init__.py b/src/eddn/conf/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/eddn/conf/tests/testLoadSettings.json b/src/eddn/conf/tests/testLoadSettings.json deleted file mode 100644 index 647a9ce..0000000 --- a/src/eddn/conf/tests/testLoadSettings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ -"RELAY_DECOMPRESS_MESSAGES" : true -} \ No newline at end of file diff --git a/src/eddn/core/DuplicateMessages.py b/src/eddn/core/DuplicateMessages.py index dece230..d19e91d 100644 --- a/src/eddn/core/DuplicateMessages.py +++ b/src/eddn/core/DuplicateMessages.py @@ -2,7 +2,7 @@ import hashlib import re import simplejson -import zlib +import copy from datetime import datetime, timedelta from eddn.conf.Settings import Settings @@ -31,37 +31,40 @@ class DuplicateMessages(Thread): if self.caches[key] + timedelta(minutes=self.max_minutes) < maxTime: del self.caches[key] - def isDuplicated(self, message): + def isDuplicated(self, json): with self.lock: + jsonTest = copy.deepcopy(json) + # Test messages are not duplicate - if re.search('test', message['$schemaRef'], re.I): + if re.search('test', jsonTest['$schemaRef'], re.I): return False - - if 'gatewayTimestamp' in message['header']: - del message['header']['gatewayTimestamp'] # Prevent dupe with new timestamp - if 'timestamp' in message['message']: - del message['message']['timestamp'] # Prevent dupe with new timestamp - if 'softwareName' in message['header']: - del message['header']['softwareName'] # Prevent dupe with different software - if 'softwareVersion' in message['header']: - del message['header']['softwareVersion'] # Prevent dupe with different software version - if 'uploaderID' in message['header']: - del message['header']['uploaderID'] # Prevent dupe with different uploaderID + + # Remove headers + if 'gatewayTimestamp' in jsonTest['header']: + del jsonTest['header']['gatewayTimestamp'] # Prevent dupe with new timestamp + if 'timestamp' in jsonTest['message']: + del jsonTest['message']['timestamp'] # Prevent dupe with new timestamp + if 'softwareName' in jsonTest['header']: + del jsonTest['header']['softwareName'] # Prevent dupe with different software + if 'softwareVersion' in jsonTest['header']: + del jsonTest['header']['softwareVersion'] # Prevent dupe with different software version + if 'uploaderID' in jsonTest['header']: + del jsonTest['header']['uploaderID'] # Prevent dupe with different uploaderID # Convert starPos to avoid software modification in dupe messages - if 'StarPos' in message['message']: - if message['message']['StarPos'][0]: - message['message']['StarPos'][0] = round(message['message']['StarPos'][0] *32) - if message['message']['StarPos'][1]: - message['message']['StarPos'][1] = round(message['message']['StarPos'][1] *32) - if message['message']['StarPos'][2]: - message['message']['StarPos'][2] = round(message['message']['StarPos'][2] *32) + if 'StarPos' in jsonTest['message']: + if jsonTest['message']['StarPos'][0]: + jsonTest['message']['StarPos'][0] = round(jsonTest['message']['StarPos'][0] *32) + if jsonTest['message']['StarPos'][1]: + jsonTest['message']['StarPos'][1] = round(jsonTest['message']['StarPos'][1] *32) + if jsonTest['message']['StarPos'][2]: + jsonTest['message']['StarPos'][2] = round(jsonTest['message']['StarPos'][2] *32) # Prevent Docked event with small difference in distance from start - if 'DistFromStarLS' in message['message']: - message['message']['DistFromStarLS'] = round(message['message']['DistFromStarLS']) + if 'DistFromStarLS' in jsonTest['message']: + jsonTest['message']['DistFromStarLS'] = round(jsonTest['message']['DistFromStarLS']) - message = simplejson.dumps(message, sort_keys=True) # Ensure most duplicate messages will get the same key + message = simplejson.dumps(jsonTest, sort_keys=True) # Ensure most duplicate messages will get the same key key = hashlib.sha256(message).hexdigest() if key not in self.caches: diff --git a/src/eddn/core/tests/TestStatsCollector.py b/src/eddn/core/tests/TestStatsCollector.py deleted file mode 100644 index d4330b1..0000000 --- a/src/eddn/core/tests/TestStatsCollector.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest - -from eddn.core.StatsCollector import StatsCollector -from time import sleep - - -class Test(unittest.TestCase): - - def testStatsCollectorTally(self): - print "This test will take about 60 seconds to run!" - statsCollector = StatsCollector() - statsCollector.start() - - self.assertEqual(0, statsCollector.getCount("test", 1)) - statsCollector.tally("test") - sleep(61) - self.assertEqual(1, statsCollector.getCount("test", 1)) diff --git a/src/eddn/core/tests/__init__.py b/src/eddn/core/tests/__init__.py deleted file mode 100644 index e69de29..0000000