Fix duplicate message not copied

This commit is contained in:
AnthorNet committed 2018-01-05 16:35:57 +01:00
1 parent dd65ba50d1
commit 32ed86da91
10 files changed
+63 -91

No files matched your search

+1 -1
View File
@@ -144,7 +144,7 @@ def parse_and_error_handle(data):
if validationResults.severity <= ValidationSeverity.WARN:
parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z'
#parsed_message['header']['uploaderIP'] = get_remote_address()
#parsed_message['header']['uploaderIP'] = get_remote_address()
# Sends the parsed to the Relay/Monitor as compressed JSON.
gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef'])
+25 -21
View File
@@ -11,6 +11,7 @@ import sqlite3
import datetime
import collections
import zmq.green as zmq
import re
from bottle import get, request, response, run as bottle_run
from eddn.conf.Settings import Settings, loadConfig
@@ -206,55 +207,58 @@ class Monitor(Thread):
else:
message = message[0]
message = zlib.decompress(message)
json = simplejson.loads(message)
# Default variables
schemaID = json['$schemaRef']
softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8')
uploaderID = json['header']['uploaderID'].encode('utf8')
uploaderIP = None
if 'uploaderIP' in json['header']:
uploaderIP = json['header']['uploaderIP'].encode('utf8')
# Duplicates?
if Settings.RELAY_DUPLICATE_MAX_MINUTES:
if duplicateMessages.isDuplicated(message):
if duplicateMessages.isDuplicated(json):
schemaID = 'DUPLICATE MESSAGE'
c = db.cursor()
c.execute('UPDATE schemas SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (schemaID, ))
c.execute('INSERT OR IGNORE INTO schemas (name, dateStats) VALUES (?, DATE("now", "utc"))', (schemaID, ))
db.commit()
db.close()
analytics.hit('DUPLICATE', uploaderID, uploaderIP)
return
if Settings.MONITOR_DECOMPRESS_MESSAGES:
message = zlib.decompress(message)
json = simplejson.loads(message)
# Update software count
softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8')
c = db.cursor()
c.execute('UPDATE softwares SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (softwareID, ))
c.execute('INSERT OR IGNORE INTO softwares (name, dateStats) VALUES (?, DATE("now", "utc"))', (softwareID, ))
db.commit()
# Update uploader count
uploaderID = json['header']['uploaderID'].encode('utf8')
if uploaderID: # Don't get empty uploaderID
c = db.cursor()
c.execute('UPDATE uploaders SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (uploaderID, ))
c.execute('INSERT OR IGNORE INTO uploaders (name, dateStats) VALUES (?, DATE("now", "utc"))', (uploaderID, ))
db.commit()
# Update schemas count
schemaID = json['$schemaRef']
c = db.cursor()
c.execute('UPDATE schemas SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (schemaID, ))
c.execute('INSERT OR IGNORE INTO schemas (name, dateStats) VALUES (?, DATE("now", "utc"))', (schemaID, ))
db.commit()
db.close()
uploaderIP = None
if 'uploaderIP' in json['header']:
uploaderIP = json['header']['uploaderIP'].encode('utf8')
analytics.hit(schemaID, uploaderID, uploaderIP)
if re.search('test', schemaID, re.I):
analytics.hit(Settings.GATEWAY_JSON_SCHEMAS[schemaID] + '#test', uploaderID, uploaderIP)
else:
analytics.hit(Settings.GATEWAY_JSON_SCHEMAS[schemaID], uploaderID, uploaderIP)
while True:
inboundMessage = receiver.recv()
+9 -9
View File
@@ -86,26 +86,26 @@ class Relay(Thread):
message = message[0]
message = zlib.decompress(message)
message = simplejson.loads(message)
json = simplejson.loads(message)
# Handle duplicate message
if Settings.RELAY_DUPLICATE_MAX_MINUTES:
if duplicateMessages.isDuplicated(message):
if duplicateMessages.isDuplicated(json):
# We've already seen this message recently. Discard it.
statsCollector.tally("duplicate")
return
# Remove IP to end consumer
#if 'uploaderIP' in message['header']:
# del message['header']['uploaderIP']
if 'uploaderIP' in json['header']:
del json['header']['uploaderIP']
# Convert messgae back to JSON
message = simplejson.dumps(message, sort_keys=True)
# Convert message back to JSON
message = simplejson.dumps(json, sort_keys=True)
# Recompress message when needed
if not Settings.RELAY_DECOMPRESS_MESSAGES:
message = zlib.compress(message)
# Recompress message
message = zlib.compress(message)
# Send message
sender.send(message)
statsCollector.tally("outbound")
+1 -5
View File
@@ -27,8 +27,6 @@ class _Settings(object):
RELAY_SENDER_BINDINGS = ["tcp://*:9500"]
RELAY_DECOMPRESS_MESSAGES = False
# If set to False, no deduplicate is made
RELAY_DUPLICATE_MAX_MINUTES = 15
@@ -93,9 +91,7 @@ class _Settings(object):
]
MONITOR_DB = "/home/EDDN_Monitor.s3db"
MONITOR_DB = "D:/EDDN_Monitor.s3db" #DEBUG
MONITOR_DECOMPRESS_MESSAGES = True
#MONITOR_DB = "D:/EDDN_Monitor.s3db" #DEBUG
MONITOR_UA = "UA-496332-23"
-11
View File
@@ -1,11 +0,0 @@
import os
import unittest
class TestSettings(unittest.TestCase):
def testLoadSettings(self):
from eddn.conf.Settings import Settings
self.assertEqual(Settings.RELAY_DECOMPRESS_MESSAGES, False)
Settings.loadFrom(os.path.join(os.path.dirname(__file__), "testLoadSettings.json"))
self.assertEqual(Settings.RELAY_DECOMPRESS_MESSAGES, True)
View File
Whitespace-only changes.
@@ -1,3 +0,0 @@
{
"RELAY_DECOMPRESS_MESSAGES" : true
}
+27 -24
View File
@@ -2,7 +2,7 @@
import hashlib
import re
import simplejson
import zlib
import copy
from datetime import datetime, timedelta
from eddn.conf.Settings import Settings
@@ -31,37 +31,40 @@ class DuplicateMessages(Thread):
if self.caches[key] + timedelta(minutes=self.max_minutes) < maxTime:
del self.caches[key]
def isDuplicated(self, message):
def isDuplicated(self, json):
with self.lock:
jsonTest = copy.deepcopy(json)
# Test messages are not duplicate
if re.search('test', message['$schemaRef'], re.I):
if re.search('test', jsonTest['$schemaRef'], re.I):
return False
if 'gatewayTimestamp' in message['header']:
del message['header']['gatewayTimestamp'] # Prevent dupe with new timestamp
if 'timestamp' in message['message']:
del message['message']['timestamp'] # Prevent dupe with new timestamp
if 'softwareName' in message['header']:
del message['header']['softwareName'] # Prevent dupe with different software
if 'softwareVersion' in message['header']:
del message['header']['softwareVersion'] # Prevent dupe with different software version
if 'uploaderID' in message['header']:
del message['header']['uploaderID'] # Prevent dupe with different uploaderID
# Remove headers
if 'gatewayTimestamp' in jsonTest['header']:
del jsonTest['header']['gatewayTimestamp'] # Prevent dupe with new timestamp
if 'timestamp' in jsonTest['message']:
del jsonTest['message']['timestamp'] # Prevent dupe with new timestamp
if 'softwareName' in jsonTest['header']:
del jsonTest['header']['softwareName'] # Prevent dupe with different software
if 'softwareVersion' in jsonTest['header']:
del jsonTest['header']['softwareVersion'] # Prevent dupe with different software version
if 'uploaderID' in jsonTest['header']:
del jsonTest['header']['uploaderID'] # Prevent dupe with different uploaderID
# Convert starPos to avoid software modification in dupe messages
if 'StarPos' in message['message']:
if message['message']['StarPos'][0]:
message['message']['StarPos'][0] = round(message['message']['StarPos'][0] *32)
if message['message']['StarPos'][1]:
message['message']['StarPos'][1] = round(message['message']['StarPos'][1] *32)
if message['message']['StarPos'][2]:
message['message']['StarPos'][2] = round(message['message']['StarPos'][2] *32)
if 'StarPos' in jsonTest['message']:
if jsonTest['message']['StarPos'][0]:
jsonTest['message']['StarPos'][0] = round(jsonTest['message']['StarPos'][0] *32)
if jsonTest['message']['StarPos'][1]:
jsonTest['message']['StarPos'][1] = round(jsonTest['message']['StarPos'][1] *32)
if jsonTest['message']['StarPos'][2]:
jsonTest['message']['StarPos'][2] = round(jsonTest['message']['StarPos'][2] *32)
# Prevent Docked event with small difference in distance from start
if 'DistFromStarLS' in message['message']:
message['message']['DistFromStarLS'] = round(message['message']['DistFromStarLS'])
if 'DistFromStarLS' in jsonTest['message']:
jsonTest['message']['DistFromStarLS'] = round(jsonTest['message']['DistFromStarLS'])
message = simplejson.dumps(message, sort_keys=True) # Ensure most duplicate messages will get the same key
message = simplejson.dumps(jsonTest, sort_keys=True) # Ensure most duplicate messages will get the same key
key = hashlib.sha256(message).hexdigest()
if key not in self.caches:
-17
View File
@@ -1,17 +0,0 @@
import unittest
from eddn.core.StatsCollector import StatsCollector
from time import sleep
class Test(unittest.TestCase):
def testStatsCollectorTally(self):
print "This test will take about 60 seconds to run!"
statsCollector = StatsCollector()
statsCollector.start()
self.assertEqual(0, statsCollector.getCount("test", 1))
statsCollector.tally("test")
sleep(61)
self.assertEqual(1, statsCollector.getCount("test", 1))
View File
Whitespace-only changes.