mirror of
https://github.com/EDCD/EDDN.git
synced 2026-11-22 01:46:44 +03:00
Fix duplicate message not copied
This commit is contained in:
1 parent
dd65ba50d1
commit
32ed86da91
10 files changed
+63
-91
No files matched your search
+1
-1
@@ -144,7 +144,7 @@ def parse_and_error_handle(data):
|
||||
|
||||
if validationResults.severity <= ValidationSeverity.WARN:
|
||||
parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z'
|
||||
#parsed_message['header']['uploaderIP'] = get_remote_address()
|
||||
#parsed_message['header']['uploaderIP'] = get_remote_address()
|
||||
|
||||
# Sends the parsed to the Relay/Monitor as compressed JSON.
|
||||
gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef'])
|
||||
|
||||
+25
-21
@@ -11,6 +11,7 @@ import sqlite3
|
||||
import datetime
|
||||
import collections
|
||||
import zmq.green as zmq
|
||||
import re
|
||||
|
||||
from bottle import get, request, response, run as bottle_run
|
||||
from eddn.conf.Settings import Settings, loadConfig
|
||||
@@ -206,55 +207,58 @@ class Monitor(Thread):
|
||||
else:
|
||||
message = message[0]
|
||||
|
||||
message = zlib.decompress(message)
|
||||
json = simplejson.loads(message)
|
||||
|
||||
# Default variables
|
||||
schemaID = json['$schemaRef']
|
||||
softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8')
|
||||
|
||||
uploaderID = json['header']['uploaderID'].encode('utf8')
|
||||
uploaderIP = None
|
||||
if 'uploaderIP' in json['header']:
|
||||
uploaderIP = json['header']['uploaderIP'].encode('utf8')
|
||||
|
||||
# Duplicates?
|
||||
if Settings.RELAY_DUPLICATE_MAX_MINUTES:
|
||||
if duplicateMessages.isDuplicated(message):
|
||||
if duplicateMessages.isDuplicated(json):
|
||||
schemaID = 'DUPLICATE MESSAGE'
|
||||
|
||||
c = db.cursor()
|
||||
c.execute('UPDATE schemas SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (schemaID, ))
|
||||
c.execute('INSERT OR IGNORE INTO schemas (name, dateStats) VALUES (?, DATE("now", "utc"))', (schemaID, ))
|
||||
db.commit()
|
||||
|
||||
db.close()
|
||||
analytics.hit('DUPLICATE', uploaderID, uploaderIP)
|
||||
|
||||
return
|
||||
|
||||
if Settings.MONITOR_DECOMPRESS_MESSAGES:
|
||||
message = zlib.decompress(message)
|
||||
|
||||
json = simplejson.loads(message)
|
||||
|
||||
|
||||
# Update software count
|
||||
softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8')
|
||||
|
||||
c = db.cursor()
|
||||
c.execute('UPDATE softwares SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (softwareID, ))
|
||||
c.execute('INSERT OR IGNORE INTO softwares (name, dateStats) VALUES (?, DATE("now", "utc"))', (softwareID, ))
|
||||
db.commit()
|
||||
|
||||
|
||||
# Update uploader count
|
||||
uploaderID = json['header']['uploaderID'].encode('utf8')
|
||||
|
||||
if uploaderID: # Don't get empty uploaderID
|
||||
c = db.cursor()
|
||||
c.execute('UPDATE uploaders SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (uploaderID, ))
|
||||
c.execute('INSERT OR IGNORE INTO uploaders (name, dateStats) VALUES (?, DATE("now", "utc"))', (uploaderID, ))
|
||||
db.commit()
|
||||
|
||||
|
||||
# Update schemas count
|
||||
schemaID = json['$schemaRef']
|
||||
|
||||
c = db.cursor()
|
||||
c.execute('UPDATE schemas SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (schemaID, ))
|
||||
c.execute('INSERT OR IGNORE INTO schemas (name, dateStats) VALUES (?, DATE("now", "utc"))', (schemaID, ))
|
||||
db.commit()
|
||||
|
||||
|
||||
db.close()
|
||||
|
||||
uploaderIP = None
|
||||
if 'uploaderIP' in json['header']:
|
||||
uploaderIP = json['header']['uploaderIP'].encode('utf8')
|
||||
|
||||
analytics.hit(schemaID, uploaderID, uploaderIP)
|
||||
if re.search('test', schemaID, re.I):
|
||||
analytics.hit(Settings.GATEWAY_JSON_SCHEMAS[schemaID] + '#test', uploaderID, uploaderIP)
|
||||
else:
|
||||
analytics.hit(Settings.GATEWAY_JSON_SCHEMAS[schemaID], uploaderID, uploaderIP)
|
||||
|
||||
while True:
|
||||
inboundMessage = receiver.recv()
|
||||
|
||||
+9
-9
@@ -86,26 +86,26 @@ class Relay(Thread):
|
||||
message = message[0]
|
||||
|
||||
message = zlib.decompress(message)
|
||||
message = simplejson.loads(message)
|
||||
json = simplejson.loads(message)
|
||||
|
||||
# Handle duplicate message
|
||||
if Settings.RELAY_DUPLICATE_MAX_MINUTES:
|
||||
if duplicateMessages.isDuplicated(message):
|
||||
if duplicateMessages.isDuplicated(json):
|
||||
# We've already seen this message recently. Discard it.
|
||||
statsCollector.tally("duplicate")
|
||||
return
|
||||
|
||||
# Remove IP to end consumer
|
||||
#if 'uploaderIP' in message['header']:
|
||||
# del message['header']['uploaderIP']
|
||||
if 'uploaderIP' in json['header']:
|
||||
del json['header']['uploaderIP']
|
||||
|
||||
# Convert messgae back to JSON
|
||||
message = simplejson.dumps(message, sort_keys=True)
|
||||
# Convert message back to JSON
|
||||
message = simplejson.dumps(json, sort_keys=True)
|
||||
|
||||
# Recompress message when needed
|
||||
if not Settings.RELAY_DECOMPRESS_MESSAGES:
|
||||
message = zlib.compress(message)
|
||||
# Recompress message
|
||||
message = zlib.compress(message)
|
||||
|
||||
# Send message
|
||||
sender.send(message)
|
||||
statsCollector.tally("outbound")
|
||||
|
||||
|
||||
@@ -27,8 +27,6 @@ class _Settings(object):
|
||||
|
||||
RELAY_SENDER_BINDINGS = ["tcp://*:9500"]
|
||||
|
||||
RELAY_DECOMPRESS_MESSAGES = False
|
||||
|
||||
# If set to False, no deduplicate is made
|
||||
RELAY_DUPLICATE_MAX_MINUTES = 15
|
||||
|
||||
@@ -93,9 +91,7 @@ class _Settings(object):
|
||||
]
|
||||
|
||||
MONITOR_DB = "/home/EDDN_Monitor.s3db"
|
||||
MONITOR_DB = "D:/EDDN_Monitor.s3db" #DEBUG
|
||||
|
||||
MONITOR_DECOMPRESS_MESSAGES = True
|
||||
#MONITOR_DB = "D:/EDDN_Monitor.s3db" #DEBUG
|
||||
|
||||
MONITOR_UA = "UA-496332-23"
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
|
||||
|
||||
class TestSettings(unittest.TestCase):
|
||||
|
||||
def testLoadSettings(self):
|
||||
from eddn.conf.Settings import Settings
|
||||
self.assertEqual(Settings.RELAY_DECOMPRESS_MESSAGES, False)
|
||||
Settings.loadFrom(os.path.join(os.path.dirname(__file__), "testLoadSettings.json"))
|
||||
self.assertEqual(Settings.RELAY_DECOMPRESS_MESSAGES, True)
|
||||
Whitespace-only changes.
@@ -1,3 +0,0 @@
|
||||
{
|
||||
"RELAY_DECOMPRESS_MESSAGES" : true
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
import hashlib
|
||||
import re
|
||||
import simplejson
|
||||
import zlib
|
||||
import copy
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from eddn.conf.Settings import Settings
|
||||
@@ -31,37 +31,40 @@ class DuplicateMessages(Thread):
|
||||
if self.caches[key] + timedelta(minutes=self.max_minutes) < maxTime:
|
||||
del self.caches[key]
|
||||
|
||||
def isDuplicated(self, message):
|
||||
def isDuplicated(self, json):
|
||||
with self.lock:
|
||||
jsonTest = copy.deepcopy(json)
|
||||
|
||||
# Test messages are not duplicate
|
||||
if re.search('test', message['$schemaRef'], re.I):
|
||||
if re.search('test', jsonTest['$schemaRef'], re.I):
|
||||
return False
|
||||
|
||||
if 'gatewayTimestamp' in message['header']:
|
||||
del message['header']['gatewayTimestamp'] # Prevent dupe with new timestamp
|
||||
if 'timestamp' in message['message']:
|
||||
del message['message']['timestamp'] # Prevent dupe with new timestamp
|
||||
if 'softwareName' in message['header']:
|
||||
del message['header']['softwareName'] # Prevent dupe with different software
|
||||
if 'softwareVersion' in message['header']:
|
||||
del message['header']['softwareVersion'] # Prevent dupe with different software version
|
||||
if 'uploaderID' in message['header']:
|
||||
del message['header']['uploaderID'] # Prevent dupe with different uploaderID
|
||||
|
||||
# Remove headers
|
||||
if 'gatewayTimestamp' in jsonTest['header']:
|
||||
del jsonTest['header']['gatewayTimestamp'] # Prevent dupe with new timestamp
|
||||
if 'timestamp' in jsonTest['message']:
|
||||
del jsonTest['message']['timestamp'] # Prevent dupe with new timestamp
|
||||
if 'softwareName' in jsonTest['header']:
|
||||
del jsonTest['header']['softwareName'] # Prevent dupe with different software
|
||||
if 'softwareVersion' in jsonTest['header']:
|
||||
del jsonTest['header']['softwareVersion'] # Prevent dupe with different software version
|
||||
if 'uploaderID' in jsonTest['header']:
|
||||
del jsonTest['header']['uploaderID'] # Prevent dupe with different uploaderID
|
||||
|
||||
# Convert starPos to avoid software modification in dupe messages
|
||||
if 'StarPos' in message['message']:
|
||||
if message['message']['StarPos'][0]:
|
||||
message['message']['StarPos'][0] = round(message['message']['StarPos'][0] *32)
|
||||
if message['message']['StarPos'][1]:
|
||||
message['message']['StarPos'][1] = round(message['message']['StarPos'][1] *32)
|
||||
if message['message']['StarPos'][2]:
|
||||
message['message']['StarPos'][2] = round(message['message']['StarPos'][2] *32)
|
||||
if 'StarPos' in jsonTest['message']:
|
||||
if jsonTest['message']['StarPos'][0]:
|
||||
jsonTest['message']['StarPos'][0] = round(jsonTest['message']['StarPos'][0] *32)
|
||||
if jsonTest['message']['StarPos'][1]:
|
||||
jsonTest['message']['StarPos'][1] = round(jsonTest['message']['StarPos'][1] *32)
|
||||
if jsonTest['message']['StarPos'][2]:
|
||||
jsonTest['message']['StarPos'][2] = round(jsonTest['message']['StarPos'][2] *32)
|
||||
|
||||
# Prevent Docked event with small difference in distance from start
|
||||
if 'DistFromStarLS' in message['message']:
|
||||
message['message']['DistFromStarLS'] = round(message['message']['DistFromStarLS'])
|
||||
if 'DistFromStarLS' in jsonTest['message']:
|
||||
jsonTest['message']['DistFromStarLS'] = round(jsonTest['message']['DistFromStarLS'])
|
||||
|
||||
message = simplejson.dumps(message, sort_keys=True) # Ensure most duplicate messages will get the same key
|
||||
message = simplejson.dumps(jsonTest, sort_keys=True) # Ensure most duplicate messages will get the same key
|
||||
key = hashlib.sha256(message).hexdigest()
|
||||
|
||||
if key not in self.caches:
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from eddn.core.StatsCollector import StatsCollector
|
||||
from time import sleep
|
||||
|
||||
|
||||
class Test(unittest.TestCase):
|
||||
|
||||
def testStatsCollectorTally(self):
|
||||
print "This test will take about 60 seconds to run!"
|
||||
statsCollector = StatsCollector()
|
||||
statsCollector.start()
|
||||
|
||||
self.assertEqual(0, statsCollector.getCount("test", 1))
|
||||
statsCollector.tally("test")
|
||||
sleep(61)
|
||||
self.assertEqual(1, statsCollector.getCount("test", 1))
|
||||
Whitespace-only changes.
Reference in new issue
Block a user