Merge pull request #15 from jamesremuscat/dev

Dev to Master

.gitignore

@@ -52,5 +52,5 @@ docs/_build/
 
 # PyBuilder
 target/
-*.prices
+
 *.prices

contrib/monitor/js/eddn.js

@@ -357,7 +357,7 @@ var start       = function(){
     Highcharts.setOptions({global: {useUTC: false}});
     
     // Grab gateways
-    gateways = gateways.sort();
+    //gateways = gateways.sort();
     $.each(gateways, function(k, gateway){
         gateway = gateway.replace('tcp://', '');
         gateway = gateway.replace(':8500', '');
@@ -406,7 +406,7 @@ var start       = function(){
     }, updateInterval);
     
     // Grab relays
-    relays = relays.sort();
+    //relays = relays.sort();
     $.each(relays, function(k, relay){
         $("select[name=relays]").append($('<option>', { 
             value: 'http://' + relay + ':' + relayBottlePort + '/stats/',

examples/Python 2.7/Client_Complete.py

@@ -209,8 +209,10 @@ def main():
         except zmq.ZMQError, e:
             echoLog('')
             echoLog('ZMQSocketException: ' + str(e))
+            subscriber.disconnect(__relayEDDN)
+            echoLog('Disconnect from ' + __relayEDDN)
             echoLog('')
-            time.sleep(10)
+            time.sleep(5)
             
         
 

examples/Python 2.7/Client_Simple.py

@@ -2,6 +2,7 @@ import zlib
 import zmq
 import simplejson
 import sys
+import time
 
 """
  "  Configuration
@@ -42,7 +43,8 @@ def main():
         except zmq.ZMQError, e:
             print 'ZMQSocketException: ' + str(e)
             sys.stdout.flush()
-            time.sleep(10)
+            subscriber.disconnect(__relayEDDN)
+            time.sleep(5)
             
         
 

examples/Python 3.4/Client_Complete.py

@@ -209,8 +209,10 @@ def main():
         except zmq.ZMQError as e:
             echoLog('')
             echoLog('ZMQSocketException: ' + str(e))
+            subscriber.disconnect(__relayEDDN)
+            echoLog('Disconnect from ' + __relayEDDN)
             echoLog('')
-            time.sleep(10)
+            time.sleep(5)
             
         
 

examples/Python 3.4/Client_Simple.py

@@ -2,6 +2,7 @@ import zlib
 import zmq
 import simplejson
 import sys
+import time
 
 """
  "  Configuration
@@ -42,7 +43,8 @@ def main():
         except zmq.ZMQError as e:
             print ('ZMQSocketException: ' + str(e))
             sys.stdout.flush()
-            time.sleep(10)
+            subscriber.disconnect(__relayEDDN)
+            time.sleep(5)
             
         
 

setup.py

@@ -3,7 +3,7 @@ import re
 import glob
 
 
-VERSIONFILE = "src/eddn/_Conf/Version.py"
+VERSIONFILE = "src/eddn/conf/Version.py"
 verstr      = "unknown"
 try:
     verstrline = open(VERSIONFILE, "rt").read()

src/eddn/Gateway.py

@@ -1,3 +1,5 @@
+# coding: utf8
+
 """
 Contains the necessary ZeroMQ socket and a helper function to publish
 market data to the Announcer daemons.
@@ -11,10 +13,11 @@ import zlib
 import zmq.green as zmq
 from datetime import datetime
 
-import os
+from pkg_resources import resource_string
+# import os
 
-from eddn._Conf.Settings import Settings, loadConfig
+from eddn.conf.Settings import Settings, loadConfig
-from eddn._Core.Validator import Validator, ValidationSeverity
+from eddn.core.Validator import Validator, ValidationSeverity
 
 from gevent import monkey
 monkey.patch_all()
@@ -29,7 +32,7 @@ sender = context.socket(zmq.PUB)
 validator = Validator()
 
 # This import must be done post-monkey-patching!
-from eddn._Core.StatsCollector import StatsCollector
+from eddn.core.StatsCollector import StatsCollector
 statsCollector = StatsCollector()
 statsCollector.start()
 
@@ -42,7 +45,7 @@ def configure():
         sender.bind(binding)
 
     for schemaRef, schemaFile in Settings.GATEWAY_JSON_SCHEMAS.iteritems():
-        validator.addSchemaResource(schemaRef, os.path.dirname(__file__) + '/' + schemaFile)
+        validator.addSchemaResource(schemaRef, resource_string(__name__, schemaFile))
 
 
 def push_message(string_message, topic):
@@ -130,16 +133,12 @@ def parse_and_error_handle(data):
     validationResults = validator.validate(parsed_message)
 
     if validationResults.severity <= ValidationSeverity.WARN:
-
-        statsCollector.tally(parsed_message["$schemaRef"])
-        statsCollector.tally(parsed_message['header']['softwareName'] + " " + parsed_message['header']['softwareVersion'])
-
         parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat()
 
         ip_hash_salt = Settings.GATEWAY_IP_KEY_SALT
         if ip_hash_salt:
             # If an IP hash is set, salt+hash the uploader's IP address and set
-            # it as the EMDR upload key value.
+            # it as the EDDN upload key value.
             ip_hash = hashlib.sha1(ip_hash_salt + get_remote_address()).hexdigest()
             parsed_message['header']['uploaderKey'] = ip_hash
 

src/eddn/Monitor.py

@@ -1,3 +1,5 @@
+# coding: utf8
+
 """
 Monitor sit below gateways, or another relay, and simply parse what it receives over SUB.
 """
@@ -10,16 +12,18 @@ import datetime
 import collections
 import zmq.green as zmq
 from bottle import get, request, response, run as bottle_run
-from eddn._Conf.Settings import Settings, loadConfig
+from eddn.conf.Settings import Settings, loadConfig
 
 from gevent import monkey
 monkey.patch_all()
 
+# This import must be done post-monkey-patching!
 if Settings.RELAY_DUPLICATE_MAX_MINUTES:
-    from eddn._Core.DuplicateMessages import DuplicateMessages
+    from eddn.core.DuplicateMessages import DuplicateMessages
     duplicateMessages = DuplicateMessages()
     duplicateMessages.start()
 
+
 def date(__format):
     d = datetime.datetime.utcnow()
     return d.strftime(__format)
@@ -32,7 +36,7 @@ def getTotalSoftwares():
     softwares = collections.OrderedDict()
 
     maxDays = request.GET.get('maxDays', '31').strip()
-    maxDays     = int(maxDays) -1;
+    maxDays = int(maxDays) - 1
 
     query = """SELECT name, SUM(hits) AS total, MAX(dateStats) AS maxDate
                FROM softwares
@@ -42,12 +46,13 @@ def getTotalSoftwares():
     results = db.execute(query)
 
     for row in results:
-        softwares[str(row[0])] = str(row[1])
+        softwares[row[0].encode('utf8')] = str(row[1])
 
     db.close()
 
     return simplejson.dumps(softwares)
 
+
 @get('/getSoftwares/')
 def getSoftwares():
     response.set_header("Access-Control-Allow-Origin", "*")
@@ -64,15 +69,16 @@ def getSoftwares():
     results = db.execute(query, (dateStart, dateEnd))
 
     for row in results:
-        if not str(row[2]) in softwares.keys():
+        if not str(row[2].encode('utf8')) in softwares.keys():
-            softwares[str(row[2])] = collections.OrderedDict()
+            softwares[row[2].encode('utf8')] = collections.OrderedDict()
 
-        softwares[str(row[2])][str(row[0])] = str(row[1])
+        softwares[row[2].encode('utf8')][str(row[0])] = str(row[1])
 
     db.close()
 
     return simplejson.dumps(softwares)
 
+
 @get('/getTotalUploaders/')
 def getTotalUploaders():
     response.set_header("Access-Control-Allow-Origin", "*")
@@ -89,12 +95,13 @@ def getTotalUploaders():
     results = db.execute(query)
 
     for row in results:
-        uploaders[str(row[0])] = str(row[1])
+        uploaders[row[0].encode('utf8')] = row[1]
 
     db.close()
 
     return simplejson.dumps(uploaders)
 
+
 @get('/getUploaders/')
 def getUploaders():
     response.set_header("Access-Control-Allow-Origin", "*")
@@ -111,15 +118,16 @@ def getUploaders():
     results = db.execute(query, (dateStart, dateEnd))
 
     for row in results:
-        if not str(row[2]) in uploaders.keys():
+        if not row[2].encode('utf8') in uploaders.keys():
-            uploaders[str(row[2])] = collections.OrderedDict()
+            uploaders[row[2].encode('utf8')] = collections.OrderedDict()
 
-        uploaders[str(row[2])][str(row[0])] = str(row[1])
+        uploaders[row[2]][row[0].encode('utf8')] = row[1]
 
     db.close()
 
     return simplejson.dumps(uploaders)
 
+
 @get('/getTotalSchemas/')
 def getTotalSchemas():
     response.set_header("Access-Control-Allow-Origin", "*")
@@ -139,6 +147,7 @@ def getTotalSchemas():
 
     return simplejson.dumps(schemas)
 
+
 @get('/getSchemas/')
 def getSchemas():
     response.set_header("Access-Control-Allow-Origin", "*")
@@ -165,7 +174,6 @@ def getSchemas():
     return simplejson.dumps(schemas)
 
 
-
 class Monitor(Thread):
 
     def run(self):
@@ -206,7 +214,7 @@ class Monitor(Thread):
             json = simplejson.loads(message)
 
             # Update software count
-            softwareID = json['header']['softwareName'] + ' | ' + json['header']['softwareVersion']
+            softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8')
 
             c = db.cursor()
             c.execute('UPDATE softwares SET hits = hits + 1 WHERE `name` = ? AND `dateStats` = DATE("now", "utc")', (softwareID, ))
@@ -214,7 +222,7 @@ class Monitor(Thread):
             db.commit()
 
             # Update uploader count
-            uploaderID = json['header']['uploaderID']
+            uploaderID = json['header']['uploaderID'].encode('utf8')
 
             if uploaderID:  # Don't get empty uploaderID
                 c = db.cursor()

src/eddn/Relay.py

@@ -1,3 +1,5 @@
+# coding: utf8
+
 """
 Relays sit below an announcer, or another relay, and simply repeat what
 they receive over PUB/SUB.
@@ -13,18 +15,19 @@ import gevent
 import simplejson
 import zmq.green as zmq
 from bottle import get, response, run as bottle_run
-from eddn._Conf.Settings import Settings, loadConfig
+from eddn.conf.Settings import Settings, loadConfig
 
 from gevent import monkey
 monkey.patch_all()
 
-from eddn._Core.StatsCollector import StatsCollector
+# This import must be done post-monkey-patching!
-
+from eddn.core.StatsCollector import StatsCollector
 statsCollector = StatsCollector()
 statsCollector.start()
 
+# This import must be done post-monkey-patching!
 if Settings.RELAY_DUPLICATE_MAX_MINUTES:
-    from eddn._Core.DuplicateMessages import DuplicateMessages
+    from eddn.core.DuplicateMessages import DuplicateMessages
     duplicateMessages = DuplicateMessages()
     duplicateMessages.start()
 

src/eddn/conf/Settings.py

@@ -1,12 +1,8 @@
-'''
+# coding: utf8
-Created on 15 Nov 2014
-
-@author: james
-'''
 
 import argparse
 import simplejson
-from eddn._Conf.Version import __version__ as version
+from eddn.conf.Version import __version__ as version
 
 
 class _Settings(object):
@@ -17,7 +13,6 @@ class _Settings(object):
     # Relay settings
     ###############################################################################
 
-    #RELAY_RECEIVER_BINDINGS = ["tcp://localhost:8500"]
     RELAY_RECEIVER_BINDINGS = ["tcp://eddn-gateway.elite-markets.net:8500", "tcp://eddn-gateway.ed-td.space:8500"]
 
     RELAY_SENDER_BINDINGS = ["tcp://*:9500"]
@@ -60,7 +55,6 @@ class _Settings(object):
 
     MONITOR_DECOMPRESS_MESSAGES = True
 
-
     def loadFrom(self, fileName):
         f = open(fileName, 'r')
         conf = simplejson.load(f)

src/eddn/conf/Version.py

@@ -1 +1,3 @@
+# coding: utf8
+
 __version__ = "0.4"

src/eddn/core/DuplicateMessages.py

@@ -1,10 +1,14 @@
+# coding: utf8
+
 from datetime import datetime, timedelta
 from threading import Lock, Thread
 from time import sleep
 import hashlib
 import zlib
+import re
+
 import simplejson
-from eddn._Conf.Settings import Settings, loadConfig
+from eddn.conf.Settings import Settings, loadConfig
 
 
 class DuplicateMessages(Thread):
@@ -33,6 +37,10 @@ class DuplicateMessages(Thread):
             message = zlib.decompress(message)
             message = simplejson.loads(message)
 
+            # Test messages are not duplicate
+            if re.search('test', message['$schemaRef'], re.I):
+                return False
+
             if message['header']['gatewayTimestamp']:
                 del message['header']['gatewayTimestamp']  # Prevent dupe with new timestamp ^^
             if message['message']['timestamp']:

src/eddn/core/StatsCollector.py

@@ -1,3 +1,5 @@
+# coding: utf8
+
 from collections import deque
 from datetime import datetime
 from itertools import islice

src/eddn/core/Validator.py

@@ -1,3 +1,5 @@
+# coding: utf8
+
 import simplejson
 from enum import IntEnum
 from jsonschema import validate as jsValidate, ValidationError
@@ -7,10 +9,10 @@ class Validator(object):
 
     schemas = {"http://example.com": {}}
 
-    def addSchemaResource(self, schemaRef, schemaFile):
+    def addSchemaResource(self, schemaRef, schema):
         if schemaRef in self.schemas.keys():
             raise Exception("Attempted to redefine schema for " + schemaRef)
-        schema = simplejson.load(open(schemaFile, "r"))
+        schema = simplejson.loads(schema)
         self.schemas[schemaRef] = schema
 
     def validate(self, json_object):