Merge branch 'develop'

setup.py

@@ -163,16 +163,6 @@ shutil.copy(
     '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV )
 )
 
-# Ensure the service log file archiving script is in place
-print """
-******************************************************************************
-Ensuring the service log file archiving script is in place
-"""
-shutil.copy(
-    'contrib/eddn-logs-archive',
-    START_SCRIPT_BIN
-)
-
 # Ensure the latest monitor files are in place
 old_umask = os.umask(022)
 print """

src/eddn/Bouncer.py

@@ -26,6 +26,7 @@ Architecture:
   4. `push_message()` then sends the message to the configured live/real
     Gateway.
 """
+import argparse
 import gevent
 import hashlib
 import logging
@@ -47,8 +48,15 @@ from bottle import Bottle, run, request, response, get, post
 app = Bottle()
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-logger.addHandler(logging.StreamHandler())
+logger.setLevel(logging.INFO)
+__logger_channel = logging.StreamHandler()
+__logger_formatter = logging.Formatter(
+    '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s'
+    )
+__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S'
+__logger_formatter.default_msec_format = '%s.%03d'
+__logger_channel.setFormatter(__logger_formatter)
+logger.addHandler(__logger_channel)
 logger.info('Made logger')
 
 
@@ -57,6 +65,27 @@ from eddn.core.StatsCollector import StatsCollector
 statsCollector = StatsCollector()
 statsCollector.start()
 
+
+def parse_cl_args():
+    parser = argparse.ArgumentParser(
+        prog='Gateway',
+        description='EDDN Gateway server',
+    )
+
+    parser.add_argument(
+        '--loglevel',
+        help='Logging level to output at',
+    )
+
+    parser.add_argument(
+        '-c', '--config',
+        metavar='config filename',
+        nargs='?',
+        default=None,
+    )
+
+    return parser.parse_args()
+
 def push_message(message_body):
     """
     Spawned as a greenlet to push messages (strings) through ZeroMQ.
@@ -243,8 +272,12 @@ class CustomLogging(object):
         return _log_to_logger
 
 def main():
+    cl_args = parse_cl_args()
+    if cl_args.loglevel:
+        logger.setLevel(cl_args.loglevel)
+
     logger.info('Loading config...')
-    loadConfig()
+    loadConfig(cl_args)
 
     logger.info('Installing EnableCors ...')
     app.install(EnableCors())

src/eddn/Gateway.py

@@ -4,6 +4,7 @@
 Contains the necessary ZeroMQ socket and a helper function to publish
 market data to the Announcer daemons.
 """
+import argparse
 import gevent
 import hashlib
 import logging
@@ -51,6 +52,27 @@ statsCollector = StatsCollector()
 statsCollector.start()
 
 
+def parse_cl_args():
+    parser = argparse.ArgumentParser(
+        prog='Gateway',
+        description='EDDN Gateway server',
+    )
+
+    parser.add_argument(
+        '--loglevel',
+        help='Logging level to output at',
+    )
+
+    parser.add_argument(
+        '-c', '--config',
+        metavar='config filename',
+        nargs='?',
+        default=None,
+    )
+
+    return parser.parse_args()
+
+
 def extract_message_details(parsed_message):
     uploader_id = '<<UNKNOWN>>'
     software_name = '<<UNKNOWN>>'
@@ -138,6 +160,7 @@ def get_decompressed_message():
             # Auto header checking.
             logger.debug('Trying zlib.decompress (15 + 32)...')
             message_body = zlib.decompress(request.body.read(), 15 + 32)
+
         except zlib.error:
             logger.error('zlib.error, trying zlib.decompress (-15)')
             # Negative wbits suppresses adler32 checksumming.
@@ -149,12 +172,14 @@ def get_decompressed_message():
         # body. If it's not form-encoded, this will return an empty dict.
         form_enc_parsed = urlparse.parse_qs(message_body)
         if form_enc_parsed:
-            logger.debug('Request is form-encoded')
+            logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address()))
             # This is a form-encoded POST. The value of the data attrib will
             # be the body we're looking for.
             try:
                 message_body = form_enc_parsed['data'][0]
+
             except (KeyError, IndexError):
+                logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address())
                 raise MalformedUploadError(
                     "No 'data' POST key/value found. Check your POST key "
                     "name for spelling, and make sure you're passing a value."
@@ -170,7 +195,7 @@ def get_decompressed_message():
         # POST key/vals, or un-encoded body.
         data_key = request.forms.get('data')
         if data_key:
-            logger.debug('form-encoded POST request detected...')
+            logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address()))
             # This is a form-encoded POST. Support the silly people.
             message_body = data_key
 
@@ -342,7 +367,12 @@ class EnableCors(object):
 
 
 def main():
-    loadConfig()
+
+    cl_args = parse_cl_args()
+    if cl_args.loglevel:
+        logger.setLevel(cl_args.loglevel)
+
+    loadConfig(cl_args)
     configure()
 
     app.install(EnableCors())

src/eddn/Monitor.py

@@ -4,6 +4,7 @@
 Monitor sit below gateways, or another relay, and simply parse what it receives over SUB.
 """
 from threading import Thread
+import argparse
 import zlib
 import gevent
 import simplejson
@@ -27,6 +28,26 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES:
     duplicateMessages.start()
 
 
+def parse_cl_args():
+    parser = argparse.ArgumentParser(
+        prog='Gateway',
+        description='EDDN Gateway server',
+    )
+
+    parser.add_argument(
+        '--loglevel',
+        help='CURRENTLY NO EFFECT - Logging level to output at',
+    )
+
+    parser.add_argument(
+        '-c', '--config',
+        metavar='config filename',
+        nargs='?',
+        default=None,
+    )
+
+    return parser.parse_args()
+
 def date(__format):
     d = datetime.datetime.utcnow()
     return d.strftime(__format)
@@ -237,7 +258,9 @@ class EnableCors(object):
         return _enable_cors
 
 def main():
-    loadConfig()
+    cl_args = parse_cl_args()
+    loadConfig(cl_args)
+
     m = Monitor()
     m.start()
     app.install(EnableCors())

src/eddn/Relay.py

@@ -4,20 +4,32 @@
 Relays sit below an announcer, or another relay, and simply repeat what
 they receive over PUB/SUB.
 """
-# Logging has to be configured first before we do anything.
+import argparse
+import gevent
+import hashlib
 import logging
+import simplejson
+import time
+import uuid
+import zlib
 from threading import Thread
 
-import time
-
-logger = logging.getLogger(__name__)
-import zlib
-
-import gevent
-import simplejson
-import hashlib
-import uuid
 import zmq.green as zmq
+
+
+# Logging has to be configured first before we do anything.
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+__logger_channel = logging.StreamHandler()
+__logger_formatter = logging.Formatter(
+    '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s'
+    )
+__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S'
+__logger_formatter.default_msec_format = '%s.%03d'
+__logger_channel.setFormatter(__logger_formatter)
+logger.addHandler(__logger_channel)
+logger.info('Made logger')
+
 from eddn.conf.Settings import Settings, loadConfig
 
 from gevent import monkey
@@ -25,6 +37,7 @@ monkey.patch_all()
 from bottle import Bottle, get, request, response, run
 app = Bottle()
 
+
 # This import must be done post-monkey-patching!
 from eddn.core.StatsCollector import StatsCollector
 statsCollector = StatsCollector()
@@ -37,6 +50,26 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES:
     duplicateMessages.start()
 
 
+def parse_cl_args():
+    parser = argparse.ArgumentParser(
+        prog='Gateway',
+        description='EDDN Gateway server',
+    )
+
+    parser.add_argument(
+        '--loglevel',
+        help='Logging level to output at',
+    )
+
+    parser.add_argument(
+        '-c', '--config',
+        metavar='config filename',
+        nargs='?',
+        default=None,
+    )
+
+    return parser.parse_args()
+
 @app.route('/stats/', method=['OPTIONS', 'GET'])
 def stats():
     stats = statsCollector.getSummary()
@@ -171,7 +204,12 @@ class EnableCors(object):
 
 
 def main():
-    loadConfig()
+    cl_args = parse_cl_args()
+    if cl_args.loglevel:
+        logger.setLevel(cl_args.loglevel)
+
+    loadConfig(cl_args)
+
     r = Relay()
     r.start()
 

src/eddn/conf/Settings.py

@@ -1,6 +1,5 @@
 # coding: utf8
 
-import argparse
 import simplejson
 from eddn.conf.Version import __version__ as version
 
@@ -134,15 +133,14 @@ class _Settings(object):
 Settings = _Settings()
 
 
-def loadConfig():
-    '''
-    Loads in a settings file specified on the commandline if one has been specified.
+def loadConfig(cl_args):
+    """
+    Load in a commandline-specified settings file, if applicable.
+
     A convenience method if you don't need other things specified as commandline
     options. Otherwise, point the filename to Settings.loadFrom().
-    '''
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-c", "--config", nargs="?", default=None)
-    args = parser.parse_args()
 
-    if args.config:
-        Settings.loadFrom(args.config)
+    :param cl_args: An `argparse.parse_args()` return.
+    """
+    if cl_args.config:
+        Settings.loadFrom(cl_args.config)