diff --git a/setup.py b/setup.py index d765672..2c04b93 100644 --- a/setup.py +++ b/setup.py @@ -163,16 +163,6 @@ shutil.copy( '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) ) -# Ensure the service log file archiving script is in place -print """ -****************************************************************************** -Ensuring the service log file archiving script is in place -""" -shutil.copy( - 'contrib/eddn-logs-archive', - START_SCRIPT_BIN -) - # Ensure the latest monitor files are in place old_umask = os.umask(022) print """ diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index b2e51f1..93fa673 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -26,6 +26,7 @@ Architecture: 4. `push_message()` then sends the message to the configured live/real Gateway. """ +import argparse import gevent import hashlib import logging @@ -47,8 +48,15 @@ from bottle import Bottle, run, request, response, get, post app = Bottle() logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.INFO) +__logger_channel = logging.StreamHandler() +__logger_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' + ) +__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' +__logger_formatter.default_msec_format = '%s.%03d' +__logger_channel.setFormatter(__logger_formatter) +logger.addHandler(__logger_channel) logger.info('Made logger') @@ -57,6 +65,27 @@ from eddn.core.StatsCollector import StatsCollector statsCollector = StatsCollector() statsCollector.start() + +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + def push_message(message_body): """ Spawned as a greenlet to push messages (strings) through ZeroMQ. @@ -243,8 +272,12 @@ class CustomLogging(object): return _log_to_logger def main(): + cl_args = parse_cl_args() + if cl_args.loglevel: + logger.setLevel(cl_args.loglevel) + logger.info('Loading config...') - loadConfig() + loadConfig(cl_args) logger.info('Installing EnableCors ...') app.install(EnableCors()) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d9e7dfd..19912ed 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -4,6 +4,7 @@ Contains the necessary ZeroMQ socket and a helper function to publish market data to the Announcer daemons. """ +import argparse import gevent import hashlib import logging @@ -51,6 +52,27 @@ statsCollector = StatsCollector() statsCollector.start() +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + + def extract_message_details(parsed_message): uploader_id = '<>' software_name = '<>' @@ -138,6 +160,7 @@ def get_decompressed_message(): # Auto header checking. logger.debug('Trying zlib.decompress (15 + 32)...') message_body = zlib.decompress(request.body.read(), 15 + 32) + except zlib.error: logger.error('zlib.error, trying zlib.decompress (-15)') # Negative wbits suppresses adler32 checksumming. @@ -149,12 +172,14 @@ def get_decompressed_message(): # body. If it's not form-encoded, this will return an empty dict. form_enc_parsed = urlparse.parse_qs(message_body) if form_enc_parsed: - logger.debug('Request is form-encoded') + logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address())) # This is a form-encoded POST. The value of the data attrib will # be the body we're looking for. try: message_body = form_enc_parsed['data'][0] + except (KeyError, IndexError): + logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address()) raise MalformedUploadError( "No 'data' POST key/value found. Check your POST key " "name for spelling, and make sure you're passing a value." @@ -170,7 +195,7 @@ def get_decompressed_message(): # POST key/vals, or un-encoded body. data_key = request.forms.get('data') if data_key: - logger.debug('form-encoded POST request detected...') + logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address())) # This is a form-encoded POST. Support the silly people. message_body = data_key @@ -342,7 +367,12 @@ class EnableCors(object): def main(): - loadConfig() + + cl_args = parse_cl_args() + if cl_args.loglevel: + logger.setLevel(cl_args.loglevel) + + loadConfig(cl_args) configure() app.install(EnableCors()) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 4c188d4..a31853f 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -4,6 +4,7 @@ Monitor sit below gateways, or another relay, and simply parse what it receives over SUB. """ from threading import Thread +import argparse import zlib import gevent import simplejson @@ -27,6 +28,26 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicateMessages.start() +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='CURRENTLY NO EFFECT - Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + def date(__format): d = datetime.datetime.utcnow() return d.strftime(__format) @@ -237,7 +258,9 @@ class EnableCors(object): return _enable_cors def main(): - loadConfig() + cl_args = parse_cl_args() + loadConfig(cl_args) + m = Monitor() m.start() app.install(EnableCors()) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 10629d4..4fbd990 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -4,20 +4,32 @@ Relays sit below an announcer, or another relay, and simply repeat what they receive over PUB/SUB. """ -# Logging has to be configured first before we do anything. +import argparse +import gevent +import hashlib import logging +import simplejson +import time +import uuid +import zlib from threading import Thread -import time - -logger = logging.getLogger(__name__) -import zlib - -import gevent -import simplejson -import hashlib -import uuid import zmq.green as zmq + + +# Logging has to be configured first before we do anything. +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +__logger_channel = logging.StreamHandler() +__logger_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' + ) +__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' +__logger_formatter.default_msec_format = '%s.%03d' +__logger_channel.setFormatter(__logger_formatter) +logger.addHandler(__logger_channel) +logger.info('Made logger') + from eddn.conf.Settings import Settings, loadConfig from gevent import monkey @@ -25,6 +37,7 @@ monkey.patch_all() from bottle import Bottle, get, request, response, run app = Bottle() + # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector statsCollector = StatsCollector() @@ -37,6 +50,26 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicateMessages.start() +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + @app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): stats = statsCollector.getSummary() @@ -171,7 +204,12 @@ class EnableCors(object): def main(): - loadConfig() + cl_args = parse_cl_args() + if cl_args.loglevel: + logger.setLevel(cl_args.loglevel) + + loadConfig(cl_args) + r = Relay() r.start() diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index ce876b5..faac1c0 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -1,6 +1,5 @@ # coding: utf8 -import argparse import simplejson from eddn.conf.Version import __version__ as version @@ -134,15 +133,14 @@ class _Settings(object): Settings = _Settings() -def loadConfig(): - ''' - Loads in a settings file specified on the commandline if one has been specified. +def loadConfig(cl_args): + """ + Load in a commandline-specified settings file, if applicable. + A convenience method if you don't need other things specified as commandline options. Otherwise, point the filename to Settings.loadFrom(). - ''' - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--config", nargs="?", default=None) - args = parser.parse_args() - if args.config: - Settings.loadFrom(args.config) + :param cl_args: An `argparse.parse_args()` return. + """ + if cl_args.config: + Settings.loadFrom(cl_args.config)