From 872c65c280d33db54833cd29ee66fa1e864fa723 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 14:16:45 +0000 Subject: [PATCH 1/7] setup.py: Do *not* install eddn-logs-archive script This should be done manually, so as to be using the correct version from the correct branch on the live service. --- setup.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/setup.py b/setup.py index d765672..2c04b93 100644 --- a/setup.py +++ b/setup.py @@ -163,16 +163,6 @@ shutil.copy( '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) ) -# Ensure the service log file archiving script is in place -print """ -****************************************************************************** -Ensuring the service log file archiving script is in place -""" -shutil.copy( - 'contrib/eddn-logs-archive', - START_SCRIPT_BIN -) - # Ensure the latest monitor files are in place old_umask = os.umask(022) print """ From fb3c0348adab2c585cfb58428f9ef114a8c41345 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 15:02:38 +0000 Subject: [PATCH 2/7] Gateway: Support `--loglevel ` CL arg NB: This required changes to src/eddn/conf/Settings.py as well, which means other scripts, e.g. Monitor.py, will also need changing. --- src/eddn/Gateway.py | 29 ++++++++++++++++++++++++++++- src/eddn/conf/Settings.py | 17 ++++++++--------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d9e7dfd..3e33132 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -4,6 +4,7 @@ Contains the necessary ZeroMQ socket and a helper function to publish market data to the Announcer daemons. """ +import argparse import gevent import hashlib import logging @@ -51,6 +52,27 @@ statsCollector = StatsCollector() statsCollector.start() +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + + def extract_message_details(parsed_message): uploader_id = '<>' software_name = '<>' @@ -342,7 +364,12 @@ class EnableCors(object): def main(): - loadConfig() + + cl_args = parse_cl_args() + if cl_args.loglevel: + logger.setLevel(cl_args.loglevel) + + loadConfig(cl_args) configure() app.install(EnableCors()) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index ce876b5..92753df 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -134,15 +134,14 @@ class _Settings(object): Settings = _Settings() -def loadConfig(): - ''' - Loads in a settings file specified on the commandline if one has been specified. +def loadConfig(cl_args): + """ + Load in a commandline-specified settings file, if applicable. + A convenience method if you don't need other things specified as commandline options. Otherwise, point the filename to Settings.loadFrom(). - ''' - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--config", nargs="?", default=None) - args = parser.parse_args() - if args.config: - Settings.loadFrom(args.config) + :param cl_args: An `argparse.parse_args()` return. + """ + if cl_args.config: + Settings.loadFrom(cl_args.config) From 86b283f920818c6132b8f64405ab64b9a8b3306c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 15:09:56 +0000 Subject: [PATCH 3/7] Monitor.py: Minimal changes to allow `--loglevel ` The CL arg won't actually *do* anything yet, because this script doesn't actually have a logger set up at all. But we need to fake this else eddn.conf.Settings.loadConfig() won't be called correctly or work. --- src/eddn/Monitor.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 4c188d4..a31853f 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -4,6 +4,7 @@ Monitor sit below gateways, or another relay, and simply parse what it receives over SUB. """ from threading import Thread +import argparse import zlib import gevent import simplejson @@ -27,6 +28,26 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicateMessages.start() +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='CURRENTLY NO EFFECT - Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + def date(__format): d = datetime.datetime.utcnow() return d.strftime(__format) @@ -237,7 +258,9 @@ class EnableCors(object): return _enable_cors def main(): - loadConfig() + cl_args = parse_cl_args() + loadConfig(cl_args) + m = Monitor() m.start() app.install(EnableCors()) From fe214583aac2d80287bf35c0771ed8a2c7615ffb Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 15:16:32 +0000 Subject: [PATCH 4/7] Relay: Support `--loglevel ` fully There's no real logging in Relay yet, but this makes it support switching level anyway. --- src/eddn/Relay.py | 60 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 10629d4..4fbd990 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -4,20 +4,32 @@ Relays sit below an announcer, or another relay, and simply repeat what they receive over PUB/SUB. """ -# Logging has to be configured first before we do anything. +import argparse +import gevent +import hashlib import logging +import simplejson +import time +import uuid +import zlib from threading import Thread -import time - -logger = logging.getLogger(__name__) -import zlib - -import gevent -import simplejson -import hashlib -import uuid import zmq.green as zmq + + +# Logging has to be configured first before we do anything. +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +__logger_channel = logging.StreamHandler() +__logger_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' + ) +__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' +__logger_formatter.default_msec_format = '%s.%03d' +__logger_channel.setFormatter(__logger_formatter) +logger.addHandler(__logger_channel) +logger.info('Made logger') + from eddn.conf.Settings import Settings, loadConfig from gevent import monkey @@ -25,6 +37,7 @@ monkey.patch_all() from bottle import Bottle, get, request, response, run app = Bottle() + # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector statsCollector = StatsCollector() @@ -37,6 +50,26 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicateMessages.start() +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + @app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): stats = statsCollector.getSummary() @@ -171,7 +204,12 @@ class EnableCors(object): def main(): - loadConfig() + cl_args = parse_cl_args() + if cl_args.loglevel: + logger.setLevel(cl_args.loglevel) + + loadConfig(cl_args) + r = Relay() r.start() From 6986af7ca0d5e0592e3a5105d543acba7cdf2c3f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 15:17:45 +0000 Subject: [PATCH 5/7] Settings.py: Remove `import argparse` CL arg parsing is done in main scripts now. --- src/eddn/conf/Settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 92753df..faac1c0 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -1,6 +1,5 @@ # coding: utf8 -import argparse import simplejson from eddn.conf.Version import __version__ as version From 8f7910a5d966934992b994c998876ed6b708d004 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 15:22:02 +0000 Subject: [PATCH 6/7] Bouncer: Support `--loglevel ` CL arg This also aligns the general logger setup with the other scripts --- src/eddn/Bouncer.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index b2e51f1..93fa673 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -26,6 +26,7 @@ Architecture: 4. `push_message()` then sends the message to the configured live/real Gateway. """ +import argparse import gevent import hashlib import logging @@ -47,8 +48,15 @@ from bottle import Bottle, run, request, response, get, post app = Bottle() logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -logger.addHandler(logging.StreamHandler()) +logger.setLevel(logging.INFO) +__logger_channel = logging.StreamHandler() +__logger_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' + ) +__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' +__logger_formatter.default_msec_format = '%s.%03d' +__logger_channel.setFormatter(__logger_formatter) +logger.addHandler(__logger_channel) logger.info('Made logger') @@ -57,6 +65,27 @@ from eddn.core.StatsCollector import StatsCollector statsCollector = StatsCollector() statsCollector.start() + +def parse_cl_args(): + parser = argparse.ArgumentParser( + prog='Gateway', + description='EDDN Gateway server', + ) + + parser.add_argument( + '--loglevel', + help='Logging level to output at', + ) + + parser.add_argument( + '-c', '--config', + metavar='config filename', + nargs='?', + default=None, + ) + + return parser.parse_args() + def push_message(message_body): """ Spawned as a greenlet to push messages (strings) through ZeroMQ. @@ -243,8 +272,12 @@ class CustomLogging(object): return _log_to_logger def main(): + cl_args = parse_cl_args() + if cl_args.loglevel: + logger.setLevel(cl_args.loglevel) + logger.info('Loading config...') - loadConfig() + loadConfig(cl_args) logger.info('Installing EnableCors ...') app.install(EnableCors()) From 8d5b03915bb142d79a621a7490f320d01b481c21 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 18 Jan 2022 15:53:06 +0000 Subject: [PATCH 7/7] Gateway: Log where upload data is form-encoded We suspect no-one is actually using this possible form of upload, and ideally would remove the code for it entirely. First we need to log any uploaders that do use this form to be sure. --- src/eddn/Gateway.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 3e33132..19912ed 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -160,6 +160,7 @@ def get_decompressed_message(): # Auto header checking. logger.debug('Trying zlib.decompress (15 + 32)...') message_body = zlib.decompress(request.body.read(), 15 + 32) + except zlib.error: logger.error('zlib.error, trying zlib.decompress (-15)') # Negative wbits suppresses adler32 checksumming. @@ -171,12 +172,14 @@ def get_decompressed_message(): # body. If it's not form-encoded, this will return an empty dict. form_enc_parsed = urlparse.parse_qs(message_body) if form_enc_parsed: - logger.debug('Request is form-encoded') + logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address())) # This is a form-encoded POST. The value of the data attrib will # be the body we're looking for. try: message_body = form_enc_parsed['data'][0] + except (KeyError, IndexError): + logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address()) raise MalformedUploadError( "No 'data' POST key/value found. Check your POST key " "name for spelling, and make sure you're passing a value." @@ -192,7 +195,7 @@ def get_decompressed_message(): # POST key/vals, or un-encoded body. data_key = request.forms.get('data') if data_key: - logger.debug('form-encoded POST request detected...') + logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address())) # This is a form-encoded POST. Support the silly people. message_body = data_key