third-party-mirrors/EDDN
1
0
Fork 0
mirror of https://github.com/EDCD/EDDN.git synced 2025-04-27 21:52:14 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Gateway: flake8 and mypy pass

Browse Source
This commit is contained in:
Athanasius 2022-03-12 12:27:11 +00:00
parent 9867d8dc29
commit 71f86b6a4e
1 changed files with 60 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/eddn/Gateway.py
View File

@ -8,7 +8,6 @@ Contains the necessary ZeroMQ socket and a helper function to publish
market data to the Announcer daemons. market data to the Announcer daemons.
""" """
import argparse import argparse
import hashlib
import logging import logging
import zlib import zlib
from datetime import datetime from datetime import datetime
@ -18,7 +17,6 @@ from urllib.parse import parse_qs
import gevent import gevent
import simplejson import simplejson
import zmq.green as zmq import zmq.green as zmq
from bottle import Bottle, request, response
from gevent import monkey from gevent import monkey
from pkg_resources import resource_string from pkg_resources import resource_string
from zmq import PUB as ZMQ_PUB from zmq import PUB as ZMQ_PUB
@ -27,8 +25,9 @@ from eddn.conf.Settings import Settings, load_config
from eddn.core.Validator import ValidationSeverity, Validator from eddn.core.Validator import ValidationSeverity, Validator
monkey.patch_all() monkey.patch_all()
import bottle import bottle # noqa: E402
from bottle import Bottle, request, response from bottle import Bottle, request, response # noqa: E402
bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB
app = Bottle() app = Bottle()
@ -60,6 +59,7 @@ stats_collector.start()
def parse_cl_args(): def parse_cl_args():
"""Parse command-line arguments."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog='Gateway', prog='Gateway',
description='EDDN Gateway server', description='EDDN Gateway server',
@ -80,7 +80,13 @@ def parse_cl_args():
return parser.parse_args() return parser.parse_args()
def extract_message_details(parsed_message): def extract_message_details(parsed_message): # noqa: CCR001
"""
Extract the details of an EDDN message.
:param parsed_message: The message to process
:return: Tuple of (uploader_id, software_name, software_version, schema_ref, journal_event)
"""
uploader_id = '<<UNKNOWN>>' uploader_id = '<<UNKNOWN>>'
software_name = '<<UNKNOWN>>' software_name = '<<UNKNOWN>>'
software_version = '<<UNKNOWN>>' software_version = '<<UNKNOWN>>'
@ -100,7 +106,6 @@ def extract_message_details(parsed_message):
if '$schemaRef' in parsed_message: if '$schemaRef' in parsed_message:
schema_ref = parsed_message['$schemaRef'] schema_ref = parsed_message['$schemaRef']
if '/journal/' in schema_ref: if '/journal/' in schema_ref:
if 'message' in parsed_message: if 'message' in parsed_message:
if 'event' in parsed_message['message']: if 'event' in parsed_message['message']:
@ -111,6 +116,7 @@ def extract_message_details(parsed_message):
return uploader_id, software_name, software_version, schema_ref, journal_event return uploader_id, software_name, software_version, schema_ref, journal_event
def configure() -> None: def configure() -> None:
""" """
Get the list of transports to bind from settings. Get the list of transports to bind from settings.
@ -178,21 +184,24 @@ def get_decompressed_message() -> bytes:
logger.error('zlib.error, trying zlib.decompress (-15)') logger.error('zlib.error, trying zlib.decompress (-15)')
# Negative wbits suppresses adler32 checksumming. # Negative wbits suppresses adler32 checksumming.
message_body = zlib.decompress(request.body.read(), -15) message_body = zlib.decompress(request.body.read(), -15)
logger.debug('Resulting message_body:\n%s\n' % (message_body)) logger.debug('Resulting message_body:\n%s\n', message_body)
# At this point, we're not sure whether we're dealing with a straight # At this point, we're not sure whether we're dealing with a straight
# un-encoded POST body, or a form-encoded POST. Attempt to parse the # un-encoded POST body, or a form-encoded POST. Attempt to parse the
# body. If it's not form-encoded, this will return an empty dict. # body. If it's not form-encoded, this will return an empty dict.
form_enc_parsed = parse_qs(message_body) form_enc_parsed = parse_qs(message_body)
if form_enc_parsed: if form_enc_parsed:
logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address())) logger.info('Request is form-encoded, compressed, from %s', get_remote_address())
# This is a form-encoded POST. The value of the data attrib will # This is a form-encoded POST. The value of the data attrib will
# be the body we're looking for. # be the body we're looking for.
try: try:
message_body = form_enc_parsed[b'data'][0] message_body = form_enc_parsed[b'data'][0]
except (KeyError, IndexError): except (KeyError, IndexError):
logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address()) logger.error(
'form-encoded, compressed, upload did not contain a "data" key. From %s',
get_remote_address()
)
raise MalformedUploadError( raise MalformedUploadError(
"No 'data' POST key/value found. Check your POST key " "No 'data' POST key/value found. Check your POST key "
"name for spelling, and make sure you're passing a value." "name for spelling, and make sure you're passing a value."
@ -208,7 +217,7 @@ def get_decompressed_message() -> bytes:
# POST key/vals, or un-encoded body. # POST key/vals, or un-encoded body.
data_key = request.forms.get('data') data_key = request.forms.get('data')
if data_key: if data_key:
logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address())) logger.info('Request is form-encoded, uncompressed, from %s', get_remote_address())
# This is a form-encoded POST. Support the silly people. # This is a form-encoded POST. Support the silly people.
message_body = data_key message_body = data_key
@ -234,7 +243,8 @@ def parse_and_error_handle(data: bytes) -> str:
# Something bad happened. We know this will return at least a # Something bad happened. We know this will return at least a
# semi-useful error message, so do so. # semi-useful error message, so do so.
try: try:
logger.error('Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n' % ( logger.error(
'Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n',
request.content_length, request.content_length,
'<<UNKNOWN>>', '<<UNKNOWN>>',
'<<UNKNOWN>>', '<<UNKNOWN>>',
@ -243,15 +253,15 @@ def parse_and_error_handle(data: bytes) -> str:
'<<UNKNOWN>>', '<<UNKNOWN>>',
get_remote_address(), get_remote_address(),
data[:512] data[:512]
)) )
except Exception as e: except Exception as e:
print('Logging of "JSON parse failed" failed: %s' % (e.message)) # TODO: Maybe just `{e}` ?
print(f"Logging of 'JSON parse failed' failed: {str(e)}")
pass pass
response.status = 400 response.status = 400
logger.error(f"Error to {get_remote_address()}: {exc}") logger.error(f"Error to {get_remote_address()}: {exc}")
return str(exc)
return 'FAIL: JSON parsing: ' + str(exc) return 'FAIL: JSON parsing: ' + str(exc)
# Here we check if an outdated schema has been passed # Here we check if an outdated schema has been passed
@ -271,36 +281,40 @@ def parse_and_error_handle(data: bytes) -> str:
gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef'])
try: try:
uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501
logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( logger.info(
'Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s',
request.content_length, request.content_length,
uploader_id, software_name, software_version, schema_ref, journal_event, uploader_id, software_name, software_version, schema_ref, journal_event,
get_remote_address() get_remote_address()
)) )
except Exception as e: except Exception as e:
print('Logging of Accepted request failed: %s' % (e.message)) # TODO: Maybe just `{e}` ?
print(f"Logging of Accepted request failed: {str(e)}")
pass pass
return 'OK' return 'OK'
else: else:
try: try:
uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501
logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( logger.error(
str(validationResults.messages), 'Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s',
str(validation_results.messages),
request.content_length, request.content_length,
uploader_id, software_name, software_version, schema_ref, journal_event, uploader_id, software_name, software_version, schema_ref, journal_event,
get_remote_address() get_remote_address()
)) )
except Exception as e: except Exception as e:
print('Logging of Failed Validation failed: %s' % (e.message)) # TODO: Maybe just `{e}` ?
print(f"Logging of Failed Validation failed: {str(e)}")
pass pass
response.status = 400 response.status = 400
stats_collector.tally("invalid") stats_collector.tally("invalid")
return "FAIL: Schema Validation: " + str(validationResults.messages) return "FAIL: Schema Validation: " + str(validation_results.messages)
@app.route('/upload/', method=['OPTIONS', 'POST']) @app.route('/upload/', method=['OPTIONS', 'POST'])
@ -320,20 +334,25 @@ def upload() -> str:
# the correct direction. # the correct direction.
response.status = 400 response.status = 400
try: try:
logger.error(f'gzip error ({request.content_length}, "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>") from {get_remote_address()}') logger.error(
f'gzip error ({request.content_length}, "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>",'
' "<<UNKNOWN>>", "<<UNKNOWN>>") from {get_remote_address()}'
)
except Exception as e: except Exception as e:
print('Logging of "gzip error" failed: %s' % (e.message)) # TODO: Maybe just `{e}` ?
print(f"Logging of 'gzip error' failed: {str(e)}")
pass pass
return 'FAIL: zlib.error: ' + exc.message return 'FAIL: zlib.error: ' + str(exc)
except MalformedUploadError as exc: except MalformedUploadError as exc:
# They probably sent an encoded POST, but got the key/val wrong. # They probably sent an encoded POST, but got the key/val wrong.
response.status = 400 response.status = 400
logger.error(f"MalformedUploadError from {get_remote_address()}: {exc.message}") # TODO: Maybe just `{exc}` ?
logger.error("MalformedUploadError from %s: %s", get_remote_address(), str(exc))
return 'FAIL: Malformed Upload: ' + exc.message return 'FAIL: Malformed Upload: ' + str(exc)
stats_collector.tally("inbound") stats_collector.tally("inbound")
return parse_and_error_handle(message_body) return parse_and_error_handle(message_body)