third-party-mirrors/EDDN
1
0
Fork 0
mirror of https://github.com/EDCD/EDDN.git synced 2025-04-25 12:42:14 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Gateway: flake8 and mypy pass

Browse Source
This commit is contained in:
Athanasius 2022-03-12 12:27:11 +00:00
parent 9867d8dc29
commit 71f86b6a4e
1 changed files with 60 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
src/eddn/Gateway.py
View File

@ -8,7 +8,6 @@ Contains the necessary ZeroMQ socket and a helper function to publish
market data to the Announcer daemons.
"""
import argparse
import hashlib
import logging
import zlib
from datetime import datetime
@ -18,7 +17,6 @@ from urllib.parse import parse_qs
import gevent
import simplejson
import zmq.green as zmq
from bottle import Bottle, request, response
from gevent import monkey
from pkg_resources import resource_string
from zmq import PUB as ZMQ_PUB
@ -27,9 +25,10 @@ from eddn.conf.Settings import Settings, load_config
from eddn.core.Validator import ValidationSeverity, Validator
monkey.patch_all()
import bottle
from bottle import Bottle, request, response
bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB
import bottle # noqa: E402
from bottle import Bottle, request, response # noqa: E402
bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB
app = Bottle()
@ -60,6 +59,7 @@ stats_collector.start()
def parse_cl_args():
"""Parse command-line arguments."""
parser = argparse.ArgumentParser(
prog='Gateway',
description='EDDN Gateway server',
@ -80,7 +80,13 @@ def parse_cl_args():
return parser.parse_args()
def extract_message_details(parsed_message):
def extract_message_details(parsed_message): # noqa: CCR001
"""
Extract the details of an EDDN message.
:param parsed_message: The message to process
:return: Tuple of (uploader_id, software_name, software_version, schema_ref, journal_event)
"""
uploader_id = '<<UNKNOWN>>'
software_name = '<<UNKNOWN>>'
software_version = '<<UNKNOWN>>'
@ -100,7 +106,6 @@ def extract_message_details(parsed_message):
if '$schemaRef' in parsed_message:
schema_ref = parsed_message['$schemaRef']
if '/journal/' in schema_ref:
if 'message' in parsed_message:
if 'event' in parsed_message['message']:
@ -111,6 +116,7 @@ def extract_message_details(parsed_message):
return uploader_id, software_name, software_version, schema_ref, journal_event
def configure() -> None:
"""
Get the list of transports to bind from settings.
@ -178,21 +184,24 @@ def get_decompressed_message() -> bytes:
logger.error('zlib.error, trying zlib.decompress (-15)')
# Negative wbits suppresses adler32 checksumming.
message_body = zlib.decompress(request.body.read(), -15)
logger.debug('Resulting message_body:\n%s\n' % (message_body))
logger.debug('Resulting message_body:\n%s\n', message_body)
# At this point, we're not sure whether we're dealing with a straight
# un-encoded POST body, or a form-encoded POST. Attempt to parse the
# body. If it's not form-encoded, this will return an empty dict.
form_enc_parsed = parse_qs(message_body)
if form_enc_parsed:
logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address()))
logger.info('Request is form-encoded, compressed, from %s', get_remote_address())
# This is a form-encoded POST. The value of the data attrib will
# be the body we're looking for.
try:
message_body = form_enc_parsed[b'data'][0]
except (KeyError, IndexError):
logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address())
logger.error(
'form-encoded, compressed, upload did not contain a "data" key. From %s',
get_remote_address()
)
raise MalformedUploadError(
"No 'data' POST key/value found. Check your POST key "
"name for spelling, and make sure you're passing a value."
@ -208,7 +217,7 @@ def get_decompressed_message() -> bytes:
# POST key/vals, or un-encoded body.
data_key = request.forms.get('data')
if data_key:
logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address()))
logger.info('Request is form-encoded, uncompressed, from %s', get_remote_address())
# This is a form-encoded POST. Support the silly people.
message_body = data_key
@ -234,24 +243,25 @@ def parse_and_error_handle(data: bytes) -> str:
# Something bad happened. We know this will return at least a
# semi-useful error message, so do so.
try:
logger.error('Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n' % (
request.content_length,
'<<UNKNOWN>>',
'<<UNKNOWN>>',
'<<UNKNOWN>>',
'<<UNKNOWN>>',
'<<UNKNOWN>>',
get_remote_address(),
data[:512]
))
logger.error(
'Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n',
request.content_length,
'<<UNKNOWN>>',
'<<UNKNOWN>>',
'<<UNKNOWN>>',
'<<UNKNOWN>>',
'<<UNKNOWN>>',
get_remote_address(),
data[:512]
)
except Exception as e:
print('Logging of "JSON parse failed" failed: %s' % (e.message))
# TODO: Maybe just `{e}` ?
print(f"Logging of 'JSON parse failed' failed: {str(e)}")
pass
response.status = 400
logger.error(f"Error to {get_remote_address()}: {exc}")
return str(exc)
return 'FAIL: JSON parsing: ' + str(exc)
# Here we check if an outdated schema has been passed
@ -271,36 +281,40 @@ def parse_and_error_handle(data: bytes) -> str:
gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef'])
try:
uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message)
logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s' % (
uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501
logger.info(
'Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s',
request.content_length,
uploader_id, software_name, software_version, schema_ref, journal_event,
get_remote_address()
))
)
except Exception as e:
print('Logging of Accepted request failed: %s' % (e.message))
# TODO: Maybe just `{e}` ?
print(f"Logging of Accepted request failed: {str(e)}")
pass
return 'OK'
else:
try:
uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message)
logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s' % (
str(validationResults.messages),
request.content_length,
uploader_id, software_name, software_version, schema_ref, journal_event,
get_remote_address()
))
uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501
logger.error(
'Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s',
str(validation_results.messages),
request.content_length,
uploader_id, software_name, software_version, schema_ref, journal_event,
get_remote_address()
)
except Exception as e:
print('Logging of Failed Validation failed: %s' % (e.message))
# TODO: Maybe just `{e}` ?
print(f"Logging of Failed Validation failed: {str(e)}")
pass
response.status = 400
stats_collector.tally("invalid")
return "FAIL: Schema Validation: " + str(validationResults.messages)
return "FAIL: Schema Validation: " + str(validation_results.messages)
@app.route('/upload/', method=['OPTIONS', 'POST'])
@ -320,20 +334,25 @@ def upload() -> str:
# the correct direction.
response.status = 400
try:
logger.error(f'gzip error ({request.content_length}, "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>") from {get_remote_address()}')
logger.error(
f'gzip error ({request.content_length}, "<<UNKNOWN>>", "<<UNKNOWN>>", "<<UNKNOWN>>",'
' "<<UNKNOWN>>", "<<UNKNOWN>>") from {get_remote_address()}'
)
except Exception as e:
print('Logging of "gzip error" failed: %s' % (e.message))
# TODO: Maybe just `{e}` ?
print(f"Logging of 'gzip error' failed: {str(e)}")
pass
return 'FAIL: zlib.error: ' + exc.message
return 'FAIL: zlib.error: ' + str(exc)
except MalformedUploadError as exc:
# They probably sent an encoded POST, but got the key/val wrong.
response.status = 400
logger.error(f"MalformedUploadError from {get_remote_address()}: {exc.message}")
# TODO: Maybe just `{exc}` ?
logger.error("MalformedUploadError from %s: %s", get_remote_address(), str(exc))
return 'FAIL: Malformed Upload: ' + exc.message
return 'FAIL: Malformed Upload: ' + str(exc)
stats_collector.tally("inbound")
return parse_and_error_handle(message_body)