From 3814f9ebb9eba2f4607f3674ec8a9bf7936188e9 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 20 Jan 2022 23:20:10 +0200 Subject: [PATCH 01/18] Quick fix to a type warning --- plugins/eddn.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/eddn.py b/plugins/eddn.py index 37165ea1..f16c571e 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -168,6 +168,10 @@ class EDDN: def flush(self): """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" + if self.replayfile is None: + logger.warning('replayfile is None!') + return + self.replayfile.seek(0, SEEK_SET) self.replayfile.truncate() for line in self.replaylog: From d65bcbbf0aeb5419f4d1da164849054f7797f9dd Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 21 Jan 2022 00:09:39 +0200 Subject: [PATCH 02/18] Added util directory and http util file HTTP utils are anything generally useful for HTTP things, currently thats just compressing a string if its larger than a given number of bytes. These libraries are intended to be available to plugins --- util/http.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 util/http.py diff --git a/util/http.py b/util/http.py new file mode 100644 index 00000000..441ff386 --- /dev/null +++ b/util/http.py @@ -0,0 +1,24 @@ +"""Utilities for dealing with HTTP.""" +from __future__ import annotations + +from gzip import compress + +__all__ = ['gzip'] + + +def gzip(data: str | bytes, max_size: int = 512, encoding='utf-8') -> tuple[bytes, bool]: + """ + Compress the given data if the max size is greater than specified. + + :param data: The data to compress + :param max_size: The max size of data, in bytes, defaults to 512 + :param encoding: The encoding to use if data is a str, defaults to 'utf-8' + :return: the payload to send, and a bool indicating compression state + """ + if isinstance(data, str): + data = data.encode(encoding=encoding) + + if len(data) <= max_size: + return data, False + + return compress(data), True From b780b1ab8e64d3c4a091014a0b3f5baa57e42575 Mon Sep 17 00:00:00 2001 From: A_D Date: Fri, 21 Jan 2022 00:12:51 +0200 Subject: [PATCH 03/18] Compress outgoing EDDN data if its large This is a sidestep solution to #1390. It doesn't attempt to directly resend data, only compressing with gzip over a given size. If that STILL returns a 413, its dropped, as without introspection of the message we cannot make it any smaller --- plugins/eddn.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index f16c571e..6275349f 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -10,6 +10,7 @@ from collections import OrderedDict from os import SEEK_SET from os.path import join from platform import system +from textwrap import dedent from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Mapping, MutableMapping, Optional from typing import OrderedDict as OrderedDictT from typing import TextIO, Tuple, Union @@ -27,6 +28,7 @@ from monitor import monitor from myNotebook import Frame from prefs import prefsVersion from ttkHyperlinkLabel import HyperlinkLabel +from util import http if sys.platform != 'win32': from fcntl import LOCK_EX, LOCK_NB, lockf @@ -217,7 +219,12 @@ class EDDN: ('message', msg['message']), ]) - r = self.session.post(self.eddn_url, data=json.dumps(to_send), timeout=self.TIMEOUT) + encoded, compressed = http.gzip(json.dumps(to_send)) + headers: None | dict[str, str] = None + if compressed: + headers = {'Content-Encoding': 'gzip'} + + r = self.session.post(self.eddn_url, data=encoded, timeout=self.TIMEOUT, headers=headers) if r.status_code != requests.codes.ok: # Check if EDDN is still objecting to an empty commodities list @@ -230,15 +237,31 @@ class EDDN: logger.trace_if('plugin.eddn', "EDDN is still objecting to empty commodities data") return # We want to silence warnings otherwise + from base64 import b64encode # we dont need this to be around until this point, which may never hit + if r.status_code == 413: + logger.debug(dedent( + f"""\ + Got a 413 while POSTing data + URL:\t{r.url} + Headers:\t{r.headers} + Sent Data Len:\t {len(encoded)} + Content:\n{r.text}\n + Msg:\n{msg} + Encoded:\n{b64encode(encoded).decode(errors="replace")} + """ + )) + + return # drop the error + if not self.UNKNOWN_SCHEMA_RE.match(r.text): - logger.debug( + logger.debug(dedent( f'''Status from POST wasn't OK: -Status\t{r.status_code} -URL\t{r.url} -Headers\t{r.headers} -Content:\n{r.text} -Msg:\n{msg}''' - ) + Status\t{r.status_code} + URL\t{r.url} + Headers\t{r.headers} + Content:\n{r.text} + Msg:\n{msg}''' + )) r.raise_for_status() From 2b8fe57bc755a4c20643c95a6798b37ec8f192ca Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 24 Jan 2022 17:21:02 +0200 Subject: [PATCH 04/18] gzip decompress support for debug_webserver --- debug_webserver.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/debug_webserver.py b/debug_webserver.py index 3d1a671b..9b717ec0 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -1,4 +1,5 @@ """Simple HTTP listener to be used with debugging various EDMC sends.""" +import gzip import json import pathlib import tempfile @@ -30,7 +31,15 @@ class LoggingHandler(server.BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 # I cant change it """Handle POST.""" logger.info(f"Received a POST for {self.path!r}!") - data = self.rfile.read(int(self.headers['Content-Length'])).decode('utf-8', errors='replace') + data_raw: bytes = self.rfile.read(int(self.headers['Content-Length'])) + data: str | bytes + + if self.headers.get('Content-Encoding') == 'gzip': + data = gzip.decompress(data_raw).decode('utf-8', errors='replace') + + else: + data = data_raw.decode('utf-8', errors='replace') + to_save = data target_path = self.path @@ -64,7 +73,7 @@ class LoggingHandler(server.BaseHTTPRequestHandler): target_file = output_data_path / (safe_file_name(target_path) + '.log') if target_file.parent != output_data_path: logger.warning(f"REFUSING TO WRITE FILE THAT ISN'T IN THE RIGHT PLACE! {target_file=}") - logger.warning(f'DATA FOLLOWS\n{data}') + logger.warning(f'DATA FOLLOWS\n{data}') # type: ignore # mypy thinks data is a byte string here return with output_lock, target_file.open('a') as f: From 73f45e37cde8147d722986b1df555e600a78effa Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 24 Jan 2022 17:21:31 +0200 Subject: [PATCH 05/18] always compress eddn data, comment rationale --- plugins/eddn.py | 11 ++++++++++- util/http.py | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 6275349f..dd8e6cab 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -219,7 +219,16 @@ class EDDN: ('message', msg['message']), ]) - encoded, compressed = http.gzip(json.dumps(to_send)) + # About the smallest request is going to be (newlines added for brevity): + # {"$schemaRef":"https://eddn.edcd.io/schemas/shipyard/2","header":{"softwareName":"E:D Market + # Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"messa + # ge":{"systemName":"delphi","stationName":"The Oracle","marketID":128782803,"timestamp":"xxxx + # -xx-xxTxx:xx:xxZ","ships":[]}} + # + # Which comes to about around 307 bytes. Lets play it safe and make our minimum 0 bytes. + # Which compresses everything + + encoded, compressed = http.gzip(json.dumps(to_send, separators=(',', ':')), max_size=0) headers: None | dict[str, str] = None if compressed: headers = {'Content-Encoding': 'gzip'} diff --git a/util/http.py b/util/http.py index 441ff386..8a8578b9 100644 --- a/util/http.py +++ b/util/http.py @@ -10,6 +10,9 @@ def gzip(data: str | bytes, max_size: int = 512, encoding='utf-8') -> tuple[byte """ Compress the given data if the max size is greater than specified. + The default was chosen somewhat arbitrarily, see eddn.py for some more careful + work towards keeping the data almost always compressed + :param data: The data to compress :param max_size: The max size of data, in bytes, defaults to 512 :param encoding: The encoding to use if data is a str, defaults to 'utf-8' From 5f7234ce890c927df51065c95272d1fd46be37ea Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 24 Jan 2022 21:17:50 +0200 Subject: [PATCH 06/18] address PR comments --- plugins/eddn.py | 21 ++++++++++----------- util/{http.py => text.py} | 0 2 files changed, 10 insertions(+), 11 deletions(-) rename util/{http.py => text.py} (100%) diff --git a/plugins/eddn.py b/plugins/eddn.py index dd8e6cab..964aa093 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -28,7 +28,7 @@ from monitor import monitor from myNotebook import Frame from prefs import prefsVersion from ttkHyperlinkLabel import HyperlinkLabel -from util import http +from util import text if sys.platform != 'win32': from fcntl import LOCK_EX, LOCK_NB, lockf @@ -171,7 +171,7 @@ class EDDN: def flush(self): """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" if self.replayfile is None: - logger.warning('replayfile is None!') + logger.error('replayfile is None!') return self.replayfile.seek(0, SEEK_SET) @@ -220,15 +220,14 @@ class EDDN: ]) # About the smallest request is going to be (newlines added for brevity): - # {"$schemaRef":"https://eddn.edcd.io/schemas/shipyard/2","header":{"softwareName":"E:D Market - # Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"messa - # ge":{"systemName":"delphi","stationName":"The Oracle","marketID":128782803,"timestamp":"xxxx - # -xx-xxTxx:xx:xxZ","ships":[]}} + # {"$schemaRef":"https://eddn.edcd.io/schemas/commodity/3","header":{"softwareName":"E:D Market + # Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"messag + # e":{"systemName":"delphi","stationName":"The Oracle","marketId":128782803,"timestamp":"2022-0 + # 1-26T12:00:00Z","commodities":[]}} # - # Which comes to about around 307 bytes. Lets play it safe and make our minimum 0 bytes. - # Which compresses everything + # Which comes to 315 bytes (including \n) and compresses to 244 bytes. So lets just compress everything - encoded, compressed = http.gzip(json.dumps(to_send, separators=(',', ':')), max_size=0) + encoded, compressed = text.gzip(json.dumps(to_send, separators=(',', ':')), max_size=0) headers: None | dict[str, str] = None if compressed: headers = {'Content-Encoding': 'gzip'} @@ -249,7 +248,7 @@ class EDDN: from base64 import b64encode # we dont need this to be around until this point, which may never hit if r.status_code == 413: logger.debug(dedent( - f"""\ + f'''\ Got a 413 while POSTing data URL:\t{r.url} Headers:\t{r.headers} @@ -257,7 +256,7 @@ class EDDN: Content:\n{r.text}\n Msg:\n{msg} Encoded:\n{b64encode(encoded).decode(errors="replace")} - """ + ''' )) return # drop the error diff --git a/util/http.py b/util/text.py similarity index 100% rename from util/http.py rename to util/text.py From 45dba2ba880142ae5e6891569a17dce9e5a7ed41 Mon Sep 17 00:00:00 2001 From: A_D Date: Mon, 24 Jan 2022 23:17:51 +0200 Subject: [PATCH 07/18] change module doc comment for text.py --- util/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/text.py b/util/text.py index 8a8578b9..0755f0f3 100644 --- a/util/text.py +++ b/util/text.py @@ -1,4 +1,4 @@ -"""Utilities for dealing with HTTP.""" +"""Utilities for dealing with text (and byte representations thereof).""" from __future__ import annotations from gzip import compress From 10824250e179aaad5bc2c9fc9a3cf6c146c65a24 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 25 Jan 2022 15:09:30 +0200 Subject: [PATCH 08/18] add deflate support --- debug_webserver.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/debug_webserver.py b/debug_webserver.py index 9b717ec0..1defdc5b 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -4,6 +4,7 @@ import json import pathlib import tempfile import threading +import zlib from http import server from typing import Any, Callable, Tuple, Union from urllib.parse import parse_qs @@ -34,6 +35,16 @@ class LoggingHandler(server.BaseHTTPRequestHandler): data_raw: bytes = self.rfile.read(int(self.headers['Content-Length'])) data: str | bytes + match self.headers.get('Content-Encoding'): + case 'gzip': + data = gzip.decompress(data_raw).decode('utf-8', errors='replace') + + case 'deflate': + zlib.decompress(data_raw).decode('utf-8', errors='replace') + + case _: + data = data_raw.decode('utf-8', errors='replace') + if self.headers.get('Content-Encoding') == 'gzip': data = gzip.decompress(data_raw).decode('utf-8', errors='replace') From ec6f333cfa2bd51b95c6aeac767dee2eeb16ac09 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 25 Jan 2022 15:18:23 +0200 Subject: [PATCH 09/18] removed old code --- debug_webserver.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/debug_webserver.py b/debug_webserver.py index 1defdc5b..f660ea05 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -45,12 +45,6 @@ class LoggingHandler(server.BaseHTTPRequestHandler): case _: data = data_raw.decode('utf-8', errors='replace') - if self.headers.get('Content-Encoding') == 'gzip': - data = gzip.decompress(data_raw).decode('utf-8', errors='replace') - - else: - data = data_raw.decode('utf-8', errors='replace') - to_save = data target_path = self.path From 1c190dd76fb2ef5b010ff4ffc944f2d6baf74718 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 25 Jan 2022 17:11:23 +0200 Subject: [PATCH 10/18] drop match statement --- debug_webserver.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/debug_webserver.py b/debug_webserver.py index f660ea05..024f39b9 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -35,15 +35,13 @@ class LoggingHandler(server.BaseHTTPRequestHandler): data_raw: bytes = self.rfile.read(int(self.headers['Content-Length'])) data: str | bytes - match self.headers.get('Content-Encoding'): - case 'gzip': - data = gzip.decompress(data_raw).decode('utf-8', errors='replace') + encoding = self.headers.get('Content-Encoding') - case 'deflate': - zlib.decompress(data_raw).decode('utf-8', errors='replace') + if encoding in ('gzip', 'deflate'): + data = zlib.decompress(data_raw).decode('utf-8', errors='replace') - case _: - data = data_raw.decode('utf-8', errors='replace') + else: + data = data_raw.decode('utf-8', errors='replace') to_save = data From e6ffe7e5208615e79fa2b2ee4c080e2d21d03116 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 25 Jan 2022 17:11:35 +0200 Subject: [PATCH 11/18] refactor error logging --- plugins/eddn.py | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 964aa093..74bab79d 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -245,34 +245,40 @@ class EDDN: logger.trace_if('plugin.eddn', "EDDN is still objecting to empty commodities data") return # We want to silence warnings otherwise - from base64 import b64encode # we dont need this to be around until this point, which may never hit if r.status_code == 413: - logger.debug(dedent( - f'''\ - Got a 413 while POSTing data - URL:\t{r.url} - Headers:\t{r.headers} - Sent Data Len:\t {len(encoded)} - Content:\n{r.text}\n - Msg:\n{msg} - Encoded:\n{b64encode(encoded).decode(errors="replace")} - ''' - )) - + self._log_response(r, header_msg='Got a 413 while POSTing data', sent_data_len=str(len(encoded))) return # drop the error if not self.UNKNOWN_SCHEMA_RE.match(r.text): - logger.debug(dedent( - f'''Status from POST wasn't OK: - Status\t{r.status_code} - URL\t{r.url} - Headers\t{r.headers} - Content:\n{r.text} - Msg:\n{msg}''' - )) + self._log_response(r, header_msg='Status from POST wasn\'t 200 (OK)') r.raise_for_status() + def _log_response( + self, + response: requests.Response, + header_msg='Failed to POST to EDDN', + **kwargs + ) -> None: + """ + Log a response object with optional additional data. + + :param response: The response to log + :param header_msg: A header message to add to the log, defaults to 'Failed to POST to EDDN' + :param kwargs: Any other notes to add, will be added below the main data in the same format. + """ + additional_data = "\n".join( + f'''{name.replace('_', ' ').title():<8}:\t{value}''' for name, value in kwargs.items() + ) + + logger.debug(dedent(f''' + {header_msg}: + Status :\t{response.status_code} + URL :\t{response.url} + Headers :\t{response.headers} + Content :\t{response.text} + '''+additional_data)) + def sendreplay(self) -> None: # noqa: CCR001 """Send cached Journal lines to EDDN.""" if not self.replayfile: From e891ee4da026816951532f59d86c3189795a65c6 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 26 Jan 2022 19:05:00 +0200 Subject: [PATCH 12/18] log more contextual data if possible on a 413 --- plugins/eddn.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 74bab79d..76e2f227 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -6,6 +6,7 @@ import pathlib import re import sys import tkinter as tk +from base64 import encode from collections import OrderedDict from os import SEEK_SET from os.path import join @@ -246,7 +247,15 @@ class EDDN: return # We want to silence warnings otherwise if r.status_code == 413: - self._log_response(r, header_msg='Got a 413 while POSTing data', sent_data_len=str(len(encoded))) + extra_data = { + 'schema_ref': msg.get('$schemaRef', 'Unset $schemaRef!'), + 'sent_data_len': str(len(encoded)), + } + + if '/journal/' in extra_data['schema_ref']: + extra_data['event'] = msg.get('message', {}).get('event', 'No Event Set') + + self._log_response(r, header_msg='Got a 413 while POSTing data', **extra_data) return # drop the error if not self.UNKNOWN_SCHEMA_RE.match(r.text): From 6c0cb45e61b9fd1d553bfcc8cab676bdaf1daa19 Mon Sep 17 00:00:00 2001 From: A_D Date: Wed, 26 Jan 2022 22:40:09 +0200 Subject: [PATCH 13/18] removed unused import --- debug_webserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug_webserver.py b/debug_webserver.py index 024f39b9..ccce5f9d 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -1,5 +1,4 @@ """Simple HTTP listener to be used with debugging various EDMC sends.""" -import gzip import json import pathlib import tempfile From d6dcebf545ba13e4d8e25b4369172bcb97b440dc Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 27 Jan 2022 14:59:27 +0200 Subject: [PATCH 14/18] Remove unused import --- plugins/eddn.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 76e2f227..a423876e 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -6,7 +6,6 @@ import pathlib import re import sys import tkinter as tk -from base64 import encode from collections import OrderedDict from os import SEEK_SET from os.path import join From 38f35403c92320254d2d0cab879b0eaa79612c1d Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 27 Jan 2022 16:02:06 +0200 Subject: [PATCH 15/18] fix gzip support in debug_webserver --- debug_webserver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debug_webserver.py b/debug_webserver.py index ccce5f9d..0a456dc9 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -1,4 +1,5 @@ """Simple HTTP listener to be used with debugging various EDMC sends.""" +import gzip import json import pathlib import tempfile @@ -36,7 +37,10 @@ class LoggingHandler(server.BaseHTTPRequestHandler): encoding = self.headers.get('Content-Encoding') - if encoding in ('gzip', 'deflate'): + if encoding == 'gzip': + data = gzip.decompress(data_raw).decode('utf-8', errors='replace') + + elif encoding == 'deflate': data = zlib.decompress(data_raw).decode('utf-8', errors='replace') else: From 17c886af84f7f1c03779ade0c5da989cf3c1d424 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 27 Jan 2022 16:03:03 +0200 Subject: [PATCH 16/18] fix log --- plugins/eddn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index a423876e..e3e4e198 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -279,13 +279,13 @@ class EDDN: f'''{name.replace('_', ' ').title():<8}:\t{value}''' for name, value in kwargs.items() ) - logger.debug(dedent(f''' + logger.debug(dedent(f'''\ {header_msg}: Status :\t{response.status_code} URL :\t{response.url} Headers :\t{response.headers} Content :\t{response.text} - '''+additional_data)) + ''')+additional_data) def sendreplay(self) -> None: # noqa: CCR001 """Send cached Journal lines to EDDN.""" From f4ba4775f2424244ed5467d55760bd3a3a819642 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 27 Jan 2022 19:40:06 +0200 Subject: [PATCH 17/18] pull decode/decompress out into its own function --- debug_webserver.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/debug_webserver.py b/debug_webserver.py index 0a456dc9..26e02714 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -6,7 +6,7 @@ import tempfile import threading import zlib from http import server -from typing import Any, Callable, Tuple, Union +from typing import Any, Callable, Literal, Tuple, Union from urllib.parse import parse_qs from config import appname @@ -37,16 +37,7 @@ class LoggingHandler(server.BaseHTTPRequestHandler): encoding = self.headers.get('Content-Encoding') - if encoding == 'gzip': - data = gzip.decompress(data_raw).decode('utf-8', errors='replace') - - elif encoding == 'deflate': - data = zlib.decompress(data_raw).decode('utf-8', errors='replace') - - else: - data = data_raw.decode('utf-8', errors='replace') - - to_save = data + to_save = self.get_printable(data_raw, encoding) target_path = self.path if len(target_path) > 1 and target_path[0] == '/': @@ -57,7 +48,7 @@ class LoggingHandler(server.BaseHTTPRequestHandler): response: Union[Callable[[str], str], str, None] = DEFAULT_RESPONSES.get(target_path) if callable(response): - response = response(data) + response = response(to_save) self.send_response_only(200, "OK") if response is not None: @@ -85,6 +76,31 @@ class LoggingHandler(server.BaseHTTPRequestHandler): with output_lock, target_file.open('a') as f: f.write(to_save + "\n\n") + @staticmethod + def get_printable(data: bytes, compression: Literal['deflate'] | Literal['gzip'] | str | None = None) -> str: + """ + Convert an incoming data stream into a string. + + :param data: The data to convert + :param compression: The compression to remove, defaults to None + :raises ValueError: If compression is unknown + :return: printable strings + """ + ret: bytes = b'' + if compression is None: + ret = data + + elif compression == 'deflate': + ret = zlib.decompress(data) + + elif compression == 'gzip': + ret = gzip.decompress(data) + + else: + raise ValueError(f'Unknown encoding for data {compression!r}') + + return ret.decode('utf-8', errors='replace') + def safe_file_name(name: str): """ From b180f34f167f651c3bd30eda8bc977a2f4cd0271 Mon Sep 17 00:00:00 2001 From: A_D Date: Thu, 27 Jan 2022 19:52:51 +0200 Subject: [PATCH 18/18] fix bugs that were hidden --- debug_webserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug_webserver.py b/debug_webserver.py index 26e02714..ace82a33 100644 --- a/debug_webserver.py +++ b/debug_webserver.py @@ -33,11 +33,10 @@ class LoggingHandler(server.BaseHTTPRequestHandler): """Handle POST.""" logger.info(f"Received a POST for {self.path!r}!") data_raw: bytes = self.rfile.read(int(self.headers['Content-Length'])) - data: str | bytes encoding = self.headers.get('Content-Encoding') - to_save = self.get_printable(data_raw, encoding) + to_save = data = self.get_printable(data_raw, encoding) target_path = self.path if len(target_path) > 1 and target_path[0] == '/': @@ -134,6 +133,7 @@ def generate_inara_response(raw_data: str) -> str: def extract_edsm_data(data: str) -> dict[str, Any]: + """Extract relevant data from edsm data.""" res = parse_qs(data) return {name: data[0] for name, data in res.items()}