From fe0e752c9b84cc428ae208f5903f7f2d83b5c287 Mon Sep 17 00:00:00 2001 From: A_D Date: Tue, 25 May 2021 11:06:48 +0200 Subject: [PATCH] Added support for arbitrary plugins for POST debug This works by replacing --eddn-local with --debug-sender, and making the webserver more generic. support has been added to EDSM, EDDN, and INARA. --- EDMarketConnector.py | 21 +++++-- PLUGINS.md | 31 ++++++++- config.py | 2 + debug_webserver.py | 146 +++++++++++++++++++++++++++++++++++++++++++ eddnListener.py | 44 ------------- edmc_data.py | 4 +- plugins/eddn.py | 10 +-- plugins/edsm.py | 12 +++- plugins/inara.py | 7 ++- 9 files changed, 218 insertions(+), 59 deletions(-) create mode 100644 debug_webserver.py delete mode 100644 eddnListener.py diff --git a/EDMarketConnector.py b/EDMarketConnector.py index fe32c5e1..17cfdd01 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -93,7 +93,11 @@ if __name__ == '__main__': # noqa: C901 action='store_true' ) - parser.add_argument('--eddn-local', help='Redirect EDDN requests to a local webserver', action='store_true') + parser.add_argument( + '--debug-sender', + help='Mark the selected sender as in debug mode. This generally results in data being written to disk', + action='append', + ) auth_options = parser.add_mutually_exclusive_group(required=False) auth_options.add_argument('--force-localserver-for-auth', @@ -131,11 +135,16 @@ if __name__ == '__main__': # noqa: C901 parser.print_help() exit(1) - if args.eddn_local: - import eddnListener - import edmc_data - eddnListener.run_listener() - edmc_data.EDDN_DEBUG_SERVER = True + if args.debug_sender and len(args.debug_sender) > 0: + import config as conf_module + import debug_webserver + from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT + + conf_module.debug_senders = [x.casefold() for x in args.debug_sender] # duplicate the list just in case + for d in conf_module.debug_senders: + logger.info(f'marked {d} for debug') + + debug_webserver.run_listener(DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT) def handle_edmc_callback_or_foregrounding() -> None: # noqa: CCR001 """Handle any edmc:// auth callback, else foreground existing window.""" diff --git a/PLUGINS.md b/PLUGINS.md index e7bab342..3670bd3f 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -603,7 +603,7 @@ Content of `state` (updated to the current journal entry): | `Data` | `dict` | 'Data' MicroResources in Odyssey, `int` count each. | | `BackPack` | `dict` | `dict` of Odyssey MicroResources in backpack. | | `BackpackJSON` | `dict` | Content of Backpack.json as of last read. | -| `ShipLockerJSON` | `dict` | Content of ShipLocker.json as of last read. | +| `ShipLockerJSON` | `dict` | Content of ShipLocker.json as of last read. | | `SuitCurrent` | `dict` | CAPI-returned data of currently worn suit. NB: May be `None` if no data. | | `Suits` | `dict`[1] | CAPI-returned data of owned suits. NB: May be `None` if no data. | | `SuitLoadoutCurrent` | `dict` | CAPI-returned data of current Suit Loadout. NB: May be `None` if no data. | @@ -1043,6 +1043,35 @@ step for the extra module(s) until it works. --- +## Debug HTTP POST requests + +You can debug your http post requests using the builtin debug webserver. + +To add support for said debug webserver to your plugin, you need to check `config.debug_senders` (`list[str]`) for +some indicator string for your plugin. `debug_senders` is generated from args to `--debug-sender` on the invocation +command line. + +If said string exists, `DEBUG_WEBSERVER_HOST` and `DEBUG_WEBSERVER_PORT` in +`edmc_data` will contain the host and port for the currently running local webserver. Simply redirect your requests +there, and your requests will be logged to disk. For organisation, rewrite your request path to simply be `/pluginname`. + +logs exist in `$TEMP/EDMarketConnector/http_debug/$path.log`. If somehow you manage to cause a directory traversal, your +data will not be saved to disk at all. You will see this in EDMCs log. + +The simplest way to go about adding support is: + +```py +from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT +from config import debug_senders + +TARGET_URL = "https://host.tld/path/to/api/magic" +if 'my_plugin' in debug_senders: + TARGET_URL = f'http://{DEBUG_WEBSERVER_HOST}:{DEBUG_WEBSERVER_PORT}/my_plugin' + +# Code that uses TARGET_URL to post info from your plugin below. + +``` + ## Disable a plugin EDMC now lets you disable a plugin without deleting it, simply rename the diff --git a/config.py b/config.py index c42a7e51..9a699fe4 100644 --- a/config.py +++ b/config.py @@ -38,6 +38,8 @@ copyright = '© 2015-2019 Jonathan Harris, 2020-2021 EDCD' update_feed = 'https://raw.githubusercontent.com/EDCD/EDMarketConnector/releases/edmarketconnector.xml' update_interval = 8*60*60 +# Providers marked to be in debug mode. Generally this is expected to switch to sending data to a log file +debug_senders: List[str] = [] # This must be done here in order to avoid an import cycle with EDMCLogging. # Other code should use EDMCLogging.get_main_logger diff --git a/debug_webserver.py b/debug_webserver.py new file mode 100644 index 00000000..a3f60b4c --- /dev/null +++ b/debug_webserver.py @@ -0,0 +1,146 @@ +"""Simple HTTP listener to be used with debugging various EDMC sends.""" +import json +import pathlib +import tempfile +import threading +from http import server +from typing import Any, Callable, Tuple, Union +from urllib.parse import parse_qs + +from config import appname +from EDMCLogging import get_main_logger + +logger = get_main_logger() + +output_lock = threading.Lock() +output_data_path = pathlib.Path(tempfile.gettempdir()) / f'{appname}' / 'http_debug' +SAFE_TRANSLATE = str.maketrans({x: '_' for x in "!@#$%^&*()./\\\r\n[]-+='\";:?<>,~`"}) + + +class LoggingHandler(server.BaseHTTPRequestHandler): + """HTTP Handler implementation that logs to EDMCs logger and writes data to files on disk.""" + + def __init__(self, request: bytes, client_address: Tuple[str, int], server) -> None: + super().__init__(request, client_address, server) + + def log_message(self, format: str, *args: Any) -> None: + """Override default handler logger with EDMC logger.""" + logger.info(format % args) + + def do_POST(self) -> None: # noqa: N802 # I cant change it + """Handle POST.""" + logger.info(f"Received a POST for {self.path!r}!") + data = self.rfile.read(int(self.headers['Content-Length'])).decode('utf-8', errors='replace') + to_save = data + + target_path = self.path + if len(target_path) > 1 and target_path[0] == '/': + target_path = target_path[1:] + + elif len(target_path) == 1 and target_path[0] == '/': + target_path = 'WEB_ROOT' + + response: Union[Callable[[str], str], str, None] = DEFAULT_RESPONSES.get(target_path) + if callable(response): + response = response(data) + + self.send_response_only(200, "OK") + if response is not None: + self.send_header('Content-Length', str(len(response))) + self.end_headers() # This is needed because send_response_only DOESN'T ACTUALLY SEND THE RESPONSE + if response is not None: + self.wfile.write(response.encode()) + self.wfile.flush() + + if target_path == 'edsm': + # attempt to extract data from urlencoded stream + try: + edsm_data = extract_edsm_data(data) + data = data + "\n" + json.dumps(edsm_data) + except Exception: + pass + + target_file = output_data_path / (safe_file_name(target_path) + '.log') + if target_file.parent != output_data_path: + logger.warning(f"REFUSING TO WRITE FILE THAT ISN'T IN THE RIGHT PLACE! {target_file=}") + logger.warn(f'DATA FOLLOWS\n{data}') + return + + with output_lock, target_file.open('a') as f: + f.write(to_save + "\n\n") + + +def safe_file_name(name: str): + """ + Escape special characters out of a file name. + + This is a nicety. Don't rely on it to be ultra secure. + """ + return name.translate(SAFE_TRANSLATE) + + +def generate_inara_response(raw_data: str) -> str: + """Generate nonstatic data for inara plugin.""" + try: + data = json.loads(raw_data) + except json.JSONDecodeError: + return "UNKNOWN REQUEST" + + out = { + 'header': { + 'eventStatus': 200 + }, + + 'events': [ + { + 'eventName': e['eventName'], 'eventStatus': 200, 'eventStatusText': "DEBUG STUFF" + } for e in data.get('events') + ] + } + + return json.dumps(out) + + +def extract_edsm_data(data: str) -> dict[str, Any]: + res = parse_qs(data) + return {name: data[0] for name, data in res.items()} + + +def generate_edsm_response(raw_data: str) -> str: + """Generate nonstatic data for edsm plugin.""" + try: + data = extract_edsm_data(raw_data) + events = json.loads(data['message']) + except (json.JSONDecodeError, Exception): + logger.exception("????") + return "UNKNOWN REQUEST" + + out = { + 'msgnum': 100, # Ok + 'msg': 'debug stuff', + 'events': [ + {'event': e['event'], 'msgnum': 100, 'msg': 'debug stuff'} for e in events + ] + } + + return json.dumps(out) + + +DEFAULT_RESPONSES = { + 'inara': generate_inara_response, + 'edsm': generate_edsm_response +} + + +def run_listener(host: str = "127.0.0.1", port: int = 9090) -> None: + """Run a listener thread.""" + output_data_path.mkdir(exist_ok=True) + logger.info(f'Starting HTTP listener on {host=} {port=}!') + listener = server.HTTPServer((host, port), LoggingHandler) + logger.info(listener) + threading.Thread(target=listener.serve_forever, daemon=True).start() + + +if __name__ == "__main__": + output_data_path.mkdir(exist_ok=True) + server.HTTPServer(("127.0.0.1", 9090), LoggingHandler).serve_forever() diff --git a/eddnListener.py b/eddnListener.py deleted file mode 100644 index 48a92914..00000000 --- a/eddnListener.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Simple HTTP listener to be used with debugging EDDN sends.""" -import pathlib -import tempfile -import threading -from http import server -from typing import Any, Tuple - -from config import appname -from EDMCLogging import get_main_logger - -logger = get_main_logger() - - -class LoggingHandler(server.BaseHTTPRequestHandler): - """HTTP Handler implementation that logs to EDMCs logger.""" - - def __init__(self, request: bytes, client_address: Tuple[str, int], server) -> None: - super().__init__(request, client_address, server) - self.output_lock = threading.Lock() - self.output_file_path = pathlib.Path(tempfile.gettempdir()) / f'{appname}' / 'eddn-listener.jsonl' - self.output_file = self.output_file_path.open('w') - - def log_message(self, format: str, *args: Any) -> None: - """Override default handler logger with EDMC logger.""" - logger.info(format % args) - - def do_POST(self) -> None: # noqa: N802 # I cant change it - """Handle POST.""" - logger.info("Received a POST!") - data = self.rfile.read(int(self.headers['Content-Length'])) - - with self.output_lock: - self.output_file.write(data.decode('utf-8', errors='replace')) - - -def run_listener(port: int = 9090) -> None: - """Run a listener thread.""" - logger.info('Starting HTTP listener on 127.0.0.1:{port}!') - listener = server.HTTPServer(("127.0.0.1", port), LoggingHandler) - threading.Thread(target=listener.serve_forever, daemon=True) - - -if __name__ == "__main__": - server.HTTPServer(("127.0.0.1", 8080), LoggingHandler).serve_forever() diff --git a/edmc_data.py b/edmc_data.py index 473e59cb..ff746196 100644 --- a/edmc_data.py +++ b/edmc_data.py @@ -571,4 +571,6 @@ edmc_suit_symbol_localised = { }, } -EDDN_DEBUG_SERVER = False # Are we using a local server for debugging? +# Local webserver for debugging. See implementation in debug_webserver.py +DEBUG_WEBSERVER_HOST = '127.0.0.1' +DEBUG_WEBSERVER_PORT = 9090 diff --git a/plugins/eddn.py b/plugins/eddn.py index d412053d..194a52de 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -21,7 +21,7 @@ import killswitch import myNotebook as nb # noqa: N813 import plug from companion import CAPIData, category_map -from config import applongname, appversion_nobuild, config +from config import applongname, appversion_nobuild, config, debug_senders from EDMCLogging import get_main_logger from monitor import monitor from myNotebook import Frame @@ -87,12 +87,14 @@ HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' class EDDN: """EDDN Data export.""" - + DEBUG = 'eddn' in debug_senders SERVER = 'https://eddn.edcd.io:4430' - if edmc_data.EDDN_DEBUG_SERVER: - SERVER = '127.0.0.1:9090' + if DEBUG: + SERVER = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}' UPLOAD = f'{SERVER}/upload/' + if DEBUG: + UPLOAD = f'{SERVER}/eddn' REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds diff --git a/plugins/edsm.py b/plugins/edsm.py index b5dbf30a..c1f8bf7f 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -22,7 +22,8 @@ import killswitch import myNotebook as nb # noqa: N813 import plug from companion import CAPIData -from config import applongname, appversion, config +from config import applongname, appversion, config, debug_senders +from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT from EDMCLogging import get_main_logger from ttkHyperlinkLabel import HyperlinkLabel @@ -528,7 +529,13 @@ def cmdr_data(data: CAPIData, is_beta: bool) -> None: this.system_link.update_idletasks() +TARGET_URL = 'https://www.edsm.net/api-journal-v1' +if 'edsm' in debug_senders: + TARGET_URL = f'http://{DEBUG_WEBSERVER_HOST}:{DEBUG_WEBSERVER_PORT}/edsm' + # Worker thread + + def worker() -> None: """ Handle uploading events to EDSM API. @@ -623,9 +630,10 @@ def worker() -> None: # logger.trace(f'Overall POST data (elided) is:\n{data_elided}') - r = this.session.post('https://www.edsm.net/api-journal-v1', data=data, timeout=_TIMEOUT) + r = this.session.post(TARGET_URL, data=data, timeout=_TIMEOUT) # logger.trace(f'API response content: {r.content}') r.raise_for_status() + reply = r.json() msg_num = reply['msgnum'] msg = reply['msg'] diff --git a/plugins/inara.py b/plugins/inara.py index a4e33e81..d469b3ba 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -1,6 +1,7 @@ """Inara Sync.""" import json +from logging import debug import threading import time import tkinter as tk @@ -17,8 +18,9 @@ import killswitch import myNotebook as nb # noqa: N813 import plug import timeout_session +import edmc_data from companion import CAPIData -from config import applongname, appversion, config +from config import applongname, appversion, config, debug_senders from EDMCLogging import get_main_logger from ttkHyperlinkLabel import HyperlinkLabel @@ -126,6 +128,9 @@ STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00 TARGET_URL = 'https://inara.cz/inapi/v1/' +DEBUG = 'inara' in debug_senders +if DEBUG: + TARGET_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/inara' def system_url(system_name: str) -> str: