Merge pull request #1093 from A-UNDERSCORE-D/enhancement/1064-fake-eddn-listener

Added debugging tools for POST based data senders

EDMarketConnector.py

@@ -93,6 +93,12 @@ if __name__ == '__main__':  # noqa: C901
                         action='store_true'
                         )
 
+    parser.add_argument(
+        '--debug-sender',
+        help='Mark the selected sender as in debug mode. This generally results in data being written to disk',
+        action='append',
+    )
+
     auth_options = parser.add_mutually_exclusive_group(required=False)
     auth_options.add_argument('--force-localserver-for-auth',
                               help='Force EDMC to use a localhost webserver for Frontier Auth callback',
@@ -129,6 +135,17 @@ if __name__ == '__main__':  # noqa: C901
             parser.print_help()
             exit(1)
 
+    if args.debug_sender and len(args.debug_sender) > 0:
+        import config as conf_module
+        import debug_webserver
+        from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT
+
+        conf_module.debug_senders = [x.casefold() for x in args.debug_sender]  # duplicate the list just in case
+        for d in conf_module.debug_senders:
+            logger.info(f'marked {d} for debug')
+
+        debug_webserver.run_listener(DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT)
+
     def handle_edmc_callback_or_foregrounding() -> None:  # noqa: CCR001
         """Handle any edmc:// auth callback, else foreground existing window."""
         logger.trace('Begin...')

PLUGINS.md

@@ -603,7 +603,7 @@ Content of `state` (updated to the current journal entry):
 | `Data`               |           `dict`            | 'Data' MicroResources in Odyssey, `int` count each.                                                             |
 | `BackPack`           |           `dict`            | `dict` of Odyssey MicroResources in backpack.                                                                   |
 | `BackpackJSON`       |           `dict`            | Content of Backpack.json as of last read.                                                                       |
-| `ShipLockerJSON`     |           `dict`            | Content of ShipLocker.json as of last read.                                                                      |
+| `ShipLockerJSON`     |           `dict`            | Content of ShipLocker.json as of last read.                                                                     |
 | `SuitCurrent`        |           `dict`            | CAPI-returned data of currently worn suit.  NB: May be `None` if no data.                                       |
 | `Suits`              |          `dict`[1]          | CAPI-returned data of owned suits.  NB: May be `None` if no data.                                               |
 | `SuitLoadoutCurrent` |           `dict`            | CAPI-returned data of current Suit Loadout.  NB: May be `None` if no data.                                      |
@@ -1043,6 +1043,39 @@ step for the extra module(s) until it works.
 
 ---
 
+## Debug HTTP POST requests
+
+You can debug your http post requests using the builtin debug webserver.
+
+To add support for said debug webserver to your plugin, you need to check `config.debug_senders` (`list[str]`) for
+some indicator string for your plugin. `debug_senders` is generated from args to `--debug-sender` on the invocation
+command line.
+
+If said string exists, `DEBUG_WEBSERVER_HOST` and `DEBUG_WEBSERVER_PORT` in
+`edmc_data` will contain the host and port for the currently running local webserver. Simply redirect your requests
+there, and your requests will be logged to disk. For organisation, rewrite your request path to simply be `/pluginname`.
+
+Logs exist in `$TEMP/EDMarketConnector/http_debug/$path.log`. If somehow you manage to cause a directory traversal, your
+data will not be saved to disk at all. You will see this in EDMCs log.
+
+The simplest way to go about adding support is:
+
+```py
+from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT
+from config import debug_senders
+
+TARGET_URL = "https://host.tld/path/to/api/magic"
+if 'my_plugin' in debug_senders:
+    TARGET_URL = f'http://{DEBUG_WEBSERVER_HOST}:{DEBUG_WEBSERVER_PORT}/my_plugin'
+
+# Code that uses TARGET_URL to post info from your plugin below.
+
+```
+
+For returned data, you can modify `debug_webserver.DEFAULT_RESPONSES` (`dict[str, Union[Callable[[str], str]], str])`
+with either a function that accepts a single string (the raw post data) and returns a single string
+(the response to send), or with a string if your required response is simple.
+
 ## Disable a plugin
 
 EDMC now lets you disable a plugin without deleting it, simply rename the

config.py

@@ -38,6 +38,8 @@ copyright = '© 2015-2019 Jonathan Harris, 2020-2021 EDCD'
 
 update_feed = 'https://raw.githubusercontent.com/EDCD/EDMarketConnector/releases/edmarketconnector.xml'
 update_interval = 8*60*60
+# Providers marked to be in debug mode. Generally this is expected to switch to sending data to a log file
+debug_senders: List[str] = []
 
 # This must be done here in order to avoid an import cycle with EDMCLogging.
 # Other code should use EDMCLogging.get_main_logger

debug_webserver.py

@@ -0,0 +1,147 @@
+"""Simple HTTP listener to be used with debugging various EDMC sends."""
+import json
+import pathlib
+import tempfile
+import threading
+from http import server
+from typing import Any, Callable, Tuple, Union
+from urllib.parse import parse_qs
+
+from config import appname
+from EDMCLogging import get_main_logger
+
+logger = get_main_logger()
+
+output_lock = threading.Lock()
+output_data_path = pathlib.Path(tempfile.gettempdir()) / f'{appname}' / 'http_debug'
+SAFE_TRANSLATE = str.maketrans({x: '_' for x in "!@#$%^&*()./\\\r\n[]-+='\";:?<>,~`"})
+
+
+class LoggingHandler(server.BaseHTTPRequestHandler):
+    """HTTP Handler implementation that logs to EDMCs logger and writes data to files on disk."""
+
+    def __init__(self, request: bytes, client_address: Tuple[str, int], server) -> None:
+        super().__init__(request, client_address, server)
+
+    def log_message(self, format: str, *args: Any) -> None:
+        """Override default handler logger with EDMC logger."""
+        logger.info(format % args)
+
+    def do_POST(self) -> None:  # noqa: N802 # I cant change it
+        """Handle POST."""
+        logger.info(f"Received a POST for {self.path!r}!")
+        data = self.rfile.read(int(self.headers['Content-Length'])).decode('utf-8', errors='replace')
+        to_save = data
+
+        target_path = self.path
+        if len(target_path) > 1 and target_path[0] == '/':
+            target_path = target_path[1:]
+
+        elif len(target_path) == 1 and target_path[0] == '/':
+            target_path = 'WEB_ROOT'
+
+        response: Union[Callable[[str], str], str, None] = DEFAULT_RESPONSES.get(target_path)
+        if callable(response):
+            response = response(data)
+
+        self.send_response_only(200, "OK")
+        if response is not None:
+            self.send_header('Content-Length', str(len(response)))
+
+        self.end_headers()  # This is needed because send_response_only DOESN'T ACTUALLY SEND THE RESPONSE </rant>
+        if response is not None:
+            self.wfile.write(response.encode())
+            self.wfile.flush()
+
+        if target_path == 'edsm':
+            # attempt to extract data from urlencoded stream
+            try:
+                edsm_data = extract_edsm_data(data)
+                data = data + "\n" + json.dumps(edsm_data)
+            except Exception:
+                pass
+
+        target_file = output_data_path / (safe_file_name(target_path) + '.log')
+        if target_file.parent != output_data_path:
+            logger.warning(f"REFUSING TO WRITE FILE THAT ISN'T IN THE RIGHT PLACE! {target_file=}")
+            logger.warning(f'DATA FOLLOWS\n{data}')
+            return
+
+        with output_lock, target_file.open('a') as f:
+            f.write(to_save + "\n\n")
+
+
+def safe_file_name(name: str):
+    """
+    Escape special characters out of a file name.
+
+    This is a nicety. Don't rely on it to be ultra secure.
+    """
+    return name.translate(SAFE_TRANSLATE)
+
+
+def generate_inara_response(raw_data: str) -> str:
+    """Generate nonstatic data for inara plugin."""
+    try:
+        data = json.loads(raw_data)
+    except json.JSONDecodeError:
+        return "UNKNOWN REQUEST"
+
+    out = {
+        'header': {
+            'eventStatus': 200
+        },
+
+        'events': [
+            {
+                'eventName': e['eventName'], 'eventStatus': 200, 'eventStatusText': "DEBUG STUFF"
+            } for e in data.get('events')
+        ]
+    }
+
+    return json.dumps(out)
+
+
+def extract_edsm_data(data: str) -> dict[str, Any]:
+    res = parse_qs(data)
+    return {name: data[0] for name, data in res.items()}
+
+
+def generate_edsm_response(raw_data: str) -> str:
+    """Generate nonstatic data for edsm plugin."""
+    try:
+        data = extract_edsm_data(raw_data)
+        events = json.loads(data['message'])
+    except (json.JSONDecodeError, Exception):
+        logger.exception("????")
+        return "UNKNOWN REQUEST"
+
+    out = {
+        'msgnum': 100,  # Ok
+        'msg': 'debug stuff',
+        'events': [
+            {'event': e['event'], 'msgnum': 100, 'msg': 'debug stuff'} for e in events
+        ]
+    }
+
+    return json.dumps(out)
+
+
+DEFAULT_RESPONSES = {
+    'inara': generate_inara_response,
+    'edsm': generate_edsm_response
+}
+
+
+def run_listener(host: str = "127.0.0.1", port: int = 9090) -> None:
+    """Run a listener thread."""
+    output_data_path.mkdir(exist_ok=True)
+    logger.info(f'Starting HTTP listener on {host=} {port=}!')
+    listener = server.HTTPServer((host, port), LoggingHandler)
+    logger.info(listener)
+    threading.Thread(target=listener.serve_forever, daemon=True).start()
+
+
+if __name__ == "__main__":
+    output_data_path.mkdir(exist_ok=True)
+    server.HTTPServer(("127.0.0.1", 9090), LoggingHandler).serve_forever()

edmc_data.py

@@ -570,3 +570,7 @@ edmc_suit_symbol_localised = {
         'utilitysuit':     'Traje Maverick',
     },
 }
+
+# Local webserver for debugging. See implementation in debug_webserver.py
+DEBUG_WEBSERVER_HOST = '127.0.0.1'
+DEBUG_WEBSERVER_PORT = 9090

plugins/eddn.py

@@ -16,11 +16,12 @@ from typing import TextIO, Tuple
 
 import requests
 
+import edmc_data
 import killswitch
 import myNotebook as nb  # noqa: N813
 import plug
 from companion import CAPIData, category_map
-from config import applongname, appversion_nobuild, config
+from config import applongname, appversion_nobuild, config, debug_senders
 from EDMCLogging import get_main_logger
 from monitor import monitor
 from myNotebook import Frame
@@ -86,10 +87,15 @@ HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'
 
 class EDDN:
     """EDDN Data export."""
-
-    # SERVER = 'http://localhost:8081'	# testing
+    DEBUG = 'eddn' in debug_senders
     SERVER = 'https://eddn.edcd.io:4430'
+    if DEBUG:
+        SERVER = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}'
+
     UPLOAD = f'{SERVER}/upload/'
+    if DEBUG:
+        UPLOAD = f'{SERVER}/eddn'
+
     REPLAYPERIOD = 400  # Roughly two messages per second, accounting for send delays [ms]
     REPLAYFLUSH = 20  # Update log on disk roughly every 10 seconds
     TIMEOUT = 10  # requests timeout

plugins/edsm.py

@@ -22,7 +22,8 @@ import killswitch
 import myNotebook as nb  # noqa: N813
 import plug
 from companion import CAPIData
-from config import applongname, appversion, config
+from config import applongname, appversion, config, debug_senders
+from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT
 from EDMCLogging import get_main_logger
 from ttkHyperlinkLabel import HyperlinkLabel
 
@@ -528,8 +529,14 @@ def cmdr_data(data: CAPIData, is_beta: bool) -> None:
         this.system_link.update_idletasks()
 
 
+TARGET_URL = 'https://www.edsm.net/api-journal-v1'
+if 'edsm' in debug_senders:
+    TARGET_URL = f'http://{DEBUG_WEBSERVER_HOST}:{DEBUG_WEBSERVER_PORT}/edsm'
+
 # Worker thread
-def worker() -> None:
+
+
+def worker() -> None:  # noqa: CCR001 C901 # Cant be broken up currently
     """
     Handle uploading events to EDSM API.
 
@@ -623,9 +630,10 @@ def worker() -> None:
 
                     #     logger.trace(f'Overall POST data (elided) is:\n{data_elided}')
 
-                    r = this.session.post('https://www.edsm.net/api-journal-v1', data=data, timeout=_TIMEOUT)
+                    r = this.session.post(TARGET_URL, data=data, timeout=_TIMEOUT)
                     # logger.trace(f'API response content: {r.content}')
                     r.raise_for_status()
+
                     reply = r.json()
                     msg_num = reply['msgnum']
                     msg = reply['msg']

plugins/inara.py

@@ -13,12 +13,13 @@ from typing import Sequence, Union, cast
 
 import requests
 
+import edmc_data
 import killswitch
 import myNotebook as nb  # noqa: N813
 import plug
 import timeout_session
 from companion import CAPIData
-from config import applongname, appversion, config
+from config import applongname, appversion, config, debug_senders
 from EDMCLogging import get_main_logger
 from ttkHyperlinkLabel import HyperlinkLabel
 
@@ -126,6 +127,9 @@ STATION_UNDOCKED: str = '×'  # "Station" name to display when not docked = U+00
 
 
 TARGET_URL = 'https://inara.cz/inapi/v1/'
+DEBUG = 'inara' in debug_senders
+if DEBUG:
+    TARGET_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/inara'
 
 
 def system_url(system_name: str) -> str: