ED/EDMarketConnector
1
0
Fork 0
mirror of https://github.com/EDCD/EDMarketConnector.git synced 2025-04-13 15:57:14 +03:00
Code Issues Projects Releases Wiki Activity

Added support for arbitrary plugins for POST debug

Browse Source 
This works by replacing --eddn-local with --debug-sender, and making the
webserver more generic. support has been added to EDSM, EDDN, and INARA.
This commit is contained in:
A_D 2021-05-25 11:06:48 +02:00
parent 57d7889da9
commit fe0e752c9b
No known key found for this signature in database
GPG Key ID: 4BE9EB7DF45076C4
9 changed files with 218 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
EDMarketConnector.py
View File

@ -93,7 +93,11 @@ if __name__ == '__main__': # noqa: C901
action='store_true'
)
parser.add_argument('--eddn-local', help='Redirect EDDN requests to a local webserver', action='store_true')
parser.add_argument(
'--debug-sender',
help='Mark the selected sender as in debug mode. This generally results in data being written to disk',
action='append',
)
auth_options = parser.add_mutually_exclusive_group(required=False)
auth_options.add_argument('--force-localserver-for-auth',
@ -131,11 +135,16 @@ if __name__ == '__main__': # noqa: C901
parser.print_help()
exit(1)
if args.eddn_local:
import eddnListener
import edmc_data
eddnListener.run_listener()
edmc_data.EDDN_DEBUG_SERVER = True
if args.debug_sender and len(args.debug_sender) > 0:
import config as conf_module
import debug_webserver
from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT
conf_module.debug_senders = [x.casefold() for x in args.debug_sender] # duplicate the list just in case
for d in conf_module.debug_senders:
logger.info(f'marked {d} for debug')
debug_webserver.run_listener(DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT)
def handle_edmc_callback_or_foregrounding() -> None: # noqa: CCR001
"""Handle any edmc:// auth callback, else foreground existing window."""

31
PLUGINS.md
View File

@ -603,7 +603,7 @@ Content of `state` (updated to the current journal entry):
| `Data` | `dict` | 'Data' MicroResources in Odyssey, `int` count each. |
| `BackPack` | `dict` | `dict` of Odyssey MicroResources in backpack. |
| `BackpackJSON` | `dict` | Content of Backpack.json as of last read. |
| `ShipLockerJSON` | `dict` | Content of ShipLocker.json as of last read. |
| `ShipLockerJSON` | `dict` | Content of ShipLocker.json as of last read. |
| `SuitCurrent` | `dict` | CAPI-returned data of currently worn suit. NB: May be `None` if no data. |
| `Suits` | `dict`[1] | CAPI-returned data of owned suits. NB: May be `None` if no data. |
| `SuitLoadoutCurrent` | `dict` | CAPI-returned data of current Suit Loadout. NB: May be `None` if no data. |
@ -1043,6 +1043,35 @@ step for the extra module(s) until it works.
---
## Debug HTTP POST requests
You can debug your http post requests using the builtin debug webserver.
To add support for said debug webserver to your plugin, you need to check `config.debug_senders` (`list[str]`) for
some indicator string for your plugin. `debug_senders` is generated from args to `--debug-sender` on the invocation
command line.
If said string exists, `DEBUG_WEBSERVER_HOST` and `DEBUG_WEBSERVER_PORT` in
`edmc_data` will contain the host and port for the currently running local webserver. Simply redirect your requests
there, and your requests will be logged to disk. For organisation, rewrite your request path to simply be `/pluginname`.
logs exist in `$TEMP/EDMarketConnector/http_debug/$path.log`. If somehow you manage to cause a directory traversal, your
data will not be saved to disk at all. You will see this in EDMCs log.
The simplest way to go about adding support is:
```py
from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT
from config import debug_senders
TARGET_URL = "https://host.tld/path/to/api/magic"
if 'my_plugin' in debug_senders:
TARGET_URL = f'http://{DEBUG_WEBSERVER_HOST}:{DEBUG_WEBSERVER_PORT}/my_plugin'
# Code that uses TARGET_URL to post info from your plugin below.
```
## Disable a plugin
EDMC now lets you disable a plugin without deleting it, simply rename the

2
config.py
View File

@ -38,6 +38,8 @@ copyright = '© 2015-2019 Jonathan Harris, 2020-2021 EDCD'
update_feed = 'https://raw.githubusercontent.com/EDCD/EDMarketConnector/releases/edmarketconnector.xml'
update_interval = 8*60*60
# Providers marked to be in debug mode. Generally this is expected to switch to sending data to a log file
debug_senders: List[str] = []
# This must be done here in order to avoid an import cycle with EDMCLogging.
# Other code should use EDMCLogging.get_main_logger

146
debug_webserver.py Normal file
View File

@ -0,0 +1,146 @@
"""Simple HTTP listener to be used with debugging various EDMC sends."""
import json
import pathlib
import tempfile
import threading
from http import server
from typing import Any, Callable, Tuple, Union
from urllib.parse import parse_qs
from config import appname
from EDMCLogging import get_main_logger
logger = get_main_logger()
output_lock = threading.Lock()
output_data_path = pathlib.Path(tempfile.gettempdir()) / f'{appname}' / 'http_debug'
SAFE_TRANSLATE = str.maketrans({x: '_' for x in "!@#$%^&*()./\\\r\n[]-+='\";:?<>,~`"})
class LoggingHandler(server.BaseHTTPRequestHandler):
"""HTTP Handler implementation that logs to EDMCs logger and writes data to files on disk."""
def __init__(self, request: bytes, client_address: Tuple[str, int], server) -> None:
super().__init__(request, client_address, server)
def log_message(self, format: str, *args: Any) -> None:
"""Override default handler logger with EDMC logger."""
logger.info(format % args)
def do_POST(self) -> None: # noqa: N802 # I cant change it
"""Handle POST."""
logger.info(f"Received a POST for {self.path!r}!")
data = self.rfile.read(int(self.headers['Content-Length'])).decode('utf-8', errors='replace')
to_save = data
target_path = self.path
if len(target_path) > 1 and target_path[0] == '/':
target_path = target_path[1:]
elif len(target_path) == 1 and target_path[0] == '/':
target_path = 'WEB_ROOT'
response: Union[Callable[[str], str], str, None] = DEFAULT_RESPONSES.get(target_path)
if callable(response):
response = response(data)
self.send_response_only(200, "OK")
if response is not None:
self.send_header('Content-Length', str(len(response)))
self.end_headers() # This is needed because send_response_only DOESN'T ACTUALLY SEND THE RESPONSE </rant>
if response is not None:
self.wfile.write(response.encode())
self.wfile.flush()
if target_path == 'edsm':
# attempt to extract data from urlencoded stream
try:
edsm_data = extract_edsm_data(data)
data = data + "\n" + json.dumps(edsm_data)
except Exception:
pass
target_file = output_data_path / (safe_file_name(target_path) + '.log')
if target_file.parent != output_data_path:
logger.warning(f"REFUSING TO WRITE FILE THAT ISN'T IN THE RIGHT PLACE! {target_file=}")
logger.warn(f'DATA FOLLOWS\n{data}')
return
with output_lock, target_file.open('a') as f:
f.write(to_save + "\n\n")
def safe_file_name(name: str):
"""
Escape special characters out of a file name.
This is a nicety. Don't rely on it to be ultra secure.
"""
return name.translate(SAFE_TRANSLATE)
def generate_inara_response(raw_data: str) -> str:
"""Generate nonstatic data for inara plugin."""
try:
data = json.loads(raw_data)
except json.JSONDecodeError:
return "UNKNOWN REQUEST"
out = {
'header': {
'eventStatus': 200
},
'events': [
{
'eventName': e['eventName'], 'eventStatus': 200, 'eventStatusText': "DEBUG STUFF"
} for e in data.get('events')
]
}
return json.dumps(out)
def extract_edsm_data(data: str) -> dict[str, Any]:
res = parse_qs(data)
return {name: data[0] for name, data in res.items()}
def generate_edsm_response(raw_data: str) -> str:
"""Generate nonstatic data for edsm plugin."""
try:
data = extract_edsm_data(raw_data)
events = json.loads(data['message'])
except (json.JSONDecodeError, Exception):
logger.exception("????")
return "UNKNOWN REQUEST"
out = {
'msgnum': 100, # Ok
'msg': 'debug stuff',
'events': [
{'event': e['event'], 'msgnum': 100, 'msg': 'debug stuff'} for e in events
]
}
return json.dumps(out)
DEFAULT_RESPONSES = {
'inara': generate_inara_response,
'edsm': generate_edsm_response
}
def run_listener(host: str = "127.0.0.1", port: int = 9090) -> None:
"""Run a listener thread."""
output_data_path.mkdir(exist_ok=True)
logger.info(f'Starting HTTP listener on {host=} {port=}!')
listener = server.HTTPServer((host, port), LoggingHandler)
logger.info(listener)
threading.Thread(target=listener.serve_forever, daemon=True).start()
if __name__ == "__main__":
output_data_path.mkdir(exist_ok=True)
server.HTTPServer(("127.0.0.1", 9090), LoggingHandler).serve_forever()

44
eddnListener.py
View File

@ -1,44 +0,0 @@
"""Simple HTTP listener to be used with debugging EDDN sends."""
import pathlib
import tempfile
import threading
from http import server
from typing import Any, Tuple
from config import appname
from EDMCLogging import get_main_logger
logger = get_main_logger()
class LoggingHandler(server.BaseHTTPRequestHandler):
"""HTTP Handler implementation that logs to EDMCs logger."""
def __init__(self, request: bytes, client_address: Tuple[str, int], server) -> None:
super().__init__(request, client_address, server)
self.output_lock = threading.Lock()
self.output_file_path = pathlib.Path(tempfile.gettempdir()) / f'{appname}' / 'eddn-listener.jsonl'
self.output_file = self.output_file_path.open('w')
def log_message(self, format: str, *args: Any) -> None:
"""Override default handler logger with EDMC logger."""
logger.info(format % args)
def do_POST(self) -> None: # noqa: N802 # I cant change it
"""Handle POST."""
logger.info("Received a POST!")
data = self.rfile.read(int(self.headers['Content-Length']))
with self.output_lock:
self.output_file.write(data.decode('utf-8', errors='replace'))
def run_listener(port: int = 9090) -> None:
"""Run a listener thread."""
logger.info('Starting HTTP listener on 127.0.0.1:{port}!')
listener = server.HTTPServer(("127.0.0.1", port), LoggingHandler)
threading.Thread(target=listener.serve_forever, daemon=True)
if __name__ == "__main__":
server.HTTPServer(("127.0.0.1", 8080), LoggingHandler).serve_forever()

4
edmc_data.py
View File

@ -571,4 +571,6 @@ edmc_suit_symbol_localised = {
},
}
EDDN_DEBUG_SERVER = False # Are we using a local server for debugging?
# Local webserver for debugging. See implementation in debug_webserver.py
DEBUG_WEBSERVER_HOST = '127.0.0.1'
DEBUG_WEBSERVER_PORT = 9090

10
plugins/eddn.py
View File

@ -21,7 +21,7 @@ import killswitch
import myNotebook as nb # noqa: N813
import plug
from companion import CAPIData, category_map
from config import applongname, appversion_nobuild, config
from config import applongname, appversion_nobuild, config, debug_senders
from EDMCLogging import get_main_logger
from monitor import monitor
from myNotebook import Frame
@ -87,12 +87,14 @@ HORIZ_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS'
class EDDN:
"""EDDN Data export."""
DEBUG = 'eddn' in debug_senders
SERVER = 'https://eddn.edcd.io:4430'
if edmc_data.EDDN_DEBUG_SERVER:
SERVER = '127.0.0.1:9090'
if DEBUG:
SERVER = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}'
UPLOAD = f'{SERVER}/upload/'
if DEBUG:
UPLOAD = f'{SERVER}/eddn'
REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms]
REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds

12
plugins/edsm.py
View File

@ -22,7 +22,8 @@ import killswitch
import myNotebook as nb # noqa: N813
import plug
from companion import CAPIData
from config import applongname, appversion, config
from config import applongname, appversion, config, debug_senders
from edmc_data import DEBUG_WEBSERVER_HOST, DEBUG_WEBSERVER_PORT
from EDMCLogging import get_main_logger
from ttkHyperlinkLabel import HyperlinkLabel
@ -528,7 +529,13 @@ def cmdr_data(data: CAPIData, is_beta: bool) -> None:
this.system_link.update_idletasks()
TARGET_URL = 'https://www.edsm.net/api-journal-v1'
if 'edsm' in debug_senders:
TARGET_URL = f'http://{DEBUG_WEBSERVER_HOST}:{DEBUG_WEBSERVER_PORT}/edsm'
# Worker thread
def worker() -> None:
"""
Handle uploading events to EDSM API.
@ -623,9 +630,10 @@ def worker() -> None:
# logger.trace(f'Overall POST data (elided) is:\n{data_elided}')
r = this.session.post('https://www.edsm.net/api-journal-v1', data=data, timeout=_TIMEOUT)
r = this.session.post(TARGET_URL, data=data, timeout=_TIMEOUT)
# logger.trace(f'API response content: {r.content}')
r.raise_for_status()
reply = r.json()
msg_num = reply['msgnum']
msg = reply['msg']

7
plugins/inara.py
View File

@ -1,6 +1,7 @@
"""Inara Sync."""
import json
from logging import debug
import threading
import time
import tkinter as tk
@ -17,8 +18,9 @@ import killswitch
import myNotebook as nb # noqa: N813
import plug
import timeout_session
import edmc_data
from companion import CAPIData
from config import applongname, appversion, config
from config import applongname, appversion, config, debug_senders
from EDMCLogging import get_main_logger
from ttkHyperlinkLabel import HyperlinkLabel
@ -126,6 +128,9 @@ STATION_UNDOCKED: str = '×' # "Station" name to display when not docked = U+00
TARGET_URL = 'https://inara.cz/inapi/v1/'
DEBUG = 'inara' in debug_senders
if DEBUG:
TARGET_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/inara'
def system_url(system_name: str) -> str: