From f318b3256b8ccd8b8c1267b770409b38f68dfa7e Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Tue, 26 Dec 2023 23:02:39 -0500 Subject: [PATCH 1/4] [662] Apply Rate Limit --- plugins/edsm.py | 132 +++++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 47 deletions(-) diff --git a/plugins/edsm.py b/plugins/edsm.py index 754ffb61..46d05f4c 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -28,7 +28,7 @@ from queue import Queue from threading import Thread from time import sleep from tkinter import ttk -from typing import TYPE_CHECKING, Any, Literal, Mapping, MutableMapping, cast +from typing import TYPE_CHECKING, Any, Literal, Mapping, MutableMapping, cast, Sequence import requests import killswitch import monitor @@ -722,6 +722,87 @@ def get_discarded_events_list() -> None: logger.warning('Exception while trying to set this.discarded_events:', exc_info=e) +def process_discarded_events() -> None: + """Process discarded events until the discarded events list is retrieved or the shutdown signal is received.""" + while not this.discarded_events: + if this.shutting_down: + logger.debug(f'returning from discarded_events loop due to {this.shutting_down=}') + return + get_discarded_events_list() + if this.discarded_events: + break + sleep(DISCARDED_EVENTS_SLEEP) + + logger.debug('Got "events to discard" list, commencing queue consumption...') + + +def send_to_edsm( # noqa: CCR001 + data: dict[str, Sequence[object]], pending: list[Mapping[str, Any]], closing: bool +) -> list[Mapping[str, Any]]: + """Send data to the EDSM API endpoint and handle the API response.""" + response = this.session.post(TARGET_URL, data=data, timeout=_TIMEOUT) + logger.trace_if('plugin.edsm.api', f'API response content: {response.content!r}') + + # Check for rate limit headers + rate_limit_remaining = response.headers.get('X-Rate-Limit-Remaining') + rate_limit_reset = response.headers.get('X-Rate-Limit-Reset') + + # Convert headers to integers if they exist + try: + remaining = int(rate_limit_remaining) if rate_limit_remaining else None + reset = int(rate_limit_reset) if rate_limit_reset else None + except ValueError: + remaining = reset = None + + if remaining is not None and reset is not None: + # Respect rate limits if they exist + if remaining == 0: + # Calculate sleep time until the rate limit reset time + reset_time = datetime.utcfromtimestamp(reset) + current_time = datetime.utcnow() + + sleep_time = (reset_time - current_time).total_seconds() + + if sleep_time > 0: + sleep(sleep_time) + + response.raise_for_status() + reply = response.json() + msg_num = reply['msgnum'] + msg = reply['msg'] + # 1xx = OK + # 2xx = fatal error + # 3&4xx not generated at top-level + # 5xx = error but events saved for later processing + + if msg_num // 100 == 2: + logger.warning(f'EDSM\t{msg_num} {msg}\t{json.dumps(pending, separators=(",", ": "))}') + # LANG: EDSM Plugin - Error message from EDSM API + plug.show_error(_('Error: EDSM {MSG}').format(MSG=msg)) + else: + if msg_num // 100 == 1: + logger.trace_if('plugin.edsm.api', 'Overall OK') + pass + elif msg_num // 100 == 5: + logger.trace_if('plugin.edsm.api', 'Event(s) not currently processed, but saved for later') + pass + else: + logger.warning(f'EDSM API call status not 1XX, 2XX or 5XX: {msg.num}') + + for e, r in zip(pending, reply['events']): + if not closing and e['event'] in ('StartUp', 'Location', 'FSDJump', 'CarrierJump'): + # Update main window's system status + this.lastlookup = r + # calls update_status in main thread + if not config.shutting_down and this.system_link is not None: + this.system_link.event_generate('<>', when="tail") + if r['msgnum'] // 100 != 1: + logger.warning(f'EDSM event with not-1xx status:\n{r["msgnum"]}\n' + f'{r["msg"]}\n{json.dumps(e, separators=(",", ": "))}') + pending = [] + return pending + + def worker() -> None: # noqa: CCR001 C901 """ Handle uploading events to EDSM API. @@ -738,17 +819,9 @@ def worker() -> None: # noqa: CCR001 C901 last_game_version = "" last_game_build = "" - while not this.discarded_events: - if this.shutting_down: - logger.debug(f'returning from discarded_events loop due to {this.shutting_down=}') - return - get_discarded_events_list() - if this.discarded_events: - break + # Process the Discard Queue + process_discarded_events() - sleep(DISCARDED_EVENTS_SLEEP) - - logger.debug('Got "events to discard" list, commencing queue consumption...') while True: if this.shutting_down: logger.debug(f'{this.shutting_down=}, so setting closing = True') @@ -861,43 +934,8 @@ def worker() -> None: # noqa: CCR001 C901 'journal.locations', f'Overall POST data (elided) is:\n{json.dumps(data_elided, indent=2)}' ) - response = this.session.post(TARGET_URL, data=data, timeout=_TIMEOUT) - logger.trace_if('plugin.edsm.api', f'API response content: {response.content!r}') - response.raise_for_status() + pending = send_to_edsm(data, pending, closing) - reply = response.json() - msg_num = reply['msgnum'] - msg = reply['msg'] - # 1xx = OK - # 2xx = fatal error - # 3&4xx not generated at top-level - # 5xx = error but events saved for later processing - - if msg_num // 100 == 2: - logger.warning(f'EDSM\t{msg_num} {msg}\t{json.dumps(pending, separators=(",", ": "))}') - # LANG: EDSM Plugin - Error message from EDSM API - plug.show_error(_('Error: EDSM {MSG}').format(MSG=msg)) - else: - if msg_num // 100 == 1: - logger.trace_if('plugin.edsm.api', 'Overall OK') - pass - elif msg_num // 100 == 5: - logger.trace_if('plugin.edsm.api', 'Event(s) not currently processed, but saved for later') - pass - else: - logger.warning(f'EDSM API call status not 1XX, 2XX or 5XX: {msg.num}') - - for e, r in zip(pending, reply['events']): - if not closing and e['event'] in ('StartUp', 'Location', 'FSDJump', 'CarrierJump'): - # Update main window's system status - this.lastlookup = r - # calls update_status in main thread - if not config.shutting_down and this.system_link is not None: - this.system_link.event_generate('<>', when="tail") - if r['msgnum'] // 100 != 1: - logger.warning(f'EDSM event with not-1xx status:\n{r["msgnum"]}\n' - f'{r["msg"]}\n{json.dumps(e, separators = (",", ": "))}') - pending = [] break # No exception, so assume success except Exception as e: From 48311ae70c5f82381dfe62181d14dd6cc3d9bd5c Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 4 Jan 2024 19:31:59 -0500 Subject: [PATCH 2/4] [Minor] Bump Version String for internal --- config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/__init__.py b/config/__init__.py index 5dfb0d24..7d4aa43f 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -54,7 +54,7 @@ appcmdname = 'EDMC' # # Major.Minor.Patch(-prerelease)(+buildmetadata) # NB: Do *not* import this, use the functions appversion() and appversion_nobuild() -_static_appversion = '5.10.1' +_static_appversion = '5.10.2-alpha0' _cached_version: semantic_version.Version | None = None copyright = '© 2015-2019 Jonathan Harris, 2020-2024 EDCD' From 29c4bd402804899974fdb57b739c4b5d57caf39d Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Fri, 5 Jan 2024 15:35:25 -0500 Subject: [PATCH 3/4] [Minor] Update Type Hint --- EDMC.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/EDMC.py b/EDMC.py index a237f4ea..39d98982 100755 --- a/EDMC.py +++ b/EDMC.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# flake8: noqa TAE001 """ EDMC.py - Command-line interface. Requires prior setup through the GUI. @@ -27,7 +26,7 @@ from EDMCLogging import edmclogger, logger, logging if TYPE_CHECKING: from logging import TRACE # type: ignore # noqa: F401 # needed to make mypy happy - def _(x): return x + def _(x: str): return x edmclogger.set_channels_loglevel(logging.INFO) From 0b90ca770885515097a62ecc4fdb9d6c3bd24bd0 Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Fri, 5 Jan 2024 15:39:26 -0500 Subject: [PATCH 4/4] [Minor] More Type Hints --- companion.py | 2 +- update.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/companion.py b/companion.py index 4a827769..d0030748 100644 --- a/companion.py +++ b/companion.py @@ -41,7 +41,7 @@ from monitor import monitor logger = get_main_logger() if TYPE_CHECKING: - def _(x): return x + def _(x: str): return x UserDict = collections.UserDict[str, Any] # indicate to our type checkers what this generic class holds normally else: diff --git a/update.py b/update.py index 5e780a37..024aeb09 100644 --- a/update.py +++ b/update.py @@ -21,7 +21,7 @@ from EDMCLogging import get_main_logger if TYPE_CHECKING: import tkinter as tk - def _(x): return x + def _(x: str): return x logger = get_main_logger()