From 2a461a9e465b944b2ef058b116ccc2edb394159c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 6 Jan 2023 09:00:09 +0000 Subject: [PATCH 1/8] eddn-report: Bump all version checks to latest & EDDI report exception * Raised all softwareVersion checks to latest releases. * EDDI has an approachsettlement/1 issue that's been reported. --- scripts/eddn-report-log-errors | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 20a6fd7..9a48b54 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -75,7 +75,7 @@ def process_file(input_file: str) -> None: ################################################################### if matches.group('software_name') == 'EDDiscovery': # https://github.com/EDDiscovery/EDDiscovery/releases/latest - if software_version >= semantic_version.Version.coerce('15.1.4.0'): + if software_version >= semantic_version.Version.coerce('16.0.5.0'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/outfitting/2': err_msg = matches.group('err_msg') if ( @@ -93,17 +93,26 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDDLite': # https://github.com/EDDiscovery/EDDLite/releases/latest - if software_version >= semantic_version.Version.coerce('2.3.0'): + if software_version >= semantic_version.Version.coerce('2.5.0'): print(line) elif matches.group('software_name') == 'EDDI': # https://github.com/EDCD/EDDI/releases/latest - if software_version >= semantic_version.Version.coerce('4.0.1'): - print(line) + if software_version >= semantic_version.Version.coerce('4.0.2-rc1'): + + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': + if matches.group('err_msg').startswith( + 'Failed Validation "[]"' + ): + # Reported on Discord: ]"> + pass + + else: + print(line) elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest - if software_version >= semantic_version.Version.coerce('5.5.0'): + if software_version >= semantic_version.Version.coerce('5.7.0'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': if matches.group('err_msg').startswith( 'Failed Validation "[ Date: Mon, 9 Jan 2023 22:53:31 +0100 Subject: [PATCH 2/8] Update readme ellisions for journal based commodity messages --- schemas/commodity-README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index 220363a..6b95bdd 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -51,6 +51,10 @@ In the list of commodites: Limpets - not purchasable in station market) or a *non-empty*`"legality":` string (not normally traded at this station market). +If the data is sourced from the journal folder: +- Remove the `$` prefix and `_name;` suffix from the `Name` field. +- There will be no `prohibited` and `economies` to send. Exclude these from the message. + #### Item Category Remove not only the `Category_Localised` key:values, but also the `Category` key:value pair from each Item. From 177a5383b9ec02b27db65bdb20917dba7dbb7a2c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 10 Jan 2023 09:00:05 +0000 Subject: [PATCH 3/8] eddn-report: Bump EDDI to 4.2.0, and comment out latest reported issue --- scripts/eddn-report-log-errors | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 9a48b54..7616e4d 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -98,17 +98,17 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDDI': # https://github.com/EDCD/EDDI/releases/latest - if software_version >= semantic_version.Version.coerce('4.0.2-rc1'): + if software_version >= semantic_version.Version.coerce('4.0.2'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': - if matches.group('err_msg').startswith( - 'Failed Validation "[]"' - ): - # Reported on Discord: ]"> - pass + # if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': + # if matches.group('err_msg').startswith( + # 'Failed Validation "[]"' + # ): + # # Reported on Discord: ]"> + # pass - else: - print(line) + # else: + print(line) elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest From ee217d24a2f362bac69b97db5b3f7cec3ffed0a8 Mon Sep 17 00:00:00 2001 From: jixxed Date: Tue, 10 Jan 2023 15:09:42 +0100 Subject: [PATCH 4/8] Improve description --- schemas/commodity-README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index 6b95bdd..6140346 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -53,7 +53,8 @@ In the list of commodites: If the data is sourced from the journal folder: - Remove the `$` prefix and `_name;` suffix from the `Name` field. -- There will be no `prohibited` and `economies` to send. Exclude these from the message. +- As the Journal Market.json doesn't contain `economies` or `prohibited` data, + leave these entirely out of the message. You **MUST NOT** send empty lists. #### Item Category Remove not only the `Category_Localised` key:values, but also the From 10dcdb9a2f3c9a5565ff36f8b084bfa62adee886 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 08:44:01 +0000 Subject: [PATCH 5/8] eddn-report: EDDI 4.0.2 fsssignaldiscovered without StarPos --- scripts/eddn-report-log-errors | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 7616e4d..6f763b0 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -100,15 +100,15 @@ def process_file(input_file: str) -> None: # https://github.com/EDCD/EDDI/releases/latest if software_version >= semantic_version.Version.coerce('4.0.2'): - # if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': - # if matches.group('err_msg').startswith( - # 'Failed Validation "[]"' - # ): - # # Reported on Discord: ]"> - # pass - - # else: - print(line) + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fsssignaldiscovered/1': + if matches.group('err_msg').startswith( + 'Failed Validation "[ + pass + + else: + print(line) elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest From 50cd2b0f3531d81ddcfb968b9adcebe3906aa881 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 08:54:51 +0000 Subject: [PATCH 6/8] eddn-report: EDDI / navroute / missing Route --- scripts/eddn-report-log-errors | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 6f763b0..62e3683 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -107,6 +107,13 @@ def process_file(input_file: str) -> None: # Reported on Discord: pass + elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/navroute/1': + if matches.group('err_msg').startswith( + 'Failed Validation "[ + pass + else: print(line) From d983a6c8065c4e72059d116b3f6a0711b9b9a23a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 19 Jan 2023 15:29:57 +0000 Subject: [PATCH 7/8] Gateway: Expand logging on Accepted or Validation Error --- src/eddn/Gateway.py | 49 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 0524478..186a6f6 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -76,8 +76,14 @@ def extract_message_details(parsed_message): uploader_id = '<>' software_name = '<>' software_version = '<>' + game_build = '<>' + game_version = '<>' schema_ref = '<>' journal_event = '<>' + system_name = '<>' + system_address = '<>' + station_name = '<>' + station_marketid = '<>' if 'header' in parsed_message: if 'uploaderID' in parsed_message['header']: @@ -89,6 +95,12 @@ def extract_message_details(parsed_message): if 'softwareVersion' in parsed_message['header']: software_version = parsed_message['header']['softwareVersion'] + if 'gamebuild' in parsed_message['header']: + game_build = parsed_message['header']['gamebuild'] + + if 'gameversion' in parsed_message['header']: + game_version = parsed_message['header']['gameversion'] + if '$schemaRef' in parsed_message: schema_ref = parsed_message['$schemaRef'] @@ -101,7 +113,26 @@ def extract_message_details(parsed_message): else: journal_event = '-' - return uploader_id, software_name, software_version, schema_ref, journal_event + if 'SystemName' in parsed_message['message']: + system_name = parsed_message['message']['SystemName'] + + elif 'StarSystem' in parsed_message['message']: + system_name = parsed_message['message']['StarSystem'] + + if 'SystemAddress' in parsed_message['message']: + system_address = parsed_message['message']['SystemAddress'] + + if 'StationName' in parsed_message['message']: + station_name = parsed_message['message']['StationName'] + + if 'marketId' in parsed_message['message']: + station_marketid = parsed_message['message']['marketId'] + + elif 'MarketID' in parsed_message['message']: + station_marketid = parsed_message['message']['MarketID'] + + return uploader_id, software_name, software_version, schema_ref, journal_event, game_build, game_version, \ + system_name, system_address, station_name, station_marketid def configure(): # Get the list of transports to bind from settings. This allows us to PUB @@ -218,10 +249,14 @@ def parse_and_error_handle(data): gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) try: - uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) - logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( + uploader_id, software_name, software_version, schema_ref, journal_event, game_build, game_version, \ + system_name, system_address, station_name, station_marketid = extract_message_details(parsed_message) + logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") from %s' % ( request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, + game_build, game_version, + system_name, system_address, + station_name, station_marketid, get_remote_address() )) @@ -233,11 +268,15 @@ def parse_and_error_handle(data): else: try: - uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) - logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( + uploader_id, software_name, software_version, schema_ref, journal_event, game_build, game_version, \ + system_name, system_address, station_name, station_marketid = extract_message_details(parsed_message) + logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") from %s' % ( str(validationResults.messages), request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, + game_build, game_version, + system_name, system_address, + station_name, station_marketid, get_remote_address() )) From 6a514d3bae37a73d3ab86deec42fe6ba37c097b5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 19 Jan 2023 16:01:53 +0000 Subject: [PATCH 8/8] Gateway: Improve the new logging elements * There are different source-keys for much of this. * Have game_version *then* game_build. * Where a message doesn't have a relevant field, set to '-'. --- src/eddn/Gateway.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 186a6f6..b6a8fba 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -76,8 +76,8 @@ def extract_message_details(parsed_message): uploader_id = '<>' software_name = '<>' software_version = '<>' - game_build = '<>' game_version = '<>' + game_build = '<>' schema_ref = '<>' journal_event = '<>' system_name = '<>' @@ -95,12 +95,12 @@ def extract_message_details(parsed_message): if 'softwareVersion' in parsed_message['header']: software_version = parsed_message['header']['softwareVersion'] - if 'gamebuild' in parsed_message['header']: - game_build = parsed_message['header']['gamebuild'] - if 'gameversion' in parsed_message['header']: game_version = parsed_message['header']['gameversion'] + if 'gamebuild' in parsed_message['header']: + game_build = parsed_message['header']['gamebuild'] + if '$schemaRef' in parsed_message: schema_ref = parsed_message['$schemaRef'] @@ -113,17 +113,32 @@ def extract_message_details(parsed_message): else: journal_event = '-' - if 'SystemName' in parsed_message['message']: + if 'systemName' in parsed_message['message']: + system_name = parsed_message['message']['systemName'] + + elif 'SystemName' in parsed_message['message']: system_name = parsed_message['message']['SystemName'] elif 'StarSystem' in parsed_message['message']: system_name = parsed_message['message']['StarSystem'] + else: + system_name = '-' + if 'SystemAddress' in parsed_message['message']: system_address = parsed_message['message']['SystemAddress'] - if 'StationName' in parsed_message['message']: + else: + system_address = '-' + + if 'stationName' in parsed_message['message']: + station_name = parsed_message['message']['stationName'] + + elif 'StationName' in parsed_message['message']: station_name = parsed_message['message']['StationName'] + + else: + station_name = '-' if 'marketId' in parsed_message['message']: station_marketid = parsed_message['message']['marketId'] @@ -131,7 +146,10 @@ def extract_message_details(parsed_message): elif 'MarketID' in parsed_message['message']: station_marketid = parsed_message['message']['MarketID'] - return uploader_id, software_name, software_version, schema_ref, journal_event, game_build, game_version, \ + else: + station_marketid = '-' + + return uploader_id, software_name, software_version, schema_ref, journal_event, game_version, game_build, \ system_name, system_address, station_name, station_marketid def configure(): @@ -249,12 +267,12 @@ def parse_and_error_handle(data): gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) try: - uploader_id, software_name, software_version, schema_ref, journal_event, game_build, game_version, \ + uploader_id, software_name, software_version, schema_ref, journal_event, game_version, game_build, \ system_name, system_address, station_name, station_marketid = extract_message_details(parsed_message) logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") from %s' % ( request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, - game_build, game_version, + game_version, game_build, system_name, system_address, station_name, station_marketid, get_remote_address() @@ -268,13 +286,13 @@ def parse_and_error_handle(data): else: try: - uploader_id, software_name, software_version, schema_ref, journal_event, game_build, game_version, \ + uploader_id, software_name, software_version, schema_ref, journal_event, game_version, game_build, \ system_name, system_address, station_name, station_marketid = extract_message_details(parsed_message) logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") from %s' % ( str(validationResults.messages), request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, - game_build, game_version, + game_version, game_build, system_name, system_address, station_name, station_marketid, get_remote_address()