From 05d5d0215ae141a2773d854a64b819c500b8b4ec Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 17 Feb 2022 11:06:01 +0000 Subject: [PATCH 1/4] scripts/eddn-report-log-errors: Use a semantic_version comparison Hardcoding to only **equal** to 'current known latest version' requires maintenance on any new release of a Sender. So leverage `semantic_version.Version.coerce('')` instead so we catch "this or later versions". --- scripts/eddn-report-log-errors | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 73307fe..ffa1c83 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -5,6 +5,8 @@ import argparse import re +import semantic_version + def parse_cl_args() -> str: """ @@ -59,13 +61,14 @@ def process_file(input_file: str) -> None: # print(matches.group('sender_ip')) # print('') + software_version = semantic_version.Version.coerce(matches.group('software_version')) ################################################################### # Issues we know about and HAVE already alerted their # developers to. ################################################################### if matches.group('software_name') == 'EDDiscovery': # https://github.com/EDDiscovery/EDDiscovery/releases/latest - if matches.group('software_version') == '12.1.7.0': + if software_version >= semantic_version.Version.coerce('12.1.7.0'): if matches.group('schema_ref') in ( 'https://eddn.edcd.io/schemas/shipyard/2', 'https://eddn.edcd.io/schemas/outfitting/2', @@ -79,7 +82,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDDLite': # https://github.com/EDDiscovery/EDDLite/releases/tag/latest - if matches.group('software_version') == '2.0.0': + if software_version >= semantic_version.Version.coerce('2.0.0'): if matches.group('schema_ref') in ( 'https://eddn.edcd.io/schemas/shipyard/2', 'https://eddn.edcd.io/schemas/outfitting/2', @@ -93,12 +96,12 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDDI': # https://github.com/EDCD/EDDI/releases/latest - if matches.group('software_version') == '4.0.1': + if software_version >= semantic_version.Version.coerce('4.0.1'): print(line) - elif matches.group('software_name') == 'E:D Market Connector [Windows]': + elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest - if matches.group('software_version') == '5.2.4': + if software_version >= semantic_version.Version.coerce('5.2.4'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/codexentry/1': # if matches.group('err_msg') != 'Failed Validation "[]"': @@ -117,6 +120,10 @@ def process_file(input_file: str) -> None: # pass + else: + print(matches.group('err_msg')) + print(line) + elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fssdiscoveryscan/1': if matches.group('err_msg') == 'Failed Validation "[]"': # @@ -127,7 +134,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'Elite G19s Companion App': # - if matches.group('software_version') == '3.7.7888.21039': + if software_version >= semantic_version.Version.coerce('3.7.7888.21039'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/commodity/3': # Reported via Frontier forums: if matches.group('err_msg') != 'Failed Validation "[]"': @@ -139,7 +146,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDSM': # It's in-browser, no public source/releases - if matches.group('software_version') == '1.0.1': + if software_version >= semantic_version.Version.coerce('1.0.1'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': if matches.group('journal_event') == 'Scan': # @@ -157,7 +164,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDSM - Console': # It's in-browser, no public source/releases - if matches.group('software_version') == '1.0': + if software_version >= semantic_version.Version.coerce('1.0'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': if matches.group('journal_event') == 'Scan': # @@ -175,7 +182,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDAOS': # Apparently a Barry Carylon project, but no home page ? - if matches.group('software_version') == '1.2.3': + if software_version >= semantic_version.Version.coerce('1.2.3'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': if matches.group('journal_event') == 'Docked': # @@ -193,7 +200,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EliteLogAgent': # - if matches.group('software_version') == '2.0.0.660': + if software_version >= semantic_version.Version.coerce('2.0.0.660'): print(line) # Abandoned/unmaintained project From 8c2a5dca6f48e20984283137cd7e0cf94d1c9d57 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 18 Feb 2022 18:07:14 +0000 Subject: [PATCH 2/4] scripts/eddn-report-log-errors: Use fileinput & update for latest * Use fileinput, makes it easy to use '-' as 'stdin please'. * EDDLite 2.0.0.0: Bad date-time * EDMC >= 5.2.4: approachsettelement/1 issues --- scripts/eddn-report-log-errors | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index ffa1c83..1e555dc 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -3,6 +3,7 @@ """Produce a report on the provided EDDN Gateway log file's ERRORs.""" import argparse +import fileinput import re import semantic_version @@ -45,7 +46,7 @@ def process_file(input_file: str) -> None: r' from (?P.+)$' ) # TODO: Make this handle gzipped files - with open(input_file, 'r') as input: + with fileinput.FileInput(files=(input_file), mode='r') as input: line = input.readline() while line: line = line.strip() @@ -87,10 +88,22 @@ def process_file(input_file: str) -> None: 'https://eddn.edcd.io/schemas/shipyard/2', 'https://eddn.edcd.io/schemas/outfitting/2', ): - # Reported via Discord PM to Robby 2022-01-07 - if matches.group('err_msg') != 'Failed Validation "[]"': + # Failed Validation "[]" + if ( + matches.group('err_msg').startswith('Failed Validation "[]"') + ): + # + pass + + elif matches.group('err_msg') == 'Failed Validation "[]"': + # Reported via Discord PM to Robby 2022-01-07 + pass + + else: print(line) + else: print(line) @@ -129,6 +142,15 @@ def process_file(input_file: str) -> None: # pass + elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': + if matches.group('err_msg') == 'Failed Validation "[]"': + # + pass + + elif matches.group('err_msg') == 'Failed Validation "[]"': + # + pass + else: print(line) From 77a2d41d96b7a82a410e08220e6991e43732cd0e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 19 Feb 2022 11:29:19 +0000 Subject: [PATCH 3/4] approachsettlement/1: Allow MarketID to be optional & document why --- schemas/approachsettlement-README.md | 34 ++++++++++++++++++++++++++++ schemas/approachsettlement-v1.0.json | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md index 9dfe6d1..13bfcd7 100644 --- a/schemas/approachsettlement-README.md +++ b/schemas/approachsettlement-README.md @@ -19,6 +19,40 @@ discrepancy.** The primary data source for this schema is the ED Journal event `ApproachSettlement`. +### MarketID +Whilst the `MarketID` property is not in the required list **YOU MUST +ABSOLUTELY SEND THIS WHEN IT IS PRESENT IN THE SOURCE DATA**. + +The only reason it is optional is that there are `ApproachSettlement` +Journal events for things like visitor beacons that do not have a market, and +thus no MarketID. + +Examples: + +```json +{ + "timestamp":"2022-02-18T14:33:35Z", + "event":"ApproachSettlement", + "Name":"Battlegroup's Disappearance", + "SystemAddress":1109989017963, + "BodyID":8, + "BodyName":"Alioth 1 a", + "Latitude":59.972752, + "Longitude":-84.506294 +}, +{ + "timestamp": "2022-02-18T15:02:04Z", + "event": "ApproachSettlement", + "Name": "$Ancient:#index=1;", + "Name_Localised": "Ancient Ruins (1)", + "SystemAddress": 3515254557027, + "BodyID": 13, + "BodyName": "Synuefe XR-H d11-102 1 b", + "Latitude": -46.576923, + "Longitude": 133.985107 +}, +``` + ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json index 571db77..b6e5ed8 100644 --- a/schemas/approachsettlement-v1.0.json +++ b/schemas/approachsettlement-v1.0.json @@ -33,7 +33,7 @@ "type" : "object", "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", "additionalProperties" : false, - "required" : [ "timestamp", "event", "StarSystem", "StarPos", "SystemAddress", "Name", "MarketID", "BodyID", "BodyName", "Latitude", "Longitude" ], + "required" : [ "timestamp", "event", "StarSystem", "StarPos", "SystemAddress", "Name", "BodyID", "BodyName", "Latitude", "Longitude" ], "properties" : { "timestamp": { "type" : "string", From 42866fdda97ec5a6d00af7aef1395dba1a6c9ae6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 19 Feb 2022 11:38:56 +0000 Subject: [PATCH 4/4] approachesettlement/1: Add no-MarketID samples for testing --- .../approachsettlement-ancient_ruins.json | 22 +++++++++++++++++++ .../approachsettlement-visitor_beacon.json | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 scripts/testing/gateway-responses/approachsettlement-ancient_ruins.json create mode 100644 scripts/testing/gateway-responses/approachsettlement-visitor_beacon.json diff --git a/scripts/testing/gateway-responses/approachsettlement-ancient_ruins.json b/scripts/testing/gateway-responses/approachsettlement-ancient_ruins.json new file mode 100644 index 0000000..e059145 --- /dev/null +++ b/scripts/testing/gateway-responses/approachsettlement-ancient_ruins.json @@ -0,0 +1,22 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/approachsettlement/1", + "header": { + "uploaderID": "from Athanasius Testing", + "softwareName": "Athanasius Testing script", + "softwareVersion": "v0.0.1" + }, + "message": { + "timestamp": "2022-02-18T15:02:04Z", + "event": "ApproachSettlement", + "Name": "$Ancient:#index=1;", + "SystemAddress": 3515254557027, + "StarSystem": "Synuefe XR-H d11-102", + "BodyID": 13, + "BodyName": "Synuefe XR-H d11-102 1 b", + "Latitude": -46.576923, + "Longitude": 133.985107, + "StarPos": [ + 357.34375, -49.34375, -74.75 + ] + } +} diff --git a/scripts/testing/gateway-responses/approachsettlement-visitor_beacon.json b/scripts/testing/gateway-responses/approachsettlement-visitor_beacon.json new file mode 100644 index 0000000..5656970 --- /dev/null +++ b/scripts/testing/gateway-responses/approachsettlement-visitor_beacon.json @@ -0,0 +1,22 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/approachsettlement/1", + "header": { + "uploaderID": "from Athanasius Testing", + "softwareName": "Athanasius Testing script", + "softwareVersion": "v0.0.1" + }, + "message": { + "timestamp":"2022-02-18T14:33:35Z", + "event":"ApproachSettlement", + "Name":"Battlegroup's Disappearance", + "StarSystem": "Alioth", + "BodyID":8, + "BodyName":"Alioth 1 a", + "Latitude":59.972752, + "Longitude":-84.506294, + "SystemAddress":1109989017963, + "StarPos": [ + -33.65625, 72.46875, -20.65625 + ] + } +}