From 40a1d11e76b8a3d03adf367643469ac7d3576bb9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 14 Jan 2022 11:02:32 +0000 Subject: [PATCH 001/234] contrib/eddn-logs-archive: Produce report on rotated logfile --- contrib/eddn-logs-archive | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/eddn-logs-archive b/contrib/eddn-logs-archive index ef30aef..972f710 100755 --- a/contrib/eddn-logs-archive +++ b/contrib/eddn-logs-archive @@ -87,7 +87,8 @@ do log " Archiving ${service}.log ..." # We have no means to tell the service to close and re-open output, it's # to stdout/err anyway. So we copy it. - COMPRESSED_NAME="${service}.log.$(date --iso-8601=seconds)" + TIMESTAMP="$(date --iso-8601=seconds)" + COMPRESSED_NAME="${service}.log.${TIMESTAMP}" cp ${service}.log "${COMPRESSED_NAME}" if [ $? -ne 0 ]; then @@ -97,6 +98,11 @@ do fi # Truncate the live file. :> ${service}.log + + # Produce a report on the just-rotated log + ${HOME}/.local/bin/eddn-report-log-errors "${COMPRESSED_NAME}" > \ + "${HOME}/reports/eddn-errors/by-log-rotation/eddn-errors-${TIMESTMAP}.txt" + # Now compress the newly archived log gzip -9v "${COMPRESSED_NAME}" log " DONE" From e69f7a938af2ee41ad324651617928667bb57df8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 15 Jan 2022 14:09:47 +0000 Subject: [PATCH 002/234] scripts/eddn-report: Add Moonlight/1.3.4/Scan case --- scripts/eddn-report-log-errors | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index a626e23..6a87003 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -179,6 +179,24 @@ def process_file(input_file: str) -> None: if matches.group('software_version') == '2.0.0.660': print(line) + # + # + elif matches.group('software_name') == 'Moonlight': + if matches.group('software_version') == '1.3.4': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': + if matches.group('journal_event') == 'Scan': + # Ref: + if not matches.group('err_msg').startswith( + 'Failed Validation "[ Date: Sun, 16 Jan 2022 09:33:45 +0000 Subject: [PATCH 003/234] scripts/eddn-report: Current G19s version only / EVA is abandoned --- scripts/eddn-report-log-errors | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 6a87003..45a95c7 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -117,9 +117,6 @@ def process_file(input_file: str) -> None: print(matches.group('err_msg')) print(line) - else: - print(line) - elif matches.group('software_name') == 'EDSM': # It's in-browser, no public source/releases if matches.group('software_version') == '1.0.1': @@ -197,7 +194,14 @@ def process_file(input_file: str) -> None: else: print(line) - ################################################################### + + # Abandoned/unmaintained project + # + # + elif matches.group('software_name') == 'EVA [iPhone]': + pass + + #################################################################### # Issues we know about, but haven't yet alerted developers to ################################################################### ################################################################### From 61661926bc3300b3280c9fd766e9e9301a616826 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 20 Mar 2022 09:12:56 +0000 Subject: [PATCH 004/234] eddn-report-log-errors: New EDAOS shipyard/2 error --- scripts/eddn-report-log-errors | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 673e42e..3325a66 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/en # vim: wrapmargin=0 textwidth=0 smarttab expandtab tabstop=2 shiftwidth=2 """Produce a report on the provided EDDN Gateway log file's ERRORs.""" @@ -254,6 +254,16 @@ def process_file(input_file: str) -> None: else: print(line) + elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/shipyard/2': + # + if matches.group('err_msg').startswith( + 'Failed Validation "[]"' + ): + pass + + else: + print(line) + else: print(line) From de6551002f32d2dab4eb59555a5b34665c8bc335 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Mar 2022 08:20:55 +0000 Subject: [PATCH 005/234] eddn-report-log-errors: fix #! line --- scripts/eddn-report-log-errors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 3325a66..e730443 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -1,4 +1,4 @@ -#!/usr/bin/en +#!/usr/bin/env python3 # vim: wrapmargin=0 textwidth=0 smarttab expandtab tabstop=2 shiftwidth=2 """Produce a report on the provided EDDN Gateway log file's ERRORs.""" From 91c6ef0108b52c936a50f3932c7a02e556e51e90 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 24 Apr 2022 10:08:19 +0000 Subject: [PATCH 006/234] scripts/eddn-report-log-errors: Handle non-parsable softwareVersion --- scripts/eddn-report-log-errors | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index e730443..35e7806 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -62,7 +62,13 @@ def process_file(input_file: str) -> None: # print(matches.group('sender_ip')) # print('') - software_version = semantic_version.Version.coerce(matches.group('software_version')) + try: + software_version = semantic_version.Version.coerce(matches.group('software_version')) + + except ValueError as e: + print(f"Error parsing sofwareVersion for:\n{matches.group('software_version')}\n{line}\n") + next + ################################################################### # Issues we know about and HAVE already alerted their # developers to. From bece26156eb147ad754fac13615bc18feff108db Mon Sep 17 00:00:00 2001 From: Gareth Harper Date: Wed, 18 May 2022 15:30:31 +0000 Subject: [PATCH 007/234] added fss body signals journal event --- schemas/fssbodysignals-README.md | 39 ++++++++++++++ schemas/fssbodysignals-v1.0.json | 92 ++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 schemas/fssbodysignals-README.md create mode 100644 schemas/fssbodysignals-v1.0.json diff --git a/schemas/fssbodysignals-README.md b/schemas/fssbodysignals-README.md new file mode 100644 index 0000000..1d8108e --- /dev/null +++ b/schemas/fssbodysignals-README.md @@ -0,0 +1,39 @@ +# EDDN FSSAllBodiesFound Schema + +## Introduction +Here we document how to take data from an ED `FSSBodySignals` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The primary data source for this schema is the ED Journal event +`FSSBodySignals`. + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. + +#### Remove _Localised key/values +All keys whose name ends with `_Localised`, i.e. the `Type_Localised` +key/values in Signals. + +#### Examples: + +```json +{ "timestamp":"2022-05-18T00:10:57Z", "event":"FSSBodySignals", "BodyName":"Phoi Auwsy ZY-Z d132 7 a", "BodyID":37, "SystemAddress":4546986398603, "Signals":[ { "Type":"$SAA_SignalType_Geological;", "Type_Localised":"Geological", "Count":2 } ] } +``` diff --git a/schemas/fssbodysignals-v1.0.json b/schemas/fssbodysignals-v1.0.json new file mode 100644 index 0000000..6c3e25c --- /dev/null +++ b/schemas/fssbodysignals-v1.0.json @@ -0,0 +1,92 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fssbodysignals/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal, minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event", "SystemName", "StarPos", "SystemAddress", "BodyID", "Signals" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "FSSBodySignals" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "SystemName": { + "type" : "string", + "minLength" : 1 + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender if not present in the journal event" + }, + "SystemAddress": { + "type" : "integer" + }, + "BodyID" : { + "type" : "integer" + }, + "Signals": { + "type" : "array", + "items" : { + "type" : "object", + "additionalProperties" : false, + "required" : [ "Type", "Count" ], + "properties" : { + "Type" : { + "type" : "string" + }, + "Count" : { + "type" : "integer" + } + } + } + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} From b9e624a3510e317085861360c43167957a4ea403 Mon Sep 17 00:00:00 2001 From: Gareth Harper Date: Tue, 24 May 2022 10:57:39 +0000 Subject: [PATCH 008/234] make gateway use new schema --- src/eddn/conf/Settings.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 5ca2a6f..c09e2ad 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -59,25 +59,28 @@ class _Settings(object): "https://eddn.edcd.io/schemas/journal/1" : "schemas/journal-v1.0.json", "https://eddn.edcd.io/schemas/journal/1/test" : "schemas/journal-v1.0.json", - "https://eddn.edcd.io/schemas/scanbarycentre/1" : "schemas/scanbarycentre-v1.0.json", - "https://eddn.edcd.io/schemas/scanbarycentre/1/test" : "schemas/scanbarycentre-v1.0.json", + "https://eddn.edcd.io/schemas/scanbarycentre/1" : "schemas/scanbarycentre-v1.0.json", + "https://eddn.edcd.io/schemas/scanbarycentre/1/test" : "schemas/scanbarycentre-v1.0.json", - "https://eddn.edcd.io/schemas/fssdiscoveryscan/1" : "schemas/fssdiscoveryscan-v1.0.json", - "https://eddn.edcd.io/schemas/fssdiscoveryscan/1/test" : "schemas/fssdiscoveryscan-v1.0.json", + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1" : "schemas/fssdiscoveryscan-v1.0.json", + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1/test" : "schemas/fssdiscoveryscan-v1.0.json", - "https://eddn.edcd.io/schemas/codexentry/1" : "schemas/codexentry-v1.0.json", - "https://eddn.edcd.io/schemas/codexentry/1/test" : "schemas/codexentry-v1.0.json", + "https://eddn.edcd.io/schemas/codexentry/1" : "schemas/codexentry-v1.0.json", + "https://eddn.edcd.io/schemas/codexentry/1/test" : "schemas/codexentry-v1.0.json", - "https://eddn.edcd.io/schemas/navbeaconscan/1" : "schemas/navbeaconscan-v1.0.json", - "https://eddn.edcd.io/schemas/navbeaconscan/1/test" : "schemas/navbeaconscan-v1.0.json", + "https://eddn.edcd.io/schemas/navbeaconscan/1" : "schemas/navbeaconscan-v1.0.json", + "https://eddn.edcd.io/schemas/navbeaconscan/1/test" : "schemas/navbeaconscan-v1.0.json", - "https://eddn.edcd.io/schemas/navroute/1" : "schemas/navroute-v1.0.json", - "https://eddn.edcd.io/schemas/navroute/1/test" : "schemas/navroute-v1.0.json", + "https://eddn.edcd.io/schemas/navroute/1" : "schemas/navroute-v1.0.json", + "https://eddn.edcd.io/schemas/navroute/1/test" : "schemas/navroute-v1.0.json", - "https://eddn.edcd.io/schemas/approachsettlement/1" : "schemas/approachsettlement-v1.0.json", - "https://eddn.edcd.io/schemas/approachsettlement/1/test" : "schemas/approachsettlement-v1.0.json", - "https://eddn.edcd.io/schemas/fssallbodiesfound/1" : "schemas/fssallbodiesfound-v1.0.json", - "https://eddn.edcd.io/schemas/fssallbodiesfound/1/test" : "schemas/fssallbodiesfound-v1.0.json", + "https://eddn.edcd.io/schemas/approachsettlement/1" : "schemas/approachsettlement-v1.0.json", + "https://eddn.edcd.io/schemas/approachsettlement/1/test" : "schemas/approachsettlement-v1.0.json", + "https://eddn.edcd.io/schemas/fssallbodiesfound/1" : "schemas/fssallbodiesfound-v1.0.json", + "https://eddn.edcd.io/schemas/fssallbodiesfound/1/test" : "schemas/fssallbodiesfound-v1.0.json", + + "https://eddn.edcd.io/schemas/fssbodysignals/1" : "schemas/fssbodysignals-v1.0.json", + "https://eddn.edcd.io/schemas/fssbodysignals/1/test" : "schemas/fssbodysignals-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ From cda25f488281e3b8a58ef154e41a1624b7eca47d Mon Sep 17 00:00:00 2001 From: Gareth Harper Date: Wed, 25 May 2022 15:46:13 +0000 Subject: [PATCH 009/234] added missing BodyName field --- schemas/fssbodysignals-v1.0.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/fssbodysignals-v1.0.json b/schemas/fssbodysignals-v1.0.json index 6c3e25c..a0317e3 100644 --- a/schemas/fssbodysignals-v1.0.json +++ b/schemas/fssbodysignals-v1.0.json @@ -67,6 +67,10 @@ "BodyID" : { "type" : "integer" }, + "BodyName": { + "type" : "string", + "minLength" : 1 + }, "Signals": { "type" : "array", "items" : { From 70a04a4f771483c25209ecd9014b525265049b84 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 12:05:05 +0100 Subject: [PATCH 010/234] docs/Developers: Call out that Listeners need to pay attention to Augmentations --- docs/Developers.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/Developers.md b/docs/Developers.md index 6170e6b..247e381 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -435,7 +435,11 @@ data you will first need to zlib-decompress each message. Then you will have a textual JSON object as per the Schemas. In general, check the guidance for [Uploading messages](#uploading-messages) -for the expected format of the messages. +for the expected format of the messages. **Pay particular attention to any +schema-specific Augmentations**. Whilst Senders MUST make every effort to +ensure such data is correct it is possible that bugs in either their code, or +the game itself, could mean it is incorrect. Listeners use such data at +their own risk. Consumers can utilise the `$schemaRef` value to determine which Schema a particular message is for. There is no need to validate the messages From 8e5b17acdd68747b08f37f883b0547d0b4b4e9f1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 11:18:53 +0000 Subject: [PATCH 011/234] schemas/fssbodysignals: Augment also with SystemName (as per schema) --- schemas/fssbodysignals-README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/fssbodysignals-README.md b/schemas/fssbodysignals-README.md index 1d8108e..051a825 100644 --- a/schemas/fssbodysignals-README.md +++ b/schemas/fssbodysignals-README.md @@ -24,6 +24,10 @@ The primary data source for this schema is the ED Journal event Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. +#### SystemName +You MUST add a `SystemName` string containing the name of the system from the +last `FSDJump`, `CarrierJump`, or `Location` event. + #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. From 370cef676f1c854c35ea4f9404b5f09924a539c8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 12:36:53 +0000 Subject: [PATCH 012/234] schemas/fssbodysignals: Use `StarSystem` not `SystemName` This matches what's done in other schemas, and allows for use of common code in EDMC. --- schemas/fssbodysignals-README.md | 4 ++-- schemas/fssbodysignals-v1.0.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/schemas/fssbodysignals-README.md b/schemas/fssbodysignals-README.md index 051a825..d66c028 100644 --- a/schemas/fssbodysignals-README.md +++ b/schemas/fssbodysignals-README.md @@ -24,8 +24,8 @@ The primary data source for this schema is the ED Journal event Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. -#### SystemName -You MUST add a `SystemName` string containing the name of the system from the +#### StarSystem +You MUST add a `StarSystem` string containing the name of the system from the last `FSDJump`, `CarrierJump`, or `Location` event. #### StarPos diff --git a/schemas/fssbodysignals-v1.0.json b/schemas/fssbodysignals-v1.0.json index a0317e3..68ee08e 100644 --- a/schemas/fssbodysignals-v1.0.json +++ b/schemas/fssbodysignals-v1.0.json @@ -33,7 +33,7 @@ "type" : "object", "description" : "Contains all properties from the listed events in the client's journal, minus the Localised strings and the properties marked below as 'disallowed'", "additionalProperties" : false, - "required" : [ "timestamp", "event", "SystemName", "StarPos", "SystemAddress", "BodyID", "Signals" ], + "required" : [ "timestamp", "event", "StarSystem", "StarPos", "SystemAddress", "BodyID", "Signals" ], "properties" : { "timestamp": { "type" : "string", @@ -50,7 +50,7 @@ "type" : "boolean", "description" : "Whether the sending Cmdr has an Odyssey expansion." }, - "SystemName": { + "StarSystem": { "type" : "string", "minLength" : 1 }, From d10b081be4d284c64112176a08b37a98cf96f9b4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 14:17:30 +0100 Subject: [PATCH 013/234] docs: Outline rules and caveats about data Augmentations * Many will need a location cross-check. * Call out this need in the fssbodysignals README. --- docs/Developers.md | 42 ++++++++++++++++++++++++++++++++ schemas/fssbodysignals-README.md | 6 +++++ 2 files changed, 48 insertions(+) diff --git a/docs/Developers.md b/docs/Developers.md index 247e381..ef846ef 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -301,6 +301,48 @@ Horizons-only features disabled. active, but in the non-Odyssey game client case you only get the Horizons boolean. +#### Other data Augmentations +Some schemas mandate that extra data be added, beyond what is in the source +data, to aid Listeners. + +This is usually related to specifying which system an event took place in, and +usually means ensuring there is the full set of: + +1. `StarSystem` - the name of the system. +2. `SystemAddress` - the game's unique numerical identifier for the system. +3. `StarPos` - The system's co-ordinates. + +Whilst it can be argued that any Listener should see preceding event(s) that +give any missing information where at least the system name or `SystemAddress` +is already in the event data, this might not always be true. So Senders MUST +add this data where required. It helps to fill out basic system information +(name, SystemAddress and co-ordinates). + +However, there is a known game bug that can result in it stopping writing to +the game journal, and some observed behaviour implies that it might then later +resume writing to that file, but with events missing. This means any Sender +that blindly assumes it knows the current system/location and uses that for +these Augmentations might send erroneous data. + +1. **Senders MUST cross-check available event data with prior 'location' + event(s) to be sure the correct extra data is being added.** +2. **Listeners SHOULD realise that any data added as an Augmentation might be + in error.** + +For Senders, if the source data only has `SystemAddress` then you MUST check +that it matches that from the prior `Location`, `FSDJump` or `CarrierJump` +event before adding `StarSystem` and `StarPos` data to a message. Drop the +message entirely if it does not match. Apply similar logic if it is only +the system's name that is already present in data. Do not blindly add +`SystemAddress` or `StarPos`. Likewise, do not blindly add `StarPos` if the +other data is already in the source, without cross-checking the system name +and `SystemAddress`. + +Listeners might be able to apply their own cross-check on received messages, +and use any mismatch with respect to what they already know to make a decision +whether to trust the augmented data. Flagging it for manual review is probably +wise. + ### Server responses There are three possible sources of HTTP responses when sending an upload to EDDN. diff --git a/schemas/fssbodysignals-README.md b/schemas/fssbodysignals-README.md index d66c028..10122a3 100644 --- a/schemas/fssbodysignals-README.md +++ b/schemas/fssbodysignals-README.md @@ -28,10 +28,16 @@ in the Developers' documentation. You MUST add a `StarSystem` string containing the name of the system from the last `FSDJump`, `CarrierJump`, or `Location` event. +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + #### Remove _Localised key/values All keys whose name ends with `_Localised`, i.e. the `Type_Localised` key/values in Signals. From 3b84e851b8c8f3e8eea700e9fb4b3769c48d3580 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 14:48:42 +0100 Subject: [PATCH 014/234] schemas/approachsettlement: Senders MUST apply cross-checks on augmentations --- schemas/approachsettlement-README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md index 13bfcd7..93c8237 100644 --- a/schemas/approachsettlement-README.md +++ b/schemas/approachsettlement-README.md @@ -64,6 +64,12 @@ You MUST add a StarSystem key/value pair representing the name of the system this event occurred in. Source this from either Location, FSDJump or CarrierJump as appropriate. +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** From 509521dd8398364b7df3bee1012a05353e1468c9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 14:57:54 +0100 Subject: [PATCH 015/234] schemas/codexentry: Senders MUST apply cross-check on StarPos augmentation --- schemas/codexentry-README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/codexentry-README.md b/schemas/codexentry-README.md index 1c675d3..beb6555 100644 --- a/schemas/codexentry-README.md +++ b/schemas/codexentry-README.md @@ -27,6 +27,9 @@ in the Developers' documentation. You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + #### BodyID and BodyName You SHOULD attempt to track the BodyName and BodyID where the player is and add keys/values for these. From fbdd531372c62f62cccb8403faa1fe5a6ea36449 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 15:00:44 +0100 Subject: [PATCH 016/234] schemas/fssallbodiesfound: Senders MUST apply cross-check on StarPos augmentation --- schemas/fssallbodiesfound-README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md index 350a318..e535b32 100644 --- a/schemas/fssallbodiesfound-README.md +++ b/schemas/fssallbodiesfound-README.md @@ -27,3 +27,6 @@ in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** From 209a02a3bbf34e557338442c60f1f9413b9f5227 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 15:03:17 +0100 Subject: [PATCH 017/234] schemas/fssdiscoveryscan: Senders MUST apply cross-check on StarPos augmentation --- schemas/fssdiscoveryscan-README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/fssdiscoveryscan-README.md b/schemas/fssdiscoveryscan-README.md index ba5a5eb..f7c8757 100644 --- a/schemas/fssdiscoveryscan-README.md +++ b/schemas/fssdiscoveryscan-README.md @@ -27,3 +27,6 @@ in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** From c47699f11a2e9aaca2be6f0c59136a10b559d226 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 15:05:01 +0100 Subject: [PATCH 018/234] schemas/navbeaconscan: Senders MUST apply cross-check on augmentations --- schemas/navbeaconscan-README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/schemas/navbeaconscan-README.md b/schemas/navbeaconscan-README.md index 39167a2..09ecdef 100644 --- a/schemas/navbeaconscan-README.md +++ b/schemas/navbeaconscan-README.md @@ -29,6 +29,12 @@ You MUST add a `StarSystem` key/value pair representing the name of the system this event occurred in. Source this from either `Location`, `FSDJump` or `CarrierJump` as appropriate. +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** From e78ecc4ae70474dee8ed11bab68409db37bf2bb9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 15:07:36 +0100 Subject: [PATCH 019/234] schemas/scanbarycentre: Senders MUST apply cross-check on augmentations --- schemas/scanbarycentre-README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/scanbarycentre-README.md b/schemas/scanbarycentre-README.md index d7b5f01..89c1ce6 100644 --- a/schemas/scanbarycentre-README.md +++ b/schemas/scanbarycentre-README.md @@ -30,3 +30,6 @@ in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** From bc628bff6f2e92cda3df7af29a75b30cfb69fd8b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 15:37:44 +0100 Subject: [PATCH 020/234] schemas/journal: All relevant events already have SystemAddress So there's no need to document it as an augmentation. This was an error based on examining the EDMC code. --- schemas/journal-README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/schemas/journal-README.md b/schemas/journal-README.md index 5dd7cea..5553afe 100644 --- a/schemas/journal-README.md +++ b/schemas/journal-README.md @@ -76,11 +76,6 @@ You MUST add a `StarSystem` key/value pair representing the name of the system this event occurred in. Source this from either `Location`, `FSDJump` or `CarrierJump` as appropriate. -#### SystemAddress -You MUST add a `SystemAddress` key/value pair representing the numerical ID -of the system this event occurred in. Source this from either `Location`, -`FSDJump` or `CarrierJump` as appropriate. - #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. From c3149f06bc054a5605c74afe9d540a57d01f2fd2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 29 May 2022 15:41:13 +0100 Subject: [PATCH 021/234] schemas/journal: Better word StarSystem and StarPos Augmentation docs And also include the MUST about cross-checks. --- schemas/journal-README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/schemas/journal-README.md b/schemas/journal-README.md index 5553afe..c2aa572 100644 --- a/schemas/journal-README.md +++ b/schemas/journal-README.md @@ -72,10 +72,20 @@ You SHOULD add this key/value pair, using the value from the `LoadGame` event. You SHOULD add this key/value pair, using the value from the `LoadGame` event. #### StarSystem -You MUST add a `StarSystem` key/value pair representing the name of the -system this event occurred in. Source this from either `Location`, -`FSDJump` or `CarrierJump` as appropriate. +If not already present, you MUST add a `StarSystem` string containing the +name of the system from the last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + +This should only apply to `SAASignalsFound` events. #### StarPos -You MUST add a `StarPos` array containing the system co-ordinates from the -last `FSDJump`, `CarrierJump`, or `Location` event. +If not already present, you MUST add a `StarPos` array containing the +system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` +event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + +This should only apply to `Docked`, `Scan` and `SAASignalsFound` events. From 85031ba2b01b058545be5d35b15fedeb462624be Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 31 May 2022 13:21:49 +0100 Subject: [PATCH 022/234] schemas/template: Update template for `StarSystem` default & cross-checks * New schemas will need to use `StarSystem`, not `SystemName` or `System`, if applicable, so have that example here. * Any such augmentations need the cross-check mandating. --- schemas/TEMPLATES/journalevent-README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index f85fc86..890d782 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -106,10 +106,20 @@ value is what the name would have been in the source Journal data. Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. +#### StarSystem +You MUST add a `StarSystem` string containing the name of the system from the +last `FSDJump`, `CarrierJump`, or `Location` event. + +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. +**You MUST apply a location cross-check, as per +[Other data augmentations](../docs/Developers.md#other-data-augmentations).** + ## Listeners The advice above for [Senders](#senders), combined with the actual Schema file *should* provide all the information you need to process these events. From 9a8ca522e6ebed05c293ecf96a6f5633234f66ca Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 13 Jun 2022 10:33:13 +0100 Subject: [PATCH 023/234] docs/Developers: Fix two minor typos --- docs/Developers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index ef846ef..6936bbb 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -1,7 +1,7 @@ ## Introduction EDDN is a -[zermoq](https://zeromq.org/) service which allows players of the game +[zeromq](https://zeromq.org/) service which allows players of the game [Elite Dangerous](https://www.elitedangerous.com/), published by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so that interested listeners can receive a copy. @@ -15,7 +15,7 @@ representing this game data and then passes it on to any interested listeners. ## Sources There are two sources of game data, both provided by the publisher of the game, -Frontier Developerments. They are both explicitly approved for use by +Frontier Developments. They are both explicitly approved for use by third-party software. ### Journal Files From 912fa0e0647871e8b74d7188d5dfc0ce6ec7c8bd Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 16:39:51 +0300 Subject: [PATCH 024/234] WIP: FSSSignalDiscovered event support --- schemas/fsssignaldiscovered-README.md | 1 + schemas/fsssignaldiscovered-v1.0.json | 81 +++++++++++++++++++++++++++ src/eddn/conf/Settings.py | 3 + 3 files changed, 85 insertions(+) create mode 100644 schemas/fsssignaldiscovered-README.md create mode 100644 schemas/fsssignaldiscovered-v1.0.json diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md new file mode 100644 index 0000000..30404ce --- /dev/null +++ b/schemas/fsssignaldiscovered-README.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json new file mode 100644 index 0000000..51bd1f0 --- /dev/null +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -0,0 +1,81 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fsssignaldiscovered/1#", + "description" : "EDDN schema for FSSSignalDiscovered Journal events. Full documentation at https://github.com/EDCD/EDDN/tree/master/schemas/fsssignaldiscovered-README.md", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "event", "signals"], + "properties" : { + "event": { + "enum" : [ "FSSSignalDiscovered" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "signals": { + "type": "array", + "description": "Array of FSSSignalDiscovered events", + "minItems": 1, + "items": { + "type": "object", + "required": ["SystemAddress", "timestamp", "SignalName"], + "description": "Single FSSSignalDiscovered event", + "properties": { + "SystemAddress": { "type": "integer" }, + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "SignalName": { "type": "string" }, + "IsStation": { "type": "boolean" }, + "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, + + "patternProperties": { + "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } + } + } + } + + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index c09e2ad..0b3253a 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -81,6 +81,9 @@ class _Settings(object): "https://eddn.edcd.io/schemas/fssbodysignals/1" : "schemas/fssbodysignals-v1.0.json", "https://eddn.edcd.io/schemas/fssbodysignals/1/test" : "schemas/fssbodysignals-v1.0.json", + + "https://eddn.edcd.io/schemasfsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemasfsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ From 036e918fe93f5bb008323643ac23efea546c94c5 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 17:41:42 +0300 Subject: [PATCH 025/234] FSSSignalDiscovered: optional systemName and StarPos --- schemas/fsssignaldiscovered-v1.0.json | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 51bd1f0..834cbe1 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -70,7 +70,20 @@ } } } - } + }, + "systemName": { + "type" : "string", + "minLength" : 1, + "description": "Should be added by the sender" + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Should be added by the sender" + } + } } From 5386ed2c64dc7223e6a494f235d77801fc313ab9 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 18:34:07 +0300 Subject: [PATCH 026/234] FSSSignalDiscovered: add properties --- schemas/fsssignaldiscovered-v1.0.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 834cbe1..bb03da5 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -63,7 +63,11 @@ }, "SignalName": { "type": "string" }, "IsStation": { "type": "boolean" }, + "USSType": { "type": "string" }, "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, + "SpawningState": {"type": "string"}, + "SpawningFaction" : {"type": "string"}, + "ThreatLevel": {"type": "integer" }, "patternProperties": { "_Localised$" : { "$ref" : "#/definitions/disallowed" } From 6dbc9e392a96db0d432291e54ceb5702118fd074 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 19:22:05 +0300 Subject: [PATCH 027/234] FSSSignalDiscovered: don't pass `$USS_Type_MissionTarget;` USS_Type --- schemas/fsssignaldiscovered-v1.0.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index bb03da5..d5e4fcc 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -63,7 +63,12 @@ }, "SignalName": { "type": "string" }, "IsStation": { "type": "boolean" }, - "USSType": { "type": "string" }, + "USSType": { + "type": "string", + "not": { + "pattern": "^\\$USS_Type_MissionTarget;$" + } + }, "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, "SpawningState": {"type": "string"}, "SpawningFaction" : {"type": "string"}, From 03caeaa20c14ec93b89e718fa21e708159919517 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 21:22:01 +0300 Subject: [PATCH 028/234] FSSSignalDiscovered: initial documentation --- schemas/fsssignaldiscovered-README.md | 53 ++++++++++++++++++++++++++- schemas/fsssignaldiscovered-v1.0.json | 6 +-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 30404ce..370006c 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -1 +1,52 @@ -TODO \ No newline at end of file +# EDDN FSSSignalDiscovered Schema + +## Introduction +Here we document how to take data from an ED `FSSSignalDiscovered` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +## Senders +The primary data source for this schema is the ED Journal event +`FSSSignalDiscovered`. + +### Batching +You MUST put several `FSSSignalDiscovered` events to an array `signals` which is key of `message`. Minimum size of +`signals` is 1 item. + +Do not make a request for every single event. + +Possible algorithm of batching: +1. If the event is FSSSignalDiscovered, store it to the temporal list and proceed to next event. +2. If the next event is also FSSSignalDiscovered, repeat 1. +3. If the next event is any other or there is no other event for more than 10 seconds, send the + temporal list in a single message to EDDN. + +### Elisions +You MUST remove the following key/value pairs from the data: + + - `TimeRemaining` key/value pair. + +You MUST refuse to send the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. + +### Augmentations +#### horizons flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### odyssey flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### systemName +You SHOULD add a `systemName` string containing the system name from the last `FSDJump`, `CarrierJump`, or `Location` +event There exists problem when you gets `FSSSignalDiscovered` before +`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. + +#### StarPos +You SHOULD add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. There exists problem when you gets `FSSSignalDiscovered` before +`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. + +## Receivers +Receivers should remember: `horizons`, `odyssey`, `systemName`, `StarPos` are optional key/value pairs, it means you +should not rely on it will appear in every EDDN event. diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index d5e4fcc..8d23066 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -76,10 +76,10 @@ "patternProperties": { "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } } } - } - }, + }, "systemName": { "type" : "string", "minLength" : 1, @@ -92,8 +92,6 @@ "maxItems" : 3, "description" : "Should be added by the sender" } - - } } }, From fb9904306b7b4177a2ef1fc0b41011d244d7a27a Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 23:59:20 +0300 Subject: [PATCH 029/234] FSSSignalDiscovered doc: add examples --- schemas/fsssignaldiscovered-README.md | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 370006c..5ada0e9 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -50,3 +50,73 @@ last `FSDJump`, `CarrierJump`, or `Location` event. There exists problem when yo ## Receivers Receivers should remember: `horizons`, `odyssey`, `systemName`, `StarPos` are optional key/value pairs, it means you should not rely on it will appear in every EDDN event. + +## Examples +This is a few example of messages that passes current `FSSSignalDiscovered` schema. +1. A message without `horizons`, `odyssey`, `systemName`, `StarPos` fields. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"an software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "event":"FSSSignalDiscovered", + "signals":[ + { + "timestamp":"2021-07-30T19:03:08Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1774711381, + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + } + ] + } +} +``` + +2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"an software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "event":"FSSSignalDiscovered", + "signals":[ + { + "timestamp":"2021-07-30T19:03:08Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1774711381, + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + }, + { + "timestamp":"2020-12-31T14:14:01Z", + "event":"FSSSignalDiscovered", + "SystemAddress":216054883492, + "SignalName":"$USS_NonHumanSignalSource;", + "USSType":"$USS_Type_NonHuman;", + "SpawningState":"$FactionState_None;", + "SpawningFaction":"$faction_none;", + "ThreatLevel":5 + } + ], + "StarPos": [ + 8.1875, + 124.21875, + -38.5 + ], + "StarSystem": "HIP 56186", + "horizons": true, + "odyssey": true + } +} +``` From 732de9bdd635f22a56e76220f0f5546c0cb0abf8 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Mon, 8 Nov 2021 00:00:39 +0300 Subject: [PATCH 030/234] FSSSignalDiscovered doc --- schemas/fsssignaldiscovered-README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 5ada0e9..f07d8ae 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -78,7 +78,7 @@ This is a few example of messages that passes current `FSSSignalDiscovered` sche } ``` -2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields. +2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields which says it sent from Odyssey. ```json { "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", From 329d59831fe4b00ff73979ce31b9d9502d8168b3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 14 Jun 2022 13:59:13 +0100 Subject: [PATCH 031/234] schemas/fsssignaldiscovered: Rework based on in-game findings 1. Unix, not DOS, line-endings. 2. `message`->`timestamp` instead of per signal. 3. `message`->`SystemAddress` instead of per signal. 4. `StarSystem` and `StarPos` augmentations now mandatory. --- schemas/fsssignaldiscovered-v1.0.json | 204 +++++++++++++------------- 1 file changed, 103 insertions(+), 101 deletions(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 8d23066..f649e2f 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -1,101 +1,103 @@ -{ - "$schema" : "http://json-schema.org/draft-04/schema#", - "id" : "https://eddn.edcd.io/schemas/fsssignaldiscovered/1#", - "description" : "EDDN schema for FSSSignalDiscovered Journal events. Full documentation at https://github.com/EDCD/EDDN/tree/master/schemas/fsssignaldiscovered-README.md", - "type" : "object", - "additionalProperties" : false, - "required": [ "$schemaRef", "header", "message" ], - "properties": { - "$schemaRef": { - "type" : "string" - }, - "header": { - "type" : "object", - "additionalProperties" : true, - "required" : [ "uploaderID", "softwareName", "softwareVersion" ], - "properties" : { - "uploaderID": { - "type" : "string" - }, - "softwareName": { - "type" : "string" - }, - "softwareVersion": { - "type" : "string" - }, - "gatewayTimestamp": { - "type" : "string", - "format" : "date-time", - "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." - } - } - }, - "message": { - "type" : "object", - "description" : "Contains all properties from the listed events in the client's journal minus Localised strings and the properties marked below as 'disallowed'", - "additionalProperties" : false, - "required" : [ "event", "signals"], - "properties" : { - "event": { - "enum" : [ "FSSSignalDiscovered" ] - }, - "horizons": { - "type" : "boolean", - "description" : "Whether the sending Cmdr has a Horizons pass." - }, - "odyssey": { - "type" : "boolean", - "description" : "Whether the sending Cmdr has an Odyssey expansion." - }, - "signals": { - "type": "array", - "description": "Array of FSSSignalDiscovered events", - "minItems": 1, - "items": { - "type": "object", - "required": ["SystemAddress", "timestamp", "SignalName"], - "description": "Single FSSSignalDiscovered event", - "properties": { - "SystemAddress": { "type": "integer" }, - "timestamp": { - "type" : "string", - "format" : "date-time" - }, - "SignalName": { "type": "string" }, - "IsStation": { "type": "boolean" }, - "USSType": { - "type": "string", - "not": { - "pattern": "^\\$USS_Type_MissionTarget;$" - } - }, - "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, - "SpawningState": {"type": "string"}, - "SpawningFaction" : {"type": "string"}, - "ThreatLevel": {"type": "integer" }, - - "patternProperties": { - "_Localised$" : { "$ref" : "#/definitions/disallowed" } - } - } - } - }, - "systemName": { - "type" : "string", - "minLength" : 1, - "description": "Should be added by the sender" - }, - "StarPos": { - "type" : "array", - "items" : { "type": "number" }, - "minItems" : 3, - "maxItems" : 3, - "description" : "Should be added by the sender" - } - } - } - }, - "definitions": { - "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } - } -} +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fsssignaldiscovered/1#", + "description" : "EDDN schema for FSSSignalDiscovered Journal events. Full documentation at https://github.com/EDCD/EDDN/tree/master/schemas/fsssignaldiscovered-README.md", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "event", "timestamp", "SystemAddress", "StarSystem", "StarPos", "signals"], + "properties" : { + "event": { + "enum" : [ "FSSSignalDiscovered" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "SystemAddress": { + "type": "integer" + }, + "signals": { + "type": "array", + "description": "Array of FSSSignalDiscovered events", + "minItems": 1, + "items": { + "type": "object", + "required": ["SignalName"], + "description": "Single FSSSignalDiscovered event", + "properties": { + "SignalName": { "type": "string" }, + "IsStation": { "type": "boolean" }, + "USSType": { + "type": "string", + "not": { + "pattern": "^\\$USS_Type_MissionTarget;$" + } + }, + "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, + "SpawningState": {"type": "string"}, + "SpawningFaction" : {"type": "string"}, + "ThreatLevel": {"type": "integer" }, + + "patternProperties": { + "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } + } + } + }, + "StarSystem": { + "type" : "string", + "minLength" : 1, + "description": "Should be added by the sender" + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Should be added by the sender" + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} From 86f56eaac2bc5e12574bb857c3300e13be4745e9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 14 Jun 2022 14:01:36 +0100 Subject: [PATCH 032/234] schemas/fsssignaldiscovered README: Reworked for changed schema * Purely event-based batching, no timer. * Outline Horizons/Odyssey differences. * Point out that USS are manually scanned, so different in time nature. * StarSystem renamed from systemName. * StarSystem and StarPos now mandatory. --- schemas/fsssignaldiscovered-README.md | 295 +++++++++++++++----------- 1 file changed, 173 insertions(+), 122 deletions(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index f07d8ae..6665b22 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -1,122 +1,173 @@ -# EDDN FSSSignalDiscovered Schema - -## Introduction -Here we document how to take data from an ED `FSSSignalDiscovered` Journal -Event and properly structure it for sending to EDDN. - -Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general -documentation for a schema such as this. - -## Senders -The primary data source for this schema is the ED Journal event -`FSSSignalDiscovered`. - -### Batching -You MUST put several `FSSSignalDiscovered` events to an array `signals` which is key of `message`. Minimum size of -`signals` is 1 item. - -Do not make a request for every single event. - -Possible algorithm of batching: -1. If the event is FSSSignalDiscovered, store it to the temporal list and proceed to next event. -2. If the next event is also FSSSignalDiscovered, repeat 1. -3. If the next event is any other or there is no other event for more than 10 seconds, send the - temporal list in a single message to EDDN. - -### Elisions -You MUST remove the following key/value pairs from the data: - - - `TimeRemaining` key/value pair. - -You MUST refuse to send the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. - -### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### systemName -You SHOULD add a `systemName` string containing the system name from the last `FSDJump`, `CarrierJump`, or `Location` -event There exists problem when you gets `FSSSignalDiscovered` before -`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. - -#### StarPos -You SHOULD add a `StarPos` array containing the system co-ordinates from the -last `FSDJump`, `CarrierJump`, or `Location` event. There exists problem when you gets `FSSSignalDiscovered` before -`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. - -## Receivers -Receivers should remember: `horizons`, `odyssey`, `systemName`, `StarPos` are optional key/value pairs, it means you -should not rely on it will appear in every EDDN event. - -## Examples -This is a few example of messages that passes current `FSSSignalDiscovered` schema. -1. A message without `horizons`, `odyssey`, `systemName`, `StarPos` fields. -```json -{ - "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", - "header":{ - "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", - "softwareName":"an software", - "softwareVersion":"a version", - "uploaderID":"an uploader" - }, - "message":{ - "event":"FSSSignalDiscovered", - "signals":[ - { - "timestamp":"2021-07-30T19:03:08Z", - "event":"FSSSignalDiscovered", - "SystemAddress":1774711381, - "SignalName":"EXPLORER-CLASS X2X-74M", - "IsStation":true - } - ] - } -} -``` - -2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields which says it sent from Odyssey. -```json -{ - "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", - "header":{ - "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", - "softwareName":"an software", - "softwareVersion":"a version", - "uploaderID":"an uploader" - }, - "message":{ - "event":"FSSSignalDiscovered", - "signals":[ - { - "timestamp":"2021-07-30T19:03:08Z", - "event":"FSSSignalDiscovered", - "SystemAddress":1774711381, - "SignalName":"EXPLORER-CLASS X2X-74M", - "IsStation":true - }, - { - "timestamp":"2020-12-31T14:14:01Z", - "event":"FSSSignalDiscovered", - "SystemAddress":216054883492, - "SignalName":"$USS_NonHumanSignalSource;", - "USSType":"$USS_Type_NonHuman;", - "SpawningState":"$FactionState_None;", - "SpawningFaction":"$faction_none;", - "ThreatLevel":5 - } - ], - "StarPos": [ - 8.1875, - 124.21875, - -38.5 - ], - "StarSystem": "HIP 56186", - "horizons": true, - "odyssey": true - } -} -``` +# EDDN FSSSignalDiscovered Schema + +## Introduction +Here we document how to take data from an ED `FSSSignalDiscovered` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +## Senders +The only data source for this schema is the ED Journal event +`FSSSignalDiscovered`. + +### Batching +You MUST coalesce contiguouys runs of `FSSSignalDiscovered` events into a +single `signals` array in the message. Minimum size of `signals` is 1 item. + +Do not make a request for every single event other than where they occur +singly (such as when a player utilises the FSS to zoom into USS individually). + +Suggested algorithm for batching: + +1. You will need to track the current location from `Location`, `FSDJump` and + `CarrierJump` events. This is in order to add the top-level augmentation + of `StarSystem` (system name) and `StarPos`. You will need to record: + 1. `SystemAddress` - for cross-checking. + 2. `StarSystem` - name of the star system. + 3. `StarPos` - the galactic co-ordinates of the system. +2. If the event is `FSSSignalDiscovered`, store it to the temporal list. +3. If the event is any other, then: + 1. check if it is `Location`, `FSDJump` or `CarrierJump` - if so you should + use this new location in the message. + 2. If it is not one of those events then you should use the tracked + location from the prior such event. + + Now construct the full `fsssignaldiscovered` schema message using the + tracked location and the stored list of events. *You **MUST** check that + the `SystemAddress` for each `FSSSignalDiscovered` event matches the + tracked location.* If there is a mis-match then drop that event. +4. Each batch of signals will be contiguous and thus should have the same + timestamp, perhaps +/- 1 second at the ends. As such you only need one + single `timestamp` in the `message` and should use that of the first event + you include. + +Point 3i/ii above are because in the current (3.8.0.406) Horizons client the +`FSSSignalDiscovered` events arrive after `Location`/`FSDJump`/`CarrierJump`, +but in the current (4.0.0.1302) Odyssey client they arrive before such events. + +Thus, in Horizons you use the last-tracked location, but in Odyssey you use +the "just arrived" location. + +Manually FSS-scanned USS type signals will come in one by one, possibly with +other events between them (such as `Music` due to zooming in/out in FSS). +There is no need to attempt batching these together if separated by other +events, even though you'll be using the `timestamp` of the first on the +message, despite the actual time-line being dependent on how quickly the +player scans them. + +This batching is more concerned with not causing an EDDN message per event +upon entry into a system. + +### Elisions +You MUST remove the following key/value pairs from the data: + + - `TimeRemaining` key/value pair (will be present on USS). This has a slight + PII nature and is also very ephemeral. + +You MUST drop the whole `FSSSignalDiscovered` event if `USSType` key +has `$USS_Type_MissionTarget;` value. Only the Cmdr with the mission has any +use of these. There's not even a statistical use. + +### Augmentations +#### horizons flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### odyssey flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### StarSystem +You **MUST** add a `StarSystem` string containing the system name from the last +tracked location. You **MUST** cross-check each `FSSSignalDiscovered` +->`SystemAddress` value to ensure it matches. If it does not, you **MUST** +drop the event. + +#### StarPos +You **MUST** add a `StarPos` array containing the system co-ordinates from the +tracked location. You **MUST** cross-check each `FSSSignalDiscovered` +->`SystemAddress` value to ensure it matches. If it does not, you **MUST** +drop the event. + +## Receivers +### Augmentations are 'SHOULD', not 'MUST' +Receivers should remember that `horizons`, `odyssey`, `StarSystem`, `StarPos` +are optional key/value pairs. You **SHOULD NOT** rely on them being present +in any given event. + +### Duplicate messages from 'busy' systems +When a system is particularly full of signals, such as when many Fleet Carriers +are present, it has been observed that the game repeats the identical +sequence of `FSSSignalDiscovered` events. So you might receive what looks like +a duplicate message, other than the timestamp (if the timestamp is the same +then the EDDN Relay should drop the duplicate). + +## Examples +This is a few example of messages that passes current `FSSSignalDiscovered` schema. +1. A message without `horizons` or `odyssey` augmentations. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"a software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "timestamp":"2021-11-06T22:48:42Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1774711381, + "signals":[ + { + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + } + ], + "StarSystem":"HR 1185", + "StarPos": [ + -64.66, -148.94, -330.41 + ] + } +} +``` + +2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields which says it sent from Odyssey. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"a software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "timestamp":"2021-11-06T22:48:42Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1350507186531, + "signals":[ + { + "event":"FSSSignalDiscovered", + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + }, + { + "event":"FSSSignalDiscovered", + "SignalName":"$USS_NonHumanSignalSource;", + "USSType":"$USS_Type_NonHuman;", + "SpawningState":"$FactionState_None;", + "SpawningFaction":"$faction_none;", + "ThreatLevel":5 + } + ], + "StarPos": [ + 8.1875, + 124.21875, + -38.5 + ], + "StarSystem": "HIP 56186", + "horizons": true, + "odyssey": true + } +} +``` From d33c752c36ffaf76ef68c991489c1e6ed1dd5510 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 14 Jun 2022 14:03:49 +0100 Subject: [PATCH 033/234] schemas/fsssignaldiscovered: README: Remove timestamp/SystemAddress from each signal --- schemas/fsssignaldiscovered-README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 6665b22..f5432dd 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -69,6 +69,9 @@ You MUST drop the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. Only the Cmdr with the mission has any use of these. There's not even a statistical use. +Because we only have a `message` level `timestamp` and `SystemAddress` these +should be removed from each member of the `signals` array. + ### Augmentations #### horizons flag You SHOULD add this key/value pair, using the value from the `LoadGame` event. From 3dbfd181886f32c8c81aebf3d51460188fbe6f54 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 15 Jun 2022 11:52:25 +0100 Subject: [PATCH 034/234] schemas/fsssignaldiscovered: Rework for per-signal timestamp & misc. clarifications * Per-signal `timestamp` due to the nature of USS signals. * Keeping `message`->`timestamp` so any Listener relying on it being there isn't tripped up (it's present in all other schemas). * Specifiy that each `signals` member should have `event` key/value pair removed. * Mandate removal of `SystemAddress` from each `signals` member. * Receivers: Only `horizons` and `odyssey` augmentations are optional. * Updated examples for these changes. --- schemas/fsssignaldiscovered-README.md | 35 ++++++++++++++++++--------- schemas/fsssignaldiscovered-v1.0.json | 7 +++++- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index f5432dd..ddbb9aa 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -12,11 +12,12 @@ The only data source for this schema is the ED Journal event `FSSSignalDiscovered`. ### Batching -You MUST coalesce contiguouys runs of `FSSSignalDiscovered` events into a +You MUST coalesce contiguous runs of `FSSSignalDiscovered` events into a single `signals` array in the message. Minimum size of `signals` is 1 item. Do not make a request for every single event other than where they occur -singly (such as when a player utilises the FSS to zoom into USS individually). +singly (such as when a player utilises the FSS to zoom into USS individually, +if there is a different following event). Suggested algorithm for batching: @@ -29,18 +30,16 @@ Suggested algorithm for batching: 2. If the event is `FSSSignalDiscovered`, store it to the temporal list. 3. If the event is any other, then: 1. check if it is `Location`, `FSDJump` or `CarrierJump` - if so you should - use this new location in the message. + use this new location in the message for the augmentations. 2. If it is not one of those events then you should use the tracked - location from the prior such event. + location from the prior such event for the augmentations. Now construct the full `fsssignaldiscovered` schema message using the tracked location and the stored list of events. *You **MUST** check that the `SystemAddress` for each `FSSSignalDiscovered` event matches the tracked location.* If there is a mis-match then drop that event. -4. Each batch of signals will be contiguous and thus should have the same - timestamp, perhaps +/- 1 second at the ends. As such you only need one - single `timestamp` in the `message` and should use that of the first event - you include. +4. Use the `timestamp` of the first signal in the batch as the top-level + `timestamp` in the `message` object. Point 3i/ii above are because in the current (3.8.0.406) Horizons client the `FSSSignalDiscovered` events arrive after `Location`/`FSDJump`/`CarrierJump`, @@ -60,6 +59,10 @@ This batching is more concerned with not causing an EDDN message per event upon entry into a system. ### Elisions +Remove the `event` key/pair from each member of the `signals` array. Including +this would be redundant as by definition we're sending `FSSSignalDiscovered` +events on this schema. + You MUST remove the following key/value pairs from the data: - `TimeRemaining` key/value pair (will be present on USS). This has a slight @@ -69,8 +72,13 @@ You MUST drop the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. Only the Cmdr with the mission has any use of these. There's not even a statistical use. -Because we only have a `message` level `timestamp` and `SystemAddress` these -should be removed from each member of the `signals` array. +Because of the location cross-check the `SystemAddress` is in the top-level +`message` object, and thus you **MUST** remove such from each signal in the +array. + +Do **NOT** remove the `timestamp` from each signal in the array. Whilst these +should be identical for a "just logged in or arrived in system" set of signals, +this is not true of manually FSS scanned USS signals. ### Augmentations #### horizons flag @@ -93,7 +101,7 @@ drop the event. ## Receivers ### Augmentations are 'SHOULD', not 'MUST' -Receivers should remember that `horizons`, `odyssey`, `StarSystem`, `StarPos` +Receivers should remember that `horizons` and `odyssey` augmentations are optional key/value pairs. You **SHOULD NOT** rely on them being present in any given event. @@ -122,6 +130,7 @@ This is a few example of messages that passes current `FSSSignalDiscovered` sche "SystemAddress":1774711381, "signals":[ { + "timestamp":"2021-11-06T22:48:42Z", "SignalName":"EXPLORER-CLASS X2X-74M", "IsStation":true } @@ -150,11 +159,13 @@ This is a few example of messages that passes current `FSSSignalDiscovered` sche "SystemAddress":1350507186531, "signals":[ { + "timestamp":"2021-11-06T22:48:42Z", "event":"FSSSignalDiscovered", "SignalName":"EXPLORER-CLASS X2X-74M", "IsStation":true }, - { + { + "timestamp":"2021-11-06T22:48:42Z", "event":"FSSSignalDiscovered", "SignalName":"$USS_NonHumanSignalSource;", "USSType":"$USS_Type_NonHuman;", diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index f649e2f..2664989 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -49,7 +49,8 @@ }, "timestamp": { "type" : "string", - "format" : "date-time" + "format" : "date-time", + "description" : "Duplicate of the first signal's timestamp, for commonality with other schemas." }, "SystemAddress": { "type": "integer" @@ -63,6 +64,10 @@ "required": ["SignalName"], "description": "Single FSSSignalDiscovered event", "properties": { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, "SignalName": { "type": "string" }, "IsStation": { "type": "boolean" }, "USSType": { From 5d945297e14572531e2a7113d5920096d2de154d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 15 Jun 2022 12:42:53 +0100 Subject: [PATCH 035/234] schemas/fsssignaldiscovered: `timestamp` required per signal --- schemas/fsssignaldiscovered-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 2664989..18f8fef 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -61,7 +61,7 @@ "minItems": 1, "items": { "type": "object", - "required": ["SignalName"], + "required": ["timestamp", "SignalName"], "description": "Single FSSSignalDiscovered event", "properties": { "timestamp": { From 56b4e20238479251c4dcd2d62d0b48a033ff9d9a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 15 Jun 2022 14:03:02 +0100 Subject: [PATCH 036/234] Settings.py: Add missing `/` for `fsssignaldiscovered/1` schema --- src/eddn/conf/Settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 0b3253a..28e9b21 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -82,8 +82,8 @@ class _Settings(object): "https://eddn.edcd.io/schemas/fssbodysignals/1" : "schemas/fssbodysignals-v1.0.json", "https://eddn.edcd.io/schemas/fssbodysignals/1/test" : "schemas/fssbodysignals-v1.0.json", - "https://eddn.edcd.io/schemasfsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", - "https://eddn.edcd.io/schemasfsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemas/fsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemas/fsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ From ff83ede9480f2da8b7ebe33ece4f5d4604445e65 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 16 Jun 2022 13:27:11 +0100 Subject: [PATCH 037/234] Gateway: Remove all form-encoded support This causes issues, at the least, with compressed messages that 'look' like they decompressed body is form-encoded. 18385 messages in the last month rejected due to this. No actually valid form-encoded messages in that time frame. --- docs/Developers.md | 7 +++---- src/eddn/Gateway.py | 35 +---------------------------------- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index 6936bbb..66aa1eb 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -127,14 +127,13 @@ The body of an EDDN message is a JSON object in UTF-8 encoding. If you do not compress this body then you MUST set a `Content-Type` header of `applicaton/json`. -For historical reasons URL form-encoded data *is* supported, **but this is -deprecated and no new software should attempt this method**. We -purposefully do not further document the exact format for this. - You *MAY* use gzip compression on the body of the message, but it is not required. If you do compress the body then you **MUST* send a `Content-Type` header of `gzip` instead of `application/json`. +**Due to issues when messages are compressed, form-encoded data is NO LONGER +SUPPORTED as of 2022-06-16.** + You should be prepared to handle all scenarios where sending of a message fails: diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 19912ed..0524478 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -9,7 +9,6 @@ import gevent import hashlib import logging import simplejson -import urlparse import zlib import zmq.green as zmq from datetime import datetime @@ -167,42 +166,10 @@ def get_decompressed_message(): message_body = zlib.decompress(request.body.read(), -15) logger.debug('Resulting message_body:\n%s\n' % (message_body)) - # At this point, we're not sure whether we're dealing with a straight - # un-encoded POST body, or a form-encoded POST. Attempt to parse the - # body. If it's not form-encoded, this will return an empty dict. - form_enc_parsed = urlparse.parse_qs(message_body) - if form_enc_parsed: - logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address())) - # This is a form-encoded POST. The value of the data attrib will - # be the body we're looking for. - try: - message_body = form_enc_parsed['data'][0] - - except (KeyError, IndexError): - logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address()) - raise MalformedUploadError( - "No 'data' POST key/value found. Check your POST key " - "name for spelling, and make sure you're passing a value." - ) - - else: - logger.debug('Request is *NOT* form-encoded') - else: logger.debug('Content-Encoding indicates *not* compressed...') - # Uncompressed request. Bottle handles all of the parsing of the - # POST key/vals, or un-encoded body. - data_key = request.forms.get('data') - if data_key: - logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address())) - # This is a form-encoded POST. Support the silly people. - message_body = data_key - - else: - logger.debug('Plain POST request detected...') - # This is a non form-encoded POST body. - message_body = request.body.read() + message_body = request.body.read() return message_body From 81344cb67aab614dac509e85106c46a8555cb598 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:16:04 +0000 Subject: [PATCH 038/234] scripts/eddn-errors: Remove EDMC codexentry trap --- scripts/eddn-report-log-errors | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 81618a8..1a7dd5b 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -138,17 +138,8 @@ def process_file(input_file: str) -> None: elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest - if software_version >= semantic_version.Version.coerce('5.3.0'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/codexentry/1': - # - if matches.group('err_msg') == 'Failed Validation "[]"': - pass - - else: - print(matches.group('err_msg')) - print(line) - - elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': + if software_version >= semantic_version.Version.coerce('5.4.1'): + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': # if matches.group('err_msg') == 'Failed Validation "[]"': # From 66dfedd2af5bc891a5e3083b986f36ef8e284e99 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:16:40 +0000 Subject: [PATCH 039/234] scripts/eddn-errors: Remove EDMC misc errors trap --- scripts/eddn-report-log-errors | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 1a7dd5b..a3940b4 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -139,23 +139,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest if software_version >= semantic_version.Version.coerce('5.4.1'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': - # - if matches.group('err_msg') == 'Failed Validation "[]"': - # - pass - - elif matches.group('err_msg').startswith( - 'Failed Validation "[ - pass - - else: - print(matches.group('err_msg')) - print(line) - - elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fssdiscoveryscan/1': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fssdiscoveryscan/1': if matches.group('err_msg') == 'Failed Validation "[]"': # pass From f6345b1705e46af99c30d7c62474a0bb7ffe5c7d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:17:13 +0000 Subject: [PATCH 040/234] eddn/report-errors: Remove other EDMC misc trap --- scripts/eddn-report-log-errors | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index a3940b4..3302135 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -139,15 +139,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest if software_version >= semantic_version.Version.coerce('5.4.1'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fssdiscoveryscan/1': - if matches.group('err_msg') == 'Failed Validation "[]"': - # - pass - - else: - print(line) - - elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': if matches.group('err_msg') == 'Failed Validation "[]"': # pass From 651bf998a73228961592eb34f2fbbc0324c7b635 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:18:27 +0000 Subject: [PATCH 041/234] scripts/eddn-errors: Remove EDMC approachsetttlemtn/missing Latitude trap --- scripts/eddn-report-log-errors | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 3302135..96e24ce 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -139,16 +139,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest if software_version >= semantic_version.Version.coerce('5.4.1'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': - if matches.group('err_msg') == 'Failed Validation "[]"': - # - pass - - else: - print(line) - - else: - print(line) + print(line) elif matches.group('software_name') == 'Elite G19s Companion App': # From 4ceac7bc8bc09d31d7e45152f6b817ac95fe4f3f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:20:32 +0000 Subject: [PATCH 042/234] scripts/eddn-errors: Bump EDD to 15.0.4.0 --- scripts/eddn-report-log-errors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 96e24ce..e00e0b8 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -75,7 +75,7 @@ def process_file(input_file: str) -> None: ################################################################### if matches.group('software_name') == 'EDDiscovery': # https://github.com/EDDiscovery/EDDiscovery/releases/latest - if software_version >= semantic_version.Version.coerce('15.0.0.0'): + if software_version >= semantic_version.Version.coerce('15.0.4.0'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': if matches.group('err_msg') == 'Failed Validation "[]"': # From 8e3e279592d79d7b1678095e004ef164f7f128cf Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:20:52 +0000 Subject: [PATCH 043/234] scripts/eddn-errors: EDD approachsettlement/Latitude trap removed --- scripts/eddn-report-log-errors | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index e00e0b8..737ec5b 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -77,11 +77,7 @@ def process_file(input_file: str) -> None: # https://github.com/EDDiscovery/EDDiscovery/releases/latest if software_version >= semantic_version.Version.coerce('15.0.4.0'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': - if matches.group('err_msg') == 'Failed Validation "[]"': - # - pass - - elif matches.group('err_msg') == 'Failed Validation "[]"': + if matches.group('err_msg') == 'Failed Validation "[]"': # pass From 13215806e4850cd6a169deaf5670591b6964450a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:21:41 +0000 Subject: [PATCH 044/234] scripts/eddn-errors: EDD approachsettlement/_Localised trap removed --- scripts/eddn-report-log-errors | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 737ec5b..d9b4236 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -76,18 +76,6 @@ def process_file(input_file: str) -> None: if matches.group('software_name') == 'EDDiscovery': # https://github.com/EDDiscovery/EDDiscovery/releases/latest if software_version >= semantic_version.Version.coerce('15.0.4.0'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/approachsettlement/1': - if matches.group('err_msg') == 'Failed Validation "[]"': - # - pass - - else: - print(line) - - else: - print(line) - - elif software_version >= semantic_version.Version.coerce('12.1.7.0'): if matches.group('schema_ref') in ( 'https://eddn.edcd.io/schemas/shipyard/2', 'https://eddn.edcd.io/schemas/outfitting/2', From 75f744abe56b947f230300c69b16d93bf11ef2c8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:22:22 +0000 Subject: [PATCH 045/234] scripts/eddn-errors: EDD shipyard/outfitting trap removed --- scripts/eddn-report-log-errors | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index d9b4236..700d519 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -76,19 +76,7 @@ def process_file(input_file: str) -> None: if matches.group('software_name') == 'EDDiscovery': # https://github.com/EDDiscovery/EDDiscovery/releases/latest if software_version >= semantic_version.Version.coerce('15.0.4.0'): - if matches.group('schema_ref') in ( - 'https://eddn.edcd.io/schemas/shipyard/2', - 'https://eddn.edcd.io/schemas/outfitting/2', - ): - # Reported via Discord PM to Robby 2022-01-07 - if matches.group('err_msg') == 'Failed Validation "[]"': - pass - - else: - print(line) - - else: - print(line) + print(line) elif matches.group('software_name') == 'EDDLite': # https://github.com/EDDiscovery/EDDLite/releases/tag/latest From e297f24c73e261152e3ccd2fc76cd5158231fea0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:23:03 +0000 Subject: [PATCH 046/234] scripts/eddn-errors: EDD version now 2.2.0 --- scripts/eddn-report-log-errors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 700d519..4635aed 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -80,7 +80,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDDLite': # https://github.com/EDDiscovery/EDDLite/releases/tag/latest - if software_version >= semantic_version.Version.coerce('2.0.0'): + if software_version >= semantic_version.Version.coerce('2.2.0'): if matches.group('schema_ref') in ( 'https://eddn.edcd.io/schemas/shipyard/2', 'https://eddn.edcd.io/schemas/outfitting/2', From bd7a0d6ab92d6138e03d4cbfea2a1e86a6bae472 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:23:37 +0000 Subject: [PATCH 047/234] scripts/eddn-errors: EDDLite - remove traps --- scripts/eddn-report-log-errors | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 4635aed..42032ff 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -81,27 +81,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDDLite': # https://github.com/EDDiscovery/EDDLite/releases/tag/latest if software_version >= semantic_version.Version.coerce('2.2.0'): - if matches.group('schema_ref') in ( - 'https://eddn.edcd.io/schemas/shipyard/2', - 'https://eddn.edcd.io/schemas/outfitting/2', - ): - # Failed Validation "[]" - if ( - matches.group('err_msg').startswith('Failed Validation "[]"') - ): - # - pass - - elif matches.group('err_msg') == 'Failed Validation "[]"': - # Reported via Discord PM to Robby 2022-01-07 - pass - - else: - print(line) - - else: - print(line) + print(line) elif matches.group('software_name') == 'EDDI': # https://github.com/EDCD/EDDI/releases/latest From 353c5a23812f8db55e25bf1cca9cb4a8a33e94d0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:26:31 +0000 Subject: [PATCH 048/234] scripts/eddn-errors: EDSM - Console version now 1.0.2 --- scripts/eddn-report-log-errors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 42032ff..32bf2ab 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -137,7 +137,7 @@ def process_file(input_file: str) -> None: elif matches.group('software_name') == 'EDSM - Console': # It's in-browser, no public source/releases - if software_version >= semantic_version.Version.coerce('1.0'): + if software_version >= semantic_version.Version.coerce('1.0.2'): if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': if matches.group('journal_event') == 'Scan': # From 159c02caf162b49475d603068994b0be7575fdf1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:31:19 +0000 Subject: [PATCH 049/234] scripts/eddn-errors: Also ignore "EVA [Android]" --- scripts/eddn-report-log-errors | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 32bf2ab..33234c0 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -210,7 +210,10 @@ def process_file(input_file: str) -> None: # Abandoned/unmaintained project # # - elif matches.group('software_name') == 'EVA [iPhone]': + elif ( + matches.group('software_name') == 'EVA [iPhone]' + or matches.group('software_name') == 'EVA [Android]' + ): pass #################################################################### From 83a7d451ef81e27e824a4e737df0810bdde9c598 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 19 Jun 2022 10:37:20 +0000 Subject: [PATCH 050/234] scripts/eddn-errors: EDMC - two fsssignaldiscovered issues --- scripts/eddn-report-log-errors | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 33234c0..82a3568 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -91,7 +91,20 @@ def process_file(input_file: str) -> None: elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest if software_version >= semantic_version.Version.coerce('5.4.1'): - print(line) + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fsssignaldiscovered/1': + if matches.group('err_msg') == 'Failed Validation "[]"': + # + pass + + elif matches.group('err_msg') == 'Failed Validation "[]"': + # + pass + + else: + print(line) + + else: + print(line) elif matches.group('software_name') == 'Elite G19s Companion App': # From 3fa501213981bbc28042d2590ec3c2cf2659f166 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 20 Jun 2022 09:56:40 +0000 Subject: [PATCH 051/234] scripts/eddn-errors: EDMC - Put back in trap for mystery disallowed --- scripts/eddn-report-log-errors | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 82a3568..a3c77ef 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -91,7 +91,14 @@ def process_file(input_file: str) -> None: elif matches.group('software_name').startswith('E:D Market Connector'): # https://github.com/EDCD/EDMarketConnector/releases/latest if software_version >= semantic_version.Version.coerce('5.4.1'): - if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fsssignaldiscovered/1': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': + if matches.group('err_msg').startswith( + 'Failed Validation "[ + pass + + elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fsssignaldiscovered/1': if matches.group('err_msg') == 'Failed Validation "[]"': # pass From 723e966b37092c289a5c2db3325a248318cb7fd2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jun 2022 14:48:11 +0100 Subject: [PATCH 052/234] docs/README schemas: Be explicit about sub-second timestamp resolution --- schemas/README-EDDN-schemas.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 4620dd1..7905cbb 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -49,11 +49,17 @@ contents all Schemas specify a top-level JSON Object with the data: Each `message` object must have, at bare minimum: -1. `timestamp` - string date and time in ISO8601 format. Whilst this - technically allows for any timezone to be cited you SHOULD provide this in - UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are - doing this properly. Do not claim 'Z' whilst actually using a local time - that is offset from UTC. +1. `timestamp` - string date and time in ISO8601 format. + 1. Whilst this technically allows for any timezone to be cited you SHOULD + provide this in UTC, aka 'Zulu Time' as in the example above. + You MUST ensure that you are doing this properly. + Do not claim 'Z' whilst actually using a local time that is offset from + UTC. + 2. Historically we had never been explicit about if Senders should include + sub-second resolution in the timestamps, or if Listeners should be + prepared to accept such. As of 2022-06-24 we are explicitly stating that + Senders **MAY** include sub-second resolution, and Listeners **MUST** + be prepared to accept such. If you are only utilising Journal-sourced data then simply using the value from there should be sufficient as the PC game client is meant to From a7b96c3228be49e23c1f137f052525d4181e58ce Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 24 Jun 2022 14:52:43 +0100 Subject: [PATCH 053/234] scripts/eddn errors: Use 'in' form for EVA variants --- scripts/eddn-report-log-errors | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index a3c77ef..f50900c 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -230,10 +230,7 @@ def process_file(input_file: str) -> None: # Abandoned/unmaintained project # # - elif ( - matches.group('software_name') == 'EVA [iPhone]' - or matches.group('software_name') == 'EVA [Android]' - ): + elif matches.group('software_name') in ('EVA [iPhone]', 'EVA [iPad]', 'EVA [Android]'): pass #################################################################### From 930632b0e9212bb08e8e0cbcad9db269b6b8ec10 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 26 Jul 2022 12:43:40 +0100 Subject: [PATCH 054/234] README: Change Discord URL to channel-specific one. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6749a49..02c97df 100644 --- a/README.md +++ b/README.md @@ -171,5 +171,5 @@ Hosting is currently provided by the ### Contacting the EDDN team -* [EDCD Discord](https://discord.gg/XBsdCq9) - **Use the `#eddn` channel**. +* [EDCD Discord - #eddn channel](https://discord.gg/DdqJ8nWVGc) * [E:D forum thread](https://forums.frontier.co.uk/threads/elite-dangerous-data-network-eddn.585701/#post-9400060) From 15e385287aa63281a8a8a14806db36d98c7e3000 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 11:58:38 +0000 Subject: [PATCH 055/234] Initial setup for proper development with linting etc --- .coveragerc | 6 +++ .flake8 | 43 +++++++++++++++++++++ .mypy.ini | 4 ++ .pre-commit-config.yaml | 84 +++++++++++++++++++++++++++++++++++++++++ .python-version | 1 + requirements-dev.txt | 32 ++++++++++++++++ 6 files changed, 170 insertions(+) create mode 100644 .coveragerc create mode 100644 .flake8 create mode 100644 .mypy.ini create mode 100644 .pre-commit-config.yaml create mode 100644 .python-version create mode 100644 requirements-dev.txt diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..019facb --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +omit = + # The tests themselves + tests/* + # Any venv files + venv/* diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..49e1306 --- /dev/null +++ b/.flake8 @@ -0,0 +1,43 @@ +[flake8] +# Show exactly where in a line the error happened +#show-source = True + +max-line-length = 120 +# Add _ as a builtin for localisation stuff +builtins = _ + +# check syntax in doctests +doctests = True +max-complexity = 15 +per-file-ignores = ./EDMC.py:E402 + +# Plugin configs +# required plugins: + +# https://github.com/Melevir/flake8-cognitive-complexity +# Provides cognitive complexity checking + +# https://github.com/adamchainz/flake8-comprehensions +# Checks list/dict/set/* comprehensions for simplification or replacement + +# https://github.com/PyCQA/pep8-naming +# checks names against PEP8 rules (eg CamelCase for class names) + +# https://github.com/best-doctor/flake8-annotations-coverage +# Checks code for type annotations + +# https://pypi.org/project/flake8-isort/ +# Ensures imports are well sorted + +# https://pypi.org/project/flake8-noqa/ +# Ensures that noqa statements are correctly formed + +# https://github.com/MichaelKim0407/flake8-use-fstring +# Prefers fstrings over .format and modulo based formatters + +max-cognitive-complexity = 15 + +# require that all noqa directves disable specific errors +noqa-require-code = True + +docstring-convention = numpy diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..03840fd --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,4 @@ +[mypy] +follow_imports = skip +ignore_missing_imports = True +scripts_are_modules = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a8bfb90 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,84 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: 'v3.4.0' + hooks: + - id: check-merge-conflict + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + + #- repo: https://github.com/pre-commit/mirrors-autopep8 + # rev: '' + # hooks: + # - id: autopep8 + +### # flake8 --show-source +### - repo: https://gitlab.com/pycqa/flake8 +### rev: '' +### hooks: +### - id: flake8 +# +# Try using local flake8 +- repo: local + hooks: + - id: flake8 + name: flake8 + entry: flake8 + language: system + types: [ python ] + +- repo: https://github.com/pre-commit/pygrep-hooks + rev: 'v1.8.0' + hooks: + - id: python-no-eval + - id: python-no-log-warn +# This is a pain where a comment begins with the word 'type' otherwise +# - id: python-use-type-annotations + +# mypy - static type checking +# mypy --follow-imports skip +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.812' + hooks: + - id: mypy + args: [ "--follow-imports", "skip", "--ignore-missing-imports", "--scripts-are-modules" ] + +### # pydocstyle.exe +### - repo: https://github.com/FalconSocial/pre-commit-mirrors-pep257 +### rev: '' +### hooks: +### - id: pep257 # docstring conventions +### # Alternate https://github.com/PyCQA/pydocstyle + +# - repo: https://github.com/digitalpulp/pre-commit-php +# rev: '' +# hooks: +# -id: php-unit + +# safety.exe check -r requirements.txt +- repo: https://github.com/Lucas-C/pre-commit-hooks-safety + rev: 'v1.2.1' + hooks: + - id: python-safety-dependencies-check + entry: safety + args: [check, --bare, -r] + language: system + +# Check translation comments are up to date +- repo: local + hooks: + - id: LANG_comments + name: 'LANG comments' + language: system + entry: python scripts/find_localised_strings.py --compare-lang L10n/en.template --directory . --ignore coriolis-data --ignore dist.win32 + pass_filenames: false + always_run: true + +default_language_version: + python: python3.9 + +default_stages: [ commit, push ] + +#files: '([^\.].+/)*.py' diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..f69abe4 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.9.7 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..a960986 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,32 @@ +# So that you don't get warnings like: +# Using legacy 'setup.py install' for flake8-annotations-coverage, since package 'wheel' is not installed. +wheel + +# Static analysis tools +flake8==4.0.1 +flake8-annotations-coverage==0.0.5 +flake8-cognitive-complexity==0.1.0 +flake8-comprehensions==3.7.0 +flake8-docstrings==1.6.0 +isort==5.10.0 +flake8-isort==4.1.1 +flake8-json==21.7.0 +flake8-noqa==1.2.0 +flake8-polyfill==1.0.2 +flake8-use-fstring==1.3 + +mypy==0.910 +pep8-naming==0.12.1 +safety==1.10.3 +types-requests==2.25.11 + +# Code formatting tools +autopep8==1.6.0 + +# Testing +pytest==6.2.5 +pytest-cov==3.0.0 # Pytest code coverage support +coverage[toml]==6.1.1 # pytest-cov dep. This is here to ensure that it includes TOML support for pyproject.toml configs + +# All of the normal requirements +-r requirements.txt From b93ee82c5f0a83854d8c3e58371aee05be9723da Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 12:03:54 +0000 Subject: [PATCH 056/234] requirements: Updated for python 3.9.7 --- requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2e557c5..161fa97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -# These are the versions we *know* work under Python 2.7.16 -argparse==1.2.1 -bottle==0.12.15 -enum34==1.1.6 -gevent==1.3.7 -jsonschema==2.6.0 -pyzmq==17.1.2 +# Versions that installed under Python 3.9.7 Thu Nov 4 12:03:41 GMTST 2021 +argparse==1.4.0 +bottle==0.12.19 +enum34==1.1.10 +gevent==21.8.0 +jsonschema==4.2.0 +pyzmq==22.3.0 requests==2.25.1 simplejson==3.16.0 strict_rfc3339==0.7 -mysql-connector-python==8.0.17 +mysql-connector-python==8.0.27 From fea561920a26aa034225377c3cd068b844fc32d0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 12:04:45 +0000 Subject: [PATCH 057/234] .pre-commit: No need for LANG section --- .pre-commit-config.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8bfb90..25f6535 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -66,16 +66,6 @@ repos: args: [check, --bare, -r] language: system -# Check translation comments are up to date -- repo: local - hooks: - - id: LANG_comments - name: 'LANG comments' - language: system - entry: python scripts/find_localised_strings.py --compare-lang L10n/en.template --directory . --ignore coriolis-data --ignore dist.win32 - pass_filenames: false - always_run: true - default_language_version: python: python3.9 From ca3417c9c0e7f0e4dcb06b7db5f4d620a0188d0a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 12:09:40 +0000 Subject: [PATCH 058/234] Gateway: Better top-level docstring --- src/eddn/Gateway.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 0524478..baedc27 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -1,6 +1,8 @@ # coding: utf8 """ +EDDN Gateway, which receives message from uploaders. + Contains the necessary ZeroMQ socket and a helper function to publish market data to the Announcer daemons. """ From b127c638873f2865de71652e32013f1d6fb7a9d6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 14:44:49 +0100 Subject: [PATCH 059/234] Gateway: Proper import order --- src/eddn/Gateway.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index baedc27..84b7657 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -7,25 +7,29 @@ Contains the necessary ZeroMQ socket and a helper function to publish market data to the Announcer daemons. """ import argparse -import gevent import hashlib import logging -import simplejson import zlib -import zmq.green as zmq from datetime import datetime +import gevent +import simplejson +import urlparse +import zmq.green as zmq +from gevent import monkey from pkg_resources import resource_string -# import os from eddn.conf.Settings import Settings, loadConfig -from eddn.core.Validator import Validator, ValidationSeverity +from eddn.core.Validator import ValidationSeverity, Validator + +# import os + -from gevent import monkey monkey.patch_all() import bottle -from bottle import Bottle, run, request, response, get, post +from bottle import Bottle, get, post, request, response, run bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB + app = Bottle() logger = logging.getLogger(__name__) @@ -49,6 +53,7 @@ validator = Validator() # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector + statsCollector = StatsCollector() statsCollector.start() From 13765c70d5f817f8ef4adc04d0b37f491691526c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 12:12:37 +0000 Subject: [PATCH 060/234] monitor/eddn.js: PyCharm code cleanup --- contrib/monitor/js/eddn.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/contrib/monitor/js/eddn.js b/contrib/monitor/js/eddn.js index e105f59..79aab9a 100644 --- a/contrib/monitor/js/eddn.js +++ b/contrib/monitor/js/eddn.js @@ -75,8 +75,8 @@ var drillDownSoftware = false; var currentDrillDown = false; var softwaresSort = { field: 'today', order: 'desc' }; // Very first load sort order -var softwaresData = new Array(); // The last data we got from API -var softwaresViewData = new Array(); // The data for the current view +var softwaresData = []; // The last data we got from API +var softwaresViewData = []; // The data for the current view var softwaresVersion = {}; var softwaresJsGridDataController = function () { @@ -212,7 +212,7 @@ var softwaresNewJsGrid = function () { /* * No longer drilling down, so need to reset the data */ - softwaresViewData = new Array(); + softwaresViewData = []; softwaresData.forEach(function(software, s) { softwareSplit = software.name.split(' | '); name = softwareSplit[0]; @@ -296,7 +296,7 @@ var softwaresNewJsGrid = function () { /* * The data we need for this drilldown */ - softwaresViewData = new Array(); + softwaresViewData = []; softwaresData.forEach(function(software, s) { softwareSplit = software.name.split(' | '); var name = ""; @@ -453,7 +453,7 @@ var doUpdateSoftwares = function() * key: software name, including the version * value: dictionary with counts for: today, yesterday, total (all time) */ - softwaresData = new Array(); + softwaresData = []; $.each(softwaresTotals, function(softwareName, total){ var sw = { 'name': softwareName, 'today': 0, 'yesterday': 0, 'total': parseInt(total)}; @@ -466,7 +466,7 @@ var doUpdateSoftwares = function() /* * Now the data we need for the current view (overall data or a drilldown of a software) */ - softwaresViewData = new Array(); + softwaresViewData = []; softwaresData.forEach(function(software, s) { softwareSplit = software.name.split(' | '); var name = ""; @@ -512,7 +512,7 @@ var doUpdateSoftwares = function() var schemasSort = { field: 'today', order: 'desc' }; // Very first load sort order -var schemasData = new Array(); +var schemasData = []; var doUpdateSchemas = function() { @@ -542,7 +542,7 @@ var doUpdateSchemas = function() /* * Prepare 'schemasData' dictionary */ - schemasData = new Array(); + schemasData = []; $.each(schemasTotals, function(schema, total) { schemaName = schema.replace('http://schemas.elite-markets.net/eddn/', 'https://eddn.edcd.io/schemas/'); // Due to the schema renames and us merging them there could be more than one From bf432a970827e812e579b9f4a78bdee1df647476 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:39:27 +0000 Subject: [PATCH 061/234] Gateway: Remove un-used bottle imports --- src/eddn/Gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 84b7657..d318101 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -27,7 +27,7 @@ from eddn.core.Validator import ValidationSeverity, Validator monkey.patch_all() import bottle -from bottle import Bottle, get, post, request, response, run +from bottle import Bottle, request, response bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB app = Bottle() From 123769f8d49e8c5ac71599cb09011f5ad7a8a52f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:40:05 +0000 Subject: [PATCH 062/234] Gateway: More import fixup --- src/eddn/Gateway.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d318101..d2c4e5e 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -16,6 +16,7 @@ import gevent import simplejson import urlparse import zmq.green as zmq +from bottle import Bottle, request, response from gevent import monkey from pkg_resources import resource_string @@ -52,7 +53,7 @@ sender = context.socket(zmq.PUB) validator = Validator() # This import must be done post-monkey-patching! -from eddn.core.StatsCollector import StatsCollector +from eddn.core.StatsCollector import StatsCollector # noqa: E402 statsCollector = StatsCollector() statsCollector.start() From 3b4fde42db37d0261808e6387caf88ecc16c59a0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:41:58 +0000 Subject: [PATCH 063/234] Gateway: snake_case variables --- src/eddn/Gateway.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d2c4e5e..63f7fb0 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -55,8 +55,8 @@ validator = Validator() # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector # noqa: E402 -statsCollector = StatsCollector() -statsCollector.start() +stats_collector = StatsCollector() +stats_collector.start() def parse_cl_args(): @@ -118,8 +118,8 @@ def configure(): for binding in Settings.GATEWAY_SENDER_BINDINGS: sender.bind(binding) - for schemaRef, schemaFile in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): - validator.addSchemaResource(schemaRef, resource_string('eddn.Gateway', schemaFile)) + for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): + validator.addSchemaResource(schema_ref, resource_string('eddn.Gateway', schema_file)) def push_message(parsed_message, topic): @@ -137,7 +137,7 @@ def push_message(parsed_message, topic): send_message = "%s |-| %s" % (str(topic), compressed_msg) sender.send(send_message) - statsCollector.tally("outbound") + stats_collector.tally("outbound") def get_remote_address(): @@ -212,13 +212,13 @@ def parse_and_error_handle(data): # Here we check if an outdated schema has been passed if parsed_message["$schemaRef"] in Settings.GATEWAY_OUTDATED_SCHEMAS: response.status = '426 Upgrade Required' # Bottle (and underlying httplib) don't know this one - statsCollector.tally("outdated") + stats_collector.tally("outdated") return "FAIL: Outdated Schema: The schema you have used is no longer supported. Please check for an updated " \ "version of your application." - validationResults = validator.validate(parsed_message) + validation_results = validator.validate(parsed_message) - if validationResults.severity <= ValidationSeverity.WARN: + if validation_results.severity <= ValidationSeverity.WARN: parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z' parsed_message['header']['uploaderIP'] = get_remote_address() @@ -254,7 +254,7 @@ def parse_and_error_handle(data): pass response.status = 400 - statsCollector.tally("invalid") + stats_collector.tally("invalid") return "FAIL: Schema Validation: " + str(validationResults.messages) @@ -293,7 +293,7 @@ def upload(): return 'FAIL: Malformed Upload: ' + exc.message - statsCollector.tally("inbound") + stats_collector.tally("inbound") return parse_and_error_handle(message_body) @@ -309,7 +309,7 @@ def health_check(): @app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): - stats = statsCollector.getSummary() + stats = stats_collector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) From 048e908e09415abe8c8804dd392ec1d1fbf3c26c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:42:48 +0000 Subject: [PATCH 064/234] Gateway: docstring pass --- src/eddn/Gateway.py | 62 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 63f7fb0..bfc4551 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -112,9 +112,12 @@ def extract_message_details(parsed_message): return uploader_id, software_name, software_version, schema_ref, journal_event def configure(): - # Get the list of transports to bind from settings. This allows us to PUB - # messages to multiple announcers over a variety of socket types - # (UNIX sockets and/or TCP sockets). + """ + Get the list of transports to bind from settings. + + This allows us to PUB messages to multiple announcers over a variety of + socket types (UNIX sockets and/or TCP sockets). + """ for binding in Settings.GATEWAY_SENDER_BINDINGS: sender.bind(binding) @@ -124,26 +127,30 @@ def configure(): def push_message(parsed_message, topic): """ + Push a message our to subscribed listeners. + Spawned as a greenlet to push messages (strings) through ZeroMQ. - This is a dumb method that just pushes strings; it assumes you've already validated - and serialised as you want to. + This is a dumb method that just pushes strings; it assumes you've already + validated and serialised as you want to. """ string_message = simplejson.dumps(parsed_message, ensure_ascii=False).encode('utf-8') # Push a zlib compressed JSON representation of the message to # announcers with schema as topic compressed_msg = zlib.compress(string_message) - + send_message = "%s |-| %s" % (str(topic), compressed_msg) - + sender.send(send_message) stats_collector.tally("outbound") def get_remote_address(): """ - Determines the address of the uploading client. First checks the for - proxy-forwarded headers, then falls back to request.remote_addr. + Determine the address of the uploading client. + + First checks the for proxy-forwarded headers, then falls back to + request.remote_addr. :rtype: str """ return request.headers.get('X-Forwarded-For', request.remote_addr) @@ -151,8 +158,9 @@ def get_remote_address(): def get_decompressed_message(): """ - For upload formats that support it, detect gzip Content-Encoding headers - and de-compress on the fly. + Detect gzip Content-Encoding headers and de-compress on the fly. + + For upload formats that support it. :rtype: str :returns: The de-compressed request body. """ @@ -183,6 +191,12 @@ def get_decompressed_message(): def parse_and_error_handle(data): + """ + Parse an incoming message and handle errors. + + :param data: + :return: The decoded message, or an error message. + """ try: parsed_message = simplejson.loads(data) except ( @@ -260,6 +274,11 @@ def parse_and_error_handle(data): @app.route('/upload/', method=['OPTIONS', 'POST']) def upload(): + """ + Handle an /upload/ request. + + :return: The processed message, else error string. + """ try: # Body may or may not be compressed. message_body = get_decompressed_message() @@ -300,6 +319,8 @@ def upload(): @app.route('/health_check/', method=['OPTIONS', 'GET']) def health_check(): """ + Return our version string in as an 'am I awake' signal. + This should only be used by the gateway monitoring script. It is used to detect whether the gateway is still alive, and whether it should remain in the DNS rotation. @@ -309,6 +330,11 @@ def health_check(): @app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): + """ + Return some stats about the Gateway's operation so far. + + :return: JSON stats data + """ stats = stats_collector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) @@ -316,18 +342,30 @@ def stats(): class MalformedUploadError(Exception): """ + Exception for malformed upload. + Raise this when an upload is structurally incorrect. This isn't so much to do with something like a bogus region ID, this is more like "You are missing a POST key/val, or a body". """ + pass class EnableCors(object): + """Handle enabling CORS headers in all responses.""" + name = 'enable_cors' api = 2 def apply(self, fn, context): + """ + Apply CORS headers to the calling bottle app. + + :param fn: + :param context: + :return: + """ def _enable_cors(*args, **kwargs): # set CORS headers response.headers['Access-Control-Allow-Origin'] = '*' @@ -342,7 +380,7 @@ class EnableCors(object): def main(): - + """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() if cl_args.loglevel: logger.setLevel(cl_args.loglevel) From 785378a9bc999815136fd102c87ae99937fb5aa5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:47:31 +0000 Subject: [PATCH 065/234] Gateway: % -> f-string pass --- src/eddn/Gateway.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index bfc4551..125e0cd 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -139,7 +139,7 @@ def push_message(parsed_message, topic): # announcers with schema as topic compressed_msg = zlib.compress(string_message) - send_message = "%s |-| %s" % (str(topic), compressed_msg) + send_message = f"{str(topic)} |-| {compressed_msg}" sender.send(send_message) stats_collector.tally("outbound") @@ -221,6 +221,8 @@ def parse_and_error_handle(data): pass response.status = 400 + logger.error(f"Error to {get_remote_address()}: {exc.message}") + return str(exc) return 'FAIL: JSON parsing: ' + str(exc) # Here we check if an outdated schema has been passed @@ -289,15 +291,7 @@ def upload(): # the correct direction. response.status = 400 try: - logger.error('gzip error (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( - request.content_length, - '<>', - '<>', - '<>', - '<>', - '<>', - get_remote_address() - )) + logger.error(f'gzip error ({request.content_length}, "<>", "<>", "<>", "<>", "<>") from {get_remote_address()}') except Exception as e: print('Logging of "gzip error" failed: %s' % (e.message)) @@ -308,7 +302,7 @@ def upload(): except MalformedUploadError as exc: # They probably sent an encoded POST, but got the key/val wrong. response.status = 400 - logger.error("MalformedUploadError from %s: %s" % (get_remote_address(), exc.message)) + logger.error(f"MalformedUploadError from {get_remote_address()}: {exc.message}") return 'FAIL: Malformed Upload: ' + exc.message From 24c80c31320cd4563b614dd814a7e095cb14b362 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 12:31:28 +0000 Subject: [PATCH 066/234] Gateway: misc formatting pass --- src/eddn/Gateway.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 125e0cd..e75e9b7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -235,8 +235,8 @@ def parse_and_error_handle(data): validation_results = validator.validate(parsed_message) if validation_results.severity <= ValidationSeverity.WARN: - parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z' - parsed_message['header']['uploaderIP'] = get_remote_address() + parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z' + parsed_message['header']['uploaderIP'] = get_remote_address() # Sends the parsed message to the Relay/Monitor as compressed JSON. gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) @@ -364,7 +364,8 @@ class EnableCors(object): # set CORS headers response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + response.headers['Access-Control-Allow-Headers'] = \ + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' if request.method != 'OPTIONS': # actual request; reply with the actual response @@ -384,12 +385,13 @@ def main(): app.install(EnableCors()) app.run( - host=Settings.GATEWAY_HTTP_BIND_ADDRESS, - port=Settings.GATEWAY_HTTP_PORT, - server='gevent', + host=Settings.GATEWAY_HTTP_BIND_ADDRESS, + port=Settings.GATEWAY_HTTP_PORT, + server='gevent', certfile=Settings.CERT_FILE, keyfile=Settings.KEY_FILE ) + if __name__ == '__main__': main() From a062d077990c6b7449b1918a64d39e56f030a859 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:16:01 +0000 Subject: [PATCH 067/234] requirements-dev: types-simplejson --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index a960986..46e4c2b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -19,6 +19,7 @@ mypy==0.910 pep8-naming==0.12.1 safety==1.10.3 types-requests==2.25.11 +types-simplejson==3.17.1 # Code formatting tools autopep8==1.6.0 From dbb3e7ce7d32f2d6ef2c7ad815429457d2adc149 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:17:15 +0000 Subject: [PATCH 068/234] requirements-dev: types-pkg_resources --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 46e4c2b..04c41eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,6 +18,7 @@ flake8-use-fstring==1.3 mypy==0.910 pep8-naming==0.12.1 safety==1.10.3 +types-pkg_resources==0.1.3 types-requests==2.25.11 types-simplejson==3.17.1 From 74de8a07ee09fd0a2c2488d17bd27c085e5adbe8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:17:29 +0100 Subject: [PATCH 069/234] Gateway: Start adding types/checking # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index e75e9b7..e42b7d3 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -11,6 +11,7 @@ import hashlib import logging import zlib from datetime import datetime +from typing import Dict import gevent import simplejson @@ -23,9 +24,6 @@ from pkg_resources import resource_string from eddn.conf.Settings import Settings, loadConfig from eddn.core.Validator import ValidationSeverity, Validator -# import os - - monkey.patch_all() import bottle from bottle import Bottle, request, response @@ -111,7 +109,7 @@ def extract_message_details(parsed_message): return uploader_id, software_name, software_version, schema_ref, journal_event -def configure(): +def configure() -> None: """ Get the list of transports to bind from settings. @@ -125,7 +123,7 @@ def configure(): validator.addSchemaResource(schema_ref, resource_string('eddn.Gateway', schema_file)) -def push_message(parsed_message, topic): +def push_message(parsed_message: Dict, topic: str) -> None: """ Push a message our to subscribed listeners. @@ -139,7 +137,7 @@ def push_message(parsed_message, topic): # announcers with schema as topic compressed_msg = zlib.compress(string_message) - send_message = f"{str(topic)} |-| {compressed_msg}" + send_message = f"{str(topic)!r} |-| {compressed_msg!r}" sender.send(send_message) stats_collector.tally("outbound") @@ -199,9 +197,8 @@ def parse_and_error_handle(data): """ try: parsed_message = simplejson.loads(data) - except ( - TypeError, ValueError - ) as exc: + + except (MalformedUploadError, TypeError, ValueError) as exc: # Something bad happened. We know this will return at least a # semi-useful error message, so do so. try: From 35b90de06d1ff0f5ac04a5067a6832a49501fc74 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:17:54 +0100 Subject: [PATCH 070/234] Gateway: typing: get_decompressed_message() # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index e42b7d3..6fe6db7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -143,18 +143,18 @@ def push_message(parsed_message: Dict, topic: str) -> None: stats_collector.tally("outbound") -def get_remote_address(): +def get_remote_address() -> str: """ Determine the address of the uploading client. First checks the for proxy-forwarded headers, then falls back to request.remote_addr. - :rtype: str + :returns: Best attempt at remote address. """ return request.headers.get('X-Forwarded-For', request.remote_addr) -def get_decompressed_message(): +def get_decompressed_message() -> bytes: """ Detect gzip Content-Encoding headers and de-compress on the fly. From 9be3cd82d8feefce4ea7f841cc11d900a0f63f2d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:29:19 +0000 Subject: [PATCH 071/234] Gateway: typing: parse_and_error_handle() --- src/eddn/Gateway.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 6fe6db7..5f3a527 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -188,7 +188,7 @@ def get_decompressed_message() -> bytes: return message_body -def parse_and_error_handle(data): +def parse_and_error_handle(data: str) -> str: """ Parse an incoming message and handle errors. @@ -218,7 +218,7 @@ def parse_and_error_handle(data): pass response.status = 400 - logger.error(f"Error to {get_remote_address()}: {exc.message}") + logger.error(f"Error to {get_remote_address()}: {exc}") return str(exc) return 'FAIL: JSON parsing: ' + str(exc) From 5939e7c889e5c9afd2a89ff1da339a27393d044d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:49:51 +0000 Subject: [PATCH 072/234] Gateway: typing: upload() --- src/eddn/Gateway.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 5f3a527..fd2beed 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -188,7 +188,7 @@ def get_decompressed_message() -> bytes: return message_body -def parse_and_error_handle(data: str) -> str: +def parse_and_error_handle(data: bytes) -> str: """ Parse an incoming message and handle errors. @@ -272,7 +272,7 @@ def parse_and_error_handle(data: str) -> str: @app.route('/upload/', method=['OPTIONS', 'POST']) -def upload(): +def upload() -> str: """ Handle an /upload/ request. From 8825526a1c5c38d3f20b99dd40991a7aa26883d9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:31:48 +0000 Subject: [PATCH 073/234] Gateway: typing: health_check() --- src/eddn/Gateway.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index fd2beed..c0636aa 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -308,13 +308,15 @@ def upload() -> str: @app.route('/health_check/', method=['OPTIONS', 'GET']) -def health_check(): +def health_check() -> str: """ Return our version string in as an 'am I awake' signal. This should only be used by the gateway monitoring script. It is used to detect whether the gateway is still alive, and whether it should remain in the DNS rotation. + + :returns: Version of this software. """ return Settings.EDDN_VERSION From 179dd8aebffb11ee871542f9bf0c22eac916060c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:32:14 +0000 Subject: [PATCH 074/234] Gateway: typing: stats() --- src/eddn/Gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index c0636aa..5e29da0 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -322,7 +322,7 @@ def health_check() -> str: @app.route('/stats/', method=['OPTIONS', 'GET']) -def stats(): +def stats() -> str: """ Return some stats about the Gateway's operation so far. From c11fa91162609f868166a45269c41ec34a2f00ab Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:34:17 +0000 Subject: [PATCH 075/234] Gateway: typing: main() --- src/eddn/Gateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 5e29da0..132b0f7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -373,7 +373,7 @@ class EnableCors(object): return _enable_cors -def main(): +def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() if cl_args.loglevel: From 99679d9d5bb87669245a002c15947fd71e771eff Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:37:17 +0000 Subject: [PATCH 076/234] Gateway: Minor renames to make PyCharm not gripe These are related to local variables shadowing globals. --- src/eddn/Gateway.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 132b0f7..1a49e9a 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -45,8 +45,8 @@ logger.info('Made logger') # This socket is used to push market data out to the Announcers over ZeroMQ. -context = zmq.Context() -sender = context.socket(zmq.PUB) +zmq_context = zmq.Context() +sender = zmq_context.socket(zmq.PUB) validator = Validator() @@ -328,9 +328,9 @@ def stats() -> str: :return: JSON stats data """ - stats = stats_collector.getSummary() - stats["version"] = Settings.EDDN_VERSION - return simplejson.dumps(stats) + stats_current = stats_collector.getSummary() + stats_current["version"] = Settings.EDDN_VERSION + return simplejson.dumps(stats_current) class MalformedUploadError(Exception): From 3b3e1a91c5b54e02c1e4d7a52a9755358d1b6fa1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 13:41:15 +0000 Subject: [PATCH 077/234] Gateway: Make the CORS apply() static Quietens a PyCharm warning --- src/eddn/Gateway.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 1a49e9a..0e007a3 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -351,7 +351,8 @@ class EnableCors(object): name = 'enable_cors' api = 2 - def apply(self, fn, context): + @staticmethod + def apply(fn, context): """ Apply CORS headers to the calling bottle app. From b37307e6a29710ea3d8ea581cc5b9428a31799c1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:51:17 +0000 Subject: [PATCH 078/234] Monitor: Initial flake8 pass --- src/eddn/Monitor.py | 154 ++++++++++++++++++++++++++------------------ 1 file changed, 93 insertions(+), 61 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index a31853f..5b88fe3 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -1,31 +1,29 @@ # coding: utf8 -""" -Monitor sit below gateways, or another relay, and simply parse what it receives over SUB. -""" -from threading import Thread import argparse -import zlib -import gevent -import simplejson -import mysql.connector as mariadb -import datetime import collections +import datetime +import zlib +from threading import Thread + +import gevent +import mysql.connector as mariadb +import simplejson import zmq.green as zmq -import re +from bottle import Bottle, request, response +from gevent import monkey from eddn.conf.Settings import Settings, loadConfig -from gevent import monkey monkey.patch_all() -from bottle import Bottle, get, request, response, run + app = Bottle() # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: from eddn.core.DuplicateMessages import DuplicateMessages - duplicateMessages = DuplicateMessages() - duplicateMessages.start() + duplicate_messages = DuplicateMessages() + duplicate_messages.start() def parse_cl_args(): @@ -59,13 +57,17 @@ def ping(): @app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET']) -def getTotalSoftwares(): +def get_total_softwares(): response.set_header("Access-Control-Allow-Origin", "*") - db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) + db = mariadb.connect( + user=Settings.MONITOR_DB['user'], + password=Settings.MONITOR_DB['password'], + database=Settings.MONITOR_DB['database'] + ) softwares = collections.OrderedDict() - maxDays = request.GET.get('maxDays', '31').strip() - maxDays = int(maxDays) - 1 + max_days = request.GET.get('maxDays', '31').strip() + max_days = int(max_days) - 1 query = """SELECT name, SUM(hits) AS total, MAX(dateStats) AS maxDate FROM softwares @@ -74,7 +76,7 @@ def getTotalSoftwares(): ORDER BY total DESC""" results = db.cursor() - results.execute(query, (maxDays, )) + results.execute(query, (max_days, )) for row in results: softwares[row[0].encode('utf8')] = str(row[1]) @@ -85,13 +87,17 @@ def getTotalSoftwares(): @app.route('/getSoftwares/', method=['OPTIONS', 'GET']) -def getSoftwares(): +def get_softwares(): response.set_header("Access-Control-Allow-Origin", "*") - db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) + db = mariadb.connect( + user=Settings.MONITOR_DB['user'], + password=Settings.MONITOR_DB['password'], + database=Settings.MONITOR_DB['database'] + ) softwares = collections.OrderedDict() - dateStart = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() - dateEnd = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() + date_start = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() + date_end = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() query = """SELECT * FROM `softwares` @@ -99,14 +105,14 @@ def getSoftwares(): ORDER BY `hits` DESC, `dateStats` ASC""" results = db.cursor() - results.execute(query, (dateStart, dateEnd)) + results.execute(query, (date_start, date_end)) for row in results: - currentDate = row[2].strftime('%Y-%m-%d') - if not currentDate in softwares.keys(): - softwares[currentDate] = collections.OrderedDict() + current_date = row[2].strftime('%Y-%m-%d') + if current_date not in softwares.keys(): + softwares[current_date] = collections.OrderedDict() - softwares[currentDate][str(row[0])] = str(row[1]) + softwares[current_date][str(row[0])] = str(row[1]) db.close() @@ -114,9 +120,13 @@ def getSoftwares(): @app.route('/getTotalSchemas/', method=['OPTIONS', 'GET']) -def getTotalSchemas(): +def get_total_schemas(): response.set_header("Access-Control-Allow-Origin", "*") - db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) + db = mariadb.connect( + user=Settings.MONITOR_DB['user'], + password=Settings.MONITOR_DB['password'], + database=Settings.MONITOR_DB['database'] + ) schemas = collections.OrderedDict() query = """SELECT `name`, SUM(`hits`) AS `total` @@ -136,14 +146,17 @@ def getTotalSchemas(): @app.route('/getSchemas/', method=['OPTIONS', 'GET']) -def getSchemas(): +def get_schemas(): response.set_header("Access-Control-Allow-Origin", "*") - db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) - #db.text_factory = lambda x: unicode(x, "utf-8", "ignore") + db = mariadb.connect( + user=Settings.MONITOR_DB['user'], + password=Settings.MONITOR_DB['password'], + database=Settings.MONITOR_DB['database'] + ) schemas = collections.OrderedDict() - dateStart = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() - dateEnd = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() + date_start = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() + date_end = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() query = """SELECT * FROM `schemas` @@ -151,14 +164,14 @@ def getSchemas(): ORDER BY `hits` DESC, `dateStats` ASC""" results = db.cursor() - results.execute(query, (dateStart, dateEnd)) + results.execute(query, (date_start, date_end)) for row in results: - currentDate = row[2].strftime('%Y-%m-%d') - if not currentDate in schemas.keys(): - schemas[currentDate] = collections.OrderedDict() + current_date = row[2].strftime('%Y-%m-%d') + if current_date not in schemas.keys(): + schemas[current_date] = collections.OrderedDict() - schemas[currentDate][str(row[0])] = str(row[1]) + schemas[current_date][str(row[0])] = str(row[1]) db.close() @@ -177,7 +190,11 @@ class Monitor(Thread): receiver.connect(binding) def monitor_worker(message): - db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) + db = mariadb.connect( + user=Settings.MONITOR_DB['user'], + password=Settings.MONITOR_DB['password'], + database=Settings.MONITOR_DB['database'] + ) # Separate topic from message message = message.split(' |-| ') @@ -189,25 +206,27 @@ class Monitor(Thread): message = message[0] message = zlib.decompress(message) - json = simplejson.loads(message) + json = simplejson.loads(message) # Default variables - schemaID = json['$schemaRef'] - softwareID = json['header']['softwareName'].encode('utf8') + ' | ' + json['header']['softwareVersion'].encode('utf8') - - uploaderID = json['header']['uploaderID'].encode('utf8') - uploaderIP = None - if 'uploaderIP' in json['header']: - uploaderIP = json['header']['uploaderIP'].encode('utf8') + schema_id = json['$schemaRef'] + software_id = json['header']['softwareName'].encode('utf8') + ' | ' \ + + json['header']['softwareVersion'].encode('utf8') # Duplicates? if Settings.RELAY_DUPLICATE_MAX_MINUTES: - if duplicateMessages.isDuplicated(json): - schemaID = 'DUPLICATE MESSAGE' + if duplicate_messages.isDuplicated(json): + schema_id = 'DUPLICATE MESSAGE' c = db.cursor() - c.execute('UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', (schemaID, )) - c.execute('INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', (schemaID, )) + c.execute( + 'UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', + (schema_id, ) + ) + c.execute( + 'INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', + (schema_id, ) + ) db.commit() db.close() @@ -216,21 +235,33 @@ class Monitor(Thread): # Update software count c = db.cursor() - c.execute('UPDATE `softwares` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', (softwareID, )) - c.execute('INSERT IGNORE INTO `softwares` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', (softwareID, )) + c.execute( + 'UPDATE `softwares` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', + (software_id, ) + ) + c.execute( + 'INSERT IGNORE INTO `softwares` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', + (software_id, ) + ) db.commit() # Update schemas count c = db.cursor() - c.execute('UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', (schemaID, )) - c.execute('INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', (schemaID, )) + c.execute( + 'UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', + (schema_id, ) + ) + c.execute( + 'INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', + (schema_id, ) + ) db.commit() db.close() while True: - inboundMessage = receiver.recv() - gevent.spawn(monitor_worker, inboundMessage) + inbound_message = receiver.recv() + gevent.spawn(monitor_worker, inbound_message) class EnableCors(object): @@ -249,7 +280,8 @@ class EnableCors(object): """Set CORS Headers.""" response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + response.headers['Access-Control-Allow-Headers'] = \ + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' if request.method != 'OPTIONS': # actual request; reply with the actual response @@ -257,7 +289,7 @@ class EnableCors(object): return _enable_cors -def main(): +def main() -> None: cl_args = parse_cl_args() loadConfig(cl_args) From 33a09ef8b5880d864eee491d94b01d1cfc54f720 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:51:56 +0000 Subject: [PATCH 079/234] Monitor: docstring'd --- src/eddn/Monitor.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 5b88fe3..1b7dfc7 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -46,18 +46,26 @@ def parse_cl_args(): return parser.parse_args() -def date(__format): +def date(__format) -> datetime.datetime: + """ + Make a 'now' datetime as per the supplied format. + + :param __format: + :return: + """ d = datetime.datetime.utcnow() return d.strftime(__format) @app.route('/ping', method=['OPTIONS', 'GET']) def ping(): + """Respond to a ping request.""" return 'pong' @app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET']) def get_total_softwares(): + """Respond with data about total uploading software counts.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( user=Settings.MONITOR_DB['user'], @@ -88,6 +96,7 @@ def get_total_softwares(): @app.route('/getSoftwares/', method=['OPTIONS', 'GET']) def get_softwares(): + """Respond with hit stats for all uploading software.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( user=Settings.MONITOR_DB['user'], @@ -121,6 +130,7 @@ def get_softwares(): @app.route('/getTotalSchemas/', method=['OPTIONS', 'GET']) def get_total_schemas(): + """Respond with total hit stats for all schemas.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( user=Settings.MONITOR_DB['user'], @@ -147,6 +157,7 @@ def get_total_schemas(): @app.route('/getSchemas/', method=['OPTIONS', 'GET']) def get_schemas(): + """Respond with schema hit stats between given datetimes.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( user=Settings.MONITOR_DB['user'], @@ -179,8 +190,10 @@ def get_schemas(): class Monitor(Thread): + """Monitor thread class.""" def run(self): + """Handle receiving Gateway messages and recording stats.""" context = zmq.Context() receiver = context.socket(zmq.SUB) @@ -290,6 +303,7 @@ class EnableCors(object): return _enable_cors def main() -> None: + """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() loadConfig(cl_args) From 07e332ff619ed0f51507fd12764a5d0ef4246fab Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:52:22 +0000 Subject: [PATCH 080/234] Monitor: types pass --- src/eddn/Monitor.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 1b7dfc7..50f8304 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -5,6 +5,7 @@ import collections import datetime import zlib from threading import Thread +from typing import OrderedDict import gevent import mysql.connector as mariadb @@ -58,13 +59,13 @@ def date(__format) -> datetime.datetime: @app.route('/ping', method=['OPTIONS', 'GET']) -def ping(): +def ping() -> str: """Respond to a ping request.""" return 'pong' @app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET']) -def get_total_softwares(): +def get_total_softwares() -> str: """Respond with data about total uploading software counts.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( @@ -95,7 +96,7 @@ def get_total_softwares(): @app.route('/getSoftwares/', method=['OPTIONS', 'GET']) -def get_softwares(): +def get_softwares() -> str: """Respond with hit stats for all uploading software.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( @@ -103,7 +104,7 @@ def get_softwares(): password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database'] ) - softwares = collections.OrderedDict() + softwares: OrderedDict = collections.OrderedDict() date_start = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() date_end = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() @@ -129,7 +130,7 @@ def get_softwares(): @app.route('/getTotalSchemas/', method=['OPTIONS', 'GET']) -def get_total_schemas(): +def get_total_schemas() -> str: """Respond with total hit stats for all schemas.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( @@ -156,7 +157,7 @@ def get_total_schemas(): @app.route('/getSchemas/', method=['OPTIONS', 'GET']) -def get_schemas(): +def get_schemas() -> str: """Respond with schema hit stats between given datetimes.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( @@ -164,7 +165,7 @@ def get_schemas(): password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database'] ) - schemas = collections.OrderedDict() + schemas: OrderedDict = collections.OrderedDict() date_start = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() date_end = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() @@ -192,7 +193,7 @@ def get_schemas(): class Monitor(Thread): """Monitor thread class.""" - def run(self): + def run(self) -> None: """Handle receiving Gateway messages and recording stats.""" context = zmq.Context() @@ -202,7 +203,7 @@ class Monitor(Thread): for binding in Settings.MONITOR_RECEIVER_BINDINGS: receiver.connect(binding) - def monitor_worker(message): + def monitor_worker(message: bytes) -> None: db = mariadb.connect( user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], @@ -210,16 +211,16 @@ class Monitor(Thread): ) # Separate topic from message - message = message.split(' |-| ') + message_split = message.split(b' |-| ') # Handle gateway not sending topic - if len(message) > 1: - message = message[1] + if len(message_split) > 1: + message = message_split[1] else: - message = message[0] + message = message_split[0] - message = zlib.decompress(message) - json = simplejson.loads(message) + message_text = zlib.decompress(message) + json = simplejson.loads(message_text) # Default variables schema_id = json['$schemaRef'] From 06e9442bea5d0e32e8f45071b518f915413d3210 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 14:26:36 +0000 Subject: [PATCH 081/234] Monitor: Make EnableCors.apply() static --- src/eddn/Monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 50f8304..3cdab9f 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -284,7 +284,8 @@ class EnableCors(object): name = 'enable_cors' api = 2 - def apply(self, fn, context): + @staticmethod + def apply(fn, context): """ Apply a CORS handler. From 3b5d16c6747ee8ff4d268fc202a120d856b5ae3d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:54:57 +0000 Subject: [PATCH 082/234] Relay: Full flake8/mypy pass --- src/eddn/Relay.py | 115 ++++++++++++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 4fbd990..40ac605 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -1,22 +1,14 @@ # coding: utf8 -""" -Relays sit below an announcer, or another relay, and simply repeat what -they receive over PUB/SUB. -""" +"""EDDN Relay, which passes messages from the Gateway to listeners.""" import argparse -import gevent import hashlib import logging -import simplejson import time import uuid import zlib from threading import Thread -import zmq.green as zmq - - # Logging has to be configured first before we do anything. logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -30,24 +22,29 @@ __logger_channel.setFormatter(__logger_formatter) logger.addHandler(__logger_channel) logger.info('Made logger') +import gevent +import simplejson +import zmq.green as zmq +from bottle import Bottle, request, response +from gevent import monkey + from eddn.conf.Settings import Settings, loadConfig -from gevent import monkey monkey.patch_all() -from bottle import Bottle, get, request, response, run + app = Bottle() - # This import must be done post-monkey-patching! -from eddn.core.StatsCollector import StatsCollector -statsCollector = StatsCollector() -statsCollector.start() +from eddn.core.StatsCollector import StatsCollector # noqa: E402 + +stats_collector = StatsCollector() +stats_collector.start() # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: from eddn.core.DuplicateMessages import DuplicateMessages - duplicateMessages = DuplicateMessages() - duplicateMessages.start() + duplicate_messages = DuplicateMessages() + duplicate_messages.start() def parse_cl_args(): @@ -71,13 +68,19 @@ def parse_cl_args(): return parser.parse_args() @app.route('/stats/', method=['OPTIONS', 'GET']) -def stats(): - stats = statsCollector.getSummary() +def stats() -> str: + """ + Return some stats about the Relay's operation so far. + + :return: JSON stats data + """ + stats = stats_collector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) class Relay(Thread): + """Relay thread class.""" REGENERATE_UPLOADER_NONCE_INTERVAL = 12 * 60 * 60 # 12 hrs @@ -87,20 +90,26 @@ class Relay(Thread): self.uploader_nonce_timestamp = 0 self.generate_uploader_nonce() - def generate_uploader_nonce(self): + def generate_uploader_nonce(self) -> None: + """Generate an uploader nonce.""" self.uploader_nonce = str(uuid.uuid4()) self.uploader_nonce_timestamp = time.time() - def scramble_uploader(self, uploader): + def scramble_uploader(self, uploader: str) -> str: + """ + Scramble an uploader ID. + + :param uploader: Plain text uploaderID. + :return: Scrambled version of uploader. + """ now = time.time() if now - self.uploader_nonce_timestamp > self.REGENERATE_UPLOADER_NONCE_INTERVAL: self.generate_uploader_nonce() - return hashlib.sha1("{}-{}".format(self.uploader_nonce, uploader.encode('utf8'))).hexdigest() - def run(self): - """ - Fires up the relay process. - """ + return hashlib.sha1(f"{self.uploader_nonce!r}-{uploader.encode}".encode('utf8')).hexdigest() + + def run(self) -> None: # noqa: CCR001 + """Handle receiving messages from Gateway and passing them on.""" # These form the connection to the Gateway daemon(s) upstream. context = zmq.Context() @@ -108,10 +117,10 @@ class Relay(Thread): # Filters on topics or not... if Settings.RELAY_RECEIVE_ONLY_GATEWAY_EXTRA_JSON is True: - for schemaRef, schemaFile in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): - receiver.setsockopt(zmq.SUBSCRIBE, schemaRef) - for schemaRef, schemaFile in Settings.RELAY_EXTRA_JSON_SCHEMAS.iteritems(): - receiver.setsockopt(zmq.SUBSCRIBE, schemaRef) + for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): + receiver.setsockopt(zmq.SUBSCRIBE, schema_ref) + for schema_ref, schema_file in Settings.RELAY_EXTRA_JSON_SCHEMAS.iteritems(): + receiver.setsockopt(zmq.SUBSCRIBE, schema_ref) else: receiver.setsockopt(zmq.SUBSCRIBE, '') @@ -126,29 +135,29 @@ class Relay(Thread): # End users, or other relays, may attach here. sender.bind(binding) - def relay_worker(message): + def relay_worker(message: bytes) -> None: """ - This is the worker function that re-sends the incoming messages out - to any subscribers. - :param str message: A JSON string to re-broadcast. + Worker that resends messages to any subscribers. + + :param message: Message to be passed on. """ # Separate topic from message - message = message.split(' |-| ') + message_split = message.split(b' |-| ') # Handle gateway not sending topic - if len(message) > 1: - message = message[1] + if len(message_split) > 1: + message = message_split[1] else: - message = message[0] + message = message_split[0] - message = zlib.decompress(message) - json = simplejson.loads(message) + message_text = zlib.decompress(message) + json = simplejson.loads(message_text) # Handle duplicate message if Settings.RELAY_DUPLICATE_MAX_MINUTES: - if duplicateMessages.isDuplicated(json): + if duplicate_messages.isDuplicated(json): # We've already seen this message recently. Discard it. - statsCollector.tally("duplicate") + stats_collector.tally("duplicate") return # Mask the uploader with a randomised nonce but still make it unique @@ -161,21 +170,21 @@ class Relay(Thread): del json['header']['uploaderIP'] # Convert message back to JSON - message = simplejson.dumps(json, sort_keys=True) + message_json = simplejson.dumps(json, sort_keys=True) # Recompress message - message = zlib.compress(message) + message = zlib.compress(message_json.encode('utf8')) # Send message sender.send(message) - statsCollector.tally("outbound") + stats_collector.tally("outbound") while True: # For each incoming message, spawn a greenlet using the relay_worker # function. - inboundMessage = receiver.recv() - statsCollector.tally("inbound") - gevent.spawn(relay_worker, inboundMessage) + inbound_message = receiver.recv() + stats_collector.tally("inbound") + gevent.spawn(relay_worker, inbound_message) class EnableCors(object): @@ -184,7 +193,8 @@ class EnableCors(object): name = 'enable_cors' api = 2 - def apply(self, fn, context): + @staticmethod + def apply(fn, context): """ Apply a CORS handler. @@ -194,7 +204,8 @@ class EnableCors(object): """Set CORS Headers.""" response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + response.headers['Access-Control-Allow-Headers'] = \ + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' if request.method != 'OPTIONS': # actual request; reply with the actual response @@ -203,7 +214,9 @@ class EnableCors(object): return _enable_cors -def main(): +<<<<<<< HEAD +def main() -> None: + """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() if cl_args.loglevel: logger.setLevel(cl_args.loglevel) From dd7e12fa28df048c53665d5a4a99dea1ff2ac72c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 15:31:46 +0000 Subject: [PATCH 083/234] Version: Bump to 2.0-alph0 for this python3 work --- src/eddn/conf/Version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eddn/conf/Version.py b/src/eddn/conf/Version.py index 0c1bf79..e042b67 100644 --- a/src/eddn/conf/Version.py +++ b/src/eddn/conf/Version.py @@ -1,2 +1,3 @@ +"""Declare the version of this software.""" # This should be a version number as understood by setuptools -__version__ = "1.2" +__version__ = "2.0-alpha0" From 8fb722f6c5bda2f7b0581b505b4f4a476c0591bd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 14:48:20 +0100 Subject: [PATCH 084/234] Settings: flake8 and mypy passes Yes, we want to keep the extra-spaces formatting, but that means noqa'ing E221. --- src/eddn/conf/Settings.py | 93 +++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 28e9b21..d61cd31 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -1,6 +1,11 @@ # coding: utf8 +"""EDDN default Settings.""" + +import argparse +from typing import Dict import simplejson + from eddn.conf.Version import __version__ as version @@ -12,64 +17,64 @@ class _Settings(object): # Local installation settings ############################################################################### - CERT_FILE = '/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem' - KEY_FILE = '/etc/letsencrypt/live/eddn.edcd.io/privkey.pem' + CERT_FILE = '/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem' # noqa: E221 + KEY_FILE = '/etc/letsencrypt/live/eddn.edcd.io/privkey.pem' # noqa: E221 ############################################################################### # Relay settings ############################################################################### - RELAY_HTTP_BIND_ADDRESS = "0.0.0.0" - RELAY_HTTP_PORT = 9090 + RELAY_HTTP_BIND_ADDRESS = "0.0.0.0" # noqa: E221 + RELAY_HTTP_PORT = 9090 # noqa: E221 - RELAY_RECEIVER_BINDINGS = ["tcp://127.0.0.1:8500"] + RELAY_RECEIVER_BINDINGS = ["tcp://127.0.0.1:8500"] # noqa: E221 - RELAY_SENDER_BINDINGS = ["tcp://*:9500"] + RELAY_SENDER_BINDINGS = ["tcp://*:9500"] # noqa: E221 # If set to False, no deduplicate is made - RELAY_DUPLICATE_MAX_MINUTES = 15 + RELAY_DUPLICATE_MAX_MINUTES = 15 # noqa: E221 # If set to false, don't listen to topic and accept all incoming messages - RELAY_RECEIVE_ONLY_GATEWAY_EXTRA_JSON = True + RELAY_RECEIVE_ONLY_GATEWAY_EXTRA_JSON = True # noqa: E221 - RELAY_EXTRA_JSON_SCHEMAS = {} + RELAY_EXTRA_JSON_SCHEMAS: Dict = {} # noqa: E221 ############################################################################### # Gateway settings ############################################################################### - GATEWAY_HTTP_BIND_ADDRESS = "127.0.0.1" - GATEWAY_HTTP_PORT = 8081 + GATEWAY_HTTP_BIND_ADDRESS = "127.0.0.1" # noqa: E221 + GATEWAY_HTTP_PORT = 8081 # noqa: E221 - GATEWAY_SENDER_BINDINGS = ["tcp://127.0.0.1:8500"] + GATEWAY_SENDER_BINDINGS = ["tcp://127.0.0.1:8500"] # noqa: E221 - GATEWAY_JSON_SCHEMAS = { - "https://eddn.edcd.io/schemas/commodity/3" : "schemas/commodity-v3.0.json", - "https://eddn.edcd.io/schemas/commodity/3/test" : "schemas/commodity-v3.0.json", + GATEWAY_JSON_SCHEMAS = { # noqa: E221 + "https://eddn.edcd.io/schemas/commodity/3": "schemas/commodity-v3.0.json", + "https://eddn.edcd.io/schemas/commodity/3/test": "schemas/commodity-v3.0.json", - "https://eddn.edcd.io/schemas/shipyard/2" : "schemas/shipyard-v2.0.json", - "https://eddn.edcd.io/schemas/shipyard/2/test" : "schemas/shipyard-v2.0.json", + "https://eddn.edcd.io/schemas/shipyard/2": "schemas/shipyard-v2.0.json", + "https://eddn.edcd.io/schemas/shipyard/2/test": "schemas/shipyard-v2.0.json", - "https://eddn.edcd.io/schemas/outfitting/2" : "schemas/outfitting-v2.0.json", - "https://eddn.edcd.io/schemas/outfitting/2/test" : "schemas/outfitting-v2.0.json", + "https://eddn.edcd.io/schemas/outfitting/2": "schemas/outfitting-v2.0.json", + "https://eddn.edcd.io/schemas/outfitting/2/test": "schemas/outfitting-v2.0.json", - "https://eddn.edcd.io/schemas/blackmarket/1" : "schemas/blackmarket-v1.0.json", - "https://eddn.edcd.io/schemas/blackmarket/1/test" : "schemas/blackmarket-v1.0.json", + "https://eddn.edcd.io/schemas/blackmarket/1": "schemas/blackmarket-v1.0.json", + "https://eddn.edcd.io/schemas/blackmarket/1/test": "schemas/blackmarket-v1.0.json", - "https://eddn.edcd.io/schemas/journal/1" : "schemas/journal-v1.0.json", - "https://eddn.edcd.io/schemas/journal/1/test" : "schemas/journal-v1.0.json", + "https://eddn.edcd.io/schemas/journal/1": "schemas/journal-v1.0.json", + "https://eddn.edcd.io/schemas/journal/1/test": "schemas/journal-v1.0.json", - "https://eddn.edcd.io/schemas/scanbarycentre/1" : "schemas/scanbarycentre-v1.0.json", - "https://eddn.edcd.io/schemas/scanbarycentre/1/test" : "schemas/scanbarycentre-v1.0.json", + "https://eddn.edcd.io/schemas/scanbarycentre/1": "schemas/scanbarycentre-v1.0.json", + "https://eddn.edcd.io/schemas/scanbarycentre/1/test": "schemas/scanbarycentre-v1.0.json", - "https://eddn.edcd.io/schemas/fssdiscoveryscan/1" : "schemas/fssdiscoveryscan-v1.0.json", - "https://eddn.edcd.io/schemas/fssdiscoveryscan/1/test" : "schemas/fssdiscoveryscan-v1.0.json", + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1": "schemas/fssdiscoveryscan-v1.0.json", + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1/test": "schemas/fssdiscoveryscan-v1.0.json", - "https://eddn.edcd.io/schemas/codexentry/1" : "schemas/codexentry-v1.0.json", - "https://eddn.edcd.io/schemas/codexentry/1/test" : "schemas/codexentry-v1.0.json", + "https://eddn.edcd.io/schemas/codexentry/1": "schemas/codexentry-v1.0.json", + "https://eddn.edcd.io/schemas/codexentry/1/test": "schemas/codexentry-v1.0.json", - "https://eddn.edcd.io/schemas/navbeaconscan/1" : "schemas/navbeaconscan-v1.0.json", - "https://eddn.edcd.io/schemas/navbeaconscan/1/test" : "schemas/navbeaconscan-v1.0.json", + "https://eddn.edcd.io/schemas/navbeaconscan/1": "schemas/navbeaconscan-v1.0.json", + "https://eddn.edcd.io/schemas/navbeaconscan/1/test": "schemas/navbeaconscan-v1.0.json", "https://eddn.edcd.io/schemas/navroute/1" : "schemas/navroute-v1.0.json", "https://eddn.edcd.io/schemas/navroute/1/test" : "schemas/navroute-v1.0.json", @@ -86,7 +91,7 @@ class _Settings(object): "https://eddn.edcd.io/schemas/fsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", } - GATEWAY_OUTDATED_SCHEMAS = [ + GATEWAY_OUTDATED_SCHEMAS = [ # noqa: E221 "http://schemas.elite-markets.net/eddn/commodity/1", "http://schemas.elite-markets.net/eddn/commodity/1/test", "http://schemas.elite-markets.net/eddn/commodity/2", @@ -111,40 +116,42 @@ class _Settings(object): # Monitor settings ############################################################################### - MONITOR_HTTP_BIND_ADDRESS = "0.0.0.0" - MONITOR_HTTP_PORT = 9091 + MONITOR_HTTP_BIND_ADDRESS = "0.0.0.0" # noqa: E221 + MONITOR_HTTP_PORT = 9091 # noqa: E221 - MONITOR_RECEIVER_BINDINGS = ["tcp://127.0.0.1:8500"] + MONITOR_RECEIVER_BINDINGS = ["tcp://127.0.0.1:8500"] # noqa: E221 - MONITOR_DB = { + MONITOR_DB = { # noqa: E221 "user": "eddn", "password": "cvLYM8AEqg29YTatFMEcqph3YkDWUMvC", "database": "eddn" } - MONITOR_UA = "UA-496332-23" + MONITOR_UA = "UA-496332-23" # noqa: E221 ########################################################################## # Bouncer settings ########################################################################## - BOUNCER_HTTP_BIND_ADDRESS = "127.0.0.1" - BOUNCER_HTTP_PORT = 8081 + BOUNCER_HTTP_BIND_ADDRESS = "127.0.0.1" # noqa: E221 + BOUNCER_HTTP_PORT = 8081 # noqa: E221 BOUNCER_LIVE_GATEWAY_URL = 'https://eddn.edcd.io:4430/upload/' - def loadFrom(self, fileName): - f = open(fileName, 'r') + def load_from(self, file_name: str) -> None: + f = open(file_name, 'r') conf = simplejson.load(f) for key, value in conf.iteritems(): if key in dir(self): self.__setattr__(key, value) + else: - print "Ignoring unknown setting {0}".format(key) + print(f"Ignoring unknown setting {key}") + Settings = _Settings() -def loadConfig(cl_args): +def loadConfig(cl_args) -> None: """ Load in a commandline-specified settings file, if applicable. From fcfe9e01b2d2e26878ddbc000b14a03c7d8b8e98 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:58:42 +0000 Subject: [PATCH 085/234] Settings.loadConfig() got renamed to load_config() --- src/eddn/Bouncer.py | 4 ++-- src/eddn/Gateway.py | 4 ++-- src/eddn/Monitor.py | 4 ++-- src/eddn/Relay.py | 5 ++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index 93fa673..82b5921 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -40,7 +40,7 @@ from functools import wraps from pkg_resources import resource_string # import os -from eddn.conf.Settings import Settings, loadConfig +from eddn.conf.Settings import Settings, load_config from gevent import monkey monkey.patch_all() @@ -277,7 +277,7 @@ def main(): logger.setLevel(cl_args.loglevel) logger.info('Loading config...') - loadConfig(cl_args) + load_config(cl_args) logger.info('Installing EnableCors ...') app.install(EnableCors()) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 0e007a3..c99a419 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -21,7 +21,7 @@ from bottle import Bottle, request, response from gevent import monkey from pkg_resources import resource_string -from eddn.conf.Settings import Settings, loadConfig +from eddn.conf.Settings import Settings, load_config from eddn.core.Validator import ValidationSeverity, Validator monkey.patch_all() @@ -380,7 +380,7 @@ def main() -> None: if cl_args.loglevel: logger.setLevel(cl_args.loglevel) - loadConfig(cl_args) + load_config(cl_args) configure() app.install(EnableCors()) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 3cdab9f..feeb0c0 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -14,7 +14,7 @@ import zmq.green as zmq from bottle import Bottle, request, response from gevent import monkey -from eddn.conf.Settings import Settings, loadConfig +from eddn.conf.Settings import Settings, load_config monkey.patch_all() @@ -307,7 +307,7 @@ class EnableCors(object): def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() - loadConfig(cl_args) + load_config(cl_args) m = Monitor() m.start() diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 40ac605..ecd29f5 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -28,7 +28,7 @@ import zmq.green as zmq from bottle import Bottle, request, response from gevent import monkey -from eddn.conf.Settings import Settings, loadConfig +from eddn.conf.Settings import Settings, load_config monkey.patch_all() @@ -214,14 +214,13 @@ class EnableCors(object): return _enable_cors -<<<<<<< HEAD def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() if cl_args.loglevel: logger.setLevel(cl_args.loglevel) - loadConfig(cl_args) + load_config(cl_args) r = Relay() r.start() From a44b6286b2d1eb5b38fc241a939f93028f2b927f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:13:13 +0000 Subject: [PATCH 086/234] .flake8: Drop the type annotations minimum to 50% Mostly I'm having trouble finding what else I *could* add types to in Bouncer.py. I think it's the bottle plugin classes' (EnableCors and CustomLogging) functions. --- .flake8 | 1 + 1 file changed, 1 insertion(+) diff --git a/.flake8 b/.flake8 index 49e1306..acda403 100644 --- a/.flake8 +++ b/.flake8 @@ -25,6 +25,7 @@ per-file-ignores = ./EDMC.py:E402 # https://github.com/best-doctor/flake8-annotations-coverage # Checks code for type annotations +min-coverage-percents = 50 # https://pypi.org/project/flake8-isort/ # Ensures imports are well sorted From 1f420a8f3efa75406dd0bf6028e76ccf1980a06a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 10:59:52 +0000 Subject: [PATCH 087/234] Bouncer: Full flake8 and mypy pass --- src/eddn/Bouncer.py | 155 ++++++++++++++++++++++++++++---------------- 1 file changed, 98 insertions(+), 57 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index 82b5921..652d3da 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -27,24 +27,21 @@ Architecture: Gateway. """ import argparse -import gevent -import hashlib import logging +import zlib +from datetime import datetime + +import gevent import requests import simplejson import urlparse -import zlib -from datetime import datetime -from functools import wraps - -from pkg_resources import resource_string -# import os +from bottle import Bottle, request, response +from gevent import monkey from eddn.conf.Settings import Settings, load_config -from gevent import monkey monkey.patch_all() -from bottle import Bottle, run, request, response, get, post + app = Bottle() logger = logging.getLogger(__name__) @@ -61,10 +58,7 @@ logger.info('Made logger') # This import must be done post-monkey-patching! -from eddn.core.StatsCollector import StatsCollector -statsCollector = StatsCollector() -statsCollector.start() - +from eddn.core.StatsCollector import StatsCollector # noqa: E402 def parse_cl_args(): parser = argparse.ArgumentParser( @@ -86,11 +80,17 @@ def parse_cl_args(): return parser.parse_args() -def push_message(message_body): +stats_collector = StatsCollector() +stats_collector.start() + + +def push_message(message_body: str) -> None: """ + Push a message our to subscribed listeners. + Spawned as a greenlet to push messages (strings) through ZeroMQ. - This is a dumb method that just pushes strings; it assumes you've already validated - and serialised as you want to. + This is a dumb method that just pushes strings; it assumes you've already + validated and serialised as you want to. """ try: r = requests.post( @@ -103,27 +103,28 @@ def push_message(message_body): else: if r.status_code != requests.codes.ok: - logger.error('Response from %s:\n%s\n' % (BOUNCER_LIVE_GATEWAY_URL, - r.text)) + logger.error(f'Response from {Settings.BOUNCER_LIVE_GATEWAY_URL}:\n{r.text}\n') else: - statsCollector.tally("outbound") + stats_collector.tally("outbound") -def get_remote_address(): +def get_remote_address() -> str: """ - Determines the address of the uploading client. First checks the for - proxy-forwarded headers, then falls back to request.remote_addr. - :rtype: str + Determine the address of the uploading client. + + First checks the for proxy-forwarded headers, then falls back to + request.remote_addr. + :returns: Best attempt at remote address. """ return request.headers.get('X-Forwarded-For', request.remote_addr) -def get_decompressed_message(): +def get_decompressed_message() -> bytes: """ - For upload formats that support it, detect gzip Content-Encoding headers - and de-compress on the fly. - :rtype: str + Detect gzip Content-Encoding headers and de-compress on the fly. + + For upload formats that support it. :returns: The de-compressed request body. """ content_encoding = request.headers.get('Content-Encoding', '') @@ -166,17 +167,27 @@ def get_decompressed_message(): return message_body -def forward_message(message_body): +def forward_message(message_body: bytes) -> str: + """ + Send the parsed message to the Relay/Monitor as compressed JSON. + + :param message_body: Incoming message. + :returns: 'OK' assuming it is. + """ # TODO: This instead needs to send the message to remote Gateway - # Sends the parsed message to the Relay/Monitor as compressed JSON. gevent.spawn(push_message, message_body) - logger.info("Accepted upload from %s" % ( - get_remote_address() - )) + logger.info(f'Accepted upload from {get_remote_address()}') + return 'OK' + @app.route('/upload/', method=['OPTIONS', 'POST']) -def upload(): +def upload() -> str: + """ + Handle an /upload/ request. + + :returns: The processed message, else error string. + """ try: # Body may or may not be compressed. message_body = get_decompressed_message() @@ -186,55 +197,80 @@ def upload(): # at least some kind of feedback for them to try to get pointed in # the correct direction. response.status = 400 - logger.error("gzip error with %s: %s" % (get_remote_address(), exc.message)) - return exc.message + logger.error(f'gzip error with {get_remote_address()}: {exc}') + + return f'{exc}' except MalformedUploadError as exc: # They probably sent an encoded POST, but got the key/val wrong. response.status = 400 - logger.error("Error to %s: %s" % (get_remote_address(), exc.message)) - return exc.message + logger.error(f'Error to {get_remote_address()}: {exc}') - statsCollector.tally("inbound") + return f'{exc}' + + stats_collector.tally("inbound") return forward_message(message_body) @app.route('/health_check/', method=['OPTIONS', 'GET']) -def health_check(): +def health_check() -> str: """ + Return our version string in as an 'am I awake' signal. + This should only be used by the gateway monitoring script. It is used to detect whether the gateway is still alive, and whether it should remain in the DNS rotation. + + :returns: Version of this software. """ return Settings.EDDN_VERSION @app.route('/stats/', method=['OPTIONS', 'GET']) -def stats(): - stats = statsCollector.getSummary() - stats["version"] = Settings.EDDN_VERSION - return simplejson.dumps(stats) +def stats() -> str: + """ + Return some stats about the Gateway's operation so far. + + :return: JSON stats data + """ + stats_current = stats_collector.getSummary() + stats_current["version"] = Settings.EDDN_VERSION + + return simplejson.dumps(stats_current) class MalformedUploadError(Exception): """ + Exception for malformed upload. + Raise this when an upload is structurally incorrect. This isn't so much to do with something like a bogus region ID, this is more like "You are missing a POST key/val, or a body". """ + pass class EnableCors(object): + """Handle enabling CORS headers in all responses.""" + name = 'enable_cors' api = 2 def apply(self, fn, context): + """ + Apply CORS headers to the calling bottle app. + + :param fn: + :param context: + :return: + """ def _enable_cors(*args, **kwargs): # set CORS headers response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + response.headers['Access-Control-Allow-Headers'] = \ + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' if request.method != 'OPTIONS': # actual request; reply with the actual response @@ -244,11 +280,19 @@ class EnableCors(object): class CustomLogging(object): - """Wrap a Bottle request so that a log line is emitted after it's handled. """ + """Wrap a Bottle request so that a log line is emitted after it's handled.""" + name = 'custom_logging' api = 2 def apply(self, fn, context): + """ + Apply custom logging to bottle request. + + :param fn: + :param context: + :return: + """ def _log_to_logger(*args, **kwargs): request_time = datetime.utcnow() actual_response = fn(*args, **kwargs) @@ -259,19 +303,15 @@ class CustomLogging(object): else: remote_addr = request.remote_addr - - logger.info('%s %s %s %s %s' % (remote_addr, - request_time, - request.method, - request.url, - response.status) - ) + + logger.info(f'{remote_addr} {request_time} {request.method} {request.url} {response.status}') return actual_response return _log_to_logger -def main(): +def main() -> None: + """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() if cl_args.loglevel: logger.setLevel(cl_args.loglevel) @@ -285,13 +325,14 @@ def main(): app.install(CustomLogging()) logger.info('Running bottle app ...') app.run( - host=Settings.BOUNCER_HTTP_BIND_ADDRESS, - port=Settings.BOUNCER_HTTP_PORT, - server='gevent', + host=Settings.BOUNCER_HTTP_BIND_ADDRESS, + port=Settings.BOUNCER_HTTP_PORT, + server='gevent', certfile=Settings.CERT_FILE, keyfile=Settings.KEY_FILE, quiet=True, ) + if __name__ == '__main__': main() From 485055357ba87d92cf0e7a2bcb92f80903fa002b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:16:32 +0000 Subject: [PATCH 088/234] Bouncer: Trying out 'Callable' for bottle plugin.apply() type annotation --- src/eddn/Bouncer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index 652d3da..ce823e6 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -30,6 +30,7 @@ import argparse import logging import zlib from datetime import datetime +from typing import Callable import gevent import requests @@ -257,7 +258,7 @@ class EnableCors(object): name = 'enable_cors' api = 2 - def apply(self, fn, context): + def apply(self, fn: Callable, context: str): """ Apply CORS headers to the calling bottle app. @@ -285,7 +286,7 @@ class CustomLogging(object): name = 'custom_logging' api = 2 - def apply(self, fn, context): + def apply(self, fn: Callable, context: str): """ Apply custom logging to bottle request. From a05e1d3d2c732c25f1f6bd2e15b39eb0c171cfba Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:17:08 +0000 Subject: [PATCH 089/234] .flake8: Comment out min-coverage-percents I figured out the bottle Plugin.apply() thing. Leaving as a comment for reference. --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index acda403..6c8327e 100644 --- a/.flake8 +++ b/.flake8 @@ -25,7 +25,7 @@ per-file-ignores = ./EDMC.py:E402 # https://github.com/best-doctor/flake8-annotations-coverage # Checks code for type annotations -min-coverage-percents = 50 +# min-coverage-percents = 50 # https://pypi.org/project/flake8-isort/ # Ensures imports are well sorted From e9b4afcdf67e7339ab61e3626c574230cba2a10c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:18:24 +0100 Subject: [PATCH 090/234] Use that bottle Plugin.apply() signature throughout # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 4 ++-- src/eddn/Monitor.py | 4 ++-- src/eddn/Relay.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index c99a419..2cdc02e 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -11,7 +11,7 @@ import hashlib import logging import zlib from datetime import datetime -from typing import Dict +from typing import Callable, Dict import gevent import simplejson @@ -352,7 +352,7 @@ class EnableCors(object): api = 2 @staticmethod - def apply(fn, context): + def apply(self, fn: Callable, context: str): """ Apply CORS headers to the calling bottle app. diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index feeb0c0..dc7bb5d 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -5,7 +5,7 @@ import collections import datetime import zlib from threading import Thread -from typing import OrderedDict +from typing import Callable, OrderedDict import gevent import mysql.connector as mariadb @@ -285,7 +285,7 @@ class EnableCors(object): api = 2 @staticmethod - def apply(fn, context): + def apply(self, fn: Callable, context: str): """ Apply a CORS handler. diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index ecd29f5..3ff289d 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -8,6 +8,7 @@ import time import uuid import zlib from threading import Thread +from typing import Callable # Logging has to be configured first before we do anything. logger = logging.getLogger(__name__) @@ -194,7 +195,7 @@ class EnableCors(object): api = 2 @staticmethod - def apply(fn, context): + def apply(self, fn: Callable, context: str): """ Apply a CORS handler. From 01b5ccf7ac810941a6041f6045807342c8ff5726 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:24:10 +0000 Subject: [PATCH 091/234] DuplicateMessages: Very initial flake8 pass --- src/eddn/core/DuplicateMessages.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/eddn/core/DuplicateMessages.py b/src/eddn/core/DuplicateMessages.py index b32844d..c3447b5 100644 --- a/src/eddn/core/DuplicateMessages.py +++ b/src/eddn/core/DuplicateMessages.py @@ -1,15 +1,20 @@ # coding: utf8 +"""Detect duplicate messages from senders.""" + import hashlib import re -import simplejson - from datetime import datetime, timedelta -from eddn.conf.Settings import Settings from threading import Lock, Thread from time import sleep +import simplejson + +from eddn.conf.Settings import Settings + class DuplicateMessages(Thread): + """Class holding all code for duplicate message detection.""" + max_minutes = Settings.RELAY_DUPLICATE_MAX_MINUTES caches = {} @@ -21,13 +26,14 @@ class DuplicateMessages(Thread): self.daemon = True def run(self): + """Expire duplicate messages.""" while True: sleep(60) with self.lock: - maxTime = datetime.utcnow() + max_time = datetime.utcnow() for key in self.caches.keys(): - if self.caches[key] + timedelta(minutes=self.max_minutes) < maxTime: + if self.caches[key] + timedelta(minutes=self.max_minutes) < max_time: del self.caches[key] def isDuplicated(self, json): From ee7ac9099698d082b3a6df0b675cbb78252bf17b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:24:43 +0000 Subject: [PATCH 092/234] DuplicateMessages: Refactor isDuplicated() name to snake_case --- src/eddn/Monitor.py | 2 +- src/eddn/Relay.py | 2 +- src/eddn/core/DuplicateMessages.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index dc7bb5d..dce2afb 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -229,7 +229,7 @@ class Monitor(Thread): # Duplicates? if Settings.RELAY_DUPLICATE_MAX_MINUTES: - if duplicate_messages.isDuplicated(json): + if duplicate_messages.is_duplicated(json): schema_id = 'DUPLICATE MESSAGE' c = db.cursor() diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 3ff289d..83624dc 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -156,7 +156,7 @@ class Relay(Thread): # Handle duplicate message if Settings.RELAY_DUPLICATE_MAX_MINUTES: - if duplicate_messages.isDuplicated(json): + if duplicate_messages.is_duplicated(json): # We've already seen this message recently. Discard it. stats_collector.tally("duplicate") return diff --git a/src/eddn/core/DuplicateMessages.py b/src/eddn/core/DuplicateMessages.py index c3447b5..d2475c7 100644 --- a/src/eddn/core/DuplicateMessages.py +++ b/src/eddn/core/DuplicateMessages.py @@ -36,7 +36,7 @@ class DuplicateMessages(Thread): if self.caches[key] + timedelta(minutes=self.max_minutes) < max_time: del self.caches[key] - def isDuplicated(self, json): + def is_duplicated(self, json): with self.lock: # Test messages are never duplicate, would be a pain to wait for another test :D if re.search('test', json['$schemaRef'], re.I): From e6b81a2a3f37c168a199bcd9b7821fd2a972b0a6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:28:45 +0000 Subject: [PATCH 093/234] DuplicateMessages: Full flake8 and mypy pass --- src/eddn/core/DuplicateMessages.py | 31 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/eddn/core/DuplicateMessages.py b/src/eddn/core/DuplicateMessages.py index d2475c7..418c761 100644 --- a/src/eddn/core/DuplicateMessages.py +++ b/src/eddn/core/DuplicateMessages.py @@ -6,6 +6,7 @@ import re from datetime import datetime, timedelta from threading import Lock, Thread from time import sleep +from typing import Dict import simplejson @@ -17,15 +18,15 @@ class DuplicateMessages(Thread): max_minutes = Settings.RELAY_DUPLICATE_MAX_MINUTES - caches = {} + caches: Dict = {} lock = Lock() - def __init__(self): + def __init__(self) -> None: super(DuplicateMessages, self).__init__() self.daemon = True - def run(self): + def run(self) -> None: """Expire duplicate messages.""" while True: sleep(60) @@ -36,35 +37,37 @@ class DuplicateMessages(Thread): if self.caches[key] + timedelta(minutes=self.max_minutes) < max_time: del self.caches[key] - def is_duplicated(self, json): + def is_duplicated(self, json: Dict) -> bool: + """Detect if the given message is in the duplicates cache.""" with self.lock: # Test messages are never duplicate, would be a pain to wait for another test :D if re.search('test', json['$schemaRef'], re.I): return False # Shallow copy, minus headers - jsonTest = { + json_test = { '$schemaRef': json['$schemaRef'], 'message': dict(json['message']), } # Remove timestamp (Mainly to avoid multiple scan messages and faction influences) - jsonTest['message'].pop('timestamp') + json_test['message'].pop('timestamp') # Convert journal starPos to avoid software modification in dupe messages - if 'StarPos' in jsonTest['message']: - jsonTest['message']['StarPos'] = [int(round(x * 32)) for x in jsonTest['message']['StarPos']] + if 'StarPos' in json_test['message']: + json_test['message']['StarPos'] = [int(round(x * 32)) for x in json_test['message']['StarPos']] # Prevent journal Docked event with small difference in distance from start - if 'DistFromStarLS' in jsonTest['message']: - jsonTest['message']['DistFromStarLS'] = int(jsonTest['message']['DistFromStarLS'] + 0.5) + if 'DistFromStarLS' in json_test['message']: + json_test['message']['DistFromStarLS'] = int(json_test['message']['DistFromStarLS'] + 0.5) # Remove journal ScanType and DistanceFromArrivalLS (Avoid duplicate scan messages after SAAScanComplete) - jsonTest['message'].pop('ScanType', None) - jsonTest['message'].pop('DistanceFromArrivalLS', None) + json_test['message'].pop('ScanType', None) + json_test['message'].pop('DistanceFromArrivalLS', None) - message = simplejson.dumps(jsonTest, sort_keys=True) # Ensure most duplicate messages will get the same key - key = hashlib.sha256(message).hexdigest() + message = simplejson.dumps(json_test, sort_keys=True) # Ensure most duplicate messages will get the same + # key + key = hashlib.sha256(message.encode('utf8')).hexdigest() if key not in self.caches: self.caches[key] = datetime.utcnow() From e6529060e23c3672504e9ec1957995bceae89d92 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:37:55 +0000 Subject: [PATCH 094/234] StatsCollector: Full flake8 and mypy pass A function name refactor touches other files. --- src/eddn/Bouncer.py | 2 +- src/eddn/Gateway.py | 2 +- src/eddn/Relay.py | 2 +- src/eddn/core/StatsCollector.py | 48 ++++++++++++++++++++++----------- 4 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index ce823e6..4cbb6bd 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -234,7 +234,7 @@ def stats() -> str: :return: JSON stats data """ - stats_current = stats_collector.getSummary() + stats_current = stats_collector.get_summary() stats_current["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats_current) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 2cdc02e..0ac382a 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -328,7 +328,7 @@ def stats() -> str: :return: JSON stats data """ - stats_current = stats_collector.getSummary() + stats_current = stats_collector.get_summary() stats_current["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats_current) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 83624dc..4c84d0e 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -75,7 +75,7 @@ def stats() -> str: :return: JSON stats data """ - stats = stats_collector.getSummary() + stats = stats_collector.get_summary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) diff --git a/src/eddn/core/StatsCollector.py b/src/eddn/core/StatsCollector.py index 9ce7f05..840453a 100644 --- a/src/eddn/core/StatsCollector.py +++ b/src/eddn/core/StatsCollector.py @@ -1,17 +1,16 @@ # coding: utf8 +"""Handle various stats about uploads.""" from collections import deque from datetime import datetime from itertools import islice from threading import Lock, Thread from time import sleep +from typing import Any, Dict class StatsCollector(Thread): - ''' - Collects simple statistics and aggregates them over the number of minutes - you choose, up to one hour. - ''' + """Collect simple statistics and aggregate them.""" def __init__(self): super(StatsCollector, self).__init__() @@ -23,41 +22,60 @@ class StatsCollector(Thread): self.lock = Lock() - self.starttime = 0 + self.start_time = 0 - def run(self): - self.starttime = datetime.utcnow() + def run(self) -> None: + """Update statistics once a minute.""" + self.start_time = datetime.utcnow() while True: sleep(60) with self.lock: for key in self.current.keys(): if key not in self.history: self.history[key] = deque(maxlen=self.max_minutes) + self.history[key].appendleft(self.current[key]) self.current[key] = 0 - def tally(self, key): + def tally(self, key: str) -> None: + """ + Add one to the count of the given key. + + :param key: Key for affected data. + """ with self.lock: if key not in self.current: self.current[key] = 1 else: self.current[key] += 1 - def getCount(self, key, minutes): + def get_count(self, key: str, minutes: int) -> int: + """ + Get current count for given key over requested time period. + + :param key: Key for requested data. + :param minutes: How many minutes back in time we want the count for. + :returns: Count for the requested data. + """ if key in self.history: return sum(islice(self.history[key], 0, min(minutes, self.max_minutes))) return 0 - def getSummary(self): - summary = {} + def get_summary(self) -> Dict[str, Any]: + """ + Get a summary of all current data. + + :returns: A Dict of the summary data. + """ + summary: Dict[str, Any] = {} for key in self.current.keys(): summary[key] = { - "1min": self.getCount(key, 1), - "5min": self.getCount(key, 5), - "60min": self.getCount(key, 60) + "1min": self.get_count(key, 1), + "5min": self.get_count(key, 5), + "60min": self.get_count(key, 60) } - summary['uptime'] = int((datetime.utcnow() - self.starttime).total_seconds()) + summary['uptime'] = int((datetime.utcnow() - self.start_time).total_seconds()) return summary From b5df9439ad144e3f61831ed8fc71b9322a8601e4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:41:22 +0000 Subject: [PATCH 095/234] requirements: strict_rfc3339 was unused --- requirements.txt | 1 - setup.py | 1 - src/eddn/core/Validator.py | 8 +++++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 161fa97..96210cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,4 @@ jsonschema==4.2.0 pyzmq==22.3.0 requests==2.25.1 simplejson==3.16.0 -strict_rfc3339==0.7 mysql-connector-python==8.0.27 diff --git a/setup.py b/setup.py index 2c04b93..ea26d7c 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,6 @@ setup( "gevent==1.3.7", "jsonschema==2.6.0", "pyzmq==17.1.2", - "strict_rfc3339==0.7", "simplejson==3.16.0", "mysql-connector-python==8.0.17" ], diff --git a/src/eddn/core/Validator.py b/src/eddn/core/Validator.py index fae15af..59d8285 100644 --- a/src/eddn/core/Validator.py +++ b/src/eddn/core/Validator.py @@ -1,9 +1,11 @@ # coding: utf8 +"""Handle validating incoming messages against the schemas.""" + +from enum import IntEnum import simplejson -import strict_rfc3339 -from enum import IntEnum -from jsonschema import validate as jsValidate, ValidationError, FormatChecker +from jsonschema import FormatChecker, ValidationError +from jsonschema import validate as jsValidate class Validator(object): From bd5784034a96e8c762409277453e0bbdbb8fc514 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 16:52:17 +0000 Subject: [PATCH 096/234] Validator: Full flake8 and mypy pass --- src/eddn/Gateway.py | 2 +- src/eddn/core/Validator.py | 115 +++++++++++++++++++++++-------------- 2 files changed, 74 insertions(+), 43 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 0ac382a..b937700 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -120,7 +120,7 @@ def configure() -> None: sender.bind(binding) for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): - validator.addSchemaResource(schema_ref, resource_string('eddn.Gateway', schema_file)) + validator.add_schema_resource(schema_ref, resource_string('eddn.Gateway', schema_file)) def push_message(parsed_message: Dict, topic: str) -> None: diff --git a/src/eddn/core/Validator.py b/src/eddn/core/Validator.py index 59d8285..2e78ee9 100644 --- a/src/eddn/core/Validator.py +++ b/src/eddn/core/Validator.py @@ -2,65 +2,96 @@ """Handle validating incoming messages against the schemas.""" from enum import IntEnum +from typing import Dict, List import simplejson from jsonschema import FormatChecker, ValidationError -from jsonschema import validate as jsValidate - - -class Validator(object): - - schemas = {"http://example.com": {}} - - def addSchemaResource(self, schemaRef, schema): - if schemaRef in self.schemas.keys(): - raise Exception("Attempted to redefine schema for " + schemaRef) - try: - schema = simplejson.loads(schema) - self.schemas[schemaRef] = schema - - except simplejson.errors.JSONDecodeError as e: - raise Exception('SCHEMA: Failed to load: ' + schemaRef) - - def validate(self, json_object): - results = ValidationResults() - - if "$schemaRef" not in json_object: - results.add(ValidationSeverity.FATAL, JsonValidationException("No $schemaRef found, unable to validate.")) - return results - - schemaRef = json_object["$schemaRef"] - if schemaRef not in self.schemas.keys(): - # We don't want to go out to the Internet and retrieve unknown schemas. - results.add(ValidationSeverity.FATAL, JsonValidationException("Schema " + schemaRef + " is unknown, unable to validate.")) - return results - - schema = self.schemas[schemaRef] - try: - jsValidate(json_object, schema, format_checker=FormatChecker()) - except ValidationError as e: - results.add(ValidationSeverity.ERROR, e) - - return results +from jsonschema import validate as json_validate class ValidationSeverity(IntEnum): + """Enum of validation status.""" + OK = 0, WARN = 1, ERROR = 2, FATAL = 3 +class JsonValidationException(Exception): + """Exception for JSON Validation errors.""" + + pass + + class ValidationResults(object): + """Validation results.""" - def __init__(self): + def __init__(self) -> None: self.severity = ValidationSeverity.OK - self.messages = [] + self.messages: List[JsonValidationException] = [] - def add(self, severity, exception): + def add(self, severity: ValidationSeverity, exception: JsonValidationException) -> None: + """ + Add a validation failure to the results. + + :param severity: + :param exception: + :return: + """ self.severity = max(severity, self.severity) self.messages.append(exception) -class JsonValidationException(Exception): - pass +class Validator(object): + """Perform validation on incoming messages.""" + + schemas: Dict = {"http://example.com": {}} + + def add_schema_resource(self, schema_ref, schema) -> None: + """ + Add the given schema to the list to validate against. + + :param schema_ref: Schema URL. + :param schema: The schema + """ + if schema_ref in self.schemas.keys(): + raise Exception("Attempted to redefine schema for " + schema_ref) + + try: + schema = simplejson.loads(schema) + self.schemas[schema_ref] = schema + + except simplejson.JSONDecodeError: + raise Exception(f'SCHEMA: Failed to load: {schema_ref}') + + def validate(self, json_object: Dict) -> ValidationResults: + """ + Validate the given message. + + :param json_object: The message to be validated. + :return: The results of validation. + """ + results = ValidationResults() + + if "$schemaRef" not in json_object: + results.add(ValidationSeverity.FATAL, JsonValidationException("No $schemaRef found, unable to validate.")) + return results + + schema_ref = json_object["$schemaRef"] + if schema_ref not in self.schemas.keys(): + # We don't want to go out to the Internet and retrieve unknown schemas. + results.add( + ValidationSeverity.FATAL, + JsonValidationException(f'Schema {schema_ref} is unknown, unable to validate.') + ) + return results + + schema = self.schemas[schema_ref] + try: + json_validate(json_object, schema, format_checker=FormatChecker()) + + except ValidationError as e: + results.add(ValidationSeverity.ERROR, e) + + return results From 998be94c5d1fdb44e6f61bb6c851eab86a0de8a9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:00:46 +0000 Subject: [PATCH 097/234] setup.py: General python3, flake8, mypy --- setup.py | 104 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index ea26d7c..8902962 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +"""Setup for EDDN software.""" import glob import os import re @@ -6,20 +7,23 @@ import subprocess import sys from setuptools import setup, find_packages +import setup_env +from setuptools import find_packages, setup + VERSIONFILE = "src/eddn/conf/Version.py" -verstr = "unknown" +verstr = "unknown" try: verstrline = open(VERSIONFILE, "rt").read() - VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" - mo = re.search(VSRE, verstrline, re.M) + VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]" + mo = re.search(VSRE, verstrline, re.M) if mo: verstr = mo.group(1) + except EnvironmentError: - print "unable to find version in %s" % (VERSIONFILE,) - raise RuntimeError("if %s exists, it is required to be well-formed" % (VERSIONFILE,)) + print(f'unable to find version in {VERSIONFILE}') + raise RuntimeError(f'if {VERSIONFILE} exists, it is required to be well-formed') # Read environment-specific settings -import setup_env ########################################################################### @@ -60,9 +64,9 @@ if setup_env.EDDN_ENV == 'live': ########################################################################### # Location of start-eddn-service script and its config file -START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] ) +START_SCRIPT_BIN = f'{os.environ["HOME"]}/.local/bin' # Location of web files -SHARE_EDDN_FILES='%s/.local/share/eddn/%s' % ( os.environ['HOME'], setup_env.EDDN_ENV ) +SHARE_EDDN_FILES = f'{os.environ["HOME"]}/.local/share/eddn/{setup_env.EDDN_ENV}' setup( name='eddn', @@ -79,7 +83,7 @@ setup( 'src', exclude=["*.tests"] ), - package_dir = {'':'src'}, + package_dir={'': 'src'}, # This includes them for the running code, but that doesn't help # serve them up for reference. @@ -112,26 +116,28 @@ setup( } ) -def open_file_perms_recursive(dirname): + +def open_file_perms_recursive(dirname: str) -> None: """Open up file perms on the given directory and its contents.""" - print 'open_file_perms_recursive: %s' % ( dirname ) + print(f'open_file_perms_recursive: {dirname}') names = os.listdir(dirname) for name in names: - n = '%s/%s' % ( dirname, name ) - print 'open_file_perms_recursive: %s' % ( n ) + n = f'{dirname}/{name}' + print(f'open_file_perms_recursive: {n}') if (os.path.isdir(n)): - os.chmod(n, 0755) + os.chmod(n, 0o755) # Recurse open_file_perms_recursive(n) - + else: - os.chmod(n, 0644) + os.chmod(n, 0o644) + # Ensure the systemd-required start files are in place -print """ +print(""" ****************************************************************************** Ensuring start script and its config file are in place... -""" +""") old_cwd = os.getcwd() if not os.path.isdir(START_SCRIPT_BIN): # We're still using Python 2.7, so no pathlib @@ -146,28 +152,38 @@ if not os.path.isdir(START_SCRIPT_BIN): os.chdir(pc) if not os.path.isdir(START_SCRIPT_BIN): - print "%s can't be created, aborting!!!" % (START_SCRIPT_BIN) + print(f"{START_SCRIPT_BIN} can't be created, aborting!!!") exit(-1) os.chdir(old_cwd) shutil.copy( - 'systemd/eddn_%s_config' % ( setup_env.EDDN_ENV), - '%s/eddn_%s_config' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) + f'systemd/eddn_{setup_env.EDDN_ENV}_config', + f'{START_SCRIPT_BIN}/eddn_{setup_env.EDDN_ENV}_config' ) # NB: We copy to a per-environment version so that, e.g.live use won't break # due to changes in the other environments. shutil.copy( 'systemd/start-eddn-service', - '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) + f'{START_SCRIPT_BIN}/start-eddn-{setup_env.EDDN_ENV}-service' +) + +# Ensure the service log file archiving script is in place +print(""" +****************************************************************************** +Ensuring the service log file archiving script is in place +""") +shutil.copy( + 'contrib/eddn-logs-archive', + START_SCRIPT_BIN ) # Ensure the latest monitor files are in place -old_umask = os.umask(022) -print """ +old_umask = os.umask(0o22) +print(f""" ****************************************************************************** -Ensuring %s exists... -""" % ( SHARE_EDDN_FILES ) +Ensuring {SHARE_EDDN_FILES} exists... +""") old_cwd = os.getcwd() if not os.path.isdir(SHARE_EDDN_FILES): # We're still using Python 2.7, so no pathlib @@ -182,61 +198,63 @@ if not os.path.isdir(SHARE_EDDN_FILES): os.chdir(pc) if not os.path.isdir(SHARE_EDDN_FILES): - print "%s can't be created, aborting!!!" % (SHARE_EDDN_FILES) + print(f"{SHARE_EDDN_FILES} can't be created, aborting!!!") exit(-1) os.chdir(old_cwd) -print """ +print(""" ****************************************************************************** Ensuring latest monitor files are in place... -""" +""") # Copy the monitor (Web page) files try: - shutil.rmtree('%s/monitor' % ( SHARE_EDDN_FILES )) + shutil.rmtree(f'{SHARE_EDDN_FILES}/monitor') except OSError: pass shutil.copytree( 'contrib/monitor', - '%s/monitor' % ( SHARE_EDDN_FILES ), + f'{SHARE_EDDN_FILES}/monitor', # Not in Python 2.7 # copy_function=shutil.copyfile, ) # And a copy of the schemas too -print """ +print(""" ****************************************************************************** Ensuring latest schema files are in place for web access... -""" +""") try: - shutil.rmtree('%s/schemas' % ( SHARE_EDDN_FILES )) + shutil.rmtree(f'{SHARE_EDDN_FILES}/schemas') + except OSError: pass + shutil.copytree( 'schemas', - '%s/schemas' % ( SHARE_EDDN_FILES ), + f'{SHARE_EDDN_FILES}/schemas', # Not in Python 2.7 # copy_function=shutil.copyfile, ) -print """ +print(""" ****************************************************************************** Opening up permissions on monitor and schema files... -""" -os.chmod(SHARE_EDDN_FILES, 0755) +""") +os.chmod(SHARE_EDDN_FILES, 0o755) open_file_perms_recursive(SHARE_EDDN_FILES) # You still need to make an override config file -if not os.path.isfile('%s/config.json' % ( SHARE_EDDN_FILES )): +if not os.path.isfile(f'{SHARE_EDDN_FILES}/config.json'): shutil.copy('docs/config-EXAMPLE.json', SHARE_EDDN_FILES) - print """ + print(f""" ****************************************************************************** There was no config.json file in place, so docs/config-EXAMPLE.json was copied into: - %s - + {SHARE_EDDN_FILES} + Please review, edit and rename this file to 'config.json' so that this software will actually work. See docs/Running-this-software.md for guidance. ****************************************************************************** -""" % ( SHARE_EDDN_FILES ) +""") os.umask(old_umask) From 257d3e880a65bb0517d9d72ef9bbcf83cebfaf36 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 17:53:03 +0000 Subject: [PATCH 098/234] setup.py: Don't cite module versions, let requirements.txt do that --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 8902962..e81a825 100644 --- a/setup.py +++ b/setup.py @@ -97,13 +97,13 @@ setup( # work, for instance. install_requires=[ "argparse", - "bottle==0.12.15", - "enum34==1.1.6", - "gevent==1.3.7", - "jsonschema==2.6.0", - "pyzmq==17.1.2", - "simplejson==3.16.0", - "mysql-connector-python==8.0.17" + "bottle", + "enum34", + "gevent", + "jsonschema", + "pyzmq", + "simplejson", + "mysql-connector-python" ], entry_points={ From fdee2c054f310a92ec098a3ac69e99fd92720ebe Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 18:14:01 +0000 Subject: [PATCH 099/234] python3: Gateway now working Now I'm actually trying to run the code I'm finding more that needs changing in order to run under python3 --- src/eddn/Bouncer.py | 4 +-- src/eddn/Gateway.py | 52 +++++++++++++++++---------------------- src/eddn/Monitor.py | 2 +- src/eddn/Relay.py | 6 ++--- src/eddn/conf/Settings.py | 2 +- 5 files changed, 29 insertions(+), 37 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index 4cbb6bd..730f14e 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -258,7 +258,7 @@ class EnableCors(object): name = 'enable_cors' api = 2 - def apply(self, fn: Callable, context: str): + def apply(self, fn: Callable): """ Apply CORS headers to the calling bottle app. @@ -286,7 +286,7 @@ class CustomLogging(object): name = 'custom_logging' api = 2 - def apply(self, fn: Callable, context: str): + def apply(self, fn: Callable): """ Apply custom logging to bottle request. diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index b937700..a5155d7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -119,7 +119,7 @@ def configure() -> None: for binding in Settings.GATEWAY_SENDER_BINDINGS: sender.bind(binding) - for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): + for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.items(): validator.add_schema_resource(schema_ref, resource_string('eddn.Gateway', schema_file)) @@ -137,7 +137,7 @@ def push_message(parsed_message: Dict, topic: str) -> None: # announcers with schema as topic compressed_msg = zlib.compress(string_message) - send_message = f"{str(topic)!r} |-| {compressed_msg!r}" + send_message = f"{str(topic)!r} |-| {compressed_msg!r}".encode('utf8') sender.send(send_message) stats_collector.tally("outbound") @@ -345,34 +345,26 @@ class MalformedUploadError(Exception): pass -class EnableCors(object): - """Handle enabling CORS headers in all responses.""" - - name = 'enable_cors' - api = 2 - - @staticmethod - def apply(self, fn: Callable, context: str): - """ - Apply CORS headers to the calling bottle app. - - :param fn: - :param context: - :return: - """ - def _enable_cors(*args, **kwargs): - # set CORS headers - response.headers['Access-Control-Allow-Origin'] = '*' - response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = \ - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' - - if request.method != 'OPTIONS': - # actual request; reply with the actual response - return fn(*args, **kwargs) - - return _enable_cors +def apply_cors() -> None: + """ + Apply CORS headers to the calling bottle app. + :param fn: + :param context: + :return: + """ + response.set_header( + 'Access-Control-Allow-Origin', + '*' + ) + response.set_header( + 'Access-Control-Allow-Methods', + 'GET, POST, PUT, OPTIONS' + ) + response.set_header( + 'Access-Control-Allow-Headers', + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + ) def main() -> None: """Handle setting up and running the bottle app.""" @@ -383,7 +375,7 @@ def main() -> None: load_config(cl_args) configure() - app.install(EnableCors()) + app.add_hook('after_request', apply_cors) app.run( host=Settings.GATEWAY_HTTP_BIND_ADDRESS, port=Settings.GATEWAY_HTTP_PORT, diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index dce2afb..3e2b36f 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -285,7 +285,7 @@ class EnableCors(object): api = 2 @staticmethod - def apply(self, fn: Callable, context: str): + def apply(self, fn: Callable): """ Apply a CORS handler. diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 4c84d0e..af3280f 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -118,9 +118,9 @@ class Relay(Thread): # Filters on topics or not... if Settings.RELAY_RECEIVE_ONLY_GATEWAY_EXTRA_JSON is True: - for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.iteritems(): + for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.items(): receiver.setsockopt(zmq.SUBSCRIBE, schema_ref) - for schema_ref, schema_file in Settings.RELAY_EXTRA_JSON_SCHEMAS.iteritems(): + for schema_ref, schema_file in Settings.RELAY_EXTRA_JSON_SCHEMAS.items(): receiver.setsockopt(zmq.SUBSCRIBE, schema_ref) else: receiver.setsockopt(zmq.SUBSCRIBE, '') @@ -195,7 +195,7 @@ class EnableCors(object): api = 2 @staticmethod - def apply(self, fn: Callable, context: str): + def apply(self, fn: Callable): """ Apply a CORS handler. diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index d61cd31..13012d5 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -140,7 +140,7 @@ class _Settings(object): def load_from(self, file_name: str) -> None: f = open(file_name, 'r') conf = simplejson.load(f) - for key, value in conf.iteritems(): + for key, value in conf.items(): if key in dir(self): self.__setattr__(key, value) From f7370a2f0b7e12f77d2995c7a4002a6baf191e21 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 18:19:01 +0000 Subject: [PATCH 100/234] Monitor: *Maybe* working now? It doesn't crash, including when the gateway successfully receives a message, but unclear if those messages are then making it to the Monitor OK. --- src/eddn/Monitor.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 3e2b36f..0b78858 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -278,31 +278,24 @@ class Monitor(Thread): gevent.spawn(monitor_worker, inbound_message) -class EnableCors(object): - """Enable CORS responses.""" +def apply_cors() -> None: + """ + Apply a CORS handler. - name = 'enable_cors' - api = 2 - - @staticmethod - def apply(self, fn: Callable): - """ - Apply a CORS handler. - - Ref: - """ - def _enable_cors(*args, **kwargs): - """Set CORS Headers.""" - response.headers['Access-Control-Allow-Origin'] = '*' - response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = \ - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' - - if request.method != 'OPTIONS': - # actual request; reply with the actual response - return fn(*args, **kwargs) - - return _enable_cors + Ref: + """ + response.set_header( + 'Access-Control-Allow-Origin', + '*' + ) + response.set_header( + 'Access-Control-Allow-Methods', + 'GET, POST, PUT, OPTIONS' + ) + response.set_header( + 'Access-Control-Allow-Headers', + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + ) def main() -> None: """Handle setting up and running the bottle app.""" @@ -311,7 +304,7 @@ def main() -> None: m = Monitor() m.start() - app.install(EnableCors()) + app.add_hook('after_request', apply_cors) app.run( host=Settings.MONITOR_HTTP_BIND_ADDRESS, port=Settings.MONITOR_HTTP_PORT, From cc19d390ce71652376dcb371d8c35284fe221719 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 4 Nov 2021 18:26:44 +0000 Subject: [PATCH 101/234] Monitor: python3 tweaks * setsockopt_string() * Trying to see what's up with: Traceback (most recent call last): File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run File "/home/eddn/dev/python3.9-venv/lib/python3.9/site-packages/eddn-2.0a0.dev0-py3.9.egg/eddn/Monitor.py", line 203, in monitor_worker message_text = zlib.decompress(message) zlib.error: Error -3 while decompressing data: incorrect header check 2021-11-04T18:25:03Z failed with error --- src/eddn/Monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 0b78858..9b01b49 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -198,7 +198,7 @@ class Monitor(Thread): context = zmq.Context() receiver = context.socket(zmq.SUB) - receiver.setsockopt(zmq.SUBSCRIBE, '') + receiver.setsockopt_string(zmq.SUBSCRIBE, '') for binding in Settings.MONITOR_RECEIVER_BINDINGS: receiver.connect(binding) @@ -219,6 +219,7 @@ class Monitor(Thread): else: message = message_split[0] + print(f'message: {message}') message_text = zlib.decompress(message) json = simplejson.loads(message_text) From 529725d2dbfdf90dd606ccc8ed7e624d96346f88 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 7 Nov 2021 15:16:14 +0000 Subject: [PATCH 102/234] Gateway -> Monitor now working We literally weren't making use of the 'topic' in the message. Thus, so as to avoid issues with trying to mash a string topic together with a bytes (compressed) message, I've ripped that out. --- src/eddn/Gateway.py | 6 ++---- src/eddn/Monitor.py | 13 +------------ 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index a5155d7..daaefeb 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -135,11 +135,9 @@ def push_message(parsed_message: Dict, topic: str) -> None: # Push a zlib compressed JSON representation of the message to # announcers with schema as topic - compressed_msg = zlib.compress(string_message) + compressed_message = zlib.compress(string_message) - send_message = f"{str(topic)!r} |-| {compressed_msg!r}".encode('utf8') - - sender.send(send_message) + sender.send(compressed_message) stats_collector.tally("outbound") diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 9b01b49..823b7a8 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -210,23 +210,12 @@ class Monitor(Thread): database=Settings.MONITOR_DB['database'] ) - # Separate topic from message - message_split = message.split(b' |-| ') - - # Handle gateway not sending topic - if len(message_split) > 1: - message = message_split[1] - else: - message = message_split[0] - - print(f'message: {message}') message_text = zlib.decompress(message) json = simplejson.loads(message_text) # Default variables schema_id = json['$schemaRef'] - software_id = json['header']['softwareName'].encode('utf8') + ' | ' \ - + json['header']['softwareVersion'].encode('utf8') + software_id = f'{json["header"]["softwareName"]} | {json["header"]["softwareVersion"]}' # Duplicates? if Settings.RELAY_DUPLICATE_MAX_MINUTES: From 7fe2e8880a9eebb274eb0607c8624564e4cc0845 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 7 Nov 2021 15:23:48 +0000 Subject: [PATCH 103/234] Relay: As we're not sending topic from Gateway, take out the sub code Just subscribe to everything. --- src/eddn/Relay.py | 62 +++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index af3280f..d158bb5 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -115,15 +115,7 @@ class Relay(Thread): context = zmq.Context() receiver = context.socket(zmq.SUB) - - # Filters on topics or not... - if Settings.RELAY_RECEIVE_ONLY_GATEWAY_EXTRA_JSON is True: - for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.items(): - receiver.setsockopt(zmq.SUBSCRIBE, schema_ref) - for schema_ref, schema_file in Settings.RELAY_EXTRA_JSON_SCHEMAS.items(): - receiver.setsockopt(zmq.SUBSCRIBE, schema_ref) - else: - receiver.setsockopt(zmq.SUBSCRIBE, '') + receiver.setsockopt_string(zmq.SUBSCRIBE, '') for binding in Settings.RELAY_RECEIVER_BINDINGS: # Relays bind upstream to an Announcer, or another Relay. @@ -142,15 +134,6 @@ class Relay(Thread): :param message: Message to be passed on. """ - # Separate topic from message - message_split = message.split(b' |-| ') - - # Handle gateway not sending topic - if len(message_split) > 1: - message = message_split[1] - else: - message = message_split[0] - message_text = zlib.decompress(message) json = simplejson.loads(message_text) @@ -188,31 +171,24 @@ class Relay(Thread): gevent.spawn(relay_worker, inbound_message) -class EnableCors(object): - """Enable CORS responses.""" +def apply_cors(): + """ + Apply a CORS handler. - name = 'enable_cors' - api = 2 - - @staticmethod - def apply(self, fn: Callable): - """ - Apply a CORS handler. - - Ref: - """ - def _enable_cors(*args, **kwargs): - """Set CORS Headers.""" - response.headers['Access-Control-Allow-Origin'] = '*' - response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = \ - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' - - if request.method != 'OPTIONS': - # actual request; reply with the actual response - return fn(*args, **kwargs) - - return _enable_cors + Ref: + """ + response.set_header( + 'Access-Control-Allow-Origin', + '*' + ) + response.set_header( + 'Access-Control-Allow-Methods', + 'GET, POST, PUT, OPTIONS' + ) + response.set_header( + 'Access-Control-Allow-Headers', + 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + ) def main() -> None: @@ -226,7 +202,7 @@ def main() -> None: r = Relay() r.start() - app.install(EnableCors()) + app.add_hook('after_request', apply_cors) app.run( host=Settings.RELAY_HTTP_BIND_ADDRESS, port=Settings.RELAY_HTTP_PORT, From 5e51c604a4817f14512d221beebeb9ab0fa42a8a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:18:48 +0100 Subject: [PATCH 104/234] flake8: minor cleanups # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 8 +++++--- src/eddn/Monitor.py | 2 +- src/eddn/Relay.py | 5 ++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index daaefeb..aa8e57b 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -1,3 +1,4 @@ + # coding: utf8 """ @@ -11,7 +12,7 @@ import hashlib import logging import zlib from datetime import datetime -from typing import Callable, Dict +from typing import Dict import gevent import simplejson @@ -353,8 +354,8 @@ def apply_cors() -> None: """ response.set_header( 'Access-Control-Allow-Origin', - '*' - ) + '*' + ) response.set_header( 'Access-Control-Allow-Methods', 'GET, POST, PUT, OPTIONS' @@ -364,6 +365,7 @@ def apply_cors() -> None: 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' ) + def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 823b7a8..36ec2ad 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -5,7 +5,7 @@ import collections import datetime import zlib from threading import Thread -from typing import Callable, OrderedDict +from typing import OrderedDict import gevent import mysql.connector as mariadb diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index d158bb5..131b497 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -8,7 +8,6 @@ import time import uuid import zlib from threading import Thread -from typing import Callable # Logging has to be configured first before we do anything. logger = logging.getLogger(__name__) @@ -26,7 +25,7 @@ logger.info('Made logger') import gevent import simplejson import zmq.green as zmq -from bottle import Bottle, request, response +from bottle import Bottle, response from gevent import monkey from eddn.conf.Settings import Settings, load_config @@ -109,7 +108,7 @@ class Relay(Thread): return hashlib.sha1(f"{self.uploader_nonce!r}-{uploader.encode}".encode('utf8')).hexdigest() - def run(self) -> None: # noqa: CCR001 + def run(self) -> None: """Handle receiving messages from Gateway and passing them on.""" # These form the connection to the Gateway daemon(s) upstream. context = zmq.Context() From a8c683093c52866c2e6a16141f8caea3b7bdd783 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 7 Nov 2021 15:42:49 +0000 Subject: [PATCH 105/234] github: Add dependabot config --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..586fb2d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + target-branch: "develop" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + target-branch: "develop" + schedule: + interval: "daily" From 1c07e4ccc780300a5b0d5d5d75dc18b443dc9d60 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 7 Nov 2021 15:44:11 +0000 Subject: [PATCH 106/234] github: Add pr-checks workflow * flake8 checks --- .github/workflows/pr-checks.yml | 93 +++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/pr-checks.yml diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 0000000..5ade895 --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,93 @@ +# This workflow will: +# +# * install Python dependencies +# * lint with a single version of Python +# +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: PR-Checks + +on: + pull_request: + branches: [ develop ] + +jobs: + flake8: + runs-on: ubuntu-18.04 + + steps: + + # Debug show the info we have to work with + - name: Show github context + run: cat $GITHUB_EVENT_PATH + + #################################################################### + # Checkout the necessary commits + #################################################################### + # We need the repo from the 'head' of the PR, not what it's + # based on. + - name: Checkout head commits + # https://github.com/actions/checkout + uses: actions/checkout@v2.4.0 + #with: + #ref: ${{github.head.sha}} + #repository: ${{github.event.pull_request.head.repo.full_name}} + #fetch-depth: 0 + + # But we do need the base references + - name: Fetch base commits + env: + BASE_REPO_URL: ${{github.event.pull_request.base.repo.svn_url}} + BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} + + run: | + echo "BASE_REPO_URL: ${BASE_REPO_URL}" + echo "BASE_REPO_OWNER: ${BASE_REPO_OWNER}" + # Add the 'base' repo as a new remote + git remote add ${BASE_REPO_OWNER} ${BASE_REPO_URL} + # And then fetch its references + git fetch ${BASE_REPO_OWNER} + #################################################################### + + #################################################################### + # Get Python set up + #################################################################### + - name: Set up Python 3.9 + uses: actions/setup-python@v2.2.2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install wheel flake8 + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt ; fi + #################################################################### + + # Have issues be annotated on run + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1 + + #################################################################### + # Lint with flake8 + #################################################################### + - name: Lint with flake8 + env: + BASE_REPO_URL: ${{github.event.pull_request.base.repo.svn_url}} + BASE_REPO_OWNER: ${{github.event.pull_request.base.repo.owner.login}} + BASE_REF: ${{github.base_ref}} + + run: | + echo "BASE_REPO_URL: ${BASE_REPO_URL}" + echo "BASE_REPO_OWNER: ${BASE_REPO_OWNER}" + echo "BASE_REF: ${BASE_REF}" + # Explicitly check for some errors + # E9 - Runtime (syntax and the like) + # F63 - 'tests' checking + # F7 - syntax errors + # F82 - undefined checking + git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --diff + # Can optionally add `--exit-zero` to the flake8 arguments so that + # this doesn't fail the build. + # explicitly ignore docstring errors (start with D) + git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --statistics --diff --extend-ignore D + #################################################################### From 8c3bf0eafffb912b0f42052fc10b291b9720069b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 7 Nov 2021 15:55:58 +0000 Subject: [PATCH 107/234] python: Use version 3.9.8 --- .python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.python-version b/.python-version index f69abe4..26cb485 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.9.7 +3.9.8 From 2e09c5de32de01f71b993253e395b8b20476a3ab Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 13:18:35 +0000 Subject: [PATCH 108/234] coveragerv: Exclude venv-3.9 as well --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 019facb..6281978 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,4 @@ omit = tests/* # Any venv files venv/* + venv-3.9/* From 323032f92ae11ae1e5b72ba48fc3f80f04b157df Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 13:19:45 +0000 Subject: [PATCH 109/234] .pydevproject; Update to cite Python 3.9 --- .pydevproject | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pydevproject b/.pydevproject index 6ceaf89..b033028 100644 --- a/.pydevproject +++ b/.pydevproject @@ -3,6 +3,6 @@ /${PROJECT_DIR_NAME}/src -python 2.6 +python 3.9 Default From 4b5687808f20c907cb798be330c2f42020aee569 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 13:30:41 +0000 Subject: [PATCH 110/234] Script to check that schema files are valid JSON --- scripts/check-schemas-load.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 scripts/check-schemas-load.py diff --git a/scripts/check-schemas-load.py b/scripts/check-schemas-load.py new file mode 100644 index 0000000..2d95559 --- /dev/null +++ b/scripts/check-schemas-load.py @@ -0,0 +1,26 @@ +"""Verify that all the current schema files actually load.""" + +import pathlib +import sys + +import simplejson + +# From parent of where this script is +script_path = pathlib.Path(sys.argv[0]) +root_dir = script_path.parent.parent + +# Take every file in the schemas directory +schemas_dir = root_dir / 'schemas' +failures = 0 +for schema_file in schemas_dir.glob('*-v*.*.json'): + # print(f'Schema: {schema_file}') + with open(schema_file, 'r') as sf: + try: + json = simplejson.load(sf) + + except simplejson.JSONDecodeError as e: + print(f'Failed to load {schema_file}:\n{e!r}') + failures += 1 + +if failures > 0: + exit(-1) From 49b14aa62e6c5feebda0338005c583fcbff76500 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 13:33:37 +0000 Subject: [PATCH 111/234] GitHub: pr-checks: Run scheck-schemas-load script --- .github/workflows/pr-checks.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 5ade895..d415782 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -91,3 +91,11 @@ jobs: # explicitly ignore docstring errors (start with D) git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --statistics --diff --extend-ignore D #################################################################### + + schemas_load: + runs-on: ubuntu-18.04 + + steps: + - name: Check schemas + run: | + scripts/check-schemas-load.py From 98b868a1154311ece4950d95b2f33fd479a05b5f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 13:35:52 +0000 Subject: [PATCH 112/234] GitHub: PR checks: Run schema check inside renamed linting job --- .github/workflows/pr-checks.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index d415782..93466df 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -12,7 +12,7 @@ on: branches: [ develop ] jobs: - flake8: + linting: runs-on: ubuntu-18.04 steps: @@ -92,10 +92,6 @@ jobs: git diff "refs/remotes/${BASE_REPO_OWNER}/${BASE_REF}" -- | flake8 . --count --statistics --diff --extend-ignore D #################################################################### - schemas_load: - runs-on: ubuntu-18.04 - - steps: - - name: Check schemas - run: | - scripts/check-schemas-load.py + - name: Check schemas + run: | + scripts/check-schemas-load.py From d6fa3fe1233f3c03778b90dc1fe1d9aaaf7749d6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 13:38:03 +0000 Subject: [PATCH 113/234] GitHub: PR Checks: run script via python --- .github/workflows/pr-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 93466df..1e64af1 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -94,4 +94,4 @@ jobs: - name: Check schemas run: | - scripts/check-schemas-load.py + python scripts/check-schemas-load.py From f9d423f4a67c0662a485d9008879068829af003c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 8 Nov 2021 14:07:55 +0000 Subject: [PATCH 114/234] setup.py: Updated to Python3 / pathlib paradigms --- setup.py | 91 +++++++++++++++++++++----------------------------------- 1 file changed, 34 insertions(+), 57 deletions(-) diff --git a/setup.py b/setup.py index e81a825..a7aa824 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,17 @@ """Setup for EDDN software.""" import glob import os +import pathlib import re import shutil import subprocess import sys from setuptools import setup, find_packages -import setup_env from setuptools import find_packages, setup +import setup_env + VERSIONFILE = "src/eddn/conf/Version.py" verstr = "unknown" try: @@ -64,9 +66,9 @@ if setup_env.EDDN_ENV == 'live': ########################################################################### # Location of start-eddn-service script and its config file -START_SCRIPT_BIN = f'{os.environ["HOME"]}/.local/bin' +START_SCRIPT_BIN = pathlib.Path(f'{os.environ["HOME"]}/.local/bin') # Location of web files -SHARE_EDDN_FILES = f'{os.environ["HOME"]}/.local/share/eddn/{setup_env.EDDN_ENV}' +SHARE_EDDN_FILES = pathlib.Path(f'{os.environ["HOME"]}/.local/share/eddn/{setup_env.EDDN_ENV}') setup( name='eddn', @@ -117,20 +119,18 @@ setup( ) -def open_file_perms_recursive(dirname: str) -> None: +def open_file_perms_recursive(dirname: pathlib.Path) -> None: """Open up file perms on the given directory and its contents.""" print(f'open_file_perms_recursive: {dirname}') - names = os.listdir(dirname) - for name in names: - n = f'{dirname}/{name}' - print(f'open_file_perms_recursive: {n}') - if (os.path.isdir(n)): - os.chmod(n, 0o755) - # Recurse - open_file_perms_recursive(n) + + for name in dirname.glob('*'): + print(f'open_file_perms_recursive: {name}') + if name.is_dir(): + name.chmod(0o755) + open_file_perms_recursive(name) else: - os.chmod(n, 0o644) + name.chmod(0o644) # Ensure the systemd-required start files are in place @@ -138,34 +138,22 @@ print(""" ****************************************************************************** Ensuring start script and its config file are in place... """) -old_cwd = os.getcwd() -if not os.path.isdir(START_SCRIPT_BIN): - # We're still using Python 2.7, so no pathlib - os.chdir('/') - for pc in START_SCRIPT_BIN[1:].split('/'): - try: - os.mkdir(pc) +try: + START_SCRIPT_BIN.mkdir(mode=0o700, parents=True, exist_ok=True) - except OSError: - pass - - os.chdir(pc) - - if not os.path.isdir(START_SCRIPT_BIN): - print(f"{START_SCRIPT_BIN} can't be created, aborting!!!") - exit(-1) - -os.chdir(old_cwd) +except Exception as e: + print(f"{START_SCRIPT_BIN} can't be created, aborting!!!\n{e!r}") + exit(-1) shutil.copy( f'systemd/eddn_{setup_env.EDDN_ENV}_config', - f'{START_SCRIPT_BIN}/eddn_{setup_env.EDDN_ENV}_config' + START_SCRIPT_BIN / f'eddn_{setup_env.EDDN_ENV}_config' ) # NB: We copy to a per-environment version so that, e.g.live use won't break # due to changes in the other environments. shutil.copy( 'systemd/start-eddn-service', - f'{START_SCRIPT_BIN}/start-eddn-{setup_env.EDDN_ENV}-service' + START_SCRIPT_BIN / f'start-eddn-{setup_env.EDDN_ENV}-service' ) # Ensure the service log file archiving script is in place @@ -184,38 +172,28 @@ print(f""" ****************************************************************************** Ensuring {SHARE_EDDN_FILES} exists... """) -old_cwd = os.getcwd() -if not os.path.isdir(SHARE_EDDN_FILES): - # We're still using Python 2.7, so no pathlib - os.chdir('/') - for pc in SHARE_EDDN_FILES[1:].split('/'): - try: - os.mkdir(pc) +try: + SHARE_EDDN_FILES.mkdir(mode=0o700, parents=True, exist_ok=True) - except OSError: - pass +except Exception as e: + print(f"{SHARE_EDDN_FILES} can't be created, aborting!!!\n{e!r}") + exit(-1) - os.chdir(pc) - - if not os.path.isdir(SHARE_EDDN_FILES): - print(f"{SHARE_EDDN_FILES} can't be created, aborting!!!") - exit(-1) - -os.chdir(old_cwd) print(""" ****************************************************************************** Ensuring latest monitor files are in place... """) # Copy the monitor (Web page) files try: - shutil.rmtree(f'{SHARE_EDDN_FILES}/monitor') + shutil.rmtree(SHARE_EDDN_FILES / 'monitor') + except OSError: pass + shutil.copytree( 'contrib/monitor', - f'{SHARE_EDDN_FILES}/monitor', - # Not in Python 2.7 - # copy_function=shutil.copyfile, + SHARE_EDDN_FILES / 'monitor', + copy_function=shutil.copyfile, # type: ignore ) # And a copy of the schemas too print(""" @@ -223,16 +201,15 @@ print(""" Ensuring latest schema files are in place for web access... """) try: - shutil.rmtree(f'{SHARE_EDDN_FILES}/schemas') + shutil.rmtree(SHARE_EDDN_FILES / 'schemas') except OSError: pass shutil.copytree( 'schemas', - f'{SHARE_EDDN_FILES}/schemas', - # Not in Python 2.7 - # copy_function=shutil.copyfile, + SHARE_EDDN_FILES / 'schemas', + copy_function=shutil.copyfile, # type: ignore ) print(""" @@ -243,7 +220,7 @@ os.chmod(SHARE_EDDN_FILES, 0o755) open_file_perms_recursive(SHARE_EDDN_FILES) # You still need to make an override config file -if not os.path.isfile(f'{SHARE_EDDN_FILES}/config.json'): +if not (SHARE_EDDN_FILES / 'config.json').is_file(): shutil.copy('docs/config-EXAMPLE.json', SHARE_EDDN_FILES) print(f""" ****************************************************************************** From 2b39934fd71fdd7b56201082a5f9ee9357b525d8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 7 Jan 2022 13:09:29 +0000 Subject: [PATCH 115/234] 'Guard' non-code to ensure this gets updated for bottle size limit etc --- src/eddn/Gateway.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index aa8e57b..3dd5724 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -1,6 +1,7 @@ # coding: utf8 +THIS NEEDS UPDATING FOR THE SIZE LIMIT TWEAK AND EXTRA LOGGING """ EDDN Gateway, which receives message from uploaders. From 3dba22b6cb5c15e8d48feec3c381cdaeba2e96d6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:06:47 +0000 Subject: [PATCH 116/234] Gateway: Remove "remember to do the python3 rebase" line --- src/eddn/Gateway.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 3dd5724..aa8e57b 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -1,7 +1,6 @@ # coding: utf8 -THIS NEEDS UPDATING FOR THE SIZE LIMIT TWEAK AND EXTRA LOGGING """ EDDN Gateway, which receives message from uploaders. From e9397b46e2069e34331a9b696985647eb94e9c15 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:07:28 +0000 Subject: [PATCH 117/234] Settings: Rename loadConfig -> load_config --- src/eddn/conf/Settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 13012d5..3ce0d05 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -151,7 +151,7 @@ class _Settings(object): Settings = _Settings() -def loadConfig(cl_args) -> None: +def load_config(cl_args) -> None: """ Load in a commandline-specified settings file, if applicable. From 3a55f6b005b1abceab1da37ecd36cec6986fe3b4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:14:27 +0000 Subject: [PATCH 118/234] Gateway: zmq's PUB is *only* in the main module now But we still need to `import zmq.green as zmq` so use: ```python from zmq import PUB as ZMQ_PUB ``` --- src/eddn/Gateway.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index aa8e57b..2c0b4ee 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -21,6 +21,7 @@ import zmq.green as zmq from bottle import Bottle, request, response from gevent import monkey from pkg_resources import resource_string +from zmq import PUB as ZMQ_PUB from eddn.conf.Settings import Settings, load_config from eddn.core.Validator import ValidationSeverity, Validator @@ -47,7 +48,7 @@ logger.info('Made logger') # This socket is used to push market data out to the Announcers over ZeroMQ. zmq_context = zmq.Context() -sender = zmq_context.socket(zmq.PUB) +sender = zmq_context.socket(ZMQ_PUB) validator = Validator() From 060645a8dca52f451f50cc9705d0d10d49cf9211 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:15:15 +0000 Subject: [PATCH 119/234] Settings: Rename loadFrom -> load_from --- src/eddn/conf/Settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 3ce0d05..74fba53 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -156,9 +156,9 @@ def load_config(cl_args) -> None: Load in a commandline-specified settings file, if applicable. A convenience method if you don't need other things specified as commandline - options. Otherwise, point the filename to Settings.loadFrom(). + options. Otherwise, point the filename to Settings.load_from(). :param cl_args: An `argparse.parse_args()` return. """ if cl_args.config: - Settings.loadFrom(cl_args.config) + Settings.load_from(cl_args.config) From c426885b52d348fbe991fb564b46617cbde35dab Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:58:48 +0000 Subject: [PATCH 120/234] Monitor: Tweak for zmq changes --- src/eddn/Monitor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 36ec2ad..85432fa 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -13,6 +13,8 @@ import simplejson import zmq.green as zmq from bottle import Bottle, request, response from gevent import monkey +from zmq import SUB as ZMQ_SUB +from zmq import SUBSCRIBE as ZMQ_SUBSCRIBE from eddn.conf.Settings import Settings, load_config @@ -197,8 +199,8 @@ class Monitor(Thread): """Handle receiving Gateway messages and recording stats.""" context = zmq.Context() - receiver = context.socket(zmq.SUB) - receiver.setsockopt_string(zmq.SUBSCRIBE, '') + receiver = context.socket(ZMQ_SUB) + receiver.setsockopt_string(ZMQ_SUBSCRIBE, '') for binding in Settings.MONITOR_RECEIVER_BINDINGS: receiver.connect(binding) From 95b48ed3ccec3202aaea212748755a195a5a16e0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 11:59:04 +0000 Subject: [PATCH 121/234] Relay: Tweak for zmq changes --- src/eddn/Relay.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 131b497..749c914 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -27,6 +27,10 @@ import simplejson import zmq.green as zmq from bottle import Bottle, response from gevent import monkey +from zmq import PUB as ZMQ_PUB +from zmq import SUB as ZMQ_SUB +from zmq import SNDHWM as ZMQ_SNDHWM +from zmq import SUBSCRIBE as ZMQ_SUBSCRIBE from eddn.conf.Settings import Settings, load_config @@ -113,15 +117,15 @@ class Relay(Thread): # These form the connection to the Gateway daemon(s) upstream. context = zmq.Context() - receiver = context.socket(zmq.SUB) - receiver.setsockopt_string(zmq.SUBSCRIBE, '') + receiver = context.socket(ZMQ_SUB) + receiver.setsockopt_string(ZMQ_SUBSCRIBE, '') for binding in Settings.RELAY_RECEIVER_BINDINGS: # Relays bind upstream to an Announcer, or another Relay. receiver.connect(binding) - sender = context.socket(zmq.PUB) - sender.setsockopt(zmq.SNDHWM, 500) + sender = context.socket(ZMQ_PUB) + sender.setsockopt(ZMQ_SNDHWM, 500) for binding in Settings.RELAY_SENDER_BINDINGS: # End users, or other relays, may attach here. From d43b85293504eac20290dbe63cdb330c2d7e5a84 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:10:03 +0000 Subject: [PATCH 122/234] setup.py: flake8 pass --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index a7aa824..c4a5414 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ import re import shutil import subprocess import sys -from setuptools import setup, find_packages from setuptools import find_packages, setup @@ -51,7 +50,7 @@ if setup_env.EDDN_ENV == 'live': out, err = git_cmd.communicate() except Exception as e: - print("Couldn't run git command to check branch: %s" % (e)) + print(f"Couldn't run git command to check branch: {e}") else: branch = out.decode().rstrip('\n') @@ -60,7 +59,7 @@ if setup_env.EDDN_ENV == 'live': # - For any 'detached HEAD' (i.e. specific commit ID, or tag) it # will be empty. if branch != 'live': - print("EDDN_ENV is '%s' (and CWD is %s), but branch is '%s', aborting!" % (setup_env.EDDN_ENV, cwd, branch)) + print(f"EDDN_ENV is '{setup_env.EDDN_ENV}' (and CWD is '{cwd}'), but branch is '{branch}', aborting!") sys.exit(-1) ########################################################################### From a8d0806446add9450ad0091a208f76116fdb0209 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:12:49 +0000 Subject: [PATCH 123/234] Bouncer: flake8 pass --- src/eddn/Bouncer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index 730f14e..d9e4bee 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -61,7 +61,12 @@ logger.info('Made logger') # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector # noqa: E402 +stats_collector = StatsCollector() +stats_collector.start() + + def parse_cl_args(): + """Parse command-line arguments.""" parser = argparse.ArgumentParser( prog='Gateway', description='EDDN Gateway server', @@ -81,9 +86,6 @@ def parse_cl_args(): return parser.parse_args() -stats_collector = StatsCollector() -stats_collector.start() - def push_message(message_body: str) -> None: """ @@ -311,6 +313,7 @@ class CustomLogging(object): return _log_to_logger + def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() From 241bd911dd27ca344035e45f05f9c2dbb6db885b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:19:14 +0100 Subject: [PATCH 124/234] Gateway: flake8 and mypy pass # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 92 ++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 2c0b4ee..a193fc7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -8,7 +8,6 @@ Contains the necessary ZeroMQ socket and a helper function to publish market data to the Announcer daemons. """ import argparse -import hashlib import logging import zlib from datetime import datetime @@ -18,7 +17,6 @@ import gevent import simplejson import urlparse import zmq.green as zmq -from bottle import Bottle, request, response from gevent import monkey from pkg_resources import resource_string from zmq import PUB as ZMQ_PUB @@ -27,9 +25,10 @@ from eddn.conf.Settings import Settings, load_config from eddn.core.Validator import ValidationSeverity, Validator monkey.patch_all() -import bottle -from bottle import Bottle, request, response -bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB +import bottle # noqa: E402 +from bottle import Bottle, request, response # noqa: E402 + +bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB app = Bottle() @@ -60,6 +59,7 @@ stats_collector.start() def parse_cl_args(): + """Parse command-line arguments.""" parser = argparse.ArgumentParser( prog='Gateway', description='EDDN Gateway server', @@ -80,7 +80,13 @@ def parse_cl_args(): return parser.parse_args() -def extract_message_details(parsed_message): +def extract_message_details(parsed_message): # noqa: CCR001 + """ + Extract the details of an EDDN message. + + :param parsed_message: The message to process + :return: Tuple of (uploader_id, software_name, software_version, schema_ref, journal_event) + """ uploader_id = '<>' software_name = '<>' software_version = '<>' @@ -100,7 +106,6 @@ def extract_message_details(parsed_message): if '$schemaRef' in parsed_message: schema_ref = parsed_message['$schemaRef'] - if '/journal/' in schema_ref: if 'message' in parsed_message: if 'event' in parsed_message['message']: @@ -111,6 +116,7 @@ def extract_message_details(parsed_message): return uploader_id, software_name, software_version, schema_ref, journal_event + def configure() -> None: """ Get the list of transports to bind from settings. @@ -178,7 +184,7 @@ def get_decompressed_message() -> bytes: logger.error('zlib.error, trying zlib.decompress (-15)') # Negative wbits suppresses adler32 checksumming. message_body = zlib.decompress(request.body.read(), -15) - logger.debug('Resulting message_body:\n%s\n' % (message_body)) + logger.debug('Resulting message_body:\n%s\n', message_body) else: logger.debug('Content-Encoding indicates *not* compressed...') @@ -202,24 +208,25 @@ def parse_and_error_handle(data: bytes) -> str: # Something bad happened. We know this will return at least a # semi-useful error message, so do so. try: - logger.error('Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n' % ( - request.content_length, - '<>', - '<>', - '<>', - '<>', - '<>', - get_remote_address(), - data[:512] - )) + logger.error( + 'Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n', + request.content_length, + '<>', + '<>', + '<>', + '<>', + '<>', + get_remote_address(), + data[:512] + ) except Exception as e: - print('Logging of "JSON parse failed" failed: %s' % (e.message)) + # TODO: Maybe just `{e}` ? + print(f"Logging of 'JSON parse failed' failed: {str(e)}") pass response.status = 400 logger.error(f"Error to {get_remote_address()}: {exc}") - return str(exc) return 'FAIL: JSON parsing: ' + str(exc) # Here we check if an outdated schema has been passed @@ -239,36 +246,40 @@ def parse_and_error_handle(data: bytes) -> str: gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) try: - uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) - logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( + uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501 + logger.info( + 'Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s', request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, get_remote_address() - )) + ) except Exception as e: - print('Logging of Accepted request failed: %s' % (e.message)) + # TODO: Maybe just `{e}` ? + print(f"Logging of Accepted request failed: {str(e)}") pass return 'OK' else: try: - uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) - logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s' % ( - str(validationResults.messages), - request.content_length, - uploader_id, software_name, software_version, schema_ref, journal_event, - get_remote_address() - )) + uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501 + logger.error( + 'Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s', + str(validation_results.messages), + request.content_length, + uploader_id, software_name, software_version, schema_ref, journal_event, + get_remote_address() + ) except Exception as e: - print('Logging of Failed Validation failed: %s' % (e.message)) + # TODO: Maybe just `{e}` ? + print(f"Logging of Failed Validation failed: {str(e)}") pass response.status = 400 stats_collector.tally("invalid") - return "FAIL: Schema Validation: " + str(validationResults.messages) + return "FAIL: Schema Validation: " + str(validation_results.messages) @app.route('/upload/', method=['OPTIONS', 'POST']) @@ -288,20 +299,25 @@ def upload() -> str: # the correct direction. response.status = 400 try: - logger.error(f'gzip error ({request.content_length}, "<>", "<>", "<>", "<>", "<>") from {get_remote_address()}') + logger.error( + f'gzip error ({request.content_length}, "<>", "<>", "<>",' + ' "<>", "<>") from {get_remote_address()}' + ) except Exception as e: - print('Logging of "gzip error" failed: %s' % (e.message)) + # TODO: Maybe just `{e}` ? + print(f"Logging of 'gzip error' failed: {str(e)}") pass - return 'FAIL: zlib.error: ' + exc.message + return 'FAIL: zlib.error: ' + str(exc) except MalformedUploadError as exc: # They probably sent an encoded POST, but got the key/val wrong. response.status = 400 - logger.error(f"MalformedUploadError from {get_remote_address()}: {exc.message}") + # TODO: Maybe just `{exc}` ? + logger.error("MalformedUploadError from %s: %s", get_remote_address(), str(exc)) - return 'FAIL: Malformed Upload: ' + exc.message + return 'FAIL: Malformed Upload: ' + str(exc) stats_collector.tally("inbound") return parse_and_error_handle(message_body) From f8dc28b25670a2d2a4b745819cca9a5ca27c92fd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:28:03 +0000 Subject: [PATCH 125/234] src/eddn/__init__.py: docstring --- src/eddn/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eddn/__init__.py b/src/eddn/__init__.py index e69de29..5656723 100644 --- a/src/eddn/__init__.py +++ b/src/eddn/__init__.py @@ -0,0 +1 @@ +"""Main EDDN module.""" From 35fcbd89d9522aff0c8ee887a111020f9bfc7489 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:30:13 +0000 Subject: [PATCH 126/234] Monitor: flake8 and mypy pass --- src/eddn/Monitor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 85432fa..c5a9f2c 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -1,3 +1,4 @@ +"""EDDN Monitor, which records stats about incoming messages.""" # coding: utf8 import argparse @@ -30,6 +31,7 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: def parse_cl_args(): + """Parse command-line arguments.""" parser = argparse.ArgumentParser( prog='Gateway', description='EDDN Gateway server', @@ -49,12 +51,13 @@ def parse_cl_args(): return parser.parse_args() -def date(__format) -> datetime.datetime: + +def date(__format) -> str: """ Make a 'now' datetime as per the supplied format. :param __format: - :return: + :return: String representation of 'now' """ d = datetime.datetime.utcnow() return d.strftime(__format) @@ -289,6 +292,7 @@ def apply_cors() -> None: 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' ) + def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() From 6e7f69ab95f6c2be66fba70bbfaa1da4f4d4a8dc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:37:35 +0000 Subject: [PATCH 127/234] Relay: flake8 and mypy pass --- src/eddn/Relay.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 749c914..f5a4788 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -9,6 +9,16 @@ import uuid import zlib from threading import Thread +import gevent +import simplejson +import zmq.green as zmq +from bottle import Bottle, response +from gevent import monkey +from zmq import PUB as ZMQ_PUB +from zmq import SNDHWM as ZMQ_SNDHWM +from zmq import SUB as ZMQ_SUB +from zmq import SUBSCRIBE as ZMQ_SUBSCRIBE + # Logging has to be configured first before we do anything. logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -22,17 +32,7 @@ __logger_channel.setFormatter(__logger_formatter) logger.addHandler(__logger_channel) logger.info('Made logger') -import gevent -import simplejson -import zmq.green as zmq -from bottle import Bottle, response -from gevent import monkey -from zmq import PUB as ZMQ_PUB -from zmq import SUB as ZMQ_SUB -from zmq import SNDHWM as ZMQ_SNDHWM -from zmq import SUBSCRIBE as ZMQ_SUBSCRIBE - -from eddn.conf.Settings import Settings, load_config +from eddn.conf.Settings import Settings, load_config # noqa: E402 monkey.patch_all() @@ -51,7 +51,8 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicate_messages.start() -def parse_cl_args(): +def parse_cl_args() -> argparse.Namespace: + """Parse command-line arguments.""" parser = argparse.ArgumentParser( prog='Gateway', description='EDDN Gateway server', @@ -71,6 +72,7 @@ def parse_cl_args(): return parser.parse_args() + @app.route('/stats/', method=['OPTIONS', 'GET']) def stats() -> str: """ From 4e6a6fc86979ef829ec7ca520cb3109248e49d89 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 14:52:10 +0100 Subject: [PATCH 128/234] conf/Settings: flake8 and mypy pass --- src/eddn/conf/Settings.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 74fba53..09404f2 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -1,7 +1,6 @@ # coding: utf8 """EDDN default Settings.""" -import argparse from typing import Dict import simplejson @@ -76,19 +75,19 @@ class _Settings(object): "https://eddn.edcd.io/schemas/navbeaconscan/1": "schemas/navbeaconscan-v1.0.json", "https://eddn.edcd.io/schemas/navbeaconscan/1/test": "schemas/navbeaconscan-v1.0.json", - "https://eddn.edcd.io/schemas/navroute/1" : "schemas/navroute-v1.0.json", - "https://eddn.edcd.io/schemas/navroute/1/test" : "schemas/navroute-v1.0.json", + "https://eddn.edcd.io/schemas/navroute/1": "schemas/navroute-v1.0.json", + "https://eddn.edcd.io/schemas/navroute/1/test": "schemas/navroute-v1.0.json", - "https://eddn.edcd.io/schemas/approachsettlement/1" : "schemas/approachsettlement-v1.0.json", - "https://eddn.edcd.io/schemas/approachsettlement/1/test" : "schemas/approachsettlement-v1.0.json", - "https://eddn.edcd.io/schemas/fssallbodiesfound/1" : "schemas/fssallbodiesfound-v1.0.json", - "https://eddn.edcd.io/schemas/fssallbodiesfound/1/test" : "schemas/fssallbodiesfound-v1.0.json", + "https://eddn.edcd.io/schemas/approachsettlement/1": "schemas/approachsettlement-v1.0.json", + "https://eddn.edcd.io/schemas/approachsettlement/1/test": "schemas/approachsettlement-v1.0.json", + "https://eddn.edcd.io/schemas/fssallbodiesfound/1": "schemas/fssallbodiesfound-v1.0.json", + "https://eddn.edcd.io/schemas/fssallbodiesfound/1/test": "schemas/fssallbodiesfound-v1.0.json", - "https://eddn.edcd.io/schemas/fssbodysignals/1" : "schemas/fssbodysignals-v1.0.json", - "https://eddn.edcd.io/schemas/fssbodysignals/1/test" : "schemas/fssbodysignals-v1.0.json", + "https://eddn.edcd.io/schemas/fssbodysignals/1": "schemas/fssbodysignals-v1.0.json", + "https://eddn.edcd.io/schemas/fssbodysignals/1/test": "schemas/fssbodysignals-v1.0.json", - "https://eddn.edcd.io/schemas/fsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", - "https://eddn.edcd.io/schemas/fsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemas/fsssignaldiscovered/1": "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemas/fsssignaldiscovered/1/test": "schemas/fsssignaldiscovered-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ # noqa: E221 From d540078b664fd2e9b495814b58a62bae5d3d8684 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:39:44 +0000 Subject: [PATCH 129/234] conf/Version: Bump to 3.0 --- src/eddn/conf/Version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/conf/Version.py b/src/eddn/conf/Version.py index e042b67..0363d4a 100644 --- a/src/eddn/conf/Version.py +++ b/src/eddn/conf/Version.py @@ -1,3 +1,3 @@ """Declare the version of this software.""" # This should be a version number as understood by setuptools -__version__ = "2.0-alpha0" +__version__ = "3.0" From 47a428c5d1dd018315676334931b6b9a5095855b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:46:04 +0000 Subject: [PATCH 130/234] Bouncer: Use "" for strings --- src/eddn/Bouncer.py | 88 ++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index d9e4bee..ef59b28 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -49,13 +49,13 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() __logger_formatter = logging.Formatter( - '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' + "%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s" ) -__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' -__logger_formatter.default_msec_format = '%s.%03d' +__logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" +__logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) logger.addHandler(__logger_channel) -logger.info('Made logger') +logger.info("Made logger") # This import must be done post-monkey-patching! @@ -68,19 +68,19 @@ stats_collector.start() def parse_cl_args(): """Parse command-line arguments.""" parser = argparse.ArgumentParser( - prog='Gateway', - description='EDDN Gateway server', + prog="Gateway", + description="EDDN Gateway server", ) parser.add_argument( - '--loglevel', - help='Logging level to output at', + "--loglevel", + help="Logging level to output at", ) parser.add_argument( - '-c', '--config', - metavar='config filename', - nargs='?', + "-c", "--config", + metavar="config filename", + nargs="?", default=None, ) @@ -102,11 +102,11 @@ def push_message(message_body: str) -> None: ) except Exception as e: - logger.error('Failed sending message on', exc_info=e) + logger.error("Failed sending message on", exc_info=e) else: if r.status_code != requests.codes.ok: - logger.error(f'Response from {Settings.BOUNCER_LIVE_GATEWAY_URL}:\n{r.text}\n') + logger.error(f"Response from {Settings.BOUNCER_LIVE_GATEWAY_URL}:\n{r.text}\n") else: stats_collector.tally("outbound") @@ -120,7 +120,7 @@ def get_remote_address() -> str: request.remote_addr. :returns: Best attempt at remote address. """ - return request.headers.get('X-Forwarded-For', request.remote_addr) + return request.headers.get("X-Forwarded-For", request.remote_addr) def get_decompressed_message() -> bytes: @@ -130,9 +130,9 @@ def get_decompressed_message() -> bytes: For upload formats that support it. :returns: The de-compressed request body. """ - content_encoding = request.headers.get('Content-Encoding', '') + content_encoding = request.headers.get("Content-Encoding", "") - if content_encoding in ['gzip', 'deflate']: + if content_encoding in ["gzip", "deflate"]: # Compressed request. We have to decompress the body, then figure out # if it's form-encoded. try: @@ -150,7 +150,7 @@ def get_decompressed_message() -> bytes: # This is a form-encoded POST. The value of the data attrib will # be the body we're looking for. try: - message_body = form_enc_parsed['data'][0] + message_body = form_enc_parsed["data"][0] except (KeyError, IndexError): raise MalformedUploadError( "No 'data' POST key/value found. Check your POST key " @@ -159,7 +159,7 @@ def get_decompressed_message() -> bytes: else: # Uncompressed request. Bottle handles all of the parsing of the # POST key/vals, or un-encoded body. - data_key = request.forms.get('data') + data_key = request.forms.get("data") if data_key: # This is a form-encoded POST. Support the silly people. message_body = data_key @@ -175,16 +175,16 @@ def forward_message(message_body: bytes) -> str: Send the parsed message to the Relay/Monitor as compressed JSON. :param message_body: Incoming message. - :returns: 'OK' assuming it is. + :returns: "OK" assuming it is. """ # TODO: This instead needs to send the message to remote Gateway gevent.spawn(push_message, message_body) - logger.info(f'Accepted upload from {get_remote_address()}') + logger.info(f"Accepted upload from {get_remote_address()}") - return 'OK' + return "OK" -@app.route('/upload/', method=['OPTIONS', 'POST']) +@app.route("/upload/", method=["OPTIONS", "POST"]) def upload() -> str: """ Handle an /upload/ request. @@ -200,25 +200,25 @@ def upload() -> str: # at least some kind of feedback for them to try to get pointed in # the correct direction. response.status = 400 - logger.error(f'gzip error with {get_remote_address()}: {exc}') + logger.error(f"gzip error with {get_remote_address()}: {exc}") - return f'{exc}' + return f"{exc}" except MalformedUploadError as exc: # They probably sent an encoded POST, but got the key/val wrong. response.status = 400 - logger.error(f'Error to {get_remote_address()}: {exc}') + logger.error(f"Error to {get_remote_address()}: {exc}") - return f'{exc}' + return f"{exc}" stats_collector.tally("inbound") return forward_message(message_body) -@app.route('/health_check/', method=['OPTIONS', 'GET']) +@app.route("/health_check/", method=["OPTIONS", "GET"]) def health_check() -> str: """ - Return our version string in as an 'am I awake' signal. + Return our version string in as an "am I awake" signal. This should only be used by the gateway monitoring script. It is used to detect whether the gateway is still alive, and whether it should remain @@ -229,7 +229,7 @@ def health_check() -> str: return Settings.EDDN_VERSION -@app.route('/stats/', method=['OPTIONS', 'GET']) +@app.route("/stats/", method=["OPTIONS", "GET"]) def stats() -> str: """ Return some stats about the Gateway's operation so far. @@ -257,7 +257,7 @@ class MalformedUploadError(Exception): class EnableCors(object): """Handle enabling CORS headers in all responses.""" - name = 'enable_cors' + name = "enable_cors" api = 2 def apply(self, fn: Callable): @@ -270,12 +270,12 @@ class EnableCors(object): """ def _enable_cors(*args, **kwargs): # set CORS headers - response.headers['Access-Control-Allow-Origin'] = '*' - response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' - response.headers['Access-Control-Allow-Headers'] = \ - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + response.headers["Access-Control-Allow-Origin"] = "*" + response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, OPTIONS" + response.headers["Access-Control-Allow-Headers"] = \ + "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" - if request.method != 'OPTIONS': + if request.method != "OPTIONS": # actual request; reply with the actual response return fn(*args, **kwargs) @@ -285,7 +285,7 @@ class EnableCors(object): class CustomLogging(object): """Wrap a Bottle request so that a log line is emitted after it's handled.""" - name = 'custom_logging' + name = "custom_logging" api = 2 def apply(self, fn: Callable): @@ -300,14 +300,14 @@ class CustomLogging(object): request_time = datetime.utcnow() actual_response = fn(*args, **kwargs) - # logger.info('Request:\n%s\n' % (request ) ) + # logger.info("Request:\n%s\n", request) if len(request.remote_route) > 1: remote_addr = request.remote_route[1] else: remote_addr = request.remote_addr - logger.info(f'{remote_addr} {request_time} {request.method} {request.url} {response.status}') + logger.info(f"{remote_addr} {request_time} {request.method} {request.url} {response.status}") return actual_response @@ -320,23 +320,23 @@ def main() -> None: if cl_args.loglevel: logger.setLevel(cl_args.loglevel) - logger.info('Loading config...') + logger.info("Loading config...") load_config(cl_args) - logger.info('Installing EnableCors ...') + logger.info("Installing EnableCors ...") app.install(EnableCors()) - logger.info('Installing CustomLogging ...') + logger.info("Installing CustomLogging ...") app.install(CustomLogging()) - logger.info('Running bottle app ...') + logger.info("Running bottle app ...") app.run( host=Settings.BOUNCER_HTTP_BIND_ADDRESS, port=Settings.BOUNCER_HTTP_PORT, - server='gevent', + server="gevent", certfile=Settings.CERT_FILE, keyfile=Settings.KEY_FILE, quiet=True, ) -if __name__ == '__main__': +if __name__ == "__main__": main() From 710223494d38beaa09a41ef7a4a6d683ba2b6b88 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 12:46:16 +0000 Subject: [PATCH 131/234] src/eddn/core/__init__.py: docstring --- src/eddn/core/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eddn/core/__init__.py b/src/eddn/core/__init__.py index e69de29..7921874 100644 --- a/src/eddn/core/__init__.py +++ b/src/eddn/core/__init__.py @@ -0,0 +1 @@ +"""EDDN core library code.""" From 841c7e979e110bee4ee878fe7dbe1827322f64bf Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:19:33 +0100 Subject: [PATCH 132/234] Gateway: Use "" for strings throughout # Conflicts: # src/eddn/Gateway.py # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 144 ++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index a193fc7..d556900 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -36,13 +36,13 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() __logger_formatter = logging.Formatter( - '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' + "%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s" ) -__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' -__logger_formatter.default_msec_format = '%s.%03d' +__logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" +__logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) logger.addHandler(__logger_channel) -logger.info('Made logger') +logger.info("Made logger") # This socket is used to push market data out to the Announcers over ZeroMQ. @@ -61,19 +61,19 @@ stats_collector.start() def parse_cl_args(): """Parse command-line arguments.""" parser = argparse.ArgumentParser( - prog='Gateway', - description='EDDN Gateway server', + prog="Gateway", + description="EDDN Gateway server", ) parser.add_argument( - '--loglevel', - help='Logging level to output at', + "--loglevel", + help="Logging level to output at", ) parser.add_argument( - '-c', '--config', - metavar='config filename', - nargs='?', + "-c", "--config", + metavar="config filename", + nargs="?", default=None, ) @@ -87,32 +87,32 @@ def extract_message_details(parsed_message): # noqa: CCR001 :param parsed_message: The message to process :return: Tuple of (uploader_id, software_name, software_version, schema_ref, journal_event) """ - uploader_id = '<>' - software_name = '<>' - software_version = '<>' - schema_ref = '<>' - journal_event = '<>' + uploader_id = "<>" + software_name = "<>" + software_version = "<>" + schema_ref = "<>" + journal_event = "<>" - if 'header' in parsed_message: - if 'uploaderID' in parsed_message['header']: - uploader_id = parsed_message['header']['uploaderID'] + if "header" in parsed_message: + if "uploaderID" in parsed_message["header"]: + uploader_id = parsed_message["header"]["uploaderID"] - if 'softwareName' in parsed_message['header']: - software_name = parsed_message['header']['softwareName'] + if "softwareName" in parsed_message["header"]: + software_name = parsed_message["header"]["softwareName"] - if 'softwareVersion' in parsed_message['header']: - software_version = parsed_message['header']['softwareVersion'] + if "softwareVersion" in parsed_message["header"]: + software_version = parsed_message["header"]["softwareVersion"] - if '$schemaRef' in parsed_message: - schema_ref = parsed_message['$schemaRef'] + if "$schemaRef" in parsed_message: + schema_ref = parsed_message["$schemaRef"] - if '/journal/' in schema_ref: - if 'message' in parsed_message: - if 'event' in parsed_message['message']: - journal_event = parsed_message['message']['event'] + if "/journal/" in schema_ref: + if "message" in parsed_message: + if "event" in parsed_message["message"]: + journal_event = parsed_message["message"]["event"] else: - journal_event = '-' + journal_event = "-" return uploader_id, software_name, software_version, schema_ref, journal_event @@ -128,7 +128,7 @@ def configure() -> None: sender.bind(binding) for schema_ref, schema_file in Settings.GATEWAY_JSON_SCHEMAS.items(): - validator.add_schema_resource(schema_ref, resource_string('eddn.Gateway', schema_file)) + validator.add_schema_resource(schema_ref, resource_string("eddn.Gateway", schema_file)) def push_message(parsed_message: Dict, topic: str) -> None: @@ -139,7 +139,7 @@ def push_message(parsed_message: Dict, topic: str) -> None: This is a dumb method that just pushes strings; it assumes you've already validated and serialised as you want to. """ - string_message = simplejson.dumps(parsed_message, ensure_ascii=False).encode('utf-8') + string_message = simplejson.dumps(parsed_message, ensure_ascii=False).encode("utf-8") # Push a zlib compressed JSON representation of the message to # announcers with schema as topic @@ -157,7 +157,7 @@ def get_remote_address() -> str: request.remote_addr. :returns: Best attempt at remote address. """ - return request.headers.get('X-Forwarded-For', request.remote_addr) + return request.headers.get("X-Forwarded-For", request.remote_addr) def get_decompressed_message() -> bytes: @@ -168,26 +168,26 @@ def get_decompressed_message() -> bytes: :rtype: str :returns: The de-compressed request body. """ - content_encoding = request.headers.get('Content-Encoding', '') - logger.debug('Content-Encoding: ' + content_encoding) + content_encoding = request.headers.get("Content-Encoding", "") + logger.debug("Content-Encoding: %s", content_encoding) - if content_encoding in ['gzip', 'deflate']: - logger.debug('Content-Encoding of gzip or deflate...') + if content_encoding in ["gzip", "deflate"]: + logger.debug("Content-Encoding of gzip or deflate...") # Compressed request. We have to decompress the body, then figure out # if it's form-encoded. try: # Auto header checking. - logger.debug('Trying zlib.decompress (15 + 32)...') + logger.debug("Trying zlib.decompress (15 + 32)...") message_body = zlib.decompress(request.body.read(), 15 + 32) except zlib.error: - logger.error('zlib.error, trying zlib.decompress (-15)') + logger.error("zlib.error, trying zlib.decompress (-15)") # Negative wbits suppresses adler32 checksumming. message_body = zlib.decompress(request.body.read(), -15) - logger.debug('Resulting message_body:\n%s\n', message_body) + logger.debug("Resulting message_body:\n%s\n", message_body) else: - logger.debug('Content-Encoding indicates *not* compressed...') + logger.debug("Content-Encoding indicates *not* compressed...") message_body = request.body.read() @@ -209,13 +209,13 @@ def parse_and_error_handle(data: bytes) -> str: # semi-useful error message, so do so. try: logger.error( - 'Error - JSON parse failed (%d, "%s", "%s", "%s", "%s", "%s") from %s:\n%s\n', + "Error - JSON parse failed (%d, '%s', '%s', '%s', '%s', '%s') from %s:\n%s\n", request.content_length, - '<>', - '<>', - '<>', - '<>', - '<>', + "<>", + "<>", + "<>", + "<>", + "<>", get_remote_address(), data[:512] ) @@ -227,11 +227,11 @@ def parse_and_error_handle(data: bytes) -> str: response.status = 400 logger.error(f"Error to {get_remote_address()}: {exc}") - return 'FAIL: JSON parsing: ' + str(exc) + return "FAIL: JSON parsing: " + str(exc) # Here we check if an outdated schema has been passed if parsed_message["$schemaRef"] in Settings.GATEWAY_OUTDATED_SCHEMAS: - response.status = '426 Upgrade Required' # Bottle (and underlying httplib) don't know this one + response.status = "426 Upgrade Required" # Bottle (and underlying httplib) don't know this one stats_collector.tally("outdated") return "FAIL: Outdated Schema: The schema you have used is no longer supported. Please check for an updated " \ "version of your application." @@ -239,16 +239,16 @@ def parse_and_error_handle(data: bytes) -> str: validation_results = validator.validate(parsed_message) if validation_results.severity <= ValidationSeverity.WARN: - parsed_message['header']['gatewayTimestamp'] = datetime.utcnow().isoformat() + 'Z' - parsed_message['header']['uploaderIP'] = get_remote_address() + parsed_message["header"]["gatewayTimestamp"] = datetime.utcnow().isoformat() + "Z" + parsed_message["header"]["uploaderIP"] = get_remote_address() # Sends the parsed message to the Relay/Monitor as compressed JSON. - gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef']) + gevent.spawn(push_message, parsed_message, parsed_message["$schemaRef"]) try: uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501 logger.info( - 'Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s', + "Accepted (%d, '%s', '%s', '%s', '%s', '%s') from %s", request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, get_remote_address() @@ -259,13 +259,13 @@ def parse_and_error_handle(data: bytes) -> str: print(f"Logging of Accepted request failed: {str(e)}") pass - return 'OK' + return "OK" else: try: uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501 logger.error( - 'Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s', + "Failed Validation '%s' (%d, '%s', '%s', '%s', '%s', '%s') from %s", str(validation_results.messages), request.content_length, uploader_id, software_name, software_version, schema_ref, journal_event, @@ -282,7 +282,7 @@ def parse_and_error_handle(data: bytes) -> str: return "FAIL: Schema Validation: " + str(validation_results.messages) -@app.route('/upload/', method=['OPTIONS', 'POST']) +@app.route("/upload/", method=["OPTIONS", "POST"]) def upload() -> str: """ Handle an /upload/ request. @@ -300,8 +300,8 @@ def upload() -> str: response.status = 400 try: logger.error( - f'gzip error ({request.content_length}, "<>", "<>", "<>",' - ' "<>", "<>") from {get_remote_address()}' + f"gzip error ({request.content_length}, "<>", "<>", "<>"," + " "<>", "<>") from {get_remote_address()}" ) except Exception as e: @@ -309,7 +309,7 @@ def upload() -> str: print(f"Logging of 'gzip error' failed: {str(e)}") pass - return 'FAIL: zlib.error: ' + str(exc) + return "FAIL: zlib.error: " + str(exc) except MalformedUploadError as exc: # They probably sent an encoded POST, but got the key/val wrong. @@ -317,16 +317,16 @@ def upload() -> str: # TODO: Maybe just `{exc}` ? logger.error("MalformedUploadError from %s: %s", get_remote_address(), str(exc)) - return 'FAIL: Malformed Upload: ' + str(exc) + return "FAIL: Malformed Upload: " + str(exc) stats_collector.tally("inbound") return parse_and_error_handle(message_body) -@app.route('/health_check/', method=['OPTIONS', 'GET']) +@app.route("/health_check/", method=["OPTIONS", "GET"]) def health_check() -> str: """ - Return our version string in as an 'am I awake' signal. + Return our version string in as an "am I awake" signal. This should only be used by the gateway monitoring script. It is used to detect whether the gateway is still alive, and whether it should remain @@ -337,7 +337,7 @@ def health_check() -> str: return Settings.EDDN_VERSION -@app.route('/stats/', method=['OPTIONS', 'GET']) +@app.route("/stats/", method=["OPTIONS", "GET"]) def stats() -> str: """ Return some stats about the Gateway's operation so far. @@ -370,16 +370,16 @@ def apply_cors() -> None: :return: """ response.set_header( - 'Access-Control-Allow-Origin', - '*' + "Access-Control-Allow-Origin", + "*" ) response.set_header( - 'Access-Control-Allow-Methods', - 'GET, POST, PUT, OPTIONS' + "Access-Control-Allow-Methods", + "GET, POST, PUT, OPTIONS" ) response.set_header( - 'Access-Control-Allow-Headers', - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + "Access-Control-Allow-Headers", + "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" ) @@ -392,15 +392,15 @@ def main() -> None: load_config(cl_args) configure() - app.add_hook('after_request', apply_cors) + app.add_hook("after_request", apply_cors) app.run( host=Settings.GATEWAY_HTTP_BIND_ADDRESS, port=Settings.GATEWAY_HTTP_PORT, - server='gevent', + server="gevent", certfile=Settings.CERT_FILE, keyfile=Settings.KEY_FILE ) -if __name__ == '__main__': +if __name__ == "__main__": main() From 5d3358417d6bdeb5a03fa7e174b6ba21b93394e9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:19:50 +0100 Subject: [PATCH 133/234] Gateway: Format with black # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 58 +++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d556900..4c34c21 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -1,4 +1,3 @@ - # coding: utf8 """ @@ -35,9 +34,7 @@ app = Bottle() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() -__logger_formatter = logging.Formatter( - "%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s" -) +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") __logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" __logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) @@ -71,7 +68,8 @@ def parse_cl_args(): ) parser.add_argument( - "-c", "--config", + "-c", + "--config", metavar="config filename", nargs="?", default=None, @@ -217,7 +215,7 @@ def parse_and_error_handle(data: bytes) -> str: "<>", "<>", get_remote_address(), - data[:512] + data[:512], ) except Exception as e: @@ -233,8 +231,10 @@ def parse_and_error_handle(data: bytes) -> str: if parsed_message["$schemaRef"] in Settings.GATEWAY_OUTDATED_SCHEMAS: response.status = "426 Upgrade Required" # Bottle (and underlying httplib) don't know this one stats_collector.tally("outdated") - return "FAIL: Outdated Schema: The schema you have used is no longer supported. Please check for an updated " \ - "version of your application." + return ( + "FAIL: Outdated Schema: The schema you have used is no longer supported. Please check for an updated " + "version of your application." + ) validation_results = validator.validate(parsed_message) @@ -246,12 +246,18 @@ def parse_and_error_handle(data: bytes) -> str: gevent.spawn(push_message, parsed_message, parsed_message["$schemaRef"]) try: - uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501 + (uploader_id, software_name, software_version, schema_ref, journal_event,) = extract_message_details( + parsed_message + ) # noqa: E501 logger.info( "Accepted (%d, '%s', '%s', '%s', '%s', '%s') from %s", request.content_length, - uploader_id, software_name, software_version, schema_ref, journal_event, - get_remote_address() + uploader_id, + software_name, + software_version, + schema_ref, + journal_event, + get_remote_address(), ) except Exception as e: @@ -263,13 +269,19 @@ def parse_and_error_handle(data: bytes) -> str: else: try: - uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message) # noqa: E501 + (uploader_id, software_name, software_version, schema_ref, journal_event,) = extract_message_details( + parsed_message + ) # noqa: E501 logger.error( "Failed Validation '%s' (%d, '%s', '%s', '%s', '%s', '%s') from %s", str(validation_results.messages), request.content_length, - uploader_id, software_name, software_version, schema_ref, journal_event, - get_remote_address() + uploader_id, + software_name, + software_version, + schema_ref, + journal_event, + get_remote_address(), ) except Exception as e: @@ -300,8 +312,8 @@ def upload() -> str: response.status = 400 try: logger.error( - f"gzip error ({request.content_length}, "<>", "<>", "<>"," - " "<>", "<>") from {get_remote_address()}" + f"gzip error ({request.content_length}, '<>', '<>', '<>'" + ", '<>', '<>') from {get_remote_address()}" ) except Exception as e: @@ -369,17 +381,11 @@ def apply_cors() -> None: :param context: :return: """ - response.set_header( - "Access-Control-Allow-Origin", - "*" - ) - response.set_header( - "Access-Control-Allow-Methods", - "GET, POST, PUT, OPTIONS" - ) + response.set_header("Access-Control-Allow-Origin", "*") + response.set_header("Access-Control-Allow-Methods", "GET, POST, PUT, OPTIONS") response.set_header( "Access-Control-Allow-Headers", - "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" + "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token", ) @@ -398,7 +404,7 @@ def main() -> None: port=Settings.GATEWAY_HTTP_PORT, server="gevent", certfile=Settings.CERT_FILE, - keyfile=Settings.KEY_FILE + keyfile=Settings.KEY_FILE, ) From 244e0a57da1d8746a21d14b083e6b13ec49bbf19 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 13:01:33 +0000 Subject: [PATCH 134/234] pyproject.toml: Add file so as to configure black line-length --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..55ec8d7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 120 From 03525efe82fd7c96ab852df14b7f4d50189608e2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 13:41:27 +0000 Subject: [PATCH 135/234] Bouncer: Re-format with black. --- src/eddn/Bouncer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/eddn/Bouncer.py b/src/eddn/Bouncer.py index ef59b28..d0c9ce4 100644 --- a/src/eddn/Bouncer.py +++ b/src/eddn/Bouncer.py @@ -48,9 +48,7 @@ app = Bottle() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() -__logger_formatter = logging.Formatter( - "%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s" - ) +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") __logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" __logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) @@ -78,7 +76,8 @@ def parse_cl_args(): ) parser.add_argument( - "-c", "--config", + "-c", + "--config", metavar="config filename", nargs="?", default=None, @@ -268,12 +267,14 @@ class EnableCors(object): :param context: :return: """ + def _enable_cors(*args, **kwargs): # set CORS headers response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, OPTIONS" - response.headers["Access-Control-Allow-Headers"] = \ - "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" + response.headers[ + "Access-Control-Allow-Headers" + ] = "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token" if request.method != "OPTIONS": # actual request; reply with the actual response @@ -296,6 +297,7 @@ class CustomLogging(object): :param context: :return: """ + def _log_to_logger(*args, **kwargs): request_time = datetime.utcnow() actual_response = fn(*args, **kwargs) From 83fa0557228d4fa6d4c1b321af3bbe3b68e43c0f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 13:43:44 +0000 Subject: [PATCH 136/234] Monitor: Re-format with black --- src/eddn/Monitor.py | 135 ++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 75 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index c5a9f2c..db731e3 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -26,6 +26,7 @@ app = Bottle() # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: from eddn.core.DuplicateMessages import DuplicateMessages + duplicate_messages = DuplicateMessages() duplicate_messages.start() @@ -33,19 +34,20 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: def parse_cl_args(): """Parse command-line arguments.""" parser = argparse.ArgumentParser( - prog='Gateway', - description='EDDN Gateway server', + prog="Gateway", + description="EDDN Gateway server", ) parser.add_argument( - '--loglevel', - help='CURRENTLY NO EFFECT - Logging level to output at', + "--loglevel", + help="CURRENTLY NO EFFECT - Logging level to output at", ) parser.add_argument( - '-c', '--config', - metavar='config filename', - nargs='?', + "-c", + "--config", + metavar="config filename", + nargs="?", default=None, ) @@ -54,33 +56,33 @@ def parse_cl_args(): def date(__format) -> str: """ - Make a 'now' datetime as per the supplied format. + Make a "now" datetime as per the supplied format. :param __format: - :return: String representation of 'now' + :return: String representation of "now" """ d = datetime.datetime.utcnow() return d.strftime(__format) -@app.route('/ping', method=['OPTIONS', 'GET']) +@app.route("/ping", method=["OPTIONS", "GET"]) def ping() -> str: """Respond to a ping request.""" - return 'pong' + return "pong" -@app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET']) +@app.route("/getTotalSoftwares/", method=["OPTIONS", "GET"]) def get_total_softwares() -> str: """Respond with data about total uploading software counts.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( - user=Settings.MONITOR_DB['user'], - password=Settings.MONITOR_DB['password'], - database=Settings.MONITOR_DB['database'] + user=Settings.MONITOR_DB["user"], + password=Settings.MONITOR_DB["password"], + database=Settings.MONITOR_DB["database"], ) softwares = collections.OrderedDict() - max_days = request.GET.get('maxDays', '31').strip() + max_days = request.GET.get("maxDays", "31").strip() max_days = int(max_days) - 1 query = """SELECT name, SUM(hits) AS total, MAX(dateStats) AS maxDate @@ -90,29 +92,29 @@ def get_total_softwares() -> str: ORDER BY total DESC""" results = db.cursor() - results.execute(query, (max_days, )) + results.execute(query, (max_days,)) for row in results: - softwares[row[0].encode('utf8')] = str(row[1]) + softwares[row[0].encode("utf8")] = str(row[1]) db.close() return simplejson.dumps(softwares) -@app.route('/getSoftwares/', method=['OPTIONS', 'GET']) +@app.route("/getSoftwares/", method=["OPTIONS", "GET"]) def get_softwares() -> str: """Respond with hit stats for all uploading software.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( - user=Settings.MONITOR_DB['user'], - password=Settings.MONITOR_DB['password'], - database=Settings.MONITOR_DB['database'] + user=Settings.MONITOR_DB["user"], + password=Settings.MONITOR_DB["password"], + database=Settings.MONITOR_DB["database"], ) softwares: OrderedDict = collections.OrderedDict() - date_start = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() - date_end = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() + date_start = request.GET.get("dateStart", str(date("%Y-%m-%d"))).strip() + date_end = request.GET.get("dateEnd", str(date("%Y-%m-%d"))).strip() query = """SELECT * FROM `softwares` @@ -123,7 +125,7 @@ def get_softwares() -> str: results.execute(query, (date_start, date_end)) for row in results: - current_date = row[2].strftime('%Y-%m-%d') + current_date = row[2].strftime("%Y-%m-%d") if current_date not in softwares.keys(): softwares[current_date] = collections.OrderedDict() @@ -134,14 +136,14 @@ def get_softwares() -> str: return simplejson.dumps(softwares) -@app.route('/getTotalSchemas/', method=['OPTIONS', 'GET']) +@app.route("/getTotalSchemas/", method=["OPTIONS", "GET"]) def get_total_schemas() -> str: """Respond with total hit stats for all schemas.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( - user=Settings.MONITOR_DB['user'], - password=Settings.MONITOR_DB['password'], - database=Settings.MONITOR_DB['database'] + user=Settings.MONITOR_DB["user"], + password=Settings.MONITOR_DB["password"], + database=Settings.MONITOR_DB["database"], ) schemas = collections.OrderedDict() @@ -161,19 +163,19 @@ def get_total_schemas() -> str: return simplejson.dumps(schemas) -@app.route('/getSchemas/', method=['OPTIONS', 'GET']) +@app.route("/getSchemas/", method=["OPTIONS", "GET"]) def get_schemas() -> str: """Respond with schema hit stats between given datetimes.""" response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect( - user=Settings.MONITOR_DB['user'], - password=Settings.MONITOR_DB['password'], - database=Settings.MONITOR_DB['database'] + user=Settings.MONITOR_DB["user"], + password=Settings.MONITOR_DB["password"], + database=Settings.MONITOR_DB["database"], ) schemas: OrderedDict = collections.OrderedDict() - date_start = request.GET.get('dateStart', str(date('%Y-%m-%d'))).strip() - date_end = request.GET.get('dateEnd', str(date('%Y-%m-%d'))).strip() + date_start = request.GET.get("dateStart", str(date("%Y-%m-%d"))).strip() + date_end = request.GET.get("dateEnd", str(date("%Y-%m-%d"))).strip() query = """SELECT * FROM `schemas` @@ -184,7 +186,7 @@ def get_schemas() -> str: results.execute(query, (date_start, date_end)) for row in results: - current_date = row[2].strftime('%Y-%m-%d') + current_date = row[2].strftime("%Y-%m-%d") if current_date not in schemas.keys(): schemas[current_date] = collections.OrderedDict() @@ -203,38 +205,37 @@ class Monitor(Thread): context = zmq.Context() receiver = context.socket(ZMQ_SUB) - receiver.setsockopt_string(ZMQ_SUBSCRIBE, '') + receiver.setsockopt_string(ZMQ_SUBSCRIBE, "") for binding in Settings.MONITOR_RECEIVER_BINDINGS: receiver.connect(binding) def monitor_worker(message: bytes) -> None: db = mariadb.connect( - user=Settings.MONITOR_DB['user'], - password=Settings.MONITOR_DB['password'], - database=Settings.MONITOR_DB['database'] + user=Settings.MONITOR_DB["user"], + password=Settings.MONITOR_DB["password"], + database=Settings.MONITOR_DB["database"], ) message_text = zlib.decompress(message) json = simplejson.loads(message_text) # Default variables - schema_id = json['$schemaRef'] - software_id = f'{json["header"]["softwareName"]} | {json["header"]["softwareVersion"]}' + schema_id = json["$schemaRef"] + software_id = f"{json['header']['softwareName']} | {json['header']['softwareVersion']}" # Duplicates? if Settings.RELAY_DUPLICATE_MAX_MINUTES: if duplicate_messages.is_duplicated(json): - schema_id = 'DUPLICATE MESSAGE' + schema_id = "DUPLICATE MESSAGE" c = db.cursor() c.execute( - 'UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', - (schema_id, ) + "UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()", + (schema_id,), ) c.execute( - 'INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', - (schema_id, ) + "INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())", (schema_id,) ) db.commit() @@ -245,25 +246,18 @@ class Monitor(Thread): # Update software count c = db.cursor() c.execute( - 'UPDATE `softwares` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', - (software_id, ) - ) - c.execute( - 'INSERT IGNORE INTO `softwares` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', - (software_id, ) + "UPDATE `softwares` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()", + (software_id,), ) + c.execute("INSERT IGNORE INTO `softwares` (`name`, `dateStats`) VALUES (%s, UTC_DATE())", (software_id,)) db.commit() # Update schemas count c = db.cursor() c.execute( - 'UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()', - (schema_id, ) - ) - c.execute( - 'INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())', - (schema_id, ) + "UPDATE `schemas` SET `hits` = `hits` + 1 WHERE `name` = %s AND `dateStats` = UTC_DATE()", (schema_id,) ) + c.execute("INSERT IGNORE INTO `schemas` (`name`, `dateStats`) VALUES (%s, UTC_DATE())", (schema_id,)) db.commit() db.close() @@ -279,18 +273,9 @@ def apply_cors() -> None: Ref: """ - response.set_header( - 'Access-Control-Allow-Origin', - '*' - ) - response.set_header( - 'Access-Control-Allow-Methods', - 'GET, POST, PUT, OPTIONS' - ) - response.set_header( - 'Access-Control-Allow-Headers', - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' - ) + response.set_header("Access-Control-Allow-Origin", "*") + response.set_header("Access-Control-Allow-Methods", "GET, POST, PUT, OPTIONS") + response.set_header("Access-Control-Allow-Headers", "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token") def main() -> None: @@ -300,15 +285,15 @@ def main() -> None: m = Monitor() m.start() - app.add_hook('after_request', apply_cors) + app.add_hook("after_request", apply_cors) app.run( host=Settings.MONITOR_HTTP_BIND_ADDRESS, port=Settings.MONITOR_HTTP_PORT, - server='gevent', + server="gevent", certfile=Settings.CERT_FILE, - keyfile=Settings.KEY_FILE + keyfile=Settings.KEY_FILE, ) -if __name__ == '__main__': +if __name__ == "__main__": main() From d87106fe269e65ed746a0037991a63ae5bb1ddc3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 13:45:37 +0000 Subject: [PATCH 137/234] Relay: Re-format with black --- src/eddn/Relay.py | 65 ++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index f5a4788..f63a85e 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -23,14 +23,12 @@ from zmq import SUBSCRIBE as ZMQ_SUBSCRIBE logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() -__logger_formatter = logging.Formatter( - '%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s' - ) -__logger_formatter.default_time_format = '%Y-%m-%d %H:%M:%S' -__logger_formatter.default_msec_format = '%s.%03d' +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") +__logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" +__logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) logger.addHandler(__logger_channel) -logger.info('Made logger') +logger.info("Made logger") from eddn.conf.Settings import Settings, load_config # noqa: E402 @@ -47,6 +45,7 @@ stats_collector.start() # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: from eddn.core.DuplicateMessages import DuplicateMessages + duplicate_messages = DuplicateMessages() duplicate_messages.start() @@ -54,26 +53,27 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: def parse_cl_args() -> argparse.Namespace: """Parse command-line arguments.""" parser = argparse.ArgumentParser( - prog='Gateway', - description='EDDN Gateway server', + prog="Gateway", + description="EDDN Gateway server", ) parser.add_argument( - '--loglevel', - help='Logging level to output at', + "--loglevel", + help="Logging level to output at", ) parser.add_argument( - '-c', '--config', - metavar='config filename', - nargs='?', + "-c", + "--config", + metavar="config filename", + nargs="?", default=None, ) return parser.parse_args() -@app.route('/stats/', method=['OPTIONS', 'GET']) +@app.route("/stats/", method=["OPTIONS", "GET"]) def stats() -> str: """ Return some stats about the Relay's operation so far. @@ -112,7 +112,7 @@ class Relay(Thread): if now - self.uploader_nonce_timestamp > self.REGENERATE_UPLOADER_NONCE_INTERVAL: self.generate_uploader_nonce() - return hashlib.sha1(f"{self.uploader_nonce!r}-{uploader.encode}".encode('utf8')).hexdigest() + return hashlib.sha1(f"{self.uploader_nonce!r}-{uploader.encode}".encode("utf8")).hexdigest() def run(self) -> None: """Handle receiving messages from Gateway and passing them on.""" @@ -120,7 +120,7 @@ class Relay(Thread): context = zmq.Context() receiver = context.socket(ZMQ_SUB) - receiver.setsockopt_string(ZMQ_SUBSCRIBE, '') + receiver.setsockopt_string(ZMQ_SUBSCRIBE, "") for binding in Settings.RELAY_RECEIVER_BINDINGS: # Relays bind upstream to an Announcer, or another Relay. @@ -151,18 +151,18 @@ class Relay(Thread): # Mask the uploader with a randomised nonce but still make it unique # for each uploader - if 'uploaderID' in json['header']: - json['header']['uploaderID'] = self.scramble_uploader(json['header']['uploaderID']) + if "uploaderID" in json["header"]: + json["header"]["uploaderID"] = self.scramble_uploader(json["header"]["uploaderID"]) # Remove IP to end consumer - if 'uploaderIP' in json['header']: - del json['header']['uploaderIP'] + if "uploaderIP" in json["header"]: + del json["header"]["uploaderIP"] # Convert message back to JSON message_json = simplejson.dumps(json, sort_keys=True) # Recompress message - message = zlib.compress(message_json.encode('utf8')) + message = zlib.compress(message_json.encode("utf8")) # Send message sender.send(message) @@ -182,18 +182,9 @@ def apply_cors(): Ref: """ - response.set_header( - 'Access-Control-Allow-Origin', - '*' - ) - response.set_header( - 'Access-Control-Allow-Methods', - 'GET, POST, PUT, OPTIONS' - ) - response.set_header( - 'Access-Control-Allow-Headers', - 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' - ) + response.set_header("Access-Control-Allow-Origin", "*") + response.set_header("Access-Control-Allow-Methods", "GET, POST, PUT, OPTIONS") + response.set_header("Access-Control-Allow-Headers", "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token") def main() -> None: @@ -207,15 +198,15 @@ def main() -> None: r = Relay() r.start() - app.add_hook('after_request', apply_cors) + app.add_hook("after_request", apply_cors) app.run( host=Settings.RELAY_HTTP_BIND_ADDRESS, port=Settings.RELAY_HTTP_PORT, - server='gevent', + server="gevent", certfile=Settings.CERT_FILE, - keyfile=Settings.KEY_FILE + keyfile=Settings.KEY_FILE, ) -if __name__ == '__main__': +if __name__ == "__main__": main() From 7a6314cba8121b725d29fe745f621bd0d8c23806 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 13:46:53 +0000 Subject: [PATCH 138/234] conf/__init__.py: docstring --- src/eddn/conf/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eddn/conf/__init__.py b/src/eddn/conf/__init__.py index e69de29..d46c169 100644 --- a/src/eddn/conf/__init__.py +++ b/src/eddn/conf/__init__.py @@ -0,0 +1 @@ +"""EDDN configuration module.""" From d87344459aa51da8ffa1ac1a50245b5be4ce3e8f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 13:48:57 +0000 Subject: [PATCH 139/234] conf/Settings: s/'/"/g; But not with black We **do** want the special layout of the actual config defaults, so screw black! --- src/eddn/conf/Settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 09404f2..5e90c4b 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -16,8 +16,8 @@ class _Settings(object): # Local installation settings ############################################################################### - CERT_FILE = '/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem' # noqa: E221 - KEY_FILE = '/etc/letsencrypt/live/eddn.edcd.io/privkey.pem' # noqa: E221 + CERT_FILE = "/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem" # noqa: E221 + KEY_FILE = "/etc/letsencrypt/live/eddn.edcd.io/privkey.pem" # noqa: E221 ############################################################################### # Relay settings @@ -134,10 +134,10 @@ class _Settings(object): BOUNCER_HTTP_BIND_ADDRESS = "127.0.0.1" # noqa: E221 BOUNCER_HTTP_PORT = 8081 # noqa: E221 - BOUNCER_LIVE_GATEWAY_URL = 'https://eddn.edcd.io:4430/upload/' + BOUNCER_LIVE_GATEWAY_URL = "https://eddn.edcd.io:4430/upload/" def load_from(self, file_name: str) -> None: - f = open(file_name, 'r') + f = open(file_name, "r") conf = simplejson.load(f) for key, value in conf.items(): if key in dir(self): From d823056e15d0840916832113d2523fdf6791ce46 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:07:42 +0000 Subject: [PATCH 140/234] pyproject.toml: Exclude everything but actual core EDDN source --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 55ec8d7..c92e908 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,8 @@ [tool.black] line-length = 120 +extend-exclude = """ +(src/eddn/conf/Settings.py +|contrib/ +|examples/ +|scripts/) +""" From 5e8c17222d98079b8bfff625dac347799d538b52 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:10:41 +0000 Subject: [PATCH 141/234] setup.py: black and otherwise s/'/"/g pass --- setup.py | 142 ++++++++++++++++++++++++++----------------------------- 1 file changed, 67 insertions(+), 75 deletions(-) diff --git a/setup.py b/setup.py index c4a5414..8837556 100644 --- a/setup.py +++ b/setup.py @@ -21,14 +21,14 @@ try: verstr = mo.group(1) except EnvironmentError: - print(f'unable to find version in {VERSIONFILE}') - raise RuntimeError(f'if {VERSIONFILE} exists, it is required to be well-formed') + print(f"unable to find version in {VERSIONFILE}") + raise RuntimeError(f"if {VERSIONFILE} exists, it is required to be well-formed") # Read environment-specific settings ########################################################################### -# Enforce the git status being "branch 'live' checked out, at its HEAD" +# Enforce the git status being "branch "live" checked out, at its HEAD" # if setup_env.py says this is the live environment. # # The idea is to have the `live` branch, *which includes documentation* @@ -39,13 +39,11 @@ except EnvironmentError: ########################################################################### cwd = os.getcwd() # e.g. /home/eddn/live/EDDN.git -if setup_env.EDDN_ENV == 'live': +if setup_env.EDDN_ENV == "live": try: git_cmd = subprocess.Popen( - 'git symbolic-ref -q --short HEAD'.split(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT + "git symbolic-ref -q --short HEAD".split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) out, err = git_cmd.communicate() @@ -53,47 +51,37 @@ if setup_env.EDDN_ENV == 'live': print(f"Couldn't run git command to check branch: {e}") else: - branch = out.decode().rstrip('\n') + branch = out.decode().rstrip("\n") # - For any other branch checked out at its HEAD this will be a # different name. - # - For any 'detached HEAD' (i.e. specific commit ID, or tag) it + # - For any "detached HEAD" (i.e. specific commit ID, or tag) it # will be empty. - if branch != 'live': + if branch != "live": print(f"EDDN_ENV is '{setup_env.EDDN_ENV}' (and CWD is '{cwd}'), but branch is '{branch}', aborting!") sys.exit(-1) ########################################################################### # Location of start-eddn-service script and its config file -START_SCRIPT_BIN = pathlib.Path(f'{os.environ["HOME"]}/.local/bin') +START_SCRIPT_BIN = pathlib.Path(f"{os.environ['HOME']}/.local/bin") # Location of web files -SHARE_EDDN_FILES = pathlib.Path(f'{os.environ["HOME"]}/.local/share/eddn/{setup_env.EDDN_ENV}') +SHARE_EDDN_FILES = pathlib.Path(f"{os.environ['HOME']}/.local/share/eddn/{setup_env.EDDN_ENV}") setup( - name='eddn', + name="eddn", version=verstr, - description='Elite: Dangerous Data Network', + description="Elite: Dangerous Data Network", long_description="""\ The Elite Dangerous Data Network allows ED players to share data. Not affiliated with Frontier Developments. """, - author='EDCD (https://edcd.github.io/)', - author_email='edcd@miggy.org', - url='https://github.com/EDCD/EDDN', - - packages=find_packages( - 'src', - exclude=["*.tests"] - ), - package_dir={'': 'src'}, - + author="EDCD (https://edcd.github.io/)", + author_email="edcd@miggy.org", + url="https://github.com/EDCD/EDDN", + packages=find_packages("src", exclude=["*.tests"]), + package_dir={"": "src"}, # This includes them for the running code, but that doesn't help # serve them up for reference. - data_files=[ - ( - 'eddn/schemas', glob.glob("schemas/*.json") - ) - ], - + data_files=[("eddn/schemas", glob.glob("schemas/*.json"))], # Yes, we pin versions. With python2.7 the latest pyzmq will NOT # work, for instance. install_requires=[ @@ -104,26 +92,25 @@ setup( "jsonschema", "pyzmq", "simplejson", - "mysql-connector-python" + "mysql-connector-python", ], - entry_points={ - 'console_scripts': [ - 'eddn-gateway = eddn.Gateway:main', - 'eddn-relay = eddn.Relay:main', - 'eddn-monitor = eddn.Monitor:main', - 'eddn-bouncer = eddn.Bouncer:main', + "console_scripts": [ + "eddn-gateway = eddn.Gateway:main", + "eddn-relay = eddn.Relay:main", + "eddn-monitor = eddn.Monitor:main", + "eddn-bouncer = eddn.Bouncer:main", ], - } + }, ) def open_file_perms_recursive(dirname: pathlib.Path) -> None: """Open up file perms on the given directory and its contents.""" - print(f'open_file_perms_recursive: {dirname}') + print(f"open_file_perms_recursive: {dirname}") - for name in dirname.glob('*'): - print(f'open_file_perms_recursive: {name}') + for name in dirname.glob("*"): + print(f"open_file_perms_recursive: {name}") if name.is_dir(): name.chmod(0o755) open_file_perms_recursive(name) @@ -133,10 +120,12 @@ def open_file_perms_recursive(dirname: pathlib.Path) -> None: # Ensure the systemd-required start files are in place -print(""" +print( + """ ****************************************************************************** Ensuring start script and its config file are in place... -""") +""" +) try: START_SCRIPT_BIN.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -144,33 +133,28 @@ except Exception as e: print(f"{START_SCRIPT_BIN} can't be created, aborting!!!\n{e!r}") exit(-1) -shutil.copy( - f'systemd/eddn_{setup_env.EDDN_ENV}_config', - START_SCRIPT_BIN / f'eddn_{setup_env.EDDN_ENV}_config' -) +shutil.copy(f"systemd/eddn_{setup_env.EDDN_ENV}_config", START_SCRIPT_BIN / f"eddn_{setup_env.EDDN_ENV}_config") # NB: We copy to a per-environment version so that, e.g.live use won't break # due to changes in the other environments. -shutil.copy( - 'systemd/start-eddn-service', - START_SCRIPT_BIN / f'start-eddn-{setup_env.EDDN_ENV}-service' -) +shutil.copy("systemd/start-eddn-service", START_SCRIPT_BIN / f"start-eddn-{setup_env.EDDN_ENV}-service") # Ensure the service log file archiving script is in place -print(""" +print( + """ ****************************************************************************** Ensuring the service log file archiving script is in place -""") -shutil.copy( - 'contrib/eddn-logs-archive', - START_SCRIPT_BIN +""" ) +shutil.copy("contrib/eddn-logs-archive", START_SCRIPT_BIN) # Ensure the latest monitor files are in place old_umask = os.umask(0o22) -print(f""" +print( + f""" ****************************************************************************** Ensuring {SHARE_EDDN_FILES} exists... -""") +""" +) try: SHARE_EDDN_FILES.mkdir(mode=0o700, parents=True, exist_ok=True) @@ -178,59 +162,67 @@ except Exception as e: print(f"{SHARE_EDDN_FILES} can't be created, aborting!!!\n{e!r}") exit(-1) -print(""" +print( + """ ****************************************************************************** Ensuring latest monitor files are in place... -""") +""" +) # Copy the monitor (Web page) files try: - shutil.rmtree(SHARE_EDDN_FILES / 'monitor') + shutil.rmtree(SHARE_EDDN_FILES / "monitor") except OSError: pass shutil.copytree( - 'contrib/monitor', - SHARE_EDDN_FILES / 'monitor', + "contrib/monitor", + SHARE_EDDN_FILES / "monitor", copy_function=shutil.copyfile, # type: ignore ) # And a copy of the schemas too -print(""" +print( + """ ****************************************************************************** Ensuring latest schema files are in place for web access... -""") +""" +) try: - shutil.rmtree(SHARE_EDDN_FILES / 'schemas') + shutil.rmtree(SHARE_EDDN_FILES / "schemas") except OSError: pass shutil.copytree( - 'schemas', - SHARE_EDDN_FILES / 'schemas', + "schemas", + SHARE_EDDN_FILES / "schemas", copy_function=shutil.copyfile, # type: ignore ) -print(""" +print( + """ ****************************************************************************** Opening up permissions on monitor and schema files... -""") +""" +) os.chmod(SHARE_EDDN_FILES, 0o755) open_file_perms_recursive(SHARE_EDDN_FILES) # You still need to make an override config file -if not (SHARE_EDDN_FILES / 'config.json').is_file(): - shutil.copy('docs/config-EXAMPLE.json', SHARE_EDDN_FILES) - print(f""" +if not (SHARE_EDDN_FILES / "config.json").is_file(): + shutil.copy("docs/config-EXAMPLE.json", SHARE_EDDN_FILES) + print( + f""" ****************************************************************************** There was no config.json file in place, so docs/config-EXAMPLE.json was copied into: {SHARE_EDDN_FILES} -Please review, edit and rename this file to 'config.json' so that this +Please review, edit and rename this file to "config.json" so that this software will actually work. See docs/Running-this-software.md for guidance. ****************************************************************************** -""") +""" + ) os.umask(old_umask) From 50c87ecf211213dca7c544a36320140d89661b49 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:14:31 +0000 Subject: [PATCH 142/234] core/DuplicateMessages: black / quotes pass --- src/eddn/core/DuplicateMessages.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/eddn/core/DuplicateMessages.py b/src/eddn/core/DuplicateMessages.py index 418c761..ac4316c 100644 --- a/src/eddn/core/DuplicateMessages.py +++ b/src/eddn/core/DuplicateMessages.py @@ -41,33 +41,33 @@ class DuplicateMessages(Thread): """Detect if the given message is in the duplicates cache.""" with self.lock: # Test messages are never duplicate, would be a pain to wait for another test :D - if re.search('test', json['$schemaRef'], re.I): + if re.search("test", json["$schemaRef"], re.I): return False # Shallow copy, minus headers json_test = { - '$schemaRef': json['$schemaRef'], - 'message': dict(json['message']), + "$schemaRef": json["$schemaRef"], + "message": dict(json["message"]), } # Remove timestamp (Mainly to avoid multiple scan messages and faction influences) - json_test['message'].pop('timestamp') + json_test["message"].pop("timestamp") # Convert journal starPos to avoid software modification in dupe messages - if 'StarPos' in json_test['message']: - json_test['message']['StarPos'] = [int(round(x * 32)) for x in json_test['message']['StarPos']] + if "StarPos" in json_test["message"]: + json_test["message"]["StarPos"] = [int(round(x * 32)) for x in json_test["message"]["StarPos"]] # Prevent journal Docked event with small difference in distance from start - if 'DistFromStarLS' in json_test['message']: - json_test['message']['DistFromStarLS'] = int(json_test['message']['DistFromStarLS'] + 0.5) + if "DistFromStarLS" in json_test["message"]: + json_test["message"]["DistFromStarLS"] = int(json_test["message"]["DistFromStarLS"] + 0.5) # Remove journal ScanType and DistanceFromArrivalLS (Avoid duplicate scan messages after SAAScanComplete) - json_test['message'].pop('ScanType', None) - json_test['message'].pop('DistanceFromArrivalLS', None) + json_test["message"].pop("ScanType", None) + json_test["message"].pop("DistanceFromArrivalLS", None) message = simplejson.dumps(json_test, sort_keys=True) # Ensure most duplicate messages will get the same # key - key = hashlib.sha256(message.encode('utf8')).hexdigest() + key = hashlib.sha256(message.encode("utf8")).hexdigest() if key not in self.caches: self.caches[key] = datetime.utcnow() From 8a07ae1cdc3637e185e31b38645d9bc56a60b3e8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:16:37 +0000 Subject: [PATCH 143/234] core/Validator: black pass, and remove extraneous `,` in constants --- src/eddn/core/Validator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/eddn/core/Validator.py b/src/eddn/core/Validator.py index 2e78ee9..c5584a8 100644 --- a/src/eddn/core/Validator.py +++ b/src/eddn/core/Validator.py @@ -12,9 +12,9 @@ from jsonschema import validate as json_validate class ValidationSeverity(IntEnum): """Enum of validation status.""" - OK = 0, - WARN = 1, - ERROR = 2, + OK = 0 + WARN = 1 + ERROR = 2 FATAL = 3 @@ -63,7 +63,7 @@ class Validator(object): self.schemas[schema_ref] = schema except simplejson.JSONDecodeError: - raise Exception(f'SCHEMA: Failed to load: {schema_ref}') + raise Exception(f"SCHEMA: Failed to load: {schema_ref}") def validate(self, json_object: Dict) -> ValidationResults: """ @@ -83,7 +83,7 @@ class Validator(object): # We don't want to go out to the Internet and retrieve unknown schemas. results.add( ValidationSeverity.FATAL, - JsonValidationException(f'Schema {schema_ref} is unknown, unable to validate.') + JsonValidationException(f"Schema {schema_ref} is unknown, unable to validate."), ) return results From bc5b7db0b14d6e5ba81ddd3167d53a72a29a05bc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:17:35 +0000 Subject: [PATCH 144/234] core/StatsCollector.py: black pass --- src/eddn/core/StatsCollector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/core/StatsCollector.py b/src/eddn/core/StatsCollector.py index 840453a..df3ecfe 100644 --- a/src/eddn/core/StatsCollector.py +++ b/src/eddn/core/StatsCollector.py @@ -73,9 +73,9 @@ class StatsCollector(Thread): summary[key] = { "1min": self.get_count(key, 1), "5min": self.get_count(key, 5), - "60min": self.get_count(key, 60) + "60min": self.get_count(key, 60), } - summary['uptime'] = int((datetime.utcnow() - self.start_time).total_seconds()) + summary["uptime"] = int((datetime.utcnow() - self.start_time).total_seconds()) return summary From a0ddf4a7b9a03d8b7d22eae8f417c6a176de6957 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:21:24 +0000 Subject: [PATCH 145/234] requirements-dev: Add "black" --- requirements-dev.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 04c41eb..d05951d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,9 @@ # Using legacy 'setup.py install' for flake8-annotations-coverage, since package 'wheel' is not installed. wheel +# Code formatting +black==22.1.0 + # Static analysis tools flake8==4.0.1 flake8-annotations-coverage==0.0.5 From a1a69dec1b8f10f915bc9736e6e0eee814073a05 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:22:17 +0000 Subject: [PATCH 146/234] .python-version: Specify only 3.9 in general, not specific patch We'll be using Debian bullseye's 3.9, which is currently 3.9.2 --- .python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.python-version b/.python-version index 26cb485..bd28b9c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.9.8 +3.9 From 71ab6ba6be375f72d5622124d40f9fea72ac653d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:25:22 +0000 Subject: [PATCH 147/234] Gateway: Remove extraneous noqa's --- src/eddn/Gateway.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 4c34c21..84ea154 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -248,7 +248,7 @@ def parse_and_error_handle(data: bytes) -> str: try: (uploader_id, software_name, software_version, schema_ref, journal_event,) = extract_message_details( parsed_message - ) # noqa: E501 + ) logger.info( "Accepted (%d, '%s', '%s', '%s', '%s', '%s') from %s", request.content_length, @@ -271,7 +271,7 @@ def parse_and_error_handle(data: bytes) -> str: try: (uploader_id, software_name, software_version, schema_ref, journal_event,) = extract_message_details( parsed_message - ) # noqa: E501 + ) logger.error( "Failed Validation '%s' (%d, '%s', '%s', '%s', '%s', '%s') from %s", str(validation_results.messages), From 015b4d1b2c1451c560c302a0c717e7f45010b0fe Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:50:39 +0000 Subject: [PATCH 148/234] .flake8: Some tuning for EDDN specifically --- .flake8 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index 6c8327e..dd85a17 100644 --- a/.flake8 +++ b/.flake8 @@ -1,15 +1,12 @@ [flake8] # Show exactly where in a line the error happened -#show-source = True +show-source = True max-line-length = 120 -# Add _ as a builtin for localisation stuff -builtins = _ # check syntax in doctests doctests = True max-complexity = 15 -per-file-ignores = ./EDMC.py:E402 # Plugin configs # required plugins: From ff18edff6e92e2ea732a374d340a171196b9d3f0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 14:58:50 +0000 Subject: [PATCH 149/234] setup.py: Turn isort off/on around `import setup_env.py` setup_env.py is NEVER checked into git, forcing anyone running the code to ensure they have it set up correctly before setup.py will work. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 8837556..1d16c74 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,9 @@ import sys from setuptools import find_packages, setup +# isort: off import setup_env +# isort: on VERSIONFILE = "src/eddn/conf/Version.py" verstr = "unknown" From 93fb5b93459e4fd88c6c0e591892e62c83c02358 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 15:05:10 +0000 Subject: [PATCH 150/234] scripts/check-schemas-load: black-formatted, and be verbose --- scripts/check-schemas-load.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/check-schemas-load.py b/scripts/check-schemas-load.py index 2d95559..fe17bc2 100644 --- a/scripts/check-schemas-load.py +++ b/scripts/check-schemas-load.py @@ -10,16 +10,16 @@ script_path = pathlib.Path(sys.argv[0]) root_dir = script_path.parent.parent # Take every file in the schemas directory -schemas_dir = root_dir / 'schemas' +schemas_dir = root_dir / "schemas" failures = 0 -for schema_file in schemas_dir.glob('*-v*.*.json'): - # print(f'Schema: {schema_file}') - with open(schema_file, 'r') as sf: +for schema_file in schemas_dir.glob("*-v*.*.json"): + print(f"Schema: {schema_file}") + with open(schema_file, "r") as sf: try: json = simplejson.load(sf) except simplejson.JSONDecodeError as e: - print(f'Failed to load {schema_file}:\n{e!r}') + print(f"Failed to load {schema_file}:\n{e!r}") failures += 1 if failures > 0: From 56b539fd56bd6a2cf40eda707e12bae832fe4f68 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 15:07:42 +0000 Subject: [PATCH 151/234] scripts/check-schemas-load: Standardise per-schema output If we're going to say a schema is OK, then prefix any output with the schema file location. --- scripts/check-schemas-load.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/check-schemas-load.py b/scripts/check-schemas-load.py index fe17bc2..1132a74 100644 --- a/scripts/check-schemas-load.py +++ b/scripts/check-schemas-load.py @@ -13,14 +13,16 @@ root_dir = script_path.parent.parent schemas_dir = root_dir / "schemas" failures = 0 for schema_file in schemas_dir.glob("*-v*.*.json"): - print(f"Schema: {schema_file}") with open(schema_file, "r") as sf: try: json = simplejson.load(sf) except simplejson.JSONDecodeError as e: - print(f"Failed to load {schema_file}:\n{e!r}") + print(f"{schema_file}: Failed to load:\n{e!r}\n") failures += 1 + else: + print(f"{schema_file}: OK") + if failures > 0: exit(-1) From 8b5b9142a3f08c14a1f27bb3774c86c99ad85cb1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 15:11:15 +0000 Subject: [PATCH 152/234] Settings: Remove {CERT,KEY}_FILE as first step to no more TLS --- src/eddn/conf/Settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 5e90c4b..e59ce5d 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -16,9 +16,6 @@ class _Settings(object): # Local installation settings ############################################################################### - CERT_FILE = "/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem" # noqa: E221 - KEY_FILE = "/etc/letsencrypt/live/eddn.edcd.io/privkey.pem" # noqa: E221 - ############################################################################### # Relay settings ############################################################################### From 4b0898122d50d2e14a4c8beb437205cad6ba6967 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 15:14:48 +0000 Subject: [PATCH 153/234] Gateway: Don't use TLS cert in app setup * The /upload/ functionality continues to work. --- src/eddn/Gateway.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 84ea154..ef28f17 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -403,8 +403,6 @@ def main() -> None: host=Settings.GATEWAY_HTTP_BIND_ADDRESS, port=Settings.GATEWAY_HTTP_PORT, server="gevent", - certfile=Settings.CERT_FILE, - keyfile=Settings.KEY_FILE, ) From e613767afab9965ce776d77112274d40d502e78b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 15:25:55 +0000 Subject: [PATCH 154/234] Update docs and apache contrib file for 'no more TLS' --- contrib/apache-eddn.conf | 11 +++++------ docs/Running-this-software.md | 18 +++++------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/contrib/apache-eddn.conf b/contrib/apache-eddn.conf index 23842e5..62ac7a1 100644 --- a/contrib/apache-eddn.conf +++ b/contrib/apache-eddn.conf @@ -151,15 +151,14 @@ - SSLProxyEngine On - SSLProxyVerify none ProxyPreserveHost On ProxyRequests Off - # Must be https, not http, as the Gateway process is - # expecting only https requests. - ProxyPass "/" "https://127.0.0.1:8081/" - ProxyPassReverse "/" "https://127.0.0.1:8081/" + + # Yes, plain HTTP, as the Gateway process knows nothing of + # TLS. + ProxyPass "http://127.0.0.1:8081/" + diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index ba7bd81..c1a8195 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -316,12 +316,6 @@ Default application configuration is in the file `src/eddn/conf/Settings.py`. Do **not** change anything in this file, see below about overriding using another file. -1. You will need to obtain a TLS certificate from, e.g. LetsEncrypt. The - application will need access to this and its private key file. - - CERT_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem' - KEY_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem' - 1. Network configuration 1. `RELAY_HTTP_BIND_ADDRESS` and `RELAY_HTTP_PORT` define the IP and port on which the Relay listens for, e.g. `/stats/` requests. @@ -384,7 +378,6 @@ There is an **example** of this in [eddn-settings-overrides-EXAMPLE.json](./eddn-settings-overrides-EXAMPLE.json). It sets: - 1. The TLS CERT and KEY files. 1. The gateway to listen on `0.0.0.0` rather than localhost (necessary when testing in a VM). 1. Configures the database connection and credentials. @@ -533,14 +526,13 @@ proxying: If using Apache on a Debian server then you need some ProxyPass directives: - SSLProxyEngine On - SSLProxyVerify none ProxyPreserveHost On + ProxyRequests Off - # Pass through 'gateway' upload URL to Debian VM - ProxyPass "/eddn/upload/" "https://VM_HOST:8081/upload/" - # Pass through 'monitor' URLs to Debian VM - ProxyPass "/eddn/" "https://VM_HOST/" + # Pass through anything with path prefix /eddn + + ProxPass "http://127.0.0.1:8081/" + This assumes you don't have a dedicated virtual host in this case, hence the From 402758f1d6f1ba6bc2f5e4b97fbb11f235241241 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 17:06:37 +0000 Subject: [PATCH 155/234] Gateway: Fixed logging to be consistent and use client IP * Send all the bottle server output through our logger. * Ensure gevent uses client IP, not 127.0.0.1. --- src/eddn/Gateway.py | 53 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index ef28f17..f4a7893 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -13,6 +13,7 @@ from datetime import datetime from typing import Dict import gevent +import gevent.pywsgi import simplejson import urlparse import zmq.green as zmq @@ -389,6 +390,55 @@ def apply_cors() -> None: ) +class EDDNWSGIHandler(gevent.pywsgi.WSGIHandler): + """iHandles overriding request logging.""" + + def format_request(self): + """ + Format information about the request for logging. + + The default causes, e.g.: + + - - [2022-03-12 16:44:39] "POST /upload/ HTTP/1.1" 400 399 0.000566 + + and makes no attempt to handle reverse proxying where we know that + X-Forwarded-For has been set by our reverse proxy. So this will use + that header if present to output the correct IP. + + Also, as we're pointing output at our logger, there is no need to + include a timestamp in the output. + + This is why we're overriding *this* and not `handle_one_response()`, + where we'd change self.client_address there instead. This does, + unfortunately, mean re-creating most of the super-class'es version + of the function. + + Resulting output: + + 2022-03-12 16:44:39.132 - INFO - pywsgi:1226: - - "POST /upload/ HTTP/1.1" 400 399 0.000566 + """ + # Start with the same as the super-class would use... + client_address = self.client_address[0] if isinstance(self.client_address, tuple) else self.client_address + + # ... but now over-ride it if the header is set. + if self.environ.get("HTTP_X_FORWARDED_FOR"): + client_address = self.environ["HTTP_X_FORWARDED_FOR"] + + length = self.response_length or '-' + + if self.time_finish: + d = self.time_finish - self.time_start + delta = f"{d:6f}" + + else: + delta = '-' + + # This differs from the super-class version in not having a datestamp + return f"{client_address or '-'} - - \"{self.requestline or '-'}\"" \ + f" {(self._orig_status or self.status or '000').split()[0]}" \ + f" {length} {delta}" + + def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() @@ -399,10 +449,13 @@ def main() -> None: configure() app.add_hook("after_request", apply_cors) + # app.install(custom_logging) app.run( host=Settings.GATEWAY_HTTP_BIND_ADDRESS, port=Settings.GATEWAY_HTTP_PORT, server="gevent", + log=gevent.pywsgi.LoggingLogAdapter(logger), + handler_class=EDDNWSGIHandler, ) From 8cf0d12d97164e012159eb8de3a3cea1623c5e85 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 17:12:17 +0000 Subject: [PATCH 156/234] Move EDDNWSGIHandler into its own file --- src/eddn/Gateway.py | 51 +---------------------------- src/eddn/core/EDDNWSGIHandler.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 50 deletions(-) create mode 100644 src/eddn/core/EDDNWSGIHandler.py diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index f4a7893..d10d50e 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -13,7 +13,6 @@ from datetime import datetime from typing import Dict import gevent -import gevent.pywsgi import simplejson import urlparse import zmq.green as zmq @@ -51,6 +50,7 @@ validator = Validator() # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector # noqa: E402 +from eddn.core.EDDNWSGIHandler import EDDNWSGIHandler stats_collector = StatsCollector() stats_collector.start() @@ -390,55 +390,6 @@ def apply_cors() -> None: ) -class EDDNWSGIHandler(gevent.pywsgi.WSGIHandler): - """iHandles overriding request logging.""" - - def format_request(self): - """ - Format information about the request for logging. - - The default causes, e.g.: - - - - [2022-03-12 16:44:39] "POST /upload/ HTTP/1.1" 400 399 0.000566 - - and makes no attempt to handle reverse proxying where we know that - X-Forwarded-For has been set by our reverse proxy. So this will use - that header if present to output the correct IP. - - Also, as we're pointing output at our logger, there is no need to - include a timestamp in the output. - - This is why we're overriding *this* and not `handle_one_response()`, - where we'd change self.client_address there instead. This does, - unfortunately, mean re-creating most of the super-class'es version - of the function. - - Resulting output: - - 2022-03-12 16:44:39.132 - INFO - pywsgi:1226: - - "POST /upload/ HTTP/1.1" 400 399 0.000566 - """ - # Start with the same as the super-class would use... - client_address = self.client_address[0] if isinstance(self.client_address, tuple) else self.client_address - - # ... but now over-ride it if the header is set. - if self.environ.get("HTTP_X_FORWARDED_FOR"): - client_address = self.environ["HTTP_X_FORWARDED_FOR"] - - length = self.response_length or '-' - - if self.time_finish: - d = self.time_finish - self.time_start - delta = f"{d:6f}" - - else: - delta = '-' - - # This differs from the super-class version in not having a datestamp - return f"{client_address or '-'} - - \"{self.requestline or '-'}\"" \ - f" {(self._orig_status or self.status or '000').split()[0]}" \ - f" {length} {delta}" - - def main() -> None: """Handle setting up and running the bottle app.""" cl_args = parse_cl_args() diff --git a/src/eddn/core/EDDNWSGIHandler.py b/src/eddn/core/EDDNWSGIHandler.py new file mode 100644 index 0000000..4f45357 --- /dev/null +++ b/src/eddn/core/EDDNWSGIHandler.py @@ -0,0 +1,56 @@ +""" +Sub-class of main gevent.pywsgi.WSGIHandler. + +Necessary in order to override some behaviour. +""" +import gevent +import gevent.pywsgi + + +class EDDNWSGIHandler(gevent.pywsgi.WSGIHandler): + """Handles overriding request logging.""" + + def format_request(self) -> str: + """ + Format information about the request for logging. + + The default causes, e.g.: + + - - [2022-03-12 16:44:39] "POST /upload/ HTTP/1.1" 400 399 0.000566 + + and makes no attempt to handle reverse proxying where we know that + X-Forwarded-For has been set by our reverse proxy. So this will use + that header if present to output the correct IP. + + Also, as we're pointing output at our logger, there is no need to + include a timestamp in the output. + + This is why we're overriding *this* and not `handle_one_response()`, + where we'd change self.client_address there instead. This does, + unfortunately, mean re-creating most of the super-class'es version + of the function. + + Resulting output: + + 2022-03-12 16:44:39.132 - INFO - pywsgi:1226: - - "POST /upload/ HTTP/1.1" 400 399 0.000566 + """ + # Start with the same as the super-class would use... + client_address = self.client_address[0] if isinstance(self.client_address, tuple) else self.client_address + + # ... but now over-ride it if the header is set. + if self.environ.get("HTTP_X_FORWARDED_FOR"): + client_address = self.environ["HTTP_X_FORWARDED_FOR"] + + length = self.response_length or '-' + + if self.time_finish: + d = self.time_finish - self.time_start + delta = f"{d:6f}" + + else: + delta = '-' + + # This differs from the super-class version in not having a datestamp + return f"{client_address or '-'} - - \"{self.requestline or '-'}\"" \ + f" {(self._orig_status or self.status or '000').split()[0]}" \ + f" {length} {delta}" From 9bf24f9a05d7a67c2c46790b652b01dfcd037119 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 12 Mar 2022 19:00:07 +0000 Subject: [PATCH 157/234] EDDNWSGIHandler: X-Forwarded-For can be a comma-separated list So, make it easier to pull out the IPs, single or not. --- src/eddn/core/EDDNWSGIHandler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/core/EDDNWSGIHandler.py b/src/eddn/core/EDDNWSGIHandler.py index 4f45357..b77a2c9 100644 --- a/src/eddn/core/EDDNWSGIHandler.py +++ b/src/eddn/core/EDDNWSGIHandler.py @@ -51,6 +51,6 @@ class EDDNWSGIHandler(gevent.pywsgi.WSGIHandler): delta = '-' # This differs from the super-class version in not having a datestamp - return f"{client_address or '-'} - - \"{self.requestline or '-'}\"" \ + return f"[{client_address or '-'}] - - \"{self.requestline or '-'}\"" \ f" {(self._orig_status or self.status or '000').split()[0]}" \ f" {length} {delta}" From d5dbc3262f51a38f95750bbffae46bf62ee57252 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 18 Aug 2022 15:20:12 +0100 Subject: [PATCH 158/234] Gateway: Put remote_addr in [], as it could be multiple, comma-separated # Conflicts: # src/eddn/Gateway.py # Conflicts: # src/eddn/Gateway.py --- src/eddn/Gateway.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d10d50e..79f6eb3 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -208,7 +208,7 @@ def parse_and_error_handle(data: bytes) -> str: # semi-useful error message, so do so. try: logger.error( - "Error - JSON parse failed (%d, '%s', '%s', '%s', '%s', '%s') from %s:\n%s\n", + "Error - JSON parse failed (%d, '%s', '%s', '%s', '%s', '%s') from [%s]:\n%s\n", request.content_length, "<>", "<>", @@ -251,7 +251,7 @@ def parse_and_error_handle(data: bytes) -> str: parsed_message ) logger.info( - "Accepted (%d, '%s', '%s', '%s', '%s', '%s') from %s", + "Accepted (%d, '%s', '%s', '%s', '%s', '%s') from [%s]", request.content_length, uploader_id, software_name, @@ -274,7 +274,7 @@ def parse_and_error_handle(data: bytes) -> str: parsed_message ) logger.error( - "Failed Validation '%s' (%d, '%s', '%s', '%s', '%s', '%s') from %s", + "Failed Validation '%s' (%d, '%s', '%s', '%s', '%s', '%s') from [%s]", str(validation_results.messages), request.content_length, uploader_id, @@ -314,7 +314,7 @@ def upload() -> str: try: logger.error( f"gzip error ({request.content_length}, '<>', '<>', '<>'" - ", '<>', '<>') from {get_remote_address()}" + ", '<>', '<>') from [{get_remote_address()}]" ) except Exception as e: @@ -328,7 +328,7 @@ def upload() -> str: # They probably sent an encoded POST, but got the key/val wrong. response.status = 400 # TODO: Maybe just `{exc}` ? - logger.error("MalformedUploadError from %s: %s", get_remote_address(), str(exc)) + logger.error("MalformedUploadError from [%s]: %s", get_remote_address(), str(exc)) return "FAIL: Malformed Upload: " + str(exc) From 83e2b548ad2313d0c06ed1c97c5644c4b1d64084 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 13:32:13 +0000 Subject: [PATCH 159/234] Gateway: Use a dict for kwargs to app.run() This is so we can now adjust if we're putting the TLS cert/key files in. --- src/eddn/Gateway.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 79f6eb3..943bc53 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -400,13 +400,18 @@ def main() -> None: configure() app.add_hook("after_request", apply_cors) - # app.install(custom_logging) + + # Build arg dict for args + argsd = { + 'host': Settings.GATEWAY_HTTP_BIND_ADDRESS, + 'port': Settings.GATEWAY_HTTP_PORT, + 'server': "gevent", + 'log': gevent.pywsgi.LoggingLogAdapter(logger), + 'handler_class': EDDNWSGIHandler, + } + app.run( - host=Settings.GATEWAY_HTTP_BIND_ADDRESS, - port=Settings.GATEWAY_HTTP_PORT, - server="gevent", - log=gevent.pywsgi.LoggingLogAdapter(logger), - handler_class=EDDNWSGIHandler, + **argsd ) From 128dab965caed231d8ae3b8412bbd79310940e69 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 13:46:37 +0000 Subject: [PATCH 160/234] TLS: Optionally use TLS if you set non-empty CERT_FILE and KEY_FILE Whilst we do want to go TLS-less in the actual EDDN code, for ease of setting up automated end to end functional testing, leave the possibility of running with TLS termination as well. --- contrib/apache-eddn.conf | 11 +++++++++-- docs/Running-this-software.md | 4 +++- src/eddn/Gateway.py | 5 +++++ src/eddn/conf/Settings.py | 6 ++++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/contrib/apache-eddn.conf b/contrib/apache-eddn.conf index 62ac7a1..393c8ed 100644 --- a/contrib/apache-eddn.conf +++ b/contrib/apache-eddn.conf @@ -151,12 +151,19 @@ + #################################### + # Only uncomment the following if you are setting non-empty + # CERT_FILE, and KEY_FILE in the main EDDN config.json + #################################### + # SSLProxyEngine On + # SSLProxyVerify none + #################################### ProxyPreserveHost On ProxyRequests Off - # Yes, plain HTTP, as the Gateway process knows nothing of - # TLS. + # Plain http if setting **empty** CERT_FILE and KEY_FILE in + # the EDDN config.json, else https. ProxyPass "http://127.0.0.1:8081/" diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index c1a8195..895e0d5 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -523,7 +523,9 @@ proxying: Internet -> existing server -> VM -> nginx -> EDDN scripts -If using Apache on a Debian server then you need some ProxyPass directives: +If using Apache on a Debian server then you need some ProxyPass directives. +These assume you using an empty CERT_FILE and KEY_FILE in the override +config.json. See `contrib/apache-eddn.conf` for how to use TLS instead: ProxyPreserveHost On diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 943bc53..edeba5e 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -410,6 +410,11 @@ def main() -> None: 'handler_class': EDDNWSGIHandler, } + # Empty CERT_FILE or KEY_FILE means don't put them in + if Settings.CERT_FILE != "" and Settings.KEY_FILE != "": + argsd["certfile"] = Settings.CERT_FILE + argsd["keyfile"] = Settings.KEY_FILE + app.run( **argsd ) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index e59ce5d..1eafae6 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -15,6 +15,12 @@ class _Settings(object): ############################################################################### # Local installation settings ############################################################################### + # If these are set to non-empty strings then you reverse proxt setup + # **MUST** pass TLS through properly, including to a https URL, not a + # plain http one. + CERT_FILE = "/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem" # noqa: E221 + KEY_FILE = "/etc/letsencrypt/live/eddn.edcd.io/privkey.pem" # noqa: E221 + ############################################################################### # Relay settings From d79f4a5aa18b36ce5afaa42a1d5fcf1a2e64ad94 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 13:59:15 +0000 Subject: [PATCH 161/234] Monitor: Change to be able to run TLS-less NB: Not yet changed the actual monitor web page files. --- contrib/apache-eddn.conf | 23 +++++++++++++++++++---- docs/Running-this-software.md | 7 ++++++- src/eddn/Gateway.py | 2 +- src/eddn/Monitor.py | 34 +++++++++++++++++++++++++++++----- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/contrib/apache-eddn.conf b/contrib/apache-eddn.conf index 393c8ed..77bb3c6 100644 --- a/contrib/apache-eddn.conf +++ b/contrib/apache-eddn.conf @@ -1,4 +1,4 @@ -# vim: :filetype=apache +# vim: :filetype=apache tabstop=4 shiftwidth=4 expandtab ########################################################################### # @@ -161,11 +161,26 @@ ProxyPreserveHost On ProxyRequests Off + #################################### + # Gateway, both /upload/ and /stats/ etc + #################################### - # Plain http if setting **empty** CERT_FILE and KEY_FILE in - # the EDDN config.json, else https. - ProxyPass "http://127.0.0.1:8081/" + # Plain http if setting **empty** CERT_FILE and KEY_FILE in + # the EDDN config.json, else https. + ProxyPass "http://127.0.0.1:8081/" + #################################### + + #################################### + # Monitor, /getSoftwares/ etc + #################################### + + # Plain http if setting **empty** CERT_FILE and KEY_FILE in + # the EDDN config.json, else https. + ProxyPass "http://127.0.0.1:9091/" + + #################################### + diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 895e0d5..8e925ed 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -531,10 +531,15 @@ config.json. See `contrib/apache-eddn.conf` for how to use TLS instead: ProxyPreserveHost On ProxyRequests Off - # Pass through anything with path prefix /eddn + # Pass through anything with path prefix /eddn to the Gateway ProxPass "http://127.0.0.1:8081/" + + # Anything with /monitor/ is for the Monitor + + ProxPass "http://127.0.0.1:9091/" + This assumes you don't have a dedicated virtual host in this case, hence the diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index edeba5e..d583832 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -416,7 +416,7 @@ def main() -> None: argsd["keyfile"] = Settings.KEY_FILE app.run( - **argsd + **argsd, ) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index db731e3..12d3bf7 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -4,6 +4,7 @@ import argparse import collections import datetime +import logging import zlib from threading import Thread from typing import OrderedDict @@ -23,6 +24,16 @@ monkey.patch_all() app = Bottle() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +__logger_channel = logging.StreamHandler() +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") +__logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" +__logger_formatter.default_msec_format = "%s.%03d" +__logger_channel.setFormatter(__logger_formatter) +logger.addHandler(__logger_channel) +logger.info("Made logger") + # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: from eddn.core.DuplicateMessages import DuplicateMessages @@ -30,6 +41,8 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicate_messages = DuplicateMessages() duplicate_messages.start() +from eddn.core.EDDNWSGIHandler import EDDNWSGIHandler + def parse_cl_args(): """Parse command-line arguments.""" @@ -286,12 +299,23 @@ def main() -> None: m = Monitor() m.start() app.add_hook("after_request", apply_cors) + + # Build arg dict for args + argsd = { + 'host': Settings.MONITOR_HTTP_BIND_ADDRESS, + 'port': Settings.MONITOR_HTTP_PORT, + 'server': "gevent", + 'log': gevent.pywsgi.LoggingLogAdapter(logger), + 'handler_class': EDDNWSGIHandler, + } + + # Empty CERT_FILE or KEY_FILE means don't put them in + if Settings.CERT_FILE != "" and Settings.KEY_FILE != "": + argsd["certfile"] = Settings.CERT_FILE + argsd["keyfile"] = Settings.KEY_FILE + app.run( - host=Settings.MONITOR_HTTP_BIND_ADDRESS, - port=Settings.MONITOR_HTTP_PORT, - server="gevent", - certfile=Settings.CERT_FILE, - keyfile=Settings.KEY_FILE, + **argsd, ) From 95b56ee3b0332a1ea9e69a27d17bfbc132530fa0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 15:34:09 +0000 Subject: [PATCH 162/234] Relay: Allow for running without TLS --- src/eddn/Relay.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index f63a85e..3dcf2be 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -38,6 +38,7 @@ app = Bottle() # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector # noqa: E402 +from eddn.core.EDDNWSGIHandler import EDDNWSGIHandler stats_collector = StatsCollector() stats_collector.start() @@ -199,12 +200,23 @@ def main() -> None: r.start() app.add_hook("after_request", apply_cors) + + # Build arg dict for args + argsd = { + 'host': Settings.RELAY_HTTP_BIND_ADDRESS, + 'port': Settings.RELAY_HTTP_PORT, + 'server': "gevent", + 'log': gevent.pywsgi.LoggingLogAdapter(logger), + 'handler_class': EDDNWSGIHandler, + } + + # Empty CERT_FILE or KEY_FILE means don't put them in + if Settings.CERT_FILE != "" and Settings.KEY_FILE != "": + argsd["certfile"] = Settings.CERT_FILE + argsd["keyfile"] = Settings.KEY_FILE + app.run( - host=Settings.RELAY_HTTP_BIND_ADDRESS, - port=Settings.RELAY_HTTP_PORT, - server="gevent", - certfile=Settings.CERT_FILE, - keyfile=Settings.KEY_FILE, + **argsd, ) From 4d705e4f37939911e09982146c8b40185da3ab32 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 15:34:54 +0000 Subject: [PATCH 163/234] Settings: Default Relay and Monitor to localhost listen. --- src/eddn/conf/Settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 1eafae6..1201c5a 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -26,7 +26,7 @@ class _Settings(object): # Relay settings ############################################################################### - RELAY_HTTP_BIND_ADDRESS = "0.0.0.0" # noqa: E221 + RELAY_HTTP_BIND_ADDRESS = "127.0.0.1" # noqa: E221 RELAY_HTTP_PORT = 9090 # noqa: E221 RELAY_RECEIVER_BINDINGS = ["tcp://127.0.0.1:8500"] # noqa: E221 @@ -118,7 +118,7 @@ class _Settings(object): # Monitor settings ############################################################################### - MONITOR_HTTP_BIND_ADDRESS = "0.0.0.0" # noqa: E221 + MONITOR_HTTP_BIND_ADDRESS = "127.0.0.1" # noqa: E221 MONITOR_HTTP_PORT = 9091 # noqa: E221 MONITOR_RECEIVER_BINDINGS = ["tcp://127.0.0.1:8500"] # noqa: E221 From 1ae349a4b76733dbbbb7878860be6f15f94e7839 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 15:42:53 +0000 Subject: [PATCH 164/234] Make per-component logging more obvious --- src/eddn/Gateway.py | 2 +- src/eddn/Monitor.py | 2 +- src/eddn/Relay.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index d583832..44f0a19 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -34,7 +34,7 @@ app = Bottle() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() -__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - Gateway - %(module)s:%(lineno)d: %(message)s") __logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" __logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 12d3bf7..3f1286c 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -27,7 +27,7 @@ app = Bottle() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() -__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - Monitor - %(module)s:%(lineno)d: %(message)s") __logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" __logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 3dcf2be..874b6a4 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -23,7 +23,7 @@ from zmq import SUBSCRIBE as ZMQ_SUBSCRIBE logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) __logger_channel = logging.StreamHandler() -__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(module)s:%(lineno)d: %(message)s") +__logger_formatter = logging.Formatter("%(asctime)s - %(levelname)s - Relay - %(module)s:%(lineno)d: %(message)s") __logger_formatter.default_time_format = "%Y-%m-%d %H:%M:%S" __logger_formatter.default_msec_format = "%s.%03d" __logger_channel.setFormatter(__logger_formatter) From 8f73060c74e86f77072039681093dd0946e0a021 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 13 Mar 2022 15:55:03 +0000 Subject: [PATCH 165/234] monitor: Move configuration into separate file This will allow for running more than one monitor without editing actual source. --- contrib/monitor/index.html | 2 ++ contrib/monitor/js/eddn-config.js | 22 +++++++++++++++++ contrib/monitor/js/eddn.js | 40 ++++++++++--------------------- 3 files changed, 36 insertions(+), 28 deletions(-) create mode 100644 contrib/monitor/js/eddn-config.js diff --git a/contrib/monitor/index.html b/contrib/monitor/index.html index 9f16a47..0b7c365 100644 --- a/contrib/monitor/index.html +++ b/contrib/monitor/index.html @@ -400,6 +400,8 @@ + + diff --git a/contrib/monitor/js/eddn-config.js b/contrib/monitor/js/eddn-config.js new file mode 100644 index 0000000..16b2a97 --- /dev/null +++ b/contrib/monitor/js/eddn-config.js @@ -0,0 +1,22 @@ +/* vim: wrapmargin=0 textwidth=0 tabstop=4 softtabstop=4 expandtab shiftwidth=4 + */ +var eddn_config = { + updateInterval: 60000, + + eddn_host: 'eddn.miggy.org', + monitorEndPoint: 'https://eddn.miggy.org/dev/monitor/', + + gatewayStats: '/dev/stats/', + + relayStats: '/dev/relay/stats/', + relayBottlePort: 9110, + + gateways: [ + 'eddn.miggy.org' + ], //TODO: Must find a way to bind them to monitor + + relays: [ + 'eddn.miggy.org' + ] //TODO: Must find a way to bind them to monitor + +}; diff --git a/contrib/monitor/js/eddn.js b/contrib/monitor/js/eddn.js index 79aab9a..7c18ecd 100644 --- a/contrib/monitor/js/eddn.js +++ b/contrib/monitor/js/eddn.js @@ -1,21 +1,5 @@ /* vim: wrapmargin=0 textwidth=0 tabstop=4 softtabstop=4 expandtab shiftwidth=4 */ -var updateInterval = 60000, - - monitorEndPoint = 'https://eddn.edcd.io:9091/', - - //gatewayBottlePort = 8080, - gatewayBottlePort = 4430, - relayBottlePort = 9090, - - gateways = [ - 'eddn.edcd.io' - ], //TODO: Must find a way to bind them to monitor - - relays = [ - 'eddn.edcd.io' - ]; //TODO: Must find a way to bind them to monitor - var stats = { 'gateways' : {}, 'relays' : {} @@ -435,11 +419,11 @@ var doUpdateSoftwares = function() */ $.ajax({ dataType: "json", - url: monitorEndPoint + 'getSoftwares/?dateStart=' + yesterday + '&dateEnd = ' + today, + url: eddn_config.monitorEndPoint + 'getSoftwares/?dateStart=' + yesterday + '&dateEnd = ' + today, success: function(softwaresTodayYesterday){ $.ajax({ dataType: "json", - url: monitorEndPoint + 'getTotalSoftwares/', + url: eddn_config.monitorEndPoint + 'getTotalSoftwares/', success: function(softwaresTotals){ // Might happen when nothing is received... if(softwaresTodayYesterday[yesterday] == undefined) @@ -524,7 +508,7 @@ var doUpdateSchemas = function() $.ajax({ dataType: "json", - url: monitorEndPoint + 'getSchemas/?dateStart=' + yesterday + '&dateEnd = ' + today, + url: eddn_config.monitorEndPoint + 'getSchemas/?dateStart=' + yesterday + '&dateEnd = ' + today, success: function(schemasTodayYesterday){ // Might happen when nothing is received... if(schemasTodayYesterday[yesterday] == undefined) @@ -534,7 +518,7 @@ var doUpdateSchemas = function() $.ajax({ dataType: "json", - url: monitorEndPoint + 'getTotalSchemas/', + url: eddn_config.monitorEndPoint + 'getTotalSchemas/', success: function(schemasTotals){ var chart = $('#schemas .chart').highcharts(), series = chart.get('schemas'); @@ -807,12 +791,12 @@ var start = function(){ // Grab gateways //gateways = gateways.sort(); - $.each(gateways, function(k, gateway){ + $.each(eddn_config.gateways, function(k, gateway){ gateway = gateway.replace('tcp://', ''); gateway = gateway.replace(':8500', ''); $("select[name=gateways]").append($('