From 9a8ca522e6ebed05c293ecf96a6f5633234f66ca Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 13 Jun 2022 10:33:13 +0100 Subject: [PATCH 01/15] docs/Developers: Fix two minor typos --- docs/Developers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index ef846ef..6936bbb 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -1,7 +1,7 @@ ## Introduction EDDN is a -[zermoq](https://zeromq.org/) service which allows players of the game +[zeromq](https://zeromq.org/) service which allows players of the game [Elite Dangerous](https://www.elitedangerous.com/), published by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so that interested listeners can receive a copy. @@ -15,7 +15,7 @@ representing this game data and then passes it on to any interested listeners. ## Sources There are two sources of game data, both provided by the publisher of the game, -Frontier Developerments. They are both explicitly approved for use by +Frontier Developments. They are both explicitly approved for use by third-party software. ### Journal Files From 912fa0e0647871e8b74d7188d5dfc0ce6ec7c8bd Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 16:39:51 +0300 Subject: [PATCH 02/15] WIP: FSSSignalDiscovered event support --- schemas/fsssignaldiscovered-README.md | 1 + schemas/fsssignaldiscovered-v1.0.json | 81 +++++++++++++++++++++++++++ src/eddn/conf/Settings.py | 3 + 3 files changed, 85 insertions(+) create mode 100644 schemas/fsssignaldiscovered-README.md create mode 100644 schemas/fsssignaldiscovered-v1.0.json diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md new file mode 100644 index 0000000..30404ce --- /dev/null +++ b/schemas/fsssignaldiscovered-README.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json new file mode 100644 index 0000000..51bd1f0 --- /dev/null +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -0,0 +1,81 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fsssignaldiscovered/1#", + "description" : "EDDN schema for FSSSignalDiscovered Journal events. Full documentation at https://github.com/EDCD/EDDN/tree/master/schemas/fsssignaldiscovered-README.md", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "event", "signals"], + "properties" : { + "event": { + "enum" : [ "FSSSignalDiscovered" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "signals": { + "type": "array", + "description": "Array of FSSSignalDiscovered events", + "minItems": 1, + "items": { + "type": "object", + "required": ["SystemAddress", "timestamp", "SignalName"], + "description": "Single FSSSignalDiscovered event", + "properties": { + "SystemAddress": { "type": "integer" }, + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "SignalName": { "type": "string" }, + "IsStation": { "type": "boolean" }, + "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, + + "patternProperties": { + "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } + } + } + } + + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index c09e2ad..0b3253a 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -81,6 +81,9 @@ class _Settings(object): "https://eddn.edcd.io/schemas/fssbodysignals/1" : "schemas/fssbodysignals-v1.0.json", "https://eddn.edcd.io/schemas/fssbodysignals/1/test" : "schemas/fssbodysignals-v1.0.json", + + "https://eddn.edcd.io/schemasfsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemasfsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ From 036e918fe93f5bb008323643ac23efea546c94c5 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 17:41:42 +0300 Subject: [PATCH 03/15] FSSSignalDiscovered: optional systemName and StarPos --- schemas/fsssignaldiscovered-v1.0.json | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 51bd1f0..834cbe1 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -70,7 +70,20 @@ } } } - } + }, + "systemName": { + "type" : "string", + "minLength" : 1, + "description": "Should be added by the sender" + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Should be added by the sender" + } + } } From 5386ed2c64dc7223e6a494f235d77801fc313ab9 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 18:34:07 +0300 Subject: [PATCH 04/15] FSSSignalDiscovered: add properties --- schemas/fsssignaldiscovered-v1.0.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 834cbe1..bb03da5 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -63,7 +63,11 @@ }, "SignalName": { "type": "string" }, "IsStation": { "type": "boolean" }, + "USSType": { "type": "string" }, "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, + "SpawningState": {"type": "string"}, + "SpawningFaction" : {"type": "string"}, + "ThreatLevel": {"type": "integer" }, "patternProperties": { "_Localised$" : { "$ref" : "#/definitions/disallowed" } From 6dbc9e392a96db0d432291e54ceb5702118fd074 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 19:22:05 +0300 Subject: [PATCH 05/15] FSSSignalDiscovered: don't pass `$USS_Type_MissionTarget;` USS_Type --- schemas/fsssignaldiscovered-v1.0.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index bb03da5..d5e4fcc 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -63,7 +63,12 @@ }, "SignalName": { "type": "string" }, "IsStation": { "type": "boolean" }, - "USSType": { "type": "string" }, + "USSType": { + "type": "string", + "not": { + "pattern": "^\\$USS_Type_MissionTarget;$" + } + }, "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, "SpawningState": {"type": "string"}, "SpawningFaction" : {"type": "string"}, From 03caeaa20c14ec93b89e718fa21e708159919517 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 21:22:01 +0300 Subject: [PATCH 06/15] FSSSignalDiscovered: initial documentation --- schemas/fsssignaldiscovered-README.md | 53 ++++++++++++++++++++++++++- schemas/fsssignaldiscovered-v1.0.json | 6 +-- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 30404ce..370006c 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -1 +1,52 @@ -TODO \ No newline at end of file +# EDDN FSSSignalDiscovered Schema + +## Introduction +Here we document how to take data from an ED `FSSSignalDiscovered` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +## Senders +The primary data source for this schema is the ED Journal event +`FSSSignalDiscovered`. + +### Batching +You MUST put several `FSSSignalDiscovered` events to an array `signals` which is key of `message`. Minimum size of +`signals` is 1 item. + +Do not make a request for every single event. + +Possible algorithm of batching: +1. If the event is FSSSignalDiscovered, store it to the temporal list and proceed to next event. +2. If the next event is also FSSSignalDiscovered, repeat 1. +3. If the next event is any other or there is no other event for more than 10 seconds, send the + temporal list in a single message to EDDN. + +### Elisions +You MUST remove the following key/value pairs from the data: + + - `TimeRemaining` key/value pair. + +You MUST refuse to send the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. + +### Augmentations +#### horizons flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### odyssey flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### systemName +You SHOULD add a `systemName` string containing the system name from the last `FSDJump`, `CarrierJump`, or `Location` +event There exists problem when you gets `FSSSignalDiscovered` before +`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. + +#### StarPos +You SHOULD add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. There exists problem when you gets `FSSSignalDiscovered` before +`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. + +## Receivers +Receivers should remember: `horizons`, `odyssey`, `systemName`, `StarPos` are optional key/value pairs, it means you +should not rely on it will appear in every EDDN event. diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index d5e4fcc..8d23066 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -76,10 +76,10 @@ "patternProperties": { "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } } } - } - }, + }, "systemName": { "type" : "string", "minLength" : 1, @@ -92,8 +92,6 @@ "maxItems" : 3, "description" : "Should be added by the sender" } - - } } }, From fb9904306b7b4177a2ef1fc0b41011d244d7a27a Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Sun, 7 Nov 2021 23:59:20 +0300 Subject: [PATCH 07/15] FSSSignalDiscovered doc: add examples --- schemas/fsssignaldiscovered-README.md | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 370006c..5ada0e9 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -50,3 +50,73 @@ last `FSDJump`, `CarrierJump`, or `Location` event. There exists problem when yo ## Receivers Receivers should remember: `horizons`, `odyssey`, `systemName`, `StarPos` are optional key/value pairs, it means you should not rely on it will appear in every EDDN event. + +## Examples +This is a few example of messages that passes current `FSSSignalDiscovered` schema. +1. A message without `horizons`, `odyssey`, `systemName`, `StarPos` fields. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"an software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "event":"FSSSignalDiscovered", + "signals":[ + { + "timestamp":"2021-07-30T19:03:08Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1774711381, + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + } + ] + } +} +``` + +2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"an software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "event":"FSSSignalDiscovered", + "signals":[ + { + "timestamp":"2021-07-30T19:03:08Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1774711381, + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + }, + { + "timestamp":"2020-12-31T14:14:01Z", + "event":"FSSSignalDiscovered", + "SystemAddress":216054883492, + "SignalName":"$USS_NonHumanSignalSource;", + "USSType":"$USS_Type_NonHuman;", + "SpawningState":"$FactionState_None;", + "SpawningFaction":"$faction_none;", + "ThreatLevel":5 + } + ], + "StarPos": [ + 8.1875, + 124.21875, + -38.5 + ], + "StarSystem": "HIP 56186", + "horizons": true, + "odyssey": true + } +} +``` From 732de9bdd635f22a56e76220f0f5546c0cb0abf8 Mon Sep 17 00:00:00 2001 From: norohind <60548839+norohind@users.noreply.github.com> Date: Mon, 8 Nov 2021 00:00:39 +0300 Subject: [PATCH 08/15] FSSSignalDiscovered doc --- schemas/fsssignaldiscovered-README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 5ada0e9..f07d8ae 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -78,7 +78,7 @@ This is a few example of messages that passes current `FSSSignalDiscovered` sche } ``` -2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields. +2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields which says it sent from Odyssey. ```json { "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", From 329d59831fe4b00ff73979ce31b9d9502d8168b3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 14 Jun 2022 13:59:13 +0100 Subject: [PATCH 09/15] schemas/fsssignaldiscovered: Rework based on in-game findings 1. Unix, not DOS, line-endings. 2. `message`->`timestamp` instead of per signal. 3. `message`->`SystemAddress` instead of per signal. 4. `StarSystem` and `StarPos` augmentations now mandatory. --- schemas/fsssignaldiscovered-v1.0.json | 204 +++++++++++++------------- 1 file changed, 103 insertions(+), 101 deletions(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 8d23066..f649e2f 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -1,101 +1,103 @@ -{ - "$schema" : "http://json-schema.org/draft-04/schema#", - "id" : "https://eddn.edcd.io/schemas/fsssignaldiscovered/1#", - "description" : "EDDN schema for FSSSignalDiscovered Journal events. Full documentation at https://github.com/EDCD/EDDN/tree/master/schemas/fsssignaldiscovered-README.md", - "type" : "object", - "additionalProperties" : false, - "required": [ "$schemaRef", "header", "message" ], - "properties": { - "$schemaRef": { - "type" : "string" - }, - "header": { - "type" : "object", - "additionalProperties" : true, - "required" : [ "uploaderID", "softwareName", "softwareVersion" ], - "properties" : { - "uploaderID": { - "type" : "string" - }, - "softwareName": { - "type" : "string" - }, - "softwareVersion": { - "type" : "string" - }, - "gatewayTimestamp": { - "type" : "string", - "format" : "date-time", - "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." - } - } - }, - "message": { - "type" : "object", - "description" : "Contains all properties from the listed events in the client's journal minus Localised strings and the properties marked below as 'disallowed'", - "additionalProperties" : false, - "required" : [ "event", "signals"], - "properties" : { - "event": { - "enum" : [ "FSSSignalDiscovered" ] - }, - "horizons": { - "type" : "boolean", - "description" : "Whether the sending Cmdr has a Horizons pass." - }, - "odyssey": { - "type" : "boolean", - "description" : "Whether the sending Cmdr has an Odyssey expansion." - }, - "signals": { - "type": "array", - "description": "Array of FSSSignalDiscovered events", - "minItems": 1, - "items": { - "type": "object", - "required": ["SystemAddress", "timestamp", "SignalName"], - "description": "Single FSSSignalDiscovered event", - "properties": { - "SystemAddress": { "type": "integer" }, - "timestamp": { - "type" : "string", - "format" : "date-time" - }, - "SignalName": { "type": "string" }, - "IsStation": { "type": "boolean" }, - "USSType": { - "type": "string", - "not": { - "pattern": "^\\$USS_Type_MissionTarget;$" - } - }, - "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, - "SpawningState": {"type": "string"}, - "SpawningFaction" : {"type": "string"}, - "ThreatLevel": {"type": "integer" }, - - "patternProperties": { - "_Localised$" : { "$ref" : "#/definitions/disallowed" } - } - } - } - }, - "systemName": { - "type" : "string", - "minLength" : 1, - "description": "Should be added by the sender" - }, - "StarPos": { - "type" : "array", - "items" : { "type": "number" }, - "minItems" : 3, - "maxItems" : 3, - "description" : "Should be added by the sender" - } - } - } - }, - "definitions": { - "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } - } -} +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fsssignaldiscovered/1#", + "description" : "EDDN schema for FSSSignalDiscovered Journal events. Full documentation at https://github.com/EDCD/EDDN/tree/master/schemas/fsssignaldiscovered-README.md", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "event", "timestamp", "SystemAddress", "StarSystem", "StarPos", "signals"], + "properties" : { + "event": { + "enum" : [ "FSSSignalDiscovered" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "SystemAddress": { + "type": "integer" + }, + "signals": { + "type": "array", + "description": "Array of FSSSignalDiscovered events", + "minItems": 1, + "items": { + "type": "object", + "required": ["SignalName"], + "description": "Single FSSSignalDiscovered event", + "properties": { + "SignalName": { "type": "string" }, + "IsStation": { "type": "boolean" }, + "USSType": { + "type": "string", + "not": { + "pattern": "^\\$USS_Type_MissionTarget;$" + } + }, + "TimeRemaining": {"$ref" : "#/definitions/disallowed"}, + "SpawningState": {"type": "string"}, + "SpawningFaction" : {"type": "string"}, + "ThreatLevel": {"type": "integer" }, + + "patternProperties": { + "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } + } + } + }, + "StarSystem": { + "type" : "string", + "minLength" : 1, + "description": "Should be added by the sender" + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Should be added by the sender" + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} From 86f56eaac2bc5e12574bb857c3300e13be4745e9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 14 Jun 2022 14:01:36 +0100 Subject: [PATCH 10/15] schemas/fsssignaldiscovered README: Reworked for changed schema * Purely event-based batching, no timer. * Outline Horizons/Odyssey differences. * Point out that USS are manually scanned, so different in time nature. * StarSystem renamed from systemName. * StarSystem and StarPos now mandatory. --- schemas/fsssignaldiscovered-README.md | 295 +++++++++++++++----------- 1 file changed, 173 insertions(+), 122 deletions(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index f07d8ae..6665b22 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -1,122 +1,173 @@ -# EDDN FSSSignalDiscovered Schema - -## Introduction -Here we document how to take data from an ED `FSSSignalDiscovered` Journal -Event and properly structure it for sending to EDDN. - -Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general -documentation for a schema such as this. - -## Senders -The primary data source for this schema is the ED Journal event -`FSSSignalDiscovered`. - -### Batching -You MUST put several `FSSSignalDiscovered` events to an array `signals` which is key of `message`. Minimum size of -`signals` is 1 item. - -Do not make a request for every single event. - -Possible algorithm of batching: -1. If the event is FSSSignalDiscovered, store it to the temporal list and proceed to next event. -2. If the next event is also FSSSignalDiscovered, repeat 1. -3. If the next event is any other or there is no other event for more than 10 seconds, send the - temporal list in a single message to EDDN. - -### Elisions -You MUST remove the following key/value pairs from the data: - - - `TimeRemaining` key/value pair. - -You MUST refuse to send the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. - -### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### systemName -You SHOULD add a `systemName` string containing the system name from the last `FSDJump`, `CarrierJump`, or `Location` -event There exists problem when you gets `FSSSignalDiscovered` before -`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. - -#### StarPos -You SHOULD add a `StarPos` array containing the system co-ordinates from the -last `FSDJump`, `CarrierJump`, or `Location` event. There exists problem when you gets `FSSSignalDiscovered` before -`FSDJump`, `CarrierJump`, or `Location` event, so you MUST cross-check it with `SystemAddress` or don't include at all. - -## Receivers -Receivers should remember: `horizons`, `odyssey`, `systemName`, `StarPos` are optional key/value pairs, it means you -should not rely on it will appear in every EDDN event. - -## Examples -This is a few example of messages that passes current `FSSSignalDiscovered` schema. -1. A message without `horizons`, `odyssey`, `systemName`, `StarPos` fields. -```json -{ - "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", - "header":{ - "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", - "softwareName":"an software", - "softwareVersion":"a version", - "uploaderID":"an uploader" - }, - "message":{ - "event":"FSSSignalDiscovered", - "signals":[ - { - "timestamp":"2021-07-30T19:03:08Z", - "event":"FSSSignalDiscovered", - "SystemAddress":1774711381, - "SignalName":"EXPLORER-CLASS X2X-74M", - "IsStation":true - } - ] - } -} -``` - -2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields which says it sent from Odyssey. -```json -{ - "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", - "header":{ - "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", - "softwareName":"an software", - "softwareVersion":"a version", - "uploaderID":"an uploader" - }, - "message":{ - "event":"FSSSignalDiscovered", - "signals":[ - { - "timestamp":"2021-07-30T19:03:08Z", - "event":"FSSSignalDiscovered", - "SystemAddress":1774711381, - "SignalName":"EXPLORER-CLASS X2X-74M", - "IsStation":true - }, - { - "timestamp":"2020-12-31T14:14:01Z", - "event":"FSSSignalDiscovered", - "SystemAddress":216054883492, - "SignalName":"$USS_NonHumanSignalSource;", - "USSType":"$USS_Type_NonHuman;", - "SpawningState":"$FactionState_None;", - "SpawningFaction":"$faction_none;", - "ThreatLevel":5 - } - ], - "StarPos": [ - 8.1875, - 124.21875, - -38.5 - ], - "StarSystem": "HIP 56186", - "horizons": true, - "odyssey": true - } -} -``` +# EDDN FSSSignalDiscovered Schema + +## Introduction +Here we document how to take data from an ED `FSSSignalDiscovered` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +## Senders +The only data source for this schema is the ED Journal event +`FSSSignalDiscovered`. + +### Batching +You MUST coalesce contiguouys runs of `FSSSignalDiscovered` events into a +single `signals` array in the message. Minimum size of `signals` is 1 item. + +Do not make a request for every single event other than where they occur +singly (such as when a player utilises the FSS to zoom into USS individually). + +Suggested algorithm for batching: + +1. You will need to track the current location from `Location`, `FSDJump` and + `CarrierJump` events. This is in order to add the top-level augmentation + of `StarSystem` (system name) and `StarPos`. You will need to record: + 1. `SystemAddress` - for cross-checking. + 2. `StarSystem` - name of the star system. + 3. `StarPos` - the galactic co-ordinates of the system. +2. If the event is `FSSSignalDiscovered`, store it to the temporal list. +3. If the event is any other, then: + 1. check if it is `Location`, `FSDJump` or `CarrierJump` - if so you should + use this new location in the message. + 2. If it is not one of those events then you should use the tracked + location from the prior such event. + + Now construct the full `fsssignaldiscovered` schema message using the + tracked location and the stored list of events. *You **MUST** check that + the `SystemAddress` for each `FSSSignalDiscovered` event matches the + tracked location.* If there is a mis-match then drop that event. +4. Each batch of signals will be contiguous and thus should have the same + timestamp, perhaps +/- 1 second at the ends. As such you only need one + single `timestamp` in the `message` and should use that of the first event + you include. + +Point 3i/ii above are because in the current (3.8.0.406) Horizons client the +`FSSSignalDiscovered` events arrive after `Location`/`FSDJump`/`CarrierJump`, +but in the current (4.0.0.1302) Odyssey client they arrive before such events. + +Thus, in Horizons you use the last-tracked location, but in Odyssey you use +the "just arrived" location. + +Manually FSS-scanned USS type signals will come in one by one, possibly with +other events between them (such as `Music` due to zooming in/out in FSS). +There is no need to attempt batching these together if separated by other +events, even though you'll be using the `timestamp` of the first on the +message, despite the actual time-line being dependent on how quickly the +player scans them. + +This batching is more concerned with not causing an EDDN message per event +upon entry into a system. + +### Elisions +You MUST remove the following key/value pairs from the data: + + - `TimeRemaining` key/value pair (will be present on USS). This has a slight + PII nature and is also very ephemeral. + +You MUST drop the whole `FSSSignalDiscovered` event if `USSType` key +has `$USS_Type_MissionTarget;` value. Only the Cmdr with the mission has any +use of these. There's not even a statistical use. + +### Augmentations +#### horizons flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### odyssey flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### StarSystem +You **MUST** add a `StarSystem` string containing the system name from the last +tracked location. You **MUST** cross-check each `FSSSignalDiscovered` +->`SystemAddress` value to ensure it matches. If it does not, you **MUST** +drop the event. + +#### StarPos +You **MUST** add a `StarPos` array containing the system co-ordinates from the +tracked location. You **MUST** cross-check each `FSSSignalDiscovered` +->`SystemAddress` value to ensure it matches. If it does not, you **MUST** +drop the event. + +## Receivers +### Augmentations are 'SHOULD', not 'MUST' +Receivers should remember that `horizons`, `odyssey`, `StarSystem`, `StarPos` +are optional key/value pairs. You **SHOULD NOT** rely on them being present +in any given event. + +### Duplicate messages from 'busy' systems +When a system is particularly full of signals, such as when many Fleet Carriers +are present, it has been observed that the game repeats the identical +sequence of `FSSSignalDiscovered` events. So you might receive what looks like +a duplicate message, other than the timestamp (if the timestamp is the same +then the EDDN Relay should drop the duplicate). + +## Examples +This is a few example of messages that passes current `FSSSignalDiscovered` schema. +1. A message without `horizons` or `odyssey` augmentations. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"a software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "timestamp":"2021-11-06T22:48:42Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1774711381, + "signals":[ + { + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + } + ], + "StarSystem":"HR 1185", + "StarPos": [ + -64.66, -148.94, -330.41 + ] + } +} +``` + +2. A message with `horizons`, `odyssey`, `systemName`, `StarPos` fields which says it sent from Odyssey. +```json +{ + "$schemaRef":"https://eddn.edcd.io/schemas/fsssignaldiscovered/1", + "header":{ + "gatewayTimestamp":"2021-11-06T22:48:43.483147Z", + "softwareName":"a software", + "softwareVersion":"a version", + "uploaderID":"an uploader" + }, + "message":{ + "timestamp":"2021-11-06T22:48:42Z", + "event":"FSSSignalDiscovered", + "SystemAddress":1350507186531, + "signals":[ + { + "event":"FSSSignalDiscovered", + "SignalName":"EXPLORER-CLASS X2X-74M", + "IsStation":true + }, + { + "event":"FSSSignalDiscovered", + "SignalName":"$USS_NonHumanSignalSource;", + "USSType":"$USS_Type_NonHuman;", + "SpawningState":"$FactionState_None;", + "SpawningFaction":"$faction_none;", + "ThreatLevel":5 + } + ], + "StarPos": [ + 8.1875, + 124.21875, + -38.5 + ], + "StarSystem": "HIP 56186", + "horizons": true, + "odyssey": true + } +} +``` From d33c752c36ffaf76ef68c991489c1e6ed1dd5510 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 14 Jun 2022 14:03:49 +0100 Subject: [PATCH 11/15] schemas/fsssignaldiscovered: README: Remove timestamp/SystemAddress from each signal --- schemas/fsssignaldiscovered-README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index 6665b22..f5432dd 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -69,6 +69,9 @@ You MUST drop the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. Only the Cmdr with the mission has any use of these. There's not even a statistical use. +Because we only have a `message` level `timestamp` and `SystemAddress` these +should be removed from each member of the `signals` array. + ### Augmentations #### horizons flag You SHOULD add this key/value pair, using the value from the `LoadGame` event. From 3dbfd181886f32c8c81aebf3d51460188fbe6f54 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 15 Jun 2022 11:52:25 +0100 Subject: [PATCH 12/15] schemas/fsssignaldiscovered: Rework for per-signal timestamp & misc. clarifications * Per-signal `timestamp` due to the nature of USS signals. * Keeping `message`->`timestamp` so any Listener relying on it being there isn't tripped up (it's present in all other schemas). * Specifiy that each `signals` member should have `event` key/value pair removed. * Mandate removal of `SystemAddress` from each `signals` member. * Receivers: Only `horizons` and `odyssey` augmentations are optional. * Updated examples for these changes. --- schemas/fsssignaldiscovered-README.md | 35 ++++++++++++++++++--------- schemas/fsssignaldiscovered-v1.0.json | 7 +++++- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/schemas/fsssignaldiscovered-README.md b/schemas/fsssignaldiscovered-README.md index f5432dd..ddbb9aa 100644 --- a/schemas/fsssignaldiscovered-README.md +++ b/schemas/fsssignaldiscovered-README.md @@ -12,11 +12,12 @@ The only data source for this schema is the ED Journal event `FSSSignalDiscovered`. ### Batching -You MUST coalesce contiguouys runs of `FSSSignalDiscovered` events into a +You MUST coalesce contiguous runs of `FSSSignalDiscovered` events into a single `signals` array in the message. Minimum size of `signals` is 1 item. Do not make a request for every single event other than where they occur -singly (such as when a player utilises the FSS to zoom into USS individually). +singly (such as when a player utilises the FSS to zoom into USS individually, +if there is a different following event). Suggested algorithm for batching: @@ -29,18 +30,16 @@ Suggested algorithm for batching: 2. If the event is `FSSSignalDiscovered`, store it to the temporal list. 3. If the event is any other, then: 1. check if it is `Location`, `FSDJump` or `CarrierJump` - if so you should - use this new location in the message. + use this new location in the message for the augmentations. 2. If it is not one of those events then you should use the tracked - location from the prior such event. + location from the prior such event for the augmentations. Now construct the full `fsssignaldiscovered` schema message using the tracked location and the stored list of events. *You **MUST** check that the `SystemAddress` for each `FSSSignalDiscovered` event matches the tracked location.* If there is a mis-match then drop that event. -4. Each batch of signals will be contiguous and thus should have the same - timestamp, perhaps +/- 1 second at the ends. As such you only need one - single `timestamp` in the `message` and should use that of the first event - you include. +4. Use the `timestamp` of the first signal in the batch as the top-level + `timestamp` in the `message` object. Point 3i/ii above are because in the current (3.8.0.406) Horizons client the `FSSSignalDiscovered` events arrive after `Location`/`FSDJump`/`CarrierJump`, @@ -60,6 +59,10 @@ This batching is more concerned with not causing an EDDN message per event upon entry into a system. ### Elisions +Remove the `event` key/pair from each member of the `signals` array. Including +this would be redundant as by definition we're sending `FSSSignalDiscovered` +events on this schema. + You MUST remove the following key/value pairs from the data: - `TimeRemaining` key/value pair (will be present on USS). This has a slight @@ -69,8 +72,13 @@ You MUST drop the whole `FSSSignalDiscovered` event if `USSType` key has `$USS_Type_MissionTarget;` value. Only the Cmdr with the mission has any use of these. There's not even a statistical use. -Because we only have a `message` level `timestamp` and `SystemAddress` these -should be removed from each member of the `signals` array. +Because of the location cross-check the `SystemAddress` is in the top-level +`message` object, and thus you **MUST** remove such from each signal in the +array. + +Do **NOT** remove the `timestamp` from each signal in the array. Whilst these +should be identical for a "just logged in or arrived in system" set of signals, +this is not true of manually FSS scanned USS signals. ### Augmentations #### horizons flag @@ -93,7 +101,7 @@ drop the event. ## Receivers ### Augmentations are 'SHOULD', not 'MUST' -Receivers should remember that `horizons`, `odyssey`, `StarSystem`, `StarPos` +Receivers should remember that `horizons` and `odyssey` augmentations are optional key/value pairs. You **SHOULD NOT** rely on them being present in any given event. @@ -122,6 +130,7 @@ This is a few example of messages that passes current `FSSSignalDiscovered` sche "SystemAddress":1774711381, "signals":[ { + "timestamp":"2021-11-06T22:48:42Z", "SignalName":"EXPLORER-CLASS X2X-74M", "IsStation":true } @@ -150,11 +159,13 @@ This is a few example of messages that passes current `FSSSignalDiscovered` sche "SystemAddress":1350507186531, "signals":[ { + "timestamp":"2021-11-06T22:48:42Z", "event":"FSSSignalDiscovered", "SignalName":"EXPLORER-CLASS X2X-74M", "IsStation":true }, - { + { + "timestamp":"2021-11-06T22:48:42Z", "event":"FSSSignalDiscovered", "SignalName":"$USS_NonHumanSignalSource;", "USSType":"$USS_Type_NonHuman;", diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index f649e2f..2664989 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -49,7 +49,8 @@ }, "timestamp": { "type" : "string", - "format" : "date-time" + "format" : "date-time", + "description" : "Duplicate of the first signal's timestamp, for commonality with other schemas." }, "SystemAddress": { "type": "integer" @@ -63,6 +64,10 @@ "required": ["SignalName"], "description": "Single FSSSignalDiscovered event", "properties": { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, "SignalName": { "type": "string" }, "IsStation": { "type": "boolean" }, "USSType": { From 5d945297e14572531e2a7113d5920096d2de154d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 15 Jun 2022 12:42:53 +0100 Subject: [PATCH 13/15] schemas/fsssignaldiscovered: `timestamp` required per signal --- schemas/fsssignaldiscovered-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/fsssignaldiscovered-v1.0.json b/schemas/fsssignaldiscovered-v1.0.json index 2664989..18f8fef 100644 --- a/schemas/fsssignaldiscovered-v1.0.json +++ b/schemas/fsssignaldiscovered-v1.0.json @@ -61,7 +61,7 @@ "minItems": 1, "items": { "type": "object", - "required": ["SignalName"], + "required": ["timestamp", "SignalName"], "description": "Single FSSSignalDiscovered event", "properties": { "timestamp": { From 56b4e20238479251c4dcd2d62d0b48a033ff9d9a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 15 Jun 2022 14:03:02 +0100 Subject: [PATCH 14/15] Settings.py: Add missing `/` for `fsssignaldiscovered/1` schema --- src/eddn/conf/Settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 0b3253a..28e9b21 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -82,8 +82,8 @@ class _Settings(object): "https://eddn.edcd.io/schemas/fssbodysignals/1" : "schemas/fssbodysignals-v1.0.json", "https://eddn.edcd.io/schemas/fssbodysignals/1/test" : "schemas/fssbodysignals-v1.0.json", - "https://eddn.edcd.io/schemasfsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", - "https://eddn.edcd.io/schemasfsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemas/fsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", + "https://eddn.edcd.io/schemas/fsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ From ff83ede9480f2da8b7ebe33ece4f5d4604445e65 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 16 Jun 2022 13:27:11 +0100 Subject: [PATCH 15/15] Gateway: Remove all form-encoded support This causes issues, at the least, with compressed messages that 'look' like they decompressed body is form-encoded. 18385 messages in the last month rejected due to this. No actually valid form-encoded messages in that time frame. --- docs/Developers.md | 7 +++---- src/eddn/Gateway.py | 35 +---------------------------------- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index 6936bbb..66aa1eb 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -127,14 +127,13 @@ The body of an EDDN message is a JSON object in UTF-8 encoding. If you do not compress this body then you MUST set a `Content-Type` header of `applicaton/json`. -For historical reasons URL form-encoded data *is* supported, **but this is -deprecated and no new software should attempt this method**. We -purposefully do not further document the exact format for this. - You *MAY* use gzip compression on the body of the message, but it is not required. If you do compress the body then you **MUST* send a `Content-Type` header of `gzip` instead of `application/json`. +**Due to issues when messages are compressed, form-encoded data is NO LONGER +SUPPORTED as of 2022-06-16.** + You should be prepared to handle all scenarios where sending of a message fails: diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 19912ed..0524478 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -9,7 +9,6 @@ import gevent import hashlib import logging import simplejson -import urlparse import zlib import zmq.green as zmq from datetime import datetime @@ -167,42 +166,10 @@ def get_decompressed_message(): message_body = zlib.decompress(request.body.read(), -15) logger.debug('Resulting message_body:\n%s\n' % (message_body)) - # At this point, we're not sure whether we're dealing with a straight - # un-encoded POST body, or a form-encoded POST. Attempt to parse the - # body. If it's not form-encoded, this will return an empty dict. - form_enc_parsed = urlparse.parse_qs(message_body) - if form_enc_parsed: - logger.info('Request is form-encoded, compressed, from %s' % (get_remote_address())) - # This is a form-encoded POST. The value of the data attrib will - # be the body we're looking for. - try: - message_body = form_enc_parsed['data'][0] - - except (KeyError, IndexError): - logger.error('form-encoded, compressed, upload did not contain a "data" key. From %s', get_remote_address()) - raise MalformedUploadError( - "No 'data' POST key/value found. Check your POST key " - "name for spelling, and make sure you're passing a value." - ) - - else: - logger.debug('Request is *NOT* form-encoded') - else: logger.debug('Content-Encoding indicates *not* compressed...') - # Uncompressed request. Bottle handles all of the parsing of the - # POST key/vals, or un-encoded body. - data_key = request.forms.get('data') - if data_key: - logger.info('Request is form-encoded, uncompressed, from %s' % (get_remote_address())) - # This is a form-encoded POST. Support the silly people. - message_body = data_key - - else: - logger.debug('Plain POST request detected...') - # This is a non form-encoded POST body. - message_body = request.body.read() + message_body = request.body.read() return message_body