From 4f3225c3af604f6bbdd2c5380dc0c671766c62f0 Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Thu, 27 Jan 2022 11:10:59 +0000 Subject: [PATCH 01/89] Add Approachsettlement --- schemas/approachsettlement-README.md | 33 +++++++++++ schemas/approachsettlement-v1.0.json | 89 ++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 schemas/approachsettlement-README.md create mode 100644 schemas/approachsettlement-v1.0.json diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md new file mode 100644 index 0000000..e4af2af --- /dev/null +++ b/schemas/approachsettlement-README.md @@ -0,0 +1,33 @@ +# EDDN ApproachSettlement Schema + +## Introduction +Here we document how to take data from an ED `FSSAllBodiesFound` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +## Senders +The primary data source for this schema is the ED Journal event +`ApproachSettlement`. + +### Key Renames +Many of the key names have a different case defined in this schema, make +sure you are renaming them as appropriate. + +### Elisions +None + +### Augmentations +#### horizons flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### odyssey flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### StarSystem + +You MUST add a StarSystem key/value pair representing the name of the system this event occurred in. Source this from either Location, FSDJump or CarrierJump as appropriate. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json new file mode 100644 index 0000000..958d5d3 --- /dev/null +++ b/schemas/approachsettlement-v1.0.json @@ -0,0 +1,89 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/approachsettlement/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the event key, Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "StarSystem", "StarPos", "SystemAddress", "Name", "MarketID", "BodyID", "BodyName", "Latitude", "Longitude" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "StarSystem": { + "type" : "string", + "minLength" : 1 + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender" + }, + "SystemAddress": { + "type" : "integer", + }, + "Name" : { + "type" : "string", + "description" : "Name of settlement" + }, + "MarketID": { + "type" : "integer" + }, + "BodyID": { + "type" : "integer" + }, + "BodyName": { + "type" : "string" + }, + "Latitude": { + "type" : "number" + }, + "Longitude": { + "type" : "number" + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} From 42742ae9e6a9b283a1bf5379ac250b2e3748fd5f Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Thu, 27 Jan 2022 11:30:57 +0000 Subject: [PATCH 02/89] Add back in event after discussion --- schemas/approachsettlement-v1.0.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json index 958d5d3..7f974a6 100644 --- a/schemas/approachsettlement-v1.0.json +++ b/schemas/approachsettlement-v1.0.json @@ -31,7 +31,7 @@ }, "message": { "type" : "object", - "description" : "Contains all properties from the listed events in the client's journal minus the event key, Localised strings and the properties marked below as 'disallowed'", + "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", "additionalProperties" : false, "required" : [ "timestamp", "StarSystem", "StarPos", "SystemAddress", "Name", "MarketID", "BodyID", "BodyName", "Latitude", "Longitude" ], "properties" : { @@ -39,6 +39,9 @@ "type" : "string", "format" : "date-time" }, + "event" : { + "enum" : [ "ApproachBody" ] + }, "horizons": { "type" : "boolean", "description" : "Whether the sending Cmdr has a Horizons pass." From 5c5ce32d09e2403f01e21fc363b1ac4145733efd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 27 Jan 2022 12:31:48 +0000 Subject: [PATCH 03/89] schemas/README: Preserve Journal event key:value, even if redundant --- schemas/README-EDDN-schemas.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 3de98f6..2773270 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -244,6 +244,10 @@ Each `message` object must have, at bare minimum: 2. At least one other key/value pair representing the data. In general there will be much more than this. Consult the [schemas and their documentation](./). +3. Where the data is sourced from a Journal event please do preserve the + event key and value. Yes, where we use an event-specific schema this + might seem redundant, but it might aid an EDDN listener in streamlining + their code, and it does no harm. Because the first versions of some schemas were defined when only the CAPI data was available, before Journal files existed, many of the key names chosen From e85e3d4b85f032d828bc05a1aa8f123ac4a1873f Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Thu, 27 Jan 2022 13:09:50 +0000 Subject: [PATCH 04/89] Fix tab issue, add description --- schemas/approachsettlement-v1.0.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json index 7f974a6..c5cfb2e 100644 --- a/schemas/approachsettlement-v1.0.json +++ b/schemas/approachsettlement-v1.0.json @@ -53,6 +53,7 @@ "StarSystem": { "type" : "string", "minLength" : 1 + "description" : "Must be added by the sender" }, "StarPos": { "type" : "array", @@ -67,17 +68,17 @@ "Name" : { "type" : "string", "description" : "Name of settlement" - }, + }, "MarketID": { "type" : "integer" }, - "BodyID": { + "BodyID": { "type" : "integer" }, "BodyName": { "type" : "string" }, - "Latitude": { + "Latitude": { "type" : "number" }, "Longitude": { From eba946bf99d5359bb4fef54e36abbd5ee31a5371 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 27 Jan 2022 18:22:07 +0000 Subject: [PATCH 05/89] Add .editorconfig to enforce some standards --- .editorconfig | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..977ec02 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# This is the project top-level .editorconfig +root = true + +# Defaults for all file types +[*] +# 4-space indents, no TABs +indent_style = space +tab_width = 4 +indent_size = tab + +# Hard-wrap at 120 columns +max_line_length = 119 + +# Unix EOL, single \n +end_of_line = lf + +# UTF-8 is the only sensible option, no BOM +charset = utf-8 + +# All files should have a final newline +insert_final_newline = true From 8ac2ac8ac603c315d69d65e7fbf7b822d120bf33 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 27 Jan 2022 18:23:28 +0000 Subject: [PATCH 06/89] Start a docs/Contributing.md * Call out the .editorconfig file --- docs/Contributing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/Contributing.md diff --git a/docs/Contributing.md b/docs/Contributing.md new file mode 100644 index 0000000..1eaadd3 --- /dev/null +++ b/docs/Contributing.md @@ -0,0 +1,13 @@ +# Contributing to the EDDN Project + +## Introduction + +This file is still mostly a stub. + +## Text format + +The project contains an `.editorconfig` file at its root. Please either ensure +your editor is taking note of those settings, or cross-check its contents +with the +[editorconfig documentation](https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties) +, and ensure your editor/IDE's settings match. From 3b7c002b51255f8b153babdbaa6183e09a598c93 Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Fri, 28 Jan 2022 08:14:45 +0000 Subject: [PATCH 07/89] Add schema to settings --- src/eddn/conf/Settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index faac1c0..af15e95 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -73,6 +73,9 @@ class _Settings(object): "https://eddn.edcd.io/schemas/navroute/1" : "schemas/navroute-v1.0.json", "https://eddn.edcd.io/schemas/navroute/1/test" : "schemas/navroute-v1.0.json", + + "https://eddn.edcd.io/schemas/approachsettlement/1" : "schemas/approachsettlement-v1.0.json", + "https://eddn.edcd.io/schemas/approachsettlement/1/test" : "schemas/approachsettlement-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ From 87a81aaa376c4f318d43082303569af59b05f9ab Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Fri, 28 Jan 2022 13:47:01 +0000 Subject: [PATCH 08/89] Fix comma typo in ApproachSettlement JSON and fix readme --- schemas/approachsettlement-README.md | 2 +- schemas/approachsettlement-v1.0.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md index e4af2af..44d72ca 100644 --- a/schemas/approachsettlement-README.md +++ b/schemas/approachsettlement-README.md @@ -1,7 +1,7 @@ # EDDN ApproachSettlement Schema ## Introduction -Here we document how to take data from an ED `FSSAllBodiesFound` Journal +Here we document how to take data from an ED `ApproachSettlement` Journal Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json index c5cfb2e..6685f29 100644 --- a/schemas/approachsettlement-v1.0.json +++ b/schemas/approachsettlement-v1.0.json @@ -52,7 +52,7 @@ }, "StarSystem": { "type" : "string", - "minLength" : 1 + "minLength" : 1, "description" : "Must be added by the sender" }, "StarPos": { @@ -63,7 +63,7 @@ "description" : "Must be added by the sender" }, "SystemAddress": { - "type" : "integer", + "type" : "integer" }, "Name" : { "type" : "string", From 5ff342cc8bf56451d2c605ddf6a2e33063d5b630 Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Fri, 28 Jan 2022 13:52:20 +0000 Subject: [PATCH 09/89] Add event to required list --- schemas/approachsettlement-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json index 6685f29..a656bb7 100644 --- a/schemas/approachsettlement-v1.0.json +++ b/schemas/approachsettlement-v1.0.json @@ -33,7 +33,7 @@ "type" : "object", "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", "additionalProperties" : false, - "required" : [ "timestamp", "StarSystem", "StarPos", "SystemAddress", "Name", "MarketID", "BodyID", "BodyName", "Latitude", "Longitude" ], + "required" : [ "timestamp", "event", "StarSystem", "StarPos", "SystemAddress", "Name", "MarketID", "BodyID", "BodyName", "Latitude", "Longitude" ], "properties" : { "timestamp": { "type" : "string", From 3690aab87eef04f37071fd45ca095e41317686eb Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Fri, 28 Jan 2022 15:10:36 +0000 Subject: [PATCH 10/89] Fix another typo in ApproachSettlement event tag --- schemas/approachsettlement-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json index a656bb7..571db77 100644 --- a/schemas/approachsettlement-v1.0.json +++ b/schemas/approachsettlement-v1.0.json @@ -40,7 +40,7 @@ "format" : "date-time" }, "event" : { - "enum" : [ "ApproachBody" ] + "enum" : [ "ApproachSettlement" ] }, "horizons": { "type" : "boolean", From 1678c467d7da3ef18907bc9fddbd33894662e9b6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 15:14:00 +0000 Subject: [PATCH 11/89] scripts/testing: Various additional inputs to test-sender --- .../gateway-responses/approachsettlement.json | 23 + .../commodity-just-over-50KiB.json | 2266 +++++++++++++++++ .../commodity-just-under-50KiB.json | 2206 ++++++++++++++++ .../commodity-smallest-message.json | 1 + .../gateway-responses/fssallbodiesfound.json | 18 + .../shipyard-smallest-message.json | 1 + 6 files changed, 4515 insertions(+) create mode 100644 scripts/testing/gateway-responses/approachsettlement.json create mode 100644 scripts/testing/gateway-responses/commodity-just-over-50KiB.json create mode 100644 scripts/testing/gateway-responses/commodity-just-under-50KiB.json create mode 100644 scripts/testing/gateway-responses/commodity-smallest-message.json create mode 100644 scripts/testing/gateway-responses/fssallbodiesfound.json create mode 100644 scripts/testing/gateway-responses/shipyard-smallest-message.json diff --git a/scripts/testing/gateway-responses/approachsettlement.json b/scripts/testing/gateway-responses/approachsettlement.json new file mode 100644 index 0000000..65f4906 --- /dev/null +++ b/scripts/testing/gateway-responses/approachsettlement.json @@ -0,0 +1,23 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/approachsettlement/1", + "header": { + "uploaderID": "from Athanasius Testing", + "softwareName": "Athanasius Testing script", + "softwareVersion": "v0.0.1" + }, + "message": { + "timestamp":"2021-10-14T12:37:54Z", + "event":"ApproachSettlement", + "Name":"Arnold Defence Base", + "MarketID":3915738368, + "SystemAddress":2381282543963, + "StarSystem": "Ix", + "BodyID":32, + "BodyName":"Ix 5 a a", + "Latitude":17.090912, + "Longitude":160.236679, + "StarPos": [ + -65.21875 , 7.75 , -111.03125 + ] + } +} diff --git a/scripts/testing/gateway-responses/commodity-just-over-50KiB.json b/scripts/testing/gateway-responses/commodity-just-over-50KiB.json new file mode 100644 index 0000000..c9df676 --- /dev/null +++ b/scripts/testing/gateway-responses/commodity-just-over-50KiB.json @@ -0,0 +1,2266 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/commodity/3", + "message": { + "timestamp": "2022-01-06T11:19:27Z", + "systemName": "LP 98-132", + "stationName": "Freeport", + "marketId": 128008448, + "commodities": [ + { + "name": "advancedcatalysers", + "meanPrice": 3039, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3756, + "demand": 936, + "demandBracket": 3 + }, + { + "name": "advancedmedicines", + "meanPrice": 1485, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1726, + "demand": 296, + "demandBracket": 3 + }, + { + "name": "advert1", + "meanPrice": 21542, + "buyPrice": 15267, + "stock": 0, + "stockBracket": 0, + "sellPrice": 15266, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aerialedenapple", + "meanPrice": 8331, + "buyPrice": 1660, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1659, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "agriculturalmedicines", + "meanPrice": 1231, + "buyPrice": 543, + "stock": 0, + "stockBracket": 0, + "sellPrice": 542, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "agronomictreatment", + "meanPrice": 3105, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 15596, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "alacarakmoskinart", + "meanPrice": 8899, + "buyPrice": 1858, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1857, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "albinoquechuamammoth", + "meanPrice": 9687, + "buyPrice": 2663, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2662, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "alexandrite", + "meanPrice": 217277, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 357530, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "algae", + "meanPrice": 356, + "buyPrice": 23, + "stock": 0, + "stockBracket": 0, + "sellPrice": 22, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "alieneggs", + "meanPrice": 25067, + "buyPrice": 19095, + "stock": 0, + "stockBracket": 0, + "sellPrice": 19094, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "altairianskin", + "meanPrice": 8432, + "buyPrice": 1262, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1261, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aluminium", + "meanPrice": 551, + "buyPrice": 1127, + "stock": 352, + "stockBracket": 2, + "sellPrice": 1045, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "alyabodilysoap", + "meanPrice": 8218, + "buyPrice": 1078, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1077, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientcasket", + "meanPrice": 16294, + "buyPrice": 5581, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5580, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientkey", + "meanPrice": 29931, + "buyPrice": 9760, + "stock": 0, + "stockBracket": 0, + "sellPrice": 9759, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientorb", + "meanPrice": 17415, + "buyPrice": 5967, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5966, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientrelic", + "meanPrice": 24962, + "buyPrice": 9126, + "stock": 0, + "stockBracket": 0, + "sellPrice": 9125, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancienttablet", + "meanPrice": 17415, + "buyPrice": 5967, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5966, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancienttotem", + "meanPrice": 20437, + "buyPrice": 7379, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7378, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancienturn", + "meanPrice": 14907, + "buyPrice": 3907, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3906, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "anduligafireworks", + "meanPrice": 8519, + "buyPrice": 1574, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1573, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "animalmeat", + "meanPrice": 1539, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1695, + "demand": 26, + "demandBracket": 1 + }, + { + "name": "animalmonitors", + "meanPrice": 537, + "buyPrice": 630, + "stock": 0, + "stockBracket": 0, + "sellPrice": 629, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "antiquejewellery", + "meanPrice": 183163, + "buyPrice": 158957, + "stock": 0, + "stockBracket": 0, + "sellPrice": 158949, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "anynacoffee", + "meanPrice": 9160, + "buyPrice": 2165, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2164, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "apavietii", + "meanPrice": 10362, + "buyPrice": 3386, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3385, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aquaponicsystems", + "meanPrice": 524, + "buyPrice": 321, + "stock": 0, + "stockBracket": 0, + "sellPrice": 320, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aroucaconventualsweets", + "meanPrice": 8737, + "buyPrice": 1557, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1556, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "articulationmotors", + "meanPrice": 7588, + "buyPrice": 4463, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4462, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "assaultplans", + "meanPrice": 26074, + "buyPrice": 3294, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3293, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "atmosphericextractors", + "meanPrice": 571, + "buyPrice": 331, + "stock": 0, + "stockBracket": 0, + "sellPrice": 330, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "autofabricators", + "meanPrice": 3827, + "buyPrice": 2685, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2684, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "azcancriformula42", + "meanPrice": 12440, + "buyPrice": 6106, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6105, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bakedgreebles", + "meanPrice": 8211, + "buyPrice": 1056, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1055, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "baltahsinevacuumkrill", + "meanPrice": 8479, + "buyPrice": 1502, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1501, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bankiamphibiousleather", + "meanPrice": 8338, + "buyPrice": 1138, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1137, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "basicmedicines", + "meanPrice": 493, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 963, + "demand": 904, + "demandBracket": 3 + }, + { + "name": "bastsnakegin", + "meanPrice": 8659, + "buyPrice": 1724, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1723, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "battleweapons", + "meanPrice": 7451, + "buyPrice": 4463, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4462, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bauxite", + "meanPrice": 1140, + "buyPrice": 10730, + "stock": 595, + "stockBracket": 1, + "sellPrice": 10270, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "beer", + "meanPrice": 430, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 799, + "demand": 1777, + "demandBracket": 3 + }, + { + "name": "belalansrayleather", + "meanPrice": 8519, + "buyPrice": 1605, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1604, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "benitoite", + "meanPrice": 149395, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 570449, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "bertrandite", + "meanPrice": 18817, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 21216, + "demand": 1047, + "demandBracket": 2 + }, + { + "name": "beryllium", + "meanPrice": 8243, + "buyPrice": 8041, + "stock": 2, + "stockBracket": 1, + "sellPrice": 7948, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "bioreducinglichen", + "meanPrice": 1204, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1703, + "demand": 5473, + "demandBracket": 3 + }, + { + "name": "biowaste", + "meanPrice": 358, + "buyPrice": 145, + "stock": 103, + "stockBracket": 1, + "sellPrice": 87, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "bismuth", + "meanPrice": 2441, + "buyPrice": 1607, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1606, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bluemilk", + "meanPrice": 10805, + "buyPrice": 3912, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3911, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bromellite", + "meanPrice": 30424, + "buyPrice": 70950, + "stock": 0, + "stockBracket": 0, + "sellPrice": 70946, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "buckyballbeermats", + "meanPrice": 7957, + "buyPrice": 8230, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8229, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "buildingfabricators", + "meanPrice": 2312, + "buyPrice": 1448, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1447, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cd75catcoffee", + "meanPrice": 9571, + "buyPrice": 2452, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2451, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "centaurimegagin", + "meanPrice": 10217, + "buyPrice": 3239, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3238, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ceramiccomposites", + "meanPrice": 415, + "buyPrice": 424, + "stock": 0, + "stockBracket": 0, + "sellPrice": 423, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ceremonialheiketea", + "meanPrice": 9251, + "buyPrice": 2161, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2160, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cetiaepyornisegg", + "meanPrice": 9769, + "buyPrice": 2741, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2740, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cetirabbits", + "meanPrice": 9079, + "buyPrice": 2068, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2067, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chameleoncloth", + "meanPrice": 9071, + "buyPrice": 1959, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1958, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chateaudeaegaeon", + "meanPrice": 8791, + "buyPrice": 1610, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1609, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chemicalwaste", + "meanPrice": 672, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 599, + "demand": 159, + "demandBracket": 3 + }, + { + "name": "cherbonesbloodcrystals", + "meanPrice": 16714, + "buyPrice": 10207, + "stock": 0, + "stockBracket": 0, + "sellPrice": 10206, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chieridanimarinepaste", + "meanPrice": 8450, + "buyPrice": 1426, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1425, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "classifiedexperimentalequipment", + "meanPrice": 11423, + "buyPrice": 4002, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4001, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "clothing", + "meanPrice": 546, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 963, + "demand": 955, + "demandBracket": 3 + }, + { + "name": "cmmcomposite", + "meanPrice": 5988, + "buyPrice": 3907, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3906, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cobalt", + "meanPrice": 3762, + "buyPrice": 13474, + "stock": 724, + "stockBracket": 1, + "sellPrice": 12870, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "coffee", + "meanPrice": 1499, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2303, + "demand": 527, + "demandBracket": 3 + }, + { + "name": "coltan", + "meanPrice": 6163, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7849, + "demand": 2063, + "demandBracket": 3 + }, + { + "name": "comercialsamples", + "meanPrice": 1815, + "buyPrice": 656, + "stock": 0, + "stockBracket": 0, + "sellPrice": 655, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "computercomponents", + "meanPrice": 776, + "buyPrice": 267, + "stock": 0, + "stockBracket": 0, + "sellPrice": 266, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "conductivefabrics", + "meanPrice": 709, + "buyPrice": 1937, + "stock": 55, + "stockBracket": 1, + "sellPrice": 1810, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "consumertechnology", + "meanPrice": 6690, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6883, + "demand": 0, + "demandBracket": 1 + }, + { + "name": "coolinghoses", + "meanPrice": 1886, + "buyPrice": 707, + "stock": 0, + "stockBracket": 0, + "sellPrice": 706, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "copper", + "meanPrice": 689, + "buyPrice": 1115, + "stock": 474, + "stockBracket": 2, + "sellPrice": 1042, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "coquimspongiformvictuals", + "meanPrice": 8077, + "buyPrice": 10924, + "stock": 0, + "stockBracket": 0, + "sellPrice": 10923, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cropharvesters", + "meanPrice": 2230, + "buyPrice": 1564, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1563, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cryolite", + "meanPrice": 12173, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 14970, + "demand": 1166, + "demandBracket": 3 + }, + { + "name": "crystallinespheres", + "meanPrice": 12216, + "buyPrice": 5438, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5437, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "damnacarapaces", + "meanPrice": 8120, + "buyPrice": 2925, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2924, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "datacore", + "meanPrice": 6791, + "buyPrice": 3907, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3906, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "deltaphoenicispalms", + "meanPrice": 8188, + "buyPrice": 1528, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1527, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "deuringastruffles", + "meanPrice": 9232, + "buyPrice": 2335, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2334, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "diagnosticsensor", + "meanPrice": 6727, + "buyPrice": 4463, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4462, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "diplomaticbag", + "meanPrice": 28625, + "buyPrice": 4265, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4264, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "disomacorn", + "meanPrice": 8134, + "buyPrice": 1053, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1052, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "domesticappliances", + "meanPrice": 740, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1216, + "demand": 575, + "demandBracket": 3 + }, + { + "name": "duradrives", + "meanPrice": 19356, + "buyPrice": 13002, + "stock": 0, + "stockBracket": 0, + "sellPrice": 13001, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "earthrelics", + "meanPrice": 15161, + "buyPrice": 14640, + "stock": 0, + "stockBracket": 0, + "sellPrice": 14639, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "eleuthermals", + "meanPrice": 8507, + "buyPrice": 1571, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1570, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "emergencypowercells", + "meanPrice": 2368, + "buyPrice": 1338, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1337, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "encripteddatastorage", + "meanPrice": 8313, + "buyPrice": 4049, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4048, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "encryptedcorrespondence", + "meanPrice": 8226, + "buyPrice": 3935, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3934, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "eraninpearlwhisky", + "meanPrice": 9040, + "buyPrice": 2058, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2057, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "eshuumbrellas", + "meanPrice": 9343, + "buyPrice": 2187, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2186, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "esusekucaviar", + "meanPrice": 9625, + "buyPrice": 2573, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2572, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ethgrezeteabuds", + "meanPrice": 10197, + "buyPrice": 3212, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3211, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "evacuationshelter", + "meanPrice": 522, + "buyPrice": 138, + "stock": 0, + "stockBracket": 0, + "sellPrice": 137, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "exhaustmanifold", + "meanPrice": 1873, + "buyPrice": 1009, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1008, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "explosives", + "meanPrice": 512, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2101, + "demand": 3964, + "demandBracket": 3 + }, + { + "name": "fish", + "meanPrice": 650, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1008, + "demand": 481, + "demandBracket": 2 + }, + { + "name": "foodcartridges", + "meanPrice": 265, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 767, + "demand": 1094, + "demandBracket": 3 + }, + { + "name": "fossilremnants", + "meanPrice": 11785, + "buyPrice": 7379, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7378, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "fruitandvegetables", + "meanPrice": 509, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1060, + "demand": 1637, + "demandBracket": 3 + }, + { + "name": "fujintea", + "meanPrice": 8597, + "buyPrice": 1418, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1417, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "galactictravelguide", + "meanPrice": 8627, + "buyPrice": 359, + "stock": 0, + "stockBracket": 0, + "sellPrice": 358, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "gallite", + "meanPrice": 11915, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 13829, + "demand": 1765, + "demandBracket": 3 + }, + { + "name": "gallium", + "meanPrice": 5203, + "buyPrice": 5165, + "stock": 11, + "stockBracket": 1, + "sellPrice": 5042, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "genebank", + "meanPrice": 59082, + "buyPrice": 46704, + "stock": 0, + "stockBracket": 0, + "sellPrice": 46701, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "geologicalequipment", + "meanPrice": 1886, + "buyPrice": 1186, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1185, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "geologicalsamples", + "meanPrice": 8313, + "buyPrice": 12144, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12143, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "gerasiangueuzebeer", + "meanPrice": 8215, + "buyPrice": 1067, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1066, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "giantirukamasnails", + "meanPrice": 9174, + "buyPrice": 2131, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2130, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "giantverrix", + "meanPrice": 12496, + "buyPrice": 5713, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5712, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "gold", + "meanPrice": 47610, + "buyPrice": 44874, + "stock": 2, + "stockBracket": 1, + "sellPrice": 44362, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "gomanyauponcoffee", + "meanPrice": 8921, + "buyPrice": 1843, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1842, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "goslarite", + "meanPrice": 5979, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8024, + "demand": 1290, + "demandBracket": 3 + }, + { + "name": "grain", + "meanPrice": 410, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 919, + "demand": 2980, + "demandBracket": 3 + }, + { + "name": "grandidierite", + "meanPrice": 197292, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 458141, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "haidneblackbrew", + "meanPrice": 8837, + "buyPrice": 1693, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1692, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "harmasilversearum", + "meanPrice": 9762, + "buyPrice": 4217, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4216, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "havasupaidreamcatcher", + "meanPrice": 14639, + "buyPrice": 8056, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8055, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hazardousenvironmentsuits", + "meanPrice": 570, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 915, + "demand": 4363, + "demandBracket": 3 + }, + { + "name": "heatsinkinterlink", + "meanPrice": 2100, + "buyPrice": 1161, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1160, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "heliostaticfurnaces", + "meanPrice": 434, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2010, + "demand": 537, + "demandBracket": 3 + }, + { + "name": "helvetitjpearls", + "meanPrice": 10450, + "buyPrice": 3507, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3506, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hip10175bushmeat", + "meanPrice": 9382, + "buyPrice": 2246, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2245, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hip41181squid", + "meanPrice": 8947, + "buyPrice": 1891, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1890, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hiporganophosphates", + "meanPrice": 8169, + "buyPrice": 1039, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1038, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hnshockmount", + "meanPrice": 1922, + "buyPrice": 1075, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1074, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "honestypills", + "meanPrice": 8860, + "buyPrice": 1686, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1685, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hr7221wheat", + "meanPrice": 8190, + "buyPrice": 1540, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1539, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hydrogenfuel", + "meanPrice": 113, + "buyPrice": 126, + "stock": 379, + "stockBracket": 1, + "sellPrice": 120, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "hydrogenperoxide", + "meanPrice": 3160, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3301, + "demand": 548, + "demandBracket": 3 + }, + { + "name": "indibourbon", + "meanPrice": 8806, + "buyPrice": 1638, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1637, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "indite", + "meanPrice": 11389, + "buyPrice": 10508, + "stock": 296, + "stockBracket": 1, + "sellPrice": 10238, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "indium", + "meanPrice": 5845, + "buyPrice": 5894, + "stock": 8, + "stockBracket": 1, + "sellPrice": 5824, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "insulatingmembrane", + "meanPrice": 10724, + "buyPrice": 10137, + "stock": 2, + "stockBracket": 2, + "sellPrice": 10021, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "iondistributor", + "meanPrice": 2363, + "buyPrice": 1338, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1337, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jadeite", + "meanPrice": 42383, + "buyPrice": 40631, + "stock": 0, + "stockBracket": 0, + "sellPrice": 40628, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jaquesquinentianstill", + "meanPrice": 13845, + "buyPrice": 7233, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7232, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jaradharrepuzzlebox", + "meanPrice": 16816, + "buyPrice": 10325, + "stock": 0, + "stockBracket": 0, + "sellPrice": 10324, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jarouarice", + "meanPrice": 8169, + "buyPrice": 2711, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2710, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jotunmookah", + "meanPrice": 8780, + "buyPrice": 1637, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1636, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "kachiriginleaches", + "meanPrice": 8227, + "buyPrice": 1359, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1358, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "kamitracigars", + "meanPrice": 12282, + "buyPrice": 5522, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5521, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "karetiicouture", + "meanPrice": 11582, + "buyPrice": 4748, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4747, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "karsukilocusts", + "meanPrice": 8543, + "buyPrice": 1583, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1582, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "kinagoinstruments", + "meanPrice": 13030, + "buyPrice": 6570, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6569, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "konggaale", + "meanPrice": 8310, + "buyPrice": 1578, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1577, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "korrokungpellets", + "meanPrice": 8067, + "buyPrice": 1190, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1189, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lanthanum", + "meanPrice": 8707, + "buyPrice": 6087, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6086, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "largeexplorationdatacash", + "meanPrice": 255841, + "buyPrice": 225210, + "stock": 0, + "stockBracket": 0, + "sellPrice": 225198, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lavianbrandy", + "meanPrice": 10365, + "buyPrice": 3614, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3613, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "leather", + "meanPrice": 435, + "buyPrice": 57, + "stock": 0, + "stockBracket": 0, + "sellPrice": 56, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "leestianeviljuice", + "meanPrice": 8220, + "buyPrice": 1086, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1085, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lepidolite", + "meanPrice": 771, + "buyPrice": 4193, + "stock": 1271, + "stockBracket": 1, + "sellPrice": 3999, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "lftvoidextractcoffee", + "meanPrice": 9554, + "buyPrice": 2506, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2505, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "liquidoxygen", + "meanPrice": 1474, + "buyPrice": 957, + "stock": 22, + "stockBracket": 1, + "sellPrice": 850, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "liquor", + "meanPrice": 879, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1331, + "demand": 453, + "demandBracket": 3 + }, + { + "name": "lithium", + "meanPrice": 1772, + "buyPrice": 1906, + "stock": 6, + "stockBracket": 1, + "sellPrice": 1855, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "lithiumhydroxide", + "meanPrice": 5673, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7350, + "demand": 99, + "demandBracket": 3 + }, + { + "name": "livehecateseaworms", + "meanPrice": 8737, + "buyPrice": 1899, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1898, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lowtemperaturediamond", + "meanPrice": 106353, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 293785, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "ltthypersweet", + "meanPrice": 8054, + "buyPrice": 1233, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1232, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m3_tissuesample_membrane", + "meanPrice": 18352, + "buyPrice": 12075, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12074, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m3_tissuesample_mycelium", + "meanPrice": 32825, + "buyPrice": 22634, + "stock": 0, + "stockBracket": 0, + "sellPrice": 22632, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m3_tissuesample_spores", + "meanPrice": 6031, + "buyPrice": 3380, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3379, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m_tissuesample_fluid", + "meanPrice": 6031, + "buyPrice": 3380, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3379, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m_tissuesample_nerves", + "meanPrice": 32825, + "buyPrice": 22634, + "stock": 0, + "stockBracket": 0, + "sellPrice": 22632, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m_tissuesample_soft", + "meanPrice": 17216, + "buyPrice": 11290, + "stock": 0, + "stockBracket": 0, + "sellPrice": 11289, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "magneticemittercoil", + "meanPrice": 1357, + "buyPrice": 656, + "stock": 0, + "stockBracket": 0, + "sellPrice": 655, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "marinesupplies", + "meanPrice": 4135, + "buyPrice": 2745, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2744, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mechucoshightea", + "meanPrice": 8846, + "buyPrice": 1709, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1708, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "medbstarlube", + "meanPrice": 8191, + "buyPrice": 1953, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1952, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "medicaldiagnosticequipment", + "meanPrice": 3075, + "buyPrice": 1779, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1778, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "metaalloys", + "meanPrice": 195453, + "buyPrice": 162685, + "stock": 0, + "stockBracket": 0, + "sellPrice": 162676, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "methaneclathrate", + "meanPrice": 1650, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1740, + "demand": 2873, + "demandBracket": 3 + }, + { + "name": "methanolmonohydratecrystals", + "meanPrice": 2478, + "buyPrice": 1290, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1289, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "microcontrollers", + "meanPrice": 5590, + "buyPrice": 3612, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3611, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "militarygradefabrics", + "meanPrice": 984, + "buyPrice": 3690, + "stock": 46, + "stockBracket": 1, + "sellPrice": 3527, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "mineralextractors", + "meanPrice": 801, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2647, + "demand": 1698, + "demandBracket": 3 + }, + { + "name": "mineraloil", + "meanPrice": 423, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1483, + "demand": 3336, + "demandBracket": 3 + }, + { + "name": "modularterminals", + "meanPrice": 2475, + "buyPrice": 1075, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1074, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "moissanite", + "meanPrice": 24833, + "buyPrice": 16662, + "stock": 0, + "stockBracket": 0, + "sellPrice": 16661, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mokojingbeastfeast", + "meanPrice": 9788, + "buyPrice": 2770, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2769, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "momusbogspaniel", + "meanPrice": 9184, + "buyPrice": 2054, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2053, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "monazite", + "meanPrice": 200975, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 840306, + "demand": 1, + "demandBracket": 3 + }, + { + "name": "mukusubiichitinos", + "meanPrice": 8359, + "buyPrice": 1190, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1189, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mulachigiantfungus", + "meanPrice": 7957, + "buyPrice": 8230, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8229, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "musgravite", + "meanPrice": 198613, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 382972, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "mutomimager", + "meanPrice": 6311, + "buyPrice": 4218, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4217, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mysteriousidol", + "meanPrice": 20863, + "buyPrice": 13815, + "stock": 0, + "stockBracket": 0, + "sellPrice": 13814, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "nanobreakers", + "meanPrice": 2366, + "buyPrice": 1009, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1008, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "nanomedicines", + "meanPrice": 9859, + "buyPrice": 2227, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2226, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "naturalfabrics", + "meanPrice": 688, + "buyPrice": 351, + "stock": 0, + "stockBracket": 0, + "sellPrice": 350, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "neofabricinsulation", + "meanPrice": 5978, + "buyPrice": 3380, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3379, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "neritusberries", + "meanPrice": 8497, + "buyPrice": 1616, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1615, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ngadandarifireopals", + "meanPrice": 19112, + "buyPrice": 12751, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12750, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ngunamodernantiques", + "meanPrice": 8545, + "buyPrice": 1989, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1988, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "njangarisaddles", + "meanPrice": 8356, + "buyPrice": 1639, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1638, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "noneuclidianexotanks", + "meanPrice": 8526, + "buyPrice": 1326, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1325, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "nonlethalweapons", + "meanPrice": 1943, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2417, + "demand": 274, + "demandBracket": 3 + }, + { + "name": "ochoengchillies", + "meanPrice": 8601, + "buyPrice": 1659, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1658, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionhead", + "meanPrice": 8437, + "buyPrice": 1814, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1813, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionheada", + "meanPrice": 8437, + "buyPrice": 1272, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1271, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionheadb", + "meanPrice": 8437, + "buyPrice": 1272, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1271, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionheadc", + "meanPrice": 4828, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5755, + "demand": 0, + "demandBracket": 1 + }, + { + "name": "opal", + "meanPrice": 135284, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 525075, + "demand": 1, + "demandBracket": 3 + }, + { + "name": "ophiuchiexinoartefacts", + "meanPrice": 10969, + "buyPrice": 4072, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4071, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "orrerianviciousbrew", + "meanPrice": 8342, + "buyPrice": 1146, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1145, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "osmium", + "meanPrice": 45198, + "buyPrice": 23482, + "stock": 0, + "stockBracket": 0, + "sellPrice": 23480, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "p_particulatesample", + "meanPrice": 45525, + "buyPrice": 32061, + "stock": 0, + "stockBracket": 0, + "sellPrice": 32059, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "painite", + "meanPrice": 53016, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 192955, + "demand": 4, + "demandBracket": 3 + }, + { + "name": "palladium", + "meanPrice": 50639, + "buyPrice": 47901, + "stock": 2, + "stockBracket": 1, + "sellPrice": 47360, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "pantaaprayersticks", + "meanPrice": 9177, + "buyPrice": 2043, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2042, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "performanceenhancers", + "meanPrice": 6790, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6907, + "demand": 40, + "demandBracket": 1 + }, + { + "name": "personalgifts", + "meanPrice": 16535, + "buyPrice": 10000, + "stock": 0, + "stockBracket": 0, + "sellPrice": 9999, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "pesticides", + "meanPrice": 437, + "buyPrice": 82, + "stock": 0, + "stockBracket": 0, + "sellPrice": 81, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "platinum", + "meanPrice": 58263, + "buyPrice": 39557, + "stock": 0, + "stockBracket": 0, + "sellPrice": 39555, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "platinumaloy", + "meanPrice": 18333, + "buyPrice": 12009, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12008, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "polymers", + "meanPrice": 376, + "buyPrice": 2614, + "stock": 272, + "stockBracket": 1, + "sellPrice": 1960, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "powerconverter", + "meanPrice": 1433, + "buyPrice": 707, + "stock": 0, + "stockBracket": 0, + "sellPrice": 706, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "zeesszeantglue", + "meanPrice": 8161, + "buyPrice": 1089, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1088, + "demand": 0, + "demandBracket": 0 + } + ] + }, + "header": { + "uploaderID": "Athanasius Testing", + "softwareName": "Athanasius Testing", + "softwareVersion": "v0.0.1" + } +} diff --git a/scripts/testing/gateway-responses/commodity-just-under-50KiB.json b/scripts/testing/gateway-responses/commodity-just-under-50KiB.json new file mode 100644 index 0000000..1dff8b1 --- /dev/null +++ b/scripts/testing/gateway-responses/commodity-just-under-50KiB.json @@ -0,0 +1,2206 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/commodity/3", + "message": { + "timestamp": "2022-01-06T11:19:27Z", + "systemName": "LP 98-132", + "stationName": "Freeport", + "marketId": 128008448, + "commodities": [ + { + "name": "advancedcatalysers", + "meanPrice": 3039, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3756, + "demand": 936, + "demandBracket": 3 + }, + { + "name": "advancedmedicines", + "meanPrice": 1485, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1726, + "demand": 296, + "demandBracket": 3 + }, + { + "name": "advert1", + "meanPrice": 21542, + "buyPrice": 15267, + "stock": 0, + "stockBracket": 0, + "sellPrice": 15266, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aerialedenapple", + "meanPrice": 8331, + "buyPrice": 1660, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1659, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "agriculturalmedicines", + "meanPrice": 1231, + "buyPrice": 543, + "stock": 0, + "stockBracket": 0, + "sellPrice": 542, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "agronomictreatment", + "meanPrice": 3105, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 15596, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "alacarakmoskinart", + "meanPrice": 8899, + "buyPrice": 1858, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1857, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "albinoquechuamammoth", + "meanPrice": 9687, + "buyPrice": 2663, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2662, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "alexandrite", + "meanPrice": 217277, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 357530, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "algae", + "meanPrice": 356, + "buyPrice": 23, + "stock": 0, + "stockBracket": 0, + "sellPrice": 22, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "alieneggs", + "meanPrice": 25067, + "buyPrice": 19095, + "stock": 0, + "stockBracket": 0, + "sellPrice": 19094, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "altairianskin", + "meanPrice": 8432, + "buyPrice": 1262, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1261, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aluminium", + "meanPrice": 551, + "buyPrice": 1127, + "stock": 352, + "stockBracket": 2, + "sellPrice": 1045, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "alyabodilysoap", + "meanPrice": 8218, + "buyPrice": 1078, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1077, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientcasket", + "meanPrice": 16294, + "buyPrice": 5581, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5580, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientkey", + "meanPrice": 29931, + "buyPrice": 9760, + "stock": 0, + "stockBracket": 0, + "sellPrice": 9759, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientorb", + "meanPrice": 17415, + "buyPrice": 5967, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5966, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancientrelic", + "meanPrice": 24962, + "buyPrice": 9126, + "stock": 0, + "stockBracket": 0, + "sellPrice": 9125, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancienttablet", + "meanPrice": 17415, + "buyPrice": 5967, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5966, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancienttotem", + "meanPrice": 20437, + "buyPrice": 7379, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7378, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ancienturn", + "meanPrice": 14907, + "buyPrice": 3907, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3906, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "anduligafireworks", + "meanPrice": 8519, + "buyPrice": 1574, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1573, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "animalmeat", + "meanPrice": 1539, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1695, + "demand": 26, + "demandBracket": 1 + }, + { + "name": "animalmonitors", + "meanPrice": 537, + "buyPrice": 630, + "stock": 0, + "stockBracket": 0, + "sellPrice": 629, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "antiquejewellery", + "meanPrice": 183163, + "buyPrice": 158957, + "stock": 0, + "stockBracket": 0, + "sellPrice": 158949, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "anynacoffee", + "meanPrice": 9160, + "buyPrice": 2165, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2164, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "apavietii", + "meanPrice": 10362, + "buyPrice": 3386, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3385, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aquaponicsystems", + "meanPrice": 524, + "buyPrice": 321, + "stock": 0, + "stockBracket": 0, + "sellPrice": 320, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "aroucaconventualsweets", + "meanPrice": 8737, + "buyPrice": 1557, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1556, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "articulationmotors", + "meanPrice": 7588, + "buyPrice": 4463, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4462, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "assaultplans", + "meanPrice": 26074, + "buyPrice": 3294, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3293, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "atmosphericextractors", + "meanPrice": 571, + "buyPrice": 331, + "stock": 0, + "stockBracket": 0, + "sellPrice": 330, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "autofabricators", + "meanPrice": 3827, + "buyPrice": 2685, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2684, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "azcancriformula42", + "meanPrice": 12440, + "buyPrice": 6106, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6105, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bakedgreebles", + "meanPrice": 8211, + "buyPrice": 1056, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1055, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "baltahsinevacuumkrill", + "meanPrice": 8479, + "buyPrice": 1502, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1501, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bankiamphibiousleather", + "meanPrice": 8338, + "buyPrice": 1138, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1137, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "basicmedicines", + "meanPrice": 493, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 963, + "demand": 904, + "demandBracket": 3 + }, + { + "name": "bastsnakegin", + "meanPrice": 8659, + "buyPrice": 1724, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1723, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "battleweapons", + "meanPrice": 7451, + "buyPrice": 4463, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4462, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bauxite", + "meanPrice": 1140, + "buyPrice": 10730, + "stock": 595, + "stockBracket": 1, + "sellPrice": 10270, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "beer", + "meanPrice": 430, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 799, + "demand": 1777, + "demandBracket": 3 + }, + { + "name": "belalansrayleather", + "meanPrice": 8519, + "buyPrice": 1605, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1604, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "benitoite", + "meanPrice": 149395, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 570449, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "bertrandite", + "meanPrice": 18817, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 21216, + "demand": 1047, + "demandBracket": 2 + }, + { + "name": "beryllium", + "meanPrice": 8243, + "buyPrice": 8041, + "stock": 2, + "stockBracket": 1, + "sellPrice": 7948, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "bioreducinglichen", + "meanPrice": 1204, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1703, + "demand": 5473, + "demandBracket": 3 + }, + { + "name": "biowaste", + "meanPrice": 358, + "buyPrice": 145, + "stock": 103, + "stockBracket": 1, + "sellPrice": 87, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "bismuth", + "meanPrice": 2441, + "buyPrice": 1607, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1606, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bluemilk", + "meanPrice": 10805, + "buyPrice": 3912, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3911, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "bromellite", + "meanPrice": 30424, + "buyPrice": 70950, + "stock": 0, + "stockBracket": 0, + "sellPrice": 70946, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "buckyballbeermats", + "meanPrice": 7957, + "buyPrice": 8230, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8229, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "buildingfabricators", + "meanPrice": 2312, + "buyPrice": 1448, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1447, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cd75catcoffee", + "meanPrice": 9571, + "buyPrice": 2452, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2451, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "centaurimegagin", + "meanPrice": 10217, + "buyPrice": 3239, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3238, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ceramiccomposites", + "meanPrice": 415, + "buyPrice": 424, + "stock": 0, + "stockBracket": 0, + "sellPrice": 423, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ceremonialheiketea", + "meanPrice": 9251, + "buyPrice": 2161, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2160, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cetiaepyornisegg", + "meanPrice": 9769, + "buyPrice": 2741, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2740, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cetirabbits", + "meanPrice": 9079, + "buyPrice": 2068, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2067, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chameleoncloth", + "meanPrice": 9071, + "buyPrice": 1959, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1958, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chateaudeaegaeon", + "meanPrice": 8791, + "buyPrice": 1610, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1609, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chemicalwaste", + "meanPrice": 672, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 599, + "demand": 159, + "demandBracket": 3 + }, + { + "name": "cherbonesbloodcrystals", + "meanPrice": 16714, + "buyPrice": 10207, + "stock": 0, + "stockBracket": 0, + "sellPrice": 10206, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "chieridanimarinepaste", + "meanPrice": 8450, + "buyPrice": 1426, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1425, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "classifiedexperimentalequipment", + "meanPrice": 11423, + "buyPrice": 4002, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4001, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "clothing", + "meanPrice": 546, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 963, + "demand": 955, + "demandBracket": 3 + }, + { + "name": "cmmcomposite", + "meanPrice": 5988, + "buyPrice": 3907, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3906, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cobalt", + "meanPrice": 3762, + "buyPrice": 13474, + "stock": 724, + "stockBracket": 1, + "sellPrice": 12870, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "coffee", + "meanPrice": 1499, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2303, + "demand": 527, + "demandBracket": 3 + }, + { + "name": "coltan", + "meanPrice": 6163, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7849, + "demand": 2063, + "demandBracket": 3 + }, + { + "name": "comercialsamples", + "meanPrice": 1815, + "buyPrice": 656, + "stock": 0, + "stockBracket": 0, + "sellPrice": 655, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "computercomponents", + "meanPrice": 776, + "buyPrice": 267, + "stock": 0, + "stockBracket": 0, + "sellPrice": 266, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "conductivefabrics", + "meanPrice": 709, + "buyPrice": 1937, + "stock": 55, + "stockBracket": 1, + "sellPrice": 1810, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "consumertechnology", + "meanPrice": 6690, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6883, + "demand": 0, + "demandBracket": 1 + }, + { + "name": "coolinghoses", + "meanPrice": 1886, + "buyPrice": 707, + "stock": 0, + "stockBracket": 0, + "sellPrice": 706, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "copper", + "meanPrice": 689, + "buyPrice": 1115, + "stock": 474, + "stockBracket": 2, + "sellPrice": 1042, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "coquimspongiformvictuals", + "meanPrice": 8077, + "buyPrice": 10924, + "stock": 0, + "stockBracket": 0, + "sellPrice": 10923, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cropharvesters", + "meanPrice": 2230, + "buyPrice": 1564, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1563, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "cryolite", + "meanPrice": 12173, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 14970, + "demand": 1166, + "demandBracket": 3 + }, + { + "name": "crystallinespheres", + "meanPrice": 12216, + "buyPrice": 5438, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5437, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "damnacarapaces", + "meanPrice": 8120, + "buyPrice": 2925, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2924, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "datacore", + "meanPrice": 6791, + "buyPrice": 3907, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3906, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "deltaphoenicispalms", + "meanPrice": 8188, + "buyPrice": 1528, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1527, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "deuringastruffles", + "meanPrice": 9232, + "buyPrice": 2335, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2334, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "diagnosticsensor", + "meanPrice": 6727, + "buyPrice": 4463, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4462, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "diplomaticbag", + "meanPrice": 28625, + "buyPrice": 4265, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4264, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "disomacorn", + "meanPrice": 8134, + "buyPrice": 1053, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1052, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "domesticappliances", + "meanPrice": 740, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1216, + "demand": 575, + "demandBracket": 3 + }, + { + "name": "duradrives", + "meanPrice": 19356, + "buyPrice": 13002, + "stock": 0, + "stockBracket": 0, + "sellPrice": 13001, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "earthrelics", + "meanPrice": 15161, + "buyPrice": 14640, + "stock": 0, + "stockBracket": 0, + "sellPrice": 14639, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "eleuthermals", + "meanPrice": 8507, + "buyPrice": 1571, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1570, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "emergencypowercells", + "meanPrice": 2368, + "buyPrice": 1338, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1337, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "encripteddatastorage", + "meanPrice": 8313, + "buyPrice": 4049, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4048, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "encryptedcorrespondence", + "meanPrice": 8226, + "buyPrice": 3935, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3934, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "eraninpearlwhisky", + "meanPrice": 9040, + "buyPrice": 2058, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2057, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "eshuumbrellas", + "meanPrice": 9343, + "buyPrice": 2187, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2186, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "esusekucaviar", + "meanPrice": 9625, + "buyPrice": 2573, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2572, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ethgrezeteabuds", + "meanPrice": 10197, + "buyPrice": 3212, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3211, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "evacuationshelter", + "meanPrice": 522, + "buyPrice": 138, + "stock": 0, + "stockBracket": 0, + "sellPrice": 137, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "exhaustmanifold", + "meanPrice": 1873, + "buyPrice": 1009, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1008, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "explosives", + "meanPrice": 512, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2101, + "demand": 3964, + "demandBracket": 3 + }, + { + "name": "fish", + "meanPrice": 650, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1008, + "demand": 481, + "demandBracket": 2 + }, + { + "name": "foodcartridges", + "meanPrice": 265, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 767, + "demand": 1094, + "demandBracket": 3 + }, + { + "name": "fossilremnants", + "meanPrice": 11785, + "buyPrice": 7379, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7378, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "fruitandvegetables", + "meanPrice": 509, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1060, + "demand": 1637, + "demandBracket": 3 + }, + { + "name": "fujintea", + "meanPrice": 8597, + "buyPrice": 1418, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1417, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "galactictravelguide", + "meanPrice": 8627, + "buyPrice": 359, + "stock": 0, + "stockBracket": 0, + "sellPrice": 358, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "gallite", + "meanPrice": 11915, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 13829, + "demand": 1765, + "demandBracket": 3 + }, + { + "name": "gallium", + "meanPrice": 5203, + "buyPrice": 5165, + "stock": 11, + "stockBracket": 1, + "sellPrice": 5042, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "genebank", + "meanPrice": 59082, + "buyPrice": 46704, + "stock": 0, + "stockBracket": 0, + "sellPrice": 46701, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "geologicalequipment", + "meanPrice": 1886, + "buyPrice": 1186, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1185, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "geologicalsamples", + "meanPrice": 8313, + "buyPrice": 12144, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12143, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "gerasiangueuzebeer", + "meanPrice": 8215, + "buyPrice": 1067, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1066, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "giantirukamasnails", + "meanPrice": 9174, + "buyPrice": 2131, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2130, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "giantverrix", + "meanPrice": 12496, + "buyPrice": 5713, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5712, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "gold", + "meanPrice": 47610, + "buyPrice": 44874, + "stock": 2, + "stockBracket": 1, + "sellPrice": 44362, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "gomanyauponcoffee", + "meanPrice": 8921, + "buyPrice": 1843, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1842, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "goslarite", + "meanPrice": 5979, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8024, + "demand": 1290, + "demandBracket": 3 + }, + { + "name": "grain", + "meanPrice": 410, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 919, + "demand": 2980, + "demandBracket": 3 + }, + { + "name": "grandidierite", + "meanPrice": 197292, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 458141, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "haidneblackbrew", + "meanPrice": 8837, + "buyPrice": 1693, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1692, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "harmasilversearum", + "meanPrice": 9762, + "buyPrice": 4217, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4216, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "havasupaidreamcatcher", + "meanPrice": 14639, + "buyPrice": 8056, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8055, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hazardousenvironmentsuits", + "meanPrice": 570, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 915, + "demand": 4363, + "demandBracket": 3 + }, + { + "name": "heatsinkinterlink", + "meanPrice": 2100, + "buyPrice": 1161, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1160, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "heliostaticfurnaces", + "meanPrice": 434, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2010, + "demand": 537, + "demandBracket": 3 + }, + { + "name": "helvetitjpearls", + "meanPrice": 10450, + "buyPrice": 3507, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3506, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hip10175bushmeat", + "meanPrice": 9382, + "buyPrice": 2246, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2245, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hip41181squid", + "meanPrice": 8947, + "buyPrice": 1891, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1890, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hiporganophosphates", + "meanPrice": 8169, + "buyPrice": 1039, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1038, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hnshockmount", + "meanPrice": 1922, + "buyPrice": 1075, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1074, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "honestypills", + "meanPrice": 8860, + "buyPrice": 1686, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1685, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hr7221wheat", + "meanPrice": 8190, + "buyPrice": 1540, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1539, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "hydrogenfuel", + "meanPrice": 113, + "buyPrice": 126, + "stock": 379, + "stockBracket": 1, + "sellPrice": 120, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "hydrogenperoxide", + "meanPrice": 3160, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3301, + "demand": 548, + "demandBracket": 3 + }, + { + "name": "indibourbon", + "meanPrice": 8806, + "buyPrice": 1638, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1637, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "indite", + "meanPrice": 11389, + "buyPrice": 10508, + "stock": 296, + "stockBracket": 1, + "sellPrice": 10238, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "indium", + "meanPrice": 5845, + "buyPrice": 5894, + "stock": 8, + "stockBracket": 1, + "sellPrice": 5824, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "insulatingmembrane", + "meanPrice": 10724, + "buyPrice": 10137, + "stock": 2, + "stockBracket": 2, + "sellPrice": 10021, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "iondistributor", + "meanPrice": 2363, + "buyPrice": 1338, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1337, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jadeite", + "meanPrice": 42383, + "buyPrice": 40631, + "stock": 0, + "stockBracket": 0, + "sellPrice": 40628, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jaquesquinentianstill", + "meanPrice": 13845, + "buyPrice": 7233, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7232, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jaradharrepuzzlebox", + "meanPrice": 16816, + "buyPrice": 10325, + "stock": 0, + "stockBracket": 0, + "sellPrice": 10324, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jarouarice", + "meanPrice": 8169, + "buyPrice": 2711, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2710, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "jotunmookah", + "meanPrice": 8780, + "buyPrice": 1637, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1636, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "kachiriginleaches", + "meanPrice": 8227, + "buyPrice": 1359, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1358, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "kamitracigars", + "meanPrice": 12282, + "buyPrice": 5522, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5521, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "karetiicouture", + "meanPrice": 11582, + "buyPrice": 4748, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4747, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "karsukilocusts", + "meanPrice": 8543, + "buyPrice": 1583, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1582, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "kinagoinstruments", + "meanPrice": 13030, + "buyPrice": 6570, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6569, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "konggaale", + "meanPrice": 8310, + "buyPrice": 1578, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1577, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "korrokungpellets", + "meanPrice": 8067, + "buyPrice": 1190, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1189, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lanthanum", + "meanPrice": 8707, + "buyPrice": 6087, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6086, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "largeexplorationdatacash", + "meanPrice": 255841, + "buyPrice": 225210, + "stock": 0, + "stockBracket": 0, + "sellPrice": 225198, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lavianbrandy", + "meanPrice": 10365, + "buyPrice": 3614, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3613, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "leather", + "meanPrice": 435, + "buyPrice": 57, + "stock": 0, + "stockBracket": 0, + "sellPrice": 56, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "leestianeviljuice", + "meanPrice": 8220, + "buyPrice": 1086, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1085, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lepidolite", + "meanPrice": 771, + "buyPrice": 4193, + "stock": 1271, + "stockBracket": 1, + "sellPrice": 3999, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "lftvoidextractcoffee", + "meanPrice": 9554, + "buyPrice": 2506, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2505, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "liquidoxygen", + "meanPrice": 1474, + "buyPrice": 957, + "stock": 22, + "stockBracket": 1, + "sellPrice": 850, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "liquor", + "meanPrice": 879, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1331, + "demand": 453, + "demandBracket": 3 + }, + { + "name": "lithium", + "meanPrice": 1772, + "buyPrice": 1906, + "stock": 6, + "stockBracket": 1, + "sellPrice": 1855, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "lithiumhydroxide", + "meanPrice": 5673, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 7350, + "demand": 99, + "demandBracket": 3 + }, + { + "name": "livehecateseaworms", + "meanPrice": 8737, + "buyPrice": 1899, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1898, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "lowtemperaturediamond", + "meanPrice": 106353, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 293785, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "ltthypersweet", + "meanPrice": 8054, + "buyPrice": 1233, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1232, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m3_tissuesample_membrane", + "meanPrice": 18352, + "buyPrice": 12075, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12074, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m3_tissuesample_mycelium", + "meanPrice": 32825, + "buyPrice": 22634, + "stock": 0, + "stockBracket": 0, + "sellPrice": 22632, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m3_tissuesample_spores", + "meanPrice": 6031, + "buyPrice": 3380, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3379, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m_tissuesample_fluid", + "meanPrice": 6031, + "buyPrice": 3380, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3379, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m_tissuesample_nerves", + "meanPrice": 32825, + "buyPrice": 22634, + "stock": 0, + "stockBracket": 0, + "sellPrice": 22632, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "m_tissuesample_soft", + "meanPrice": 17216, + "buyPrice": 11290, + "stock": 0, + "stockBracket": 0, + "sellPrice": 11289, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "magneticemittercoil", + "meanPrice": 1357, + "buyPrice": 656, + "stock": 0, + "stockBracket": 0, + "sellPrice": 655, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "marinesupplies", + "meanPrice": 4135, + "buyPrice": 2745, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2744, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mechucoshightea", + "meanPrice": 8846, + "buyPrice": 1709, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1708, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "medbstarlube", + "meanPrice": 8191, + "buyPrice": 1953, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1952, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "medicaldiagnosticequipment", + "meanPrice": 3075, + "buyPrice": 1779, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1778, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "metaalloys", + "meanPrice": 195453, + "buyPrice": 162685, + "stock": 0, + "stockBracket": 0, + "sellPrice": 162676, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "methaneclathrate", + "meanPrice": 1650, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1740, + "demand": 2873, + "demandBracket": 3 + }, + { + "name": "methanolmonohydratecrystals", + "meanPrice": 2478, + "buyPrice": 1290, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1289, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "microcontrollers", + "meanPrice": 5590, + "buyPrice": 3612, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3611, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "militarygradefabrics", + "meanPrice": 984, + "buyPrice": 3690, + "stock": 46, + "stockBracket": 1, + "sellPrice": 3527, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "mineralextractors", + "meanPrice": 801, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2647, + "demand": 1698, + "demandBracket": 3 + }, + { + "name": "mineraloil", + "meanPrice": 423, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1483, + "demand": 3336, + "demandBracket": 3 + }, + { + "name": "modularterminals", + "meanPrice": 2475, + "buyPrice": 1075, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1074, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "moissanite", + "meanPrice": 24833, + "buyPrice": 16662, + "stock": 0, + "stockBracket": 0, + "sellPrice": 16661, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mokojingbeastfeast", + "meanPrice": 9788, + "buyPrice": 2770, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2769, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "momusbogspaniel", + "meanPrice": 9184, + "buyPrice": 2054, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2053, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "monazite", + "meanPrice": 200975, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 840306, + "demand": 1, + "demandBracket": 3 + }, + { + "name": "mukusubiichitinos", + "meanPrice": 8359, + "buyPrice": 1190, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1189, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mulachigiantfungus", + "meanPrice": 7957, + "buyPrice": 8230, + "stock": 0, + "stockBracket": 0, + "sellPrice": 8229, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "musgravite", + "meanPrice": 198613, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 382972, + "demand": 2, + "demandBracket": 3 + }, + { + "name": "mutomimager", + "meanPrice": 6311, + "buyPrice": 4218, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4217, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "mysteriousidol", + "meanPrice": 20863, + "buyPrice": 13815, + "stock": 0, + "stockBracket": 0, + "sellPrice": 13814, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "nanobreakers", + "meanPrice": 2366, + "buyPrice": 1009, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1008, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "nanomedicines", + "meanPrice": 9859, + "buyPrice": 2227, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2226, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "naturalfabrics", + "meanPrice": 688, + "buyPrice": 351, + "stock": 0, + "stockBracket": 0, + "sellPrice": 350, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "neofabricinsulation", + "meanPrice": 5978, + "buyPrice": 3380, + "stock": 0, + "stockBracket": 0, + "sellPrice": 3379, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "neritusberries", + "meanPrice": 8497, + "buyPrice": 1616, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1615, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ngadandarifireopals", + "meanPrice": 19112, + "buyPrice": 12751, + "stock": 0, + "stockBracket": 0, + "sellPrice": 12750, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "ngunamodernantiques", + "meanPrice": 8545, + "buyPrice": 1989, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1988, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "njangarisaddles", + "meanPrice": 8356, + "buyPrice": 1639, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1638, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "noneuclidianexotanks", + "meanPrice": 8526, + "buyPrice": 1326, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1325, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "nonlethalweapons", + "meanPrice": 1943, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2417, + "demand": 274, + "demandBracket": 3 + }, + { + "name": "ochoengchillies", + "meanPrice": 8601, + "buyPrice": 1659, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1658, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionhead", + "meanPrice": 8437, + "buyPrice": 1814, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1813, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionheada", + "meanPrice": 8437, + "buyPrice": 1272, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1271, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionheadb", + "meanPrice": 8437, + "buyPrice": 1272, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1271, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "onionheadc", + "meanPrice": 4828, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 5755, + "demand": 0, + "demandBracket": 1 + }, + { + "name": "opal", + "meanPrice": 135284, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 525075, + "demand": 1, + "demandBracket": 3 + }, + { + "name": "ophiuchiexinoartefacts", + "meanPrice": 10969, + "buyPrice": 4072, + "stock": 0, + "stockBracket": 0, + "sellPrice": 4071, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "orrerianviciousbrew", + "meanPrice": 8342, + "buyPrice": 1146, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1145, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "osmium", + "meanPrice": 45198, + "buyPrice": 23482, + "stock": 0, + "stockBracket": 0, + "sellPrice": 23480, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "p_particulatesample", + "meanPrice": 45525, + "buyPrice": 32061, + "stock": 0, + "stockBracket": 0, + "sellPrice": 32059, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "painite", + "meanPrice": 53016, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 192955, + "demand": 4, + "demandBracket": 3 + }, + { + "name": "palladium", + "meanPrice": 50639, + "buyPrice": 47901, + "stock": 2, + "stockBracket": 1, + "sellPrice": 47360, + "demand": 1, + "demandBracket": 0 + }, + { + "name": "pantaaprayersticks", + "meanPrice": 9177, + "buyPrice": 2043, + "stock": 0, + "stockBracket": 0, + "sellPrice": 2042, + "demand": 0, + "demandBracket": 0 + }, + { + "name": "performanceenhancers", + "meanPrice": 6790, + "buyPrice": 0, + "stock": 0, + "stockBracket": 0, + "sellPrice": 6907, + "demand": 40, + "demandBracket": 1 + }, + { + "name": "zeesszeantglue", + "meanPrice": 8161, + "buyPrice": 1089, + "stock": 0, + "stockBracket": 0, + "sellPrice": 1088, + "demand": 0, + "demandBracket": 0 + } + ] + }, + "header": { + "uploaderID": "Athanasius Testing", + "softwareName": "Athanasius Testing", + "softwareVersion": "v0.0.1" + } +} diff --git a/scripts/testing/gateway-responses/commodity-smallest-message.json b/scripts/testing/gateway-responses/commodity-smallest-message.json new file mode 100644 index 0000000..c109d6e --- /dev/null +++ b/scripts/testing/gateway-responses/commodity-smallest-message.json @@ -0,0 +1 @@ +{"$schemaRef":"https://eddn.edcd.io/schemas/commodity/3","header":{"softwareName":"E:D Market Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"message":{"systemName":"delphi","stationName":"The Oracle","marketId":128782803,"timestamp":"2022-01-26T12:00:00Z","commodities":[]}} diff --git a/scripts/testing/gateway-responses/fssallbodiesfound.json b/scripts/testing/gateway-responses/fssallbodiesfound.json new file mode 100644 index 0000000..04d2424 --- /dev/null +++ b/scripts/testing/gateway-responses/fssallbodiesfound.json @@ -0,0 +1,18 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/fssallbodiesfound/1", + "message": { + "timestamp":"2022-01-26T16:21:00Z", + "event":"FSSAllBodiesFound", + "SystemName":"Zeta Doradus", + "StarPos": [ + 30.40625,-22.65625,-2.18750 + ], + "SystemAddress":44853889387, + "Count":1 + }, + "header": { + "uploaderID": "from Athanasius Testing", + "softwareName": "Athanasius Testing script", + "softwareVersion": "v0.0.1" + } +} diff --git a/scripts/testing/gateway-responses/shipyard-smallest-message.json b/scripts/testing/gateway-responses/shipyard-smallest-message.json new file mode 100644 index 0000000..945f008 --- /dev/null +++ b/scripts/testing/gateway-responses/shipyard-smallest-message.json @@ -0,0 +1 @@ +{"$schemaRef":"https://eddn.edcd.io/schemas/shipyard/2","header":{"softwareName":"E:D Market Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"message":{"systemName":"delphi","stationName":"The Oracle","marketId":128782803,"timestamp":"2022-01-26T12:00:00Z","ships":[]}} From b323460bf824771f3655dd243815a31ed0453c95 Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Wed, 26 Jan 2022 17:50:41 +0000 Subject: [PATCH 12/89] Add FSSAllBodiesFound --- schemas/fssallbodiesfound-README.md | 30 ++++++++++++ schemas/fssallbodiesfound-v1.0.json | 75 +++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 schemas/fssallbodiesfound-README.md create mode 100644 schemas/fssallbodiesfound-v1.0.json diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md new file mode 100644 index 0000000..a86db5c --- /dev/null +++ b/schemas/fssallbodiesfound-README.md @@ -0,0 +1,30 @@ +# EDDN FSSAllBodiesFound Schema + +## Introduction +Here we document how to take data from an ED `FSSAllBodiesFound` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +## Senders +The primary data source for this schema is the ED Journal event +`FSSAllBodiesFound`. + +### Key Renames +Many of the key names have a different case defined in this schema, make +sure you are renaming them as appropriate. + +### Elisions +None + +### Augmentations +#### horizons flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### odyssey flag +You SHOULD add this key/value pair, using the value from the `LoadGame` event. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/fssallbodiesfound-v1.0.json b/schemas/fssallbodiesfound-v1.0.json new file mode 100644 index 0000000..14dc507 --- /dev/null +++ b/schemas/fssallbodiesfound-v1.0.json @@ -0,0 +1,75 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fssallbodiesfound/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the event key, Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "SystemName", "StarPos", "SystemAddress", "Count" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "SystemName": { + "type" : "string", + "minLength" : 1 + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender if not present in the journal event" + }, + "SystemAddress": { + "type" : "integer", + "description" : "Should be added by the sender if not present in the journal event" + }, + "Count" : { + "type" : "integer", + "description" : "Number of bodies in this system" + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} From b6d25ccebcc099ac93173ec50cc07b7a71014dab Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Thu, 27 Jan 2022 11:32:46 +0000 Subject: [PATCH 13/89] Added back in the event key --- schemas/fssallbodiesfound-v1.0.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/schemas/fssallbodiesfound-v1.0.json b/schemas/fssallbodiesfound-v1.0.json index 14dc507..b3a4198 100644 --- a/schemas/fssallbodiesfound-v1.0.json +++ b/schemas/fssallbodiesfound-v1.0.json @@ -31,7 +31,7 @@ }, "message": { "type" : "object", - "description" : "Contains all properties from the listed events in the client's journal minus the event key, Localised strings and the properties marked below as 'disallowed'", + "description" : "Contains all properties from the listed events in the client's journal, minus the Localised strings and the properties marked below as 'disallowed'", "additionalProperties" : false, "required" : [ "timestamp", "SystemName", "StarPos", "SystemAddress", "Count" ], "properties" : { @@ -39,6 +39,9 @@ "type" : "string", "format" : "date-time" }, + "event" : { + "enum" : [ "FSSAllBodiesFound" ] + }, "horizons": { "type" : "boolean", "description" : "Whether the sending Cmdr has a Horizons pass." From d4e7409ae0d79a7e4c00de31c28e8cbd4b0fbb7a Mon Sep 17 00:00:00 2001 From: robbyxp1 Date: Thu, 27 Jan 2022 16:28:37 +0000 Subject: [PATCH 14/89] Add to settings.py the fssallbodiesfound schemas --- src/eddn/conf/Settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index af15e95..b0782ce 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -76,6 +76,8 @@ class _Settings(object): "https://eddn.edcd.io/schemas/approachsettlement/1" : "schemas/approachsettlement-v1.0.json", "https://eddn.edcd.io/schemas/approachsettlement/1/test" : "schemas/approachsettlement-v1.0.json", + "https://eddn.edcd.io/schemas/fssallbodiesfound/1" : "schemas/fssallbodiesfound-v1.0.json", + "https://eddn.edcd.io/schemas/fssallbodiesfound/1/test" : "schemas/fssallbodiesfound-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [ @@ -97,6 +99,8 @@ class _Settings(object): "http://schemas.elite-markets.net/eddn/blackmarket/1/test", "http://schemas.elite-markets.net/eddn/journal/1", "http://schemas.elite-markets.net/eddn/journal/1/test", + "http://schemas.elite-markets.net/eddn/fssallbodiesfound/1", + "http://schemas.elite-markets.net/eddn/fssallbodiesfound/1/test", ] ############################################################################### From de33e4460c246cc6094344a4101d38da43e97add Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:26:12 +0000 Subject: [PATCH 15/89] Schemas/FSSAllBodiesFound: Make 'event' required --- schemas/fssallbodiesfound-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/fssallbodiesfound-v1.0.json b/schemas/fssallbodiesfound-v1.0.json index b3a4198..074362a 100644 --- a/schemas/fssallbodiesfound-v1.0.json +++ b/schemas/fssallbodiesfound-v1.0.json @@ -33,7 +33,7 @@ "type" : "object", "description" : "Contains all properties from the listed events in the client's journal, minus the Localised strings and the properties marked below as 'disallowed'", "additionalProperties" : false, - "required" : [ "timestamp", "SystemName", "StarPos", "SystemAddress", "Count" ], + "required" : [ "timestamp", "event", "SystemName", "StarPos", "SystemAddress", "Count" ], "properties" : { "timestamp": { "type" : "string", From 8db880f95def1f9b22af54bf2cd62a7c029fcf3f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:26:57 +0000 Subject: [PATCH 16/89] schemas/FSSAllBodiesFound: Don't list in outdated schemas --- src/eddn/conf/Settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index b0782ce..5ca2a6f 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -99,8 +99,6 @@ class _Settings(object): "http://schemas.elite-markets.net/eddn/blackmarket/1/test", "http://schemas.elite-markets.net/eddn/journal/1", "http://schemas.elite-markets.net/eddn/journal/1/test", - "http://schemas.elite-markets.net/eddn/fssallbodiesfound/1", - "http://schemas.elite-markets.net/eddn/fssallbodiesfound/1/test", ] ############################################################################### From e50b57a5389d32e68eceb9629bff0d5e80588a89 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 12:59:46 +0000 Subject: [PATCH 17/89] docs/Contributing: Adding 'new schema' guidelines --- docs/Contributing.md | 118 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/docs/Contributing.md b/docs/Contributing.md index 1eaadd3..62b11da 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -11,3 +11,121 @@ your editor is taking note of those settings, or cross-check its contents with the [editorconfig documentation](https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties) , and ensure your editor/IDE's settings match. + +## Code Changes + +### Bug Fixes + +### Enhancements + +## Adding a New Schema + +If you think you have a good case for an additional EDDN Schema then there are +several things you should consider: + +1. What is the source of the data? In almost all circumstances the only + acceptable sources are the game Journal files and the Frontier CAPI service. + We do *NOT* want manually entered data being sent over EDDN, it's too prone + to error. + +2. Is the new Schema going to be practically useful ? + 1. What use cases are there for the new data? + 2. Who benefits ? + 3. What's the likely volume of messages compared to existing Schemas? If + there would often be many messages in a short space of time consider + requiring senders to batch them. 2022-01-28: There's no live example of + this yet, but see + [discussion of adding support for FSSSignalDiscovered](https://github.com/EDCD/EDDN/issues/152) + . + 5. What's the likely size range of the new messages? The Gateway has a + limit on the size of the *body* of `/upload/` requests. Check + [live branch src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py) + `bottle.BaseRequest.MEMFILE_MAX = ...` for the current limit. + +3. For CAPI-sourced data you need to keep in mind the possible synchronization + issues between it and any necessary data augmentations from Journal data. + This might mean needing to make such augmentations optional. + +4. For Journal-sourced data if the source is an event not yet allowed by any + existing Schema then you MUST define a wholly new Schema for the data. This + allows you to fully specify both required and forbidden information in the + resulting messages. + The Journal events that are handled in the generic `journal` Schema are only + there for historical reasons and due to the difficulties in ensuring all + listeners and senders migrate to separate Schemas in a synchronized manner. + +5. Ensure you read + [the general Schemas README](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) + so as to be aware of general requirements that your new Schema will need to + adhere to. + +7. You **MUST** + [open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) + in order to propose the new Schema. If a consensus appears to have been + reached in comments therein then start work on a Pull Request. + +8. There must be at least one working Sender implementation before the Pull + Request for a new Schema will be merged into the live service. Experience + has demonstrated that there are often caveats and gotchas discovered during + the development of a Sender for a new Schema. + Often this will end up being a Pull Request against either + [Elite Dangerous Market Connector](https://github.com/EDCD/EDMarketConnector) 's + EDDN plugin, or [ED Discovery](https://github.com/EDDiscovery/EDDiscovery). + +The Schema files are placed in the `schemas/` directory, located in the root +of the project structure. They are +[JSON](https://www.json.org/json-en.html) +files, conforming to the +[JSON Schema](https://json-schema.org/) +specification. As of 2022-01-28 we still use 'draft 04' of this specification. +We are looking into updating to the latest in +[#139 - Update to latest JSON schema version(s) ](https://github.com/EDCD/EDDN/issues/139). + +All Schema files MUST be accompanied by a MarkDown formatted README file. + +### Always start a new Schema at version 1 + +The first time a new Schema goes live it should be as version 1. + - What should policy be on incrementing the version ? I'm not confident + anything other than an integer is actually supported - Ath + +Any breaking changes **MUST** increment the version number. Use a git file +rename to update the name of the file. Examples of such breaking changes +include: + +- If you add a new required property. Senders will need to update. +- If you remove a required property *and making it optional doesn't make + sense*. Senders will need to update. Listeners will need to cope with the + data no longer being present. +- If you change a property from optional to required or disallowed. Senders + will need to update. Listeners can no longer expect it, if disallowed. + +### Necessary file edits + +1. Obviously you need to create the new file, in the `schemas/` directory. + This should be named as per the data source, i.e. Journal `event` value, and + include the Schema version, and `.json` extension. You **MUST** fold the + the `event` value to lower case for this. + An example is `fssdiscoveryscan-v1.0.json` for adding support for the Journal + `FSSDiscoveryScan` event. + +2. You **MUST** also create the README file for the new Schema. This is also + placed in the `schemas/` directory. The name should match that of the + Schema file itself, without the version, and with a `.md` extentions instead + of `.json`. + An example is `fssdiscoveryscan-README.md` documents the + `fssdiscoveryscan-v1.0.json` Schema file. + +3. You will need to add two lines to `src/eddn/conf/Settings.py` in order to + have the Gateway actually recognise the new Schema. You are adding to the + end of the `GATEWAY_JSON_SCHEMAS` dictionary. Both the live Schema *and* + the `/test` version **MUST** be added. + For `fssdiscoveryscan-v1.0.json` you would add: + ```python + + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1" : "schemas/fssdiscoveryscan-v1.0.json", + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1/test" : "schemas/fssdiscoveryscan-v1.0.json", + ``` + Please ensure you use the current hostname as per the entries for already + existing Schemas. Keep the trailing comma on the final entry, Python + allows it, and it will reduce the diff on adding any further Schemas. From 4aadbeb1464e75d03e022f29ee7e886216af4001 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 13:01:53 +0000 Subject: [PATCH 18/89] schemas/README: If using compression, set Content-Type correctly --- schemas/README-EDDN-schemas.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 2773270..82987b2 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -118,7 +118,7 @@ Messages sent to EDDN **MUST**: - Use a **POST** request, with the body containing the EDDN message. No query parameters in the URL are supported or necessary. -The body of an EDDN message is a JSON object in UTF-8 encoding. You SHOULD +The body of an EDDN message is a JSON object in UTF-8 encoding. You MUST set a `Content-Type` header of `applicaton/json`, and NOT any of: * `application/x-www-form-urlencoded` @@ -130,7 +130,8 @@ deprecated and no new software should attempt this method**. We purposefully do not further document the exact format for this. You *MAY* use gzip compression on the body of the message, but it is not -required. +required. If you do compress the body then you **MUST* send a `Content-Type` +header with `gzip` value instead of `application/json`. You should be prepared to handle all scenarios where sending of a message fails: From f0e14f5e038b1d1623cb6d941fd2fad9a8d171fc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 13:33:27 +0000 Subject: [PATCH 19/89] README: List the three services that EDCD *might* be providing Only Live is guaranteed to be available. --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 4127a10..a11d7e2 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,34 @@ Anyone planning to send data too EDDN **MUST** comply with all the advice in that document, and the individual schema README files as applicable. It's also the best resource for those listening to the EDDN data stream. +#### EDDN endpoints + +There are up to three separate services which might be running. + +| Service | Upload | Listeners | Notes | +| ------: | :-----: |:-----------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Live | `https://eddn.edcd.io:4430/upload/` | `tcp://eddn.edcd.io:9500/` | The actual live service, which should always be running. It is automatically restarted every Thursday at 07:06:00 UTC | +| Beta | `https://beta.eddn.edcd.io:4431/upload/` | `tcp://beta.eddn.edcd.io:9510/` | The beta service, which should be running the current state of the `beta` branch. Usually only active when either new schemas or forthcoming code changes are being actively tested. | +| Dev | `https://dev.eddn.edcd.io:4432/upload/` | `tcp://dev.eddn.edcd.io:9520/` | The dev service, which could be running any public branch of the code *or* a private branch. | + +In general the Beta and Dev services will only be running so as to aid the core +development team in testing changes. Don't expect them to be generally +available. + +You **MUST** use the correct hostname in the Upload URLs as these are +TLS/HTTPS connections terminated on a Reverse Proxy. + +You can get away with using alternate hostnames on the Listener URLs, as +they're purely a ZeroMQ endpoint, no TLS. But don't be surprised if either +the Beta or Dev service is actually running on a different IP, so don't +chance it. + +If you need to test some of your own changes then please read +[Running this software](docs/Running-this-software.md) for how to instantiate +your own test service. It is hoped that in the future the code will allow for +easily running in a "local only" mode, not requiring any reverse proxy or +internet-valid TLS certificates. + --- --- From f8f7b58607470614e4686a2e39df60b7701c56d1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 13:34:21 +0000 Subject: [PATCH 20/89] docs/Contributing: Start work on specific Schema/README requirements --- docs/Contributing.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 62b11da..9b27223 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -37,7 +37,7 @@ several things you should consider: this yet, but see [discussion of adding support for FSSSignalDiscovered](https://github.com/EDCD/EDDN/issues/152) . - 5. What's the likely size range of the new messages? The Gateway has a + 4. What's the likely size range of the new messages? The Gateway has a limit on the size of the *body* of `/upload/` requests. Check [live branch src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py) `bottle.BaseRequest.MEMFILE_MAX = ...` for the current limit. @@ -129,3 +129,38 @@ include: Please ensure you use the current hostname as per the entries for already existing Schemas. Keep the trailing comma on the final entry, Python allows it, and it will reduce the diff on adding any further Schemas. + +#### Schema file requirements + +1. The file **MUST** actually be valid JSON, without any special extensions + (so no comments). +2. The file **MUST** comply with the relevant JSON Schema definition. +3. The file **MUST** actually load using Python's `simplejson` module, as this + is what the Gateway code uses. +4. All new Schemas **MUST** comply with all requirements outlined in the + [general Schemas documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md). +5. If the data source is a game Journal event then you **MUST** include the + `event` key and its value as in the source data. This might seem redundant + when we mandate a separate schema for any newly handled Journal event, but + it does no harm and might make data handling for Listeners easier, i.e. + they can just pass all "oh, that's from Journal data" messages through the + same initial handling. + +#### Schema README requirements + +The per-Schema README **MUST** give both Senders and Listeners sufficient +information to correctly handle the pertinent data. You do not need to repeat +anything already specified in the general Schema README. Referring to it via +MarkDown linking is helpful. + +1. The reason(s) for any augmentations to a message must be clearly explained. + 1. **DO** outline where the additional data comes from. e.g. `StarPos` + added to many events should come from a prior `Location`, `FSDJump` or + `CarrierJump` Journal event. + +2. The reason(s) why any property is optional must be clearly explained. + Perhaps it's not always present in the source data. + +3. The reason(s) why any data in the source is not in the message, i.e. because + it's personal to the player, or maybe it's just not useful (always the same + in every instance of the source data). From 4239729632cfc401ea35d13f775b807b21295689 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 13:42:38 +0000 Subject: [PATCH 21/89] docs/Contributing: Further Schema/README requirements --- docs/Contributing.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 9b27223..a864605 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -111,7 +111,7 @@ include: 2. You **MUST** also create the README file for the new Schema. This is also placed in the `schemas/` directory. The name should match that of the - Schema file itself, without the version, and with a `.md` extentions instead + Schema file itself, without the version, and with a `.md` extention instead of `.json`. An example is `fssdiscoveryscan-README.md` documents the `fssdiscoveryscan-v1.0.json` Schema file. @@ -133,15 +133,20 @@ include: #### Schema file requirements 1. The file **MUST** actually be valid JSON, without any special extensions - (so no comments). + (so no comments). Remember that JSON does **not** allow for a trailing + comma on the last entry of an array or dictionary. 2. The file **MUST** comply with the relevant JSON Schema definition. 3. The file **MUST** actually load using Python's `simplejson` module, as this - is what the Gateway code uses. + is what the Gateway code uses. The script `contrib/test-schema.py` will + check both this and that the validation code doesn't choke on it. 4. All new Schemas **MUST** comply with all requirements outlined in the [general Schemas documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md). + If you have a good reason why your new Schema can't and shouldn't comply + with any such then consensus will need to be achieved on changing those + requirements and/or allowing the exception. 5. If the data source is a game Journal event then you **MUST** include the `event` key and its value as in the source data. This might seem redundant - when we mandate a separate schema for any newly handled Journal event, but + when we mandate a separate Schema for any newly handled Journal event, but it does no harm and might make data handling for Listeners easier, i.e. they can just pass all "oh, that's from Journal data" messages through the same initial handling. @@ -164,3 +169,10 @@ MarkDown linking is helpful. 3. The reason(s) why any data in the source is not in the message, i.e. because it's personal to the player, or maybe it's just not useful (always the same in every instance of the source data). + +4. If your Schema only works whilst not complying with any main Schema + requirements, and this has been approved, then you need to explicitly + document which requirement(s) are waived and why. + +5. If you use another Schema's README as the basis for yours then you MUST + remove any text that isn't relevant to your Schema. From f1d8a509ca392e2a9dedc9f0bb81175ea8e9cba9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 15:51:17 +0000 Subject: [PATCH 22/89] docs/Contributing: Require test files for new schemas * Let's have examples in the scripts/testing/gatewat-response directory. * Actually you usually don't actually need to actually use the word actually. --- docs/Contributing.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index a864605..ac9f0f1 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -87,7 +87,7 @@ All Schema files MUST be accompanied by a MarkDown formatted README file. The first time a new Schema goes live it should be as version 1. - What should policy be on incrementing the version ? I'm not confident - anything other than an integer is actually supported - Ath + anything other than an integer is supported - Ath Any breaking changes **MUST** increment the version number. Use a git file rename to update the name of the file. Examples of such breaking changes @@ -117,7 +117,7 @@ include: `fssdiscoveryscan-v1.0.json` Schema file. 3. You will need to add two lines to `src/eddn/conf/Settings.py` in order to - have the Gateway actually recognise the new Schema. You are adding to the + have the Gateway recognise the new Schema. You are adding to the end of the `GATEWAY_JSON_SCHEMAS` dictionary. Both the live Schema *and* the `/test` version **MUST** be added. For `fssdiscoveryscan-v1.0.json` you would add: @@ -130,13 +130,36 @@ include: existing Schemas. Keep the trailing comma on the final entry, Python allows it, and it will reduce the diff on adding any further Schemas. +4. You MUST add a file containing an example full EDDN message in the + `scripts/testing/gateway-responses/` directory. + 1. Name the file as per the Schema name. In the case of only adding a + single valid message you can use e.g. `newschema.json`. + 1. If adding variants of valid messages then please annotate the + filename appropriately, e.g. `newschema-inspace.json` and + `newschema-onbody.json`. + 2. If adding variants that are *invalid* in some manner them name + the files as e.g. `newschema-invalid-no-starpos.json`. + 2. The file MUST contain the full plain-text of an EDDN upload, as would be + sent in the body of a request to the `/upload/` endpoint. Test it with + the `scripts/testing/gateway-response/test-sender.py` script. + 4. Base the `message` part of this on actual source data, e.g. a line + from a relevant Journal file. + 5. Ensure the message `timestamp` value is sane, i.e. from a time period + in which the game was providing this data. + 6. Ensure any data added as an augmentation is correct, i.e. + the proper co-ordinates of the named StarSystem in StarPos. + + This will aid in confirming that the new schema actually works for a valid + message. You MAY add additional examples that are invalid in various ways + that the schema will detect, but this is not required. + #### Schema file requirements -1. The file **MUST** actually be valid JSON, without any special extensions +1. The file **MUST** be valid JSON, without any special extensions (so no comments). Remember that JSON does **not** allow for a trailing comma on the last entry of an array or dictionary. 2. The file **MUST** comply with the relevant JSON Schema definition. -3. The file **MUST** actually load using Python's `simplejson` module, as this +3. The file **MUST** load using Python's `simplejson` module, as this is what the Gateway code uses. The script `contrib/test-schema.py` will check both this and that the validation code doesn't choke on it. 4. All new Schemas **MUST** comply with all requirements outlined in the From 779fa75b1e4f1aab412a8091da44be94ff86d9f1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 16:37:15 +0000 Subject: [PATCH 23/89] docs/Contributing: Branches, Code Changes, Fix/Enhancement, and more --- docs/Contributing.md | 180 +++++++++++++++++++++++++++++++++---------- 1 file changed, 140 insertions(+), 40 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index ac9f0f1..3e71521 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -2,9 +2,11 @@ ## Introduction -This file is still mostly a stub. +This document is intended to solidly and usefully define necessary information +pertaining to either improving the EDDN software, or add a new Schema to the +supported set. -## Text format +## File formatting and editor configuration The project contains an `.editorconfig` file at its root. Please either ensure your editor is taking note of those settings, or cross-check its contents @@ -12,72 +14,168 @@ with the [editorconfig documentation](https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties) , and ensure your editor/IDE's settings match. +## Branches and other project miscellanea + +This project utilises a number of Git branches: + +- `live` - The Live service should, outside of bried periods during deployment, + always be running using the software and supporting files from this branch. + This is where you should be looking if checking current behaviour of the + service. + +- `master` - This is in effect the staging area for any changes considered + ready for deployment to the Live service. + +- `beta` - Changes under consideration for Live deployment that are currently + undergoing further testing. If the Beta service is running it should be + using this branch, but there might be exceptions. + There MAY be changes in `develop` *and* `master` that were not merged here. + +- `develop` - Usually any Pull Requests will be merged here first. This is the + default branch against which any new work should be undertaken. Urgent + bug fixes should be the only exception to this, and even then it would only + happen if `develop` already contains non-Live changes that are not yet + considered ready for deployment. + +You might also see 'work in progress' branches with a `fix/` or `enhancement/` +prefix. + ## Code Changes +All code changes should start with +[an open Issue on GitHub](https://github.com/EDCD/EDDN/issues?q=is%3Aissue+is%3Aopen). +If no pertinent issue already exists then please create one. + +All Pull Requests should be made against the `develop` branch unless you are +directed to do otherwise. A Pull Request that is opened without prior +discussion in a relevant Issue is liable to be closed without further +consideration, but exceptions may be made for 'obvious' changes. + +## Testing + +As of 2022-01-28 the project still does not contain any automated tests, +neither unit or functional. But you should make every effort to test any +changes, including new Schemas, or changes to existing ones, before opening +a Pull Request for those changes. + +`scripts/testing/` exists and might contain some scripts and supporting files +that will be useful in this. + ### Bug Fixes +An urgent Bug Fix may be fast-tracked through to the `master` branch, but will +by default go through the `develop` branch. + +Where changes pertain to fixing a bug they should be in a branch named as per +the convention `fix//`, e.g. +`fix/123/avoid-decompress-crash`. + ### Enhancements +Any changes to existing code or supporting files that does not address a bug +is considered an enhancement. Examples would be: + +- Changes to an existing Schema to better support actual game data and + potential uses by Listeners. If you're not sure whether the change is an + Enhancement or Fix, use your best assessment, we won't bite your head off. +- Adding a wholly new Schema. +- Improving the Monitor web page, be that adding extra output or a new way to + view or manipulate the data presented. + +Where changes pertain to adding wholly new functionality, including adding a +new schema, or improving an existing feature, then they should be in a branch +named as per the convention `enhancement//` +, e.g. `enhancement/234/add-schema-somenewevent`. + ## Adding a New Schema If you think you have a good case for an additional EDDN Schema then there are several things you should consider: -1. What is the source of the data? In almost all circumstances the only - acceptable sources are the game Journal files and the Frontier CAPI service. - We do *NOT* want manually entered data being sent over EDDN, it's too prone - to error. +1. Ensure you read + [the general Schemas README](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) + so as to be aware of general requirements that your new Schema will need to + adhere to. -2. Is the new Schema going to be practically useful ? + You might also find useful information in the other Schema-specific README + files. Certainly check those if you encounter trouble documenting a new + Schema. + +2. What is the source of the data? In almost all circumstances the only + acceptable sources are the game Journal files and the Frontier CAPI service. + We do *NOT* accept any manually entered data being sent over EDDN, it's too + prone to error. + + - Do **NOT** blindly trust that the Frontier-provided Journal documentation + is correct with respect to the current game version. Gather actual + examples of Journal output under as varied circumstances as are relevant, + and base your new Schema on what you learn. + + - Remember that there might be differences between data from a player using + a Horizons version of the game versus an Odyssey version. This is why + all Schemas should mandate augmentation with `horizons` and `odyssey` + flags, but there might be other considerations when defining a Schema. + +3. Is the new Schema going to be practically useful ? 1. What use cases are there for the new data? - 2. Who benefits ? - 3. What's the likely volume of messages compared to existing Schemas? If + 2. Given that nowhere near all players will be running an EDDN + sender, and even where they do we might still miss some relevant data, + is this data still useful ? + + e.g. the owner of a Fleet Carrier sending data about their buy and sell + orders is useful, but if they then don't log in again for a while + there'll be no update to the FC state. Likewise for an FC jumping + between systems. + + At the very least you should consider, and document, caveats about the + data for the benefit of Listeners. + 3. Who benefits ? If the only interested Listener would be a very niche + project, with no benefit to others, then perhaps you should instead + consider e.g. an EDMarket Connector plugin that sends to your own + server ? + 4. What's the likely volume of messages compared to existing Schemas? If there would often be many messages in a short space of time consider requiring senders to batch them. 2022-01-28: There's no live example of this yet, but see [discussion of adding support for FSSSignalDiscovered](https://github.com/EDCD/EDDN/issues/152) . - 4. What's the likely size range of the new messages? The Gateway has a + 5. What's the likely size range of the new messages? The Gateway has a limit on the size of the *body* of `/upload/` requests. Check [live branch src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py) `bottle.BaseRequest.MEMFILE_MAX = ...` for the current limit. -3. For CAPI-sourced data you need to keep in mind the possible synchronization - issues between it and any necessary data augmentations from Journal data. - This might mean needing to make such augmentations optional. +4. For CAPI-sourced data you need to keep in mind possible synchronization + issues between it and any necessary data augmentations from Journal data. + This might mean needing to make such augmentations optional. -4. For Journal-sourced data if the source is an event not yet allowed by any - existing Schema then you MUST define a wholly new Schema for the data. This - allows you to fully specify both required and forbidden information in the - resulting messages. - The Journal events that are handled in the generic `journal` Schema are only - there for historical reasons and due to the difficulties in ensuring all - listeners and senders migrate to separate Schemas in a synchronized manner. +5. For Journal-sourced data if the source is an event not yet allowed by any + existing Schema then you MUST define a wholly new Schema for the data. This + allows you to fully specify both required and forbidden information. -5. Ensure you read - [the general Schemas README](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) - so as to be aware of general requirements that your new Schema will need to - adhere to. + The Journal events that are handled in the generic `journal` Schema are only + there for historical reasons and due to the difficulties in ensuring all + listeners and senders migrate to separate Schemas in a synchronized manner. -7. You **MUST** - [open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) - in order to propose the new Schema. If a consensus appears to have been - reached in comments therein then start work on a Pull Request. +6. You **MUST** + [open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) + in order to propose the new Schema. If a consensus appears to have been + reached in comments therein then start work on a Pull Request. -8. There must be at least one working Sender implementation before the Pull - Request for a new Schema will be merged into the live service. Experience +7. There must be at least one working Sender implementation before the Pull + Request for a new Schema will be merged into the Live service. Experience has demonstrated that there are often caveats and gotchas discovered during the development of a Sender for a new Schema. + Often this will end up being a Pull Request against either [Elite Dangerous Market Connector](https://github.com/EDCD/EDMarketConnector) 's EDDN plugin, or [ED Discovery](https://github.com/EDDiscovery/EDDiscovery). The Schema files are placed in the `schemas/` directory, located in the root -of the project structure. They are -[JSON](https://www.json.org/json-en.html) -files, conforming to the -[JSON Schema](https://json-schema.org/) -specification. As of 2022-01-28 we still use 'draft 04' of this specification. +of the project structure. + +They are [JSON](https://www.json.org/json-en.html) files, conforming to the +[JSON Schema](https://json-schema.org/) specification. +As of 2022-01-28 we still use 'draft 04' of this specification. We are looking into updating to the latest in [#139 - Update to latest JSON schema version(s) ](https://github.com/EDCD/EDDN/issues/139). @@ -102,7 +200,7 @@ include: ### Necessary file edits -1. Obviously you need to create the new file, in the `schemas/` directory. +1. Obviously you need to create the new file in the `schemas/` directory. This should be named as per the data source, i.e. Journal `event` value, and include the Schema version, and `.json` extension. You **MUST** fold the the `event` value to lower case for this. @@ -113,7 +211,7 @@ include: placed in the `schemas/` directory. The name should match that of the Schema file itself, without the version, and with a `.md` extention instead of `.json`. - An example is `fssdiscoveryscan-README.md` documents the + An example is `fssdiscoveryscan-README.md` documenting the `fssdiscoveryscan-v1.0.json` Schema file. 3. You will need to add two lines to `src/eddn/conf/Settings.py` in order to @@ -130,18 +228,20 @@ include: existing Schemas. Keep the trailing comma on the final entry, Python allows it, and it will reduce the diff on adding any further Schemas. -4. You MUST add a file containing an example full EDDN message in the +4. You MUST add a file containing an example **valid** full EDDN message in the `scripts/testing/gateway-responses/` directory. 1. Name the file as per the Schema name. In the case of only adding a single valid message you can use e.g. `newschema.json`. 1. If adding variants of valid messages then please annotate the - filename appropriately, e.g. `newschema-inspace.json` and - `newschema-onbody.json`. + filename appropriately, e.g. `newschema-valid-inspace.json` and + `newschema-valid-onbody.json`. 2. If adding variants that are *invalid* in some manner them name the files as e.g. `newschema-invalid-no-starpos.json`. 2. The file MUST contain the full plain-text of an EDDN upload, as would be sent in the body of a request to the `/upload/` endpoint. Test it with the `scripts/testing/gateway-response/test-sender.py` script. + 3. Please have the `$schemaRef` key:value first, followed by the `header` + dictionary, and only then the `message` dictionary. 4. Base the `message` part of this on actual source data, e.g. a line from a relevant Journal file. 5. Ensure the message `timestamp` value is sane, i.e. from a time period From 43ea088f7a46471f597430b9aa2cba32de7f0134 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:37:52 +0000 Subject: [PATCH 24/89] docs/Contributing: 'bried' typo --- README.md | 2 +- docs/Contributing.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a11d7e2..43c4135 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ your own test service. It is hoped that in the future the code will allow for easily running in a "local only" mode, not requiring any reverse proxy or internet-valid TLS certificates. ---- +-- --- ## Misc diff --git a/docs/Contributing.md b/docs/Contributing.md index 3e71521..ec2ba5b 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -18,7 +18,7 @@ with the This project utilises a number of Git branches: -- `live` - The Live service should, outside of bried periods during deployment, +- `live` - The Live service should, outside of brief periods during deployment, always be running using the software and supporting files from this branch. This is where you should be looking if checking current behaviour of the service. From 0e791c53b45f2351fe9c46f711d0ca276dc7d37d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:43:30 +0000 Subject: [PATCH 25/89] docs/Contributing: Keep "must be valid JSON" etc in one place --- docs/Contributing.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index ec2ba5b..87bdeec 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -171,15 +171,8 @@ several things you should consider: EDDN plugin, or [ED Discovery](https://github.com/EDDiscovery/EDDiscovery). The Schema files are placed in the `schemas/` directory, located in the root -of the project structure. - -They are [JSON](https://www.json.org/json-en.html) files, conforming to the -[JSON Schema](https://json-schema.org/) specification. -As of 2022-01-28 we still use 'draft 04' of this specification. -We are looking into updating to the latest in -[#139 - Update to latest JSON schema version(s) ](https://github.com/EDCD/EDDN/issues/139). - -All Schema files MUST be accompanied by a MarkDown formatted README file. +of the project structure. See [Schema file requirements](#schema-file-requirements) +for more information. ### Always start a new Schema at version 1 @@ -255,10 +248,17 @@ include: #### Schema file requirements -1. The file **MUST** be valid JSON, without any special extensions - (so no comments). Remember that JSON does **not** allow for a trailing - comma on the last entry of an array or dictionary. -2. The file **MUST** comply with the relevant JSON Schema definition. + +1. The file **MUST** be valid [JSON](https://www.json.org/json-en.html), + without any special extensions (so no comments). Remember that JSON does + **not** allow for a trailing comma on the last entry of an array or + dictionary. +2. The file **MUST** comply with the relevant + [JSON Schema](https://json-schema.org/) definition. + + As of 2022-01-28 we still use 'draft 04' of this specification. + We are looking into updating to the latest in + [#139 - Update to latest JSON schema version(s) ](https://github.com/EDCD/EDDN/issues/139). 3. The file **MUST** load using Python's `simplejson` module, as this is what the Gateway code uses. The script `contrib/test-schema.py` will check both this and that the validation code doesn't choke on it. @@ -273,6 +273,8 @@ include: it does no harm and might make data handling for Listeners easier, i.e. they can just pass all "oh, that's from Journal data" messages through the same initial handling. +6. All Schema files MUST be accompanied by a MarkDown formatted + [README file](#schema-readme-requirements). #### Schema README requirements From 180fe9ec944c691d186570ba234d09e784f7c3b3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:51:32 +0000 Subject: [PATCH 26/89] docs/Contributing: Minor rewording for better grammar --- docs/Contributing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 87bdeec..7a252bc 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -265,8 +265,8 @@ include: 4. All new Schemas **MUST** comply with all requirements outlined in the [general Schemas documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md). If you have a good reason why your new Schema can't and shouldn't comply - with any such then consensus will need to be achieved on changing those - requirements and/or allowing the exception. + with these requirements, then consensus will need to be achieved on changing + those requirements and/or allowing the exception. 5. If the data source is a game Journal event then you **MUST** include the `event` key and its value as in the source data. This might seem redundant when we mandate a separate Schema for any newly handled Journal event, but From 42d21111c3bb53db781d23a8518331855b2d57a3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:51:57 +0000 Subject: [PATCH 27/89] schemas/README: Linkify to endpoint listing on mention of beta/dev services --- schemas/README-EDDN-schemas.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 82987b2..a245d11 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -100,10 +100,12 @@ value, e.g. You MUST also utilise these test forms of the schemas when first testing your code. -There might also be a beta.eddn.edcd.io, or dev.eddn.edcd.io, service -available from time to time as necessary, e.g. for testing new schemas or -changes to existing ones. Ask on the `#eddn` channel of the EDCD Discord -(see https://edcd.github.io/ for an invite link). +As well as the Live service there are also `beta` and `dev` +[endpoints](../README.md#eddn-endpoints) which might be available from time +to time as necessary, e.g. for testing new schemas or changes to existing +ones. Ask on the `#eddn` channel of the +[EDCD Discord](https://edcd.github.io/) (check at the bottom for the invite +link). Alternatively you could attempt [running your own test instance of EDDN](../docs/Running-this-software.md). From 62969a2cbba6e86468b302238cb3dc6034b4747f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:54:47 +0000 Subject: [PATCH 28/89] schemas/README: Journal-sourced messages MUST have `event` in new schemas This would be a 100% MUST, but we have live schemas not mandating it. --- schemas/README-EDDN-schemas.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index a245d11..37ac5b3 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -248,10 +248,13 @@ Each `message` object must have, at bare minimum: will be much more than this. Consult the [schemas and their documentation](./). 3. Where the data is sourced from a Journal event please do preserve the - event key and value. Yes, where we use an event-specific schema this + `event` key and value. Yes, where we use an event-specific schema this might seem redundant, but it might aid an EDDN listener in streamlining their code, and it does no harm. + Any new schema based on Journal data **MUST** make `event` a required + property of the `message` dictionary. + Because the first versions of some schemas were defined when only the CAPI data was available, before Journal files existed, many of the key names chosen in the schemas are based on the equivalent in CAPI data, not Journal events. From 60ed98c19c6619e33993bd9326cf04ed8e050de0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 17:55:35 +0000 Subject: [PATCH 29/89] schemas/README: Fix 'ouy' typo --- schemas/README-EDDN-schemas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 37ac5b3..bee1b48 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -258,7 +258,7 @@ Each `message` object must have, at bare minimum: Because the first versions of some schemas were defined when only the CAPI data was available, before Journal files existed, many of the key names chosen in the schemas are based on the equivalent in CAPI data, not Journal events. -This means ouy MUST rename many of the keys from Journal events to match the +This means you MUST rename many of the keys from Journal events to match the schemas. EDDN is intended to transport generic data not specific to any particular Cmdr From 8648718f38c39e33e5af78d7dbc621397ff64342 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 18:00:49 +0000 Subject: [PATCH 30/89] schemas/Docs: Mostly capitalising 'Schema' * Treat 'Schema', when referring to an EDDN schema, as a proper noun. * Additional minor grammar improvements for clarity. --- schemas/README-EDDN-schemas.md | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index bee1b48..88f96f8 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -73,7 +73,7 @@ data sometimes lags behind the game - usually by a few seconds, sometimes by minutes. You MUST check in the data from the CAPI that the Cmdr is docked, and that the station and system names match those reported from the Journal before using the data for the commodity, outfitting -and shipyard schemas: +and shipyard Schemas: 1. Retrieve the commander data from the `/profile` CAPI endpoint. 2. Check that `commander['docked']` is true. If not, abort. @@ -88,21 +88,21 @@ and shipyard schemas: ## Uploading messages -### Send only live data to the live schemas +### Send only live data to the live Schemas You MUST **NOT** send information from any non-live (e.g. alpha or beta) -version of the game to the main schemas on this URL. +version of the game to the main Schemas on this URL. You MAY send such to this URL so long as you append `/test` to the `$schemaRef` value, e.g. "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2/test", -You MUST also utilise these test forms of the schemas when first testing your -code. +You MUST also utilise these test forms of the Schemas whenever you are +testing your EDDN-handling code, be that new code or changes to existing code. As well as the Live service there are also `beta` and `dev` [endpoints](../README.md#eddn-endpoints) which might be available from time -to time as necessary, e.g. for testing new schemas or changes to existing +to time as necessary, e.g. for testing new Schemas or changes to existing ones. Ask on the `#eddn` channel of the [EDCD Discord](https://edcd.github.io/) (check at the bottom for the invite link). @@ -152,8 +152,8 @@ any failed message. You **MUST NOT** retry any message that received a HTTP `400` or `426` code. An exception can be made if, **and only if**, *you have manually verified that -you have fixed the issues with it (i.e. updated the schema/version to a -currently supported one and adjusted the data to fit that schema/version).* +you have fixed the issues with it (i.e. updated the Schema/version to a +currently supported one and adjusted the data to fit that Schema/version).* You **MAY** retry a message that initially received a `413` response (in the hopes that the EDDN service admins decided to increase the maximum @@ -168,7 +168,7 @@ In general: Each message is a JSON object in UTF-8 encoding containing the following key+value pairs: -1. `$schemaRef` - Which schema (including version) this message is for. +1. `$schemaRef` - Which Schema (including version) this message is for. 2. `header` - Object containing mandatory information about the upload; 1. `uploaderID` - a unique ID for the player uploading this data. Don't worry about privacy, the EDDN service will hash this with a key @@ -219,11 +219,11 @@ For example, a shipyard message, version 2, might look like: ``` ### Contents of `message` -Every message MUST comply with the schema its `$schemaRef` value cites. +Every message MUST comply with the Schema its `$schemaRef` value cites. Apart from short time windows during deployment of a new version the live EDDN service should always be using -[the schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). +[the Schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). So, be sure you're checking the live versions and not, e.g. those in the `master` or other branches. @@ -246,20 +246,20 @@ Each `message` object must have, at bare minimum: i.e. "too old". 2. At least one other key/value pair representing the data. In general there will be much more than this. Consult the - [schemas and their documentation](./). + [Schemas and their documentation](./). 3. Where the data is sourced from a Journal event please do preserve the - `event` key and value. Yes, where we use an event-specific schema this + `event` key and value. Yes, where we use an event-specific Schema this might seem redundant, but it might aid an EDDN listener in streamlining their code, and it does no harm. - Any new schema based on Journal data **MUST** make `event` a required + Any new Schema based on Journal data **MUST** make `event` a required property of the `message` dictionary. -Because the first versions of some schemas were defined when only the CAPI +Because the first versions of some Schemas were defined when only the CAPI data was available, before Journal files existed, many of the key names chosen -in the schemas are based on the equivalent in CAPI data, not Journal events. +in the Schemas are based on the equivalent in CAPI data, not Journal events. This means you MUST rename many of the keys from Journal events to match the -schemas. +Schemas. EDDN is intended to transport generic data not specific to any particular Cmdr and to reflect only the data that every player would see in-game in station @@ -267,20 +267,20 @@ services or the local map. To that end, uploading applications MUST ensure that messages do not contain any Cmdr-specific data (other than "uploaderID", the "horizons" flag, and the "odyssey" flag). -The individual schemas will instruct you on various elisions (removals) to +The individual Schemas will instruct you on various elisions (removals) to be made to comply with this. -Some of these requirements are also enforced by the schemas, and some things -the schemas enforce might not be explicitly called out here. So, **do** -check what you're sending against the relevant schema(s) when making any +Some of these requirements are also enforced by the Schemas, and some things +the Schemas enforce might not be explicitly called out here. So, **do** +check what you're sending against the relevant Schema(s) when making any changes to your code. -It is also advisable to Watch this repository on GitHub so as to be aware -of any changes to schemas. +It is also advisable to Watch this repository on GitHub so that you are aware +of any changes to Schemas. #### `horizons` and `odyssey` flags -Where the schema allows for them, add the `horizons` and `odyssey` +Where the Schema allows for them, add the `horizons` and `odyssey` keys with boolean values. `null` is not allowed in the values, so if you cannot determine a value do not include that key at all. The best source of these is the `LoadGame` event from journals. It's present @@ -378,18 +378,18 @@ For all failures the response body will contain text that begins `FAIL: `. Curr ``` 4. `FAIL: Schema Validation: ` - the message failed to validate - against the cited schema. e.g. + against the cited Schema. e.g. ``` FAIL: Schema Validation: [] ``` - The exact detail will be very much dependent on both the schema the + The exact detail will be very much dependent on both the Schema the message cited and the contents of the message that failed to pass the validation. In particular, if the message contains a key that is tagged 'disallowed' in - the schema the response will look like: + the Schema, then the response will look like: ``` FAIL: Schema Validation: "[]" @@ -403,7 +403,7 @@ For all failures the response body will contain text that begins `FAIL: `. Curr reports errors, and we are [hoping to improve this](https://github.com/EDCD/EDDN/issues/163). -2. `426` - `Upgrade Required` - This indicates that the cited schema, or +2. `426` - `Upgrade Required` - This indicates that the cited Schema, or version thereof, is outdated. The body of the response will be: ``` @@ -412,7 +412,7 @@ For all failures the response body will contain text that begins `FAIL: `. Curr The wording here is aimed at users of applications that send messages over EDDN. If you're the developer of such an application then obviously you need to update your code to use a currently supported - schema and version thereof. + Schema and version thereof. There shouldn't be any other variants of a 'FAIL' message. If you find @@ -442,14 +442,14 @@ your end, not the server. Once you've connected to that you will receive messages. To access the data you will first need to zlib-decompress each message. Then you will -have a textual JSON object as per the schemas. +have a textual JSON object as per the Schemas. In general, check the guidance for [Uploading messages](#uploading-messages) for the expected format of the messages. -Consumers can utilise the `$schemaRef` value to determine which schema a +Consumers can utilise the `$schemaRef` value to determine which Schema a particular message is for. There is no need to validate the messages -against the schemas yourself, as that is performed on the EDDN Gateway. +against the Schemas yourself, as that is performed on the EDDN Gateway. Messages that do not pass the schema validation there are not forwarded to receivers. From 6eb81e44615b0efccf65a595613be28a8f3feb3f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 18:16:17 +0000 Subject: [PATCH 31/89] schemas/README: Make some lists actual numbered lists This is to aid with e.g. "point 2 in "Contents of message" when referring to this documentation. --- schemas/README-EDDN-schemas.md | 72 +++++++++++++++++----------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 88f96f8..47bd5cd 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -227,48 +227,50 @@ EDDN service should always be using So, be sure you're checking the live versions and not, e.g. those in the `master` or other branches. -Each `message` object must have, at bare minimum: +1. Each `message` object must have, at bare minimum: -1. `timestamp` - string date and time in ISO8601 format. Whilst this - technically allows for any timezone to be cited you SHOULD provide this in - UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are - doing this properly. Do not claim 'Z' whilst actually using a local time - that is offset from UTC. + 1. `timestamp` - string date and time in ISO8601 format. Whilst this + technically allows for any timezone to be cited you SHOULD provide this in + UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are + doing this properly. Do not claim 'Z' whilst actually using a local time + that is offset from UTC. - If you are only utilising Journal-sourced data then simply using the - value from there should be sufficient as the PC game client is meant to - always be correctly citing UTC for this. Indeed it has been observed, - in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC - by 21 seconds both the in-game UI clock *and* the Journal event - timestamps are still properly UTC to the nearest second. + If you are only utilising Journal-sourced data then simply using the + value from there should be sufficient as the PC game client is meant to + always be correctly citing UTC for this. Indeed it has been observed, + in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC + by 21 seconds both the in-game UI clock *and* the Journal event + timestamps are still properly UTC to the nearest second. - Listeners MAY make decisions on accepting data based on this time stamp, - i.e. "too old". -2. At least one other key/value pair representing the data. In general there - will be much more than this. Consult the - [Schemas and their documentation](./). -3. Where the data is sourced from a Journal event please do preserve the - `event` key and value. Yes, where we use an event-specific Schema this - might seem redundant, but it might aid an EDDN listener in streamlining - their code, and it does no harm. + Listeners MAY make decisions on accepting data based on this time stamp, + i.e. "too old". + 2. At least one other key/value pair representing the data. In general there + will be much more than this. Consult the + [Schemas and their documentation](./). + 3. Where the data is sourced from a Journal event please do preserve the + `event` key and value. Yes, where we use an event-specific Schema this + might seem redundant, but it might aid an EDDN listener in streamlining + their code, and it does no harm. - Any new Schema based on Journal data **MUST** make `event` a required - property of the `message` dictionary. + Any new Schema based on Journal data **MUST** make `event` a required + property of the `message` dictionary. + 4. You should probably be including + [horizons and odyssey flags](#horizons-and-odyssey-flags). -Because the first versions of some Schemas were defined when only the CAPI -data was available, before Journal files existed, many of the key names chosen -in the Schemas are based on the equivalent in CAPI data, not Journal events. -This means you MUST rename many of the keys from Journal events to match the -Schemas. +2. Because the first versions of some Schemas were defined when only the CAPI + data was available, before Journal files existed, many of the key names chosen + in the Schemas are based on the equivalent in CAPI data, not Journal events. + This means you MUST rename many of the keys from Journal events to match the + Schemas. -EDDN is intended to transport generic data not specific to any particular Cmdr -and to reflect only the data that every player would see in-game in station -services or the local map. To that end, uploading applications MUST ensure -that messages do not contain any Cmdr-specific data (other than "uploaderID", -the "horizons" flag, and the "odyssey" flag). +3. EDDN is intended to transport generic data not specific to any particular Cmdr + and to reflect only the data that every player would see in-game in station + services or the local map. To that end, uploading applications MUST ensure + that messages do not contain any Cmdr-specific data (other than "uploaderID", + the "horizons" flag, and the "odyssey" flag). -The individual Schemas will instruct you on various elisions (removals) to -be made to comply with this. + The individual Schemas will instruct you on various elisions (removals) to + be made to comply with this. Some of these requirements are also enforced by the Schemas, and some things the Schemas enforce might not be explicitly called out here. So, **do** From 989b997d1548db62e34d7951d997bb626446f65a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 28 Jan 2022 18:34:00 +0000 Subject: [PATCH 32/89] schemas: Skeleton journal event template --- schemas/TEMPLATES/journalevent-v1.0.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 schemas/TEMPLATES/journalevent-v1.0.json diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json new file mode 100644 index 0000000..a960e49 --- /dev/null +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -0,0 +1 @@ +# This is not a valid JSON file From eac3e1af4ba057d40ceb163aa310296c1a2627e9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 15:47:17 +0000 Subject: [PATCH 33/89] schemas: Add a 'new journal event schema' template Hopefully the "not actually valid in JSON" comments make it obvious both what needs to be edited, and that they all need removing. --- schemas/TEMPLATES/journalevent-v1.0.json | 96 ++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json index a960e49..31d7943 100644 --- a/schemas/TEMPLATES/journalevent-v1.0.json +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -1 +1,97 @@ # This is not a valid JSON file +# Removed all the 'comments' as you are sure you have complied with their +# instructions! +# +# 1. Replace all instances of newjournalevent with the lower-case folded name +# of the Journal event this schema is for. +# 2. Replace all instances of NewJournalEvent with the name of the journal +# event this schema is for, as it appears in the Journal. Specifically, +# you must conserve the case. +# 3. Leave the 'header' dictionary exactly as it is, it is mandatory. +# 4. Add any additional schema-mandatory message properties to the 'required' +# array. +# 5. If, and ONLY IF, you have good reason to believe there might be additional +# valid keys in the source data, change the 'additionalProperties' value +# to 'true'. +# EMPHASIS: You should really know about all of the possible keys and their +# values and be defining them, possibly as optional (not listed in +# 'required') properties. If needs be document, in the README for +# this Schema, that all but the defined properties should be +# elided when constructing a message. +# 6. The 'horizons' and 'odyssey' properties in 'message' MUST BE RETAINED. +# Any message based on Journal data should be able to add them as +# appropriate. Note that they *are optional in the resulting message*. +# You MUST NOT add these to the 'required' list, as the LoadGame Journal +# event is not guaranteed to contain either, depending on which client +# (base, horizons, odyssey) the player is running. +# 7. 'some_disallowed_key' demonstrates how to specify that a key (and thus its +# value) is not allowed in this schema. +# 8. Note the trailing comma after the 'some_disallowed_key' entry in +# 'messages'. If all you did was remove these comments you would NOT have +# a valid JSON file. You should be adding **at least** one additional +# pertinent 'message' key:value definition. +# 9. You MUST NOT remove the 'disallowed' key, or edit its value. The purpose +# of this is to be cited as the allowed value on a disallowed key. As it +# defines that the only valid type for the value is **not** any of the +# possible JSON types, its effect is to disallow any value for the key, and +# as a result disallow the key. +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/newjournalevent/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "NewJournalEvent" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the game client logged that Horizons functionality was available." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the game client logged that Odyssey functionality was available." + }, + "some_disallowed_key" { "$ref" : "#/definitions/disallowed" }, + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} From 099eb72e7358af7b7c9971d829da9692cd45c034 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 16:29:17 +0000 Subject: [PATCH 34/89] schemas/README: Update horizons/odyssey flag section * Re-confirmed what is in the `LoadGame` event for all 3 possible circumstances. * Emphasised that the `Fileheader`->`Odyssey` value is different in nature. --- schemas/README-EDDN-schemas.md | 38 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 47bd5cd..b5c7b43 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -283,9 +283,10 @@ of any changes to Schemas. #### `horizons` and `odyssey` flags Where the Schema allows for them, add the `horizons` and `odyssey` -keys with boolean values. `null` is not allowed in the values, so if -you cannot determine a value do not include that key at all. The best -source of these is the `LoadGame` event from journals. It's present +keys with boolean values. `null` is not allowed in the values, so **if +you cannot determine a value do not include that key at all**. + +The only source of these is the `LoadGame` event from journals. It's present both in the PC local files and the CAPI journal data. If you're composing a shipyard or outfitting message from CAPI data then it is possible to synthesise the `horizons` flag. For now consult the function @@ -293,26 +294,43 @@ possible to synthesise the `horizons` flag. For now consult the function [EDMarketConnector:plugins/eddn.py](https://github.com/EDCD/EDMarketConnector/blob/stable/plugins/eddn.py) for a method to achieve this. -As of 2022-01-22 the following was observed for the `LoadGame` events as +As of 2022-01-29 the following was observed for the `LoadGame` events as present in CAPI-sourced Journal files (which were confirmed to match the PC-local files for these events): -- PC Odyssey Client, game version `4.0.0.1002`: +- PC Odyssey Client, game version `4.0.0.1100`: ```json - { "timestamp":"2022-01-20T11:15:22Z", "event":"LoadGame", "FID":"F", "Commander":"", "Horizons":true, "Odyssey":true,... + { "timestamp":"2022-01-29T16:17:02Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true, "Odyssey":true,... ``` -- PC Horizons Client, game version `3.8.0.403`, no `Odyssey` key was +- PC Horizons Client, game version `3.8.0.404`, no `Odyssey` key was present: ```json - { "timestamp":"2022-01-20T11:20:17Z", "event":"LoadGame", "FID":"F", "Commander":"", "Horizons":true,... + { "timestamp":"2022-01-29T16:15:07Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true,... ``` -- PC 'base' Client, game version `3.8.0.403`, no `Odyssey` key was +- PC 'base' Client, game version `3.8.0.404`, no `Odyssey` key was present: ```json - { "timestamp":"2022-01-20T11:22:54Z", "event":"LoadGame", "FID":"F", "Commander":"", "Horizons":false,... + { "timestamp":"2022-01-29T16:11:54Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":false,... ``` +Do not attempt to use the value(s) from a `Fileheader` event as the semantics +are different. With clients 3.8.0.404 and 4.0.0.1100 the following was observed: + +| Game Client | Fileheader | LoadGame | +| ---------------: | ---------------- | ------------------------------: | +| Base | "Odyssey":false | "Horizons":false | +| Horizons | "Odyssey":false | "Horizons":true | +| Odyssey | "Odyssey":true | "Horizons":true, "Odyssey":true | + +NB: The 'Base' client appears to simply be the Horizons client with any +Horizons-only features disabled. + +- In the `Fileheader` event it's indicating whether it's an Odyssey game client. +- In the `LoadGame` it's indicating whether Horizons and/or Odyssey features are + active, but in the non-Odyssey game client case you only get the Horizons + boolean. + ### Server responses There are three possible sources of HTTP responses when sending an upload to EDDN. From 5fe2642d03818e8829b7fb8e2a3b8463aeebe29b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 16:45:11 +0000 Subject: [PATCH 35/89] schemas: First cut of newjournalevent-README.md template --- schemas/TEMPLATES/journalevent-README.md | 98 ++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 schemas/TEMPLATES/journalevent-README.md diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md new file mode 100644 index 0000000..beb115e --- /dev/null +++ b/schemas/TEMPLATES/journalevent-README.md @@ -0,0 +1,98 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +!!! STOP !!! + +!! These are not valid MarkDown comments. + +!! You MUST read, comply with, and then remove all of these lines to have a + +!! valid Schema README file. + +1. Be careful if using any existing Schema, or its README, as reference for a + new Schema. Some of them will have some departures from best practices for + historical reasons. It's usually not easy to co-ordinate all Listeners and + Senders smoothly updating to a new version of a Schema, so less than ideal + requires/optional/disallowed definitions might be present in older Schemas. + + [The main Schema documentation](../README-EDDN-schemas.md) is the canonical + authority. +2. Replace all instances of `NewJournalEvent` with the name of the actual + Journal event's Schema you are documenting. This should have the case + preserved as per how it appears in actual game Journal files. +3. Replace all instances of `newjournalevent` with the lower-case folded + version of this Schema's Journal event name. +4. For new Journal-based schemas no key renames should be necessary. + + If there are no renames of key names for this Schema, then edit the + `Key Renames` section to contain only the text `None.`. + + Where such renames *are* required do **NOT** attempt to list them all here. + That would just require updating them both here and in the actual Schema. + + If there are any, call them out in the `description` of the affected + property in the Schema. + +5. In the `Elisions` section clearly document any keys (and thus their values) + that are in the source data, but that should not be in the resulting EDDN + message. + + **You do not need to list keys with a `_Localised` suffix.** + +6. Do **NOT** remove the `horizons and odyssey flags` section. It is + mandatory that they are allowed (but are optional) in any Journal-based + EDDN Schema. + +7. If the source Journal event contains information that includes the System + name (possibly as `StarSystem` or `SystemName`) then you MUST include the + `StarPos` section in `Augmentations`. + + Additionally, you must also include `StarPos` as required in the Schema + if the source data contains `SystemAddress`. + +The line: + + # EDDN NewJournalEvent Schema + +below should ultimately be the first line in this file, after required edits. + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# EDDN NewJournalEvent Schema + +## Introduction +Here we document how to take data from an ED `NewJournalEvent` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file you should, in the first instance, assume +that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The data source for this schema is the ED Journal event `NewJournalEvent`. + +### Key Renames +Many of the key names have a different case defined in this schema, make +sure you are renaming them as appropriate. + +### Elisions +None + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) +over in the main Schema documentation. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. + +## Listeners +The advice above for [Senders](#senders), combined with the actual Schema file *should* +provide all the information you need to process these events. From fa869c6478797998b51f17b4f36209011887b4dd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 16:47:58 +0000 Subject: [PATCH 36/89] schemas: Update horizons/odyssey descriptins in schema template --- schemas/TEMPLATES/journalevent-v1.0.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json index 31d7943..d742b61 100644 --- a/schemas/TEMPLATES/journalevent-v1.0.json +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -81,11 +81,11 @@ }, "horizons": { "type" : "boolean", - "description" : "Whether the game client logged that Horizons functionality was available." + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." }, "odyssey": { "type" : "boolean", - "description" : "Whether the game client logged that Odyssey functionality was available." + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." }, "some_disallowed_key" { "$ref" : "#/definitions/disallowed" }, } From 5f5d0afb2936eedf0c89c3d3bcea4dff844b3ac8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 16:53:42 +0000 Subject: [PATCH 37/89] schemas/templates: Minor formatting/grammer cleanups --- schemas/TEMPLATES/journalevent-README.md | 18 +++++++++++------- schemas/TEMPLATES/journalevent-v1.0.json | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index beb115e..78fe032 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -42,12 +42,16 @@ mandatory that they are allowed (but are optional) in any Journal-based EDDN Schema. -7. If the source Journal event contains information that includes the System - name (possibly as `StarSystem` or `SystemName`) then you MUST include the - `StarPos` section in `Augmentations`. +7. If: + 1. either the source Journal event contains information that includes the + System name (possibly as `StarSystem` or `SystemName`), **OR** the source + data contains a `SystemAddress` value, + 2. and a `StarPos` array is *not already present* in the source data. + + then you MUST include the `StarPos` section in `Augmentations`. - Additionally, you must also include `StarPos` as required in the Schema - if the source data contains `SystemAddress`. + If neither key is in the source data then remove the `StarPos` section from + this document. The line: @@ -67,8 +71,8 @@ Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. If you find any discrepancies between what this document says and what is -defined in the relevant Schema file you should, in the first instance, assume -that it is the Schema file that is correct. +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. **PLEASE open [an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) to report any such anomalies you find so that we can check and resolve the diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json index d742b61..7e044cb 100644 --- a/schemas/TEMPLATES/journalevent-v1.0.json +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -84,7 +84,7 @@ "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." }, "odyssey": { - "type" : "boolean", + "type" : "boolean", "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." }, "some_disallowed_key" { "$ref" : "#/definitions/disallowed" }, From 64dd5832d4ea1c7e2b2b4d638b636bd59702a8fc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 16:56:08 +0000 Subject: [PATCH 38/89] schemas: Journal event template: Couple `StarPos` instructions --- schemas/TEMPLATES/journalevent-README.md | 5 +++-- schemas/TEMPLATES/journalevent-v1.0.json | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index 78fe032..e56aa9d 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -48,10 +48,11 @@ data contains a `SystemAddress` value, 2. and a `StarPos` array is *not already present* in the source data. - then you MUST include the `StarPos` section in `Augmentations`. + then you MUST include the `StarPos` section in `Augmentations` and add + `StarPos` to the `required` message properties in the Schema file. If neither key is in the source data then remove the `StarPos` section from - this document. + this document and the Schema file. The line: diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json index 7e044cb..143f47b 100644 --- a/schemas/TEMPLATES/journalevent-v1.0.json +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -30,7 +30,9 @@ # 'messages'. If all you did was remove these comments you would NOT have # a valid JSON file. You should be adding **at least** one additional # pertinent 'message' key:value definition. -# 9. You MUST NOT remove the 'disallowed' key, or edit its value. The purpose +# 9. Consult the journalevent-README.md file, particularly the section about +# `StarPos` and remove that section of this file if it is not needed. +# 10. You MUST NOT remove the 'disallowed' key, or edit its value. The purpose # of this is to be cited as the allowed value on a disallowed key. As it # defines that the only valid type for the value is **not** any of the # possible JSON types, its effect is to disallow any value for the key, and @@ -87,6 +89,13 @@ "type" : "boolean", "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender" + }, "some_disallowed_key" { "$ref" : "#/definitions/disallowed" }, } } From c9b3dd374767b19491e9b406de0c0f0bc2ca5105 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 17:02:34 +0000 Subject: [PATCH 39/89] monitor: Update 'provided by' EDSM -> EDCD --- contrib/monitor/index.html | 2 +- contrib/monitor/schemas.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/monitor/index.html b/contrib/monitor/index.html index 990386f..9f16a47 100644 --- a/contrib/monitor/index.html +++ b/contrib/monitor/index.html @@ -53,7 +53,7 @@

- Provided by EDSM + Provided by EDCD

diff --git a/contrib/monitor/schemas.html b/contrib/monitor/schemas.html index 34ac82b..4940b3b 100644 --- a/contrib/monitor/schemas.html +++ b/contrib/monitor/schemas.html @@ -41,7 +41,7 @@

- Provided by EDSM + Provided by EDCD

From 0b17f026036bae87887534079edaa8d2a62fb904 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 17:15:40 +0000 Subject: [PATCH 40/89] README: Re-word "use the correct hostname" in endpoints --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 43c4135..4a273c5 100644 --- a/README.md +++ b/README.md @@ -111,10 +111,9 @@ available. You **MUST** use the correct hostname in the Upload URLs as these are TLS/HTTPS connections terminated on a Reverse Proxy. -You can get away with using alternate hostnames on the Listener URLs, as -they're purely a ZeroMQ endpoint, no TLS. But don't be surprised if either -the Beta or Dev service is actually running on a different IP, so don't -chance it. +The Listener URLs are ZeroMQ endpoints, no TLS. But whilst this means you +don't strictly need to use the correct hostname there is no guarantee that the +beta and dev hostnames won't be pointing at, or hosted on, a different IP. If you need to test some of your own changes then please read [Running this software](docs/Running-this-software.md) for how to instantiate @@ -122,7 +121,7 @@ your own test service. It is hoped that in the future the code will allow for easily running in a "local only" mode, not requiring any reverse proxy or internet-valid TLS certificates. --- +--- --- ## Misc From d802448e4c86772e5c28df65c6192a07d6c0f993 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 17:18:43 +0000 Subject: [PATCH 41/89] README: Linkify 'a defined format' & simplify plaintext/compression header --- schemas/README-EDDN-schemas.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index b5c7b43..930f22d 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -3,13 +3,14 @@ ## Introduction EDDN is a -[zermoq](https://zeromq.org/) service to allow players of the game +[zermoq](https://zeromq.org/) service which allows players of the game [Elite Dangerous](https://www.elitedangerous.com/), published by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so that interested listeners can receive a copy. -EDDN accepts HTTP POST uploads in a defined format representing this game data -and then passes it on to any interested listeners. +EDDN accepts HTTP POST uploads in +[a defined format](#sending-data) +representing this game data and then passes it on to any interested listeners. --- ## Sources @@ -120,12 +121,9 @@ Messages sent to EDDN **MUST**: - Use a **POST** request, with the body containing the EDDN message. No query parameters in the URL are supported or necessary. -The body of an EDDN message is a JSON object in UTF-8 encoding. You MUST -set a `Content-Type` header of `applicaton/json`, and NOT any of: - -* `application/x-www-form-urlencoded` -* `multipart/form-data` -* `text/plain` +The body of an EDDN message is a JSON object in UTF-8 encoding. If you do not +compress this body then you MUST set a `Content-Type` header of +`applicaton/json`. For historical reasons URL form-encoded data *is* supported, **but this is deprecated and no new software should attempt this method**. We @@ -133,7 +131,7 @@ purposefully do not further document the exact format for this. You *MAY* use gzip compression on the body of the message, but it is not required. If you do compress the body then you **MUST* send a `Content-Type` -header with `gzip` value instead of `application/json`. +header of `gzip` instead of `application/json`. You should be prepared to handle all scenarios where sending of a message fails: From da109f85e7fa4e86563ba762d5d537a717d37884 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 17:24:21 +0000 Subject: [PATCH 42/89] schemas/README: State that schema files trump their README --- schemas/README-EDDN-schemas.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 930f22d..a73012a 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -217,7 +217,16 @@ For example, a shipyard message, version 2, might look like: ``` ### Contents of `message` -Every message MUST comply with the Schema its `$schemaRef` value cites. +Every message MUST comply with the Schema its `$schemaRef` value cites. Each +Schema file should have a matching `-README.md` file in the project +root `schemas/` directory. Always consult this so that you're aware of any +Schema-specific requirements. + +The Schema file, `-v.json`, is considered the authority on +the format of messages for that Schema. If anything in the accompanying +documentation is in disagreement with this then please +[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +so that we can investigate and correct, or clarify, as necessary. Apart from short time windows during deployment of a new version the live EDDN service should always be using From 7337f83061be1287086a56292ebb02fa7d7cbd82 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 17:26:25 +0000 Subject: [PATCH 43/89] schemas/README: Tweak order and text of 'minimum message' --- schemas/README-EDDN-schemas.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index a73012a..e65ac59 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -251,18 +251,19 @@ So, be sure you're checking the live versions and not, e.g. those in the Listeners MAY make decisions on accepting data based on this time stamp, i.e. "too old". - 2. At least one other key/value pair representing the data. In general there - will be much more than this. Consult the - [Schemas and their documentation](./). - 3. Where the data is sourced from a Journal event please do preserve the + 2. Where the data is sourced from a Journal event please do preserve the `event` key and value. Yes, where we use an event-specific Schema this might seem redundant, but it might aid an EDDN listener in streamlining their code, and it does no harm. Any new Schema based on Journal data **MUST** make `event` a required property of the `message` dictionary. - 4. You should probably be including - [horizons and odyssey flags](#horizons-and-odyssey-flags). + 3. At least one other key/value pair representing the data. In general there + will be much more than this. Consult the + [Schemas and their documentation](./). + 4. Include + [horizons and odyssey flags](#horizons-and-odyssey-flags) where the + Schema requires them. 2. Because the first versions of some Schemas were defined when only the CAPI data was available, before Journal files existed, many of the key names chosen From c1cee82cb6ae62da55d0b5d088658143b44f0e94 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 18:26:44 +0000 Subject: [PATCH 44/89] README: Point to docs/Contribyting.md#adding-a-new-schema --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4a273c5..4d47bb7 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,14 @@ your own test service. It is hoped that in the future the code will allow for easily running in a "local only" mode, not requiring any reverse proxy or internet-valid TLS certificates. +#### New Schemas +*All* new Schema proposals **MUST** be started by opening +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose). Please +consult +[docs/Contributing 'Adding a New Schema'](docs/Contributing.md#adding-a-new-schema) +for further guidelines. + + --- --- From f3845720275c0434668312e48cc3a68956400e8d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 18:27:16 +0000 Subject: [PATCH 45/89] docs/Contributing: Small grammar tweak --- docs/Contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 7a252bc..5b55b52 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -94,7 +94,7 @@ several things you should consider: 1. Ensure you read [the general Schemas README](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) - so as to be aware of general requirements that your new Schema will need to + to be aware of general requirements that your new Schema will need to adhere to. You might also find useful information in the other Schema-specific README From 3a2b1d03097376fbaf400c2e7f55a1fac9919294 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 29 Jan 2022 18:31:02 +0000 Subject: [PATCH 46/89] docs/Contributing: Reference schema template files --- docs/Contributing.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 5b55b52..2a2cddc 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -161,7 +161,12 @@ several things you should consider: in order to propose the new Schema. If a consensus appears to have been reached in comments therein then start work on a Pull Request. -7. There must be at least one working Sender implementation before the Pull +7. Consult + [the template for new Journal-based schemas](../schemas/TEMPLATES/journalevent-v1.0.json) + and [its README template](../schemas/TEMPLATES/journalevent-README.md) for + an outline and some further guidance. + +8. There must be at least one working Sender implementation before the Pull Request for a new Schema will be merged into the Live service. Experience has demonstrated that there are often caveats and gotchas discovered during the development of a Sender for a new Schema. From 396a2f0b3a46c9531675805b45284e846d597bc7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:38:32 +0000 Subject: [PATCH 47/89] docs/schemas: Improve "what must be present or removed" --- docs/Contributing.md | 22 ++++++++++++++++++++++ schemas/README-EDDN-schemas.md | 9 ++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 2a2cddc..dee05f7 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -281,6 +281,28 @@ include: 6. All Schema files MUST be accompanied by a MarkDown formatted [README file](#schema-readme-requirements). +#### Data that should be allowed in a new Schema + +The default is to define the new Schema such that all possible data in the +source Journal event is allowed. + +However, you **MUST** consider which source data constitutes information +personal to the player, e.g. their credit balance, or the specific amount they +paid for something after the applicable discounts and mandate its removal in +the Schema. + +Furthermore, the Schema **MUST** mandate removal of all keys (and thus their +values as well) where the key name has the suffix `_Localised`. These +key:value pairs should be accompanied in the source data by a non-Localised +version which will remain, unless otherwise elided. + +You should gather as many examples as possible of actual game Journal events, +and then define the Schema such that anything (after mandated elisions as +above) that is always in the source data is *required* in the new Schema. + +Anything that does not always occur in the source data, and is not defined and +as a mandatory elision, should be defined, but remain optional. + #### Schema README requirements The per-Schema README **MUST** give both Senders and Listeners sufficient diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index e65ac59..f69d9d9 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -273,9 +273,12 @@ So, be sure you're checking the live versions and not, e.g. those in the 3. EDDN is intended to transport generic data not specific to any particular Cmdr and to reflect only the data that every player would see in-game in station - services or the local map. To that end, uploading applications MUST ensure - that messages do not contain any Cmdr-specific data (other than "uploaderID", - the "horizons" flag, and the "odyssey" flag). + services or the local map. To that end: + 1. Uploading applications MUST ensure that messages do not contain any + Cmdr-specific data (other than "uploaderID", the "horizons" flag, and + the "odyssey" flag). + 2. Uploading applications MUST remove any data where the name of the + relevant key has a `_Localised` suffix. The individual Schemas will instruct you on various elisions (removals) to be made to comply with this. From 4ade29866c19c22e17eddc88c4013f3f0c522a55 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:40:53 +0000 Subject: [PATCH 48/89] docs/Contributing.md: Add some section separators --- docs/Contributing.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index dee05f7..986c0b6 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -6,6 +6,8 @@ This document is intended to solidly and usefully define necessary information pertaining to either improving the EDDN software, or add a new Schema to the supported set. +--- + ## File formatting and editor configuration The project contains an `.editorconfig` file at its root. Please either ensure @@ -14,6 +16,8 @@ with the [editorconfig documentation](https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties) , and ensure your editor/IDE's settings match. +--- + ## Branches and other project miscellanea This project utilises a number of Git branches: @@ -40,6 +44,9 @@ This project utilises a number of Git branches: You might also see 'work in progress' branches with a `fix/` or `enhancement/` prefix. +--- +--- + ## Code Changes All code changes should start with @@ -51,7 +58,7 @@ directed to do otherwise. A Pull Request that is opened without prior discussion in a relevant Issue is liable to be closed without further consideration, but exceptions may be made for 'obvious' changes. -## Testing +### Testing As of 2022-01-28 the project still does not contain any automated tests, neither unit or functional. But you should make every effort to test any @@ -87,6 +94,9 @@ new schema, or improving an existing feature, then they should be in a branch named as per the convention `enhancement//` , e.g. `enhancement/234/add-schema-somenewevent`. +--- +--- + ## Adding a New Schema If you think you have a good case for an additional EDDN Schema then there are @@ -179,6 +189,8 @@ The Schema files are placed in the `schemas/` directory, located in the root of the project structure. See [Schema file requirements](#schema-file-requirements) for more information. +--- + ### Always start a new Schema at version 1 The first time a new Schema goes live it should be as version 1. @@ -196,6 +208,8 @@ include: - If you change a property from optional to required or disallowed. Senders will need to update. Listeners can no longer expect it, if disallowed. +--- + ### Necessary file edits 1. Obviously you need to create the new file in the `schemas/` directory. @@ -328,3 +342,8 @@ MarkDown linking is helpful. 5. If you use another Schema's README as the basis for yours then you MUST remove any text that isn't relevant to your Schema. + +--- +--- + + From 3e69b7b630c5fedf036e2ebcbae13566ffe70f27 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:46:20 +0000 Subject: [PATCH 49/89] docs: Move most of schemas/README into docs/Developers.md --- docs/Developers.md | 111 +++++++++++++++++++++++++++++++++ schemas/README-EDDN-schemas.md | 111 --------------------------------- 2 files changed, 111 insertions(+), 111 deletions(-) create mode 100644 docs/Developers.md diff --git a/docs/Developers.md b/docs/Developers.md new file mode 100644 index 0000000..88e8225 --- /dev/null +++ b/docs/Developers.md @@ -0,0 +1,111 @@ +## Introduction + +EDDN is a +[zermoq](https://zeromq.org/) service which allows players of the game +[Elite Dangerous](https://www.elitedangerous.com/), published +by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so +that interested listeners can receive a copy. + +EDDN accepts HTTP POST uploads in +[a defined format](#sending-data) +representing this game data and then passes it on to any interested listeners. + +--- +## Sources + +There are two sources of game data, both provided by the publisher of the game, +Frontier Developerments. They are both explicitly approved for use by +third-party software. + +### Journal Files + +On the PC version of the game, "Journal files" are written during any game +session. These are in newline-delimited JSON format, with each line +representing a single JSON object. Frontier Developments publishes +documentation for the various events in their +[Player Tools & API Discussions](https://forums.frontier.co.uk/forums/elite-api-and-tools/) +forum. + +In general the documentation is made available in a file named something like: + + Journal_Manual-v + +as both a MicroSoft word `.doc` file, or a `.pdf` file. Historically the +use of `_` versus `-` in those filenames has varied. + +Consult the latest of these for documentation on individual events. +However, be aware that sometimes the documentation is in error, possibly due to +not having been updated after a game client change. + +### Companion API (CAPI) data + +Frontier Developments provides an API to retrieve certain game data, even +without the game running. Historically this was for use by its short-lived +iOS "Companion" app, and was only intended to be used by that app. There was no +public documentation, or even admission of its existence. + +Eventually, after some enterprising players had snooped the connections and +figured out the login method and endpoints, Frontier Developments +[allowed general use of this](https://forums.frontier.co.uk/threads/open-letter-to-frontier-developments.218658/page-19#post-3371472) +. + +Originally the API authentication required being supplied with the email and +password as used to login to the game (but at least this was over HTTPS). + +In late 2018 Frontier switched the authentication to using an oAuth2 flow, +meaning players no longer need to supply their email and password to +third-party sites and clients. + +As of October 2021 there has still never been any official documentation about +the available endpoints and how they work. There is some +[third-party documentation](https://github.com/Athanasius/fd-api/blob/main/docs/README.md) +by Athanasius. + +It is *not* recommended to use CAPI data as the source as it's fraught with +additional issues. EDMarketConnector does so in order to facilitate +obtaining data without the player needing to open the commodities screen. + +#### Detecting CAPI data lag + +When using the Companion API please be aware that the server that supplies this +data sometimes lags behind the game - usually by a few seconds, sometimes by +minutes. You MUST check in the data from the CAPI that the Cmdr is +docked, and that the station and system names match those +reported from the Journal before using the data for the commodity, outfitting +and shipyard Schemas: + +1. Retrieve the commander data from the `/profile` CAPI endpoint. +2. Check that `commander['docked']` is true. If not, abort. +3. Retrieve the data from the `/market` and `/shipyard` CAPI endpoints. +4. Compare the system and station name from the CAPI market data, + `["lastStarport"]["name"]` and `["lastSystem"]["name"]`, + to that from the last `Docked` or `Location` journal event. If either does + not match then you MUST **abort**. This likely indicates that the CAPI + data is lagging behind the game client state and thus should not be used. + +--- + +## Uploading messages + +### Send only live data to the live Schemas +You MUST **NOT** send information from any non-live (e.g. alpha or beta) +version of the game to the main Schemas on this URL. + +You MAY send such to this URL so long as you append `/test` to the `$schemaRef` +value, e.g. + + "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2/test", + +You MUST also utilise these test forms of the Schemas whenever you are +testing your EDDN-handling code, be that new code or changes to existing code. + +As well as the Live service there are also `beta` and `dev` +[endpoints](../README.md#eddn-endpoints) which might be available from time +to time as necessary, e.g. for testing new Schemas or changes to existing +ones. Ask on the `#eddn` channel of the +[EDCD Discord](https://edcd.github.io/) (check at the bottom for the invite +link). + +Alternatively you could attempt +[running your own test instance of EDDN](../docs/Running-this-software.md). + diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index f69d9d9..8ebb1d4 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -1,116 +1,5 @@ # EDDN Schemas Documentation -## Introduction - -EDDN is a -[zermoq](https://zeromq.org/) service which allows players of the game -[Elite Dangerous](https://www.elitedangerous.com/), published -by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so -that interested listeners can receive a copy. - -EDDN accepts HTTP POST uploads in -[a defined format](#sending-data) -representing this game data and then passes it on to any interested listeners. - ---- -## Sources - -There are two sources of game data, both provided by the publisher of the game, -Frontier Developerments. They are both explicitly approved for use by -third-party software. - -### Journal Files - -On the PC version of the game, "Journal files" are written during any game -session. These are in newline-delimited JSON format, with each line -representing a single JSON object. Frontier Developments publishes -documentation for the various events in their -[Player Tools & API Discussions](https://forums.frontier.co.uk/forums/elite-api-and-tools/) -forum. - -In general the documentation is made available in a file named something like: - - Journal_Manual-v - -as both a MicroSoft word `.doc` file, or a `.pdf` file. Historically the -use of `_` versus `-` in those filenames has varied. - -Consult the latest of these for documentation on individual events. -However, be aware that sometimes the documentation is in error, possibly due to -not having been updated after a game client change. - -### Companion API (CAPI) data - -Frontier Developments provides an API to retrieve certain game data, even -without the game running. Historically this was for use by its short-lived -iOS "Companion" app, and was only intended to be used by that app. There was no -public documentation, or even admission of its existence. - -Eventually, after some enterprising players had snooped the connections and -figured out the login method and endpoints, Frontier Developments -[allowed general use of this](https://forums.frontier.co.uk/threads/open-letter-to-frontier-developments.218658/page-19#post-3371472) -. - -Originally the API authentication required being supplied with the email and -password as used to login to the game (but at least this was over HTTPS). - -In late 2018 Frontier switched the authentication to using an oAuth2 flow, -meaning players no longer need to supply their email and password to -third-party sites and clients. - -As of October 2021 there has still never been any official documentation about -the available endpoints and how they work. There is some -[third-party documentation](https://github.com/Athanasius/fd-api/blob/main/docs/README.md) -by Athanasius. - -It is *not* recommended to use CAPI data as the source as it's fraught with -additional issues. EDMarketConnector does so in order to facilitate -obtaining data without the player needing to open the commodities screen. - -#### Detecting CAPI data lag - -When using the Companion API please be aware that the server that supplies this -data sometimes lags behind the game - usually by a few seconds, sometimes by -minutes. You MUST check in the data from the CAPI that the Cmdr is -docked, and that the station and system names match those -reported from the Journal before using the data for the commodity, outfitting -and shipyard Schemas: - -1. Retrieve the commander data from the `/profile` CAPI endpoint. -2. Check that `commander['docked']` is true. If not, abort. -3. Retrieve the data from the `/market` and `/shipyard` CAPI endpoints. -4. Compare the system and station name from the CAPI market data, - `["lastStarport"]["name"]` and `["lastSystem"]["name"]`, - to that from the last `Docked` or `Location` journal event. If either does - not match then you MUST **abort**. This likely indicates that the CAPI - data is lagging behind the game client state and thus should not be used. - ---- - -## Uploading messages - -### Send only live data to the live Schemas -You MUST **NOT** send information from any non-live (e.g. alpha or beta) -version of the game to the main Schemas on this URL. - -You MAY send such to this URL so long as you append `/test` to the `$schemaRef` -value, e.g. - - "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2/test", - -You MUST also utilise these test forms of the Schemas whenever you are -testing your EDDN-handling code, be that new code or changes to existing code. - -As well as the Live service there are also `beta` and `dev` -[endpoints](../README.md#eddn-endpoints) which might be available from time -to time as necessary, e.g. for testing new Schemas or changes to existing -ones. Ask on the `#eddn` channel of the -[EDCD Discord](https://edcd.github.io/) (check at the bottom for the invite -link). - -Alternatively you could attempt -[running your own test instance of EDDN](../docs/Running-this-software.md). - ### Sending data Messages sent to EDDN **MUST**: From c9d0e30e42975be6964e0d79da4f4f1f4beaadda Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:55:33 +0000 Subject: [PATCH 50/89] schemas/README: Flesh out this in its new 'stub' form The remaining extant text has also now been moved into `docs/Developers.md`. --- schemas/README-EDDN-schemas.md | 386 ++------------------------------- 1 file changed, 16 insertions(+), 370 deletions(-) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 8ebb1d4..d0d4022 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -1,377 +1,23 @@ # EDDN Schemas Documentation -### Sending data -Messages sent to EDDN **MUST**: +EDDN message Schemas are [JSON](https://www.json.org/json-en.html) files +conforming to 'draft 04' of the [JSON Schema](https://json-schema.org/) +specification. -- Use the URL: `https://eddn.edcd.io:4430/upload/`. Note the use of - TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad - Request` response. -- Use the HTTP 1.1 protocol. HTTP/2 is not supported at this time. -- Use a **POST** request, with the body containing the EDDN message. No - query parameters in the URL are supported or necessary. +## Canonical location of Schema files -The body of an EDDN message is a JSON object in UTF-8 encoding. If you do not -compress this body then you MUST set a `Content-Type` header of -`applicaton/json`. +For the EDDN Live service you should always be checking +[the live version of the schemas, and their READMEs](https://github.com/EDCD/EDDN/tree/live/schemas). +Any other version of the Schemas is not guaranteed to be synchronized with +those actually running on the Live service. -For historical reasons URL form-encoded data *is* supported, **but this is -deprecated and no new software should attempt this method**. We -purposefully do not further document the exact format for this. +## Documentation of Schema files -You *MAY* use gzip compression on the body of the message, but it is not -required. If you do compress the body then you **MUST* send a `Content-Type` -header of `gzip` instead of `application/json`. +The Schema files themselves are considered to be the canonical definition of +the required, and allowed, contents of the relevant EDDN message. There +**SHOULD** be an accompanying README file, e.g. for `commodity-v3.0.json` there +is also a `commodity-README.md` file in the project root `schemas/` directory. -You should be prepared to handle all scenarios where sending of a message -fails: - -1. Connection refused. -2. Connection timed out. -3. Other possible responses as documented in - [Server responses](#server-responses). - -Carefully consider whether you should queue a 'failed' message for later -retry. In particular, you should ensure that one 'bad' message does not -block other messages from being successfully sent. - -You **MUST** wait some reasonable time (minimum 1 minute) before retrying -any failed message. - -You **MUST NOT** retry any message that received a HTTP `400` or `426` code. -An exception can be made if, **and only if**, *you have manually verified that -you have fixed the issues with it (i.e. updated the Schema/version to a -currently supported one and adjusted the data to fit that Schema/version).* - -You **MAY** retry a message that initially received a `413` response (in -the hopes that the EDDN service admins decided to increase the maximum -allowed request size), but should not do so too quickly or in perpetuity. - -In general: - -- No data is better than bad data. -- *Delayed* good data is better than degrading the EDDN service for others. - -### Format of uploaded messages -Each message is a JSON object in UTF-8 encoding containing the following -key+value pairs: - -1. `$schemaRef` - Which Schema (including version) this message is for. -2. `header` - Object containing mandatory information about the upload; - 1. `uploaderID` - a unique ID for the player uploading this data. - Don't worry about privacy, the EDDN service will hash this with a key - that is regularly changed so no-one knows who an uploader is in-game. - 2. `softwareName` - an identifier for the software performing the upload. - 3. `softwareVersion` - The version of that software being used. - - Listeners MAY make decisions about whether to utilise the data in any - message based on the combination of `softwareName` and `softwareVersion`. - - **DO not** add `gatewaytimestamp` yourself. The EDDN Gateway will add - this and will overwrite any that you provide, so don't bother. -4. `message` - Object containing the data for this message. Consult the - relevant README file within this documentation, e.g. - [codexentry-README.md](./codexentry-README.md). There are some general - guidelines [below](#contents-of-message). - -For example, a shipyard message, version 2, might look like: - -```JSON -{ - "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2", - "header": { - "uploaderID": "Bill", - "softwareName": "My excellent app", - "softwareVersion": "0.0.1" - }, - "message": { - "systemName": "Munfayl", - "stationName": "Samson", - "marketId": 128023552, - "horizons": true, - "timestamp": "2019-01-08T06:39:43Z", - "ships": [ - "anaconda", - "dolphin", - "eagle", - "ferdelance", - "hauler", - "krait_light", - "krait_mkii", - "mamba", - "python", - "sidewinder" - ] - } -} -``` - -### Contents of `message` -Every message MUST comply with the Schema its `$schemaRef` value cites. Each -Schema file should have a matching `-README.md` file in the project -root `schemas/` directory. Always consult this so that you're aware of any -Schema-specific requirements. - -The Schema file, `-v.json`, is considered the authority on -the format of messages for that Schema. If anything in the accompanying -documentation is in disagreement with this then please -[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) -so that we can investigate and correct, or clarify, as necessary. - -Apart from short time windows during deployment of a new version the live -EDDN service should always be using -[the Schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). -So, be sure you're checking the live versions and not, e.g. those in the -`master` or other branches. - -1. Each `message` object must have, at bare minimum: - - 1. `timestamp` - string date and time in ISO8601 format. Whilst this - technically allows for any timezone to be cited you SHOULD provide this in - UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are - doing this properly. Do not claim 'Z' whilst actually using a local time - that is offset from UTC. - - If you are only utilising Journal-sourced data then simply using the - value from there should be sufficient as the PC game client is meant to - always be correctly citing UTC for this. Indeed it has been observed, - in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC - by 21 seconds both the in-game UI clock *and* the Journal event - timestamps are still properly UTC to the nearest second. - - Listeners MAY make decisions on accepting data based on this time stamp, - i.e. "too old". - 2. Where the data is sourced from a Journal event please do preserve the - `event` key and value. Yes, where we use an event-specific Schema this - might seem redundant, but it might aid an EDDN listener in streamlining - their code, and it does no harm. - - Any new Schema based on Journal data **MUST** make `event` a required - property of the `message` dictionary. - 3. At least one other key/value pair representing the data. In general there - will be much more than this. Consult the - [Schemas and their documentation](./). - 4. Include - [horizons and odyssey flags](#horizons-and-odyssey-flags) where the - Schema requires them. - -2. Because the first versions of some Schemas were defined when only the CAPI - data was available, before Journal files existed, many of the key names chosen - in the Schemas are based on the equivalent in CAPI data, not Journal events. - This means you MUST rename many of the keys from Journal events to match the - Schemas. - -3. EDDN is intended to transport generic data not specific to any particular Cmdr - and to reflect only the data that every player would see in-game in station - services or the local map. To that end: - 1. Uploading applications MUST ensure that messages do not contain any - Cmdr-specific data (other than "uploaderID", the "horizons" flag, and - the "odyssey" flag). - 2. Uploading applications MUST remove any data where the name of the - relevant key has a `_Localised` suffix. - - The individual Schemas will instruct you on various elisions (removals) to - be made to comply with this. - -Some of these requirements are also enforced by the Schemas, and some things -the Schemas enforce might not be explicitly called out here. So, **do** -check what you're sending against the relevant Schema(s) when making any -changes to your code. - -It is also advisable to Watch this repository on GitHub so that you are aware -of any changes to Schemas. - -#### `horizons` and `odyssey` flags - -Where the Schema allows for them, add the `horizons` and `odyssey` -keys with boolean values. `null` is not allowed in the values, so **if -you cannot determine a value do not include that key at all**. - -The only source of these is the `LoadGame` event from journals. It's present -both in the PC local files and the CAPI journal data. If you're -composing a shipyard or outfitting message from CAPI data then it is -possible to synthesise the `horizons` flag. For now consult the function -`capi_is_horizons()` in -[EDMarketConnector:plugins/eddn.py](https://github.com/EDCD/EDMarketConnector/blob/stable/plugins/eddn.py) -for a method to achieve this. - -As of 2022-01-29 the following was observed for the `LoadGame` events as -present in CAPI-sourced Journal files (which were confirmed to match the -PC-local files for these events): - -- PC Odyssey Client, game version `4.0.0.1100`: - ```json - { "timestamp":"2022-01-29T16:17:02Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true, "Odyssey":true,... - ``` -- PC Horizons Client, game version `3.8.0.404`, no `Odyssey` key was - present: - ```json - { "timestamp":"2022-01-29T16:15:07Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true,... - ``` - -- PC 'base' Client, game version `3.8.0.404`, no `Odyssey` key was - present: - ```json - { "timestamp":"2022-01-29T16:11:54Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":false,... - ``` - -Do not attempt to use the value(s) from a `Fileheader` event as the semantics -are different. With clients 3.8.0.404 and 4.0.0.1100 the following was observed: - -| Game Client | Fileheader | LoadGame | -| ---------------: | ---------------- | ------------------------------: | -| Base | "Odyssey":false | "Horizons":false | -| Horizons | "Odyssey":false | "Horizons":true | -| Odyssey | "Odyssey":true | "Horizons":true, "Odyssey":true | - -NB: The 'Base' client appears to simply be the Horizons client with any -Horizons-only features disabled. - -- In the `Fileheader` event it's indicating whether it's an Odyssey game client. -- In the `LoadGame` it's indicating whether Horizons and/or Odyssey features are - active, but in the non-Odyssey game client case you only get the Horizons - boolean. - -### Server responses -There are three possible sources of HTTP responses when sending an upload -to EDDN. - -1. The reverse proxy that initially accepts the request. -2. The python `bottle` module that the Gateway uses to process the - forwarded requests. This might object to a message before the actual - EDDN code gets to process it at all. -3. The actual EDDN Gateway code. - -Once a message has cleared the EDDN Gateway then there is no mechanism for any -further issue (such as a message being detected as a duplicate in the -Monitor downstream of the Gateway) to be reported back to the sender. - -To state the obvious, if there are no issues with a request then an HTTP -200 response will be received by the sender. The body of the response -should be the string `OK`. - -#### Reverse Proxy responses -In addition to generic "you typoed the URL" and other such "you just didn't -make a valid request" responses you might experience the following: - -1. `408` - `Request Timed Out` - the sender took too long to make/complete - its request and the reverse proxy rejected it as a result. -2. `503` - `Service Unavailable` - the EDDN Gateway process is either not - running, or not responding. -3. `400` - `Bad Request` - if you attempt to use plain HTTP, rather than - HTTPS. - -#### bottle responses -1. `413` - `Payload Too Large` - `bottle` enforces a maximum request size - and the request exceeds that. As of 2022-01-07 the limit is 1MiB, - which is versus the compressed size of the body, if compression is - used. Thus compression *will* allow for sending approximately 10x - larger messages. - To verify the current limit check for the line that looks like: - - ``` - bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB - ``` - - in - [src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py), - as added in - [commit 0e80c76cb564771465f61825e694227dcc3be312](https://github.com/EDCD/EDDN/commit/0e80c76cb564771465f61825e694227dcc3be312). - -#### EDDN Gateway responses -For all failures the response body will contain text that begins `FAIL: `. Currently two different HTTP status codes are utilised: - -1. `400` - `Bad Request` - This indicates something wrong with the request - body. Possibly due to a format issue (compression, form encoding), or - the actual content of the EDDN message: - 1. `FAIL: zlib.error: ` - A failure to decompress a message that - claimed to be compressed. - - 2. `FAIL: Malformed Upload: ` - the message appeared to be - form-encoded, but either the format was bad or there was no `data` - key. - - 3. `FAIL: JSON parsing: ` - the - message couldn't be parsed as valid JSON. e.g. - - ``` - FAIL: JSON parsing: Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - ``` - - 4. `FAIL: Schema Validation: ` - the message failed to validate - against the cited Schema. e.g. - - ``` - FAIL: Schema Validation: [] - ``` - - The exact detail will be very much dependent on both the Schema the - message cited and the contents of the message that failed to pass the - validation. - - In particular, if the message contains a key that is tagged 'disallowed' in - the Schema, then the response will look like: - - ``` - FAIL: Schema Validation: "[]" - ``` - This is due to the use of a JSON schema stanza that says "don't allow - any valid type for the value of this key" to trigger the error for such - disallowed keys. - - Note how the **value** for the disallowed key is cited, not the key *name* - itself. This is a limitation of how the `jsonschema` python module - reports errors, and we are - [hoping to improve this](https://github.com/EDCD/EDDN/issues/163). - -2. `426` - `Upgrade Required` - This indicates that the cited Schema, or - version thereof, is outdated. The body of the response will be: - - ``` - FAIL: Oudated Schema: The schema you have used is no longer supported. Please check for an updated version of your application. - ``` - The wording here is aimed at users of applications that send messages - over EDDN. If you're the developer of such an application then - obviously you need to update your code to use a currently supported - Schema and version thereof. - - -There shouldn't be any other variants of a 'FAIL' message. If you find -any then please -[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) -with as much detail as possible so that we can update this documentation. - -## Receiving messages - -EDDN provides a continuous stream of information from uploaders. To use this -data you'll need to connect to the stream using ZeroMQ (a library is probably -available for your language of choice). - -The URL for the live Relay is: - - tcp://eddn.edcd.io:9500 - -Depending on the programming language and library used, you might need -to explicitly subscribe to an empty topic, `''`, in order to receive -anything. - -Unfortunately at this time we're using an old version of -ZeroMQ which does not support server-side subscription filtering. So by -all means specify a more specific topic filter if it suits your needs, -but realise that the dropping of not-matching messages will happen at -your end, not the server. - -Once you've connected to that you will receive messages. To access the -data you will first need to zlib-decompress each message. Then you will -have a textual JSON object as per the Schemas. - -In general, check the guidance for [Uploading messages](#uploading-messages) -for the expected format of the messages. - -Consumers can utilise the `$schemaRef` value to determine which Schema a -particular message is for. There is no need to validate the messages -against the Schemas yourself, as that is performed on the EDDN Gateway. -Messages that do not pass the schema validation there are not forwarded to -receivers. - -There is [example code](https://github.com/EDCD/EDDN/tree/master/examples) -available for a variety of programming languages to help you get started. +For more general documentation that all developers wanting to either Upload +messages or Listen to the stream of messages from the Relay, please consult +[the Developer documentation](../docs/Developers.md). From 245a4ba930fe0c385087462d2e9c9a18399c68e0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:57:28 +0000 Subject: [PATCH 51/89] README: Reference `docs/Developers.md` now, not schema/README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d47bb7..586e0d4 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ interact with EDDN - check the [EDDN tag](https://edcodex.info/?m=tools&cat=9). ### Developers If you are a developer of a third-party tool that could be enhanced by uploading data to EDDN then please consult -[the live branch documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) +[the live branch Developers' documentation](https://github.com/EDCD/EDDN/blob/live/docs/Developers.md) . **DO NOT** assume that any code or documentation in the `master` (or any other) branch on GitHub truly reflects the current live service! From de4900ac6778e65524c69c1d37e9b8b952fdea15 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:57:44 +0000 Subject: [PATCH 52/89] docs/Developers: Import remaining portion of old schemas/README --- docs/Developers.md | 375 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) diff --git a/docs/Developers.md b/docs/Developers.md index 88e8225..d9b2c66 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -109,3 +109,378 @@ link). Alternatively you could attempt [running your own test instance of EDDN](../docs/Running-this-software.md). +### Sending data +Messages sent to EDDN **MUST**: + +- Use the URL: `https://eddn.edcd.io:4430/upload/`. Note the use of + TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad + Request` response. +- Use the HTTP 1.1 protocol. HTTP/2 is not supported at this time. +- Use a **POST** request, with the body containing the EDDN message. No + query parameters in the URL are supported or necessary. + +The body of an EDDN message is a JSON object in UTF-8 encoding. If you do not +compress this body then you MUST set a `Content-Type` header of +`applicaton/json`. + +For historical reasons URL form-encoded data *is* supported, **but this is +deprecated and no new software should attempt this method**. We +purposefully do not further document the exact format for this. + +You *MAY* use gzip compression on the body of the message, but it is not +required. If you do compress the body then you **MUST* send a `Content-Type` +header of `gzip` instead of `application/json`. + +You should be prepared to handle all scenarios where sending of a message +fails: + +1. Connection refused. +2. Connection timed out. +3. Other possible responses as documented in + [Server responses](#server-responses). + +Carefully consider whether you should queue a 'failed' message for later +retry. In particular, you should ensure that one 'bad' message does not +block other messages from being successfully sent. + +You **MUST** wait some reasonable time (minimum 1 minute) before retrying +any failed message. + +You **MUST NOT** retry any message that received a HTTP `400` or `426` code. +An exception can be made if, **and only if**, *you have manually verified that +you have fixed the issues with it (i.e. updated the Schema/version to a +currently supported one and adjusted the data to fit that Schema/version).* + +You **MAY** retry a message that initially received a `413` response (in +the hopes that the EDDN service admins decided to increase the maximum +allowed request size), but should not do so too quickly or in perpetuity. + +In general: + +- No data is better than bad data. +- *Delayed* good data is better than degrading the EDDN service for others. + +### Format of uploaded messages +Each message is a JSON object in UTF-8 encoding containing the following +key+value pairs: + +1. `$schemaRef` - Which Schema (including version) this message is for. +2. `header` - Object containing mandatory information about the upload; + 1. `uploaderID` - a unique ID for the player uploading this data. + Don't worry about privacy, the EDDN service will hash this with a key + that is regularly changed so no-one knows who an uploader is in-game. + 2. `softwareName` - an identifier for the software performing the upload. + 3. `softwareVersion` - The version of that software being used. + + Listeners MAY make decisions about whether to utilise the data in any + message based on the combination of `softwareName` and `softwareVersion`. + + **DO not** add `gatewaytimestamp` yourself. The EDDN Gateway will add + this and will overwrite any that you provide, so don't bother. +4. `message` - Object containing the data for this message. Consult the + relevant README file within this documentation, e.g. + [codexentry-README.md](./codexentry-README.md). There are some general + guidelines [below](#contents-of-message). + +For example, a shipyard message, version 2, might look like: + +```JSON +{ + "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2", + "header": { + "uploaderID": "Bill", + "softwareName": "My excellent app", + "softwareVersion": "0.0.1" + }, + "message": { + "systemName": "Munfayl", + "stationName": "Samson", + "marketId": 128023552, + "horizons": true, + "timestamp": "2019-01-08T06:39:43Z", + "ships": [ + "anaconda", + "dolphin", + "eagle", + "ferdelance", + "hauler", + "krait_light", + "krait_mkii", + "mamba", + "python", + "sidewinder" + ] + } +} +``` + +### Contents of `message` +Every message MUST comply with the Schema its `$schemaRef` value cites. Each +Schema file should have a matching `-README.md` file in the project +root `schemas/` directory. Always consult this so that you're aware of any +Schema-specific requirements. + +The Schema file, `-v.json`, is considered the authority on +the format of messages for that Schema. If anything in the accompanying +documentation is in disagreement with this then please +[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +so that we can investigate and correct, or clarify, as necessary. + +Apart from short time windows during deployment of a new version the live +EDDN service should always be using +[the Schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). +So, be sure you're checking the live versions and not, e.g. those in the +`master` or other branches. + +1. Each `message` object must have, at bare minimum: + + 1. `timestamp` - string date and time in ISO8601 format. Whilst this + technically allows for any timezone to be cited you SHOULD provide this in + UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are + doing this properly. Do not claim 'Z' whilst actually using a local time + that is offset from UTC. + + If you are only utilising Journal-sourced data then simply using the + value from there should be sufficient as the PC game client is meant to + always be correctly citing UTC for this. Indeed it has been observed, + in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC + by 21 seconds both the in-game UI clock *and* the Journal event + timestamps are still properly UTC to the nearest second. + + Listeners MAY make decisions on accepting data based on this time stamp, + i.e. "too old". + 2. Where the data is sourced from a Journal event please do preserve the + `event` key and value. Yes, where we use an event-specific Schema this + might seem redundant, but it might aid an EDDN listener in streamlining + their code, and it does no harm. + + Any new Schema based on Journal data **MUST** make `event` a required + property of the `message` dictionary. + 3. At least one other key/value pair representing the data. In general there + will be much more than this. Consult the + [Schemas and their documentation](./). + 4. Include + [horizons and odyssey flags](#horizons-and-odyssey-flags) where the + Schema requires them. + +2. Because the first versions of some Schemas were defined when only the CAPI + data was available, before Journal files existed, many of the key names chosen + in the Schemas are based on the equivalent in CAPI data, not Journal events. + This means you MUST rename many of the keys from Journal events to match the + Schemas. + +3. EDDN is intended to transport generic data not specific to any particular Cmdr + and to reflect only the data that every player would see in-game in station + services or the local map. To that end: + 1. Uploading applications MUST ensure that messages do not contain any + Cmdr-specific data (other than "uploaderID", the "horizons" flag, and + the "odyssey" flag). + 2. Uploading applications MUST remove any data where the name of the + relevant key has a `_Localised` suffix. + + The individual Schemas will instruct you on various elisions (removals) to + be made to comply with this. + +Some of these requirements are also enforced by the Schemas, and some things +the Schemas enforce might not be explicitly called out here. So, **do** +check what you're sending against the relevant Schema(s) when making any +changes to your code. + +It is also advisable to Watch this repository on GitHub so that you are aware +of any changes to Schemas. + +#### `horizons` and `odyssey` flags + +Where the Schema allows for them, add the `horizons` and `odyssey` +keys with boolean values. `null` is not allowed in the values, so **if +you cannot determine a value do not include that key at all**. + +The only source of these is the `LoadGame` event from journals. It's present +both in the PC local files and the CAPI journal data. If you're +composing a shipyard or outfitting message from CAPI data then it is +possible to synthesise the `horizons` flag. For now consult the function +`capi_is_horizons()` in +[EDMarketConnector:plugins/eddn.py](https://github.com/EDCD/EDMarketConnector/blob/stable/plugins/eddn.py) +for a method to achieve this. + +As of 2022-01-29 the following was observed for the `LoadGame` events as +present in CAPI-sourced Journal files (which were confirmed to match the +PC-local files for these events): + +- PC Odyssey Client, game version `4.0.0.1100`: + ```json + { "timestamp":"2022-01-29T16:17:02Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true, "Odyssey":true,... + ``` +- PC Horizons Client, game version `3.8.0.404`, no `Odyssey` key was + present: + ```json + { "timestamp":"2022-01-29T16:15:07Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true,... + ``` + +- PC 'base' Client, game version `3.8.0.404`, no `Odyssey` key was + present: + ```json + { "timestamp":"2022-01-29T16:11:54Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":false,... + ``` + +Do not attempt to use the value(s) from a `Fileheader` event as the semantics +are different. With clients 3.8.0.404 and 4.0.0.1100 the following was observed: + +| Game Client | Fileheader | LoadGame | +| ---------------: | ---------------- | ------------------------------: | +| Base | "Odyssey":false | "Horizons":false | +| Horizons | "Odyssey":false | "Horizons":true | +| Odyssey | "Odyssey":true | "Horizons":true, "Odyssey":true | + +NB: The 'Base' client appears to simply be the Horizons client with any +Horizons-only features disabled. + +- In the `Fileheader` event it's indicating whether it's an Odyssey game client. +- In the `LoadGame` it's indicating whether Horizons and/or Odyssey features are + active, but in the non-Odyssey game client case you only get the Horizons + boolean. + +### Server responses +There are three possible sources of HTTP responses when sending an upload +to EDDN. + +1. The reverse proxy that initially accepts the request. +2. The python `bottle` module that the Gateway uses to process the + forwarded requests. This might object to a message before the actual + EDDN code gets to process it at all. +3. The actual EDDN Gateway code. + +Once a message has cleared the EDDN Gateway then there is no mechanism for any +further issue (such as a message being detected as a duplicate in the +Monitor downstream of the Gateway) to be reported back to the sender. + +To state the obvious, if there are no issues with a request then an HTTP +200 response will be received by the sender. The body of the response +should be the string `OK`. + +#### Reverse Proxy responses +In addition to generic "you typoed the URL" and other such "you just didn't +make a valid request" responses you might experience the following: + +1. `408` - `Request Timed Out` - the sender took too long to make/complete + its request and the reverse proxy rejected it as a result. +2. `503` - `Service Unavailable` - the EDDN Gateway process is either not + running, or not responding. +3. `400` - `Bad Request` - if you attempt to use plain HTTP, rather than + HTTPS. + +#### bottle responses +1. `413` - `Payload Too Large` - `bottle` enforces a maximum request size + and the request exceeds that. As of 2022-01-07 the limit is 1MiB, + which is versus the compressed size of the body, if compression is + used. Thus compression *will* allow for sending approximately 10x + larger messages. + To verify the current limit check for the line that looks like: + + ``` + bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB + ``` + + in + [src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py), + as added in + [commit 0e80c76cb564771465f61825e694227dcc3be312](https://github.com/EDCD/EDDN/commit/0e80c76cb564771465f61825e694227dcc3be312). + +#### EDDN Gateway responses +For all failures the response body will contain text that begins `FAIL: `. Currently two different HTTP status codes are utilised: + +1. `400` - `Bad Request` - This indicates something wrong with the request + body. Possibly due to a format issue (compression, form encoding), or + the actual content of the EDDN message: + 1. `FAIL: zlib.error: ` - A failure to decompress a message that + claimed to be compressed. + + 2. `FAIL: Malformed Upload: ` - the message appeared to be + form-encoded, but either the format was bad or there was no `data` + key. + + 3. `FAIL: JSON parsing: ` - the + message couldn't be parsed as valid JSON. e.g. + + ``` + FAIL: JSON parsing: Expecting property name enclosed in double quotes: line 1 column 2 (char 1) + ``` + + 4. `FAIL: Schema Validation: ` - the message failed to validate + against the cited Schema. e.g. + + ``` + FAIL: Schema Validation: [] + ``` + + The exact detail will be very much dependent on both the Schema the + message cited and the contents of the message that failed to pass the + validation. + + In particular, if the message contains a key that is tagged 'disallowed' in + the Schema, then the response will look like: + + ``` + FAIL: Schema Validation: "[]" + ``` + This is due to the use of a JSON schema stanza that says "don't allow + any valid type for the value of this key" to trigger the error for such + disallowed keys. + + Note how the **value** for the disallowed key is cited, not the key *name* + itself. This is a limitation of how the `jsonschema` python module + reports errors, and we are + [hoping to improve this](https://github.com/EDCD/EDDN/issues/163). + +2. `426` - `Upgrade Required` - This indicates that the cited Schema, or + version thereof, is outdated. The body of the response will be: + + ``` + FAIL: Oudated Schema: The schema you have used is no longer supported. Please check for an updated version of your application. + ``` + The wording here is aimed at users of applications that send messages + over EDDN. If you're the developer of such an application then + obviously you need to update your code to use a currently supported + Schema and version thereof. + + +There shouldn't be any other variants of a 'FAIL' message. If you find +any then please +[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) +with as much detail as possible so that we can update this documentation. + +## Receiving messages + +EDDN provides a continuous stream of information from uploaders. To use this +data you'll need to connect to the stream using ZeroMQ (a library is probably +available for your language of choice). + +The URL for the live Relay is: + + tcp://eddn.edcd.io:9500 + +Depending on the programming language and library used, you might need +to explicitly subscribe to an empty topic, `''`, in order to receive +anything. + +Unfortunately at this time we're using an old version of +ZeroMQ which does not support server-side subscription filtering. So by +all means specify a more specific topic filter if it suits your needs, +but realise that the dropping of not-matching messages will happen at +your end, not the server. + +Once you've connected to that you will receive messages. To access the +data you will first need to zlib-decompress each message. Then you will +have a textual JSON object as per the Schemas. + +In general, check the guidance for [Uploading messages](#uploading-messages) +for the expected format of the messages. + +Consumers can utilise the `$schemaRef` value to determine which Schema a +particular message is for. There is no need to validate the messages +against the Schemas yourself, as that is performed on the EDDN Gateway. +Messages that do not pass the schema validation there are not forwarded to +receivers. + +There is [example code](https://github.com/EDCD/EDDN/tree/master/examples) +available for a variety of programming languages to help you get started. From ead10a2c1e6cd42962f98fd9f17d51356c8e34ac Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 12:59:35 +0000 Subject: [PATCH 53/89] docs/Developers: Tweak to why EDMC uses CAPI --- docs/Developers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Developers.md b/docs/Developers.md index d9b2c66..399150a 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -11,6 +11,7 @@ EDDN accepts HTTP POST uploads in representing this game data and then passes it on to any interested listeners. --- + ## Sources There are two sources of game data, both provided by the publisher of the game, @@ -63,7 +64,8 @@ by Athanasius. It is *not* recommended to use CAPI data as the source as it's fraught with additional issues. EDMarketConnector does so in order to facilitate -obtaining data without the player needing to open the commodities screen. +obtaining station data without the player needing to, e.g. open the commodities +screen. #### Detecting CAPI data lag From 10c43e5304f477cd803e4217fd0a8c7e2a8c1a38 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 13:07:38 +0000 Subject: [PATCH 54/89] docs/Developers: Use relative schemas/ link & tweak horizons/odyssey flags reference --- docs/Developers.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index 399150a..05ea54d 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -218,9 +218,9 @@ For example, a shipyard message, version 2, might look like: ### Contents of `message` Every message MUST comply with the Schema its `$schemaRef` value cites. Each -Schema file should have a matching `-README.md` file in the project -root `schemas/` directory. Always consult this so that you're aware of any -Schema-specific requirements. +Schema file should have a matching `-README.md` file in the +[project root schemas/ directory](../schemas/) +. Always consult this so that you're aware of any Schema-specific requirements. The Schema file, `-v.json`, is considered the authority on the format of messages for that Schema. If anything in the accompanying @@ -261,9 +261,10 @@ So, be sure you're checking the live versions and not, e.g. those in the 3. At least one other key/value pair representing the data. In general there will be much more than this. Consult the [Schemas and their documentation](./). - 4. Include - [horizons and odyssey flags](#horizons-and-odyssey-flags) where the - Schema requires them. + + Please consult the advice pertaining to + [horizons and odyssey flags](#horizons-and-odyssey-flags) and include them + whenever possible. 2. Because the first versions of some Schemas were defined when only the CAPI data was available, before Journal files existed, many of the key names chosen From 692c28bd7512bf93348caffcbe68c35139276b86 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 14:03:16 +0000 Subject: [PATCH 55/89] docs/Developers: Be *very* explicit about the live upload URL --- docs/Developers.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index 05ea54d..c769eb8 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -114,8 +114,10 @@ Alternatively you could attempt ### Sending data Messages sent to EDDN **MUST**: -- Use the URL: `https://eddn.edcd.io:4430/upload/`. Note the use of - TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad +- Use the URL: `https://eddn.edcd.io:4430/upload/` + - Note the non-standard port `4430`. + - Yes, the trailing `/` is required. + - Note the use of TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad Request` response. - Use the HTTP 1.1 protocol. HTTP/2 is not supported at this time. - Use a **POST** request, with the body containing the EDDN message. No From 3b8845d31ac6b759705df42199f2e3426a4fe0d7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 14:14:59 +0000 Subject: [PATCH 56/89] docs: Resolving where "general message contents" lives and its format * Most of it is once more in schemas/README. * But keeping the horizons/odyssey flags in docs/Developers.md, as that's more about how you handle them in code. --- docs/Developers.md | 75 ++++++++++------------------------ schemas/README-EDDN-schemas.md | 41 +++++++++++++++++++ 2 files changed, 62 insertions(+), 54 deletions(-) diff --git a/docs/Developers.md b/docs/Developers.md index c769eb8..772ca88 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -236,69 +236,36 @@ EDDN service should always be using So, be sure you're checking the live versions and not, e.g. those in the `master` or other branches. -1. Each `message` object must have, at bare minimum: +Please consult the +[general README for Schemas](../schemas/README-EDDN-schemas.md#general-eddn-message-outline) +for more detailed information about a message's content. - 1. `timestamp` - string date and time in ISO8601 format. Whilst this - technically allows for any timezone to be cited you SHOULD provide this in - UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are - doing this properly. Do not claim 'Z' whilst actually using a local time - that is offset from UTC. +EDDN is intended to transport generic data not specific to any particular Cmdr +and to reflect only the data that every player would see in-game in station +services or the local map. To that end: +1. Uploading applications MUST ensure that messages do not contain any + Cmdr-specific data (other than "uploaderID", the "horizons" flag, and + the "odyssey" flag). +2. Uploading applications MUST remove any data where the name of the + relevant key has a `_Localised` suffix. - If you are only utilising Journal-sourced data then simply using the - value from there should be sufficient as the PC game client is meant to - always be correctly citing UTC for this. Indeed it has been observed, - in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC - by 21 seconds both the in-game UI clock *and* the Journal event - timestamps are still properly UTC to the nearest second. - - Listeners MAY make decisions on accepting data based on this time stamp, - i.e. "too old". - 2. Where the data is sourced from a Journal event please do preserve the - `event` key and value. Yes, where we use an event-specific Schema this - might seem redundant, but it might aid an EDDN listener in streamlining - their code, and it does no harm. - - Any new Schema based on Journal data **MUST** make `event` a required - property of the `message` dictionary. - 3. At least one other key/value pair representing the data. In general there - will be much more than this. Consult the - [Schemas and their documentation](./). - - Please consult the advice pertaining to - [horizons and odyssey flags](#horizons-and-odyssey-flags) and include them - whenever possible. - -2. Because the first versions of some Schemas were defined when only the CAPI - data was available, before Journal files existed, many of the key names chosen - in the Schemas are based on the equivalent in CAPI data, not Journal events. - This means you MUST rename many of the keys from Journal events to match the - Schemas. - -3. EDDN is intended to transport generic data not specific to any particular Cmdr - and to reflect only the data that every player would see in-game in station - services or the local map. To that end: - 1. Uploading applications MUST ensure that messages do not contain any - Cmdr-specific data (other than "uploaderID", the "horizons" flag, and - the "odyssey" flag). - 2. Uploading applications MUST remove any data where the name of the - relevant key has a `_Localised` suffix. - - The individual Schemas will instruct you on various elisions (removals) to - be made to comply with this. +The individual Schemas will instruct you on various elisions (removals) to +be made to comply with this. Some of these requirements are also enforced by the Schemas, and some things -the Schemas enforce might not be explicitly called out here. So, **do** -check what you're sending against the relevant Schema(s) when making any +the Schemas enforce might not be explicitly called out in documentation. So, +**do** check what you're sending against the relevant Schema(s) when making any changes to your code. -It is also advisable to Watch this repository on GitHub so that you are aware -of any changes to Schemas. +It is also advisable to Watch this repository +[on GitHub](https://github.io/EDCD/EDDN/) +so that you are aware of any changes to Schemas. #### `horizons` and `odyssey` flags -Where the Schema allows for them, add the `horizons` and `odyssey` -keys with boolean values. `null` is not allowed in the values, so **if -you cannot determine a value do not include that key at all**. +Where the Schema allows for them, `horizons` and `odyssey` keys SHOULD be +added with appropriate boolean values. `null` is not allowed in the values, +so **if you cannot determine a value do not include that key at all**. The only source of these is the `LoadGame` event from journals. It's present both in the PC local files and the CAPI journal data. If you're diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index d0d4022..ae49751 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -21,3 +21,44 @@ is also a `commodity-README.md` file in the project root `schemas/` directory. For more general documentation that all developers wanting to either Upload messages or Listen to the stream of messages from the Relay, please consult [the Developer documentation](../docs/Developers.md). + +## General EDDN message outline + +Each `message` object must have, at bare minimum: + +1. `timestamp` - string date and time in ISO8601 format. Whilst this + technically allows for any timezone to be cited you SHOULD provide this in + UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are + doing this properly. Do not claim 'Z' whilst actually using a local time + that is offset from UTC. + + If you are only utilising Journal-sourced data then simply using the + value from there should be sufficient as the PC game client is meant to + always be correctly citing UTC for this. Indeed it has been observed, + in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC + by 21 seconds both the in-game UI clock *and* the Journal event + timestamps are still properly UTC to the nearest second. + + Listeners MAY make decisions on accepting data based on this time stamp, + i.e. "too old". +2. Where the data is sourced from a Journal event please do preserve the + `event` key and value. Yes, where we use an event-specific Schema this + might seem redundant, but it might aid an EDDN listener in streamlining + their code, and it does no harm. + + Any new Schema based on Journal data **MUST** make `event` a required + property of the `message` dictionary. +3. At least one other key/value pair representing the data. In general there + will be much more than this. Consult the + [Schemas and their documentation](./). + +4. Please consult the advice pertaining to + [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) and include them + whenever possible. + +Because the first versions of some Schemas were defined when only the CAPI +data was available, before Journal files existed, many of the key names chosen +in the Schemas are based on the equivalent in CAPI data, not Journal events. +This means you MUST rename many of the keys from Journal events to match the +Schemas. Consult the relevant Schema, and its README, for details. + From e6d0240c367d31d7cb40646d2cb7afa22af04356 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 14:36:39 +0000 Subject: [PATCH 57/89] schemas/template: Define how to tag a key as having been renamed --- schemas/TEMPLATES/journalevent-README.md | 26 ++++++++---------------- schemas/TEMPLATES/journalevent-v1.0.json | 11 ++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index e56aa9d..cfb6e30 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -23,8 +23,8 @@ version of this Schema's Journal event name. 4. For new Journal-based schemas no key renames should be necessary. - If there are no renames of key names for this Schema, then edit the - `Key Renames` section to contain only the text `None.`. + If there are no renames of key names for this Schema, then remove the + `Key Renames` section. Where such renames *are* required do **NOT** attempt to list them all here. That would just require updating them both here and in the actual Schema. @@ -32,17 +32,11 @@ If there are any, call them out in the `description` of the affected property in the Schema. -5. In the `Elisions` section clearly document any keys (and thus their values) - that are in the source data, but that should not be in the resulting EDDN - message. - - **You do not need to list keys with a `_Localised` suffix.** - -6. Do **NOT** remove the `horizons and odyssey flags` section. It is +7. Do **NOT** remove the `horizons and odyssey flags` section. It is mandatory that they are allowed (but are optional) in any Journal-based EDDN Schema. -7. If: +8. If both: 1. either the source Journal event contains information that includes the System name (possibly as `StarSystem` or `SystemName`), **OR** the source data contains a `SystemAddress` value, @@ -83,11 +77,9 @@ discrepancy.** The data source for this schema is the ED Journal event `NewJournalEvent`. ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -None +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. ### Augmentations #### horizons and odyssey flags @@ -99,5 +91,5 @@ You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. ## Listeners -The advice above for [Senders](#senders), combined with the actual Schema file *should* -provide all the information you need to process these events. +The advice above for [Senders](#senders), combined with the actual Schema file +*should* provide all the information you need to process these events. diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json index 143f47b..a87c8fe 100644 --- a/schemas/TEMPLATES/journalevent-v1.0.json +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -25,7 +25,7 @@ # event is not guaranteed to contain either, depending on which client # (base, horizons, odyssey) the player is running. # 7. 'some_disallowed_key' demonstrates how to specify that a key (and thus its -# value) is not allowed in this schema. +# value) is not allowed in this schema. Note the **MANDATORY** description. # 8. Note the trailing comma after the 'some_disallowed_key' entry in # 'messages'. If all you did was remove these comments you would NOT have # a valid JSON file. You should be adding **at least** one additional @@ -96,7 +96,14 @@ "maxItems" : 3, "description" : "Must be added by the sender" }, - "some_disallowed_key" { "$ref" : "#/definitions/disallowed" }, + "ExampleRenamedKey" : { + "type" : "string", + "renamed" : "SomeOtherKey" + } + "some_disallowed_key" { + "$ref" : "#/definitions/disallowed", + "description" : "MANDATORY brief description of why this key must be removed from source data" + }, } } }, From 500ce86d51bfcc11e9aa3a75b77ec46c3ff5f354 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 14:40:46 +0000 Subject: [PATCH 58/89] schemas/template: Define `renamed` usage --- schemas/TEMPLATES/journalevent-README.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index cfb6e30..ff007ba 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -29,14 +29,30 @@ Where such renames *are* required do **NOT** attempt to list them all here. That would just require updating them both here and in the actual Schema. - If there are any, call them out in the `description` of the affected - property in the Schema. + If there are any, then the affected property object should contain a key + named `renamed` with its value being the original key name in the source + data, e.g. in the commodity/3 schema a Journal `StarSystem` is renamed + to `systemName` so we have: -7. Do **NOT** remove the `horizons and odyssey flags` section. It is + ```json + "message": { + "type" : "object", + "additionalProperties" : false, + "required" : [ "systemName", "stationName", "marketId", "timestamp", "commodities" ], + "properties" : { + "systemName": { + "type" : "string", + "renamed" : "StarSystem", + "minLength" : 1 + }, + + ``` + +6. Do **NOT** remove the `horizons and odyssey flags` section. It is mandatory that they are allowed (but are optional) in any Journal-based EDDN Schema. -8. If both: +7. If both: 1. either the source Journal event contains information that includes the System name (possibly as `StarSystem` or `SystemName`), **OR** the source data contains a `SystemAddress` value, From 43152bbce8ca3925fd5b3d44daeddec460414267 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 14:48:07 +0000 Subject: [PATCH 59/89] schemas/blackmarket: Bring in line with newer documentation --- schemas/blackmarket-README.md | 24 +++++++----------------- schemas/blackmarket-v1.0.json | 5 ++++- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/schemas/blackmarket-README.md b/schemas/blackmarket-README.md index 0054bd1..499e650 100644 --- a/schemas/blackmarket-README.md +++ b/schemas/blackmarket-README.md @@ -20,17 +20,9 @@ documentation for a schema such as this. The primary data source for this schema is the ED Journal event `MarketSell`. ### Key Renames -#### name -Due to how the EDDN schema is defined the `Type` key/value should -have the key renamed to `name`. - -#### prohibited -Due to how the EDDN schema is defined the `IllegalGoods` key/value should -have the key renamed to `prohibited`. - -#### marketID -The Journal documentation says this is `MarketID`, but in the schema the -`m` is lower case. +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. ### Elisions You MUST remove the following key/value pairs from the data: @@ -41,15 +33,13 @@ You MUST remove the following key/value pairs from the data: - `BlackMarket` - Because we're using this schema, so this is un-necessary. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) +over in the main Schema documentation. #### systemName The star system name for where this market is. Use the `StarSystem` value from the prior `Docked` or `Location` event. #### stationName -From the `StationName` value on the prior `Docked` or `Location` event. \ No newline at end of file +From the `StationName` value on the prior `Docked` or `Location` event. diff --git a/schemas/blackmarket-v1.0.json b/schemas/blackmarket-v1.0.json index 362916a..15cf3ed 100644 --- a/schemas/blackmarket-v1.0.json +++ b/schemas/blackmarket-v1.0.json @@ -44,7 +44,8 @@ "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "timestamp": { "type" : "string", @@ -52,6 +53,7 @@ }, "name": { "type" : "string", + "renamed" : "Type", "minLength" : 1, "description" : "Commodity name as returned by the MarketSell entry in the Journal" }, @@ -61,6 +63,7 @@ }, "prohibited": { "type" : "boolean", + "renamed" : "IllegalGoods", "description" : "Whether the commodity is prohibited at this station" } } From 93713ffa9b4c202923271722f65c1cbd808720de Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 15:16:46 +0000 Subject: [PATCH 60/89] schemas/codexentry: Re-align README with current standards --- schemas/codexentry-README.md | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/schemas/codexentry-README.md b/schemas/codexentry-README.md index 58da99e..b3c5fc6 100644 --- a/schemas/codexentry-README.md +++ b/schemas/codexentry-README.md @@ -7,31 +7,25 @@ properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `CodexEntry`. -### Elisions -You MUST remove any key where the key name ends in -`_Localised`. - -You MUST remove the two keys `IsNewEntry` and `NewTraitsDiscovered`. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) +over in the main Schema documentation. #### StarPos -You MUST **add** a `StarPos` key with value of type `array` containing the -galaxy co-ordinates of the system. You will need to have obtained these -from prior event(s) upon the player arriving, or logging into, the system. - -e.g. if the system is `Alpha Centauri`: -```json - "StarPos": [3.03125, -0.09375, 3.15625] -``` +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. #### BodyID and BodyName You SHOULD attempt to track the BodyName and BodyID where the player is @@ -103,7 +97,7 @@ e.g. for `Bestia A 2 a` If you cannot properly obtain the values for `BodyName` or `BodyID` then you MUST NOT include them. -## Receivers +## Listeners As per ['BodyID and BodyName'](#bodyid-and-bodyname) above be aware that you are not guaranteed to receive these values for any given event. Some From 8de80a868af498eda0a8eff694ac5a46fab3fbc5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 15:29:41 +0000 Subject: [PATCH 61/89] docs: More DRY and getting things in the correct places --- docs/Contributing.md | 2 +- docs/Developers.md | 29 +++++++-------------- schemas/README-EDDN-schemas.md | 33 +++++++++++++++++++----- schemas/TEMPLATES/journalevent-README.md | 12 ++++++--- 4 files changed, 44 insertions(+), 32 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 986c0b6..9a5cb2b 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -282,7 +282,7 @@ include: is what the Gateway code uses. The script `contrib/test-schema.py` will check both this and that the validation code doesn't choke on it. 4. All new Schemas **MUST** comply with all requirements outlined in the - [general Schemas documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md). + [general Schemas documentation](../schemas/README-EDDN-schemas.md). If you have a good reason why your new Schema can't and shouldn't comply with these requirements, then consensus will need to be achieved on changing those requirements and/or allowing the exception. diff --git a/docs/Developers.md b/docs/Developers.md index 772ca88..4dddb87 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -165,26 +165,9 @@ In general: - *Delayed* good data is better than degrading the EDDN service for others. ### Format of uploaded messages -Each message is a JSON object in UTF-8 encoding containing the following -key+value pairs: - -1. `$schemaRef` - Which Schema (including version) this message is for. -2. `header` - Object containing mandatory information about the upload; - 1. `uploaderID` - a unique ID for the player uploading this data. - Don't worry about privacy, the EDDN service will hash this with a key - that is regularly changed so no-one knows who an uploader is in-game. - 2. `softwareName` - an identifier for the software performing the upload. - 3. `softwareVersion` - The version of that software being used. - - Listeners MAY make decisions about whether to utilise the data in any - message based on the combination of `softwareName` and `softwareVersion`. - - **DO not** add `gatewaytimestamp` yourself. The EDDN Gateway will add - this and will overwrite any that you provide, so don't bother. -4. `message` - Object containing the data for this message. Consult the - relevant README file within this documentation, e.g. - [codexentry-README.md](./codexentry-README.md). There are some general - guidelines [below](#contents-of-message). +Each message is a JSON object in UTF-8 encoding with contents that comply with +the relevant Schema. There is an outline in the general +[EDDN Message README](../schemas/README-EDDN-schemas.md#mandatory-schema-file-contents). For example, a shipyard message, version 2, might look like: @@ -252,6 +235,12 @@ services or the local map. To that end: The individual Schemas will instruct you on various elisions (removals) to be made to comply with this. +Because the first versions of some Schemas were defined when only the CAPI +data was available, before Journal files existed, many of the key names chosen +in the Schemas are based on the equivalent in CAPI data, not Journal events. +This means you MUST rename many of the keys from Journal events to match the +Schemas. Consult the relevant Schema for details. + Some of these requirements are also enforced by the Schemas, and some things the Schemas enforce might not be explicitly called out in documentation. So, **do** check what you're sending against the relevant Schema(s) when making any diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index ae49751..4620dd1 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -22,7 +22,30 @@ For more general documentation that all developers wanting to either Upload messages or Listen to the stream of messages from the Relay, please consult [the Developer documentation](../docs/Developers.md). -## General EDDN message outline +### Mandatory Schema file contents + +It is best to base any new Schema file on +[the provided template](./TEMPLATES/journalevent-v1.0.json). As per its +contents all Schemas specify a top-level JSON Object with the data: + +1. `$schemaRef` - Which Schema (including version) this message is for. +2. `header` - Object containing mandatory information about the upload; + 1. `uploaderID` - a unique ID for the player uploading this data. + Don't worry about privacy, the EDDN service will hash this with a key + that is regularly changed so no-one knows who an uploader is in-game. + 2. `softwareName` - an identifier for the software performing the upload. + 3. `softwareVersion` - The version of that software being used. + + Listeners MAY make decisions about whether to utilise the data in any + message based on the combination of `softwareName` and `softwareVersion`. + + **DO not** add `gatewaytimestamp` yourself. The EDDN Gateway will add + this and will overwrite any that you provide, so don't bother. +4. `message` - Object containing the data for this message. Consult the + relevant README file within this documentation, e.g. + [codexentry-README.md](./codexentry-README.md). + +### General EDDN message outline Each `message` object must have, at bare minimum: @@ -56,9 +79,5 @@ Each `message` object must have, at bare minimum: [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) and include them whenever possible. -Because the first versions of some Schemas were defined when only the CAPI -data was available, before Journal files existed, many of the key names chosen -in the Schemas are based on the equivalent in CAPI data, not Journal events. -This means you MUST rename many of the keys from Journal events to match the -Schemas. Consult the relevant Schema, and its README, for details. - +Where a key has to be renamed this will be specified in the Schema through a +`renamed` property on the object in question. diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index ff007ba..2ba8d3e 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -48,11 +48,11 @@ ``` -6. Do **NOT** remove the `horizons and odyssey flags` section. It is +5. Do **NOT** remove the `horizons and odyssey flags` section. It is mandatory that they are allowed (but are optional) in any Journal-based EDDN Schema. -7. If both: +6. If both: 1. either the source Journal event contains information that includes the System name (possibly as `StarSystem` or `SystemName`), **OR** the source data contains a `SystemAddress` value, @@ -64,6 +64,10 @@ If neither key is in the source data then remove the `StarPos` section from this document and the Schema file. +7. Do **NOT** add an 'Elisions'/'Removals' section. Leave the Schema as the + sole reference for any data that is in the source but should not be in the + final EDDN message. + The line: # EDDN NewJournalEvent Schema @@ -99,8 +103,8 @@ value is what the name would have been in the source Journal data. ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) -over in the main Schema documentation. +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +over in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the From df09dd064e86185349949f4bbc0e78e7fc833c3c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 15:45:19 +0000 Subject: [PATCH 62/89] contrib/test-schema: Make this a little more useful * Can now supply only the Schema file name to test its loading, without needing to supply a test data file as well. * Added some status output, so it's more obvious when things worked. --- contrib/test-schema.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/contrib/test-schema.py b/contrib/test-schema.py index ebf0e51..da56adf 100644 --- a/contrib/test-schema.py +++ b/contrib/test-schema.py @@ -7,7 +7,7 @@ from jsonschema import validate as json_validate if len(sys.argv) < 2: print( f""" -Usage: {sys.argv[0]} +Usage: {sys.argv[0]} [] Note that the entire file will be loaded by simpljson.load() and should only contain one JSON object. @@ -16,12 +16,18 @@ only contain one JSON object. sys.exit(-1) schema_file_name = sys.argv[1] -test_file_name = sys.argv[2] +test_file_name = None +if len(sys.argv) == 3: + test_file_name = sys.argv[2] -with open(test_file_name, 'r') as test_file: - test_event = simplejson.load(test_file) +with open(schema_file_name, 'r') as schema_file: + schema = simplejson.load(schema_file) + print('Schema file loaded OK...') - with open(schema_file_name, 'r') as schema_file: - schema = simplejson.load(schema_file) + if test_file_name is not None: + with open(test_file_name, 'r') as test_file: + test_event = simplejson.load(test_file) - json_validate(test_event, schema, format_checker=FormatChecker()) + json_validate(test_event, schema, format_checker=FormatChecker()) + + print('Input file validated against schema OK.') From 0ee415deb1c8069b5fd3b456d5626f6daf8d0022 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 15:57:26 +0000 Subject: [PATCH 63/89] schemas/commodity: Bring in line with new standards * There's a lot that's still only in the README, which I would have added when creating the README based on EDMC code. * Specific ellisions now marked in the schema. * All key renames now marked in the schema. --- schemas/commodity-README.md | 32 ++++++++++++------------------ schemas/commodity-v3.0.json | 39 +++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index c2a4516..18c9cc5 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -24,18 +24,12 @@ See [Using CAPI data](#using-capi-data) below. So, as per the schema, do include it if available. ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -#### StarSystem to systemName -Rename the `StarSystem` key name to `systemName`. +Some key names in this Schema are different from how they appear in source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. The names +used are as found in the CAPI source data. ### Elisions -#### Remove _Localised key/values -All keys whose name ends with `_Localised`, i.e. the `Name_Localised` -key/values in Items. - -#### Other Elisions You MUST remove the following key/value pairs from the data: - `StationType` key/value. @@ -50,15 +44,13 @@ In the list of commodites: string (not normally traded at this station market). #### Item Category -Remove not only the `Category_Localised` key/value, as above, but also the -`Category` key/value pair from each Item. +Remove not only the `Category_Localised` key:values, but also the +`Category` key:value pair from each Item. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +over in the Developers' documentation. ### Using CAPI data It is *not* recommended to use CAPI data as the source as it's fraught with @@ -66,10 +58,10 @@ additional issues. EDMarketConnector does so in order to facilitate obtaining data without the player needing to open the commodities screen. Please read -[the guidance on checking for CAPI lag](README-EDDN-schemas.md#detecting-capi-data-lag) +[the guidance on checking for CAPI lag](../docs/Developers.md#detecting-capi-data-lag) before utilising CAPI data for EDDN messages. -Note that CAPI `/market` data will sometimes have the `StatusFlasg` per +Note that CAPI `/market` data will sometimes have the `statusFlasg` per item, which are defined as optional in this schema (because they're not in the Market.json data). You SHOULD include this data in your message if using CAPI as the source. @@ -91,4 +83,4 @@ any of the listed ships or modules have a `sku` value of #### CAPI odyssey flag Unfortunately there is no method to be *certain* of this from CAPI data, so you will have to trust in the system/station name check and use the value -from the Journal `LoadGame` event. \ No newline at end of file +from the Journal `LoadGame` event. diff --git a/schemas/commodity-v3.0.json b/schemas/commodity-v3.0.json index 3d4450b..60c5c78 100644 --- a/schemas/commodity-v3.0.json +++ b/schemas/commodity-v3.0.json @@ -40,10 +40,12 @@ }, "stationName": { "type" : "string", + "renamed" : "StarSystem", "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "horizons": { "type" : "boolean", @@ -67,31 +69,39 @@ "properties" : { "name": { "type" : "string", + "renamed" : "Name", "minLength" : 1, "description" : "Symbolic name as returned by the Companion API" }, "meanPrice": { - "type" : "integer" + "type" : "integer", + "renamed" : "MeanPrice" }, "buyPrice": { "type" : "integer", + "renaamed" : "BuyPrice", "description" : "Price to buy from the market" }, "stock": { - "type" : "integer" + "type" : "integer", + "renamed" : "Stock" }, "stockBracket": { - "$ref" : "#/definitions/levelType" + "$ref" : "#/definitions/levelType", + "renamed" : "StockBracket" }, "sellPrice": { "type" : "integer", + "renamed" : "SellPrice", "description" : "Price to sell to the market" }, "demand": { - "type" : "integer" + "type" : "integer", + "renamed" : "Demand" }, "demandBracket": { - "$ref" : "#/definitions/levelType" + "$ref" : "#/definitions/levelType", + "renamed" : "DemandBracket" }, "statusFlags": { "type" : "array", @@ -101,6 +111,18 @@ "type" : "string", "minLength" : 1 } + }, + "Producer": { + "$ref" : "#/definitions/disallowed", + "description" : "Not present in CAPI data, so removed from Journal-sourced data" + }, + "Rare" : { + "$ref" : "#/definitions/disallowed", + "description" : "Not present in CAPI data, so removed from Journal-sourced data" + }, + "id": { + "$ref" : "#/definitions/disallowed", + "description" : "Not wanted for historical reasons?" } } } @@ -130,11 +152,16 @@ "type" : "string", "minLength" : 1 } + }, + "StationType": { + "$ref" : "#/definitions/disallowed", + "description" : "Not present in CAPI data, so removed from Journal-sourced data" } } } }, "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } }, "levelType": { "enum" : [0, 1, 2, 3, ""], "description" : "Note: A value of \"\" indicates that the commodity is not normally sold/purchased at this station, but is currently temporarily for sale/purchase" From 2684d7bcf1b3bdb6ee8a298e09d58598b64cd1c2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:04:26 +0000 Subject: [PATCH 64/89] schemas/fssallbodiesfound: Bring in line with current standards * Remove un-necessary README text. * SystemAddress is always in the Journal events, so lose the misleading description. --- schemas/fssallbodiesfound-README.md | 15 +++------------ schemas/fssallbodiesfound-v1.0.json | 3 +-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md index a86db5c..8c9e585 100644 --- a/schemas/fssallbodiesfound-README.md +++ b/schemas/fssallbodiesfound-README.md @@ -11,19 +11,10 @@ documentation for a schema such as this. The primary data source for this schema is the ED Journal event `FSSAllBodiesFound`. -### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -None - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +over in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the diff --git a/schemas/fssallbodiesfound-v1.0.json b/schemas/fssallbodiesfound-v1.0.json index 074362a..61ec184 100644 --- a/schemas/fssallbodiesfound-v1.0.json +++ b/schemas/fssallbodiesfound-v1.0.json @@ -62,8 +62,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" + "type" : "integer" }, "Count" : { "type" : "integer", From 2efa760009c9d61d979d128543e20ce820f1361a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:07:51 +0000 Subject: [PATCH 65/89] schemas/fssdiscoveryscan: Bring in line with current standards * Remove un-necessary README text. * Remove un-necessary description on `SystemAddress` (it's always present). --- schemas/fssdiscoveryscan-README.md | 27 ++++++++++++--------------- schemas/fssdiscoveryscan-v1.0.json | 3 +-- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/schemas/fssdiscoveryscan-README.md b/schemas/fssdiscoveryscan-README.md index 6d4d844..db8db15 100644 --- a/schemas/fssdiscoveryscan-README.md +++ b/schemas/fssdiscoveryscan-README.md @@ -7,26 +7,23 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `FSSDiscoveryScan`. -### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -You MUST remove the following key/value pairs from the data: - - - `Progress` key/value pair. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the -last `FSDJump`, `CarrierJump`, or `Location` event. \ No newline at end of file +last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/fssdiscoveryscan-v1.0.json b/schemas/fssdiscoveryscan-v1.0.json index 0c3e7ad..9751db1 100644 --- a/schemas/fssdiscoveryscan-v1.0.json +++ b/schemas/fssdiscoveryscan-v1.0.json @@ -62,8 +62,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" + "type" : "integer" }, "Progress" : { "$ref" : "#/definitions/disallowed", From c73b3d6e083786c9b51f05424b1bc88cb18197c3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:11:13 +0000 Subject: [PATCH 66/89] schemas: Align Introduction blurbs --- schemas/approachsettlement-README.md | 30 +++++++++++++++------------- schemas/commodity-README.md | 8 ++++++++ schemas/fssallbodiesfound-README.md | 8 ++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md index 44d72ca..a4852f6 100644 --- a/schemas/approachsettlement-README.md +++ b/schemas/approachsettlement-README.md @@ -7,27 +7,29 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `ApproachSettlement`. -### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -None - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarSystem -You MUST add a StarSystem key/value pair representing the name of the system this event occurred in. Source this from either Location, FSDJump or CarrierJump as appropriate. +You MUST add a StarSystem key/value pair representing the name of the system +this event occurred in. Source this from either Location, FSDJump or +CarrierJump as appropriate. #### StarPos -You MUST add a `StarPos` array containing the system co-ordinates from the last `FSDJump`, `CarrierJump`, or `Location` event. +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index 18c9cc5..d319026 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -7,6 +7,14 @@ properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `Market`, and the additional file, `Market.json`, that it signals the writing of. diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md index 8c9e585..8a5d446 100644 --- a/schemas/fssallbodiesfound-README.md +++ b/schemas/fssallbodiesfound-README.md @@ -7,6 +7,14 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `FSSAllBodiesFound`. From adcf6b60101bc7445b854f87aa8048ef3534b3c8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:13:56 +0000 Subject: [PATCH 67/89] schemas/navbeaconscan: Bring in line with current standards --- schemas/navbeaconscan-README.md | 19 +++++++++++-------- schemas/navbeaconscan-v1.0.json | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/schemas/navbeaconscan-README.md b/schemas/navbeaconscan-README.md index c12e91b..7a3bdd4 100644 --- a/schemas/navbeaconscan-README.md +++ b/schemas/navbeaconscan-README.md @@ -7,19 +7,22 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `NavBeaconScan`. -### Elisions -There are no elisions in this schema. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +over in the Developers' documentation. #### StarSystem You MUST add a `StarSystem` key/value pair representing the name of the diff --git a/schemas/navbeaconscan-v1.0.json b/schemas/navbeaconscan-v1.0.json index 64d55bf..b40cfa9 100644 --- a/schemas/navbeaconscan-v1.0.json +++ b/schemas/navbeaconscan-v1.0.json @@ -52,7 +52,8 @@ }, "StarSystem": { "type" : "string", - "minLength" : 1 + "minLength" : 1, + "description" : "Should be added by the sender if not present in the journal event" }, "StarPos": { "type" : "array", @@ -63,7 +64,6 @@ }, "SystemAddress": { "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" }, "NumBodies" : { "type" : "integer" From 2969bb2eb11b734ac2746fc990cc52b9867e0de0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:16:46 +0000 Subject: [PATCH 68/89] schemas/navroute: Bring in line with current standards --- schemas/navroute-README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/schemas/navroute-README.md b/schemas/navroute-README.md index 3025baa..fe5646e 100644 --- a/schemas/navroute-README.md +++ b/schemas/navroute-README.md @@ -7,6 +7,14 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the `NavRoute.json` file. That it has been freshly written is signalled by the ED Journal event `NavRoute`. @@ -18,12 +26,8 @@ data you got from reading this file, not merely the Journal event. The primary data to be sent is the `Route` array from the contents of the separate file. -### Elisions -There are no elisions in this schema. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +over in the Developers' documentation. -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. \ No newline at end of file From cb849f3d9edf3a3df1ad089cbee6d7ba3c96f26c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:38:43 +0000 Subject: [PATCH 69/89] schemas/template: Minor grammar tweak to README --- schemas/TEMPLATES/journalevent-README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index 2ba8d3e..82d08f7 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -104,7 +104,7 @@ value is what the name would have been in the source Journal data. ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) -over in the Developers' documentation. +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the From c903df1dd55f7547e65bbc3769377cac625bda29 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:39:35 +0000 Subject: [PATCH 70/89] schemas/outfitting: Bring in line with current standards --- schemas/outfitting-README.md | 27 ++++++++++++++++++++------- schemas/outfitting-v2.0.json | 6 +++++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/schemas/outfitting-README.md b/schemas/outfitting-README.md index 090fc61..ac9ae8f 100644 --- a/schemas/outfitting-README.md +++ b/schemas/outfitting-README.md @@ -7,6 +7,14 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `Outfitting`. @@ -19,8 +27,15 @@ before utilising CAPI data for EDDN messages. You only need the `name` key's value for each member of the `modules` array. ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. + +### The modules/Items list +The source data, Journal or CAPI, contains more than just the names of the +available items. This Schema is only concerned with the names, so the list +you build will have only strings as its members, not including other information +such as id, category, cost/BuyPrice, sku or stock. ### Elisions Remove items whose availability depends on the Cmdr's status rather than on the @@ -35,8 +50,6 @@ station. Namely: - The `"Int_PlanetApproachSuite"` module (for historical reasons). ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. \ No newline at end of file +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. diff --git a/schemas/outfitting-v2.0.json b/schemas/outfitting-v2.0.json index 3a8e49d..a8959e0 100644 --- a/schemas/outfitting-v2.0.json +++ b/schemas/outfitting-v2.0.json @@ -36,14 +36,17 @@ "properties" : { "systemName": { "type" : "string", + "renamed" : "StarSystem", "minLength" : 1 }, "stationName": { "type" : "string", + "renamed" : "StationName", "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "horizons": { "type" : "boolean", @@ -59,6 +62,7 @@ }, "modules": { "type" : "array", + "renamed" : "Items", "minItems" : 1, "uniqueItems" : true, "items" : { From ec781aa94fc3d53602198bbe482f9f3a1a1f220e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:40:28 +0000 Subject: [PATCH 71/89] schemas/misc READMEs: Remove extraneous 'over ' --- schemas/blackmarket-README.md | 2 +- schemas/codexentry-README.md | 2 +- schemas/commodity-README.md | 2 +- schemas/fssallbodiesfound-README.md | 2 +- schemas/navbeaconscan-README.md | 2 +- schemas/navroute-README.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/schemas/blackmarket-README.md b/schemas/blackmarket-README.md index 499e650..f3079d6 100644 --- a/schemas/blackmarket-README.md +++ b/schemas/blackmarket-README.md @@ -35,7 +35,7 @@ You MUST remove the following key/value pairs from the data: ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) -over in the main Schema documentation. +in the main Schema documentation. #### systemName The star system name for where this market is. Use the `StarSystem` value diff --git a/schemas/codexentry-README.md b/schemas/codexentry-README.md index b3c5fc6..1e4f6ea 100644 --- a/schemas/codexentry-README.md +++ b/schemas/codexentry-README.md @@ -21,7 +21,7 @@ The primary data source for this schema is the ED Journal event `CodexEntry`. ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) -over in the main Schema documentation. +in the main Schema documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index d319026..9e6a5ca 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -58,7 +58,7 @@ Remove not only the `Category_Localised` key:values, but also the ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) -over in the Developers' documentation. +in the Developers' documentation. ### Using CAPI data It is *not* recommended to use CAPI data as the source as it's fraught with diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md index 8a5d446..059f26e 100644 --- a/schemas/fssallbodiesfound-README.md +++ b/schemas/fssallbodiesfound-README.md @@ -22,7 +22,7 @@ The primary data source for this schema is the ED Journal event ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) -over in the Developers' documentation. +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the diff --git a/schemas/navbeaconscan-README.md b/schemas/navbeaconscan-README.md index 7a3bdd4..b78ff36 100644 --- a/schemas/navbeaconscan-README.md +++ b/schemas/navbeaconscan-README.md @@ -22,7 +22,7 @@ The primary data source for this schema is the ED Journal event ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) -over in the Developers' documentation. +in the Developers' documentation. #### StarSystem You MUST add a `StarSystem` key/value pair representing the name of the diff --git a/schemas/navroute-README.md b/schemas/navroute-README.md index fe5646e..050a778 100644 --- a/schemas/navroute-README.md +++ b/schemas/navroute-README.md @@ -29,5 +29,5 @@ separate file. ### Augmentations #### horizons and odyssey flags Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) -over in the Developers' documentation. +in the Developers' documentation. From 6b213795e529b8fddd69abe51828585da3df0df3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 16:54:58 +0000 Subject: [PATCH 72/89] schemas/scanbarycentre: Bring in line with current standards --- schemas/scanbarycentre-README.md | 20 +++++++++++++------- schemas/scanbarycentre-v1.0.json | 3 +-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/schemas/scanbarycentre-README.md b/schemas/scanbarycentre-README.md index da0e178..efe75c5 100644 --- a/schemas/scanbarycentre-README.md +++ b/schemas/scanbarycentre-README.md @@ -7,19 +7,25 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `ScanBaryCentre`. -### Elisions -There are no elisions in this schema. +Although most of the event-specific data is not specified as `required`, +senders SHOULD include any defined in the schema if it's in the source data. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the diff --git a/schemas/scanbarycentre-v1.0.json b/schemas/scanbarycentre-v1.0.json index 15a8985..e019326 100644 --- a/schemas/scanbarycentre-v1.0.json +++ b/schemas/scanbarycentre-v1.0.json @@ -62,8 +62,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" + "type" : "integer" }, "BodyID": { "type" : "integer" From 5492b4fb60e1ccc3296f5856ec5d5b3c368449bd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:14:59 +0000 Subject: [PATCH 73/89] schemas/shipyard: Align with current standards --- schemas/shipyard-README.md | 34 ++++++++++++++++++---------------- schemas/shipyard-v2.0.json | 6 +++++- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/schemas/shipyard-README.md b/schemas/shipyard-README.md index 33016a4..bb69d84 100644 --- a/schemas/shipyard-README.md +++ b/schemas/shipyard-README.md @@ -7,34 +7,36 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `Shipyard`. You MAY also source this data from the CAPI `/shipyard` endpoint. Please read -[the guidance on checking for CAPI lag](README-EDDN-schemas.md#detecting-capi-data-lag) +[the guidance on checking for CAPI lag](../docs/Developers.md#detecting-capi-data-lag) before utilising CAPI data for EDDN messages. - You only need the `name` key's value for each member of the `PriceList` -array (if using Journal, it will be from the `ships` array if using CAPI -data). +The `ships` array is built from *only* the `name` values of either the Journal +`PriceList` array in the `Shipyard.json` file, or from the `ships` array of +CAPI `/shipyard` data. When using CAPI data *include* ships listed in the `"unavailable_list"` property (i.e. available at this station, but not to this Cmdr). -This list of ship names will go in the `ships` array in the EDDN message. - ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -There are no elisions in this schema. +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. diff --git a/schemas/shipyard-v2.0.json b/schemas/shipyard-v2.0.json index 9de2cc8..bb4fabd 100644 --- a/schemas/shipyard-v2.0.json +++ b/schemas/shipyard-v2.0.json @@ -36,14 +36,17 @@ "properties" : { "systemName": { "type" : "string", + "renamed" : "StarSystem", "minLength" : 1 }, "stationName": { "type" : "string", + "renamed" : "StationName", "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "horizons": { "type" : "boolean", @@ -63,6 +66,7 @@ }, "ships": { "type" : "array", + "renamed" : "PriceList", "minItems" : 1, "uniqueItems" : true, "items" : { From a07e2ad86a1c7cc5eb9e671491905bec2ea10ee4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:16:30 +0000 Subject: [PATCH 74/89] schemas/outfitting: Fix 'capi lag' link to new location --- schemas/outfitting-README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/outfitting-README.md b/schemas/outfitting-README.md index ac9ae8f..33406f6 100644 --- a/schemas/outfitting-README.md +++ b/schemas/outfitting-README.md @@ -21,7 +21,7 @@ The primary data source for this schema is the ED Journal event You MAY also source this data from the CAPI `/shipyard` endpoint. Please read -[the guidance on checking for CAPI lag](README-EDDN-schemas.md#detecting-capi-data-lag) +[the guidance on checking for CAPI lag](../docs/Developers.md#detecting-capi-data-lag) before utilising CAPI data for EDDN messages. You only need the `name` key's value for each member of the `modules` array. From 3474fcc8da67aab7cbda73424bc57ac996bd6d6b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:25:28 +0000 Subject: [PATCH 75/89] schemas/READMEs: Fix the 'horizons and odyssey' link It was 'correct' in the template, but that's a directory deeper. Some instances were still pointing at the wrong file. --- schemas/TEMPLATES/journalevent-README.md | 2 +- schemas/approachsettlement-README.md | 2 +- schemas/blackmarket-README.md | 4 ++-- schemas/codexentry-README.md | 4 ++-- schemas/commodity-README.md | 2 +- schemas/fssallbodiesfound-README.md | 2 +- schemas/fssdiscoveryscan-README.md | 2 +- schemas/navbeaconscan-README.md | 2 +- schemas/navroute-README.md | 2 +- schemas/outfitting-README.md | 2 +- schemas/scanbarycentre-README.md | 2 +- schemas/shipyard-README.md | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md index 82d08f7..f85fc86 100644 --- a/schemas/TEMPLATES/journalevent-README.md +++ b/schemas/TEMPLATES/journalevent-README.md @@ -103,7 +103,7 @@ value is what the name would have been in the source Journal data. ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. #### StarPos diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md index a4852f6..9dfe6d1 100644 --- a/schemas/approachsettlement-README.md +++ b/schemas/approachsettlement-README.md @@ -21,7 +21,7 @@ The primary data source for this schema is the ED Journal event ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. #### StarSystem diff --git a/schemas/blackmarket-README.md b/schemas/blackmarket-README.md index f3079d6..c5a8eda 100644 --- a/schemas/blackmarket-README.md +++ b/schemas/blackmarket-README.md @@ -34,8 +34,8 @@ You MUST remove the following key/value pairs from the data: ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) -in the main Schema documentation. +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### systemName The star system name for where this market is. Use the `StarSystem` value diff --git a/schemas/codexentry-README.md b/schemas/codexentry-README.md index 1e4f6ea..cfef4d7 100644 --- a/schemas/codexentry-README.md +++ b/schemas/codexentry-README.md @@ -20,8 +20,8 @@ The primary data source for this schema is the ED Journal event `CodexEntry`. ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../README-EDDN-schemas.md#horizons-and-odyssey-flags) -in the main Schema documentation. +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index 9e6a5ca..3e155b6 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -57,7 +57,7 @@ Remove not only the `Category_Localised` key:values, but also the ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. ### Using CAPI data diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md index 059f26e..350a318 100644 --- a/schemas/fssallbodiesfound-README.md +++ b/schemas/fssallbodiesfound-README.md @@ -21,7 +21,7 @@ The primary data source for this schema is the ED Journal event ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. #### StarPos diff --git a/schemas/fssdiscoveryscan-README.md b/schemas/fssdiscoveryscan-README.md index db8db15..ba5a5eb 100644 --- a/schemas/fssdiscoveryscan-README.md +++ b/schemas/fssdiscoveryscan-README.md @@ -21,7 +21,7 @@ The primary data source for this schema is the ED Journal event ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. #### StarPos diff --git a/schemas/navbeaconscan-README.md b/schemas/navbeaconscan-README.md index b78ff36..39167a2 100644 --- a/schemas/navbeaconscan-README.md +++ b/schemas/navbeaconscan-README.md @@ -21,7 +21,7 @@ The primary data source for this schema is the ED Journal event ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. #### StarSystem diff --git a/schemas/navroute-README.md b/schemas/navroute-README.md index 050a778..703c794 100644 --- a/schemas/navroute-README.md +++ b/schemas/navroute-README.md @@ -28,6 +28,6 @@ separate file. ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. diff --git a/schemas/outfitting-README.md b/schemas/outfitting-README.md index 33406f6..e3e18bf 100644 --- a/schemas/outfitting-README.md +++ b/schemas/outfitting-README.md @@ -51,5 +51,5 @@ station. Namely: ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. diff --git a/schemas/scanbarycentre-README.md b/schemas/scanbarycentre-README.md index efe75c5..d7b5f01 100644 --- a/schemas/scanbarycentre-README.md +++ b/schemas/scanbarycentre-README.md @@ -24,7 +24,7 @@ senders SHOULD include any defined in the schema if it's in the source data. ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. #### StarPos diff --git a/schemas/shipyard-README.md b/schemas/shipyard-README.md index bb69d84..b1f32d9 100644 --- a/schemas/shipyard-README.md +++ b/schemas/shipyard-README.md @@ -38,5 +38,5 @@ value is what the name would have been in the source Journal data. ### Augmentations #### horizons and odyssey flags -Please read [horizons and odyssey flags](../../docs/Developers.md#horizons-and-odyssey-flags) +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) in the Developers' documentation. From 36ca116095fbbd4a436d96cb4fb54dfd5633346c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:35:03 +0000 Subject: [PATCH 76/89] docs/Contribyting: Schema READMEs shouldn't repeat what the JSON file states --- docs/Contributing.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/Contributing.md b/docs/Contributing.md index 9a5cb2b..72437b8 100644 --- a/docs/Contributing.md +++ b/docs/Contributing.md @@ -319,12 +319,21 @@ as a mandatory elision, should be defined, but remain optional. #### Schema README requirements -The per-Schema README **MUST** give both Senders and Listeners sufficient -information to correctly handle the pertinent data. You do not need to repeat -anything already specified in the general Schema README. Referring to it via -MarkDown linking is helpful. +The combination of the Schema file itself and its README **MUST** give both +Senders and Listeners sufficient information to correctly handle the pertinent +data. -1. The reason(s) for any augmentations to a message must be clearly explained. +If the Schema signifies a field should be removed then you do not need +to explicitly call it out in the README. + +Likewise, the Schema MUST note, with a `renamed` property, where the key name +for some data differs from that in the Journal source data. Because this is +mandatory you do not then need to list such in the README. + +You do not need to repeat anything already specified in the general +Schema README. Referring to it via MarkDown linking is helpful. + +1. Any augmentations to a message must be clearly explained. 1. **DO** outline where the additional data comes from. e.g. `StarPos` added to many events should come from a prior `Location`, `FSDJump` or `CarrierJump` Journal event. @@ -345,5 +354,3 @@ MarkDown linking is helpful. --- --- - - From a1a20f511a546408ebaf29d4fc31276866e6dc68 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:38:38 +0000 Subject: [PATCH 77/89] schemas/codexentry: Don't linkify 'Journal' That section is gone/moved/renamed, and this is un-necessary anyway. --- schemas/codexentry-README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schemas/codexentry-README.md b/schemas/codexentry-README.md index cfef4d7..1c675d3 100644 --- a/schemas/codexentry-README.md +++ b/schemas/codexentry-README.md @@ -32,9 +32,8 @@ You SHOULD attempt to track the BodyName and BodyID where the player is and add keys/values for these. You MUST track `BodyName` both from Status.json *and* also from some -[Journal](./README-EDDN-schemas.md#journal-files) -events in order to cross-check it before using the `BodyID` from -[Journal](./README-EDDN-schemas.md#journal-files) events. +Journal events in order to cross-check it before using the `BodyID` from +Journal events. The following is correct as of game version 4.0.0.801 (Odyssey initial release, Update 7, plus one patch). From 7f6fe9cd76eeb124b4ba61ef2d43c93b25d2cbc9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:43:34 +0000 Subject: [PATCH 78/89] docs/Adding-A-New-Schema: Reference Contributing.md section We might just remove this file, but it can stay for now. --- docs/Adding-A-New-Schema.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/Adding-A-New-Schema.md b/docs/Adding-A-New-Schema.md index fcdcf1d..1759d06 100644 --- a/docs/Adding-A-New-Schema.md +++ b/docs/Adding-A-New-Schema.md @@ -1,11 +1,16 @@ # Adding A New Schema ## Introduction + As of September 2021 it was decided that all new Journal events will be added to their own, new, schemas. This better facilitates defining any values that should be elided, or augmentations added, without twisting schema definitions into knots. +Consult +[Contributing.md#adding-a-new-schema](./Contributing.md#adding-a-new-schema) +for guidelines on the specifics of proposing and designing a new Scema. + In the future we will likely migrate all of the events currently supported in the journal schema into their own schemas, and later still deprecate the journal schema. From 88cabbf032cb565b72bc8f5f4b55cce7a866f544 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sun, 30 Jan 2022 17:50:49 +0000 Subject: [PATCH 79/89] docs/Developers: Simplify link to 'Running-this-software.md' --- docs/Developers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Developers.md b/docs/Developers.md index 4dddb87..6170e6b 100644 --- a/docs/Developers.md +++ b/docs/Developers.md @@ -109,7 +109,7 @@ ones. Ask on the `#eddn` channel of the link). Alternatively you could attempt -[running your own test instance of EDDN](../docs/Running-this-software.md). +[running your own test instance of EDDN](./Running-this-software.md). ### Sending data Messages sent to EDDN **MUST**: From 245a111248809d64a24323edebd16b9934a18cc8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 31 Jan 2022 10:23:30 +0000 Subject: [PATCH 80/89] scripts/testing: Example codexentry file --- scripts/testing/gateway-responses/codexentry.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 scripts/testing/gateway-responses/codexentry.json diff --git a/scripts/testing/gateway-responses/codexentry.json b/scripts/testing/gateway-responses/codexentry.json new file mode 100644 index 0000000..6b885fe --- /dev/null +++ b/scripts/testing/gateway-responses/codexentry.json @@ -0,0 +1 @@ +{ "timestamp":"2021-09-24T14:29:39Z", "event":"CodexEntry", "EntryID":1400414, "Name":"$Codex_Ent_Gas_Vents_SilicateVapourGeysers_Name;", "Name_Localised":"Silicate Vapour Gas Vent", "SubCategory":"$Codex_SubCategory_Geology_and_Anomalies;", "SubCategory_Localised":"Geology and anomalies", "Category":"$Codex_Category_Biology;", "Category_Localised":"Biological and Geological", "Region":"$Codex_RegionName_18;", "Region_Localised":"Inner Orion Spur", "System":"Bestia", "SystemAddress":147916327267, "Latitude":23.197777, "Longitude":51.803349, "IsNewEntry":true, "VoucherAmount":50000 } From 13287083c1bf9ccd958c547a16e045d5abbd4adf Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 31 Jan 2022 10:24:18 +0000 Subject: [PATCH 81/89] scripts/testing: Actual 'EDDN message' version of codexentry.json --- .../testing/gateway-responses/codexentry.json | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/testing/gateway-responses/codexentry.json b/scripts/testing/gateway-responses/codexentry.json index 6b885fe..01cabee 100644 --- a/scripts/testing/gateway-responses/codexentry.json +++ b/scripts/testing/gateway-responses/codexentry.json @@ -1 +1,26 @@ -{ "timestamp":"2021-09-24T14:29:39Z", "event":"CodexEntry", "EntryID":1400414, "Name":"$Codex_Ent_Gas_Vents_SilicateVapourGeysers_Name;", "Name_Localised":"Silicate Vapour Gas Vent", "SubCategory":"$Codex_SubCategory_Geology_and_Anomalies;", "SubCategory_Localised":"Geology and anomalies", "Category":"$Codex_Category_Biology;", "Category_Localised":"Biological and Geological", "Region":"$Codex_RegionName_18;", "Region_Localised":"Inner Orion Spur", "System":"Bestia", "SystemAddress":147916327267, "Latitude":23.197777, "Longitude":51.803349, "IsNewEntry":true, "VoucherAmount":50000 } +{ + "$schemaRef": "https://eddn.edcd.io/schemas/codexentry-v1.0.json", + "header": { + "uploaderID": "Ath Testing", + "softwareName": "Athanasius test code", + "softwareVersion": "v0.0.1" + }, + "message": { + "timestamp":"2021-09-24T14:29:39Z", + "event":"CodexEntry", + "EntryID":1400414, + "Name":"$Codex_Ent_Gas_Vents_SilicateVapourGeysers_Name;", + "SubCategory":"$Codex_SubCategory_Geology_and_Anomalies;", + "Category":"$Codex_Category_Biology;", + "Region":"$Codex_RegionName_18;", + "System":"Bestia", + "SystemAddress":147916327267, + "StarPos": [ + 1.000, + 2.000, + 3.000 + ], + "Latitude":23.197777, + "Longitude":51.803349 + } +} From 6ac735b1acab0909e47c0a4f4cf110f9826ba145 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 31 Jan 2022 10:31:07 +0000 Subject: [PATCH 82/89] schemas/navbeacon: Removing dangling comma --- schemas/navbeaconscan-v1.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/navbeaconscan-v1.0.json b/schemas/navbeaconscan-v1.0.json index b40cfa9..b3fdc94 100644 --- a/schemas/navbeaconscan-v1.0.json +++ b/schemas/navbeaconscan-v1.0.json @@ -63,7 +63,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", + "type" : "integer" }, "NumBodies" : { "type" : "integer" From a619a2a2ec2cf6f3dc6c114982bddb3fd650bf02 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 31 Jan 2022 10:34:26 +0000 Subject: [PATCH 83/89] scripts/testing: Correct example codexentry $schemaRef --- scripts/testing/gateway-responses/codexentry.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/testing/gateway-responses/codexentry.json b/scripts/testing/gateway-responses/codexentry.json index 01cabee..25f8889 100644 --- a/scripts/testing/gateway-responses/codexentry.json +++ b/scripts/testing/gateway-responses/codexentry.json @@ -1,5 +1,5 @@ { - "$schemaRef": "https://eddn.edcd.io/schemas/codexentry-v1.0.json", + "$schemaRef": "https://eddn.edcd.io/schemas/codexentry/1", "header": { "uploaderID": "Ath Testing", "softwareName": "Athanasius test code", From c797bdb677a987fcbe8a939333cb5d51190a0345 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 8 Feb 2022 16:56:28 +0000 Subject: [PATCH 84/89] README: Add "Archives and data dumps" section --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 586e0d4..6749a49 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ archive or "current state" available to anyone. What it provides is a stream of live data to any interested parties. Some of those then make aggregated data available for general use. +If you want an archive of messages, or a dump of the current known state of +the game galaxy, check ["Archives and data dumps"](#archives-and-data-dumps) +below. + --- --- @@ -73,6 +77,28 @@ intended. a 'Neutron Star' route plotter, but has since expanded into offering many other route plotting tools and general data searching. +##### Archives and data dumps + +Alternatively if you want an archive of past EDDN messages, or a data dump to +use: + +- [edgalaxydata](https://edgalaxydata.space/) has various data captures, + including 'all' (some listener downtime is inevitable) EDDN messages for + many years. + +- [spansh dumps](https://www.spansh.co.uk/dumps) are a "whole galaxy" data set, + of systems, bodies and stations. The full `galaxy.json.gz` is **very** + large, but is currently the only source of an "all known bodies" dump. + Pay attention to the 'Generated' "time ago" column. + +- [EDDB dumps](https://eddb.io/api) represent a snapshot of the data EDDB uses. + NB: There has been no "bodies" data for years now, EDDB itself stopped + updating or adding to this. + +- [EDSM nightly dumps](https://www.edsm.net/en/nightly-dumps) represent a + snapshot of the data EDSM uses. NB: there's only a "last 7 days" bodies + dump as the full data proved too large to dump in a timely manner. + --- There are many other third-party tools for Elite Dangerous, both for From cb1991739e4185321c4196cf123e05d0e5cb3428 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 10 Feb 2022 15:46:47 +0000 Subject: [PATCH 85/89] scripts/apache-log-rate: Determine highest message rate from apache logs --- scripts/apache-log-rate | 98 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100755 scripts/apache-log-rate diff --git a/scripts/apache-log-rate b/scripts/apache-log-rate new file mode 100755 index 0000000..d5e9106 --- /dev/null +++ b/scripts/apache-log-rate @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# vim: wrapmargin=0 textwidth=0 smarttab expandtab tabstop=2 shiftwidth=2 +"""Process Apache access.log lines to find highest rate of /upload/.""" + +import argparse +import datetime +import dateutil.parser +import fileinput +import re + + +def process_log_file( + input_file: str ='-', + request_text: str = '/upload/', + window_size: int = 1, +) -> None: + """ + Process the indicated log file to determine peak rate of interesting lines. + + :param input_file: Name of input file, `-` for stdin + :param request_text: The text that denotes an interesting line + :param window_size: Time, in seconds, for the window to assess + """ + print(f'With:\n\tinput_file: "{input_file}"\n\trequest_text: "{request_text}"') + with fileinput.FileInput(files=(input_file)) as f: + apache_re = re.compile(r'^(?P[.:0-9a-fA-F]{3,39}) - - \[(?P[^\]]+)\] (?P.*' + request_text + '.*)$') + apache_datetime_re = re.compile( + r'^(?P[0-9]{2})/(?P[^/]{3})/(?P[0-9]{4}):(?P