diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 7905cbb..8423755 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -15,7 +15,7 @@ those actually running on the Live service. The Schema files themselves are considered to be the canonical definition of the required, and allowed, contents of the relevant EDDN message. There -**SHOULD** be an accompanying README file, e.g. for `commodity-v3.0.json` there +**MUST** be an accompanying README file, e.g. for `commodity-v3.0.json` there is also a `commodity-README.md` file in the project root `schemas/` directory. For more general documentation that all developers wanting to either Upload @@ -29,7 +29,16 @@ It is best to base any new Schema file on contents all Schemas specify a top-level JSON Object with the data: 1. `$schemaRef` - Which Schema (including version) this message is for. -2. `header` - Object containing mandatory information about the upload; +2. `$id` - The canonical URL for this schema once it is in live service. + 1. Remember to have the version as in `journal/1` not `journal-v1.0`. + 2. Do **NOT** end this with a `#` empty fragment. This is + [documented](https://json-schema.org/draft/2020-12/json-schema-core.html#section-8.2.1) + as unnecessary. + 3. Where there are two separate schemas for the same kind of data, but one + is for Journal-sourced, and the other for CAPI-sourced, you should have + the "filename" of the schema end with `_`, e.g. + `fcmaterials_journal/1` and `fcmaterials_capi/1`. +3. `header` - Object containing mandatory information about the upload; 1. `uploaderID` - a unique ID for the player uploading this data. Don't worry about privacy, the EDDN service will hash this with a key that is regularly changed so no-one knows who an uploader is in-game. diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json index a87c8fe..0b64ec9 100644 --- a/schemas/TEMPLATES/journalevent-v1.0.json +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -39,7 +39,7 @@ # as a result disallow the key. { "$schema" : "http://json-schema.org/draft-04/schema#", - "id" : "https://eddn.edcd.io/schemas/newjournalevent/1#", + "id" : "https://eddn.edcd.io/schemas/newjournalevent/1", "type" : "object", "additionalProperties" : false, "required": [ "$schemaRef", "header", "message" ], diff --git a/schemas/fcmaterials_capi-README.md b/schemas/fcmaterials_capi-README.md new file mode 100644 index 0000000..a76f136 --- /dev/null +++ b/schemas/fcmaterials_capi-README.md @@ -0,0 +1,78 @@ +# EDDN FCMaterials Schema + +## Introduction +This is the documentation for how to take data from an the ED CAPI `/market` +endpoint and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The data source for this schema is the `/market` ED CAPI endpoint. You only +want selected parts of the full data returned for this schema. + +You **MUST NOT** construct the message by starting with the entirety of the +CAPI data and then removing everything but what you need. That risks Frontier +adding more data to the endpoint and your `fcmaterials_capi` messages being +rejected as invalid. Instead, construct the message content by setting just +the data that is necessary. + +Your `message` object **MUST**: +1. Have an `"event":"FCMaterials"` member to aid Listeners who pass this + through a "usually from the Journal" code path. +2. Set a `"MarketID"` key with the value from `"id"` in the CAPI data. +3. Set a `"CarrierID"` key with the value from the `"name"` in the CAPI data. +4. Set the `"Items"` key's contents directly from the `/market` -> `orders` + -> `onfootmicroresources` CAPI data. +5. Remove any data where the key is `"locName"` from the `"Items"` data. + +You **MUST NOT**: +1. Attempt to set a `"CarrierName"` from any source, the `CarrierID` is + sufficient. + +### Example algorithm + +1. Make a CAPI `/market` query. +2. Set `event`, `MarketID` and `CarrierID` as outlined above. +3. Set `Items` value to the data in `orders.onfootmicroresources`. +4. Process the contents of `Items`, removing any data with a key of `locName`. + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. + +You **SHOULD** set these flags from the Journal `FileHeader` data if you are +reasonably sure you have a live game session against which you are performing +CAPI queries. +You **MUST NOT** set them otherwise, as e.g. the player could be active in +the game on another computer, using a different game mode and the CAPI data +will be for that game mode. + +## Listeners +The advice above for [Senders](#senders), combined with the actual Schema file +*should* provide all the information you need to process these events. + +Do note that the data source for this is the CAPI, and as such the data is not +the same as for the `fcmaterials_journal` schema: + +1. There is no good source of `CarrierName` in CAPI `/market` endpoint data, so + that is not included. +2. The `sales` and `purchases` values do **not** contain the same form of data. +3. The `sales` member of `Items` will be `[]` if there are no sales orders, but + when there are orders: + 1. It will be an object/dictionary. + 2. The keys are the commodity ID. + 3. The value of that key is the rest of the data for that sales order. +4. The `purchases` value: + 1. Is always an array, unlike `sales`. + 2. As a consequence does **not** provide the commodity id at all, only + the name. diff --git a/schemas/fcmaterials_capi-v1.0.json b/schemas/fcmaterials_capi-v1.0.json new file mode 100644 index 0000000..49edb06 --- /dev/null +++ b/schemas/fcmaterials_capi-v1.0.json @@ -0,0 +1,144 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fcmaterials_capi/1", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event", "MarketID", "CarrierID", "Items" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "FCMaterials" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." + }, + "odyssey": { + "type" : "boolean", + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." + }, + + "MarketID": { + "type" : "integer" + }, + + "CarrierID": { + "type" : "string", + "minLength" : 1 + }, + + "Items": { + "properties": { + "purchases": { + "type": "array", + "items": { + "type": "object", + "required": [ "name", "price", "outstanding", "total" ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "locName": { + "$ref": "#/definitions/disallowed" + }, + "outstanding": { + "type": "integer" + }, + "price": { + "type": "integer" + }, + "total": { + "type": "integer" + } + } + } + }, + "sales": { + "anyOf": [ + { + "type": "array", + "$comment": "If there are no items then sales is an empty array", + "minItems": 0, + "maxItems": 0 + }, + { + "type": "object", + "$comment": "If there ARE items then sales is an object, *NOT* an array", + "patternProperties": { + "^[0-9]+$": { + "type" : "object", + "required" : [ "id", "name", "price", "stock" ], + "additionalProperties": false, + "properties": { + "id" : { + "type" : "integer" + }, + "name" : { + "type" : "string", + "minLength": 1 + }, + "locName" : { + "$ref": "#/definitions/disallowed" + }, + "price" : { + "type": "integer" + }, + "stock": { + "type": "integer" + } + } + } + } + } + ] + } + } + } + } + } + }, + "definitions": { + "disallowed": { + "not" : { + "type": [ + "array", "boolean", "integer", "number", "null", "object", "string" + ] + } + } + } +} diff --git a/schemas/fcmaterials_journal-README.md b/schemas/fcmaterials_journal-README.md new file mode 100644 index 0000000..357da91 --- /dev/null +++ b/schemas/fcmaterials_journal-README.md @@ -0,0 +1,38 @@ +# EDDN FCMaterials Schema + +## Introduction +This is the documentation for how to take data from an ED `FCMaterials.json` +file and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The data source for this schema is the file `FCMaterials.json`. That it has +been freshly written is signalled by the ED Journal event `FCMaterials`. +**NB: This schema is not, currently, for sending CAPI `/market`-sourced data +about these materials.** + +So, monitor the Journal as normal, and when you see a `FCMaterials` event open +the `FCMaterials.json` file for reading, read it, and close it again. Use the +data you got from reading this file, not merely the Journal event. + +Your `message` should primarily be the contents of this file, with the addition +of any augmentations, as noted below. + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. + +## Listeners +The advice above for [Senders](#senders), combined with the actual Schema file +*should* provide all the information you need to process these events. diff --git a/schemas/fcmaterials_journal-v1.0.json b/schemas/fcmaterials_journal-v1.0.json new file mode 100644 index 0000000..35bb080 --- /dev/null +++ b/schemas/fcmaterials_journal-v1.0.json @@ -0,0 +1,99 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fcmaterials_journal/1", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event", "MarketID", "CarrierName", "CarrierID", "Items" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "FCMaterials" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." + }, + "odyssey": { + "type" : "boolean", + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." + }, + + "MarketID": { + "type" : "integer" + }, + + "CarrierName": { + "type" : "string", + "minLength" : 1 + }, + + "CarrierID": { + "type" : "string", + "minLength" : 1 + }, + + "Items": { + "type" : "array", + "required" : [ "id", "Name", "Price", "Stock", "Demand" ], + "properties" : { + "id" : { + "type" : "integer" + }, + "Name": { + "type" : "string", + "minLength" : 1 + }, + "Price": { + "type" : "integer" + }, + "Stock": { + "type" : "integer" + }, + "Demand": { + "type" : "integer" + } + }, + "patternProperties": { + "_Localised$" : { "$ref" : "#/definitions/disallowed" } + } + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} diff --git a/scripts/test-schema.py b/scripts/test-schema.py new file mode 100644 index 0000000..8df8617 --- /dev/null +++ b/scripts/test-schema.py @@ -0,0 +1,17 @@ +import sys + +import simplejson +import jsonschema + +schema_filename = sys.argv[1] +message_filename = sys.argv[2] + +schema_file = open(schema_filename, 'r') +schema_data = schema_file.read() +schema = simplejson.loads(schema_data) + +message_file = open(message_filename, 'r') +message_data = message_file.read() +message = simplejson.loads(message_data) + +jsonschema.validate(message, schema, format_checker=jsonschema.FormatChecker()) diff --git a/src/eddn/conf/Settings.py b/src/eddn/conf/Settings.py index 28e9b21..9e2f6a5 100644 --- a/src/eddn/conf/Settings.py +++ b/src/eddn/conf/Settings.py @@ -84,6 +84,12 @@ class _Settings(object): "https://eddn.edcd.io/schemas/fsssignaldiscovered/1" : "schemas/fsssignaldiscovered-v1.0.json", "https://eddn.edcd.io/schemas/fsssignaldiscovered/1/test" : "schemas/fsssignaldiscovered-v1.0.json", + + "https://eddn.edcd.io/schemas/fcmaterials_journal/1" : "schemas/fcmaterials_journal-v1.0.json", + "https://eddn.edcd.io/schemas/fcmaterials_journal/1/test" : "schemas/fcmaterials_journal-v1.0.json", + + "https://eddn.edcd.io/schemas/fcmaterials_capi/1" : "schemas/fcmaterials_capi-v1.0.json", + "https://eddn.edcd.io/schemas/fcmaterials_capi/1/test" : "schemas/fcmaterials_capi-v1.0.json", } GATEWAY_OUTDATED_SCHEMAS = [