diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..977ec02 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# This is the project top-level .editorconfig +root = true + +# Defaults for all file types +[*] +# 4-space indents, no TABs +indent_style = space +tab_width = 4 +indent_size = tab + +# Hard-wrap at 120 columns +max_line_length = 119 + +# Unix EOL, single \n +end_of_line = lf + +# UTF-8 is the only sensible option, no BOM +charset = utf-8 + +# All files should have a final newline +insert_final_newline = true diff --git a/README.md b/README.md index 4127a10..6749a49 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ archive or "current state" available to anyone. What it provides is a stream of live data to any interested parties. Some of those then make aggregated data available for general use. +If you want an archive of messages, or a dump of the current known state of +the game galaxy, check ["Archives and data dumps"](#archives-and-data-dumps) +below. + --- --- @@ -73,6 +77,28 @@ intended. a 'Neutron Star' route plotter, but has since expanded into offering many other route plotting tools and general data searching. +##### Archives and data dumps + +Alternatively if you want an archive of past EDDN messages, or a data dump to +use: + +- [edgalaxydata](https://edgalaxydata.space/) has various data captures, + including 'all' (some listener downtime is inevitable) EDDN messages for + many years. + +- [spansh dumps](https://www.spansh.co.uk/dumps) are a "whole galaxy" data set, + of systems, bodies and stations. The full `galaxy.json.gz` is **very** + large, but is currently the only source of an "all known bodies" dump. + Pay attention to the 'Generated' "time ago" column. + +- [EDDB dumps](https://eddb.io/api) represent a snapshot of the data EDDB uses. + NB: There has been no "bodies" data for years now, EDDB itself stopped + updating or adding to this. + +- [EDSM nightly dumps](https://www.edsm.net/en/nightly-dumps) represent a + snapshot of the data EDSM uses. NB: there's only a "last 7 days" bodies + dump as the full data proved too large to dump in a timely manner. + --- There are many other third-party tools for Elite Dangerous, both for @@ -85,7 +111,7 @@ interact with EDDN - check the [EDDN tag](https://edcodex.info/?m=tools&cat=9). ### Developers If you are a developer of a third-party tool that could be enhanced by uploading data to EDDN then please consult -[the live branch documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) +[the live branch Developers' documentation](https://github.com/EDCD/EDDN/blob/live/docs/Developers.md) . **DO NOT** assume that any code or documentation in the `master` (or any other) branch on GitHub truly reflects the current live service! @@ -94,6 +120,41 @@ Anyone planning to send data too EDDN **MUST** comply with all the advice in that document, and the individual schema README files as applicable. It's also the best resource for those listening to the EDDN data stream. +#### EDDN endpoints + +There are up to three separate services which might be running. + +| Service | Upload | Listeners | Notes | +| ------: | :-----: |:-----------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Live | `https://eddn.edcd.io:4430/upload/` | `tcp://eddn.edcd.io:9500/` | The actual live service, which should always be running. It is automatically restarted every Thursday at 07:06:00 UTC | +| Beta | `https://beta.eddn.edcd.io:4431/upload/` | `tcp://beta.eddn.edcd.io:9510/` | The beta service, which should be running the current state of the `beta` branch. Usually only active when either new schemas or forthcoming code changes are being actively tested. | +| Dev | `https://dev.eddn.edcd.io:4432/upload/` | `tcp://dev.eddn.edcd.io:9520/` | The dev service, which could be running any public branch of the code *or* a private branch. | + +In general the Beta and Dev services will only be running so as to aid the core +development team in testing changes. Don't expect them to be generally +available. + +You **MUST** use the correct hostname in the Upload URLs as these are +TLS/HTTPS connections terminated on a Reverse Proxy. + +The Listener URLs are ZeroMQ endpoints, no TLS. But whilst this means you +don't strictly need to use the correct hostname there is no guarantee that the +beta and dev hostnames won't be pointing at, or hosted on, a different IP. + +If you need to test some of your own changes then please read +[Running this software](docs/Running-this-software.md) for how to instantiate +your own test service. It is hoped that in the future the code will allow for +easily running in a "local only" mode, not requiring any reverse proxy or +internet-valid TLS certificates. + +#### New Schemas +*All* new Schema proposals **MUST** be started by opening +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose). Please +consult +[docs/Contributing 'Adding a New Schema'](docs/Contributing.md#adding-a-new-schema) +for further guidelines. + + --- --- diff --git a/contrib/monitor/index.html b/contrib/monitor/index.html index 990386f..9f16a47 100644 --- a/contrib/monitor/index.html +++ b/contrib/monitor/index.html @@ -53,7 +53,7 @@

- Provided by EDSM + Provided by EDCD

diff --git a/contrib/monitor/schemas.html b/contrib/monitor/schemas.html index 34ac82b..4940b3b 100644 --- a/contrib/monitor/schemas.html +++ b/contrib/monitor/schemas.html @@ -41,7 +41,7 @@

- Provided by EDSM + Provided by EDCD

diff --git a/contrib/test-schema.py b/contrib/test-schema.py index ebf0e51..da56adf 100644 --- a/contrib/test-schema.py +++ b/contrib/test-schema.py @@ -7,7 +7,7 @@ from jsonschema import validate as json_validate if len(sys.argv) < 2: print( f""" -Usage: {sys.argv[0]} +Usage: {sys.argv[0]} [] Note that the entire file will be loaded by simpljson.load() and should only contain one JSON object. @@ -16,12 +16,18 @@ only contain one JSON object. sys.exit(-1) schema_file_name = sys.argv[1] -test_file_name = sys.argv[2] +test_file_name = None +if len(sys.argv) == 3: + test_file_name = sys.argv[2] -with open(test_file_name, 'r') as test_file: - test_event = simplejson.load(test_file) +with open(schema_file_name, 'r') as schema_file: + schema = simplejson.load(schema_file) + print('Schema file loaded OK...') - with open(schema_file_name, 'r') as schema_file: - schema = simplejson.load(schema_file) + if test_file_name is not None: + with open(test_file_name, 'r') as test_file: + test_event = simplejson.load(test_file) - json_validate(test_event, schema, format_checker=FormatChecker()) + json_validate(test_event, schema, format_checker=FormatChecker()) + + print('Input file validated against schema OK.') diff --git a/docs/Adding-A-New-Schema.md b/docs/Adding-A-New-Schema.md index fcdcf1d..1759d06 100644 --- a/docs/Adding-A-New-Schema.md +++ b/docs/Adding-A-New-Schema.md @@ -1,11 +1,16 @@ # Adding A New Schema ## Introduction + As of September 2021 it was decided that all new Journal events will be added to their own, new, schemas. This better facilitates defining any values that should be elided, or augmentations added, without twisting schema definitions into knots. +Consult +[Contributing.md#adding-a-new-schema](./Contributing.md#adding-a-new-schema) +for guidelines on the specifics of proposing and designing a new Scema. + In the future we will likely migrate all of the events currently supported in the journal schema into their own schemas, and later still deprecate the journal schema. diff --git a/docs/Contributing.md b/docs/Contributing.md new file mode 100644 index 0000000..72437b8 --- /dev/null +++ b/docs/Contributing.md @@ -0,0 +1,356 @@ +# Contributing to the EDDN Project + +## Introduction + +This document is intended to solidly and usefully define necessary information +pertaining to either improving the EDDN software, or add a new Schema to the +supported set. + +--- + +## File formatting and editor configuration + +The project contains an `.editorconfig` file at its root. Please either ensure +your editor is taking note of those settings, or cross-check its contents +with the +[editorconfig documentation](https://github.com/editorconfig/editorconfig/wiki/EditorConfig-Properties) +, and ensure your editor/IDE's settings match. + +--- + +## Branches and other project miscellanea + +This project utilises a number of Git branches: + +- `live` - The Live service should, outside of brief periods during deployment, + always be running using the software and supporting files from this branch. + This is where you should be looking if checking current behaviour of the + service. + +- `master` - This is in effect the staging area for any changes considered + ready for deployment to the Live service. + +- `beta` - Changes under consideration for Live deployment that are currently + undergoing further testing. If the Beta service is running it should be + using this branch, but there might be exceptions. + There MAY be changes in `develop` *and* `master` that were not merged here. + +- `develop` - Usually any Pull Requests will be merged here first. This is the + default branch against which any new work should be undertaken. Urgent + bug fixes should be the only exception to this, and even then it would only + happen if `develop` already contains non-Live changes that are not yet + considered ready for deployment. + +You might also see 'work in progress' branches with a `fix/` or `enhancement/` +prefix. + +--- +--- + +## Code Changes + +All code changes should start with +[an open Issue on GitHub](https://github.com/EDCD/EDDN/issues?q=is%3Aissue+is%3Aopen). +If no pertinent issue already exists then please create one. + +All Pull Requests should be made against the `develop` branch unless you are +directed to do otherwise. A Pull Request that is opened without prior +discussion in a relevant Issue is liable to be closed without further +consideration, but exceptions may be made for 'obvious' changes. + +### Testing + +As of 2022-01-28 the project still does not contain any automated tests, +neither unit or functional. But you should make every effort to test any +changes, including new Schemas, or changes to existing ones, before opening +a Pull Request for those changes. + +`scripts/testing/` exists and might contain some scripts and supporting files +that will be useful in this. + +### Bug Fixes + +An urgent Bug Fix may be fast-tracked through to the `master` branch, but will +by default go through the `develop` branch. + +Where changes pertain to fixing a bug they should be in a branch named as per +the convention `fix//`, e.g. +`fix/123/avoid-decompress-crash`. + +### Enhancements + +Any changes to existing code or supporting files that does not address a bug +is considered an enhancement. Examples would be: + +- Changes to an existing Schema to better support actual game data and + potential uses by Listeners. If you're not sure whether the change is an + Enhancement or Fix, use your best assessment, we won't bite your head off. +- Adding a wholly new Schema. +- Improving the Monitor web page, be that adding extra output or a new way to + view or manipulate the data presented. + +Where changes pertain to adding wholly new functionality, including adding a +new schema, or improving an existing feature, then they should be in a branch +named as per the convention `enhancement//` +, e.g. `enhancement/234/add-schema-somenewevent`. + +--- +--- + +## Adding a New Schema + +If you think you have a good case for an additional EDDN Schema then there are +several things you should consider: + +1. Ensure you read + [the general Schemas README](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) + to be aware of general requirements that your new Schema will need to + adhere to. + + You might also find useful information in the other Schema-specific README + files. Certainly check those if you encounter trouble documenting a new + Schema. + +2. What is the source of the data? In almost all circumstances the only + acceptable sources are the game Journal files and the Frontier CAPI service. + We do *NOT* accept any manually entered data being sent over EDDN, it's too + prone to error. + + - Do **NOT** blindly trust that the Frontier-provided Journal documentation + is correct with respect to the current game version. Gather actual + examples of Journal output under as varied circumstances as are relevant, + and base your new Schema on what you learn. + + - Remember that there might be differences between data from a player using + a Horizons version of the game versus an Odyssey version. This is why + all Schemas should mandate augmentation with `horizons` and `odyssey` + flags, but there might be other considerations when defining a Schema. + +3. Is the new Schema going to be practically useful ? + 1. What use cases are there for the new data? + 2. Given that nowhere near all players will be running an EDDN + sender, and even where they do we might still miss some relevant data, + is this data still useful ? + + e.g. the owner of a Fleet Carrier sending data about their buy and sell + orders is useful, but if they then don't log in again for a while + there'll be no update to the FC state. Likewise for an FC jumping + between systems. + + At the very least you should consider, and document, caveats about the + data for the benefit of Listeners. + 3. Who benefits ? If the only interested Listener would be a very niche + project, with no benefit to others, then perhaps you should instead + consider e.g. an EDMarket Connector plugin that sends to your own + server ? + 4. What's the likely volume of messages compared to existing Schemas? If + there would often be many messages in a short space of time consider + requiring senders to batch them. 2022-01-28: There's no live example of + this yet, but see + [discussion of adding support for FSSSignalDiscovered](https://github.com/EDCD/EDDN/issues/152) + . + 5. What's the likely size range of the new messages? The Gateway has a + limit on the size of the *body* of `/upload/` requests. Check + [live branch src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py) + `bottle.BaseRequest.MEMFILE_MAX = ...` for the current limit. + +4. For CAPI-sourced data you need to keep in mind possible synchronization + issues between it and any necessary data augmentations from Journal data. + This might mean needing to make such augmentations optional. + +5. For Journal-sourced data if the source is an event not yet allowed by any + existing Schema then you MUST define a wholly new Schema for the data. This + allows you to fully specify both required and forbidden information. + + The Journal events that are handled in the generic `journal` Schema are only + there for historical reasons and due to the difficulties in ensuring all + listeners and senders migrate to separate Schemas in a synchronized manner. + +6. You **MUST** + [open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) + in order to propose the new Schema. If a consensus appears to have been + reached in comments therein then start work on a Pull Request. + +7. Consult + [the template for new Journal-based schemas](../schemas/TEMPLATES/journalevent-v1.0.json) + and [its README template](../schemas/TEMPLATES/journalevent-README.md) for + an outline and some further guidance. + +8. There must be at least one working Sender implementation before the Pull + Request for a new Schema will be merged into the Live service. Experience + has demonstrated that there are often caveats and gotchas discovered during + the development of a Sender for a new Schema. + + Often this will end up being a Pull Request against either + [Elite Dangerous Market Connector](https://github.com/EDCD/EDMarketConnector) 's + EDDN plugin, or [ED Discovery](https://github.com/EDDiscovery/EDDiscovery). + +The Schema files are placed in the `schemas/` directory, located in the root +of the project structure. See [Schema file requirements](#schema-file-requirements) +for more information. + +--- + +### Always start a new Schema at version 1 + +The first time a new Schema goes live it should be as version 1. + - What should policy be on incrementing the version ? I'm not confident + anything other than an integer is supported - Ath + +Any breaking changes **MUST** increment the version number. Use a git file +rename to update the name of the file. Examples of such breaking changes +include: + +- If you add a new required property. Senders will need to update. +- If you remove a required property *and making it optional doesn't make + sense*. Senders will need to update. Listeners will need to cope with the + data no longer being present. +- If you change a property from optional to required or disallowed. Senders + will need to update. Listeners can no longer expect it, if disallowed. + +--- + +### Necessary file edits + +1. Obviously you need to create the new file in the `schemas/` directory. + This should be named as per the data source, i.e. Journal `event` value, and + include the Schema version, and `.json` extension. You **MUST** fold the + the `event` value to lower case for this. + An example is `fssdiscoveryscan-v1.0.json` for adding support for the Journal + `FSSDiscoveryScan` event. + +2. You **MUST** also create the README file for the new Schema. This is also + placed in the `schemas/` directory. The name should match that of the + Schema file itself, without the version, and with a `.md` extention instead + of `.json`. + An example is `fssdiscoveryscan-README.md` documenting the + `fssdiscoveryscan-v1.0.json` Schema file. + +3. You will need to add two lines to `src/eddn/conf/Settings.py` in order to + have the Gateway recognise the new Schema. You are adding to the + end of the `GATEWAY_JSON_SCHEMAS` dictionary. Both the live Schema *and* + the `/test` version **MUST** be added. + For `fssdiscoveryscan-v1.0.json` you would add: + ```python + + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1" : "schemas/fssdiscoveryscan-v1.0.json", + "https://eddn.edcd.io/schemas/fssdiscoveryscan/1/test" : "schemas/fssdiscoveryscan-v1.0.json", + ``` + Please ensure you use the current hostname as per the entries for already + existing Schemas. Keep the trailing comma on the final entry, Python + allows it, and it will reduce the diff on adding any further Schemas. + +4. You MUST add a file containing an example **valid** full EDDN message in the + `scripts/testing/gateway-responses/` directory. + 1. Name the file as per the Schema name. In the case of only adding a + single valid message you can use e.g. `newschema.json`. + 1. If adding variants of valid messages then please annotate the + filename appropriately, e.g. `newschema-valid-inspace.json` and + `newschema-valid-onbody.json`. + 2. If adding variants that are *invalid* in some manner them name + the files as e.g. `newschema-invalid-no-starpos.json`. + 2. The file MUST contain the full plain-text of an EDDN upload, as would be + sent in the body of a request to the `/upload/` endpoint. Test it with + the `scripts/testing/gateway-response/test-sender.py` script. + 3. Please have the `$schemaRef` key:value first, followed by the `header` + dictionary, and only then the `message` dictionary. + 4. Base the `message` part of this on actual source data, e.g. a line + from a relevant Journal file. + 5. Ensure the message `timestamp` value is sane, i.e. from a time period + in which the game was providing this data. + 6. Ensure any data added as an augmentation is correct, i.e. + the proper co-ordinates of the named StarSystem in StarPos. + + This will aid in confirming that the new schema actually works for a valid + message. You MAY add additional examples that are invalid in various ways + that the schema will detect, but this is not required. + +#### Schema file requirements + + +1. The file **MUST** be valid [JSON](https://www.json.org/json-en.html), + without any special extensions (so no comments). Remember that JSON does + **not** allow for a trailing comma on the last entry of an array or + dictionary. +2. The file **MUST** comply with the relevant + [JSON Schema](https://json-schema.org/) definition. + + As of 2022-01-28 we still use 'draft 04' of this specification. + We are looking into updating to the latest in + [#139 - Update to latest JSON schema version(s) ](https://github.com/EDCD/EDDN/issues/139). +3. The file **MUST** load using Python's `simplejson` module, as this + is what the Gateway code uses. The script `contrib/test-schema.py` will + check both this and that the validation code doesn't choke on it. +4. All new Schemas **MUST** comply with all requirements outlined in the + [general Schemas documentation](../schemas/README-EDDN-schemas.md). + If you have a good reason why your new Schema can't and shouldn't comply + with these requirements, then consensus will need to be achieved on changing + those requirements and/or allowing the exception. +5. If the data source is a game Journal event then you **MUST** include the + `event` key and its value as in the source data. This might seem redundant + when we mandate a separate Schema for any newly handled Journal event, but + it does no harm and might make data handling for Listeners easier, i.e. + they can just pass all "oh, that's from Journal data" messages through the + same initial handling. +6. All Schema files MUST be accompanied by a MarkDown formatted + [README file](#schema-readme-requirements). + +#### Data that should be allowed in a new Schema + +The default is to define the new Schema such that all possible data in the +source Journal event is allowed. + +However, you **MUST** consider which source data constitutes information +personal to the player, e.g. their credit balance, or the specific amount they +paid for something after the applicable discounts and mandate its removal in +the Schema. + +Furthermore, the Schema **MUST** mandate removal of all keys (and thus their +values as well) where the key name has the suffix `_Localised`. These +key:value pairs should be accompanied in the source data by a non-Localised +version which will remain, unless otherwise elided. + +You should gather as many examples as possible of actual game Journal events, +and then define the Schema such that anything (after mandated elisions as +above) that is always in the source data is *required* in the new Schema. + +Anything that does not always occur in the source data, and is not defined and +as a mandatory elision, should be defined, but remain optional. + +#### Schema README requirements + +The combination of the Schema file itself and its README **MUST** give both +Senders and Listeners sufficient information to correctly handle the pertinent +data. + +If the Schema signifies a field should be removed then you do not need +to explicitly call it out in the README. + +Likewise, the Schema MUST note, with a `renamed` property, where the key name +for some data differs from that in the Journal source data. Because this is +mandatory you do not then need to list such in the README. + +You do not need to repeat anything already specified in the general +Schema README. Referring to it via MarkDown linking is helpful. + +1. Any augmentations to a message must be clearly explained. + 1. **DO** outline where the additional data comes from. e.g. `StarPos` + added to many events should come from a prior `Location`, `FSDJump` or + `CarrierJump` Journal event. + +2. The reason(s) why any property is optional must be clearly explained. + Perhaps it's not always present in the source data. + +3. The reason(s) why any data in the source is not in the message, i.e. because + it's personal to the player, or maybe it's just not useful (always the same + in every instance of the source data). + +4. If your Schema only works whilst not complying with any main Schema + requirements, and this has been approved, then you need to explicitly + document which requirement(s) are waived and why. + +5. If you use another Schema's README as the basis for yours then you MUST + remove any text that isn't relevant to your Schema. + +--- +--- diff --git a/docs/Developers.md b/docs/Developers.md new file mode 100644 index 0000000..6170e6b --- /dev/null +++ b/docs/Developers.md @@ -0,0 +1,447 @@ +## Introduction + +EDDN is a +[zermoq](https://zeromq.org/) service which allows players of the game +[Elite Dangerous](https://www.elitedangerous.com/), published +by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so +that interested listeners can receive a copy. + +EDDN accepts HTTP POST uploads in +[a defined format](#sending-data) +representing this game data and then passes it on to any interested listeners. + +--- + +## Sources + +There are two sources of game data, both provided by the publisher of the game, +Frontier Developerments. They are both explicitly approved for use by +third-party software. + +### Journal Files + +On the PC version of the game, "Journal files" are written during any game +session. These are in newline-delimited JSON format, with each line +representing a single JSON object. Frontier Developments publishes +documentation for the various events in their +[Player Tools & API Discussions](https://forums.frontier.co.uk/forums/elite-api-and-tools/) +forum. + +In general the documentation is made available in a file named something like: + + Journal_Manual-v + +as both a MicroSoft word `.doc` file, or a `.pdf` file. Historically the +use of `_` versus `-` in those filenames has varied. + +Consult the latest of these for documentation on individual events. +However, be aware that sometimes the documentation is in error, possibly due to +not having been updated after a game client change. + +### Companion API (CAPI) data + +Frontier Developments provides an API to retrieve certain game data, even +without the game running. Historically this was for use by its short-lived +iOS "Companion" app, and was only intended to be used by that app. There was no +public documentation, or even admission of its existence. + +Eventually, after some enterprising players had snooped the connections and +figured out the login method and endpoints, Frontier Developments +[allowed general use of this](https://forums.frontier.co.uk/threads/open-letter-to-frontier-developments.218658/page-19#post-3371472) +. + +Originally the API authentication required being supplied with the email and +password as used to login to the game (but at least this was over HTTPS). + +In late 2018 Frontier switched the authentication to using an oAuth2 flow, +meaning players no longer need to supply their email and password to +third-party sites and clients. + +As of October 2021 there has still never been any official documentation about +the available endpoints and how they work. There is some +[third-party documentation](https://github.com/Athanasius/fd-api/blob/main/docs/README.md) +by Athanasius. + +It is *not* recommended to use CAPI data as the source as it's fraught with +additional issues. EDMarketConnector does so in order to facilitate +obtaining station data without the player needing to, e.g. open the commodities +screen. + +#### Detecting CAPI data lag + +When using the Companion API please be aware that the server that supplies this +data sometimes lags behind the game - usually by a few seconds, sometimes by +minutes. You MUST check in the data from the CAPI that the Cmdr is +docked, and that the station and system names match those +reported from the Journal before using the data for the commodity, outfitting +and shipyard Schemas: + +1. Retrieve the commander data from the `/profile` CAPI endpoint. +2. Check that `commander['docked']` is true. If not, abort. +3. Retrieve the data from the `/market` and `/shipyard` CAPI endpoints. +4. Compare the system and station name from the CAPI market data, + `["lastStarport"]["name"]` and `["lastSystem"]["name"]`, + to that from the last `Docked` or `Location` journal event. If either does + not match then you MUST **abort**. This likely indicates that the CAPI + data is lagging behind the game client state and thus should not be used. + +--- + +## Uploading messages + +### Send only live data to the live Schemas +You MUST **NOT** send information from any non-live (e.g. alpha or beta) +version of the game to the main Schemas on this URL. + +You MAY send such to this URL so long as you append `/test` to the `$schemaRef` +value, e.g. + + "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2/test", + +You MUST also utilise these test forms of the Schemas whenever you are +testing your EDDN-handling code, be that new code or changes to existing code. + +As well as the Live service there are also `beta` and `dev` +[endpoints](../README.md#eddn-endpoints) which might be available from time +to time as necessary, e.g. for testing new Schemas or changes to existing +ones. Ask on the `#eddn` channel of the +[EDCD Discord](https://edcd.github.io/) (check at the bottom for the invite +link). + +Alternatively you could attempt +[running your own test instance of EDDN](./Running-this-software.md). + +### Sending data +Messages sent to EDDN **MUST**: + +- Use the URL: `https://eddn.edcd.io:4430/upload/` + - Note the non-standard port `4430`. + - Yes, the trailing `/` is required. + - Note the use of TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad + Request` response. +- Use the HTTP 1.1 protocol. HTTP/2 is not supported at this time. +- Use a **POST** request, with the body containing the EDDN message. No + query parameters in the URL are supported or necessary. + +The body of an EDDN message is a JSON object in UTF-8 encoding. If you do not +compress this body then you MUST set a `Content-Type` header of +`applicaton/json`. + +For historical reasons URL form-encoded data *is* supported, **but this is +deprecated and no new software should attempt this method**. We +purposefully do not further document the exact format for this. + +You *MAY* use gzip compression on the body of the message, but it is not +required. If you do compress the body then you **MUST* send a `Content-Type` +header of `gzip` instead of `application/json`. + +You should be prepared to handle all scenarios where sending of a message +fails: + +1. Connection refused. +2. Connection timed out. +3. Other possible responses as documented in + [Server responses](#server-responses). + +Carefully consider whether you should queue a 'failed' message for later +retry. In particular, you should ensure that one 'bad' message does not +block other messages from being successfully sent. + +You **MUST** wait some reasonable time (minimum 1 minute) before retrying +any failed message. + +You **MUST NOT** retry any message that received a HTTP `400` or `426` code. +An exception can be made if, **and only if**, *you have manually verified that +you have fixed the issues with it (i.e. updated the Schema/version to a +currently supported one and adjusted the data to fit that Schema/version).* + +You **MAY** retry a message that initially received a `413` response (in +the hopes that the EDDN service admins decided to increase the maximum +allowed request size), but should not do so too quickly or in perpetuity. + +In general: + +- No data is better than bad data. +- *Delayed* good data is better than degrading the EDDN service for others. + +### Format of uploaded messages +Each message is a JSON object in UTF-8 encoding with contents that comply with +the relevant Schema. There is an outline in the general +[EDDN Message README](../schemas/README-EDDN-schemas.md#mandatory-schema-file-contents). + +For example, a shipyard message, version 2, might look like: + +```JSON +{ + "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2", + "header": { + "uploaderID": "Bill", + "softwareName": "My excellent app", + "softwareVersion": "0.0.1" + }, + "message": { + "systemName": "Munfayl", + "stationName": "Samson", + "marketId": 128023552, + "horizons": true, + "timestamp": "2019-01-08T06:39:43Z", + "ships": [ + "anaconda", + "dolphin", + "eagle", + "ferdelance", + "hauler", + "krait_light", + "krait_mkii", + "mamba", + "python", + "sidewinder" + ] + } +} +``` + +### Contents of `message` +Every message MUST comply with the Schema its `$schemaRef` value cites. Each +Schema file should have a matching `-README.md` file in the +[project root schemas/ directory](../schemas/) +. Always consult this so that you're aware of any Schema-specific requirements. + +The Schema file, `-v.json`, is considered the authority on +the format of messages for that Schema. If anything in the accompanying +documentation is in disagreement with this then please +[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +so that we can investigate and correct, or clarify, as necessary. + +Apart from short time windows during deployment of a new version the live +EDDN service should always be using +[the Schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). +So, be sure you're checking the live versions and not, e.g. those in the +`master` or other branches. + +Please consult the +[general README for Schemas](../schemas/README-EDDN-schemas.md#general-eddn-message-outline) +for more detailed information about a message's content. + +EDDN is intended to transport generic data not specific to any particular Cmdr +and to reflect only the data that every player would see in-game in station +services or the local map. To that end: +1. Uploading applications MUST ensure that messages do not contain any + Cmdr-specific data (other than "uploaderID", the "horizons" flag, and + the "odyssey" flag). +2. Uploading applications MUST remove any data where the name of the + relevant key has a `_Localised` suffix. + +The individual Schemas will instruct you on various elisions (removals) to +be made to comply with this. + +Because the first versions of some Schemas were defined when only the CAPI +data was available, before Journal files existed, many of the key names chosen +in the Schemas are based on the equivalent in CAPI data, not Journal events. +This means you MUST rename many of the keys from Journal events to match the +Schemas. Consult the relevant Schema for details. + +Some of these requirements are also enforced by the Schemas, and some things +the Schemas enforce might not be explicitly called out in documentation. So, +**do** check what you're sending against the relevant Schema(s) when making any +changes to your code. + +It is also advisable to Watch this repository +[on GitHub](https://github.io/EDCD/EDDN/) +so that you are aware of any changes to Schemas. + +#### `horizons` and `odyssey` flags + +Where the Schema allows for them, `horizons` and `odyssey` keys SHOULD be +added with appropriate boolean values. `null` is not allowed in the values, +so **if you cannot determine a value do not include that key at all**. + +The only source of these is the `LoadGame` event from journals. It's present +both in the PC local files and the CAPI journal data. If you're +composing a shipyard or outfitting message from CAPI data then it is +possible to synthesise the `horizons` flag. For now consult the function +`capi_is_horizons()` in +[EDMarketConnector:plugins/eddn.py](https://github.com/EDCD/EDMarketConnector/blob/stable/plugins/eddn.py) +for a method to achieve this. + +As of 2022-01-29 the following was observed for the `LoadGame` events as +present in CAPI-sourced Journal files (which were confirmed to match the +PC-local files for these events): + +- PC Odyssey Client, game version `4.0.0.1100`: + ```json + { "timestamp":"2022-01-29T16:17:02Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true, "Odyssey":true,... + ``` +- PC Horizons Client, game version `3.8.0.404`, no `Odyssey` key was + present: + ```json + { "timestamp":"2022-01-29T16:15:07Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":true,... + ``` + +- PC 'base' Client, game version `3.8.0.404`, no `Odyssey` key was + present: + ```json + { "timestamp":"2022-01-29T16:11:54Z", "event":"LoadGame", "FID":"", "Commander":"", "Horizons":false,... + ``` + +Do not attempt to use the value(s) from a `Fileheader` event as the semantics +are different. With clients 3.8.0.404 and 4.0.0.1100 the following was observed: + +| Game Client | Fileheader | LoadGame | +| ---------------: | ---------------- | ------------------------------: | +| Base | "Odyssey":false | "Horizons":false | +| Horizons | "Odyssey":false | "Horizons":true | +| Odyssey | "Odyssey":true | "Horizons":true, "Odyssey":true | + +NB: The 'Base' client appears to simply be the Horizons client with any +Horizons-only features disabled. + +- In the `Fileheader` event it's indicating whether it's an Odyssey game client. +- In the `LoadGame` it's indicating whether Horizons and/or Odyssey features are + active, but in the non-Odyssey game client case you only get the Horizons + boolean. + +### Server responses +There are three possible sources of HTTP responses when sending an upload +to EDDN. + +1. The reverse proxy that initially accepts the request. +2. The python `bottle` module that the Gateway uses to process the + forwarded requests. This might object to a message before the actual + EDDN code gets to process it at all. +3. The actual EDDN Gateway code. + +Once a message has cleared the EDDN Gateway then there is no mechanism for any +further issue (such as a message being detected as a duplicate in the +Monitor downstream of the Gateway) to be reported back to the sender. + +To state the obvious, if there are no issues with a request then an HTTP +200 response will be received by the sender. The body of the response +should be the string `OK`. + +#### Reverse Proxy responses +In addition to generic "you typoed the URL" and other such "you just didn't +make a valid request" responses you might experience the following: + +1. `408` - `Request Timed Out` - the sender took too long to make/complete + its request and the reverse proxy rejected it as a result. +2. `503` - `Service Unavailable` - the EDDN Gateway process is either not + running, or not responding. +3. `400` - `Bad Request` - if you attempt to use plain HTTP, rather than + HTTPS. + +#### bottle responses +1. `413` - `Payload Too Large` - `bottle` enforces a maximum request size + and the request exceeds that. As of 2022-01-07 the limit is 1MiB, + which is versus the compressed size of the body, if compression is + used. Thus compression *will* allow for sending approximately 10x + larger messages. + To verify the current limit check for the line that looks like: + + ``` + bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB + ``` + + in + [src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py), + as added in + [commit 0e80c76cb564771465f61825e694227dcc3be312](https://github.com/EDCD/EDDN/commit/0e80c76cb564771465f61825e694227dcc3be312). + +#### EDDN Gateway responses +For all failures the response body will contain text that begins `FAIL: `. Currently two different HTTP status codes are utilised: + +1. `400` - `Bad Request` - This indicates something wrong with the request + body. Possibly due to a format issue (compression, form encoding), or + the actual content of the EDDN message: + 1. `FAIL: zlib.error: ` - A failure to decompress a message that + claimed to be compressed. + + 2. `FAIL: Malformed Upload: ` - the message appeared to be + form-encoded, but either the format was bad or there was no `data` + key. + + 3. `FAIL: JSON parsing: ` - the + message couldn't be parsed as valid JSON. e.g. + + ``` + FAIL: JSON parsing: Expecting property name enclosed in double quotes: line 1 column 2 (char 1) + ``` + + 4. `FAIL: Schema Validation: ` - the message failed to validate + against the cited Schema. e.g. + + ``` + FAIL: Schema Validation: [] + ``` + + The exact detail will be very much dependent on both the Schema the + message cited and the contents of the message that failed to pass the + validation. + + In particular, if the message contains a key that is tagged 'disallowed' in + the Schema, then the response will look like: + + ``` + FAIL: Schema Validation: "[]" + ``` + This is due to the use of a JSON schema stanza that says "don't allow + any valid type for the value of this key" to trigger the error for such + disallowed keys. + + Note how the **value** for the disallowed key is cited, not the key *name* + itself. This is a limitation of how the `jsonschema` python module + reports errors, and we are + [hoping to improve this](https://github.com/EDCD/EDDN/issues/163). + +2. `426` - `Upgrade Required` - This indicates that the cited Schema, or + version thereof, is outdated. The body of the response will be: + + ``` + FAIL: Oudated Schema: The schema you have used is no longer supported. Please check for an updated version of your application. + ``` + The wording here is aimed at users of applications that send messages + over EDDN. If you're the developer of such an application then + obviously you need to update your code to use a currently supported + Schema and version thereof. + + +There shouldn't be any other variants of a 'FAIL' message. If you find +any then please +[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) +with as much detail as possible so that we can update this documentation. + +## Receiving messages + +EDDN provides a continuous stream of information from uploaders. To use this +data you'll need to connect to the stream using ZeroMQ (a library is probably +available for your language of choice). + +The URL for the live Relay is: + + tcp://eddn.edcd.io:9500 + +Depending on the programming language and library used, you might need +to explicitly subscribe to an empty topic, `''`, in order to receive +anything. + +Unfortunately at this time we're using an old version of +ZeroMQ which does not support server-side subscription filtering. So by +all means specify a more specific topic filter if it suits your needs, +but realise that the dropping of not-matching messages will happen at +your end, not the server. + +Once you've connected to that you will receive messages. To access the +data you will first need to zlib-decompress each message. Then you will +have a textual JSON object as per the Schemas. + +In general, check the guidance for [Uploading messages](#uploading-messages) +for the expected format of the messages. + +Consumers can utilise the `$schemaRef` value to determine which Schema a +particular message is for. There is no need to validate the messages +against the Schemas yourself, as that is performed on the EDDN Gateway. +Messages that do not pass the schema validation there are not forwarded to +receivers. + +There is [example code](https://github.com/EDCD/EDDN/tree/master/examples) +available for a variety of programming languages to help you get started. diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 3de98f6..4620dd1 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -1,171 +1,34 @@ # EDDN Schemas Documentation -## Introduction +EDDN message Schemas are [JSON](https://www.json.org/json-en.html) files +conforming to 'draft 04' of the [JSON Schema](https://json-schema.org/) +specification. -EDDN is a -[zermoq](https://zeromq.org/) service to allow players of the game -[Elite Dangerous](https://www.elitedangerous.com/), published -by [Frontier Developments](https://www.frontier.co.uk/), to upload game data so -that interested listeners can receive a copy. +## Canonical location of Schema files -EDDN accepts HTTP POST uploads in a defined format representing this game data -and then passes it on to any interested listeners. +For the EDDN Live service you should always be checking +[the live version of the schemas, and their READMEs](https://github.com/EDCD/EDDN/tree/live/schemas). +Any other version of the Schemas is not guaranteed to be synchronized with +those actually running on the Live service. ---- -## Sources +## Documentation of Schema files -There are two sources of game data, both provided by the publisher of the game, -Frontier Developerments. They are both explicitly approved for use by -third-party software. +The Schema files themselves are considered to be the canonical definition of +the required, and allowed, contents of the relevant EDDN message. There +**SHOULD** be an accompanying README file, e.g. for `commodity-v3.0.json` there +is also a `commodity-README.md` file in the project root `schemas/` directory. -### Journal Files +For more general documentation that all developers wanting to either Upload +messages or Listen to the stream of messages from the Relay, please consult +[the Developer documentation](../docs/Developers.md). -On the PC version of the game, "Journal files" are written during any game -session. These are in newline-delimited JSON format, with each line -representing a single JSON object. Frontier Developments publishes -documentation for the various events in their -[Player Tools & API Discussions](https://forums.frontier.co.uk/forums/elite-api-and-tools/) -forum. +### Mandatory Schema file contents -In general the documentation is made available in a file named something like: +It is best to base any new Schema file on +[the provided template](./TEMPLATES/journalevent-v1.0.json). As per its +contents all Schemas specify a top-level JSON Object with the data: - Journal_Manual-v - -as both a MicroSoft word `.doc` file, or a `.pdf` file. Historically the -use of `_` versus `-` in those filenames has varied. - -Consult the latest of these for documentation on individual events. -However, be aware that sometimes the documentation is in error, possibly due to -not having been updated after a game client change. - -### Companion API (CAPI) data - -Frontier Developments provides an API to retrieve certain game data, even -without the game running. Historically this was for use by its short-lived -iOS "Companion" app, and was only intended to be used by that app. There was no -public documentation, or even admission of its existence. - -Eventually, after some enterprising players had snooped the connections and -figured out the login method and endpoints, Frontier Developments -[allowed general use of this](https://forums.frontier.co.uk/threads/open-letter-to-frontier-developments.218658/page-19#post-3371472) -. - -Originally the API authentication required being supplied with the email and -password as used to login to the game (but at least this was over HTTPS). - -In late 2018 Frontier switched the authentication to using an oAuth2 flow, -meaning players no longer need to supply their email and password to -third-party sites and clients. - -As of October 2021 there has still never been any official documentation about -the available endpoints and how they work. There is some -[third-party documentation](https://github.com/Athanasius/fd-api/blob/main/docs/README.md) -by Athanasius. - -It is *not* recommended to use CAPI data as the source as it's fraught with -additional issues. EDMarketConnector does so in order to facilitate -obtaining data without the player needing to open the commodities screen. - -#### Detecting CAPI data lag - -When using the Companion API please be aware that the server that supplies this -data sometimes lags behind the game - usually by a few seconds, sometimes by -minutes. You MUST check in the data from the CAPI that the Cmdr is -docked, and that the station and system names match those -reported from the Journal before using the data for the commodity, outfitting -and shipyard schemas: - -1. Retrieve the commander data from the `/profile` CAPI endpoint. -2. Check that `commander['docked']` is true. If not, abort. -3. Retrieve the data from the `/market` and `/shipyard` CAPI endpoints. -4. Compare the system and station name from the CAPI market data, - `["lastStarport"]["name"]` and `["lastSystem"]["name"]`, - to that from the last `Docked` or `Location` journal event. If either does - not match then you MUST **abort**. This likely indicates that the CAPI - data is lagging behind the game client state and thus should not be used. - ---- - -## Uploading messages - -### Send only live data to the live schemas -You MUST **NOT** send information from any non-live (e.g. alpha or beta) -version of the game to the main schemas on this URL. - -You MAY send such to this URL so long as you append `/test` to the `$schemaRef` -value, e.g. - - "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2/test", - -You MUST also utilise these test forms of the schemas when first testing your -code. - -There might also be a beta.eddn.edcd.io, or dev.eddn.edcd.io, service -available from time to time as necessary, e.g. for testing new schemas or -changes to existing ones. Ask on the `#eddn` channel of the EDCD Discord -(see https://edcd.github.io/ for an invite link). - -Alternatively you could attempt -[running your own test instance of EDDN](../docs/Running-this-software.md). - -### Sending data -Messages sent to EDDN **MUST**: - -- Use the URL: `https://eddn.edcd.io:4430/upload/`. Note the use of - TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad - Request` response. -- Use the HTTP 1.1 protocol. HTTP/2 is not supported at this time. -- Use a **POST** request, with the body containing the EDDN message. No - query parameters in the URL are supported or necessary. - -The body of an EDDN message is a JSON object in UTF-8 encoding. You SHOULD -set a `Content-Type` header of `applicaton/json`, and NOT any of: - -* `application/x-www-form-urlencoded` -* `multipart/form-data` -* `text/plain` - -For historical reasons URL form-encoded data *is* supported, **but this is -deprecated and no new software should attempt this method**. We -purposefully do not further document the exact format for this. - -You *MAY* use gzip compression on the body of the message, but it is not -required. - -You should be prepared to handle all scenarios where sending of a message -fails: - -1. Connection refused. -2. Connection timed out. -3. Other possible responses as documented in - [Server responses](#server-responses). - -Carefully consider whether you should queue a 'failed' message for later -retry. In particular, you should ensure that one 'bad' message does not -block other messages from being successfully sent. - -You **MUST** wait some reasonable time (minimum 1 minute) before retrying -any failed message. - -You **MUST NOT** retry any message that received a HTTP `400` or `426` code. -An exception can be made if, **and only if**, *you have manually verified that -you have fixed the issues with it (i.e. updated the schema/version to a -currently supported one and adjusted the data to fit that schema/version).* - -You **MAY** retry a message that initially received a `413` response (in -the hopes that the EDDN service admins decided to increase the maximum -allowed request size), but should not do so too quickly or in perpetuity. - -In general: - -- No data is better than bad data. -- *Delayed* good data is better than degrading the EDDN service for others. - -### Format of uploaded messages -Each message is a JSON object in UTF-8 encoding containing the following -key+value pairs: - -1. `$schemaRef` - Which schema (including version) this message is for. +1. `$schemaRef` - Which Schema (including version) this message is for. 2. `header` - Object containing mandatory information about the upload; 1. `uploaderID` - a unique ID for the player uploading this data. Don't worry about privacy, the EDDN service will hash this with a key @@ -175,54 +38,14 @@ key+value pairs: Listeners MAY make decisions about whether to utilise the data in any message based on the combination of `softwareName` and `softwareVersion`. - + **DO not** add `gatewaytimestamp` yourself. The EDDN Gateway will add this and will overwrite any that you provide, so don't bother. 4. `message` - Object containing the data for this message. Consult the relevant README file within this documentation, e.g. - [codexentry-README.md](./codexentry-README.md). There are some general - guidelines [below](#contents-of-message). + [codexentry-README.md](./codexentry-README.md). -For example, a shipyard message, version 2, might look like: - -```JSON -{ - "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2", - "header": { - "uploaderID": "Bill", - "softwareName": "My excellent app", - "softwareVersion": "0.0.1" - }, - "message": { - "systemName": "Munfayl", - "stationName": "Samson", - "marketId": 128023552, - "horizons": true, - "timestamp": "2019-01-08T06:39:43Z", - "ships": [ - "anaconda", - "dolphin", - "eagle", - "ferdelance", - "hauler", - "krait_light", - "krait_mkii", - "mamba", - "python", - "sidewinder" - ] - } -} -``` - -### Contents of `message` -Every message MUST comply with the schema its `$schemaRef` value cites. - -Apart from short time windows during deployment of a new version the live -EDDN service should always be using -[the schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). -So, be sure you're checking the live versions and not, e.g. those in the -`master` or other branches. +### General EDDN message outline Each `message` object must have, at bare minimum: @@ -231,217 +54,30 @@ Each `message` object must have, at bare minimum: UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are doing this properly. Do not claim 'Z' whilst actually using a local time that is offset from UTC. - - If you are only utilising Journal-sourced data then simply using the - value from there should be sufficient as the PC game client is meant to - always be correctly citing UTC for this. Indeed it has been observed, - in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC - by 21 seconds both the in-game UI clock *and* the Journal event + + If you are only utilising Journal-sourced data then simply using the + value from there should be sufficient as the PC game client is meant to + always be correctly citing UTC for this. Indeed it has been observed, + in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC + by 21 seconds both the in-game UI clock *and* the Journal event timestamps are still properly UTC to the nearest second. Listeners MAY make decisions on accepting data based on this time stamp, i.e. "too old". -2. At least one other key/value pair representing the data. In general there +2. Where the data is sourced from a Journal event please do preserve the + `event` key and value. Yes, where we use an event-specific Schema this + might seem redundant, but it might aid an EDDN listener in streamlining + their code, and it does no harm. + + Any new Schema based on Journal data **MUST** make `event` a required + property of the `message` dictionary. +3. At least one other key/value pair representing the data. In general there will be much more than this. Consult the - [schemas and their documentation](./). + [Schemas and their documentation](./). -Because the first versions of some schemas were defined when only the CAPI -data was available, before Journal files existed, many of the key names chosen -in the schemas are based on the equivalent in CAPI data, not Journal events. -This means ouy MUST rename many of the keys from Journal events to match the -schemas. +4. Please consult the advice pertaining to + [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) and include them + whenever possible. -EDDN is intended to transport generic data not specific to any particular Cmdr -and to reflect only the data that every player would see in-game in station -services or the local map. To that end, uploading applications MUST ensure -that messages do not contain any Cmdr-specific data (other than "uploaderID", -the "horizons" flag, and the "odyssey" flag). - -The individual schemas will instruct you on various elisions (removals) to -be made to comply with this. - -Some of these requirements are also enforced by the schemas, and some things -the schemas enforce might not be explicitly called out here. So, **do** -check what you're sending against the relevant schema(s) when making any -changes to your code. - -It is also advisable to Watch this repository on GitHub so as to be aware -of any changes to schemas. - -#### `horizons` and `odyssey` flags - -Where the schema allows for them, add the `horizons` and `odyssey` -keys with boolean values. `null` is not allowed in the values, so if -you cannot determine a value do not include that key at all. The best -source of these is the `LoadGame` event from journals. It's present -both in the PC local files and the CAPI journal data. If you're -composing a shipyard or outfitting message from CAPI data then it is -possible to synthesise the `horizons` flag. For now consult the function -`capi_is_horizons()` in -[EDMarketConnector:plugins/eddn.py](https://github.com/EDCD/EDMarketConnector/blob/stable/plugins/eddn.py) -for a method to achieve this. - -As of 2022-01-22 the following was observed for the `LoadGame` events as -present in CAPI-sourced Journal files (which were confirmed to match the -PC-local files for these events): - -- PC Odyssey Client, game version `4.0.0.1002`: - ```json - { "timestamp":"2022-01-20T11:15:22Z", "event":"LoadGame", "FID":"F", "Commander":"", "Horizons":true, "Odyssey":true,... - ``` -- PC Horizons Client, game version `3.8.0.403`, no `Odyssey` key was - present: - ```json - { "timestamp":"2022-01-20T11:20:17Z", "event":"LoadGame", "FID":"F", "Commander":"", "Horizons":true,... - ``` - -- PC 'base' Client, game version `3.8.0.403`, no `Odyssey` key was - present: - ```json - { "timestamp":"2022-01-20T11:22:54Z", "event":"LoadGame", "FID":"F", "Commander":"", "Horizons":false,... - ``` - -### Server responses -There are three possible sources of HTTP responses when sending an upload -to EDDN. - -1. The reverse proxy that initially accepts the request. -2. The python `bottle` module that the Gateway uses to process the - forwarded requests. This might object to a message before the actual - EDDN code gets to process it at all. -3. The actual EDDN Gateway code. - -Once a message has cleared the EDDN Gateway then there is no mechanism for any -further issue (such as a message being detected as a duplicate in the -Monitor downstream of the Gateway) to be reported back to the sender. - -To state the obvious, if there are no issues with a request then an HTTP -200 response will be received by the sender. The body of the response -should be the string `OK`. - -#### Reverse Proxy responses -In addition to generic "you typoed the URL" and other such "you just didn't -make a valid request" responses you might experience the following: - -1. `408` - `Request Timed Out` - the sender took too long to make/complete - its request and the reverse proxy rejected it as a result. -2. `503` - `Service Unavailable` - the EDDN Gateway process is either not - running, or not responding. -3. `400` - `Bad Request` - if you attempt to use plain HTTP, rather than - HTTPS. - -#### bottle responses -1. `413` - `Payload Too Large` - `bottle` enforces a maximum request size - and the request exceeds that. As of 2022-01-07 the limit is 1MiB, - which is versus the compressed size of the body, if compression is - used. Thus compression *will* allow for sending approximately 10x - larger messages. - To verify the current limit check for the line that looks like: - - ``` - bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB - ``` - - in - [src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/live/src/eddn/Gateway.py), - as added in - [commit 0e80c76cb564771465f61825e694227dcc3be312](https://github.com/EDCD/EDDN/commit/0e80c76cb564771465f61825e694227dcc3be312). - -#### EDDN Gateway responses -For all failures the response body will contain text that begins `FAIL: `. Currently two different HTTP status codes are utilised: - -1. `400` - `Bad Request` - This indicates something wrong with the request - body. Possibly due to a format issue (compression, form encoding), or - the actual content of the EDDN message: - 1. `FAIL: zlib.error: ` - A failure to decompress a message that - claimed to be compressed. - - 2. `FAIL: Malformed Upload: ` - the message appeared to be - form-encoded, but either the format was bad or there was no `data` - key. - - 3. `FAIL: JSON parsing: ` - the - message couldn't be parsed as valid JSON. e.g. - - ``` - FAIL: JSON parsing: Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - ``` - - 4. `FAIL: Schema Validation: ` - the message failed to validate - against the cited schema. e.g. - - ``` - FAIL: Schema Validation: [] - ``` - - The exact detail will be very much dependent on both the schema the - message cited and the contents of the message that failed to pass the - validation. - - In particular, if the message contains a key that is tagged 'disallowed' in - the schema the response will look like: - - ``` - FAIL: Schema Validation: "[]" - ``` - This is due to the use of a JSON schema stanza that says "don't allow - any valid type for the value of this key" to trigger the error for such - disallowed keys. - - Note how the **value** for the disallowed key is cited, not the key *name* - itself. This is a limitation of how the `jsonschema` python module - reports errors, and we are - [hoping to improve this](https://github.com/EDCD/EDDN/issues/163). - -2. `426` - `Upgrade Required` - This indicates that the cited schema, or - version thereof, is outdated. The body of the response will be: - - ``` - FAIL: Oudated Schema: The schema you have used is no longer supported. Please check for an updated version of your application. - ``` - The wording here is aimed at users of applications that send messages - over EDDN. If you're the developer of such an application then - obviously you need to update your code to use a currently supported - schema and version thereof. - - -There shouldn't be any other variants of a 'FAIL' message. If you find -any then please -[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) -with as much detail as possible so that we can update this documentation. - -## Receiving messages - -EDDN provides a continuous stream of information from uploaders. To use this -data you'll need to connect to the stream using ZeroMQ (a library is probably -available for your language of choice). - -The URL for the live Relay is: - - tcp://eddn.edcd.io:9500 - -Depending on the programming language and library used, you might need -to explicitly subscribe to an empty topic, `''`, in order to receive -anything. - -Unfortunately at this time we're using an old version of -ZeroMQ which does not support server-side subscription filtering. So by -all means specify a more specific topic filter if it suits your needs, -but realise that the dropping of not-matching messages will happen at -your end, not the server. - -Once you've connected to that you will receive messages. To access the -data you will first need to zlib-decompress each message. Then you will -have a textual JSON object as per the schemas. - -In general, check the guidance for [Uploading messages](#uploading-messages) -for the expected format of the messages. - -Consumers can utilise the `$schemaRef` value to determine which schema a -particular message is for. There is no need to validate the messages -against the schemas yourself, as that is performed on the EDDN Gateway. -Messages that do not pass the schema validation there are not forwarded to -receivers. - -There is [example code](https://github.com/EDCD/EDDN/tree/master/examples) -available for a variety of programming languages to help you get started. +Where a key has to be renamed this will be specified in the Schema through a +`renamed` property on the object in question. diff --git a/schemas/TEMPLATES/journalevent-README.md b/schemas/TEMPLATES/journalevent-README.md new file mode 100644 index 0000000..f85fc86 --- /dev/null +++ b/schemas/TEMPLATES/journalevent-README.md @@ -0,0 +1,115 @@ +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +!!! STOP !!! + +!! These are not valid MarkDown comments. + +!! You MUST read, comply with, and then remove all of these lines to have a + +!! valid Schema README file. + +1. Be careful if using any existing Schema, or its README, as reference for a + new Schema. Some of them will have some departures from best practices for + historical reasons. It's usually not easy to co-ordinate all Listeners and + Senders smoothly updating to a new version of a Schema, so less than ideal + requires/optional/disallowed definitions might be present in older Schemas. + + [The main Schema documentation](../README-EDDN-schemas.md) is the canonical + authority. +2. Replace all instances of `NewJournalEvent` with the name of the actual + Journal event's Schema you are documenting. This should have the case + preserved as per how it appears in actual game Journal files. +3. Replace all instances of `newjournalevent` with the lower-case folded + version of this Schema's Journal event name. +4. For new Journal-based schemas no key renames should be necessary. + + If there are no renames of key names for this Schema, then remove the + `Key Renames` section. + + Where such renames *are* required do **NOT** attempt to list them all here. + That would just require updating them both here and in the actual Schema. + + If there are any, then the affected property object should contain a key + named `renamed` with its value being the original key name in the source + data, e.g. in the commodity/3 schema a Journal `StarSystem` is renamed + to `systemName` so we have: + + ```json + "message": { + "type" : "object", + "additionalProperties" : false, + "required" : [ "systemName", "stationName", "marketId", "timestamp", "commodities" ], + "properties" : { + "systemName": { + "type" : "string", + "renamed" : "StarSystem", + "minLength" : 1 + }, + + ``` + +5. Do **NOT** remove the `horizons and odyssey flags` section. It is + mandatory that they are allowed (but are optional) in any Journal-based + EDDN Schema. + +6. If both: + 1. either the source Journal event contains information that includes the + System name (possibly as `StarSystem` or `SystemName`), **OR** the source + data contains a `SystemAddress` value, + 2. and a `StarPos` array is *not already present* in the source data. + + then you MUST include the `StarPos` section in `Augmentations` and add + `StarPos` to the `required` message properties in the Schema file. + + If neither key is in the source data then remove the `StarPos` section from + this document and the Schema file. + +7. Do **NOT** add an 'Elisions'/'Removals' section. Leave the Schema as the + sole reference for any data that is in the source but should not be in the + final EDDN message. + +The line: + + # EDDN NewJournalEvent Schema + +below should ultimately be the first line in this file, after required edits. + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# EDDN NewJournalEvent Schema + +## Introduction +Here we document how to take data from an ED `NewJournalEvent` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The data source for this schema is the ED Journal event `NewJournalEvent`. + +### Key Renames +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. + +## Listeners +The advice above for [Senders](#senders), combined with the actual Schema file +*should* provide all the information you need to process these events. diff --git a/schemas/TEMPLATES/journalevent-v1.0.json b/schemas/TEMPLATES/journalevent-v1.0.json new file mode 100644 index 0000000..a87c8fe --- /dev/null +++ b/schemas/TEMPLATES/journalevent-v1.0.json @@ -0,0 +1,113 @@ +# This is not a valid JSON file +# Removed all the 'comments' as you are sure you have complied with their +# instructions! +# +# 1. Replace all instances of newjournalevent with the lower-case folded name +# of the Journal event this schema is for. +# 2. Replace all instances of NewJournalEvent with the name of the journal +# event this schema is for, as it appears in the Journal. Specifically, +# you must conserve the case. +# 3. Leave the 'header' dictionary exactly as it is, it is mandatory. +# 4. Add any additional schema-mandatory message properties to the 'required' +# array. +# 5. If, and ONLY IF, you have good reason to believe there might be additional +# valid keys in the source data, change the 'additionalProperties' value +# to 'true'. +# EMPHASIS: You should really know about all of the possible keys and their +# values and be defining them, possibly as optional (not listed in +# 'required') properties. If needs be document, in the README for +# this Schema, that all but the defined properties should be +# elided when constructing a message. +# 6. The 'horizons' and 'odyssey' properties in 'message' MUST BE RETAINED. +# Any message based on Journal data should be able to add them as +# appropriate. Note that they *are optional in the resulting message*. +# You MUST NOT add these to the 'required' list, as the LoadGame Journal +# event is not guaranteed to contain either, depending on which client +# (base, horizons, odyssey) the player is running. +# 7. 'some_disallowed_key' demonstrates how to specify that a key (and thus its +# value) is not allowed in this schema. Note the **MANDATORY** description. +# 8. Note the trailing comma after the 'some_disallowed_key' entry in +# 'messages'. If all you did was remove these comments you would NOT have +# a valid JSON file. You should be adding **at least** one additional +# pertinent 'message' key:value definition. +# 9. Consult the journalevent-README.md file, particularly the section about +# `StarPos` and remove that section of this file if it is not needed. +# 10. You MUST NOT remove the 'disallowed' key, or edit its value. The purpose +# of this is to be cited as the allowed value on a disallowed key. As it +# defines that the only valid type for the value is **not** any of the +# possible JSON types, its effect is to disallow any value for the key, and +# as a result disallow the key. +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/newjournalevent/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "NewJournalEvent" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." + }, + "odyssey": { + "type" : "boolean", + "description" : "Boolean value copied from the Journal LoadGame event, when it is present there." + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender" + }, + "ExampleRenamedKey" : { + "type" : "string", + "renamed" : "SomeOtherKey" + } + "some_disallowed_key" { + "$ref" : "#/definitions/disallowed", + "description" : "MANDATORY brief description of why this key must be removed from source data" + }, + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} diff --git a/schemas/approachsettlement-README.md b/schemas/approachsettlement-README.md new file mode 100644 index 0000000..9dfe6d1 --- /dev/null +++ b/schemas/approachsettlement-README.md @@ -0,0 +1,35 @@ +# EDDN ApproachSettlement Schema + +## Introduction +Here we document how to take data from an ED `ApproachSettlement` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The primary data source for this schema is the ED Journal event +`ApproachSettlement`. + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. + +#### StarSystem + +You MUST add a StarSystem key/value pair representing the name of the system +this event occurred in. Source this from either Location, FSDJump or +CarrierJump as appropriate. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/approachsettlement-v1.0.json b/schemas/approachsettlement-v1.0.json new file mode 100644 index 0000000..571db77 --- /dev/null +++ b/schemas/approachsettlement-v1.0.json @@ -0,0 +1,93 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/approachsettlement/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event", "StarSystem", "StarPos", "SystemAddress", "Name", "MarketID", "BodyID", "BodyName", "Latitude", "Longitude" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "ApproachSettlement" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "StarSystem": { + "type" : "string", + "minLength" : 1, + "description" : "Must be added by the sender" + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender" + }, + "SystemAddress": { + "type" : "integer" + }, + "Name" : { + "type" : "string", + "description" : "Name of settlement" + }, + "MarketID": { + "type" : "integer" + }, + "BodyID": { + "type" : "integer" + }, + "BodyName": { + "type" : "string" + }, + "Latitude": { + "type" : "number" + }, + "Longitude": { + "type" : "number" + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} diff --git a/schemas/blackmarket-README.md b/schemas/blackmarket-README.md index 0054bd1..c5a8eda 100644 --- a/schemas/blackmarket-README.md +++ b/schemas/blackmarket-README.md @@ -20,17 +20,9 @@ documentation for a schema such as this. The primary data source for this schema is the ED Journal event `MarketSell`. ### Key Renames -#### name -Due to how the EDDN schema is defined the `Type` key/value should -have the key renamed to `name`. - -#### prohibited -Due to how the EDDN schema is defined the `IllegalGoods` key/value should -have the key renamed to `prohibited`. - -#### marketID -The Journal documentation says this is `MarketID`, but in the schema the -`m` is lower case. +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. ### Elisions You MUST remove the following key/value pairs from the data: @@ -41,15 +33,13 @@ You MUST remove the following key/value pairs from the data: - `BlackMarket` - Because we're using this schema, so this is un-necessary. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### systemName The star system name for where this market is. Use the `StarSystem` value from the prior `Docked` or `Location` event. #### stationName -From the `StationName` value on the prior `Docked` or `Location` event. \ No newline at end of file +From the `StationName` value on the prior `Docked` or `Location` event. diff --git a/schemas/blackmarket-v1.0.json b/schemas/blackmarket-v1.0.json index 362916a..15cf3ed 100644 --- a/schemas/blackmarket-v1.0.json +++ b/schemas/blackmarket-v1.0.json @@ -44,7 +44,8 @@ "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "timestamp": { "type" : "string", @@ -52,6 +53,7 @@ }, "name": { "type" : "string", + "renamed" : "Type", "minLength" : 1, "description" : "Commodity name as returned by the MarketSell entry in the Journal" }, @@ -61,6 +63,7 @@ }, "prohibited": { "type" : "boolean", + "renamed" : "IllegalGoods", "description" : "Whether the commodity is prohibited at this station" } } diff --git a/schemas/codexentry-README.md b/schemas/codexentry-README.md index 58da99e..1c675d3 100644 --- a/schemas/codexentry-README.md +++ b/schemas/codexentry-README.md @@ -7,40 +7,33 @@ properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `CodexEntry`. -### Elisions -You MUST remove any key where the key name ends in -`_Localised`. - -You MUST remove the two keys `IsNewEntry` and `NewTraitsDiscovered`. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarPos -You MUST **add** a `StarPos` key with value of type `array` containing the -galaxy co-ordinates of the system. You will need to have obtained these -from prior event(s) upon the player arriving, or logging into, the system. - -e.g. if the system is `Alpha Centauri`: -```json - "StarPos": [3.03125, -0.09375, 3.15625] -``` +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. #### BodyID and BodyName You SHOULD attempt to track the BodyName and BodyID where the player is and add keys/values for these. You MUST track `BodyName` both from Status.json *and* also from some -[Journal](./README-EDDN-schemas.md#journal-files) -events in order to cross-check it before using the `BodyID` from -[Journal](./README-EDDN-schemas.md#journal-files) events. +Journal events in order to cross-check it before using the `BodyID` from +Journal events. The following is correct as of game version 4.0.0.801 (Odyssey initial release, Update 7, plus one patch). @@ -103,7 +96,7 @@ e.g. for `Bestia A 2 a` If you cannot properly obtain the values for `BodyName` or `BodyID` then you MUST NOT include them. -## Receivers +## Listeners As per ['BodyID and BodyName'](#bodyid-and-bodyname) above be aware that you are not guaranteed to receive these values for any given event. Some diff --git a/schemas/commodity-README.md b/schemas/commodity-README.md index c2a4516..3e155b6 100644 --- a/schemas/commodity-README.md +++ b/schemas/commodity-README.md @@ -7,6 +7,14 @@ properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `Market`, and the additional file, `Market.json`, that it signals the writing of. @@ -24,18 +32,12 @@ See [Using CAPI data](#using-capi-data) below. So, as per the schema, do include it if available. ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -#### StarSystem to systemName -Rename the `StarSystem` key name to `systemName`. +Some key names in this Schema are different from how they appear in source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. The names +used are as found in the CAPI source data. ### Elisions -#### Remove _Localised key/values -All keys whose name ends with `_Localised`, i.e. the `Name_Localised` -key/values in Items. - -#### Other Elisions You MUST remove the following key/value pairs from the data: - `StationType` key/value. @@ -50,15 +52,13 @@ In the list of commodites: string (not normally traded at this station market). #### Item Category -Remove not only the `Category_Localised` key/value, as above, but also the -`Category` key/value pair from each Item. +Remove not only the `Category_Localised` key:values, but also the +`Category` key:value pair from each Item. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. ### Using CAPI data It is *not* recommended to use CAPI data as the source as it's fraught with @@ -66,10 +66,10 @@ additional issues. EDMarketConnector does so in order to facilitate obtaining data without the player needing to open the commodities screen. Please read -[the guidance on checking for CAPI lag](README-EDDN-schemas.md#detecting-capi-data-lag) +[the guidance on checking for CAPI lag](../docs/Developers.md#detecting-capi-data-lag) before utilising CAPI data for EDDN messages. -Note that CAPI `/market` data will sometimes have the `StatusFlasg` per +Note that CAPI `/market` data will sometimes have the `statusFlasg` per item, which are defined as optional in this schema (because they're not in the Market.json data). You SHOULD include this data in your message if using CAPI as the source. @@ -91,4 +91,4 @@ any of the listed ships or modules have a `sku` value of #### CAPI odyssey flag Unfortunately there is no method to be *certain* of this from CAPI data, so you will have to trust in the system/station name check and use the value -from the Journal `LoadGame` event. \ No newline at end of file +from the Journal `LoadGame` event. diff --git a/schemas/commodity-v3.0.json b/schemas/commodity-v3.0.json index 3d4450b..60c5c78 100644 --- a/schemas/commodity-v3.0.json +++ b/schemas/commodity-v3.0.json @@ -40,10 +40,12 @@ }, "stationName": { "type" : "string", + "renamed" : "StarSystem", "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "horizons": { "type" : "boolean", @@ -67,31 +69,39 @@ "properties" : { "name": { "type" : "string", + "renamed" : "Name", "minLength" : 1, "description" : "Symbolic name as returned by the Companion API" }, "meanPrice": { - "type" : "integer" + "type" : "integer", + "renamed" : "MeanPrice" }, "buyPrice": { "type" : "integer", + "renaamed" : "BuyPrice", "description" : "Price to buy from the market" }, "stock": { - "type" : "integer" + "type" : "integer", + "renamed" : "Stock" }, "stockBracket": { - "$ref" : "#/definitions/levelType" + "$ref" : "#/definitions/levelType", + "renamed" : "StockBracket" }, "sellPrice": { "type" : "integer", + "renamed" : "SellPrice", "description" : "Price to sell to the market" }, "demand": { - "type" : "integer" + "type" : "integer", + "renamed" : "Demand" }, "demandBracket": { - "$ref" : "#/definitions/levelType" + "$ref" : "#/definitions/levelType", + "renamed" : "DemandBracket" }, "statusFlags": { "type" : "array", @@ -101,6 +111,18 @@ "type" : "string", "minLength" : 1 } + }, + "Producer": { + "$ref" : "#/definitions/disallowed", + "description" : "Not present in CAPI data, so removed from Journal-sourced data" + }, + "Rare" : { + "$ref" : "#/definitions/disallowed", + "description" : "Not present in CAPI data, so removed from Journal-sourced data" + }, + "id": { + "$ref" : "#/definitions/disallowed", + "description" : "Not wanted for historical reasons?" } } } @@ -130,11 +152,16 @@ "type" : "string", "minLength" : 1 } + }, + "StationType": { + "$ref" : "#/definitions/disallowed", + "description" : "Not present in CAPI data, so removed from Journal-sourced data" } } } }, "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } }, "levelType": { "enum" : [0, 1, 2, 3, ""], "description" : "Note: A value of \"\" indicates that the commodity is not normally sold/purchased at this station, but is currently temporarily for sale/purchase" diff --git a/schemas/fssallbodiesfound-README.md b/schemas/fssallbodiesfound-README.md new file mode 100644 index 0000000..350a318 --- /dev/null +++ b/schemas/fssallbodiesfound-README.md @@ -0,0 +1,29 @@ +# EDDN FSSAllBodiesFound Schema + +## Introduction +Here we document how to take data from an ED `FSSAllBodiesFound` Journal +Event and properly structure it for sending to EDDN. + +Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general +documentation for a schema such as this. + +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + +## Senders +The primary data source for this schema is the ED Journal event +`FSSAllBodiesFound`. + +### Augmentations +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. + +#### StarPos +You MUST add a `StarPos` array containing the system co-ordinates from the +last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/fssallbodiesfound-v1.0.json b/schemas/fssallbodiesfound-v1.0.json new file mode 100644 index 0000000..61ec184 --- /dev/null +++ b/schemas/fssallbodiesfound-v1.0.json @@ -0,0 +1,77 @@ +{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "https://eddn.edcd.io/schemas/fssallbodiesfound/1#", + "type" : "object", + "additionalProperties" : false, + "required": [ "$schemaRef", "header", "message" ], + "properties": { + "$schemaRef": { + "type" : "string" + }, + "header": { + "type" : "object", + "additionalProperties" : true, + "required" : [ "uploaderID", "softwareName", "softwareVersion" ], + "properties" : { + "uploaderID": { + "type" : "string" + }, + "softwareName": { + "type" : "string" + }, + "softwareVersion": { + "type" : "string" + }, + "gatewayTimestamp": { + "type" : "string", + "format" : "date-time", + "description" : "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property." + } + } + }, + "message": { + "type" : "object", + "description" : "Contains all properties from the listed events in the client's journal, minus the Localised strings and the properties marked below as 'disallowed'", + "additionalProperties" : false, + "required" : [ "timestamp", "event", "SystemName", "StarPos", "SystemAddress", "Count" ], + "properties" : { + "timestamp": { + "type" : "string", + "format" : "date-time" + }, + "event" : { + "enum" : [ "FSSAllBodiesFound" ] + }, + "horizons": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has a Horizons pass." + }, + "odyssey": { + "type" : "boolean", + "description" : "Whether the sending Cmdr has an Odyssey expansion." + }, + "SystemName": { + "type" : "string", + "minLength" : 1 + }, + "StarPos": { + "type" : "array", + "items" : { "type": "number" }, + "minItems" : 3, + "maxItems" : 3, + "description" : "Must be added by the sender if not present in the journal event" + }, + "SystemAddress": { + "type" : "integer" + }, + "Count" : { + "type" : "integer", + "description" : "Number of bodies in this system" + } + } + } + }, + "definitions": { + "disallowed" : { "not" : { "type": [ "array", "boolean", "integer", "number", "null", "object", "string" ] } } + } +} diff --git a/schemas/fssdiscoveryscan-README.md b/schemas/fssdiscoveryscan-README.md index 6d4d844..ba5a5eb 100644 --- a/schemas/fssdiscoveryscan-README.md +++ b/schemas/fssdiscoveryscan-README.md @@ -7,26 +7,23 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `FSSDiscoveryScan`. -### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -You MUST remove the following key/value pairs from the data: - - - `Progress` key/value pair. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the -last `FSDJump`, `CarrierJump`, or `Location` event. \ No newline at end of file +last `FSDJump`, `CarrierJump`, or `Location` event. diff --git a/schemas/fssdiscoveryscan-v1.0.json b/schemas/fssdiscoveryscan-v1.0.json index 0c3e7ad..9751db1 100644 --- a/schemas/fssdiscoveryscan-v1.0.json +++ b/schemas/fssdiscoveryscan-v1.0.json @@ -62,8 +62,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" + "type" : "integer" }, "Progress" : { "$ref" : "#/definitions/disallowed", diff --git a/schemas/navbeaconscan-README.md b/schemas/navbeaconscan-README.md index c12e91b..39167a2 100644 --- a/schemas/navbeaconscan-README.md +++ b/schemas/navbeaconscan-README.md @@ -7,19 +7,22 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `NavBeaconScan`. -### Elisions -There are no elisions in this schema. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarSystem You MUST add a `StarSystem` key/value pair representing the name of the diff --git a/schemas/navbeaconscan-v1.0.json b/schemas/navbeaconscan-v1.0.json index 64d55bf..b3fdc94 100644 --- a/schemas/navbeaconscan-v1.0.json +++ b/schemas/navbeaconscan-v1.0.json @@ -52,7 +52,8 @@ }, "StarSystem": { "type" : "string", - "minLength" : 1 + "minLength" : 1, + "description" : "Should be added by the sender if not present in the journal event" }, "StarPos": { "type" : "array", @@ -62,8 +63,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" + "type" : "integer" }, "NumBodies" : { "type" : "integer" diff --git a/schemas/navroute-README.md b/schemas/navroute-README.md index 3025baa..703c794 100644 --- a/schemas/navroute-README.md +++ b/schemas/navroute-README.md @@ -7,6 +7,14 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the `NavRoute.json` file. That it has been freshly written is signalled by the ED Journal event `NavRoute`. @@ -18,12 +26,8 @@ data you got from reading this file, not merely the Journal event. The primary data to be sent is the `Route` array from the contents of the separate file. -### Elisions -There are no elisions in this schema. - ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. \ No newline at end of file diff --git a/schemas/outfitting-README.md b/schemas/outfitting-README.md index 090fc61..e3e18bf 100644 --- a/schemas/outfitting-README.md +++ b/schemas/outfitting-README.md @@ -7,20 +7,35 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `Outfitting`. You MAY also source this data from the CAPI `/shipyard` endpoint. Please read -[the guidance on checking for CAPI lag](README-EDDN-schemas.md#detecting-capi-data-lag) +[the guidance on checking for CAPI lag](../docs/Developers.md#detecting-capi-data-lag) before utilising CAPI data for EDDN messages. You only need the `name` key's value for each member of the `modules` array. ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. + +### The modules/Items list +The source data, Journal or CAPI, contains more than just the names of the +available items. This Schema is only concerned with the names, so the list +you build will have only strings as its members, not including other information +such as id, category, cost/BuyPrice, sku or stock. ### Elisions Remove items whose availability depends on the Cmdr's status rather than on the @@ -35,8 +50,6 @@ station. Namely: - The `"Int_PlanetApproachSuite"` module (for historical reasons). ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. \ No newline at end of file +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. diff --git a/schemas/outfitting-v2.0.json b/schemas/outfitting-v2.0.json index 3a8e49d..a8959e0 100644 --- a/schemas/outfitting-v2.0.json +++ b/schemas/outfitting-v2.0.json @@ -36,14 +36,17 @@ "properties" : { "systemName": { "type" : "string", + "renamed" : "StarSystem", "minLength" : 1 }, "stationName": { "type" : "string", + "renamed" : "StationName", "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "horizons": { "type" : "boolean", @@ -59,6 +62,7 @@ }, "modules": { "type" : "array", + "renamed" : "Items", "minItems" : 1, "uniqueItems" : true, "items" : { diff --git a/schemas/scanbarycentre-README.md b/schemas/scanbarycentre-README.md index da0e178..d7b5f01 100644 --- a/schemas/scanbarycentre-README.md +++ b/schemas/scanbarycentre-README.md @@ -7,19 +7,25 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `ScanBaryCentre`. -### Elisions -There are no elisions in this schema. +Although most of the event-specific data is not specified as `required`, +senders SHOULD include any defined in the schema if it's in the source data. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. #### StarPos You MUST add a `StarPos` array containing the system co-ordinates from the diff --git a/schemas/scanbarycentre-v1.0.json b/schemas/scanbarycentre-v1.0.json index 15a8985..e019326 100644 --- a/schemas/scanbarycentre-v1.0.json +++ b/schemas/scanbarycentre-v1.0.json @@ -62,8 +62,7 @@ "description" : "Must be added by the sender if not present in the journal event" }, "SystemAddress": { - "type" : "integer", - "description" : "Should be added by the sender if not present in the journal event" + "type" : "integer" }, "BodyID": { "type" : "integer" diff --git a/schemas/shipyard-README.md b/schemas/shipyard-README.md index 33016a4..b1f32d9 100644 --- a/schemas/shipyard-README.md +++ b/schemas/shipyard-README.md @@ -7,34 +7,36 @@ Event and properly structure it for sending to EDDN. Please consult [EDDN Schemas README](./README-EDDN-schemas.md) for general documentation for a schema such as this. +If you find any discrepancies between what this document says and what is +defined in the relevant Schema file, then you should, in the first instance, +assume that it is the Schema file that is correct. +**PLEASE open +[an issue on GitHub](https://github.com/EDCD/EDDN/issues/new/choose) +to report any such anomalies you find so that we can check and resolve the +discrepancy.** + ## Senders The primary data source for this schema is the ED Journal event `Shipyard`. You MAY also source this data from the CAPI `/shipyard` endpoint. Please read -[the guidance on checking for CAPI lag](README-EDDN-schemas.md#detecting-capi-data-lag) +[the guidance on checking for CAPI lag](../docs/Developers.md#detecting-capi-data-lag) before utilising CAPI data for EDDN messages. - You only need the `name` key's value for each member of the `PriceList` -array (if using Journal, it will be from the `ships` array if using CAPI -data). +The `ships` array is built from *only* the `name` values of either the Journal +`PriceList` array in the `Shipyard.json` file, or from the `ships` array of +CAPI `/shipyard` data. When using CAPI data *include* ships listed in the `"unavailable_list"` property (i.e. available at this station, but not to this Cmdr). -This list of ship names will go in the `ships` array in the EDDN message. - ### Key Renames -Many of the key names have a different case defined in this schema, make -sure you are renaming them as appropriate. - -### Elisions -There are no elisions in this schema. +Some key names in this Schema are different from how they appear in the source +Journal data. Look for keys where the object contains a `renamed` key - the +value is what the name would have been in the source Journal data. ### Augmentations -#### horizons flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. - -#### odyssey flag -You SHOULD add this key/value pair, using the value from the `LoadGame` event. +#### horizons and odyssey flags +Please read [horizons and odyssey flags](../docs/Developers.md#horizons-and-odyssey-flags) +in the Developers' documentation. diff --git a/schemas/shipyard-v2.0.json b/schemas/shipyard-v2.0.json index 9de2cc8..bb4fabd 100644 --- a/schemas/shipyard-v2.0.json +++ b/schemas/shipyard-v2.0.json @@ -36,14 +36,17 @@ "properties" : { "systemName": { "type" : "string", + "renamed" : "StarSystem", "minLength" : 1 }, "stationName": { "type" : "string", + "renamed" : "StationName", "minLength" : 1 }, "marketId": { - "type" : "integer" + "type" : "integer", + "renamed" : "MarketID" }, "horizons": { "type" : "boolean", @@ -63,6 +66,7 @@ }, "ships": { "type" : "array", + "renamed" : "PriceList", "minItems" : 1, "uniqueItems" : true, "items" : { diff --git a/scripts/apache-log-rate b/scripts/apache-log-rate new file mode 100755 index 0000000..5e63280 --- /dev/null +++ b/scripts/apache-log-rate @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# vim: wrapmargin=0 textwidth=0 smarttab expandtab tabstop=2 shiftwidth=2 +"""Process Apache access.log lines to find highest rate of /upload/.""" + +import argparse +from collections import deque +import datetime +import dateutil.parser +import fileinput +import re + + +def process_log_file( + input_file: str ='-', + request_text: str = '/upload/', + window_size: int = 1, +) -> None: + """ + Process the indicated log file to determine peak rate of interesting lines. + + :param input_file: Name of input file, `-` for stdin + :param request_text: The text that denotes an interesting line + :param window_size: Time, in seconds, for the window to assess + """ + print(f'With:\n\tinput_file: "{input_file}"\n\trequest_text: "{request_text}"') + with fileinput.FileInput(files=(input_file)) as f: + apache_re = re.compile(r'^(?P[.:0-9a-fA-F]{3,39}) - - \[(?P[^\]]+)\] (?P.*' + request_text + '.*)$') + print(f'Apache RE:\n{apache_re}\n') + apache_datetime_re = re.compile( + r'^(?P[0-9]{2})/(?P[^/]{3})/(?P[0-9]{4}):(?P