diff --git a/README.md b/README.md index 62bd29c..42978ee 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,72 @@ -## EDDN - Elite: Dangerous Data Network +# EDDN - Elite Dangerous Data Network -The **Elite: Dangerous Data Network** is a system for willing Commanders to share dynamic data about the galaxy with others. -By pooling data in a common format, tools and analyses can be produced that add an even greater depth and vibrancy to the in-game universe. +## About EDDN +Elite Dangerous Data Network is a tool that facilitates the players of the game +[Elite Dangerous](https://www.elitedangerous.com/), including its +expansions, sharing data about the game galaxy with others. +By pooling data in a common format, tools and analyses can be produced that add +an even greater depth and vibrancy to the in-game universe. -EDDN is not run by or affiliated with [Frontier Developments](http://www.frontier.co.uk/). +EDDN is not run by or affiliated with the developer of the game - [Frontier +Developments](http://www.frontier.co.uk/). -Hosting has been very generously provided by [Vivio Technologies](https://www.viviotech.net/), until 2017. -Hosting is now provided by the [EDCD community](https://edcd.github.io/). +The live EDDN service itself does not store any data, and thus makes no +archive or "current state" available to anyone. What it provides is a +stream of live data to any interested parties. Some of those then make +aggregated data available for general use. -### [Using EDDN](https://github.com/EDSM-NET/EDDN/wiki) +--- + +## Using EDDN +### Game players +If you are a player of the game and only want to help out by sharing the +data available to you over EDDN then please consult the +[EDCD Cmdr's Guide](https://edcd.github.io/cmdrs-guide.html). For the most +part if you want to share data then you will need to be playing the game on a +PC, but there are some tools that utilise an API provided by the game +developer that can supply some data if you are playing on a console. + +If you're looking for tools that utilise EDDN data to enhance your experience +then you're probably looking for one of the sites listed below. NB: These are +listed in name-alphabetical order and no particular ranking or endorsement is +intended. + +- [EDDB](https://eddb.io/) - a website which tries to act as a database of all + the data available in the game. In general EDDB tries to help finding + stuff which players are looking for. +- [EDSM](https://www.edsm.net/) - originally focused on being a 'Star Map', + but has since expanded its functionality. Of particular interest to + in-game explorers. +- [Inara](https://inara.cz/) - a popular alternative to EDDB, with a lot of + its own unique functionality. +- [Spansh](https://www.spansh.co.uk/plotter) - originally this had one tool, + a 'Neutron Star' route plotter, but has since expanded into offering many + other route plotting tools and general data searching. + +There are many other third-party tools for Elite Dangerous listed on +[Elite: Dangerous Codex](https://edcodex.info/), some of which will +interact with EDDN. Check the [EDDN tag](https://edcodex.info/?m=tools&cat=9). + +### Developers +If you are a developer of a third-party tool that could be enhanced by +uploading data to EDDN then please consult +[the live branch documentation](https://github.com/EDCD/EDDN/blob/live/schemas/README-EDDN-schemas.md) +. +**DO NOT** assume that any code or documentation in the `master` (or +any other) branch on GitHub truly reflects the current live service! + +### Misc +There is also a [wiki page](https://github.com/EDSM-NET/EDDN/wiki), but its +contents are currently being migrated into the source code tree (so that +they always match the in-use code). + +Consult [EDDN Status](https://eddn.edcd.io/) for some information about, +and statistics for, the live service. + +--- + +## Hosting of the live service + +Hosting is currently provided by the +[Elite: Dangerous Community Developers](https://edcd.github.io/). -### [EDDN Status](https://eddn.edcd.io/) diff --git a/contrib/test-schema.py b/contrib/test-schema.py new file mode 100644 index 0000000..ebf0e51 --- /dev/null +++ b/contrib/test-schema.py @@ -0,0 +1,27 @@ +"""Check if a file's JSON message passes the given schema.""" +import simplejson +import sys +from jsonschema import FormatChecker, ValidationError +from jsonschema import validate as json_validate + +if len(sys.argv) < 2: + print( +f""" +Usage: {sys.argv[0]} + +Note that the entire file will be loaded by simpljson.load() and should +only contain one JSON object. +""" + ) + sys.exit(-1) + +schema_file_name = sys.argv[1] +test_file_name = sys.argv[2] + +with open(test_file_name, 'r') as test_file: + test_event = simplejson.load(test_file) + + with open(schema_file_name, 'r') as schema_file: + schema = simplejson.load(schema_file) + + json_validate(test_event, schema, format_checker=FormatChecker()) diff --git a/schemas/README-EDDN-schemas.md b/schemas/README-EDDN-schemas.md index 83f6742..170955f 100644 --- a/schemas/README-EDDN-schemas.md +++ b/schemas/README-EDDN-schemas.md @@ -98,24 +98,71 @@ value, e.g. "$schemaRef": "https://eddn.edcd.io/schemas/shipyard/2/test", You MUST also utilise these test forms of the schemas when first testing your -code. There might also be a beta.eddn.edcd.io, or dev.eddn.edcd.io, service +code. + +There might also be a beta.eddn.edcd.io, or dev.eddn.edcd.io, service available from time to time as necessary, e.g. for testing new schemas or -changes to existing ones. +changes to existing ones. Ask on the `#eddn` channel of the EDCD Discord +(see https://edcd.github.io/ for an invite link). + +Alternatively you could attempt +[running your own test instance of EDDN](../docs/Running-this-software.md). ### Sending data -To upload market data to EDDN, you'll need to make a POST request to the URL: +Messages sent to EDDN **MUST**: -* https://eddn.edcd.io:4430/upload/ +- Use the URL: `https://eddn.edcd.io:4430/upload/`. Note the use of + TLS-encrypted HTTPS. A plain HTTP request will elicit a `400 Bad + Request` response. +- Use the HTTP 1.1 protocol. HTTP/2 is not supported at this time. +- Use a **POST** request, with the body containing the EDDN message. No + query parameters in the URL are supported or necessary. -The body of this is a JSON object, so you SHOULD set a `Content-Type` header of -`applicaton/json`, and NOT any of: +The body of an EDDN message is a JSON object in UTF-8 encoding. You SHOULD +set a `Content-Type` header of `applicaton/json`, and NOT any of: * `application/x-www-form-urlencoded` * `multipart/form-data` * `text/plain` +For historical reasons URL form-encoded data *is* supported, **but this is +deprecated and no new software should attempt this method**. We +purposefully do not further document the exact format for this. + +You *MAY* use gzip compression on the body of the message, but it is not +required. + +You should be prepared to handle all scenarios where sending of a message +fails: + +1. Connection refused. +2. Connection timed out. +3. Other possible responses as documented in + [Server responses](#server-responses). + +Carefully consider whether you should queue a 'failed' message for later +retry. In particular, you should ensure that one 'bad' message does not +block other messages from being successfully sent. + +You **MUST** wait some reasonable time (minimum 1 minute) before retrying +any failed message. + +You **MUST NOT** retry any message that received a HTTP `400` or `426` code. +An exception can be made if, **and only if**, *you have manually verified that +you have fixed the issues with it (i.e. updated the schema/version to a +currently supported one and adjusted the data to fit that schema/version).* + +You **MAY** retry a message that initially received a `413` response (in +the hopes that the EDDN service admins decided to increase the maximum +allowed request size), but should not do so too quickly or in perpetuity. + +In general: + +- No data is better than bad data. +- *Delayed* good data is better than degrading the EDDN service for others. + ### Format of uploaded messages -Each message is a JSON object in utf-8 encoding containing the following +Each message is a JSON object in UTF-8 encoding containing the following key+value pairs: 1. `$schemaRef` - Which schema (including version) this message is for. @@ -169,38 +216,157 @@ For example, a shipyard message, version 2, might look like: ``` ### Contents of `message` +Every message MUST comply with the schema its `$schemaRef` value cites. + +Apart from short time windows during deployment of a new version the live +EDDN service should always be using +[the schemas as present in the live branch](https://github.com/EDCD/EDDN/tree/live/schemas). +So, be sure you're checking the live versions and not, e.g. those in the +`master` or other branches. Each `message` object must have, at bare minimum: -1. `timestamp` - string date and time in ISO8601 format. Whilst that +1. `timestamp` - string date and time in ISO8601 format. Whilst this technically allows for any timezone to be cited you SHOULD provide this in UTC, aka 'Zulu Time' as in the example above. You MUST ensure that you are doing this properly. Do not claim 'Z' whilst actually using a local time that is offset from UTC. + If you are only utilising Journal-sourced data then simply using the + value from there should be sufficient as the PC game client is meant to + always be correctly citing UTC for this. Indeed it has been observed, + in the Odyssey 4.0.0.1002 client, that with the Windows clock behind UTC + by 21 seconds both the in-game UI clock *and* the Journal event + timestamps are still properly UTC to the nearest second. + Listeners MAY make decisions on accepting data based on this time stamp, i.e. "too old". -2. One other key/value pair representing the data. In general there will be - much more than this. Again, consult the +2. At least one other key/value pair representing the data. In general there + will be much more than this. Consult the [schemas and their documentation](./). -Note that many of the key names chosen in the schemas are based on the CAPI -data, not Journal events, because the CAPI came first. This means renaming -many of the keys from Journal events to match the schema. +Because the first versions of some schemas were defined when only the CAPI +data was available, before Journal files existed, many of the key names chosen +in the schemas are based on the equivalent in CAPI data, not Journal events. +This means ouy MUST rename many of the keys from Journal events to match the +schemas. EDDN is intended to transport generic data not specific to any particular Cmdr -and to reflect the data that a player would see in-game in station services or -the local map. To that end, uploading applications MUST ensure that messages do -not contain any Cmdr-specific data (other than "uploaderID" and the "horizons" -flag). +and to reflect only the data that every player would see in-game in station +services or the local map. To that end, uploading applications MUST ensure +that messages do not contain any Cmdr-specific data (other than "uploaderID", +the "horizons" flag, and the "odyssey" flag). The individual schemas will instruct you on various elisions (removals) to be made to comply with this. Some of these requirements are also enforced by the schemas, and some things -the schemas enforce might not be explicitly called out here, so **do** -check what you're sending against the schema when implementing sending new -events. +the schemas enforce might not be explicitly called out here. So, **do** +check what you're sending against the relevant schema(s) when making any +changes to your code. + +It is also advisable to Watch this repository on GitHub so as to be aware +of any changes to schemas. + +### Server responses +There are three possible sources of HTTP responses when sending an upload +to EDDN. + +1. The reverse proxy that initially accepts the request. +2. The python `bottle` module that the Gateway uses to process the + forwarded requests. This might object to a message before the actual + EDDN code gets to process it at all. +3. The actual EDDN Gateway code. + +Once a message has cleared the EDDN Gateway then there is no mechanism for any +further issue (such as a message being detected as a duplicate in the +Monitor downstream of the Gateway) to be reported back to the sender. + +To state the obvious, if there are no issues with a request then an HTTP +200 response will be received by the sender. The body of the response +should be the string `OK`. + +#### Reverse Proxy responses +In addition to generic "you typoed the URL" and other such "you just didn't +make a valid request" responses you might experience the following: + +1. `408` - `Request Timed Out` - the sender took too long to make/complete + its request and the reverse proxy rejected it as a result. +2. `503` - `Service Unavailable` - the EDDN Gateway process is either not + running, or not responding. + +#### bottle responses +1. `413` - `Payload Too Large` - `bottle` enforces a maximum request size + and the request exceeds that. As of 2022-01-07 the limit is 1MiB, and + pertains to the plain-text size, not after gzip compression if used. + To verify the current limit check for the line that looks like: + + ``` + bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 # 1MiB, default is/was 100KiB + ``` + + in + [src/eddn/Gateway.py](https://github.com/EDCD/EDDN/blob/master/src/eddn/Gateway.py), + as added in + [commit 0e80c76cb564771465f61825e694227dcc3be312](https://github.com/EDCD/EDDN/commit/0e80c76cb564771465f61825e694227dcc3be312). + +#### EDDN Gateway responses +For all failures the response body will contain text that begins `FAIL: `. Currently two different HTTP status codes are utilised: + +1. `400` - `Bad Request` - This indicates something wrong with the request + body. Possibly due to a format issue (compression, form encoding), or + the actual content of the EDDN message: + 1. `FAIL: zlib.error: ` - A failure to decompress a message that + claimed to be compressed. + + 2. `FAIL: Malformed Upload: ` - the message appeared to be + form-encoded, but either the format was bad or there was no `data` + key. + + 3. `FAIL: JSON parsing: ` - the + message couldn't be parsed as valid JSON. e.g. + + ``` + FAIL: JSON parsing: Expecting property name enclosed in double quotes: line 1 column 2 (char 1) + ``` + + 4. `FAIL: Schema Validation: ` - the message failed to validate + against the cited schema. e.g. + + ``` + FAIL: Schema Validation: [] + ``` + + The exact detail will be very much dependent on both the schema the + message cited and the contents of the message that failed to pass the + validation. + + In particular, if the message contains a key that is tagged 'disallowed' in + the schema the response will look like: + + ``` + FAIL: Schema Validation: "[]" + ``` + This is due to the use of a JSON schema stanza that says "don't allow + any valid type for the value of this key" to trigger the error for such + disallowed keys. + +2. `426` - `Upgrade Required` - This indicates that the cited schema, or + version thereof, is outdated. The body of the response will be: + + ``` + FAIL: Oudated Schema: The schema you have used is no longer supported. Please check for an updated version of your application. + ``` + The wording here is aimed at users of applications that send messages + over EDDN. If you're the developer of such an application then + obviously you need to update your code to use a currently supported + schema and version thereof. + + +There shouldn't be any other variants of a 'FAIL' message. If you find +any then please +[open an issue on GitHub](https://github.com/EDCD/EDDN/issues/new) +with as much detail as possible so that we can update this documentation. ## Receiving messages diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors new file mode 100755 index 0000000..5d2186f --- /dev/null +++ b/scripts/eddn-report-log-errors @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# vim: wrapmargin=0 textwidth=0 smarttab expandtab tabstop=2 shiftwidth=2 +"""Produce a report on the provided EDDN Gateway log file's ERRORs.""" + +import argparse +import re + + +def parse_cl_args() -> str: + """ + Check command-line arguments for input file name. + + :returns: str - input file name + """ + parser = argparse.ArgumentParser( + prog='eddn-report-log-errors', + description='Process an EDDN Gateway log file and report on any ERROR lines found' + ) + + parser.add_argument( + 'inputfile', + metavar='', + help='Name of an EDDN Gateway log file' + ) + + args = parser.parse_args() + + return args.inputfile + + +def process_file(input_file: str) -> None: + print(f'Input file: {input_file}') + + _RE_ERROR = re.compile( + r'^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}[\.,][0-9]{3} - ERROR - Gateway:[0-9]+:' + r' (?P.+)' + r' \((?P[0-9]+),' + r' "(?P[^"]*)",' + r' "(?P[^"]*)",' + r' "(?P[^"]*)",' + r' "(?P[^"]*)",' + r' "(?P[^"]*)"\)' + r' from (?P.+)$' + ) + # TODO: Make this handle gzipped files + with open(input_file, 'r') as input: + line = input.readline() + while line: + line = line.strip() + matches = _RE_ERROR.search(line) + if matches: + # print(matches.group('err_msg')) + # print(matches.group('request_size')) + # print(matches.group('uploader_id')) + # print(matches.group('software_name')) + # print(matches.group('software_version')) + # print(matches.group('schema_ref')) + # print(matches.group('journal_event')) + # print(matches.group('sender_ip')) + # print('') + + ################################################################### + # Issues we know about and HAVE already alerted their + # developers to. + ################################################################### + if matches.group('software_name') == 'EDDiscovery': + # https://github.com/EDDiscovery/EDDiscovery/releases/latest + if matches.group('software_version') == '12.1.7.0': + if matches.group('schema_ref') in ( + 'https://eddn.edcd.io/schemas/shipyard/2', + 'https://eddn.edcd.io/schemas/outfitting/2', + ): + # Reported via Discord PM to Robby 2022-01-07 + if matches.group('err_msg') != 'Failed Validation "[]"': + print(line) + + else: + print(line) + + elif matches.group('software_name') == 'EDDLite': + # https://github.com/EDDiscovery/EDDLite/releases/tag/latest + if matches.group('software_version') == '2.0.0': + if matches.group('schema_ref') in ( + 'https://eddn.edcd.io/schemas/shipyard/2', + 'https://eddn.edcd.io/schemas/outfitting/2', + ): + # Reported via Discord PM to Robby 2022-01-07 + if matches.group('err_msg') != 'Failed Validation "[]"': + print(line) + + else: + print(line) + + elif matches.group('software_name') == 'EDDI': + # https://github.com/EDCD/EDDI/releases/latest + if matches.group('software_version') == '4.0.1': + print(line) + + elif matches.group('software_name') == 'E:D Market Connector [Windows]': + # https://github.com/EDCD/EDMarketConnector/releases/latest + if matches.group('software_version') == '5.2.4': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/codexentry/1': + # + if matches.group('err_msg') != 'Failed Validation "[]"': + print(matches.group('err_msg')) + print(line) + + else: + print(line) + + elif matches.group('software_name') == 'Elite G19s Companion App': + # + if matches.group('software_version') == '3.7.7888.21039': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/commodity/3': + # Reported via Frontier forums: + if matches.group('err_msg') != 'Failed Validation "[]"': + print(matches.group('err_msg')) + print(line) + + else: + print(line) + + elif matches.group('software_name') == 'EDSM': + # It's in-browser, no public source/releases + if matches.group('software_version') == '1.0.1': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': + if matches.group('journal_event') == 'Scan': + # + if not matches.group('err_msg').startswith( + 'Failed Validation "[ + if not matches.group('err_msg').startswith( + 'Failed Validation "[ + if not matches.group('err_msg').startswith( + 'Failed Validation "[ Market.json') + sys.exit(-1) + +CANONICALISE_RE = re.compile(r'\$(.+)_name;') + +def canonicalise(item) -> str: + match = CANONICALISE_RE.match(item) + return match and match.group(1) or item + +entry = json.load(open(sys.argv[1], 'r')) + +items = entry.get('Items') + +commodities = sorted((OrderedDict([ + ('name', canonicalise(commodity['Name'])), + ('meanPrice', commodity['MeanPrice']), + ('buyPrice', commodity['BuyPrice']), + ('stock', commodity['Stock']), + ('stockBracket', commodity['StockBracket']), + ('sellPrice', commodity['SellPrice']), + ('demand', commodity['Demand']), + ('demandBracket', commodity['DemandBracket']), +]) for commodity in items), key=lambda c: c['name']) + +msg = { + '$schemaRef': 'https://eddn.edcd.io/schemas/commodity/3', + 'message': OrderedDict([ + ('timestamp', entry['timestamp']), + ('systemName', entry['StarSystem']), + ('stationName', entry['StationName']), + ('marketId', entry['MarketID']), + ('commodities', commodities), + ]), + +} + +msg['header'] = { + 'uploaderID': 'Athanasius Testing', + 'softwareName': 'Athanasius Testing', + 'softwareVersion': 'v0.0.1', +} +print(json.dumps(msg, indent=2)) diff --git a/scripts/testing/gateway-responses/scan-invalid-JSON.json b/scripts/testing/gateway-responses/scan-invalid-JSON.json new file mode 100644 index 0000000..de93807 --- /dev/null +++ b/scripts/testing/gateway-responses/scan-invalid-JSON.json @@ -0,0 +1,37 @@ +{] + "$schemaRef": "https://eddn.edcd.io/schemas/journal/1", + "header": { + "uploaderID": "Athanasius Testing", + "softwareName": "Athanasius Testing", + "softwareVersion": "v0.0.1" + }, + "message": { + "timestamp":"2021-11-05T15:46:28Z", + "event":"Scan", + "ScanType":"AutoScan", + "BodyName":"Elphin", + "BodyID":1, + "Parents":[ {"Null":0} ], + "StarSystem":"Elphin", + "StarPos":[-30.12500,8.18750,-17.00000], + "SystemAddress":3932076118738, + "DistanceFromArrivalLS":0.000000, + "StarType":"K", + "Subclass":3, + "StellarMass":0.769531, + "Radius":587464832.000000, + "AbsoluteMagnitude":6.294067, + "Age_MY":9558, + "SurfaceTemperature":4796.000000, + "Luminosity":"V", + "SemiMajorAxis":1704360246658.325195, + "Eccentricity":0.348740, + "OrbitalInclination":-72.647343, + "Periapsis":86.347190, + "OrbitalPeriod":7189218699.932098, + "AscendingNode":0.000000, + "MeanAnomaly":351.262353, + "RotationPeriod":248957.736717, + "AxialTilt":-0.126915 + } +} diff --git a/scripts/testing/gateway-responses/scan-invalid-no-softwarename.json b/scripts/testing/gateway-responses/scan-invalid-no-softwarename.json new file mode 100644 index 0000000..a359890 --- /dev/null +++ b/scripts/testing/gateway-responses/scan-invalid-no-softwarename.json @@ -0,0 +1,36 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/journal/1", + "message": { + "timestamp":"2021-11-05T15:46:28Z", + "event":"Scan", + "ScanType":"AutoScan", + "BodyName":"Elphin", + "BodyID":1, + "Parents":[ {"Null":0} ], + "StarSystem":"Elphin", + "StarPos":[-30.12500,8.18750,-17.00000], + "SystemAddress":3932076118738, + "DistanceFromArrivalLS":0.000000, + "StarType":"K", + "Subclass":3, + "StellarMass":0.769531, + "Radius":587464832.000000, + "AbsoluteMagnitude":6.294067, + "Age_MY":9558, + "SurfaceTemperature":4796.000000, + "Luminosity":"V", + "SemiMajorAxis":1704360246658.325195, + "Eccentricity":0.348740, + "OrbitalInclination":-72.647343, + "Periapsis":86.347190, + "OrbitalPeriod":7189218699.932098, + "AscendingNode":0.000000, + "MeanAnomaly":351.262353, + "RotationPeriod":248957.736717, + "AxialTilt":-0.126915 + }, + "header": { + "uploaderID": "Athanasius Testing", + "softwareVersion": "v0.0.1" + } +} diff --git a/scripts/testing/gateway-responses/scan-invalid-no-starpos.json b/scripts/testing/gateway-responses/scan-invalid-no-starpos.json new file mode 100644 index 0000000..12fb537 --- /dev/null +++ b/scripts/testing/gateway-responses/scan-invalid-no-starpos.json @@ -0,0 +1,37 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/journal/1", + "message": { + "timestamp":"2021-11-05T15:46:28Z", + "event":"Scan", + "ScanType":"AutoScan", + "BodyName":"Elphin", + "BodyID":1, + "Parents":[ {"Null":0} ], + "StarSystem":"Elphin", + "NOTStarPos":[-30.12500,8.18750,-17.00000], + "SystemAddress":3932076118738, + "DistanceFromArrivalLS":0.000000, + "StarType":"K", + "Subclass":3, + "StellarMass":0.769531, + "Radius":587464832.000000, + "AbsoluteMagnitude":6.294067, + "Age_MY":9558, + "SurfaceTemperature":4796.000000, + "Luminosity":"V", + "SemiMajorAxis":1704360246658.325195, + "Eccentricity":0.348740, + "OrbitalInclination":-72.647343, + "Periapsis":86.347190, + "OrbitalPeriod":7189218699.932098, + "AscendingNode":0.000000, + "MeanAnomaly":351.262353, + "RotationPeriod":248957.736717, + "AxialTilt":-0.126915 + }, + "header": { + "uploaderID": "Athanasius Testing", + "softwareName": "Athanasius Testing", + "softwareVersion": "v0.0.1" + } +} diff --git a/scripts/testing/gateway-responses/scan-valid.json b/scripts/testing/gateway-responses/scan-valid.json new file mode 100644 index 0000000..b2a17eb --- /dev/null +++ b/scripts/testing/gateway-responses/scan-valid.json @@ -0,0 +1,37 @@ +{ + "$schemaRef": "https://eddn.edcd.io/schemas/journal/1", + "message": { + "timestamp":"2021-11-05T15:46:28Z", + "event":"Scan", + "ScanType":"AutoScan", + "BodyName":"Elphin", + "BodyID":1, + "Parents":[ {"Null":0} ], + "StarSystem":"Elphin", + "StarPos":[-30.12500,8.18750,-17.00000], + "SystemAddress":3932076118738, + "DistanceFromArrivalLS":0.000000, + "StarType":"K", + "Subclass":3, + "StellarMass":0.769531, + "Radius":587464832.000000, + "AbsoluteMagnitude":6.294067, + "Age_MY":9558, + "SurfaceTemperature":4796.000000, + "Luminosity":"V", + "SemiMajorAxis":1704360246658.325195, + "Eccentricity":0.348740, + "OrbitalInclination":-72.647343, + "Periapsis":86.347190, + "OrbitalPeriod":7189218699.932098, + "AscendingNode":0.000000, + "MeanAnomaly":351.262353, + "RotationPeriod":248957.736717, + "AxialTilt":-0.126915 + }, + "header": { + "uploaderID": "from Athanasius Testing", + "softwareName": "Athanasius Testing script", + "softwareVersion": "v0.0.1" + } +} diff --git a/scripts/testing/gateway-responses/test-bad-gzip.py b/scripts/testing/gateway-responses/test-bad-gzip.py new file mode 100644 index 0000000..ff5ff11 --- /dev/null +++ b/scripts/testing/gateway-responses/test-bad-gzip.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# 2022-01-10: THIS SCRIPT DOES NOT PERFORM THE INTENDED PURPOSE +# BECAUSE IT SEEMS THAT `requests` (or underlying modules) IS TOO CLEVER +# AND APPLIES COMPRESSION WHEN WE SET THE `Content-Encoding: gzip` +# HEADER + +import json +import requests +import sys + +print(''' +DO NOT USE THIS SCRIPT, IT DOES NOT PERFORM THE INTENDED PURPOSE. + +USE THE `test-bad-gzip.sh` SCRIPT INSTEAD. + +''') +sys.exit(-1) + +if len(sys.argv) != 2: + print('test-sender.py ') + sys.exit(-1) + +with open(sys.argv[1], 'r') as f: + msg = f.read() + + s = requests.Session() + + # This apparently causes compression to actually happen + s.headers['Content-Encoding'] = 'gzip' + r = s.post( + 'https://beta.eddn.edcd.io:4431/upload/', + data=msg, + ) + + print(f'Response: {r!r}') + print(f'Body: {r.content.decode()}') + diff --git a/scripts/testing/gateway-responses/test-bad-gzip.sh b/scripts/testing/gateway-responses/test-bad-gzip.sh new file mode 100644 index 0000000..befde64 --- /dev/null +++ b/scripts/testing/gateway-responses/test-bad-gzip.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# +# python `requests` appears to perform compression when you set the +# 'Content-Encoding: gzip' header, so do this with curl. + +curl --verbose -d 'wegiuweuygtfawgep9aqe8fpq2387lfbr;iufvypq38764tpgf' -H 'Content-Encoding: gzip' 'https://beta.eddn.edcd.io:4431/upload/' diff --git a/scripts/testing/gateway-responses/test-gzip-bad-formdata.py b/scripts/testing/gateway-responses/test-gzip-bad-formdata.py new file mode 100644 index 0000000..0ae79fd --- /dev/null +++ b/scripts/testing/gateway-responses/test-gzip-bad-formdata.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import json +import requests +import sys +import urllib3 +import zlib + +if len(sys.argv) != 2: + print('test-sender.py ') + sys.exit(-1) + +with open(sys.argv[1], 'r') as f: + # Read from provided file + msg = f.read() + + # Fake form-encode it + msg = 'wibble=' + msg + + # Compress it + msg_gzip = zlib.compress(msg.encode('utf-8')) + + http = urllib3.PoolManager() + + # Send that compressed data as a POST body + r = http.request( + 'POST', + 'https://beta.eddn.edcd.io:4431/upload/', + headers={ + 'Content-Encoding': 'gzip' + }, + body=msg_gzip + ) + + print(f'Response: {r.status!r}') + print(f'Body:\n{r.data.decode()}\n') + diff --git a/scripts/testing/gateway-responses/test-gzip-correct-formdata.py b/scripts/testing/gateway-responses/test-gzip-correct-formdata.py new file mode 100644 index 0000000..066f617 --- /dev/null +++ b/scripts/testing/gateway-responses/test-gzip-correct-formdata.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import json +import requests +import sys +import urllib3 +import zlib + +if len(sys.argv) != 2: + print('test-sender.py ') + sys.exit(-1) + +with open(sys.argv[1], 'r') as f: + # Read from provided file + msg = f.read() + + # Fake form-encode it + msg = 'data=' + msg + + # Compress it + msg_gzip = zlib.compress(msg.encode('utf-8')) + + http = urllib3.PoolManager() + + # Send that compressed data as a POST body + r = http.request( + 'POST', + 'https://beta.eddn.edcd.io:4431/upload/', + headers={ + 'Content-Encoding': 'gzip' + }, + body=msg_gzip + ) + + print(f'Response: {r.status!r}') + print(f'Body:\n{r.data.decode()}\n') + diff --git a/scripts/testing/gateway-responses/test-plain-bad-formdata.py b/scripts/testing/gateway-responses/test-plain-bad-formdata.py new file mode 100644 index 0000000..f0fccb4 --- /dev/null +++ b/scripts/testing/gateway-responses/test-plain-bad-formdata.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import json +import sys +import urllib3 + +if len(sys.argv) != 2: + print('test-sender.py ') + sys.exit(-1) + +with open(sys.argv[1], 'r') as f: + # Read from provided file + msg = f.read() + + # Fake form-encode it + msg = 'wibble=' + msg + + http = urllib3.PoolManager() + + # Send that data as a POST body + r = http.request( + 'POST', + 'https://beta.eddn.edcd.io:4431/upload/', + body=msg + ) + + print(f'Response: {r.status!r}') + print(f'Body:\n{r.data.decode()}\n') + diff --git a/scripts/testing/gateway-responses/test-plain-correct-formdata.py b/scripts/testing/gateway-responses/test-plain-correct-formdata.py new file mode 100644 index 0000000..99ccf63 --- /dev/null +++ b/scripts/testing/gateway-responses/test-plain-correct-formdata.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import json +import sys +import urllib3 + +if len(sys.argv) != 2: + print('test-sender.py ') + sys.exit(-1) + +with open(sys.argv[1], 'r') as f: + # Read from provided file + msg = f.read() + + # Fake form-encode it + msg = 'data=' + msg + + http = urllib3.PoolManager() + + # Send that data as a POST body + r = http.request( + 'POST', + 'https://beta.eddn.edcd.io:4431/upload/', + body=msg + ) + + print(f'Response: {r.status!r}') + print(f'Body:\n{r.data.decode()}\n') + diff --git a/scripts/testing/gateway-responses/test-sender.py b/scripts/testing/gateway-responses/test-sender.py new file mode 100644 index 0000000..8bd9cde --- /dev/null +++ b/scripts/testing/gateway-responses/test-sender.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import json +import requests +import sys + +if len(sys.argv) != 2: + print('test-sender.py ') + sys.exit(-1) + +with open(sys.argv[1], 'r') as f: + msg = f.read() + + s = requests.Session() + + r = s.post('https://beta.eddn.edcd.io:4431/upload/', data=msg) + + print(f'Response: {r!r}') + print(f'Body: {r.content.decode()}') + diff --git a/setup.py b/setup.py index bb187fe..d765672 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,10 @@ import glob import os import re import shutil +import subprocess +import sys from setuptools import setup, find_packages - VERSIONFILE = "src/eddn/conf/Version.py" verstr = "unknown" try: @@ -20,6 +21,44 @@ except EnvironmentError: # Read environment-specific settings import setup_env + +########################################################################### +# Enforce the git status being "branch 'live' checked out, at its HEAD" +# if setup_env.py says this is the live environment. +# +# The idea is to have the `live` branch, *which includes documentation* +# always match what is actually running as the live service (modulo the +# small window between pull/install/restart). Thus it shouldn't use +# `master`, or any other branch than `live`, which may have changes merged +# some time before they become live. +########################################################################### +cwd = os.getcwd() +# e.g. /home/eddn/live/EDDN.git +if setup_env.EDDN_ENV == 'live': + + try: + git_cmd = subprocess.Popen( + 'git symbolic-ref -q --short HEAD'.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + out, err = git_cmd.communicate() + + except Exception as e: + print("Couldn't run git command to check branch: %s" % (e)) + + else: + branch = out.decode().rstrip('\n') + # - For any other branch checked out at its HEAD this will be a + # different name. + # - For any 'detached HEAD' (i.e. specific commit ID, or tag) it + # will be empty. + if branch != 'live': + print("EDDN_ENV is '%s' (and CWD is %s), but branch is '%s', aborting!" % (setup_env.EDDN_ENV, cwd, branch)) + sys.exit(-1) + +########################################################################### + # Location of start-eddn-service script and its config file START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] ) # Location of web files diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 9aee4f5..ee360f1 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -128,22 +128,28 @@ def get_decompressed_message(): :returns: The de-compressed request body. """ content_encoding = request.headers.get('Content-Encoding', '') + logger.debug('Content-Encoding: ' + content_encoding) if content_encoding in ['gzip', 'deflate']: + logger.debug('Content-Encoding of gzip or deflate...') # Compressed request. We have to decompress the body, then figure out # if it's form-encoded. try: # Auto header checking. + logger.debug('Trying zlib.decompress (15 + 32)...') message_body = zlib.decompress(request.body.read(), 15 + 32) except zlib.error: + logger.error('zlib.error, trying zlib.decompress (-15)') # Negative wbits suppresses adler32 checksumming. message_body = zlib.decompress(request.body.read(), -15) + logger.debug('Resulting message_body:\n%s\n' % (message_body)) # At this point, we're not sure whether we're dealing with a straight # un-encoded POST body, or a form-encoded POST. Attempt to parse the # body. If it's not form-encoded, this will return an empty dict. form_enc_parsed = urlparse.parse_qs(message_body) if form_enc_parsed: + logger.debug('Request is form-encoded') # This is a form-encoded POST. The value of the data attrib will # be the body we're looking for. try: @@ -153,14 +159,33 @@ def get_decompressed_message(): "No 'data' POST key/value found. Check your POST key " "name for spelling, and make sure you're passing a value." ) - else: - # Uncompressed request. Bottle handles all of the parsing of the - # POST key/vals, or un-encoded body. - data_key = request.forms.get('data') - if data_key: - # This is a form-encoded POST. Support the silly people. - message_body = data_key + else: + logger.debug('Request is *NOT* form-encoded') + + else: + logger.debug('Content-Encoding indicates *not* compressed...') + + form_enc_parsed = urlparse.parse_qs(request.body.read()) + if form_enc_parsed: + logger.debug('Request is form-encoded') + + # Uncompressed request. Bottle handles all of the parsing of the + # POST key/vals, or un-encoded body. + data_key = request.forms.get('data') + if data_key: + logger.debug('form-encoded POST request detected...') + # This is a form-encoded POST. Support the silly people. + message_body = data_key + + else: + raise MalformedUploadError( + "No 'data' POST key/value found. Check your POST key " + "name for spelling, and make sure you're passing a value." + ) + + else: + logger.debug('Plain POST request detected...') # This is a non form-encoded POST body. message_body = request.body.read() @@ -171,7 +196,7 @@ def parse_and_error_handle(data): try: parsed_message = simplejson.loads(data) except ( - MalformedUploadError, TypeError, ValueError + TypeError, ValueError ) as exc: # Something bad happened. We know this will return at least a # semi-useful error message, so do so. @@ -192,13 +217,14 @@ def parse_and_error_handle(data): pass response.status = 400 - return 'FAIL: ' + str(exc) + return 'FAIL: JSON parsing: ' + str(exc) # Here we check if an outdated schema has been passed if parsed_message["$schemaRef"] in Settings.GATEWAY_OUTDATED_SCHEMAS: response.status = '426 Upgrade Required' # Bottle (and underlying httplib) don't know this one statsCollector.tally("outdated") - return "FAIL: The schema you have used is no longer supported. Please check for an updated version of your application." + return "FAIL: Outdated Schema: The schema you have used is no longer supported. Please check for an updated " \ + "version of your application." validationResults = validator.validate(parsed_message) @@ -239,7 +265,7 @@ def parse_and_error_handle(data): response.status = 400 statsCollector.tally("invalid") - return "FAIL: " + str(validationResults.messages) + return "FAIL: Schema Validation: " + str(validationResults.messages) @app.route('/upload/', method=['OPTIONS', 'POST']) @@ -268,14 +294,14 @@ def upload(): print('Logging of "gzip error" failed: %s' % (e.message)) pass - return exc.message + return 'FAIL: zlib.error: ' + exc.message except MalformedUploadError as exc: # They probably sent an encoded POST, but got the key/val wrong. response.status = 400 logger.error("MalformedUploadError from %s: %s" % (get_remote_address(), exc.message)) - return exc.message + return 'FAIL: Malformed Upload: ' + exc.message statsCollector.tally("inbound") return parse_and_error_handle(message_body)