Merge branch 'master' into live

schemas/commodity-README.md

@@ -51,6 +51,11 @@ In the list of commodites:
   Limpets - not purchasable in station market) or a *non-empty*`"legality":` 
   string (not normally traded at this station market).
 
+If the data is sourced from the journal folder:
+- Remove the `$` prefix and `_name;` suffix from the `Name` field.
+- As the Journal Market.json doesn't contain `economies` or `prohibited` data, 
+  leave these entirely out of the message.  You **MUST NOT** send empty lists.
+
 #### Item Category
 Remove not only the `Category_Localised` key:values, but also the
 `Category` key:value pair from each Item.

scripts/eddn-report-log-errors

@@ -75,7 +75,7 @@ def process_file(input_file: str) -> None:
         ###################################################################
         if matches.group('software_name') == 'EDDiscovery':
           # https://github.com/EDDiscovery/EDDiscovery/releases/latest
-          if software_version >= semantic_version.Version.coerce('15.1.4.0'):
+          if software_version >= semantic_version.Version.coerce('16.0.5.0'):
             if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/outfitting/2':
               err_msg = matches.group('err_msg')
               if (
@@ -93,17 +93,33 @@ def process_file(input_file: str) -> None:
 
         elif matches.group('software_name') == 'EDDLite':
           # https://github.com/EDDiscovery/EDDLite/releases/latest
-          if software_version >= semantic_version.Version.coerce('2.3.0'):
+          if software_version >= semantic_version.Version.coerce('2.5.0'):
             print(line)
 
         elif matches.group('software_name') == 'EDDI':
           # https://github.com/EDCD/EDDI/releases/latest
-          if software_version >= semantic_version.Version.coerce('4.0.1'):
+          if software_version >= semantic_version.Version.coerce('4.0.2'):
-            print(line)
+
+            if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/fsssignaldiscovered/1':
+              if matches.group('err_msg').startswith(
+                'Failed Validation "[<ValidationError: "\'StarPos\' is a required property"'
+                ):
+                # Reported on Discord: <https://discord.com/channels/164411426939600896/353595704658231299/1062652620986134608>
+                pass
+             
+            elif matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/navroute/1':
+              if matches.group('err_msg').startswith(
+                'Failed Validation "[<ValidationError: "\'Route\' is a required property"'
+                ):
+                # Reported on Discord: <https://discord.com/channels/164411426939600896/353595704658231299/1063017819752648775>
+                pass
+             
+            else:
+              print(line)
 
         elif matches.group('software_name').startswith('E:D Market Connector'):
           # https://github.com/EDCD/EDMarketConnector/releases/latest
-          if software_version >= semantic_version.Version.coerce('5.5.0'):
+          if software_version >= semantic_version.Version.coerce('5.7.0'):
             if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1':
               if matches.group('err_msg').startswith(
                 'Failed Validation "[<ValidationError: "{\'type\': [\'array\', \'boolean\', \'integer\', \'number\', \'null\', \'object\', \'string\']} is not allowed for'

src/eddn/Gateway.py

@@ -76,8 +76,14 @@ def extract_message_details(parsed_message):
     uploader_id = '<<UNKNOWN>>'
     software_name = '<<UNKNOWN>>'
     software_version = '<<UNKNOWN>>'
+    game_version = '<<UNKNOWN>>'
+    game_build = '<<UNKNOWN>>'
     schema_ref = '<<UNKNOWN>>'
     journal_event = '<<UNKNOWN>>'
+    system_name = '<<UNKNOWN>>'
+    system_address = '<<UNKNOWN>>'
+    station_name = '<<UNKNOWN>>'
+    station_marketid = '<<UNKNOWN>>'
 
     if 'header' in parsed_message:
         if 'uploaderID' in parsed_message['header']:
@@ -89,6 +95,12 @@ def extract_message_details(parsed_message):
         if 'softwareVersion' in parsed_message['header']:
             software_version = parsed_message['header']['softwareVersion']
 
+        if 'gameversion' in parsed_message['header']:
+            game_version = parsed_message['header']['gameversion']
+
+        if 'gamebuild' in parsed_message['header']:
+            game_build = parsed_message['header']['gamebuild']
+
     if '$schemaRef' in parsed_message:
         schema_ref = parsed_message['$schemaRef']
 
@@ -101,7 +113,44 @@ def extract_message_details(parsed_message):
         else:
             journal_event = '-'
 
-    return uploader_id, software_name, software_version, schema_ref, journal_event
+        if 'systemName' in parsed_message['message']:
+            system_name = parsed_message['message']['systemName']
+
+        elif 'SystemName' in parsed_message['message']:
+            system_name = parsed_message['message']['SystemName']
+
+        elif 'StarSystem' in parsed_message['message']:
+            system_name = parsed_message['message']['StarSystem']
+
+        else:
+            system_name = '-'
+
+        if 'SystemAddress' in parsed_message['message']:
+            system_address = parsed_message['message']['SystemAddress']
+
+        else:
+            system_address = '-'
+
+        if 'stationName' in parsed_message['message']:
+            station_name = parsed_message['message']['stationName']
+        
+        elif 'StationName' in parsed_message['message']:
+            station_name = parsed_message['message']['StationName']
+
+        else:
+            station_name = '-'
+        
+        if 'marketId' in parsed_message['message']:
+            station_marketid = parsed_message['message']['marketId']
+
+        elif 'MarketID' in parsed_message['message']:
+            station_marketid = parsed_message['message']['MarketID']
+
+        else:
+            station_marketid = '-'
+
+    return uploader_id, software_name, software_version, schema_ref, journal_event, game_version, game_build, \
+           system_name, system_address, station_name, station_marketid
 
 def configure():
     # Get the list of transports to bind from settings. This allows us to PUB
@@ -218,10 +267,14 @@ def parse_and_error_handle(data):
         gevent.spawn(push_message, parsed_message, parsed_message['$schemaRef'])
 
         try:
-            uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message)
+            uploader_id, software_name, software_version, schema_ref, journal_event, game_version, game_build, \
-            logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s") from %s' % (
+                system_name, system_address, station_name, station_marketid = extract_message_details(parsed_message)
+            logger.info('Accepted (%d, "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") from %s' % (
                 request.content_length,
                 uploader_id, software_name, software_version, schema_ref, journal_event,
+                game_version, game_build,
+                system_name, system_address,
+                station_name, station_marketid,
                 get_remote_address()
             ))
 
@@ -233,11 +286,15 @@ def parse_and_error_handle(data):
 
     else:
         try:
-            uploader_id, software_name, software_version, schema_ref, journal_event = extract_message_details(parsed_message)
+            uploader_id, software_name, software_version, schema_ref, journal_event, game_version, game_build, \
-            logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s") from %s' % (
+                system_name, system_address, station_name, station_marketid = extract_message_details(parsed_message)
+            logger.error('Failed Validation "%s" (%d, "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s") from %s' % (
                     str(validationResults.messages),
                     request.content_length,
                     uploader_id, software_name, software_version, schema_ref, journal_event,
+                    game_version, game_build,
+                    system_name, system_address,
+                    station_name, station_marketid,
                     get_remote_address()
             ))