From 02b2ea6e18d6631b08dd3fcec0fa3dd3b7e501f4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 11 Jan 2022 16:08:59 +0000 Subject: [PATCH 01/11] scripts/test-sender.py: Allow for changing target URL --- .../testing/gateway-responses/test-sender.py | 44 ++++++++++++++----- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/scripts/testing/gateway-responses/test-sender.py b/scripts/testing/gateway-responses/test-sender.py index de563a7..caa09f7 100644 --- a/scripts/testing/gateway-responses/test-sender.py +++ b/scripts/testing/gateway-responses/test-sender.py @@ -1,20 +1,42 @@ #!/usr/bin/env python3 +# vim: tabstop=4 shiftwidth=4 expandtab smarttab textwidth=0 wrapmargin=0 -import json +import argparse import requests -import sys -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) +upload_url = 'https://dev.eddn.edcd.io:4432/upload/' -with open(sys.argv[1], 'r') as f: - msg = f.read() +def send_message(url, message_filename): + with open(message_filename, 'r') as f: + msg = f.read() - s = requests.Session() + s = requests.Session() - r = s.post('https://dev.eddn.edcd.io:4432/upload/', data=msg) + r = s.post(upload_url, data=msg) - print(f'Response: {r!r}') - print(f'Body: {r.content.decode()}') + print(f'Response: {r!r}') + print(f'Body: {r.content.decode()}') +if __name__ == "__main__": + __parser = argparse.ArgumentParser( + description='Send test messages to an EDDN /upload/ endpoint', + ) + + __parser.add_argument( + '--url', + metavar='', + help='The full URL of an EDDN /upload/ endpoint', + ) + + __parser.add_argument( + 'messagefile', + metavar='', + help='Name of a file containing the body of the EDDN message to be sent', + ) + + args = __parser.parse_args() + + if args.url: + upload_url = args.url + + send_message(upload_url, args.messagefile) From ecd3afbbbc986406b9dea687c44964eda489a034 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 11 Jan 2022 16:12:21 +0000 Subject: [PATCH 02/11] scripts/test-sender.py: Allow 'beta' and 'dev' --url aliases Also explicitly states the URL and input file it uses. --- scripts/testing/gateway-responses/test-sender.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/testing/gateway-responses/test-sender.py b/scripts/testing/gateway-responses/test-sender.py index caa09f7..c364ef8 100644 --- a/scripts/testing/gateway-responses/test-sender.py +++ b/scripts/testing/gateway-responses/test-sender.py @@ -7,6 +7,12 @@ import requests upload_url = 'https://dev.eddn.edcd.io:4432/upload/' def send_message(url, message_filename): + print(f''' +send_message: + URL: {url} + input file: "{message_filename}" +''') + with open(message_filename, 'r') as f: msg = f.read() @@ -37,6 +43,14 @@ if __name__ == "__main__": args = __parser.parse_args() if args.url: - upload_url = args.url + # Allow for some short aliases, but NOT!!! for live !!! + if args.url == 'beta': + upload_url = 'https://beta.eddn.edcd.io:4431/upload/' + + elif args.url == 'dev': + upload_url = 'https://dev.eddn.edcd.io:4432/upload/' + + else: + upload_url = args.url send_message(upload_url, args.messagefile) From fb83aae240613f13edad2430ce6093893a491333 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 11 Jan 2022 16:27:20 +0000 Subject: [PATCH 03/11] scripts/test-sender: Support for requesting good or bad form encoding --- .../testing/gateway-responses/test-sender.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/testing/gateway-responses/test-sender.py b/scripts/testing/gateway-responses/test-sender.py index c364ef8..d5ac96b 100644 --- a/scripts/testing/gateway-responses/test-sender.py +++ b/scripts/testing/gateway-responses/test-sender.py @@ -6,16 +6,23 @@ import requests upload_url = 'https://dev.eddn.edcd.io:4432/upload/' -def send_message(url, message_filename): +def send_message(url, args): print(f''' send_message: URL: {url} - input file: "{message_filename}" + input file: "{args.messagefile}" ''') - with open(message_filename, 'r') as f: + with open(args.messagefile, 'r') as f: msg = f.read() + if args.formdata: + if args.formdata == 'good': + msg = 'data=' + msg + + elif args.formdata == 'bad': + msg = 'BADLYENCODED=' + msg + s = requests.Session() r = s.post(upload_url, data=msg) @@ -34,6 +41,12 @@ if __name__ == "__main__": help='The full URL of an EDDN /upload/ endpoint', ) + __parser.add_argument( + '--formdata', + choices=('good', 'bad'), + help='Specify to form-encode the request body', + ) + __parser.add_argument( 'messagefile', metavar='', @@ -53,4 +66,4 @@ if __name__ == "__main__": else: upload_url = args.url - send_message(upload_url, args.messagefile) + send_message(upload_url, args) From 7537a300c0f0b465507d1b36d8383c17ec1da465 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 11 Jan 2022 16:31:05 +0000 Subject: [PATCH 04/11] scripts/test-sender: Support for valid gzip compression --- scripts/testing/gateway-responses/test-sender.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/testing/gateway-responses/test-sender.py b/scripts/testing/gateway-responses/test-sender.py index d5ac96b..52e7aca 100644 --- a/scripts/testing/gateway-responses/test-sender.py +++ b/scripts/testing/gateway-responses/test-sender.py @@ -3,6 +3,7 @@ import argparse import requests +import zlib upload_url = 'https://dev.eddn.edcd.io:4432/upload/' @@ -25,6 +26,11 @@ send_message: s = requests.Session() + if args.gzip: + if args.gzip == 'good': + msg = zlib.compress(msg.encode('utf-8')) + s.headers['Content-Encoding'] = 'gzip' + r = s.post(upload_url, data=msg) print(f'Response: {r!r}') @@ -47,6 +53,12 @@ if __name__ == "__main__": help='Specify to form-encode the request body', ) + __parser.add_argument( + '--gzip', + choices=('good'), + help='Specify to gzip-compress the request body', + ) + __parser.add_argument( 'messagefile', metavar='', From ad8fc57df3dfd4f7cbaa2ee7a20bb54c765150b0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 11 Jan 2022 16:41:25 +0000 Subject: [PATCH 05/11] scripts/test-sender: Support --gzip=bad --- scripts/testing/gateway-responses/test-sender.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/testing/gateway-responses/test-sender.py b/scripts/testing/gateway-responses/test-sender.py index 52e7aca..93f2fa0 100644 --- a/scripts/testing/gateway-responses/test-sender.py +++ b/scripts/testing/gateway-responses/test-sender.py @@ -27,9 +27,14 @@ send_message: s = requests.Session() if args.gzip: - if args.gzip == 'good': - msg = zlib.compress(msg.encode('utf-8')) - s.headers['Content-Encoding'] = 'gzip' + # We assume that the argparse setup is enforcing the value being + # valid, i.e. `'good'` if it's not `'bad'`. + msg = zlib.compress(msg.encode('utf-8')) + s.headers['Content-Encoding'] = 'gzip' + + if args.gzip == 'bad': + # Prepend a character so it's not a valid gzip header + msg = b'w' + msg r = s.post(upload_url, data=msg) @@ -55,7 +60,7 @@ if __name__ == "__main__": __parser.add_argument( '--gzip', - choices=('good'), + choices=('good', 'bad'), help='Specify to gzip-compress the request body', ) From 00071ba7e9be9643e03bcadb3e2c1ffc62ddea3e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 11 Jan 2022 16:43:30 +0000 Subject: [PATCH 06/11] scripts/testing: Remove per-scenario scripts The required functionality is now all in `test-sender.py`. --- .../gateway-responses/test-bad-gzip.py | 38 ------------------- .../gateway-responses/test-bad-gzip.sh | 6 --- .../test-gzip-bad-formdata.py | 37 ------------------ .../test-gzip-correct-formdata.py | 37 ------------------ .../gateway-responses/test-gzip-plain-json.py | 34 ----------------- .../test-plain-bad-formdata.py | 29 -------------- .../test-plain-correct-formdata.py | 29 -------------- 7 files changed, 210 deletions(-) delete mode 100644 scripts/testing/gateway-responses/test-bad-gzip.py delete mode 100644 scripts/testing/gateway-responses/test-bad-gzip.sh delete mode 100644 scripts/testing/gateway-responses/test-gzip-bad-formdata.py delete mode 100644 scripts/testing/gateway-responses/test-gzip-correct-formdata.py delete mode 100644 scripts/testing/gateway-responses/test-gzip-plain-json.py delete mode 100644 scripts/testing/gateway-responses/test-plain-bad-formdata.py delete mode 100644 scripts/testing/gateway-responses/test-plain-correct-formdata.py diff --git a/scripts/testing/gateway-responses/test-bad-gzip.py b/scripts/testing/gateway-responses/test-bad-gzip.py deleted file mode 100644 index 9d50f1f..0000000 --- a/scripts/testing/gateway-responses/test-bad-gzip.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -# -# 2022-01-10: THIS SCRIPT DOES NOT PERFORM THE INTENDED PURPOSE -# BECAUSE IT SEEMS THAT `requests` (or underlying modules) IS TOO CLEVER -# AND APPLIES COMPRESSION WHEN WE SET THE `Content-Encoding: gzip` -# HEADER - -import json -import requests -import sys - -print(''' -DO NOT USE THIS SCRIPT, IT DOES NOT PERFORM THE INTENDED PURPOSE. - -USE THE `test-bad-gzip.sh` SCRIPT INSTEAD. - -''') -sys.exit(-1) - -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) - -with open(sys.argv[1], 'r') as f: - msg = f.read() - - s = requests.Session() - - # This apparently causes compression to actually happen - s.headers['Content-Encoding'] = 'gzip' - r = s.post( - 'https://dev.eddn.edcd.io:4432/upload/', - data=msg, - ) - - print(f'Response: {r!r}') - print(f'Body: {r.content.decode()}') - diff --git a/scripts/testing/gateway-responses/test-bad-gzip.sh b/scripts/testing/gateway-responses/test-bad-gzip.sh deleted file mode 100644 index 77592bf..0000000 --- a/scripts/testing/gateway-responses/test-bad-gzip.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# -# python `requests` appears to perform compression when you set the -# 'Content-Encoding: gzip' header, so do this with curl. - -curl --verbose -d 'wegiuweuygtfawgep9aqe8fpq2387lfbr;iufvypq38764tpgf' -H 'Content-Encoding: gzip' 'https://dev.eddn.edcd.io:4432/upload/' diff --git a/scripts/testing/gateway-responses/test-gzip-bad-formdata.py b/scripts/testing/gateway-responses/test-gzip-bad-formdata.py deleted file mode 100644 index 0e3e1f0..0000000 --- a/scripts/testing/gateway-responses/test-gzip-bad-formdata.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 - -import json -import requests -import sys -import urllib3 -import zlib - -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) - -with open(sys.argv[1], 'r') as f: - # Read from provided file - msg = f.read() - - # Fake form-encode it - msg = 'wibble=' + msg - - # Compress it - msg_gzip = zlib.compress(msg.encode('utf-8')) - - http = urllib3.PoolManager() - - # Send that compressed data as a POST body - r = http.request( - 'POST', - 'https://dev.eddn.edcd.io:4432/upload/', - headers={ - 'Content-Encoding': 'gzip' - }, - body=msg_gzip - ) - - print(f'Response: {r.status!r}') - print(f'Body:\n{r.data.decode()}\n') - diff --git a/scripts/testing/gateway-responses/test-gzip-correct-formdata.py b/scripts/testing/gateway-responses/test-gzip-correct-formdata.py deleted file mode 100644 index e58c1d9..0000000 --- a/scripts/testing/gateway-responses/test-gzip-correct-formdata.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 - -import json -import requests -import sys -import urllib3 -import zlib - -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) - -with open(sys.argv[1], 'r') as f: - # Read from provided file - msg = f.read() - - # Fake form-encode it - msg = 'data=' + msg - - # Compress it - msg_gzip = zlib.compress(msg.encode('utf-8')) - - http = urllib3.PoolManager() - - # Send that compressed data as a POST body - r = http.request( - 'POST', - 'https://dev.eddn.edcd.io:4432/upload/', - headers={ - 'Content-Encoding': 'gzip' - }, - body=msg_gzip - ) - - print(f'Response: {r.status!r}') - print(f'Body:\n{r.data.decode()}\n') - diff --git a/scripts/testing/gateway-responses/test-gzip-plain-json.py b/scripts/testing/gateway-responses/test-gzip-plain-json.py deleted file mode 100644 index db7042a..0000000 --- a/scripts/testing/gateway-responses/test-gzip-plain-json.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -import json -import requests -import sys -import urllib3 -import zlib - -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) - -with open(sys.argv[1], 'r') as f: - # Read from provided file - msg = f.read() - - # Compress it - msg_gzip = zlib.compress(msg.encode('utf-8')) - - http = urllib3.PoolManager() - - # Send that compressed data as a POST body - r = http.request( - 'POST', - 'https://dev.eddn.edcd.io:4432/upload/', - headers={ - 'Content-Encoding': 'gzip' - }, - body=msg_gzip - ) - - print(f'Response: {r.status!r}') - print(f'Body:\n{r.data.decode()}\n') - diff --git a/scripts/testing/gateway-responses/test-plain-bad-formdata.py b/scripts/testing/gateway-responses/test-plain-bad-formdata.py deleted file mode 100644 index 04cc0bd..0000000 --- a/scripts/testing/gateway-responses/test-plain-bad-formdata.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys -import urllib3 - -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) - -with open(sys.argv[1], 'r') as f: - # Read from provided file - msg = f.read() - - # Fake form-encode it - msg = 'wibble=' + msg - - http = urllib3.PoolManager() - - # Send that data as a POST body - r = http.request( - 'POST', - 'https://dev.eddn.edcd.io:4432/upload/', - body=msg - ) - - print(f'Response: {r.status!r}') - print(f'Body:\n{r.data.decode()}\n') - diff --git a/scripts/testing/gateway-responses/test-plain-correct-formdata.py b/scripts/testing/gateway-responses/test-plain-correct-formdata.py deleted file mode 100644 index a00815f..0000000 --- a/scripts/testing/gateway-responses/test-plain-correct-formdata.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys -import urllib3 - -if len(sys.argv) != 2: - print('test-sender.py ') - sys.exit(-1) - -with open(sys.argv[1], 'r') as f: - # Read from provided file - msg = f.read() - - # Fake form-encode it - msg = 'data=' + msg - - http = urllib3.PoolManager() - - # Send that data as a POST body - r = http.request( - 'POST', - 'https://dev.eddn.edcd.io:4432/upload/', - body=msg - ) - - print(f'Response: {r.status!r}') - print(f'Body:\n{r.data.decode()}\n') - From e93c04bb214ced0b116a37391b5b731548fda1ac Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 12 Jan 2022 09:40:47 +0000 Subject: [PATCH 07/11] scripts/eddn-report: EliteLogAgent latest is 2.0.0.660 This allows for ignoring the ancient 0.9.2.412 that showed some errors yesterday. --- scripts/eddn-report-log-errors | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 5d2186f..b6921fe 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -174,6 +174,11 @@ def process_file(input_file: str) -> None: else: print(line) + elif matches.group('software_name') == 'EliteLogAgent': + # Apparently a Barry Carylon project, but no home page ? + if matches.group('software_version') == '2.0.0.660': + print(line) + ################################################################### # Issues we know about, but haven't yet alerted developers to ################################################################### From cc5ae322278ab8d1aab76ba24724cc6ae8662aa2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 12 Jan 2022 15:43:59 +0000 Subject: [PATCH 08/11] scripts/eddn-report: Correct EliteLogAgent comment --- scripts/eddn-report-log-errors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index b6921fe..a626e23 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -175,7 +175,7 @@ def process_file(input_file: str) -> None: print(line) elif matches.group('software_name') == 'EliteLogAgent': - # Apparently a Barry Carylon project, but no home page ? + # if matches.group('software_version') == '2.0.0.660': print(line) From 40a1d11e76b8a3d03adf367643469ac7d3576bb9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 14 Jan 2022 11:02:32 +0000 Subject: [PATCH 09/11] contrib/eddn-logs-archive: Produce report on rotated logfile --- contrib/eddn-logs-archive | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/eddn-logs-archive b/contrib/eddn-logs-archive index ef30aef..972f710 100755 --- a/contrib/eddn-logs-archive +++ b/contrib/eddn-logs-archive @@ -87,7 +87,8 @@ do log " Archiving ${service}.log ..." # We have no means to tell the service to close and re-open output, it's # to stdout/err anyway. So we copy it. - COMPRESSED_NAME="${service}.log.$(date --iso-8601=seconds)" + TIMESTAMP="$(date --iso-8601=seconds)" + COMPRESSED_NAME="${service}.log.${TIMESTAMP}" cp ${service}.log "${COMPRESSED_NAME}" if [ $? -ne 0 ]; then @@ -97,6 +98,11 @@ do fi # Truncate the live file. :> ${service}.log + + # Produce a report on the just-rotated log + ${HOME}/.local/bin/eddn-report-log-errors "${COMPRESSED_NAME}" > \ + "${HOME}/reports/eddn-errors/by-log-rotation/eddn-errors-${TIMESTMAP}.txt" + # Now compress the newly archived log gzip -9v "${COMPRESSED_NAME}" log " DONE" From e69f7a938af2ee41ad324651617928667bb57df8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 15 Jan 2022 14:09:47 +0000 Subject: [PATCH 10/11] scripts/eddn-report: Add Moonlight/1.3.4/Scan case --- scripts/eddn-report-log-errors | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index a626e23..6a87003 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -179,6 +179,24 @@ def process_file(input_file: str) -> None: if matches.group('software_version') == '2.0.0.660': print(line) + # + # + elif matches.group('software_name') == 'Moonlight': + if matches.group('software_version') == '1.3.4': + if matches.group('schema_ref') == 'https://eddn.edcd.io/schemas/journal/1': + if matches.group('journal_event') == 'Scan': + # Ref: + if not matches.group('err_msg').startswith( + 'Failed Validation "[ Date: Sun, 16 Jan 2022 09:33:45 +0000 Subject: [PATCH 11/11] scripts/eddn-report: Current G19s version only / EVA is abandoned --- scripts/eddn-report-log-errors | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/eddn-report-log-errors b/scripts/eddn-report-log-errors index 6a87003..45a95c7 100755 --- a/scripts/eddn-report-log-errors +++ b/scripts/eddn-report-log-errors @@ -117,9 +117,6 @@ def process_file(input_file: str) -> None: print(matches.group('err_msg')) print(line) - else: - print(line) - elif matches.group('software_name') == 'EDSM': # It's in-browser, no public source/releases if matches.group('software_version') == '1.0.1': @@ -197,7 +194,14 @@ def process_file(input_file: str) -> None: else: print(line) - ################################################################### + + # Abandoned/unmaintained project + # + # + elif matches.group('software_name') == 'EVA [iPhone]': + pass + + #################################################################### # Issues we know about, but haven't yet alerted developers to ################################################################### ###################################################################