From d185618d57091296408d1ea1da83eb22a9611f48 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 14:43:24 +0100 Subject: [PATCH 01/39] setup.py: Update author and URL edcd@miggy.org is a placeholder, can be tweaked by Athanasius to anything required. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e4d2943..79e101b 100644 --- a/setup.py +++ b/setup.py @@ -20,9 +20,9 @@ setup( name='eddn', version=verstr, description='Elite: Dangerous Data Network', - author='Anthor (EDSM)', - author_email='contact@edsm.net', - url='https://github.com/EDSM-NET/EDDN', + author='EDCD (https://edcd.github.io/)', + author_email='edcd@miggy.org', + url='https://github.com/EDCD/EDDN', packages=find_packages('src', exclude=["*.tests"]), package_dir = {'':'src'}, data_files=[('eddn/schemas', glob.glob("schemas/*.json"))], From f033b3621b48addfbfbc30b2e4f3ffe092dfdc45 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 14:53:54 +0100 Subject: [PATCH 02/39] setup.py: Pin versions of install_requires packages Yes, I know that: says not to do this in install_requires. But there's at least the issue of pyzmq's later versions just not working with python 2.7. These are the versions we *know* work currently. --- setup.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 79e101b..245b578 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,19 @@ setup( long_description="""\ The Elite: Dangerous Data Network allows E:D players to share data. Not affiliated with Frontier Developments. """, - install_requires=["argparse", "bottle", "enum34", "gevent", "jsonschema", "pyzmq", "strict_rfc3339", "simplejson", "mysql-connector-python"], + # Yes, we pin versions. With python2.7 the latest pyzmq will NOT + # work, for instance. + install_requires=[ + "argparse", + "bottle==0.12.15", + "enum34==1.1.6", + "gevent==1.3.7", + "jsonschema==2.6.0", + "pyzmq==17.1.2", + "strict_rfc3339==0.7", + "simplejson==3.16.0", + "mysql-connector-python==8.0.17" + ], entry_points={ 'console_scripts': [ 'eddn-gateway = eddn.Gateway:main', From 4edb1e7e21983c339dfb846619d160d9e29f9e89 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 15:01:07 +0100 Subject: [PATCH 03/39] setup.py: Comment about schemas only being for code, not website --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 245b578..fd0d018 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,8 @@ setup( url='https://github.com/EDCD/EDDN', packages=find_packages('src', exclude=["*.tests"]), package_dir = {'':'src'}, + # This includes them for the running code, but that doesn't help + # serve them up for reference. data_files=[('eddn/schemas', glob.glob("schemas/*.json"))], long_description="""\ The Elite: Dangerous Data Network allows E:D players to share data. Not affiliated with Frontier Developments. From 165a00599fe340c99e6e272604edcf2403c127e8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 15:15:10 +0100 Subject: [PATCH 04/39] docs/Running: We should use a python venv --- docs/Running-this-software.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 924ac5f..6254a9a 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -23,7 +23,7 @@ A specific user was created: Some additional Debian packages and python modules are required: - apt install python-pip + apt install python-pip virtualenv You will need a mysql/mariab database: @@ -115,11 +115,26 @@ appropriate `` section: ## In the 'eddn' account +### Set up a python virtual environment +So as to not have any python package version requirements clash with +anything else it's best to use a Python virtual environment (venv). You +will have installed the Debian package 'virtualenv' above for this purpose. + +We'll put the venv in `~/eddn/python2.7-venv` with the following +command: + + virtualenv -p /usr/bin/python2.7 ${HOME}/python2.7-venv + +And now start using this venv: + + . python2.7-venv/bin/activate + ### Clone a copy of the application project from gitub mkdir -p ~/eddn/dev cd ~/eddn/dev git clone https://github.com/EDCD/EDDN.git + cd EDDN We'll assume this `~/eddn/dev/EDDN` path elsewhere in this document. From 415f5bb17dc31166d94a1eaab5503d317ca20be5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 15:22:58 +0100 Subject: [PATCH 05/39] systemd: Change to a single script for starting services * Pass the EDDN service to be started as $1. * eddn_config now defines a python venv activate script to use. * And this new central script uses that. --- contrib/systemd/eddn@.service | 2 +- contrib/systemd/eddn_config | 1 + contrib/systemd/start-eddn-gateway | 16 ------------- contrib/systemd/start-eddn-monitor | 16 ------------- contrib/systemd/start-eddn-relay | 16 ------------- contrib/systemd/start-eddn-service | 37 ++++++++++++++++++++++++++++++ 6 files changed, 39 insertions(+), 49 deletions(-) delete mode 100755 contrib/systemd/start-eddn-gateway delete mode 100755 contrib/systemd/start-eddn-monitor delete mode 100755 contrib/systemd/start-eddn-relay create mode 100755 contrib/systemd/start-eddn-service diff --git a/contrib/systemd/eddn@.service b/contrib/systemd/eddn@.service index 541516b..7779172 100644 --- a/contrib/systemd/eddn@.service +++ b/contrib/systemd/eddn@.service @@ -18,7 +18,7 @@ After=network.target Type=simple User=eddn Group=eddn -ExecStart=/home/eddn/.local/bin/start-%i +ExecStart=/home/eddn/.local/bin/start-eddn-service %i TimeoutStartSec=10s TimeoutStopSec=10s SyslogIdentifier=eddn@%i diff --git a/contrib/systemd/eddn_config b/contrib/systemd/eddn_config index 6fbe7d3..68368d1 100644 --- a/contrib/systemd/eddn_config +++ b/contrib/systemd/eddn_config @@ -1,2 +1,3 @@ CONFIG_OVERRIDE="${HOME}/.local/share/eddn/config.json" LOG_DIR="${HOME}/.var/log/eddn" +PYTHON_VENV_ACTIVATE="${HOME}/eddn/python2.7-venv/bin/activate" diff --git a/contrib/systemd/start-eddn-gateway b/contrib/systemd/start-eddn-gateway deleted file mode 100755 index 6c71da4..0000000 --- a/contrib/systemd/start-eddn-gateway +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab -# -# Start the EDDN Gateway, including redirecting output to a log file. - -EXEC_PATH=$(dirname $0) -#echo "EXEC_PATH: ${EXEC_PATH}" - -# Ensure we're in the correct place -cd ${EXEC_PATH} -#pwd - -# Bring in some common configuration -. ./eddn_config - -./eddn-gateway --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-gateway.log" 2>&1 diff --git a/contrib/systemd/start-eddn-monitor b/contrib/systemd/start-eddn-monitor deleted file mode 100755 index 717e3e2..0000000 --- a/contrib/systemd/start-eddn-monitor +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab -# -# Start the EDDN Gateway, including redirecting output to a log file. - -EXEC_PATH=$(dirname $0) -#echo "EXEC_PATH: ${EXEC_PATH}" - -# Ensure we're in the correct place -cd ${EXEC_PATH} -#pwd - -# Bring in some common configuration -. ./eddn_config - -./eddn-monitor --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-monitor.log" 2>&1 diff --git a/contrib/systemd/start-eddn-relay b/contrib/systemd/start-eddn-relay deleted file mode 100755 index cb5bd82..0000000 --- a/contrib/systemd/start-eddn-relay +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab -# -# Start the EDDN Gateway, including redirecting output to a log file. - -EXEC_PATH=$(dirname $0) -#echo "EXEC_PATH: ${EXEC_PATH}" - -# Ensure we're in the correct place -cd ${EXEC_PATH} -#pwd - -# Bring in some common configuration -. ./eddn_config - -./eddn-relay --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-relay.log" 2>&1 diff --git a/contrib/systemd/start-eddn-service b/contrib/systemd/start-eddn-service new file mode 100755 index 0000000..64b66bd --- /dev/null +++ b/contrib/systemd/start-eddn-service @@ -0,0 +1,37 @@ +#!/bin/bash -x +# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab +# +# Start an EDDN Service, including redirecting output to a log file. + +if [ -z "${1}" ]; +then + echo "No EDDN service specified. One of: ${SERVICE}, monitor, relay" + exit 3 +fi +SERVICE="${1}" + +EXEC_PATH=$(dirname $0) +#echo "EXEC_PATH: ${EXEC_PATH}" + +# Ensure we're in the correct place +cd ${EXEC_PATH} +#pwd + +# Bring in some common configuration +if [ ! -f eddn_config ]; +then + echo "eddn_config is missing from $(pwd)" + exit 1 +fi +. ./eddn_config + +# Use the python venv +. "${PYTHON_VENV_ACTIVATE}" + +if [ ! -f eddn-${SERVICE} ]; +then + echo "eddn-${SERVICE} is missing from $(pwd)" + exit 2 +fi + +./eddn-${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-${SERVICE}.log" 2>&1 From 4c2f3794803b053cc71222d42efbfcd0a07202d6 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 15:52:39 +0100 Subject: [PATCH 06/39] contrib/systemd: Adjusted service file and script for python venv --- contrib/systemd/eddn_config | 2 +- contrib/systemd/start-eddn-service | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/systemd/eddn_config b/contrib/systemd/eddn_config index 68368d1..22d7121 100644 --- a/contrib/systemd/eddn_config +++ b/contrib/systemd/eddn_config @@ -1,3 +1,3 @@ CONFIG_OVERRIDE="${HOME}/.local/share/eddn/config.json" LOG_DIR="${HOME}/.var/log/eddn" -PYTHON_VENV_ACTIVATE="${HOME}/eddn/python2.7-venv/bin/activate" +PYTHON_VENV="${HOME}/eddn/python-venv" diff --git a/contrib/systemd/start-eddn-service b/contrib/systemd/start-eddn-service index 64b66bd..63fa806 100755 --- a/contrib/systemd/start-eddn-service +++ b/contrib/systemd/start-eddn-service @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/sh # vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab # # Start an EDDN Service, including redirecting output to a log file. @@ -26,12 +26,12 @@ fi . ./eddn_config # Use the python venv -. "${PYTHON_VENV_ACTIVATE}" +. "${PYTHON_VENV}/bin/activate" -if [ ! -f eddn-${SERVICE} ]; +if [ ! -f "${PYTHON_VENV}/bin/${SERVICE}" ]; then - echo "eddn-${SERVICE} is missing from $(pwd)" + echo "${SERVICE} is missing from ${PYTHON_VENV}/bin" exit 2 fi -./eddn-${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-${SERVICE}.log" 2>&1 +${PYTHON_VENV}/bin/${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1 From 152bd86818b2ef418a124475072ae8779477ef65 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 15:55:49 +0100 Subject: [PATCH 07/39] setup.py: A little reformatting for readability --- setup.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index fd0d018..5037d94 100644 --- a/setup.py +++ b/setup.py @@ -20,17 +20,27 @@ setup( name='eddn', version=verstr, description='Elite: Dangerous Data Network', + long_description="""\ + The Elite Dangerous Data Network allows ED players to share data. Not affiliated with Frontier Developments. + """, author='EDCD (https://edcd.github.io/)', author_email='edcd@miggy.org', url='https://github.com/EDCD/EDDN', - packages=find_packages('src', exclude=["*.tests"]), + + packages=find_packages( + 'src', + exclude=["*.tests"] + ), package_dir = {'':'src'}, + # This includes them for the running code, but that doesn't help # serve them up for reference. - data_files=[('eddn/schemas', glob.glob("schemas/*.json"))], - long_description="""\ - The Elite: Dangerous Data Network allows E:D players to share data. Not affiliated with Frontier Developments. - """, + data_files=[ + ( + 'eddn/schemas', glob.glob("schemas/*.json") + ) + ], + # Yes, we pin versions. With python2.7 the latest pyzmq will NOT # work, for instance. install_requires=[ @@ -44,11 +54,12 @@ setup( "simplejson==3.16.0", "mysql-connector-python==8.0.17" ], + entry_points={ 'console_scripts': [ 'eddn-gateway = eddn.Gateway:main', 'eddn-relay = eddn.Relay:main', 'eddn-monitor = eddn.Monitor:main', - ], - } - ) + ], + } +) From 548f7c3cb30a470c0f4bf5427c797df52d8ff9e3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 16:30:11 +0100 Subject: [PATCH 08/39] setup.py: Get 'extra' files in place - monitor, schemas, systemd files * 'monitor' files are for the web page showing status/stats. * 'schema' files are a publically available copy of the schemas in-use. * 'systemd' files are the start- script and its config file. --- setup.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5037d94..72bacb2 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ -from setuptools import setup, find_packages -import re import glob +import os +import re +import shutil +from setuptools import setup, find_packages VERSIONFILE = "src/eddn/conf/Version.py" @@ -15,6 +17,10 @@ except EnvironmentError: print "unable to find version in %s" % (VERSIONFILE,) raise RuntimeError("if %s exists, it is required to be well-formed" % (VERSIONFILE,)) +# Location of start-eddn-service script and its config file +START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] ) +# Location of web files +SHARE_EDDN_FILES='%s/.local/share/eddn' % ( os.environ['HOME'] ) setup( name='eddn', @@ -63,3 +69,56 @@ setup( ], } ) + +# Ensure the systemd-required start files are in place +if not os.path.isdir(START_SCRIPT_BIN): + # We're still using Python 2.7, so no pathlib + os.chdir('/') + for pc in START_SCRIPT_BIN[1:].split('/'): + try: + os.mkdir(pc) + + except OSError: + pass + + os.chdir(pc) + + if not os.path.isdir(START_SCRIPT_BIN): + print "%s can't be created, aborting!!!" % (START_SCRIPT_BIN) + exit(-1) + +for f in ( 'contrib/systemd/start-eddn-service', 'contrib/systemd/eddn_config'): + shutil.copy(f, START_SCRIPT_BIN) + +# Ensure the latest monitor files are in place +if not os.path.isdir(SHARE_EDDN_FILES): + # We're still using Python 2.7, so no pathlib + os.chdir('/') + for pc in SHARE_EDDN_FILES[1:].split('/'): + try: + os.mkdir(pc) + + except OSError: + pass + + os.chdir(pc) + + if not os.path.isdir(SHARE_EDDN_FILES): + print "%s can't be created, aborting!!!" % (SHARE_EDDN_FILES) + exit(-1) + +# Copy the monitor (Web page) files +try: + shutil.rmtree('%s/monitor' % ( SHARE_EDDN_FILES )) +except OSError: + pass +shutil.copytree('contrib/monitor', '%s/monitor' % ( SHARE_EDDN_FILES )) +# And a copy of the schemas too +try: + shutil.rmtree('%s/schemas' % ( SHARE_EDDN_FILES )) +except OSError: + pass +shutil.copytree('schemas', '%s/schemas' % ( SHARE_EDDN_FILES )) + +# You still need to make an override config file +print "You now NEED to create and populate %s/config.json" % ( SHARE_EDDN_FILES ) From e8c1e3b03a4173e8cdc5b34886e2396a08baf48e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 16:42:02 +0100 Subject: [PATCH 09/39] setup.py: Ensure CWD is as expected & copy example config if needs be --- setup.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 72bacb2..6bd9b33 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ setup( ) # Ensure the systemd-required start files are in place +old_cwd = os.getcwd() if not os.path.isdir(START_SCRIPT_BIN): # We're still using Python 2.7, so no pathlib os.chdir('/') @@ -87,10 +88,12 @@ if not os.path.isdir(START_SCRIPT_BIN): print "%s can't be created, aborting!!!" % (START_SCRIPT_BIN) exit(-1) +os.chdir(old_cwd) for f in ( 'contrib/systemd/start-eddn-service', 'contrib/systemd/eddn_config'): shutil.copy(f, START_SCRIPT_BIN) # Ensure the latest monitor files are in place +old_cwd = os.getcwd() if not os.path.isdir(SHARE_EDDN_FILES): # We're still using Python 2.7, so no pathlib os.chdir('/') @@ -107,6 +110,7 @@ if not os.path.isdir(SHARE_EDDN_FILES): print "%s can't be created, aborting!!!" % (SHARE_EDDN_FILES) exit(-1) +os.chdir(old_cwd) # Copy the monitor (Web page) files try: shutil.rmtree('%s/monitor' % ( SHARE_EDDN_FILES )) @@ -121,4 +125,17 @@ except OSError: shutil.copytree('schemas', '%s/schemas' % ( SHARE_EDDN_FILES )) # You still need to make an override config file -print "You now NEED to create and populate %s/config.json" % ( SHARE_EDDN_FILES ) +if not os.path.isfile('%s/config.json' % ( SHARE_EDDN_FILES )): + shutil.copy('docs/config-EXAMPLE.json', SHARE_EDDN_FILES) + print """ +****************************************************************************** +There was no config.json file in place, so docs/config-EXAMPLE.json was +copied into: + + %s + +Please review, edit and rename this file to 'config.json' so that this +software will actually work. +See docs/Running-this-software.md for guidance. +****************************************************************************** +""" % ( SHARE_EDDN_FILES ) From b58acec1c904408c04d9d8aa7d6dab80c5fcd3cc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 16:45:41 +0100 Subject: [PATCH 10/39] setup.py: Call out what we're doing in these extra steps --- setup.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/setup.py b/setup.py index 6bd9b33..8b01793 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,10 @@ setup( ) # Ensure the systemd-required start files are in place +print """ +****************************************************************************** +Ensuring start script and its config file are in place... +""" old_cwd = os.getcwd() if not os.path.isdir(START_SCRIPT_BIN): # We're still using Python 2.7, so no pathlib @@ -93,6 +97,10 @@ for f in ( 'contrib/systemd/start-eddn-service', 'contrib/systemd/eddn_config'): shutil.copy(f, START_SCRIPT_BIN) # Ensure the latest monitor files are in place +print """ +****************************************************************************** +Ensuring %s exists... +""" % ( SHARE_EDDN_FILES ) old_cwd = os.getcwd() if not os.path.isdir(SHARE_EDDN_FILES): # We're still using Python 2.7, so no pathlib @@ -111,6 +119,10 @@ if not os.path.isdir(SHARE_EDDN_FILES): exit(-1) os.chdir(old_cwd) +print """ +****************************************************************************** +Ensuring latest monitor files are in place... +""" # Copy the monitor (Web page) files try: shutil.rmtree('%s/monitor' % ( SHARE_EDDN_FILES )) @@ -118,6 +130,10 @@ except OSError: pass shutil.copytree('contrib/monitor', '%s/monitor' % ( SHARE_EDDN_FILES )) # And a copy of the schemas too +print """ +****************************************************************************** +Ensuring latest schema files are in place for web access... +""" try: shutil.rmtree('%s/schemas' % ( SHARE_EDDN_FILES )) except OSError: From c6db4b27334205dc0e977725ba5059fca0cfcaa4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 16:52:04 +0100 Subject: [PATCH 11/39] docs/Running: ${HOME} not ~ / python venv un-versioned symbolic link * Let's stick with Bourne Shell, not BASH-isms. * If we make everything use ~/eddn/python-venv symbolic link then there's less to update in the future. --- docs/Running-this-software.md | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 6254a9a..1acfe19 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -58,7 +58,7 @@ If you don't yet have nginx installed then start with: #### LetsEncrypt TLS Certificates -You will need a LetsEncrupt/ACME client in order to keep the TLS certificate +You will need a LetsEncrypt/ACME client in order to keep the TLS certificate renewed. cd /etc/letsencrypt @@ -120,23 +120,29 @@ So as to not have any python package version requirements clash with anything else it's best to use a Python virtual environment (venv). You will have installed the Debian package 'virtualenv' above for this purpose. -We'll put the venv in `~/eddn/python2.7-venv` with the following +We'll put the venv in `${HOME}/eddn/python2.7-venv` with the following command: + mkdir -p ${HOME}/eddn + cd ${HOME}/eddn virtualenv -p /usr/bin/python2.7 ${HOME}/python2.7-venv +And for future ease of changing python versions: + + ln -s python2.7-venv python-venv + And now start using this venv: - . python2.7-venv/bin/activate + . python-venv/bin/activate ### Clone a copy of the application project from gitub - mkdir -p ~/eddn/dev - cd ~/eddn/dev + mkdir -p ${HOME}/eddn/dev + cd ${HOME}/eddn/dev git clone https://github.com/EDCD/EDDN.git cd EDDN -We'll assume this `~/eddn/dev/EDDN` path elsewhere in this document. +We'll assume this `${HOME}/eddn/dev/EDDN` path elsewhere in this document. ### Ensure necessary python modules are installed Installing extra necessary python modules is simple: @@ -146,7 +152,7 @@ Installing extra necessary python modules is simple: ### Initialise Database Schema You will need to get the database schema in place: - mysql -p eddn < ~/eddn/dev/EDDN/schema.sql + mysql -p eddn < ${HOME}/eddn/dev/EDDN/schema.sql ### Monitor and Schema files @@ -155,14 +161,14 @@ script, so you'll need to manually copy them into somewhere convenient, e.g.: mkdir -p ${HOME}/.local/share/eddn - cp -r ~/eddn/dev/EDDN/contrib/monitor ${HOME}/.local/share/eddn + cp -r ${HOME}/eddn/dev/EDDN/contrib/monitor ${HOME}/.local/share/eddn chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn You will need to ensure that the Monitor nginx setup can see the schema files in order to serve them for use by the Gateway. So perform, e.g.: mkdir -p ${HOME}/.local/share/eddn - cp -r ~/eddn/dev/EDDN/schemas ${HOME}/.local/share/eddn + cp -r ${HOME}/eddn/dev/EDDN/schemas ${HOME}/.local/share/eddn chmod -R og+rX ${HOME}/.local/share/eddn/schemas # Concepts @@ -280,7 +286,7 @@ You have some choices for how to run the application components: python setup.py install --user - to install under `~/.local/` instead. + to install under `${HOME}/.local/` instead. There is an example systemd setup in `contrib/systemd` that assumes this local installation. @@ -290,12 +296,12 @@ You have some choices for how to run the application components: `DAEMON` lines tweaking for running from another location. 1. For quick testing purposes you can run them as follows, assuming you - installed into `~/.local/`, and have your override settings in + installed into `${HOME}/.local/`, and have your override settings in `${HOME}/etc/eddn-settings-overrides.json`: - ~/.local/bin/eddn-gateway --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-gateway.log 2>&1 & - ~/.local/bin/eddn-monitor --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-monitor.log 2>&1 & - ~/.local/bin/eddn-relay --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-relay.log 2>&1 & + ${HOME}/.local/bin/eddn-gateway --config ${HOME}/etc/eddn-settings-overrides.json >> ${HOME}/logs/eddn-gateway.log 2>&1 & + ${HOME}/.local/bin/eddn-monitor --config ${HOME}/etc/eddn-settings-overrides.json >> ${HOME}/logs/eddn-monitor.log 2>&1 & + ${HOME}/.local/bin/eddn-relay --config ${HOME}/etc/eddn-settings-overrides.json >> ${HOME}/logs/eddn-relay.log 2>&1 & # Accessing the Monitor There is an EDDN Status web page usually provided at, e.g. From 6bc7d3042d9540ebf43fa85b39d1fd3e4a52e86b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 17:08:47 +0100 Subject: [PATCH 12/39] contrib/init.d: Tweak DAEMON setting for python venv install --- contrib/init.d/eddn-gateway | 4 ++-- contrib/init.d/eddn-monitor | 4 ++-- contrib/init.d/eddn-relay | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/init.d/eddn-gateway b/contrib/init.d/eddn-gateway index 3b87b73..cd90dde 100644 --- a/contrib/init.d/eddn-gateway +++ b/contrib/init.d/eddn-gateway @@ -13,7 +13,7 @@ DESC="eddn-gateway" PIDFILE="/var/run/${NAME}.pid" LOGFILE="/var/log/eddn/${NAME}.log" -DAEMON="/usr/local/bin/${NAME}" +DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}" EXEC_AS_USER="root" @@ -50,4 +50,4 @@ restart|force-reload) ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/contrib/init.d/eddn-monitor b/contrib/init.d/eddn-monitor index 2d73441..ec5727c 100644 --- a/contrib/init.d/eddn-monitor +++ b/contrib/init.d/eddn-monitor @@ -13,7 +13,7 @@ DESC="eddn-monitor" PIDFILE="/var/run/${NAME}.pid" LOGFILE="/var/log/eddn/${NAME}.log" -DAEMON="/usr/local/bin/${NAME}" +DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}" EXEC_AS_USER="root" @@ -50,4 +50,4 @@ restart|force-reload) ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/contrib/init.d/eddn-relay b/contrib/init.d/eddn-relay index 24b3d79..bd9bcff 100644 --- a/contrib/init.d/eddn-relay +++ b/contrib/init.d/eddn-relay @@ -13,7 +13,7 @@ DESC="eddn-relay" PIDFILE="/var/run/${NAME}.pid" LOGFILE="/var/log/eddn/${NAME}.log" -DAEMON="/usr/local/bin/${NAME}" +DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}" EXEC_AS_USER="root" @@ -50,4 +50,4 @@ restart|force-reload) ;; esac -exit 0 \ No newline at end of file +exit 0 From db7b69473b55b6efa34d3a56fe50be3c12a331e1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 17:12:28 +0100 Subject: [PATCH 13/39] docs/: Add example override config file --- docs/config-EXAMPLE.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 docs/config-EXAMPLE.json diff --git a/docs/config-EXAMPLE.json b/docs/config-EXAMPLE.json new file mode 100644 index 0000000..45e81bf --- /dev/null +++ b/docs/config-EXAMPLE.json @@ -0,0 +1,12 @@ +{ + "CERT_FILE": "/home/eddn/etc/fullchain.pem", + "KEY_FILE": "/home/eddn/etc/privkey.pem", + + "GATEWAY_HTTP_BIND_ADDRESS": "0.0.0.0", + + "MONITOR_DB": { + "database": "eddn", + "user": "eddn", + "password": "SOME SECURE PASSWORD" + }, +} From d16e222a99148f16290aa1c6bd521754046d549f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 17:12:43 +0100 Subject: [PATCH 14/39] docs/Running: Mandate the python venv installation Let's stop guessing at how the reader wants to do it and just *tell* them how to do it instead. Python virtual environment, done. --- docs/Running-this-software.md | 59 +++++++++++++++-------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 1acfe19..1aa569e 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -155,22 +155,6 @@ You will need to get the database schema in place: mysql -p eddn < ${HOME}/eddn/dev/EDDN/schema.sql -### Monitor and Schema files -The Monitor files are not currently installed anywhere by the `setup.py` -script, so you'll need to manually copy them into somewhere convenient, -e.g.: - - mkdir -p ${HOME}/.local/share/eddn - cp -r ${HOME}/eddn/dev/EDDN/contrib/monitor ${HOME}/.local/share/eddn - chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn - -You will need to ensure that the Monitor nginx setup can see the schema files -in order to serve them for use by the Gateway. So perform, e.g.: - - mkdir -p ${HOME}/.local/share/eddn - cp -r ${HOME}/eddn/dev/EDDN/schemas ${HOME}/.local/share/eddn - chmod -R og+rX ${HOME}/.local/share/eddn/schemas - # Concepts There are three components to this application. @@ -277,32 +261,27 @@ It sets: # Running You have some choices for how to run the application components: -1. You can choose to run this application directly from the source using the - provided script in `contrib/run-from-source.sh`. +1. If you are just testing out code changes then you can choose to run + this application directly from the source using the provided script in + `contrib/run-from-source.sh`. -1. Or you can utilise the `setup.py` file to build and install the application - files. By default this requires write permissions under `/usr/local`, but - you can run: +1. Otherwise you will want to utilise the `setup.py` file to build and + install the application files. As we're using a python venv we can just + run: - python setup.py install --user + python setup.py install - to install under `${HOME}/.local/` instead. + to install it all. This will install a python egg into the python + venv, and then also ensure that the monitor and schema files are in + place. There is an example systemd setup in `contrib/systemd` that assumes this local installation. - If you install into `/usr/local/` then there are SysV style init.d scripts - in `contrib/init.d/` for running the components. They will need the - `DAEMON` lines tweaking for running from another location. + There are also some SysV style init.d scripts in `contrib/init.d/` for + running the components. They will need the `DAEMON` lines tweaking for + running from another location. -1. For quick testing purposes you can run them as follows, assuming you - installed into `${HOME}/.local/`, and have your override settings in - `${HOME}/etc/eddn-settings-overrides.json`: - - ${HOME}/.local/bin/eddn-gateway --config ${HOME}/etc/eddn-settings-overrides.json >> ${HOME}/logs/eddn-gateway.log 2>&1 & - ${HOME}/.local/bin/eddn-monitor --config ${HOME}/etc/eddn-settings-overrides.json >> ${HOME}/logs/eddn-monitor.log 2>&1 & - ${HOME}/.local/bin/eddn-relay --config ${HOME}/etc/eddn-settings-overrides.json >> ${HOME}/logs/eddn-relay.log 2>&1 & - # Accessing the Monitor There is an EDDN Status web page usually provided at, e.g. https://eddn.edcd.io/. This is enabled by the Monitor component through @@ -312,6 +291,18 @@ by the Monitor process itself. You will need to configure a reverse proxy to actually enable access to this. There is an example nginx configuration in `contrib/nginx-eddn.conf`. +The necessary files should be put in place by + +The 'monitor' files are what form the status/statistics page at +https://eddn.edcd.io/, so they need to be installed somewhere in a +static manner accessible to nginx. + +Although setup.py installs the files you might still need to ensure the +permissions are correct for your web server to access them. + + chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn + chmod -R og+rX ${HOME}/.local/share/eddn/schemas + ## Testing all of this in a VM In order to test all of this in a VM you might need to set up a double proxying: From f98809fd2e9e55fe1aa40cddcf4037067f0bf946 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 17:32:13 +0100 Subject: [PATCH 15/39] systemd: Correct AssertPathExists to python venv --- contrib/systemd/eddn@.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/systemd/eddn@.service b/contrib/systemd/eddn@.service index 7779172..9ef01de 100644 --- a/contrib/systemd/eddn@.service +++ b/contrib/systemd/eddn@.service @@ -7,7 +7,7 @@ [Unit] Description=EDDN Service %i -AssertPathExists=/home/eddn/.local/bin/%i +AssertPathExists=/home/eddn/eddn/python-venv/bin/%i PartOf=eddn.service ReloadPropagatedFrom=eddn.service Before=eddn.service From 2c06b8e79e202ed56b3a932fe881173d1501ba37 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 14 Jun 2021 17:34:34 +0100 Subject: [PATCH 16/39] systemd: start script - exec the python --- contrib/systemd/start-eddn-service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/systemd/start-eddn-service b/contrib/systemd/start-eddn-service index 63fa806..38d8c7d 100755 --- a/contrib/systemd/start-eddn-service +++ b/contrib/systemd/start-eddn-service @@ -34,4 +34,4 @@ then exit 2 fi -${PYTHON_VENV}/bin/${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1 +exec ${PYTHON_VENV}/bin/${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1 From 451b93db1d41c9971e8fb7a584268e6fe0d4cb1b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 1 Jul 2021 15:41:44 +0100 Subject: [PATCH 17/39] docs/Running: Correct placement of "Reverse Proxy with nginx" section --- docs/Running-this-software.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 1aa569e..4712429 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -51,11 +51,6 @@ alternative ACME client). apt install certbot -### Reverse Proxy with nginx -If you don't yet have nginx installed then start with: - - apt install nginx-light - #### LetsEncrypt TLS Certificates You will need a LetsEncrypt/ACME client in order to keep the TLS certificate @@ -74,6 +69,11 @@ renewed. ln -s ../../archive/eddn.edcd.io/fullchain1.pem fullchain.pem ln -s ../../archive/eddn.edcd.io/privkey1.pem privkey.pem +### Reverse Proxy with nginx +If you don't yet have nginx installed then start with: + + apt install nginx-light + #### nginx configuration There is an example configuration in `contrib/nginx-eddn.conf` which makes some assumptions: From 086493b5bffbbc3c8f84feb035e5d2471848fd17 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 1 Jul 2021 16:52:25 +0000 Subject: [PATCH 18/39] Make it possible to split dev and live installs * Have 'dev' and 'live' components to many paths and file names. * Have separate dev/live systemd start script configs. * Have the systemd start script take a dev/live second argument. * Try to ensure permissions on 'web' files by explicitly setting umask. --- contrib/systemd/eddn_config | 3 --- contrib/systemd/eddn_dev_config | 3 +++ contrib/systemd/eddn_live_config | 3 +++ contrib/systemd/start-eddn-service | 25 +++++++++++++++++++------ setup.py | 16 +++++++++++++--- 5 files changed, 38 insertions(+), 12 deletions(-) delete mode 100644 contrib/systemd/eddn_config create mode 100644 contrib/systemd/eddn_dev_config create mode 100644 contrib/systemd/eddn_live_config diff --git a/contrib/systemd/eddn_config b/contrib/systemd/eddn_config deleted file mode 100644 index 22d7121..0000000 --- a/contrib/systemd/eddn_config +++ /dev/null @@ -1,3 +0,0 @@ -CONFIG_OVERRIDE="${HOME}/.local/share/eddn/config.json" -LOG_DIR="${HOME}/.var/log/eddn" -PYTHON_VENV="${HOME}/eddn/python-venv" diff --git a/contrib/systemd/eddn_dev_config b/contrib/systemd/eddn_dev_config new file mode 100644 index 0000000..cdfdee2 --- /dev/null +++ b/contrib/systemd/eddn_dev_config @@ -0,0 +1,3 @@ +CONFIG_OVERRIDE="${HOME}/.local/share/eddn/dev/config.json" +LOG_DIR="${HOME}/dev/logs" +PYTHON_VENV="${HOME}/dev/python-venv" diff --git a/contrib/systemd/eddn_live_config b/contrib/systemd/eddn_live_config new file mode 100644 index 0000000..1ce0827 --- /dev/null +++ b/contrib/systemd/eddn_live_config @@ -0,0 +1,3 @@ +CONFIG_OVERRIDE="${HOME}/.local/share/eddn/live/config.json" +LOG_DIR="${HOME}/live/logs" +PYTHON_VENV="${HOME}/live/python-venv" diff --git a/contrib/systemd/start-eddn-service b/contrib/systemd/start-eddn-service index 38d8c7d..a252872 100755 --- a/contrib/systemd/start-eddn-service +++ b/contrib/systemd/start-eddn-service @@ -3,13 +3,26 @@ # # Start an EDDN Service, including redirecting output to a log file. +usage() { + echo "Usage: $(basename $0) [ gateway | monitor | relay ] [ dev | live ]" +} + if [ -z "${1}" ]; then - echo "No EDDN service specified. One of: ${SERVICE}, monitor, relay" + usage + echo "No EDDN service specified." exit 3 fi SERVICE="${1}" +if [ -z "${2}" ]; +then + usage + echo "No EDDN environment specified." + exit 3 +fi +EDDN_ENV="${2}" + EXEC_PATH=$(dirname $0) #echo "EXEC_PATH: ${EXEC_PATH}" @@ -18,20 +31,20 @@ cd ${EXEC_PATH} #pwd # Bring in some common configuration -if [ ! -f eddn_config ]; +if [ ! -f "eddn_${EDDN_ENV}_config" ]; then - echo "eddn_config is missing from $(pwd)" + echo "eddn_${EDDN_ENV}_config is missing from $(pwd)" exit 1 fi -. ./eddn_config +. "./eddn_${EDDN_ENV}_config" # Use the python venv . "${PYTHON_VENV}/bin/activate" -if [ ! -f "${PYTHON_VENV}/bin/${SERVICE}" ]; +if [ ! -f "${PYTHON_VENV}/bin/eddn-${SERVICE}" ]; then echo "${SERVICE} is missing from ${PYTHON_VENV}/bin" exit 2 fi -exec ${PYTHON_VENV}/bin/${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1 +exec ${PYTHON_VENV}/bin/eddn-${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1 diff --git a/setup.py b/setup.py index 8b01793..c651055 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,11 @@ except EnvironmentError: print "unable to find version in %s" % (VERSIONFILE,) raise RuntimeError("if %s exists, it is required to be well-formed" % (VERSIONFILE,)) +EDDN_ENV="dev" # Location of start-eddn-service script and its config file START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] ) # Location of web files -SHARE_EDDN_FILES='%s/.local/share/eddn' % ( os.environ['HOME'] ) +SHARE_EDDN_FILES='%s/.local/share/eddn/%s' % ( os.environ['HOME'], EDDN_ENV ) setup( name='eddn', @@ -93,10 +94,18 @@ if not os.path.isdir(START_SCRIPT_BIN): exit(-1) os.chdir(old_cwd) -for f in ( 'contrib/systemd/start-eddn-service', 'contrib/systemd/eddn_config'): - shutil.copy(f, START_SCRIPT_BIN) + +shutil.copy( + 'contrib/systemd/eddn_%s_config' % ( EDDN_ENV), + '%s/eddn_%s_config' % ( START_SCRIPT_BIN, EDDN_ENV ) +) +shutil.copy( + 'contrib/systemd/start-eddn-service', + '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, EDDN_ENV ) +) # Ensure the latest monitor files are in place +old_umask = os.umask(022) print """ ****************************************************************************** Ensuring %s exists... @@ -155,3 +164,4 @@ software will actually work. See docs/Running-this-software.md for guidance. ****************************************************************************** """ % ( SHARE_EDDN_FILES ) +os.umask(old_umask) From 4e343fe4446658be933f316bad955be005fd133c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 1 Jul 2021 18:11:13 +0100 Subject: [PATCH 19/39] docs/running: a start on apache config --- docs/Running-this-software.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 4712429..6dd9df9 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -1,3 +1,5 @@ + These instructions are based on getting the software up and running from scratch on a Debian Buster (10.9, stable as of 2021-05-16) system. @@ -69,6 +71,16 @@ renewed. ln -s ../../archive/eddn.edcd.io/fullchain1.pem fullchain.pem ln -s ../../archive/eddn.edcd.io/privkey1.pem privkey.pem +### Reverse Proxy with Apache +If you already have an Apache installation it will be easier to just use +it for the reverse proxy. + +Ensure you have the module installed and active: + + a2enmod proxy + +#### Apache configuration + ### Reverse Proxy with nginx If you don't yet have nginx installed then start with: From 7684cec4d5b5a810e4a033329b377c70c927724c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 1 Jul 2021 17:30:37 +0000 Subject: [PATCH 20/39] .gitignore: VIM swap files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index fdebc55..08c8a19 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,5 @@ target/ # Editors .idea +# VIM swap files +.*.sw? From 096ccab9fb0580dced1713910775be9412e0903c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 1 Jul 2021 17:30:48 +0000 Subject: [PATCH 21/39] docs/Running: Outline for using Apache --- contrib/apache-eddn.conf | 97 +++++++++++++++++++++++++++++++++++ docs/Running-this-software.md | 40 +++++++++------ 2 files changed, 121 insertions(+), 16 deletions(-) create mode 100644 contrib/apache-eddn.conf diff --git a/contrib/apache-eddn.conf b/contrib/apache-eddn.conf new file mode 100644 index 0000000..4428f5f --- /dev/null +++ b/contrib/apache-eddn.conf @@ -0,0 +1,97 @@ +# vim: :filetype=apache +## dev.eddn.edcd.io + + ServerName dev.eddn.edcd.io + + DocumentRoot //home/eddn/.local/share/eddn/dev + + ErrorLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/access.log combined + + Redirect / https://dev.eddn.edcd.io/ + RedirectMatch "/^(.*)$" "https://dev.eddn.edcd.io/$1" + + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + + + + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + AllowOverride All + + Require all granted + + + Require all denied + + + Include partials/default-directory.conf + + + + + + + SSLEngine On + SSLCertificateFile /etc/letsencrypt/live/dev.eddn.edcd.io/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/dev.eddn.edcd.io/privkey.pem + + ServerName dev.eddn.edcd.io + + DocumentRoot /home/eddn/.local/share/eddn/dev/monitor + + ErrorLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/access.log combined + + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + + + + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + AllowOverride All + + Require all granted + + + Require all denied + + + + # Serve the schemas + Alias /schemas/ /home/eddn/.local/share/eddn/dev/schemas/ + + # netdata (performance info) + + Redirect /netdata /netdata/ + + + SetOutputFilter DEFLATE + + + Require all granted + + + Require all denied + + + + SSLProxyEngine On + SSLProxyVerify none + ProxyPreserveHost On + + ProxyPass "/netdata/" "http://127.0.0.1:19999/" + + + + diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 6dd9df9..8dc723f 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -80,6 +80,28 @@ Ensure you have the module installed and active: a2enmod proxy #### Apache configuration +There is an example VirtualHost configuration in +`contrib/apache-eddn.conf` which makes the following assumptions: + + 1. The usual Apache default configuration is in place elsewhere. + 1. The hostname being used - `ServerName`. + 1. The location of the monitor files - `DocumentRoot`. + 1. The location of the schema files - `Alias /schemas/ ...`. + 1. The location of the TLS certificate files - `SSLCertificateFile` and + `SSLCertificateKeyFile. + +You should be able to: + + 1. Copy `contrib/apache-eddn.conf` into `/etc/apache/sites-available/` + *as an appropriate filename for the hostname you're using*. + 1. Edit to suit the local situation/setup. **Remember to ensure the + configured log directory exists.** + 1. Enable the site: + + a2ensite + apache2ctl configtest + # CHECK THE OUTPUT + apache2ctl graceful ### Reverse Proxy with nginx If you don't yet have nginx installed then start with: @@ -95,7 +117,7 @@ some assumptions: 1. The location of the monitor files - `root` directive. 1. The location of the schema files - `location` directive. 1. The location of the TLS certificate files - `ssl_certificate` and - `ssl_certificate_key` directives. + `ssl_certificate_key` directives. You should be able to: @@ -107,24 +129,10 @@ You should be able to: ln -s /etc/nginx/sites-available/eddn systemctl restart nginx.service -If you're already using another web server, such as Apache, you'll need to +If you're already using another web server you'll need to duplicate at least the use of a TLS certificate and the Reverse Proxying as required. -For Apache you would reverse proxy using something like the following in an -appropriate `` section: - - - SSLProxyEngine On - SSLProxyVerify none - ProxyPreserveHost On - - # Pass through 'gateway' upload URL to Debian VM - ProxyPass "/upload/" "https://EDDNHOST:8081/upload/" - # Pass through 'monitor' URLs to Debian VM - ProxyPass "/" "https://EDDNHOST/" - - ## In the 'eddn' account ### Set up a python virtual environment From 2a4f8488cd7b2dee95b6cc44480a0b72911f50e3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 13:55:25 +0000 Subject: [PATCH 22/39] Remove setup.cfg from git, to avoid merges over-writing any per-env changes The hope is this can be used to easily switch between live, beta and dev for setup.py without worrying that a git merge will suddenly turn beta into dev, or live into beta or dev. --- .gitignore | 4 ++++ setup.cfg | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 08c8a19..c3ce6ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Hoping this can be a per environment config, so needs to not be merged +# by git, so not *in* git. +setup.cfg + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a35e2cd..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[egg_info] -tag_build = .dev \ No newline at end of file From 18047995070cfb1a7e92355a214be742049fb54a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 14:12:43 +0000 Subject: [PATCH 23/39] setup: Use setup_env.py for per-environment settings --- .gitignore | 4 ++-- setup.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index c3ce6ef..9bff30b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -# Hoping this can be a per environment config, so needs to not be merged -# by git, so not *in* git. +# These need to be per-environment, so not changed by git merges. setup.cfg +setup_env.py # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/setup.py b/setup.py index c651055..127c984 100644 --- a/setup.py +++ b/setup.py @@ -17,11 +17,13 @@ except EnvironmentError: print "unable to find version in %s" % (VERSIONFILE,) raise RuntimeError("if %s exists, it is required to be well-formed" % (VERSIONFILE,)) -EDDN_ENV="dev" +# Read environment-specific settings +import setup_env + # Location of start-eddn-service script and its config file START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] ) # Location of web files -SHARE_EDDN_FILES='%s/.local/share/eddn/%s' % ( os.environ['HOME'], EDDN_ENV ) +SHARE_EDDN_FILES='%s/.local/share/eddn/%s' % ( os.environ['HOME'], setup_env.EDDN_ENV ) setup( name='eddn', @@ -96,12 +98,12 @@ if not os.path.isdir(START_SCRIPT_BIN): os.chdir(old_cwd) shutil.copy( - 'contrib/systemd/eddn_%s_config' % ( EDDN_ENV), - '%s/eddn_%s_config' % ( START_SCRIPT_BIN, EDDN_ENV ) + 'contrib/systemd/eddn_%s_config' % ( setup_env.EDDN_ENV), + '%s/eddn_%s_config' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) ) shutil.copy( 'contrib/systemd/start-eddn-service', - '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, EDDN_ENV ) + '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) ) # Ensure the latest monitor files are in place From a340b027fce0d08a7d7c1f8e149b24b380a499a4 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 14:16:36 +0000 Subject: [PATCH 24/39] beta: Add systemd config --- contrib/systemd/eddn_beta_config | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contrib/systemd/eddn_beta_config diff --git a/contrib/systemd/eddn_beta_config b/contrib/systemd/eddn_beta_config new file mode 100644 index 0000000..4677abc --- /dev/null +++ b/contrib/systemd/eddn_beta_config @@ -0,0 +1,3 @@ +CONFIG_OVERRIDE="${HOME}/.local/share/eddn/beta/config.json" +LOG_DIR="${HOME}/beta/logs" +PYTHON_VENV="${HOME}/beta/python-venv" From c72f583476f0c9dfaf4c46123626954e50299fe8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 14:17:21 +0000 Subject: [PATCH 25/39] beta: Support in systemd start-eddn-service script --- contrib/systemd/start-eddn-service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/systemd/start-eddn-service b/contrib/systemd/start-eddn-service index a252872..4f27af1 100755 --- a/contrib/systemd/start-eddn-service +++ b/contrib/systemd/start-eddn-service @@ -4,7 +4,7 @@ # Start an EDDN Service, including redirecting output to a log file. usage() { - echo "Usage: $(basename $0) [ gateway | monitor | relay ] [ dev | live ]" + echo "Usage: $(basename $0) [ gateway | monitor | relay ] [ live | beta | dev ]" } if [ -z "${1}" ]; From 2eb8a9fb8806ba010402635d2a163dfd157a5ea0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 16:32:19 +0000 Subject: [PATCH 26/39] Gateway: Implement a CORS plugin for all routes * Switches all of the routes to use .route(..., method=[...]) so as to include OPTIONS as well as either POST or GET. --- src/eddn/Gateway.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 1531156..3eed61f 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -159,7 +159,7 @@ def parse_and_error_handle(data): return "FAIL: " + str(validationResults.messages) -@app.post('/upload/') +@app.route('/upload/', methods=['OPTIONS', 'POST']) def upload(): response.set_header("Access-Control-Allow-Origin", "*") try: @@ -182,7 +182,7 @@ def upload(): return parse_and_error_handle(message_body) -@app.get('/health_check/') +@app.route('/health_check/', method=['OPTIONS', 'GET']) def health_check(): """ This should only be used by the gateway monitoring script. It is used @@ -192,9 +192,8 @@ def health_check(): return Settings.EDDN_VERSION -@app.get('/stats/') +@app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): - response.set_header("Access-Control-Allow-Origin", "*") stats = statsCollector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) @@ -209,9 +208,29 @@ class MalformedUploadError(Exception): pass +class EnableCors(object): + name = 'enable_cors' + api = 2 + + def apply(self, fn, context): + def _enable_cors(*args, **kwargs): + # set CORS headers + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' + response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + + if request.method != 'OPTIONS': + # actual request; reply with the actual response + return fn(*args, **kwargs) + + return _enable_cors + + def main(): loadConfig() configure() + + app.install(EnableCors()) app.run( host=Settings.GATEWAY_HTTP_BIND_ADDRESS, port=Settings.GATEWAY_HTTP_PORT, From 1feb7c2deec1ac3ca09e1b00f9bdb660c37256db Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 16:54:37 +0000 Subject: [PATCH 27/39] Monitor: Converted to app. including CORS handler --- src/eddn/Monitor.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 0d9cc3d..dc8150c 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -13,11 +13,12 @@ import collections import zmq.green as zmq import re -from bottle import get, request, response, run as bottle_run from eddn.conf.Settings import Settings, loadConfig from gevent import monkey monkey.patch_all() +from bottle import Bottle, get, request, response, run +app = Bottle() # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: @@ -31,12 +32,12 @@ def date(__format): return d.strftime(__format) -@get('/ping') +@app.route('/ping', method=['OPTIONS', 'GET']) def ping(): return 'pong' -@get('/getTotalSoftwares/') +@app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET']) def getTotalSoftwares(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -62,7 +63,7 @@ def getTotalSoftwares(): return simplejson.dumps(softwares) -@get('/getSoftwares/') +@app.route('/getSoftwares/', method=['OPTIONS', 'GET']) def getSoftwares(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -91,7 +92,7 @@ def getSoftwares(): return simplejson.dumps(softwares) -@get('/getTotalSchemas/') +@app.route('/getTotalSchemas/', method=['OPTIONS', 'GET']) def getTotalSchemas(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -113,7 +114,7 @@ def getTotalSchemas(): return simplejson.dumps(schemas) -@get('/getSchemas/') +@app.route('/getSchemas/', method=['OPTIONS', 'GET']) def getSchemas(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -211,11 +212,35 @@ class Monitor(Thread): gevent.spawn(monitor_worker, inboundMessage) +class EnableCors(object): + """Enable CORS responses.""" + + name = 'enable_cors' + api = 2 + + def apply(self, fn, context): + """ + Apply a CORS handler. + + Ref: + """ + def _enable_cors(*args, **kwargs): + """Set CORS Headers.""" + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' + response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + + if request.method != 'OPTIONS': + # actual request; reply with the actual response + return fn(*args, **kwargs) + + return _enable_cors + def main(): loadConfig() m = Monitor() m.start() - bottle_run( + app.run( host=Settings.MONITOR_HTTP_BIND_ADDRESS, port=Settings.MONITOR_HTTP_PORT, server='gevent', From a8b3fc33e114be183a51bbf1db33226c7aae6ed9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 16:57:06 +0000 Subject: [PATCH 28/39] Monitor: Forgot the `app.install(EnableCors())` --- src/eddn/Monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index dc8150c..4c188d4 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -240,6 +240,7 @@ def main(): loadConfig() m = Monitor() m.start() + app.install(EnableCors()) app.run( host=Settings.MONITOR_HTTP_BIND_ADDRESS, port=Settings.MONITOR_HTTP_PORT, From 9359b783018b87eec257fed7f31aaaaac62484f1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 17:02:07 +0000 Subject: [PATCH 29/39] Gateway: Correct methodS to singular & remove extraneous header * methods != method * There shouldn't be any need to manually set Access-Control-Allow-Origin headers now. --- src/eddn/Gateway.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 3eed61f..63d10d7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -159,9 +159,8 @@ def parse_and_error_handle(data): return "FAIL: " + str(validationResults.messages) -@app.route('/upload/', methods=['OPTIONS', 'POST']) +@app.route('/upload/', method=['OPTIONS', 'POST']) def upload(): - response.set_header("Access-Control-Allow-Origin", "*") try: # Body may or may not be compressed. message_body = get_decompressed_message() From 1e9edf94f725d371a4dc33f85934aa8ea6d250a2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 17:02:53 +0000 Subject: [PATCH 30/39] Relay: app.-ise and CORS-ise * Use the "make an explicit Bottle() and use it" change. * Use app.route with OPTIONS in method= list. * Remove extraneous setting of Access-Control-Allow-Origin header. --- src/eddn/Relay.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index f69b495..9843c79 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -18,11 +18,12 @@ import simplejson import hashlib import uuid import zmq.green as zmq -from bottle import get, response, run as bottle_run from eddn.conf.Settings import Settings, loadConfig from gevent import monkey monkey.patch_all() +from bottle import Bottle, get, response, run +app = Bottle() # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector @@ -36,9 +37,8 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicateMessages.start() -@get('/stats/') +@app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): - response.set_header("Access-Control-Allow-Origin", "*") stats = statsCollector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) @@ -145,11 +145,38 @@ class Relay(Thread): gevent.spawn(relay_worker, inboundMessage) +class EnableCors(object): + """Enable CORS responses.""" + + name = 'enable_cors' + api = 2 + + def apply(self, fn, context): + """ + Apply a CORS handler. + + Ref: + """ + def _enable_cors(*args, **kwargs): + """Set CORS Headers.""" + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' + response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + + if request.method != 'OPTIONS': + # actual request; reply with the actual response + return fn(*args, **kwargs) + + return _enable_cors + + def main(): loadConfig() r = Relay() r.start() - bottle_run( + + app.install(EnableCors()) + app.run( host=Settings.RELAY_HTTP_BIND_ADDRESS, port=Settings.RELAY_HTTP_PORT, server='gevent', From 1df2fb749b716b0e140ee763d338927fe1769587 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 17:07:46 +0000 Subject: [PATCH 31/39] Relay: Add missing `from bottle import ..., request, ...` --- src/eddn/Relay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index 9843c79..10629d4 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -22,7 +22,7 @@ from eddn.conf.Settings import Settings, loadConfig from gevent import monkey monkey.patch_all() -from bottle import Bottle, get, response, run +from bottle import Bottle, get, request, response, run app = Bottle() # This import must be done post-monkey-patching! From de761f653b51b7792009568aceab72fedf55fb89 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 17:34:29 +0000 Subject: [PATCH 32/39] contrib: Add LetsEncrypt example deploy script files --- contrib/letsencrypt/certbot-common | 35 ++++++++++ contrib/letsencrypt/deploy-changed-certs | 89 ++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 contrib/letsencrypt/certbot-common create mode 100755 contrib/letsencrypt/deploy-changed-certs diff --git a/contrib/letsencrypt/certbot-common b/contrib/letsencrypt/certbot-common new file mode 100644 index 0000000..23f8e47 --- /dev/null +++ b/contrib/letsencrypt/certbot-common @@ -0,0 +1,35 @@ +########################################################################### +# Copy a certificate's files into place, with appropriate ownership and +# mode. +# +# $1 - Name of certificate (i.e. letsencrypt directory names). +# $2 - Source Directory +# $3 - Destination filename for fullchain.pem +# $4 - Destination filename for privkey.pem +# $5 - File ownership to set (user:group) +# $6 - File mode to set (as passed to 'chmod') +########################################################################### +copy_cert() { + CERT_NAME="$1" + SRC_DIR="$2" + DST_FILE_FULLCHAIN="$3" + DST_FILE_PRIVKEY="$4" + CERT_NEW_OWNER="$5" + CERT_NEW_PERMS="$6" + + echo "${CERT_NAME}: Copying new files into place..." + + # Preserve only the mode as it should be 0600, and thus we won't + # temporarily open up the files for *all* users to read, + # BUT don't preserve the timestamp as we want it to be 'now' so + # that a `find ... -newer ` check works later. + cp -v --preserve=mode ${SRC_DIR}/fullchain.pem ${DST_FILE_FULLCHAIN} + cp -v --preserve=mode ${SRC_DIR}/privkey.pem ${DST_FILE_PRIVKEY} + chown -v ${CERT_NEW_OWNER} ${DST_FILE_FULLCHAIN} ${DST_FILE_PRIVKEY} + chmod -v ${CERT_NEW_PERMS} ${DST_FILE_FULLCHAIN} ${DST_FILE_PRIVKEY} + + echo "${CERT_NAME}: Copying new files into place DONE" +} +########################################################################### + +# vim: :set filetype=sh tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0 diff --git a/contrib/letsencrypt/deploy-changed-certs b/contrib/letsencrypt/deploy-changed-certs new file mode 100755 index 0000000..0986a54 --- /dev/null +++ b/contrib/letsencrypt/deploy-changed-certs @@ -0,0 +1,89 @@ +#!/bin/bash +# Add " -x" above to debug +# +# certbot deploy hook +# +# This should be triggered by being present in: +# +# /etc/letsencrypt/renewal-hooks/deploy/ +# +# It can be linked into the 'post' directory for testing with: +# +# certbot renew --dry-run +# +# which you might want to do because deploy hooks aren't run for that +# command. +# +# You can also just straight up run this script, including to get into place +# any certificate files it's configured for, but have never been deployed. + +# Paranoia re-enforcement of no group/other perms on created files +chmod -R og-rwx /etc/letsencrypt/archive + +echo "$0 - Running in: $(pwd)" +# Import common code and settings. +. /etc/scripts/certbot-common + +# As of 2021-07-02 and certbot 0.31.0 (current in Debian buster) +# there is **zero** information passed in (CL args or environment) to +# this hook. So we just need to check each potentially renewed +# certificate. + +########################################################################### +# MAIN_HOST_NAME +########################################################################### +CERT_NAME="MAIN_HOST_NAME" +# We're only interested if it's newer than when the files were last copied +SRC_DIR="/etc/letsencrypt/live/${CERT_NAME}" +DST_FILE_FULLCHAIN="/etc/exim4/exim.crt" +DST_FILE_PRIVKEY="/etc/exim4/exim.key" +CERT_NEW_OWNER="root:Debian-exim" +CERT_NEW_PERMS="440" + +############################################################# +# Needs to be in place for exim to use +############################################################# +# 'find' doesn't set exit status depending on if it found anything, that's +# for actual errors, so we test against the output. +if [ "$(find ${SRC_DIR} -newer ${DST_FILE_FULLCHAIN} -o -newer ${DST_FILE_PRIVKEY} )" != "" ]; +then + echo "${CERT_NAME}: (Re)new(ed) certificate..." + + copy_cert "${CERT_NAME}" "${SRC_DIR}" "${DST_FILE_FULLCHAIN}" "${DST_FILE_PRIVKEY}" "${CERT_NEW_OWNER}" "${CERT_NEW_PERMS}" + + echo "${CERT_NAME}: DONE" +fi +############################################################# + +########################################################################### + +########################################################################### +# eddn.edcd.io and related names +########################################################################### +CERT_NEW_OWNER="eddn:eddn" +CERT_NEW_PERMS="400" + +for eddn in eddn.edcd.io test.eddn.edcd.io staging.eddn.edcd.io ; +do + CERT_NAME="${eddn}" + SRC_DIR="/etc/letsencrypt/live/${CERT_NAME}" + DST_FILE_FULLCHAIN="/home/eddn/etc/${CERT_NAME}-fullchain.pem" + DST_FILE_PRIVKEY="/home/eddn/etc/${CERT_NAME}-privkey.pem" + + if [ -d "${SRC_DIR}" ]; + then + if [ ! -f "${DST_FILE_FULLCHAIN}" \ + -o ! -f "${DST_FILE_PRIVKEY}" \ + -o "$(find ${SRC_DIR} -newer ${DST_FILE_FULLCHAIN} -o -newer ${DST_FILE_PRIVKEY} )" != "" ]; + then + echo "${CERT_NAME}: (Re)New(ed) certificate..." + + copy_cert "${CERT_NAME}" "${SRC_DIR}" "${DST_FILE_FULLCHAIN}" "${DST_FILE_PRIVKEY}" "${CERT_NEW_OWNER}" "${CERT_NEW_PERMS}" + + echo "${CERT_NAME}: DONE" + fi + fi +done +########################################################################### + +# vim: tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0 From fcd316c7d4b504f1e69499859b4b6a9707588ace Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 2 Jul 2021 18:04:34 +0000 Subject: [PATCH 33/39] docs/Running: Edit and expand LetsEncrypt section --- docs/Running-this-software.md | 54 +++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 8dc723f..f0c7798 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -47,16 +47,25 @@ need to install netdata. On Debian-based systems: The default configuration should be all you need, listening on `127.0.0.1:19999`. -### LetsEncrypt: certbot -It will be necessary to renew the TLS certificate using certbot (or some -alternative ACME client). +### LetsEncrypt +We assume that you're using a TLS certificate from +[LetsEncrypt](https://letsencrypt.org/), it's free! + +It will be necessary to renew the TLS certificate using certbot, or some +alternative ACME client. We'll assume certbot. + +#### Install certbot +On a Debian system simply: apt install certbot -#### LetsEncrypt TLS Certificates +Although this version might be a little old now, it does work. -You will need a LetsEncrypt/ACME client in order to keep the TLS certificate -renewed. +#### LetsEncrypt TLS Certificates +If you are taking over hosting the EDDN relay then hopefully you have access +to the existing certificate files. + +So, first copy those into place: cd /etc/letsencrypt mkdir -p archive/eddn.edcd.io @@ -71,6 +80,39 @@ renewed. ln -s ../../archive/eddn.edcd.io/fullchain1.pem fullchain.pem ln -s ../../archive/eddn.edcd.io/privkey1.pem privkey.pem +After this you need to ensure that the certificate stays renewed. With a +Debian system using certbot: + +1. There should already be a systemd timer set up: + + `systemctl status certbot.timer` + + If that doesn't show "`; enabled;`" in: + + `Loaded: loaded (/lib/systemd/system/certbot.timer; enabled; vendor preset: enabled)` + + then: + + `systemctl enable certbot.timer` + + This will renew the certificate as necessary (i.e. when <= 30 days until + it expires, or whatever current LetsEncrypt and certbot policy causes). + But it will not ensure the files are in all the places you might need + them to be. + +1. Ensure the certificate files are deployed to where they're needed. When + using the certbot timer the easiest thing to do is to utilise a script in + `/etc/letsencrypt/renewal-hooks/deploy/`. + + There are example files for this in `contrib/letsencrypt/`: + + mkdir -p /etc/letsencrypt/renewal-hooks/deploy + cp contrib/letsencrypt/deploy-changed-certs /etc/letsencrypt/renewal-hooks/deploy + mkdir -p /etc/scripts + cp contrib/letsencrypt/certbot-common /etc/scripts/ + + **Remember to edit them to suit your setup!** + ### Reverse Proxy with Apache If you already have an Apache installation it will be easier to just use it for the reverse proxy. From b1056c12392af848ecc031fdd9e8ebb6f31a8a47 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 3 Jul 2021 14:02:35 +0000 Subject: [PATCH 34/39] contrib/run-from-source: Handle missing custom config --- contrib/run-from-source.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/contrib/run-from-source.sh b/contrib/run-from-source.sh index 871534c..85e587e 100644 --- a/contrib/run-from-source.sh +++ b/contrib/run-from-source.sh @@ -17,8 +17,17 @@ do echo "$d: Already running as $(cat ${LOGPATH}/${d}.pid)" continue fi + if [ -f "${BASEPATH}/etc/settings.json" ]; + then + CONFIG="--config ${BASEPATH}/etc/settings.json" + else + echo "WARNING: No override settings found, you'll be using defaults" + echo "WARNING: Did you forget to make ${BASEPATH}/etc/settings.json ?" + echo " Continuing anyway..." + CONFIG="" + fi ${PYTHON} -m eddn.${d} \ - --config ${BASEPATH}/etc/settings.json \ + ${CONFIG} \ > ${LOGPATH}/$d.log \ 2>&1 & echo $! > "${LOGPATH}/${d}.pid" From 0bcb9cc545d02800d42ddbc1163e616e026567d0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 5 Jul 2021 12:52:50 +0000 Subject: [PATCH 35/39] docs/Running: Specify network/firewall config, correct some paths and ... *
between sections * Document the ports that *must* be open to the internet. * Apache will require the proxy_http module. * Move the github clone earlier in instructions, as part of ensuring paths exist when they should. * Note about needing to do hostname and port substitutions in monitor files if not using the standard values. * Link to other sections where appropriate. --- docs/Running-this-software.md | 135 +++++++++++++++++++++++++++------- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index f0c7798..7fee97b 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -19,6 +19,8 @@ A specific user was created: useradd -c 'EDDN Gateway' -m -s /bin/bash eddn +--- + # Further installation ## As 'root' @@ -38,6 +40,8 @@ You will need a mysql/mariab database: > GRANT ALL PRIVILEGES on eddn.* TO 'eddn'@'localhost'; > \q +--- + ### Netdata In order to get host performance metrics (CPU, RAM and network usage) you will need to install netdata. On Debian-based systems: @@ -47,6 +51,8 @@ need to install netdata. On Debian-based systems: The default configuration should be all you need, listening on `127.0.0.1:19999`. +--- + ### LetsEncrypt We assume that you're using a TLS certificate from [LetsEncrypt](https://letsencrypt.org/), it's free! @@ -113,15 +119,63 @@ Debian system using certbot: **Remember to edit them to suit your setup!** -### Reverse Proxy with Apache +--- + +### Network Configuration +There are multiple ports that you'll have to ensure are allowed through any +firewall, and some of them also require being reverse proxied correctly. + +The reverse proxies pertain to: + +1. The port for the Gateway to receive uploads from senders (e.g. Elite + Dangerous Market Connector). This is also used for the 'monitor' web + page to obtain stats about messages passing through the Gateway. + +1. A set of URLs for accessing [netdata](#netdata). + +#### Necessary ports +These all for TCP, no UDP: + +1. `443` - a web server capable of reverse proxying set up for TLS on the + public host name of the EDDN service. This is used to serve the schemas, + the monitor web page, and to reverse proxy URLs beginning `/netdata/` to + the [netdata](#netdata) service. + +1. Default: `4430` - Gateway 'http' port, used both for EDDN senders to + upload, and also for the Gateway message rate stats on the monitor web + page. + + But that's the *public* port. The Gateway process itself listens on `8081`. + So you'll need a reverse proxy listening on port `4430` and forwarding + *all* requests to `127.0.0.1:8081`. + +1. Default: `9091` - Monitor 'http' port, used for the monitor web page to + query schema and software statistics. No reverse proxy setup. + +1. Default: `9500` - The port on the Relay that EDDN listeners connect to in + order to receive the zeromq stream. No reverse proxy setup. + +1. Default: `9090` - The Relay 'http' port for its portion of the message + statistics on the monitor web page. No reverse proxy setup. + +There's also the internal `8500` port, but that's literally only used for +the Monitor and Relay to pick up zeromq messages forwarded from the +Gateway, so all over localhost. + +See [Configuration](#configuration) for guidance on what override config +settings can be used to change any of these ports. + +--- + +#### Reverse Proxy with Apache If you already have an Apache installation it will be easier to just use it for the reverse proxy. -Ensure you have the module installed and active: +Ensure you have these modules installed and active: - a2enmod proxy + a2enmod proxy proxy_http -#### Apache configuration +##### Apache configuration There is an example VirtualHost configuration in `contrib/apache-eddn.conf` which makes the following assumptions: @@ -145,12 +199,14 @@ You should be able to: # CHECK THE OUTPUT apache2ctl graceful -### Reverse Proxy with nginx +--- + +#### Reverse Proxy with nginx If you don't yet have nginx installed then start with: apt install nginx-light -#### nginx configuration +##### nginx configuration There is an example configuration in `contrib/nginx-eddn.conf` which makes some assumptions: @@ -175,18 +231,29 @@ If you're already using another web server you'll need to duplicate at least the use of a TLS certificate and the Reverse Proxying as required. +--- + ## In the 'eddn' account +### Clone a copy of the application project from gitub + + mkdir -p ${HOME}/dev + cd ${HOME}/dev + git clone https://github.com/EDCD/EDDN.git EDDN.git + cd EDDN.git + +We'll assume this `${HOME}/dev/EDDN.git` path elsewhere in this document. + ### Set up a python virtual environment So as to not have any python package version requirements clash with anything else it's best to use a Python virtual environment (venv). You -will have installed the Debian package 'virtualenv' above for this purpose. +will have installed the Debian package 'virtualenv' [above](#as-root) for +this purpose. -We'll put the venv in `${HOME}/eddn/python2.7-venv` with the following +We'll put the venv in `${HOME}/dev/python2.7-venv` with the following command: - mkdir -p ${HOME}/eddn - cd ${HOME}/eddn + cd ${HOME}/dev virtualenv -p /usr/bin/python2.7 ${HOME}/python2.7-venv And for future ease of changing python versions: @@ -197,15 +264,6 @@ And now start using this venv: . python-venv/bin/activate -### Clone a copy of the application project from gitub - - mkdir -p ${HOME}/eddn/dev - cd ${HOME}/eddn/dev - git clone https://github.com/EDCD/EDDN.git - cd EDDN - -We'll assume this `${HOME}/eddn/dev/EDDN` path elsewhere in this document. - ### Ensure necessary python modules are installed Installing extra necessary python modules is simple: @@ -217,6 +275,10 @@ You will need to get the database schema in place: mysql -p eddn < ${HOME}/eddn/dev/EDDN/schema.sql +Ref: [As root](#as-root). + +--- + # Concepts There are three components to this application. @@ -248,7 +310,11 @@ test host. The files in question are: monitor/js/eddn.js monitor/schemas.html -Replace the string `eddn.edcd.io` with the hostname you're using. +Replace the string `eddn.edcd.io` with the hostname you're using. You'll need +to perform similar substitutions if you change the configuration to use any +different port numbers. + +--- # Configuration Default application configuration is in the file `src/eddn/conf/Settings.py`. @@ -258,36 +324,45 @@ another file. 1. You will need to obtain a TLS certificate from, e.g. LetsEncrypt. The application will need access to this and its private key file. - CERT_FILE = '/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem' - KEY_FILE = '/etc/letsencrypt/live/eddn.edcd.io/privkey.pem' + CERT_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem' + KEY_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem' 1. Network configuration 1. `RELAY_HTTP_BIND_ADDRESS` and `RELAY_HTTP_PORT` define the IP and port on which the Relay listens for, e.g. `/stats/` requests. + 1. `RELAY_RECEIVER_BINDINGS` defines where the Relay connects in order to subscribe to messages from the Gateway. Should match `GATEWAY_SENDER_BINDINGS`. + 1. `RELAY_SENDER_BINDINGS` defines the address the application listens on for connections from listeners such as eddb.io. + 1. `RELAY_DUPLICATE_MAX_MINUTES` how many minutes to keep messages hashes cached for so as to detect, and not Relay out, duplicate messages. If you set this to the literal string `false` the duplication checks will be disabled. This is **very handy** when testing the code. + 1. `GATEWAY_HTTP_BIND_ADDRESS` and `GATEWAY_HTTP_PORT` define where the Gateway listens to for incoming messages from senders. Might be forwarded from nginx or other reverse proxy. + 1. `GATEWAY_SENDER_BINDINGS` is where the Gateway listens for connections from the Relay and Monitor in order to send them messages that passed schema checks. + 1. `GATEWAY_JSON_SCHEMAS` defines the schemas used for validation. Note - that these are full public URLs which are served by nginx (or whatever - else you're using as the reverse proxy). + that these are full public URLs which are served by your web server. + 1. `GATEWAY_OUTDATED_SCHEMAS` any past schemas that are no longer valid. + 1. `MONITOR_HTTP_BIND_ADDRESS` and `MONITOR_HTTP_PORT` define where the Monitor listens to for web connections, e.g. the statistics page. + 1. `MONITOR_RECEIVER_BINDINGS` defines where the Monitor connects in order to subscribe to messages from the Gateway. Should match `GATEWAY_SENDER_BINDINGS`. + 1. `MONITOR_UA` appears to be unused. 1. Database Configuration @@ -295,8 +370,8 @@ another file. connect to a mysql/mariadb database for storing stats. 1. `database` - the name of the database 1. `user` - the user to connect as - 1. `password` - the secure password you set above when installing and - configuring mariadb/mysql. + 1. `password` - the secure password you set [above](#as-root) when + installing and configuring mariadb/mysql. It is assumed that the database is on `localhost`. @@ -320,6 +395,8 @@ It sets: 1. Configures the database connection and credentials. 1. Turns off the relay duplicate check. +--- + # Running You have some choices for how to run the application components: @@ -344,6 +421,8 @@ You have some choices for how to run the application components: running the components. They will need the `DAEMON` lines tweaking for running from another location. +--- + # Accessing the Monitor There is an EDDN Status web page usually provided at, e.g. https://eddn.edcd.io/. This is enabled by the Monitor component through @@ -365,7 +444,9 @@ permissions are correct for your web server to access them. chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn chmod -R og+rX ${HOME}/.local/share/eddn/schemas -## Testing all of this in a VM +--- + +# Testing all of this in a VM In order to test all of this in a VM you might need to set up a double proxying: From 1476ff824d7e7cefc2bb3b187cd05ff26dccbe9a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 5 Jul 2021 12:56:25 +0000 Subject: [PATCH 36/39] contrib/apache: Full, confirmed working, config --- contrib/apache-eddn.conf | 213 ++++++++++++++++++++++++++------------- 1 file changed, 141 insertions(+), 72 deletions(-) diff --git a/contrib/apache-eddn.conf b/contrib/apache-eddn.conf index 4428f5f..23842e5 100644 --- a/contrib/apache-eddn.conf +++ b/contrib/apache-eddn.conf @@ -1,97 +1,166 @@ # vim: :filetype=apache -## dev.eddn.edcd.io + +########################################################################### +# +# Read **ALL** the comments in this file, don't blindly use it! +# +# Be sure to replace 'YOUROWN.eddn.edcd.io' with your hostname. +# +# Also edit the DocumentRoot and related statements if you use a +# different path. +# +# Ensure the CustomLog directory actually exists, else apache will not +# start, or die on a restart/reload. +# +########################################################################### + +## YOUROWN.eddn.edcd.io - ServerName dev.eddn.edcd.io + ServerName YOUROWN.eddn.edcd.io - DocumentRoot //home/eddn/.local/share/eddn/dev + DocumentRoot /home/eddn/.local/share/eddn/dev - ErrorLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/error.log - # Possible values include: debug, info, notice, warn, error, crit, - # alert, emerg. - LogLevel warn - CustomLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/access.log combined + ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined - Redirect / https://dev.eddn.edcd.io/ - RedirectMatch "/^(.*)$" "https://dev.eddn.edcd.io/$1" + # Comment these out when initially requesting a LetsEncrypt cert + Redirect / https://YOUROWN.eddn.edcd.io/ + RedirectMatch "/^(.*)$" "https://YOUROWN.eddn.edcd.io/$1" - # LetsEncrypt - Alias /.well-known/ /var/www/letsencrypt/.well-known/ - - Options -Indexes - + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + - - Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec - AllowOverride All - - Require all granted - - - Require all denied - + + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + AllowOverride All + + Require all granted + + + Require all denied + - Include partials/default-directory.conf - + Include partials/default-directory.conf + +# This will need to be commented out/disabled for initial LetsEncrypt +# certificate request, as you don't have the certificate yet! - SSLEngine On - SSLCertificateFile /etc/letsencrypt/live/dev.eddn.edcd.io/fullchain.pem - SSLCertificateKeyFile /etc/letsencrypt/live/dev.eddn.edcd.io/privkey.pem + SSLEngine On + SSLCertificateFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem - ServerName dev.eddn.edcd.io + ServerName YOUROWN.eddn.edcd.io - DocumentRoot /home/eddn/.local/share/eddn/dev/monitor + DocumentRoot /home/eddn/.local/share/eddn/YOUROWN/monitor - ErrorLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/error.log - # Possible values include: debug, info, notice, warn, error, crit, - # alert, emerg. - LogLevel warn - CustomLog ${APACHE_LOG_DIR}/dev.eddn.edcd.io/access.log combined + ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined - # LetsEncrypt - Alias /.well-known/ /var/www/letsencrypt/.well-known/ - - Options -Indexes - + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + - - Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec - AllowOverride All - - Require all granted - - - Require all denied - - + + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + AllowOverride All + + Require all granted + + + Require all denied + + - # Serve the schemas - Alias /schemas/ /home/eddn/.local/share/eddn/dev/schemas/ + # Serve the schemas + Alias /schemas/ /home/eddn/.local/share/eddn/YOUROWN/schemas/ - # netdata (performance info) - - Redirect /netdata /netdata/ - - - SetOutputFilter DEFLATE + # netdata (performance info) + + Redirect /netdata /netdata/ + + + SetOutputFilter DEFLATE - - Require all granted - - - Require all denied - - - - SSLProxyEngine On - SSLProxyVerify none - ProxyPreserveHost On + + Require all granted + + + Require all denied + + + + SSLProxyEngine On + SSLProxyVerify none + ProxyPreserveHost On - ProxyPass "/netdata/" "http://127.0.0.1:19999/" - + # Yes, plain http for this. + ProxyPass "/netdata/" "http://127.0.0.1:19999/" + + +# This is for the Gateway public URLs + +# This will need to be commented out/disabled for initial LetsEncrypt +# certificate request, as you don't have the certificate yet! +# You also need to ensure `Listen 4430` is in ports.conf + + SSLEngine On + SSLCertificateFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem + + ServerName YOUROWN.eddn.edcd.io + + DocumentRoot /home/eddn/.local/share/eddn/YOUROWN/monitor + + ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined + + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + + + + + + Require all granted + + + Require all denied + + + + SSLProxyEngine On + SSLProxyVerify none + ProxyPreserveHost On + ProxyRequests Off + + # Must be https, not http, as the Gateway process is + # expecting only https requests. + ProxyPass "/" "https://127.0.0.1:8081/" + ProxyPassReverse "/" "https://127.0.0.1:8081/" + + + + From f9cf4142559ffc6eb3f7152637acf20fc59ed7aa Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 5 Jul 2021 13:42:51 +0000 Subject: [PATCH 37/39] contrib: Log archival script This will rotate to datetime stamped filenames (not a serial), and compress. Will also only keep 28 days of logs at most. --- contrib/logs-archive | 86 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 contrib/logs-archive diff --git a/contrib/logs-archive b/contrib/logs-archive new file mode 100755 index 0000000..7f35dd4 --- /dev/null +++ b/contrib/logs-archive @@ -0,0 +1,86 @@ +#!/bin/bash +# Add ' -x' above to debug +# + +########################################################################### +# Configuration +########################################################################### +# Maximum age, in days, of log files to keep +MAX_LOGFILE_AGE=28 +# Minimum size of live log before rotating, see find(1) -size for format +MIN_ROTATE_SIZE="100M" +########################################################################### + +########################################################################### +# Helper functions +########################################################################### +################################################## +# Print program usage information. +################################################## +usage() { + echo "Usage: $(basename $1) [ live | beta | dev ]" +} +################################################## + +################################################## +################################################## +################################################## + +########################################################################### + +########################################################################### +# Check command line arguments +########################################################################### +EDDN_ENV="$1" +if [ -z "${EDDN_ENV}" ]; +then + usage $0 + exit 1 +fi +########################################################################### + +########################################################################### +# Perform rotation +########################################################################### +LOGS_DIR="${HOME}/${EDDN_ENV}/logs" +if [ ! -d "${LOGS_DIR}" ]; +then + echo "$(dirname): Logs directory doesn't exist: ${LOGS_DIR}" + exit 2 +fi + +cd ${LOGS_DIR} || exit 3 + +for service in gateway monitor relay ; +do + echo "Service: ${service}" + echo " Expiring old logs..." + find . -name "${service}.log.*.gz" -a -atime +${MAX_LOGFILE_AGE} -exec rm -fv {} \; + echo " DONE" + + echo " Checking if current logfile needs archiving..." + if [ ! -z "$(find . -name ${service}.log -a -size +${MIN_ROTATE_SIZE})" ]; + then + echo " Archiving ${service}.log ..." + # We have no means to tell the service to close and re-open output, it's + # to stdout/err anyway. So we copy it. + COMPRESSED_NAME="${service}.log.$(date --iso-8601=seconds)" + cp ${service}.log "${COMPRESSED_NAME}" + if [ $? -ne 0 ]; + then + echo " FAILED copying live log file to new archive!!!" + echo " Exiting from any further processing." + exit 4 + fi + # Truncate the live file. + :> ${service}.log + # Now compress the newly archived log + gzip -9v "${COMPRESSED_NAME}" + echo " DONE" + else + echo " No" + fi +done +########################################################################### + +# vim: tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0 From 1e9a5e6212e558b410e79f035389b3956dd0af1e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 5 Jul 2021 13:46:31 +0000 Subject: [PATCH 38/39] contrib: Rename logs-archive to eddn-logs-archive --- contrib/{logs-archive => eddn-logs-archive} | 4 ---- 1 file changed, 4 deletions(-) rename contrib/{logs-archive => eddn-logs-archive} (94%) diff --git a/contrib/logs-archive b/contrib/eddn-logs-archive similarity index 94% rename from contrib/logs-archive rename to contrib/eddn-logs-archive index 7f35dd4..79b10dd 100755 --- a/contrib/logs-archive +++ b/contrib/eddn-logs-archive @@ -22,10 +22,6 @@ usage() { } ################################################## -################################################## -################################################## -################################################## - ########################################################################### ########################################################################### From aa42aaf8b8bcec6429e6ddaff293200f9ebf8500 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Mon, 5 Jul 2021 13:50:45 +0000 Subject: [PATCH 39/39] setup.py: Install the logs archivel script --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index 127c984..bfd0c6b 100644 --- a/setup.py +++ b/setup.py @@ -106,6 +106,16 @@ shutil.copy( '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) ) +# Ensure the service log file archiving script is in place +print """ +****************************************************************************** +Ensuring the service log file archiving script is in place +""" +shutil.copy( + 'contrib/eddn-logs-archive', + START_SCRIPT_BIN +) + # Ensure the latest monitor files are in place old_umask = os.umask(022) print """