Merge pull request #124 from EDCD/enhancement/118/venv-and-setup.py

Generally get setup.py and docs into better shape
This commit is contained in:
Athanasius 2021-07-05 14:53:49 +01:00 committed by GitHub
commit dd57ca3ac0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 931 additions and 160 deletions

6
.gitignore vendored
View File

@ -1,3 +1,7 @@
# These need to be per-environment, so not changed by git merges.
setup.cfg
setup_env.py
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
@ -61,3 +65,5 @@ target/
# Editors
.idea
# VIM swap files
.*.sw?

166
contrib/apache-eddn.conf Normal file
View File

@ -0,0 +1,166 @@
# vim: :filetype=apache
###########################################################################
#
# Read **ALL** the comments in this file, don't blindly use it!
#
# Be sure to replace 'YOUROWN.eddn.edcd.io' with your hostname.
#
# Also edit the DocumentRoot and related statements if you use a
# different path.
#
# Ensure the CustomLog directory actually exists, else apache will not
# start, or die on a restart/reload.
#
###########################################################################
## YOUROWN.eddn.edcd.io
<VirtualHost *:80>
ServerName YOUROWN.eddn.edcd.io
DocumentRoot /home/eddn/.local/share/eddn/dev
ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined
# Comment these out when initially requesting a LetsEncrypt cert
Redirect / https://YOUROWN.eddn.edcd.io/
RedirectMatch "/^(.*)$" "https://YOUROWN.eddn.edcd.io/$1"
# LetsEncrypt
Alias /.well-known/ /var/www/letsencrypt/.well-known/
<Directory /var/www/letsencrypt/.well-known/>
Options -Indexes
</Directory>
<Directory /home/eddn/.local/share/eddn/dev>
Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec
AllowOverride All
<Limit GET POST OPTIONS>
Require all granted
</Limit>
<LimitExcept GET POST OPTIONS>
Require all denied
</LimitExcept>
Include partials/default-directory.conf
</Directory>
</VirtualHost>
# This will need to be commented out/disabled for initial LetsEncrypt
# certificate request, as you don't have the certificate yet!
<IfModule mod_ssl.c>
<VirtualHost *:443>
SSLEngine On
SSLCertificateFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem
ServerName YOUROWN.eddn.edcd.io
DocumentRoot /home/eddn/.local/share/eddn/YOUROWN/monitor
ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined
# LetsEncrypt
Alias /.well-known/ /var/www/letsencrypt/.well-known/
<Directory /var/www/letsencrypt/.well-known/>
Options -Indexes
</Directory>
<Directory /home/eddn/.local/share/eddn/dev>
Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec
AllowOverride All
<Limit GET POST OPTIONS>
Require all granted
</Limit>
<LimitExcept GET POST OPTIONS>
Require all denied
</LimitExcept>
</Directory>
# Serve the schemas
Alias /schemas/ /home/eddn/.local/share/eddn/YOUROWN/schemas/
# netdata (performance info)
<IfModule mod_alias.c>
Redirect /netdata /netdata/
</IfModule>
<LocationMatch /netdata*>
SetOutputFilter DEFLATE
<Limit GET POST OPTIONS>
Require all granted
</Limit>
<LimitExcept GET POST OPTIONS>
Require all denied
</LimitExcept>
</LocationMatch>
<IfModule mod_proxy.c>
SSLProxyEngine On
SSLProxyVerify none
ProxyPreserveHost On
# Yes, plain http for this.
ProxyPass "/netdata/" "http://127.0.0.1:19999/"
</IfModule>
</VirtualHost>
</IfModule>
# This is for the Gateway public URLs
<IfModule mod_ssl.c>
# This will need to be commented out/disabled for initial LetsEncrypt
# certificate request, as you don't have the certificate yet!
# You also need to ensure `Listen 4430` is in ports.conf
<VirtualHost *:4430>
SSLEngine On
SSLCertificateFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem
ServerName YOUROWN.eddn.edcd.io
DocumentRoot /home/eddn/.local/share/eddn/YOUROWN/monitor
ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined
# LetsEncrypt
Alias /.well-known/ /var/www/letsencrypt/.well-known/
<Directory /var/www/letsencrypt/.well-known/>
Options -Indexes
</Directory>
<LocationMatch /*>
<Limit GET POST OPTIONS>
Require all granted
</Limit>
<LimitExcept GET POST OPTIONS>
Require all denied
</LimitExcept>
</LocationMatch>
<IfModule mod_proxy.c>
SSLProxyEngine On
SSLProxyVerify none
ProxyPreserveHost On
ProxyRequests Off
# Must be https, not http, as the Gateway process is
# expecting only https requests.
ProxyPass "/" "https://127.0.0.1:8081/"
ProxyPassReverse "/" "https://127.0.0.1:8081/"
</IfModule>
</VirtualHost>
</IfModule>

82
contrib/eddn-logs-archive Executable file
View File

@ -0,0 +1,82 @@
#!/bin/bash
# Add ' -x' above to debug
#
###########################################################################
# Configuration
###########################################################################
# Maximum age, in days, of log files to keep
MAX_LOGFILE_AGE=28
# Minimum size of live log before rotating, see find(1) -size for format
MIN_ROTATE_SIZE="100M"
###########################################################################
###########################################################################
# Helper functions
###########################################################################
##################################################
# Print program usage information.
##################################################
usage() {
echo "Usage: $(basename $1) [ live | beta | dev ]"
}
##################################################
###########################################################################
###########################################################################
# Check command line arguments
###########################################################################
EDDN_ENV="$1"
if [ -z "${EDDN_ENV}" ];
then
usage $0
exit 1
fi
###########################################################################
###########################################################################
# Perform rotation
###########################################################################
LOGS_DIR="${HOME}/${EDDN_ENV}/logs"
if [ ! -d "${LOGS_DIR}" ];
then
echo "$(dirname): Logs directory doesn't exist: ${LOGS_DIR}"
exit 2
fi
cd ${LOGS_DIR} || exit 3
for service in gateway monitor relay ;
do
echo "Service: ${service}"
echo " Expiring old logs..."
find . -name "${service}.log.*.gz" -a -atime +${MAX_LOGFILE_AGE} -exec rm -fv {} \;
echo " DONE"
echo " Checking if current logfile needs archiving..."
if [ ! -z "$(find . -name ${service}.log -a -size +${MIN_ROTATE_SIZE})" ];
then
echo " Archiving ${service}.log ..."
# We have no means to tell the service to close and re-open output, it's
# to stdout/err anyway. So we copy it.
COMPRESSED_NAME="${service}.log.$(date --iso-8601=seconds)"
cp ${service}.log "${COMPRESSED_NAME}"
if [ $? -ne 0 ];
then
echo " FAILED copying live log file to new archive!!!"
echo " Exiting from any further processing."
exit 4
fi
# Truncate the live file.
:> ${service}.log
# Now compress the newly archived log
gzip -9v "${COMPRESSED_NAME}"
echo " DONE"
else
echo " No"
fi
done
###########################################################################
# vim: tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0

View File

@ -13,7 +13,7 @@ DESC="eddn-gateway"
PIDFILE="/var/run/${NAME}.pid"
LOGFILE="/var/log/eddn/${NAME}.log"
DAEMON="/usr/local/bin/${NAME}"
DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}"
EXEC_AS_USER="root"
@ -50,4 +50,4 @@ restart|force-reload)
;;
esac
exit 0
exit 0

View File

@ -13,7 +13,7 @@ DESC="eddn-monitor"
PIDFILE="/var/run/${NAME}.pid"
LOGFILE="/var/log/eddn/${NAME}.log"
DAEMON="/usr/local/bin/${NAME}"
DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}"
EXEC_AS_USER="root"
@ -50,4 +50,4 @@ restart|force-reload)
;;
esac
exit 0
exit 0

View File

@ -13,7 +13,7 @@ DESC="eddn-relay"
PIDFILE="/var/run/${NAME}.pid"
LOGFILE="/var/log/eddn/${NAME}.log"
DAEMON="/usr/local/bin/${NAME}"
DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}"
EXEC_AS_USER="root"
@ -50,4 +50,4 @@ restart|force-reload)
;;
esac
exit 0
exit 0

View File

@ -0,0 +1,35 @@
###########################################################################
# Copy a certificate's files into place, with appropriate ownership and
# mode.
#
# $1 - Name of certificate (i.e. letsencrypt directory names).
# $2 - Source Directory
# $3 - Destination filename for fullchain.pem
# $4 - Destination filename for privkey.pem
# $5 - File ownership to set (user:group)
# $6 - File mode to set (as passed to 'chmod')
###########################################################################
copy_cert() {
CERT_NAME="$1"
SRC_DIR="$2"
DST_FILE_FULLCHAIN="$3"
DST_FILE_PRIVKEY="$4"
CERT_NEW_OWNER="$5"
CERT_NEW_PERMS="$6"
echo "${CERT_NAME}: Copying new files into place..."
# Preserve only the mode as it should be 0600, and thus we won't
# temporarily open up the files for *all* users to read,
# BUT don't preserve the timestamp as we want it to be 'now' so
# that a `find ... -newer <this file>` check works later.
cp -v --preserve=mode ${SRC_DIR}/fullchain.pem ${DST_FILE_FULLCHAIN}
cp -v --preserve=mode ${SRC_DIR}/privkey.pem ${DST_FILE_PRIVKEY}
chown -v ${CERT_NEW_OWNER} ${DST_FILE_FULLCHAIN} ${DST_FILE_PRIVKEY}
chmod -v ${CERT_NEW_PERMS} ${DST_FILE_FULLCHAIN} ${DST_FILE_PRIVKEY}
echo "${CERT_NAME}: Copying new files into place DONE"
}
###########################################################################
# vim: :set filetype=sh tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0

View File

@ -0,0 +1,89 @@
#!/bin/bash
# Add " -x" above to debug
#
# certbot deploy hook
#
# This should be triggered by being present in:
#
# /etc/letsencrypt/renewal-hooks/deploy/
#
# It can be linked into the 'post' directory for testing with:
#
# certbot renew --dry-run
#
# which you might want to do because deploy hooks aren't run for that
# command.
#
# You can also just straight up run this script, including to get into place
# any certificate files it's configured for, but have never been deployed.
# Paranoia re-enforcement of no group/other perms on created files
chmod -R og-rwx /etc/letsencrypt/archive
echo "$0 - Running in: $(pwd)"
# Import common code and settings.
. /etc/scripts/certbot-common
# As of 2021-07-02 and certbot 0.31.0 (current in Debian buster)
# there is **zero** information passed in (CL args or environment) to
# this hook. So we just need to check each potentially renewed
# certificate.
###########################################################################
# MAIN_HOST_NAME
###########################################################################
CERT_NAME="MAIN_HOST_NAME"
# We're only interested if it's newer than when the files were last copied
SRC_DIR="/etc/letsencrypt/live/${CERT_NAME}"
DST_FILE_FULLCHAIN="/etc/exim4/exim.crt"
DST_FILE_PRIVKEY="/etc/exim4/exim.key"
CERT_NEW_OWNER="root:Debian-exim"
CERT_NEW_PERMS="440"
#############################################################
# Needs to be in place for exim to use
#############################################################
# 'find' doesn't set exit status depending on if it found anything, that's
# for actual errors, so we test against the output.
if [ "$(find ${SRC_DIR} -newer ${DST_FILE_FULLCHAIN} -o -newer ${DST_FILE_PRIVKEY} )" != "" ];
then
echo "${CERT_NAME}: (Re)new(ed) certificate..."
copy_cert "${CERT_NAME}" "${SRC_DIR}" "${DST_FILE_FULLCHAIN}" "${DST_FILE_PRIVKEY}" "${CERT_NEW_OWNER}" "${CERT_NEW_PERMS}"
echo "${CERT_NAME}: DONE"
fi
#############################################################
###########################################################################
###########################################################################
# eddn.edcd.io and related names
###########################################################################
CERT_NEW_OWNER="eddn:eddn"
CERT_NEW_PERMS="400"
for eddn in eddn.edcd.io test.eddn.edcd.io staging.eddn.edcd.io ;
do
CERT_NAME="${eddn}"
SRC_DIR="/etc/letsencrypt/live/${CERT_NAME}"
DST_FILE_FULLCHAIN="/home/eddn/etc/${CERT_NAME}-fullchain.pem"
DST_FILE_PRIVKEY="/home/eddn/etc/${CERT_NAME}-privkey.pem"
if [ -d "${SRC_DIR}" ];
then
if [ ! -f "${DST_FILE_FULLCHAIN}" \
-o ! -f "${DST_FILE_PRIVKEY}" \
-o "$(find ${SRC_DIR} -newer ${DST_FILE_FULLCHAIN} -o -newer ${DST_FILE_PRIVKEY} )" != "" ];
then
echo "${CERT_NAME}: (Re)New(ed) certificate..."
copy_cert "${CERT_NAME}" "${SRC_DIR}" "${DST_FILE_FULLCHAIN}" "${DST_FILE_PRIVKEY}" "${CERT_NEW_OWNER}" "${CERT_NEW_PERMS}"
echo "${CERT_NAME}: DONE"
fi
fi
done
###########################################################################
# vim: tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0

View File

@ -17,8 +17,17 @@ do
echo "$d: Already running as $(cat ${LOGPATH}/${d}.pid)"
continue
fi
if [ -f "${BASEPATH}/etc/settings.json" ];
then
CONFIG="--config ${BASEPATH}/etc/settings.json"
else
echo "WARNING: No override settings found, you'll be using defaults"
echo "WARNING: Did you forget to make ${BASEPATH}/etc/settings.json ?"
echo " Continuing anyway..."
CONFIG=""
fi
${PYTHON} -m eddn.${d} \
--config ${BASEPATH}/etc/settings.json \
${CONFIG} \
> ${LOGPATH}/$d.log \
2>&1 &
echo $! > "${LOGPATH}/${d}.pid"

View File

@ -7,7 +7,7 @@
[Unit]
Description=EDDN Service %i
AssertPathExists=/home/eddn/.local/bin/%i
AssertPathExists=/home/eddn/eddn/python-venv/bin/%i
PartOf=eddn.service
ReloadPropagatedFrom=eddn.service
Before=eddn.service
@ -18,7 +18,7 @@ After=network.target
Type=simple
User=eddn
Group=eddn
ExecStart=/home/eddn/.local/bin/start-%i
ExecStart=/home/eddn/.local/bin/start-eddn-service %i
TimeoutStartSec=10s
TimeoutStopSec=10s
SyslogIdentifier=eddn@%i

View File

@ -0,0 +1,3 @@
CONFIG_OVERRIDE="${HOME}/.local/share/eddn/beta/config.json"
LOG_DIR="${HOME}/beta/logs"
PYTHON_VENV="${HOME}/beta/python-venv"

View File

@ -1,2 +0,0 @@
CONFIG_OVERRIDE="${HOME}/.local/share/eddn/config.json"
LOG_DIR="${HOME}/.var/log/eddn"

View File

@ -0,0 +1,3 @@
CONFIG_OVERRIDE="${HOME}/.local/share/eddn/dev/config.json"
LOG_DIR="${HOME}/dev/logs"
PYTHON_VENV="${HOME}/dev/python-venv"

View File

@ -0,0 +1,3 @@
CONFIG_OVERRIDE="${HOME}/.local/share/eddn/live/config.json"
LOG_DIR="${HOME}/live/logs"
PYTHON_VENV="${HOME}/live/python-venv"

View File

@ -1,16 +0,0 @@
#!/bin/sh
# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab
#
# Start the EDDN Gateway, including redirecting output to a log file.
EXEC_PATH=$(dirname $0)
#echo "EXEC_PATH: ${EXEC_PATH}"
# Ensure we're in the correct place
cd ${EXEC_PATH}
#pwd
# Bring in some common configuration
. ./eddn_config
./eddn-gateway --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-gateway.log" 2>&1

View File

@ -1,16 +0,0 @@
#!/bin/sh
# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab
#
# Start the EDDN Gateway, including redirecting output to a log file.
EXEC_PATH=$(dirname $0)
#echo "EXEC_PATH: ${EXEC_PATH}"
# Ensure we're in the correct place
cd ${EXEC_PATH}
#pwd
# Bring in some common configuration
. ./eddn_config
./eddn-monitor --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-monitor.log" 2>&1

View File

@ -1,16 +0,0 @@
#!/bin/sh
# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab
#
# Start the EDDN Gateway, including redirecting output to a log file.
EXEC_PATH=$(dirname $0)
#echo "EXEC_PATH: ${EXEC_PATH}"
# Ensure we're in the correct place
cd ${EXEC_PATH}
#pwd
# Bring in some common configuration
. ./eddn_config
./eddn-relay --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-relay.log" 2>&1

View File

@ -0,0 +1,50 @@
#!/bin/sh
# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab
#
# Start an EDDN Service, including redirecting output to a log file.
usage() {
echo "Usage: $(basename $0) [ gateway | monitor | relay ] [ live | beta | dev ]"
}
if [ -z "${1}" ];
then
usage
echo "No EDDN service specified."
exit 3
fi
SERVICE="${1}"
if [ -z "${2}" ];
then
usage
echo "No EDDN environment specified."
exit 3
fi
EDDN_ENV="${2}"
EXEC_PATH=$(dirname $0)
#echo "EXEC_PATH: ${EXEC_PATH}"
# Ensure we're in the correct place
cd ${EXEC_PATH}
#pwd
# Bring in some common configuration
if [ ! -f "eddn_${EDDN_ENV}_config" ];
then
echo "eddn_${EDDN_ENV}_config is missing from $(pwd)"
exit 1
fi
. "./eddn_${EDDN_ENV}_config"
# Use the python venv
. "${PYTHON_VENV}/bin/activate"
if [ ! -f "${PYTHON_VENV}/bin/eddn-${SERVICE}" ];
then
echo "${SERVICE} is missing from ${PYTHON_VENV}/bin"
exit 2
fi
exec ${PYTHON_VENV}/bin/eddn-${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1

View File

@ -1,3 +1,5 @@
<!-- vim: tabstop=2 softtabstop=2 shiftwidth=2 expandtab smartindent smarttab wrapmargin=0 textwidth=0
-->
These instructions are based on getting the software up and running from
scratch on a Debian Buster (10.9, stable as of 2021-05-16) system.
@ -17,13 +19,15 @@ A specific user was created:
useradd -c 'EDDN Gateway' -m -s /bin/bash eddn
---
# Further installation
## As 'root'
Some additional Debian packages and python modules are required:
apt install python-pip
apt install python-pip virtualenv
You will need a mysql/mariab database:
@ -36,6 +40,8 @@ You will need a mysql/mariab database:
> GRANT ALL PRIVILEGES on eddn.* TO 'eddn'@'localhost';
> \q
---
### Netdata
In order to get host performance metrics (CPU, RAM and network usage) you will
need to install netdata. On Debian-based systems:
@ -45,21 +51,27 @@ need to install netdata. On Debian-based systems:
The default configuration should be all you need, listening on
`127.0.0.1:19999`.
### LetsEncrypt: certbot
It will be necessary to renew the TLS certificate using certbot (or some
alternative ACME client).
---
### LetsEncrypt
We assume that you're using a TLS certificate from
[LetsEncrypt](https://letsencrypt.org/), it's free!
It will be necessary to renew the TLS certificate using certbot, or some
alternative ACME client. We'll assume certbot.
#### Install certbot
On a Debian system simply:
apt install certbot
### Reverse Proxy with nginx
If you don't yet have nginx installed then start with:
apt install nginx-light
Although this version might be a little old now, it does work.
#### LetsEncrypt TLS Certificates
If you are taking over hosting the EDDN relay then hopefully you have access
to the existing certificate files.
You will need a LetsEncrupt/ACME client in order to keep the TLS certificate
renewed.
So, first copy those into place:
cd /etc/letsencrypt
mkdir -p archive/eddn.edcd.io
@ -74,7 +86,127 @@ renewed.
ln -s ../../archive/eddn.edcd.io/fullchain1.pem fullchain.pem
ln -s ../../archive/eddn.edcd.io/privkey1.pem privkey.pem
#### nginx configuration
After this you need to ensure that the certificate stays renewed. With a
Debian system using certbot:
1. There should already be a systemd timer set up:
`systemctl status certbot.timer`
If that doesn't show "`; enabled;`" in:
`Loaded: loaded (/lib/systemd/system/certbot.timer; enabled; vendor preset: enabled)`
then:
`systemctl enable certbot.timer`
This will renew the certificate as necessary (i.e. when <= 30 days until
it expires, or whatever current LetsEncrypt and certbot policy causes).
But it will not ensure the files are in all the places you might need
them to be.
1. Ensure the certificate files are deployed to where they're needed. When
using the certbot timer the easiest thing to do is to utilise a script in
`/etc/letsencrypt/renewal-hooks/deploy/`.
There are example files for this in `contrib/letsencrypt/`:
mkdir -p /etc/letsencrypt/renewal-hooks/deploy
cp contrib/letsencrypt/deploy-changed-certs /etc/letsencrypt/renewal-hooks/deploy
mkdir -p /etc/scripts
cp contrib/letsencrypt/certbot-common /etc/scripts/
**Remember to edit them to suit your setup!**
---
### Network Configuration
There are multiple ports that you'll have to ensure are allowed through any
firewall, and some of them also require being reverse proxied correctly.
The reverse proxies pertain to:
1. The port for the Gateway to receive uploads from senders (e.g. Elite
Dangerous Market Connector). This is also used for the 'monitor' web
page to obtain stats about messages passing through the Gateway.
1. A set of URLs for accessing [netdata](#netdata).
#### Necessary ports
These all for TCP, no UDP:
1. `443` - a web server capable of reverse proxying set up for TLS on the
public host name of the EDDN service. This is used to serve the schemas,
the monitor web page, and to reverse proxy URLs beginning `/netdata/` to
the [netdata](#netdata) service.
1. Default: `4430` - Gateway 'http' port, used both for EDDN senders to
upload, and also for the Gateway message rate stats on the monitor web
page.
But that's the *public* port. The Gateway process itself listens on `8081`.
So you'll need a reverse proxy listening on port `4430` and forwarding
*all* requests to `127.0.0.1:8081`.
1. Default: `9091` - Monitor 'http' port, used for the monitor web page to
query schema and software statistics. No reverse proxy setup.
1. Default: `9500` - The port on the Relay that EDDN listeners connect to in
order to receive the zeromq stream. No reverse proxy setup.
1. Default: `9090` - The Relay 'http' port for its portion of the message
statistics on the monitor web page. No reverse proxy setup.
There's also the internal `8500` port, but that's literally only used for
the Monitor and Relay to pick up zeromq messages forwarded from the
Gateway, so all over localhost.
See [Configuration](#configuration) for guidance on what override config
settings can be used to change any of these ports.
---
#### Reverse Proxy with Apache
If you already have an Apache installation it will be easier to just use
it for the reverse proxy.
Ensure you have these modules installed and active:
a2enmod proxy proxy_http
##### Apache configuration
There is an example VirtualHost configuration in
`contrib/apache-eddn.conf` which makes the following assumptions:
1. The usual Apache default configuration is in place elsewhere.
1. The hostname being used - `ServerName`.
1. The location of the monitor files - `DocumentRoot`.
1. The location of the schema files - `Alias /schemas/ ...`.
1. The location of the TLS certificate files - `SSLCertificateFile` and
`SSLCertificateKeyFile.
You should be able to:
1. Copy `contrib/apache-eddn.conf` into `/etc/apache/sites-available/`
*as an appropriate filename for the hostname you're using*.
1. Edit to suit the local situation/setup. **Remember to ensure the
configured log directory exists.**
1. Enable the site:
a2ensite <filename without trailing .conf>
apache2ctl configtest
# CHECK THE OUTPUT
apache2ctl graceful
---
#### Reverse Proxy with nginx
If you don't yet have nginx installed then start with:
apt install nginx-light
##### nginx configuration
There is an example configuration in `contrib/nginx-eddn.conf` which makes
some assumptions:
@ -83,7 +215,7 @@ some assumptions:
1. The location of the monitor files - `root` directive.
1. The location of the schema files - `location` directive.
1. The location of the TLS certificate files - `ssl_certificate` and
`ssl_certificate_key` directives.
`ssl_certificate_key` directives.
You should be able to:
@ -95,33 +227,42 @@ You should be able to:
ln -s /etc/nginx/sites-available/eddn
systemctl restart nginx.service
If you're already using another web server, such as Apache, you'll need to
If you're already using another web server you'll need to
duplicate at least the use of a TLS certificate and the Reverse Proxying as
required.
For Apache you would reverse proxy using something like the following in an
appropriate `<VirtualHost>` section:
<IfModule mod_proxy.c>
SSLProxyEngine On
SSLProxyVerify none
ProxyPreserveHost On
# Pass through 'gateway' upload URL to Debian VM
ProxyPass "/upload/" "https://EDDNHOST:8081/upload/"
# Pass through 'monitor' URLs to Debian VM
ProxyPass "/" "https://EDDNHOST/"
</IfModule>
---
## In the 'eddn' account
### Clone a copy of the application project from gitub
mkdir -p ~/eddn/dev
cd ~/eddn/dev
git clone https://github.com/EDCD/EDDN.git
mkdir -p ${HOME}/dev
cd ${HOME}/dev
git clone https://github.com/EDCD/EDDN.git EDDN.git
cd EDDN.git
We'll assume this `~/eddn/dev/EDDN` path elsewhere in this document.
We'll assume this `${HOME}/dev/EDDN.git` path elsewhere in this document.
### Set up a python virtual environment
So as to not have any python package version requirements clash with
anything else it's best to use a Python virtual environment (venv). You
will have installed the Debian package 'virtualenv' [above](#as-root) for
this purpose.
We'll put the venv in `${HOME}/dev/python2.7-venv` with the following
command:
cd ${HOME}/dev
virtualenv -p /usr/bin/python2.7 ${HOME}/python2.7-venv
And for future ease of changing python versions:
ln -s python2.7-venv python-venv
And now start using this venv:
. python-venv/bin/activate
### Ensure necessary python modules are installed
Installing extra necessary python modules is simple:
@ -131,24 +272,12 @@ Installing extra necessary python modules is simple:
### Initialise Database Schema
You will need to get the database schema in place:
mysql -p eddn < ~/eddn/dev/EDDN/schema.sql
mysql -p eddn < ${HOME}/eddn/dev/EDDN/schema.sql
<the password you set in the "CREATE USER" statement above>
### Monitor and Schema files
The Monitor files are not currently installed anywhere by the `setup.py`
script, so you'll need to manually copy them into somewhere convenient,
e.g.:
Ref: [As root](#as-root).
mkdir -p ${HOME}/.local/share/eddn
cp -r ~/eddn/dev/EDDN/contrib/monitor ${HOME}/.local/share/eddn
chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn
You will need to ensure that the Monitor nginx setup can see the schema files
in order to serve them for use by the Gateway. So perform, e.g.:
mkdir -p ${HOME}/.local/share/eddn
cp -r ~/eddn/dev/EDDN/schemas ${HOME}/.local/share/eddn
chmod -R og+rX ${HOME}/.local/share/eddn/schemas
---
# Concepts
There are three components to this application.
@ -181,7 +310,11 @@ test host. The files in question are:
monitor/js/eddn.js
monitor/schemas.html
Replace the string `eddn.edcd.io` with the hostname you're using.
Replace the string `eddn.edcd.io` with the hostname you're using. You'll need
to perform similar substitutions if you change the configuration to use any
different port numbers.
---
# Configuration
Default application configuration is in the file `src/eddn/conf/Settings.py`.
@ -191,36 +324,45 @@ another file.
1. You will need to obtain a TLS certificate from, e.g. LetsEncrypt. The
application will need access to this and its private key file.
CERT_FILE = '/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem'
KEY_FILE = '/etc/letsencrypt/live/eddn.edcd.io/privkey.pem'
CERT_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem'
KEY_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem'
1. Network configuration
1. `RELAY_HTTP_BIND_ADDRESS` and `RELAY_HTTP_PORT` define the IP and port
on which the Relay listens for, e.g. `/stats/` requests.
1. `RELAY_RECEIVER_BINDINGS` defines where the Relay connects in order to
subscribe to messages from the Gateway. Should match
`GATEWAY_SENDER_BINDINGS`.
1. `RELAY_SENDER_BINDINGS` defines the address the application listens on
for connections from listeners such as eddb.io.
1. `RELAY_DUPLICATE_MAX_MINUTES` how many minutes to keep messages hashes
cached for so as to detect, and not Relay out, duplicate messages. If
you set this to the literal string `false` the duplication checks will be
disabled. This is **very handy** when testing the code.
1. `GATEWAY_HTTP_BIND_ADDRESS` and `GATEWAY_HTTP_PORT` define where the
Gateway listens to for incoming messages from senders. Might be
forwarded from nginx or other reverse proxy.
1. `GATEWAY_SENDER_BINDINGS` is where the Gateway listens for connections
from the Relay and Monitor in order to send them messages that passed
schema checks.
1. `GATEWAY_JSON_SCHEMAS` defines the schemas used for validation. Note
that these are full public URLs which are served by nginx (or whatever
else you're using as the reverse proxy).
that these are full public URLs which are served by your web server.
1. `GATEWAY_OUTDATED_SCHEMAS` any past schemas that are no longer valid.
1. `MONITOR_HTTP_BIND_ADDRESS` and `MONITOR_HTTP_PORT` define where the
Monitor listens to for web connections, e.g. the statistics page.
1. `MONITOR_RECEIVER_BINDINGS` defines where the Monitor connects in order
to subscribe to messages from the Gateway. Should match
`GATEWAY_SENDER_BINDINGS`.
1. `MONITOR_UA` appears to be unused.
1. Database Configuration
@ -228,8 +370,8 @@ another file.
connect to a mysql/mariadb database for storing stats.
1. `database` - the name of the database
1. `user` - the user to connect as
1. `password` - the secure password you set above when installing and
configuring mariadb/mysql.
1. `password` - the secure password you set [above](#as-root) when
installing and configuring mariadb/mysql.
It is assumed that the database is on `localhost`.
@ -253,35 +395,34 @@ It sets:
1. Configures the database connection and credentials.
1. Turns off the relay duplicate check.
---
# Running
You have some choices for how to run the application components:
1. You can choose to run this application directly from the source using the
provided script in `contrib/run-from-source.sh`.
1. If you are just testing out code changes then you can choose to run
this application directly from the source using the provided script in
`contrib/run-from-source.sh`.
1. Or you can utilise the `setup.py` file to build and install the application
files. By default this requires write permissions under `/usr/local`, but
you can run:
1. Otherwise you will want to utilise the `setup.py` file to build and
install the application files. As we're using a python venv we can just
run:
python setup.py install --user
python setup.py install
to install under `~/.local/` instead.
to install it all. This will install a python egg into the python
venv, and then also ensure that the monitor and schema files are in
place.
There is an example systemd setup in `contrib/systemd` that assumes
this local installation.
If you install into `/usr/local/` then there are SysV style init.d scripts
in `contrib/init.d/` for running the components. They will need the
`DAEMON` lines tweaking for running from another location.
There are also some SysV style init.d scripts in `contrib/init.d/` for
running the components. They will need the `DAEMON` lines tweaking for
running from another location.
1. For quick testing purposes you can run them as follows, assuming you
installed into `~/.local/`, and have your override settings in
`${HOME}/etc/eddn-settings-overrides.json`:
---
~/.local/bin/eddn-gateway --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-gateway.log 2>&1 &
~/.local/bin/eddn-monitor --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-monitor.log 2>&1 &
~/.local/bin/eddn-relay --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-relay.log 2>&1 &
# Accessing the Monitor
There is an EDDN Status web page usually provided at, e.g.
https://eddn.edcd.io/. This is enabled by the Monitor component through
@ -291,7 +432,21 @@ by the Monitor process itself.
You will need to configure a reverse proxy to actually enable access to this.
There is an example nginx configuration in `contrib/nginx-eddn.conf`.
## Testing all of this in a VM
The necessary files should be put in place by
The 'monitor' files are what form the status/statistics page at
https://eddn.edcd.io/, so they need to be installed somewhere in a
static manner accessible to nginx.
Although setup.py installs the files you might still need to ensure the
permissions are correct for your web server to access them.
chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn
chmod -R og+rX ${HOME}/.local/share/eddn/schemas
---
# Testing all of this in a VM
In order to test all of this in a VM you might need to set up a double
proxying:

12
docs/config-EXAMPLE.json Normal file
View File

@ -0,0 +1,12 @@
{
"CERT_FILE": "/home/eddn/etc/fullchain.pem",
"KEY_FILE": "/home/eddn/etc/privkey.pem",
"GATEWAY_HTTP_BIND_ADDRESS": "0.0.0.0",
"MONITOR_DB": {
"database": "eddn",
"user": "eddn",
"password": "SOME SECURE PASSWORD"
},
}

View File

@ -1,2 +0,0 @@
[egg_info]
tag_build = .dev

165
setup.py
View File

@ -1,6 +1,8 @@
from setuptools import setup, find_packages
import re
import glob
import os
import re
import shutil
from setuptools import setup, find_packages
VERSIONFILE = "src/eddn/conf/Version.py"
@ -15,26 +17,163 @@ except EnvironmentError:
print "unable to find version in %s" % (VERSIONFILE,)
raise RuntimeError("if %s exists, it is required to be well-formed" % (VERSIONFILE,))
# Read environment-specific settings
import setup_env
# Location of start-eddn-service script and its config file
START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] )
# Location of web files
SHARE_EDDN_FILES='%s/.local/share/eddn/%s' % ( os.environ['HOME'], setup_env.EDDN_ENV )
setup(
name='eddn',
version=verstr,
description='Elite: Dangerous Data Network',
author='Anthor (EDSM)',
author_email='contact@edsm.net',
url='https://github.com/EDSM-NET/EDDN',
packages=find_packages('src', exclude=["*.tests"]),
package_dir = {'':'src'},
data_files=[('eddn/schemas', glob.glob("schemas/*.json"))],
long_description="""\
The Elite: Dangerous Data Network allows E:D players to share data. Not affiliated with Frontier Developments.
The Elite Dangerous Data Network allows ED players to share data. Not affiliated with Frontier Developments.
""",
install_requires=["argparse", "bottle", "enum34", "gevent", "jsonschema", "pyzmq", "strict_rfc3339", "simplejson", "mysql-connector-python"],
author='EDCD (https://edcd.github.io/)',
author_email='edcd@miggy.org',
url='https://github.com/EDCD/EDDN',
packages=find_packages(
'src',
exclude=["*.tests"]
),
package_dir = {'':'src'},
# This includes them for the running code, but that doesn't help
# serve them up for reference.
data_files=[
(
'eddn/schemas', glob.glob("schemas/*.json")
)
],
# Yes, we pin versions. With python2.7 the latest pyzmq will NOT
# work, for instance.
install_requires=[
"argparse",
"bottle==0.12.15",
"enum34==1.1.6",
"gevent==1.3.7",
"jsonschema==2.6.0",
"pyzmq==17.1.2",
"strict_rfc3339==0.7",
"simplejson==3.16.0",
"mysql-connector-python==8.0.17"
],
entry_points={
'console_scripts': [
'eddn-gateway = eddn.Gateway:main',
'eddn-relay = eddn.Relay:main',
'eddn-monitor = eddn.Monitor:main',
],
}
)
],
}
)
# Ensure the systemd-required start files are in place
print """
******************************************************************************
Ensuring start script and its config file are in place...
"""
old_cwd = os.getcwd()
if not os.path.isdir(START_SCRIPT_BIN):
# We're still using Python 2.7, so no pathlib
os.chdir('/')
for pc in START_SCRIPT_BIN[1:].split('/'):
try:
os.mkdir(pc)
except OSError:
pass
os.chdir(pc)
if not os.path.isdir(START_SCRIPT_BIN):
print "%s can't be created, aborting!!!" % (START_SCRIPT_BIN)
exit(-1)
os.chdir(old_cwd)
shutil.copy(
'contrib/systemd/eddn_%s_config' % ( setup_env.EDDN_ENV),
'%s/eddn_%s_config' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV )
)
shutil.copy(
'contrib/systemd/start-eddn-service',
'%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV )
)
# Ensure the service log file archiving script is in place
print """
******************************************************************************
Ensuring the service log file archiving script is in place
"""
shutil.copy(
'contrib/eddn-logs-archive',
START_SCRIPT_BIN
)
# Ensure the latest monitor files are in place
old_umask = os.umask(022)
print """
******************************************************************************
Ensuring %s exists...
""" % ( SHARE_EDDN_FILES )
old_cwd = os.getcwd()
if not os.path.isdir(SHARE_EDDN_FILES):
# We're still using Python 2.7, so no pathlib
os.chdir('/')
for pc in SHARE_EDDN_FILES[1:].split('/'):
try:
os.mkdir(pc)
except OSError:
pass
os.chdir(pc)
if not os.path.isdir(SHARE_EDDN_FILES):
print "%s can't be created, aborting!!!" % (SHARE_EDDN_FILES)
exit(-1)
os.chdir(old_cwd)
print """
******************************************************************************
Ensuring latest monitor files are in place...
"""
# Copy the monitor (Web page) files
try:
shutil.rmtree('%s/monitor' % ( SHARE_EDDN_FILES ))
except OSError:
pass
shutil.copytree('contrib/monitor', '%s/monitor' % ( SHARE_EDDN_FILES ))
# And a copy of the schemas too
print """
******************************************************************************
Ensuring latest schema files are in place for web access...
"""
try:
shutil.rmtree('%s/schemas' % ( SHARE_EDDN_FILES ))
except OSError:
pass
shutil.copytree('schemas', '%s/schemas' % ( SHARE_EDDN_FILES ))
# You still need to make an override config file
if not os.path.isfile('%s/config.json' % ( SHARE_EDDN_FILES )):
shutil.copy('docs/config-EXAMPLE.json', SHARE_EDDN_FILES)
print """
******************************************************************************
There was no config.json file in place, so docs/config-EXAMPLE.json was
copied into:
%s
Please review, edit and rename this file to 'config.json' so that this
software will actually work.
See docs/Running-this-software.md for guidance.
******************************************************************************
""" % ( SHARE_EDDN_FILES )
os.umask(old_umask)

View File

@ -159,9 +159,8 @@ def parse_and_error_handle(data):
return "FAIL: " + str(validationResults.messages)
@app.post('/upload/')
@app.route('/upload/', method=['OPTIONS', 'POST'])
def upload():
response.set_header("Access-Control-Allow-Origin", "*")
try:
# Body may or may not be compressed.
message_body = get_decompressed_message()
@ -182,7 +181,7 @@ def upload():
return parse_and_error_handle(message_body)
@app.get('/health_check/')
@app.route('/health_check/', method=['OPTIONS', 'GET'])
def health_check():
"""
This should only be used by the gateway monitoring script. It is used
@ -192,9 +191,8 @@ def health_check():
return Settings.EDDN_VERSION
@app.get('/stats/')
@app.route('/stats/', method=['OPTIONS', 'GET'])
def stats():
response.set_header("Access-Control-Allow-Origin", "*")
stats = statsCollector.getSummary()
stats["version"] = Settings.EDDN_VERSION
return simplejson.dumps(stats)
@ -209,9 +207,29 @@ class MalformedUploadError(Exception):
pass
class EnableCors(object):
name = 'enable_cors'
api = 2
def apply(self, fn, context):
def _enable_cors(*args, **kwargs):
# set CORS headers
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS'
response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'
if request.method != 'OPTIONS':
# actual request; reply with the actual response
return fn(*args, **kwargs)
return _enable_cors
def main():
loadConfig()
configure()
app.install(EnableCors())
app.run(
host=Settings.GATEWAY_HTTP_BIND_ADDRESS,
port=Settings.GATEWAY_HTTP_PORT,

View File

@ -13,11 +13,12 @@ import collections
import zmq.green as zmq
import re
from bottle import get, request, response, run as bottle_run
from eddn.conf.Settings import Settings, loadConfig
from gevent import monkey
monkey.patch_all()
from bottle import Bottle, get, request, response, run
app = Bottle()
# This import must be done post-monkey-patching!
if Settings.RELAY_DUPLICATE_MAX_MINUTES:
@ -31,12 +32,12 @@ def date(__format):
return d.strftime(__format)
@get('/ping')
@app.route('/ping', method=['OPTIONS', 'GET'])
def ping():
return 'pong'
@get('/getTotalSoftwares/')
@app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET'])
def getTotalSoftwares():
response.set_header("Access-Control-Allow-Origin", "*")
db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database'])
@ -62,7 +63,7 @@ def getTotalSoftwares():
return simplejson.dumps(softwares)
@get('/getSoftwares/')
@app.route('/getSoftwares/', method=['OPTIONS', 'GET'])
def getSoftwares():
response.set_header("Access-Control-Allow-Origin", "*")
db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database'])
@ -91,7 +92,7 @@ def getSoftwares():
return simplejson.dumps(softwares)
@get('/getTotalSchemas/')
@app.route('/getTotalSchemas/', method=['OPTIONS', 'GET'])
def getTotalSchemas():
response.set_header("Access-Control-Allow-Origin", "*")
db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database'])
@ -113,7 +114,7 @@ def getTotalSchemas():
return simplejson.dumps(schemas)
@get('/getSchemas/')
@app.route('/getSchemas/', method=['OPTIONS', 'GET'])
def getSchemas():
response.set_header("Access-Control-Allow-Origin", "*")
db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database'])
@ -211,11 +212,36 @@ class Monitor(Thread):
gevent.spawn(monitor_worker, inboundMessage)
class EnableCors(object):
"""Enable CORS responses."""
name = 'enable_cors'
api = 2
def apply(self, fn, context):
"""
Apply a CORS handler.
Ref: <https://stackoverflow.com/a/17262900>
"""
def _enable_cors(*args, **kwargs):
"""Set CORS Headers."""
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS'
response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'
if request.method != 'OPTIONS':
# actual request; reply with the actual response
return fn(*args, **kwargs)
return _enable_cors
def main():
loadConfig()
m = Monitor()
m.start()
bottle_run(
app.install(EnableCors())
app.run(
host=Settings.MONITOR_HTTP_BIND_ADDRESS,
port=Settings.MONITOR_HTTP_PORT,
server='gevent',

View File

@ -18,11 +18,12 @@ import simplejson
import hashlib
import uuid
import zmq.green as zmq
from bottle import get, response, run as bottle_run
from eddn.conf.Settings import Settings, loadConfig
from gevent import monkey
monkey.patch_all()
from bottle import Bottle, get, request, response, run
app = Bottle()
# This import must be done post-monkey-patching!
from eddn.core.StatsCollector import StatsCollector
@ -36,9 +37,8 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES:
duplicateMessages.start()
@get('/stats/')
@app.route('/stats/', method=['OPTIONS', 'GET'])
def stats():
response.set_header("Access-Control-Allow-Origin", "*")
stats = statsCollector.getSummary()
stats["version"] = Settings.EDDN_VERSION
return simplejson.dumps(stats)
@ -145,11 +145,38 @@ class Relay(Thread):
gevent.spawn(relay_worker, inboundMessage)
class EnableCors(object):
"""Enable CORS responses."""
name = 'enable_cors'
api = 2
def apply(self, fn, context):
"""
Apply a CORS handler.
Ref: <https://stackoverflow.com/a/17262900>
"""
def _enable_cors(*args, **kwargs):
"""Set CORS Headers."""
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS'
response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'
if request.method != 'OPTIONS':
# actual request; reply with the actual response
return fn(*args, **kwargs)
return _enable_cors
def main():
loadConfig()
r = Relay()
r.start()
bottle_run(
app.install(EnableCors())
app.run(
host=Settings.RELAY_HTTP_BIND_ADDRESS,
port=Settings.RELAY_HTTP_PORT,
server='gevent',