diff --git a/.gitignore b/.gitignore index fdebc55..9bff30b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# These need to be per-environment, so not changed by git merges. +setup.cfg +setup_env.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -61,3 +65,5 @@ target/ # Editors .idea +# VIM swap files +.*.sw? diff --git a/contrib/apache-eddn.conf b/contrib/apache-eddn.conf new file mode 100644 index 0000000..23842e5 --- /dev/null +++ b/contrib/apache-eddn.conf @@ -0,0 +1,166 @@ +# vim: :filetype=apache + +########################################################################### +# +# Read **ALL** the comments in this file, don't blindly use it! +# +# Be sure to replace 'YOUROWN.eddn.edcd.io' with your hostname. +# +# Also edit the DocumentRoot and related statements if you use a +# different path. +# +# Ensure the CustomLog directory actually exists, else apache will not +# start, or die on a restart/reload. +# +########################################################################### + +## YOUROWN.eddn.edcd.io + + ServerName YOUROWN.eddn.edcd.io + + DocumentRoot /home/eddn/.local/share/eddn/dev + + ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined + + # Comment these out when initially requesting a LetsEncrypt cert + Redirect / https://YOUROWN.eddn.edcd.io/ + RedirectMatch "/^(.*)$" "https://YOUROWN.eddn.edcd.io/$1" + + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + + + + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + AllowOverride All + + Require all granted + + + Require all denied + + + Include partials/default-directory.conf + + + + +# This will need to be commented out/disabled for initial LetsEncrypt +# certificate request, as you don't have the certificate yet! + + + SSLEngine On + SSLCertificateFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem + + ServerName YOUROWN.eddn.edcd.io + + DocumentRoot /home/eddn/.local/share/eddn/YOUROWN/monitor + + ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined + + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + + + + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + AllowOverride All + + Require all granted + + + Require all denied + + + + # Serve the schemas + Alias /schemas/ /home/eddn/.local/share/eddn/YOUROWN/schemas/ + + # netdata (performance info) + + Redirect /netdata /netdata/ + + + SetOutputFilter DEFLATE + + + Require all granted + + + Require all denied + + + + SSLProxyEngine On + SSLProxyVerify none + ProxyPreserveHost On + + # Yes, plain http for this. + ProxyPass "/netdata/" "http://127.0.0.1:19999/" + + + + + +# This is for the Gateway public URLs + +# This will need to be commented out/disabled for initial LetsEncrypt +# certificate request, as you don't have the certificate yet! +# You also need to ensure `Listen 4430` is in ports.conf + + SSLEngine On + SSLCertificateFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem + + ServerName YOUROWN.eddn.edcd.io + + DocumentRoot /home/eddn/.local/share/eddn/YOUROWN/monitor + + ErrorLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/error.log + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/YOUROWN.eddn.edcd.io/access.log combined + + # LetsEncrypt + Alias /.well-known/ /var/www/letsencrypt/.well-known/ + + Options -Indexes + + + + + + Require all granted + + + Require all denied + + + + SSLProxyEngine On + SSLProxyVerify none + ProxyPreserveHost On + ProxyRequests Off + + # Must be https, not http, as the Gateway process is + # expecting only https requests. + ProxyPass "/" "https://127.0.0.1:8081/" + ProxyPassReverse "/" "https://127.0.0.1:8081/" + + + + diff --git a/contrib/eddn-logs-archive b/contrib/eddn-logs-archive new file mode 100755 index 0000000..79b10dd --- /dev/null +++ b/contrib/eddn-logs-archive @@ -0,0 +1,82 @@ +#!/bin/bash +# Add ' -x' above to debug +# + +########################################################################### +# Configuration +########################################################################### +# Maximum age, in days, of log files to keep +MAX_LOGFILE_AGE=28 +# Minimum size of live log before rotating, see find(1) -size for format +MIN_ROTATE_SIZE="100M" +########################################################################### + +########################################################################### +# Helper functions +########################################################################### +################################################## +# Print program usage information. +################################################## +usage() { + echo "Usage: $(basename $1) [ live | beta | dev ]" +} +################################################## + +########################################################################### + +########################################################################### +# Check command line arguments +########################################################################### +EDDN_ENV="$1" +if [ -z "${EDDN_ENV}" ]; +then + usage $0 + exit 1 +fi +########################################################################### + +########################################################################### +# Perform rotation +########################################################################### +LOGS_DIR="${HOME}/${EDDN_ENV}/logs" +if [ ! -d "${LOGS_DIR}" ]; +then + echo "$(dirname): Logs directory doesn't exist: ${LOGS_DIR}" + exit 2 +fi + +cd ${LOGS_DIR} || exit 3 + +for service in gateway monitor relay ; +do + echo "Service: ${service}" + echo " Expiring old logs..." + find . -name "${service}.log.*.gz" -a -atime +${MAX_LOGFILE_AGE} -exec rm -fv {} \; + echo " DONE" + + echo " Checking if current logfile needs archiving..." + if [ ! -z "$(find . -name ${service}.log -a -size +${MIN_ROTATE_SIZE})" ]; + then + echo " Archiving ${service}.log ..." + # We have no means to tell the service to close and re-open output, it's + # to stdout/err anyway. So we copy it. + COMPRESSED_NAME="${service}.log.$(date --iso-8601=seconds)" + cp ${service}.log "${COMPRESSED_NAME}" + if [ $? -ne 0 ]; + then + echo " FAILED copying live log file to new archive!!!" + echo " Exiting from any further processing." + exit 4 + fi + # Truncate the live file. + :> ${service}.log + # Now compress the newly archived log + gzip -9v "${COMPRESSED_NAME}" + echo " DONE" + else + echo " No" + fi +done +########################################################################### + +# vim: tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0 diff --git a/contrib/init.d/eddn-gateway b/contrib/init.d/eddn-gateway index 3b87b73..cd90dde 100644 --- a/contrib/init.d/eddn-gateway +++ b/contrib/init.d/eddn-gateway @@ -13,7 +13,7 @@ DESC="eddn-gateway" PIDFILE="/var/run/${NAME}.pid" LOGFILE="/var/log/eddn/${NAME}.log" -DAEMON="/usr/local/bin/${NAME}" +DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}" EXEC_AS_USER="root" @@ -50,4 +50,4 @@ restart|force-reload) ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/contrib/init.d/eddn-monitor b/contrib/init.d/eddn-monitor index 2d73441..ec5727c 100644 --- a/contrib/init.d/eddn-monitor +++ b/contrib/init.d/eddn-monitor @@ -13,7 +13,7 @@ DESC="eddn-monitor" PIDFILE="/var/run/${NAME}.pid" LOGFILE="/var/log/eddn/${NAME}.log" -DAEMON="/usr/local/bin/${NAME}" +DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}" EXEC_AS_USER="root" @@ -50,4 +50,4 @@ restart|force-reload) ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/contrib/init.d/eddn-relay b/contrib/init.d/eddn-relay index 24b3d79..bd9bcff 100644 --- a/contrib/init.d/eddn-relay +++ b/contrib/init.d/eddn-relay @@ -13,7 +13,7 @@ DESC="eddn-relay" PIDFILE="/var/run/${NAME}.pid" LOGFILE="/var/log/eddn/${NAME}.log" -DAEMON="/usr/local/bin/${NAME}" +DAEMON="/home/eddn/eddn/python-venv/bin/${NAME}" EXEC_AS_USER="root" @@ -50,4 +50,4 @@ restart|force-reload) ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/contrib/letsencrypt/certbot-common b/contrib/letsencrypt/certbot-common new file mode 100644 index 0000000..23f8e47 --- /dev/null +++ b/contrib/letsencrypt/certbot-common @@ -0,0 +1,35 @@ +########################################################################### +# Copy a certificate's files into place, with appropriate ownership and +# mode. +# +# $1 - Name of certificate (i.e. letsencrypt directory names). +# $2 - Source Directory +# $3 - Destination filename for fullchain.pem +# $4 - Destination filename for privkey.pem +# $5 - File ownership to set (user:group) +# $6 - File mode to set (as passed to 'chmod') +########################################################################### +copy_cert() { + CERT_NAME="$1" + SRC_DIR="$2" + DST_FILE_FULLCHAIN="$3" + DST_FILE_PRIVKEY="$4" + CERT_NEW_OWNER="$5" + CERT_NEW_PERMS="$6" + + echo "${CERT_NAME}: Copying new files into place..." + + # Preserve only the mode as it should be 0600, and thus we won't + # temporarily open up the files for *all* users to read, + # BUT don't preserve the timestamp as we want it to be 'now' so + # that a `find ... -newer ` check works later. + cp -v --preserve=mode ${SRC_DIR}/fullchain.pem ${DST_FILE_FULLCHAIN} + cp -v --preserve=mode ${SRC_DIR}/privkey.pem ${DST_FILE_PRIVKEY} + chown -v ${CERT_NEW_OWNER} ${DST_FILE_FULLCHAIN} ${DST_FILE_PRIVKEY} + chmod -v ${CERT_NEW_PERMS} ${DST_FILE_FULLCHAIN} ${DST_FILE_PRIVKEY} + + echo "${CERT_NAME}: Copying new files into place DONE" +} +########################################################################### + +# vim: :set filetype=sh tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0 diff --git a/contrib/letsencrypt/deploy-changed-certs b/contrib/letsencrypt/deploy-changed-certs new file mode 100755 index 0000000..0986a54 --- /dev/null +++ b/contrib/letsencrypt/deploy-changed-certs @@ -0,0 +1,89 @@ +#!/bin/bash +# Add " -x" above to debug +# +# certbot deploy hook +# +# This should be triggered by being present in: +# +# /etc/letsencrypt/renewal-hooks/deploy/ +# +# It can be linked into the 'post' directory for testing with: +# +# certbot renew --dry-run +# +# which you might want to do because deploy hooks aren't run for that +# command. +# +# You can also just straight up run this script, including to get into place +# any certificate files it's configured for, but have never been deployed. + +# Paranoia re-enforcement of no group/other perms on created files +chmod -R og-rwx /etc/letsencrypt/archive + +echo "$0 - Running in: $(pwd)" +# Import common code and settings. +. /etc/scripts/certbot-common + +# As of 2021-07-02 and certbot 0.31.0 (current in Debian buster) +# there is **zero** information passed in (CL args or environment) to +# this hook. So we just need to check each potentially renewed +# certificate. + +########################################################################### +# MAIN_HOST_NAME +########################################################################### +CERT_NAME="MAIN_HOST_NAME" +# We're only interested if it's newer than when the files were last copied +SRC_DIR="/etc/letsencrypt/live/${CERT_NAME}" +DST_FILE_FULLCHAIN="/etc/exim4/exim.crt" +DST_FILE_PRIVKEY="/etc/exim4/exim.key" +CERT_NEW_OWNER="root:Debian-exim" +CERT_NEW_PERMS="440" + +############################################################# +# Needs to be in place for exim to use +############################################################# +# 'find' doesn't set exit status depending on if it found anything, that's +# for actual errors, so we test against the output. +if [ "$(find ${SRC_DIR} -newer ${DST_FILE_FULLCHAIN} -o -newer ${DST_FILE_PRIVKEY} )" != "" ]; +then + echo "${CERT_NAME}: (Re)new(ed) certificate..." + + copy_cert "${CERT_NAME}" "${SRC_DIR}" "${DST_FILE_FULLCHAIN}" "${DST_FILE_PRIVKEY}" "${CERT_NEW_OWNER}" "${CERT_NEW_PERMS}" + + echo "${CERT_NAME}: DONE" +fi +############################################################# + +########################################################################### + +########################################################################### +# eddn.edcd.io and related names +########################################################################### +CERT_NEW_OWNER="eddn:eddn" +CERT_NEW_PERMS="400" + +for eddn in eddn.edcd.io test.eddn.edcd.io staging.eddn.edcd.io ; +do + CERT_NAME="${eddn}" + SRC_DIR="/etc/letsencrypt/live/${CERT_NAME}" + DST_FILE_FULLCHAIN="/home/eddn/etc/${CERT_NAME}-fullchain.pem" + DST_FILE_PRIVKEY="/home/eddn/etc/${CERT_NAME}-privkey.pem" + + if [ -d "${SRC_DIR}" ]; + then + if [ ! -f "${DST_FILE_FULLCHAIN}" \ + -o ! -f "${DST_FILE_PRIVKEY}" \ + -o "$(find ${SRC_DIR} -newer ${DST_FILE_FULLCHAIN} -o -newer ${DST_FILE_PRIVKEY} )" != "" ]; + then + echo "${CERT_NAME}: (Re)New(ed) certificate..." + + copy_cert "${CERT_NAME}" "${SRC_DIR}" "${DST_FILE_FULLCHAIN}" "${DST_FILE_PRIVKEY}" "${CERT_NEW_OWNER}" "${CERT_NEW_PERMS}" + + echo "${CERT_NAME}: DONE" + fi + fi +done +########################################################################### + +# vim: tabstop=2 shiftwidth=2 expandtab wrapmargin=0 textwidth=0 diff --git a/contrib/run-from-source.sh b/contrib/run-from-source.sh index 871534c..85e587e 100644 --- a/contrib/run-from-source.sh +++ b/contrib/run-from-source.sh @@ -17,8 +17,17 @@ do echo "$d: Already running as $(cat ${LOGPATH}/${d}.pid)" continue fi + if [ -f "${BASEPATH}/etc/settings.json" ]; + then + CONFIG="--config ${BASEPATH}/etc/settings.json" + else + echo "WARNING: No override settings found, you'll be using defaults" + echo "WARNING: Did you forget to make ${BASEPATH}/etc/settings.json ?" + echo " Continuing anyway..." + CONFIG="" + fi ${PYTHON} -m eddn.${d} \ - --config ${BASEPATH}/etc/settings.json \ + ${CONFIG} \ > ${LOGPATH}/$d.log \ 2>&1 & echo $! > "${LOGPATH}/${d}.pid" diff --git a/contrib/systemd/eddn@.service b/contrib/systemd/eddn@.service index 541516b..9ef01de 100644 --- a/contrib/systemd/eddn@.service +++ b/contrib/systemd/eddn@.service @@ -7,7 +7,7 @@ [Unit] Description=EDDN Service %i -AssertPathExists=/home/eddn/.local/bin/%i +AssertPathExists=/home/eddn/eddn/python-venv/bin/%i PartOf=eddn.service ReloadPropagatedFrom=eddn.service Before=eddn.service @@ -18,7 +18,7 @@ After=network.target Type=simple User=eddn Group=eddn -ExecStart=/home/eddn/.local/bin/start-%i +ExecStart=/home/eddn/.local/bin/start-eddn-service %i TimeoutStartSec=10s TimeoutStopSec=10s SyslogIdentifier=eddn@%i diff --git a/contrib/systemd/eddn_beta_config b/contrib/systemd/eddn_beta_config new file mode 100644 index 0000000..4677abc --- /dev/null +++ b/contrib/systemd/eddn_beta_config @@ -0,0 +1,3 @@ +CONFIG_OVERRIDE="${HOME}/.local/share/eddn/beta/config.json" +LOG_DIR="${HOME}/beta/logs" +PYTHON_VENV="${HOME}/beta/python-venv" diff --git a/contrib/systemd/eddn_config b/contrib/systemd/eddn_config deleted file mode 100644 index 6fbe7d3..0000000 --- a/contrib/systemd/eddn_config +++ /dev/null @@ -1,2 +0,0 @@ -CONFIG_OVERRIDE="${HOME}/.local/share/eddn/config.json" -LOG_DIR="${HOME}/.var/log/eddn" diff --git a/contrib/systemd/eddn_dev_config b/contrib/systemd/eddn_dev_config new file mode 100644 index 0000000..cdfdee2 --- /dev/null +++ b/contrib/systemd/eddn_dev_config @@ -0,0 +1,3 @@ +CONFIG_OVERRIDE="${HOME}/.local/share/eddn/dev/config.json" +LOG_DIR="${HOME}/dev/logs" +PYTHON_VENV="${HOME}/dev/python-venv" diff --git a/contrib/systemd/eddn_live_config b/contrib/systemd/eddn_live_config new file mode 100644 index 0000000..1ce0827 --- /dev/null +++ b/contrib/systemd/eddn_live_config @@ -0,0 +1,3 @@ +CONFIG_OVERRIDE="${HOME}/.local/share/eddn/live/config.json" +LOG_DIR="${HOME}/live/logs" +PYTHON_VENV="${HOME}/live/python-venv" diff --git a/contrib/systemd/start-eddn-gateway b/contrib/systemd/start-eddn-gateway deleted file mode 100755 index 6c71da4..0000000 --- a/contrib/systemd/start-eddn-gateway +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab -# -# Start the EDDN Gateway, including redirecting output to a log file. - -EXEC_PATH=$(dirname $0) -#echo "EXEC_PATH: ${EXEC_PATH}" - -# Ensure we're in the correct place -cd ${EXEC_PATH} -#pwd - -# Bring in some common configuration -. ./eddn_config - -./eddn-gateway --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-gateway.log" 2>&1 diff --git a/contrib/systemd/start-eddn-monitor b/contrib/systemd/start-eddn-monitor deleted file mode 100755 index 717e3e2..0000000 --- a/contrib/systemd/start-eddn-monitor +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab -# -# Start the EDDN Gateway, including redirecting output to a log file. - -EXEC_PATH=$(dirname $0) -#echo "EXEC_PATH: ${EXEC_PATH}" - -# Ensure we're in the correct place -cd ${EXEC_PATH} -#pwd - -# Bring in some common configuration -. ./eddn_config - -./eddn-monitor --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-monitor.log" 2>&1 diff --git a/contrib/systemd/start-eddn-relay b/contrib/systemd/start-eddn-relay deleted file mode 100755 index cb5bd82..0000000 --- a/contrib/systemd/start-eddn-relay +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab -# -# Start the EDDN Gateway, including redirecting output to a log file. - -EXEC_PATH=$(dirname $0) -#echo "EXEC_PATH: ${EXEC_PATH}" - -# Ensure we're in the correct place -cd ${EXEC_PATH} -#pwd - -# Bring in some common configuration -. ./eddn_config - -./eddn-relay --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/eddn-relay.log" 2>&1 diff --git a/contrib/systemd/start-eddn-service b/contrib/systemd/start-eddn-service new file mode 100755 index 0000000..4f27af1 --- /dev/null +++ b/contrib/systemd/start-eddn-service @@ -0,0 +1,50 @@ +#!/bin/sh +# vim: tabstop=2 shiftwidth=2 textwidth=0 wrapmargin=0 expandtab +# +# Start an EDDN Service, including redirecting output to a log file. + +usage() { + echo "Usage: $(basename $0) [ gateway | monitor | relay ] [ live | beta | dev ]" +} + +if [ -z "${1}" ]; +then + usage + echo "No EDDN service specified." + exit 3 +fi +SERVICE="${1}" + +if [ -z "${2}" ]; +then + usage + echo "No EDDN environment specified." + exit 3 +fi +EDDN_ENV="${2}" + +EXEC_PATH=$(dirname $0) +#echo "EXEC_PATH: ${EXEC_PATH}" + +# Ensure we're in the correct place +cd ${EXEC_PATH} +#pwd + +# Bring in some common configuration +if [ ! -f "eddn_${EDDN_ENV}_config" ]; +then + echo "eddn_${EDDN_ENV}_config is missing from $(pwd)" + exit 1 +fi +. "./eddn_${EDDN_ENV}_config" + +# Use the python venv +. "${PYTHON_VENV}/bin/activate" + +if [ ! -f "${PYTHON_VENV}/bin/eddn-${SERVICE}" ]; +then + echo "${SERVICE} is missing from ${PYTHON_VENV}/bin" + exit 2 +fi + +exec ${PYTHON_VENV}/bin/eddn-${SERVICE} --config "${CONFIG_OVERRIDE}" >> "${LOG_DIR}/${SERVICE}.log" 2>&1 diff --git a/docs/Running-this-software.md b/docs/Running-this-software.md index 924ac5f..7fee97b 100644 --- a/docs/Running-this-software.md +++ b/docs/Running-this-software.md @@ -1,3 +1,5 @@ + These instructions are based on getting the software up and running from scratch on a Debian Buster (10.9, stable as of 2021-05-16) system. @@ -17,13 +19,15 @@ A specific user was created: useradd -c 'EDDN Gateway' -m -s /bin/bash eddn +--- + # Further installation ## As 'root' Some additional Debian packages and python modules are required: - apt install python-pip + apt install python-pip virtualenv You will need a mysql/mariab database: @@ -36,6 +40,8 @@ You will need a mysql/mariab database: > GRANT ALL PRIVILEGES on eddn.* TO 'eddn'@'localhost'; > \q +--- + ### Netdata In order to get host performance metrics (CPU, RAM and network usage) you will need to install netdata. On Debian-based systems: @@ -45,21 +51,27 @@ need to install netdata. On Debian-based systems: The default configuration should be all you need, listening on `127.0.0.1:19999`. -### LetsEncrypt: certbot -It will be necessary to renew the TLS certificate using certbot (or some -alternative ACME client). +--- + +### LetsEncrypt +We assume that you're using a TLS certificate from +[LetsEncrypt](https://letsencrypt.org/), it's free! + +It will be necessary to renew the TLS certificate using certbot, or some +alternative ACME client. We'll assume certbot. + +#### Install certbot +On a Debian system simply: apt install certbot -### Reverse Proxy with nginx -If you don't yet have nginx installed then start with: - - apt install nginx-light +Although this version might be a little old now, it does work. #### LetsEncrypt TLS Certificates +If you are taking over hosting the EDDN relay then hopefully you have access +to the existing certificate files. -You will need a LetsEncrupt/ACME client in order to keep the TLS certificate -renewed. +So, first copy those into place: cd /etc/letsencrypt mkdir -p archive/eddn.edcd.io @@ -74,7 +86,127 @@ renewed. ln -s ../../archive/eddn.edcd.io/fullchain1.pem fullchain.pem ln -s ../../archive/eddn.edcd.io/privkey1.pem privkey.pem -#### nginx configuration +After this you need to ensure that the certificate stays renewed. With a +Debian system using certbot: + +1. There should already be a systemd timer set up: + + `systemctl status certbot.timer` + + If that doesn't show "`; enabled;`" in: + + `Loaded: loaded (/lib/systemd/system/certbot.timer; enabled; vendor preset: enabled)` + + then: + + `systemctl enable certbot.timer` + + This will renew the certificate as necessary (i.e. when <= 30 days until + it expires, or whatever current LetsEncrypt and certbot policy causes). + But it will not ensure the files are in all the places you might need + them to be. + +1. Ensure the certificate files are deployed to where they're needed. When + using the certbot timer the easiest thing to do is to utilise a script in + `/etc/letsencrypt/renewal-hooks/deploy/`. + + There are example files for this in `contrib/letsencrypt/`: + + mkdir -p /etc/letsencrypt/renewal-hooks/deploy + cp contrib/letsencrypt/deploy-changed-certs /etc/letsencrypt/renewal-hooks/deploy + mkdir -p /etc/scripts + cp contrib/letsencrypt/certbot-common /etc/scripts/ + + **Remember to edit them to suit your setup!** + +--- + +### Network Configuration +There are multiple ports that you'll have to ensure are allowed through any +firewall, and some of them also require being reverse proxied correctly. + +The reverse proxies pertain to: + +1. The port for the Gateway to receive uploads from senders (e.g. Elite + Dangerous Market Connector). This is also used for the 'monitor' web + page to obtain stats about messages passing through the Gateway. + +1. A set of URLs for accessing [netdata](#netdata). + +#### Necessary ports +These all for TCP, no UDP: + +1. `443` - a web server capable of reverse proxying set up for TLS on the + public host name of the EDDN service. This is used to serve the schemas, + the monitor web page, and to reverse proxy URLs beginning `/netdata/` to + the [netdata](#netdata) service. + +1. Default: `4430` - Gateway 'http' port, used both for EDDN senders to + upload, and also for the Gateway message rate stats on the monitor web + page. + + But that's the *public* port. The Gateway process itself listens on `8081`. + So you'll need a reverse proxy listening on port `4430` and forwarding + *all* requests to `127.0.0.1:8081`. + +1. Default: `9091` - Monitor 'http' port, used for the monitor web page to + query schema and software statistics. No reverse proxy setup. + +1. Default: `9500` - The port on the Relay that EDDN listeners connect to in + order to receive the zeromq stream. No reverse proxy setup. + +1. Default: `9090` - The Relay 'http' port for its portion of the message + statistics on the monitor web page. No reverse proxy setup. + +There's also the internal `8500` port, but that's literally only used for +the Monitor and Relay to pick up zeromq messages forwarded from the +Gateway, so all over localhost. + +See [Configuration](#configuration) for guidance on what override config +settings can be used to change any of these ports. + +--- + +#### Reverse Proxy with Apache +If you already have an Apache installation it will be easier to just use +it for the reverse proxy. + +Ensure you have these modules installed and active: + + a2enmod proxy proxy_http + +##### Apache configuration +There is an example VirtualHost configuration in +`contrib/apache-eddn.conf` which makes the following assumptions: + + 1. The usual Apache default configuration is in place elsewhere. + 1. The hostname being used - `ServerName`. + 1. The location of the monitor files - `DocumentRoot`. + 1. The location of the schema files - `Alias /schemas/ ...`. + 1. The location of the TLS certificate files - `SSLCertificateFile` and + `SSLCertificateKeyFile. + +You should be able to: + + 1. Copy `contrib/apache-eddn.conf` into `/etc/apache/sites-available/` + *as an appropriate filename for the hostname you're using*. + 1. Edit to suit the local situation/setup. **Remember to ensure the + configured log directory exists.** + 1. Enable the site: + + a2ensite + apache2ctl configtest + # CHECK THE OUTPUT + apache2ctl graceful + +--- + +#### Reverse Proxy with nginx +If you don't yet have nginx installed then start with: + + apt install nginx-light + +##### nginx configuration There is an example configuration in `contrib/nginx-eddn.conf` which makes some assumptions: @@ -83,7 +215,7 @@ some assumptions: 1. The location of the monitor files - `root` directive. 1. The location of the schema files - `location` directive. 1. The location of the TLS certificate files - `ssl_certificate` and - `ssl_certificate_key` directives. + `ssl_certificate_key` directives. You should be able to: @@ -95,33 +227,42 @@ You should be able to: ln -s /etc/nginx/sites-available/eddn systemctl restart nginx.service -If you're already using another web server, such as Apache, you'll need to +If you're already using another web server you'll need to duplicate at least the use of a TLS certificate and the Reverse Proxying as required. -For Apache you would reverse proxy using something like the following in an -appropriate `` section: - - - SSLProxyEngine On - SSLProxyVerify none - ProxyPreserveHost On - - # Pass through 'gateway' upload URL to Debian VM - ProxyPass "/upload/" "https://EDDNHOST:8081/upload/" - # Pass through 'monitor' URLs to Debian VM - ProxyPass "/" "https://EDDNHOST/" - +--- ## In the 'eddn' account ### Clone a copy of the application project from gitub - mkdir -p ~/eddn/dev - cd ~/eddn/dev - git clone https://github.com/EDCD/EDDN.git + mkdir -p ${HOME}/dev + cd ${HOME}/dev + git clone https://github.com/EDCD/EDDN.git EDDN.git + cd EDDN.git -We'll assume this `~/eddn/dev/EDDN` path elsewhere in this document. +We'll assume this `${HOME}/dev/EDDN.git` path elsewhere in this document. + +### Set up a python virtual environment +So as to not have any python package version requirements clash with +anything else it's best to use a Python virtual environment (venv). You +will have installed the Debian package 'virtualenv' [above](#as-root) for +this purpose. + +We'll put the venv in `${HOME}/dev/python2.7-venv` with the following +command: + + cd ${HOME}/dev + virtualenv -p /usr/bin/python2.7 ${HOME}/python2.7-venv + +And for future ease of changing python versions: + + ln -s python2.7-venv python-venv + +And now start using this venv: + + . python-venv/bin/activate ### Ensure necessary python modules are installed Installing extra necessary python modules is simple: @@ -131,24 +272,12 @@ Installing extra necessary python modules is simple: ### Initialise Database Schema You will need to get the database schema in place: - mysql -p eddn < ~/eddn/dev/EDDN/schema.sql + mysql -p eddn < ${HOME}/eddn/dev/EDDN/schema.sql -### Monitor and Schema files -The Monitor files are not currently installed anywhere by the `setup.py` -script, so you'll need to manually copy them into somewhere convenient, -e.g.: +Ref: [As root](#as-root). - mkdir -p ${HOME}/.local/share/eddn - cp -r ~/eddn/dev/EDDN/contrib/monitor ${HOME}/.local/share/eddn - chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn - -You will need to ensure that the Monitor nginx setup can see the schema files -in order to serve them for use by the Gateway. So perform, e.g.: - - mkdir -p ${HOME}/.local/share/eddn - cp -r ~/eddn/dev/EDDN/schemas ${HOME}/.local/share/eddn - chmod -R og+rX ${HOME}/.local/share/eddn/schemas +--- # Concepts There are three components to this application. @@ -181,7 +310,11 @@ test host. The files in question are: monitor/js/eddn.js monitor/schemas.html -Replace the string `eddn.edcd.io` with the hostname you're using. +Replace the string `eddn.edcd.io` with the hostname you're using. You'll need +to perform similar substitutions if you change the configuration to use any +different port numbers. + +--- # Configuration Default application configuration is in the file `src/eddn/conf/Settings.py`. @@ -191,36 +324,45 @@ another file. 1. You will need to obtain a TLS certificate from, e.g. LetsEncrypt. The application will need access to this and its private key file. - CERT_FILE = '/etc/letsencrypt/live/eddn.edcd.io/fullchain.pem' - KEY_FILE = '/etc/letsencrypt/live/eddn.edcd.io/privkey.pem' + CERT_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/fullchain.pem' + KEY_FILE = '/etc/letsencrypt/live/YOUROWN.eddn.edcd.io/privkey.pem' 1. Network configuration 1. `RELAY_HTTP_BIND_ADDRESS` and `RELAY_HTTP_PORT` define the IP and port on which the Relay listens for, e.g. `/stats/` requests. + 1. `RELAY_RECEIVER_BINDINGS` defines where the Relay connects in order to subscribe to messages from the Gateway. Should match `GATEWAY_SENDER_BINDINGS`. + 1. `RELAY_SENDER_BINDINGS` defines the address the application listens on for connections from listeners such as eddb.io. + 1. `RELAY_DUPLICATE_MAX_MINUTES` how many minutes to keep messages hashes cached for so as to detect, and not Relay out, duplicate messages. If you set this to the literal string `false` the duplication checks will be disabled. This is **very handy** when testing the code. + 1. `GATEWAY_HTTP_BIND_ADDRESS` and `GATEWAY_HTTP_PORT` define where the Gateway listens to for incoming messages from senders. Might be forwarded from nginx or other reverse proxy. + 1. `GATEWAY_SENDER_BINDINGS` is where the Gateway listens for connections from the Relay and Monitor in order to send them messages that passed schema checks. + 1. `GATEWAY_JSON_SCHEMAS` defines the schemas used for validation. Note - that these are full public URLs which are served by nginx (or whatever - else you're using as the reverse proxy). + that these are full public URLs which are served by your web server. + 1. `GATEWAY_OUTDATED_SCHEMAS` any past schemas that are no longer valid. + 1. `MONITOR_HTTP_BIND_ADDRESS` and `MONITOR_HTTP_PORT` define where the Monitor listens to for web connections, e.g. the statistics page. + 1. `MONITOR_RECEIVER_BINDINGS` defines where the Monitor connects in order to subscribe to messages from the Gateway. Should match `GATEWAY_SENDER_BINDINGS`. + 1. `MONITOR_UA` appears to be unused. 1. Database Configuration @@ -228,8 +370,8 @@ another file. connect to a mysql/mariadb database for storing stats. 1. `database` - the name of the database 1. `user` - the user to connect as - 1. `password` - the secure password you set above when installing and - configuring mariadb/mysql. + 1. `password` - the secure password you set [above](#as-root) when + installing and configuring mariadb/mysql. It is assumed that the database is on `localhost`. @@ -253,35 +395,34 @@ It sets: 1. Configures the database connection and credentials. 1. Turns off the relay duplicate check. +--- + # Running You have some choices for how to run the application components: -1. You can choose to run this application directly from the source using the - provided script in `contrib/run-from-source.sh`. +1. If you are just testing out code changes then you can choose to run + this application directly from the source using the provided script in + `contrib/run-from-source.sh`. -1. Or you can utilise the `setup.py` file to build and install the application - files. By default this requires write permissions under `/usr/local`, but - you can run: +1. Otherwise you will want to utilise the `setup.py` file to build and + install the application files. As we're using a python venv we can just + run: - python setup.py install --user + python setup.py install - to install under `~/.local/` instead. + to install it all. This will install a python egg into the python + venv, and then also ensure that the monitor and schema files are in + place. There is an example systemd setup in `contrib/systemd` that assumes this local installation. - If you install into `/usr/local/` then there are SysV style init.d scripts - in `contrib/init.d/` for running the components. They will need the - `DAEMON` lines tweaking for running from another location. + There are also some SysV style init.d scripts in `contrib/init.d/` for + running the components. They will need the `DAEMON` lines tweaking for + running from another location. -1. For quick testing purposes you can run them as follows, assuming you - installed into `~/.local/`, and have your override settings in - `${HOME}/etc/eddn-settings-overrides.json`: +--- - ~/.local/bin/eddn-gateway --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-gateway.log 2>&1 & - ~/.local/bin/eddn-monitor --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-monitor.log 2>&1 & - ~/.local/bin/eddn-relay --config ${HOME}/etc/eddn-settings-overrides.json >> ~/logs/eddn-relay.log 2>&1 & - # Accessing the Monitor There is an EDDN Status web page usually provided at, e.g. https://eddn.edcd.io/. This is enabled by the Monitor component through @@ -291,7 +432,21 @@ by the Monitor process itself. You will need to configure a reverse proxy to actually enable access to this. There is an example nginx configuration in `contrib/nginx-eddn.conf`. -## Testing all of this in a VM +The necessary files should be put in place by + +The 'monitor' files are what form the status/statistics page at +https://eddn.edcd.io/, so they need to be installed somewhere in a +static manner accessible to nginx. + +Although setup.py installs the files you might still need to ensure the +permissions are correct for your web server to access them. + + chmod -R og+rX ${HOME} ${HOME}/.local ${HOME}/.local/share ${HOME}/.local/share/eddn + chmod -R og+rX ${HOME}/.local/share/eddn/schemas + +--- + +# Testing all of this in a VM In order to test all of this in a VM you might need to set up a double proxying: diff --git a/docs/config-EXAMPLE.json b/docs/config-EXAMPLE.json new file mode 100644 index 0000000..45e81bf --- /dev/null +++ b/docs/config-EXAMPLE.json @@ -0,0 +1,12 @@ +{ + "CERT_FILE": "/home/eddn/etc/fullchain.pem", + "KEY_FILE": "/home/eddn/etc/privkey.pem", + + "GATEWAY_HTTP_BIND_ADDRESS": "0.0.0.0", + + "MONITOR_DB": { + "database": "eddn", + "user": "eddn", + "password": "SOME SECURE PASSWORD" + }, +} diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index a35e2cd..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[egg_info] -tag_build = .dev \ No newline at end of file diff --git a/setup.py b/setup.py index e4d2943..bfd0c6b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ -from setuptools import setup, find_packages -import re import glob +import os +import re +import shutil +from setuptools import setup, find_packages VERSIONFILE = "src/eddn/conf/Version.py" @@ -15,26 +17,163 @@ except EnvironmentError: print "unable to find version in %s" % (VERSIONFILE,) raise RuntimeError("if %s exists, it is required to be well-formed" % (VERSIONFILE,)) +# Read environment-specific settings +import setup_env + +# Location of start-eddn-service script and its config file +START_SCRIPT_BIN='%s/.local/bin' % ( os.environ['HOME'] ) +# Location of web files +SHARE_EDDN_FILES='%s/.local/share/eddn/%s' % ( os.environ['HOME'], setup_env.EDDN_ENV ) setup( name='eddn', version=verstr, description='Elite: Dangerous Data Network', - author='Anthor (EDSM)', - author_email='contact@edsm.net', - url='https://github.com/EDSM-NET/EDDN', - packages=find_packages('src', exclude=["*.tests"]), - package_dir = {'':'src'}, - data_files=[('eddn/schemas', glob.glob("schemas/*.json"))], long_description="""\ - The Elite: Dangerous Data Network allows E:D players to share data. Not affiliated with Frontier Developments. + The Elite Dangerous Data Network allows ED players to share data. Not affiliated with Frontier Developments. """, - install_requires=["argparse", "bottle", "enum34", "gevent", "jsonschema", "pyzmq", "strict_rfc3339", "simplejson", "mysql-connector-python"], + author='EDCD (https://edcd.github.io/)', + author_email='edcd@miggy.org', + url='https://github.com/EDCD/EDDN', + + packages=find_packages( + 'src', + exclude=["*.tests"] + ), + package_dir = {'':'src'}, + + # This includes them for the running code, but that doesn't help + # serve them up for reference. + data_files=[ + ( + 'eddn/schemas', glob.glob("schemas/*.json") + ) + ], + + # Yes, we pin versions. With python2.7 the latest pyzmq will NOT + # work, for instance. + install_requires=[ + "argparse", + "bottle==0.12.15", + "enum34==1.1.6", + "gevent==1.3.7", + "jsonschema==2.6.0", + "pyzmq==17.1.2", + "strict_rfc3339==0.7", + "simplejson==3.16.0", + "mysql-connector-python==8.0.17" + ], + entry_points={ 'console_scripts': [ 'eddn-gateway = eddn.Gateway:main', 'eddn-relay = eddn.Relay:main', 'eddn-monitor = eddn.Monitor:main', - ], - } - ) + ], + } +) + +# Ensure the systemd-required start files are in place +print """ +****************************************************************************** +Ensuring start script and its config file are in place... +""" +old_cwd = os.getcwd() +if not os.path.isdir(START_SCRIPT_BIN): + # We're still using Python 2.7, so no pathlib + os.chdir('/') + for pc in START_SCRIPT_BIN[1:].split('/'): + try: + os.mkdir(pc) + + except OSError: + pass + + os.chdir(pc) + + if not os.path.isdir(START_SCRIPT_BIN): + print "%s can't be created, aborting!!!" % (START_SCRIPT_BIN) + exit(-1) + +os.chdir(old_cwd) + +shutil.copy( + 'contrib/systemd/eddn_%s_config' % ( setup_env.EDDN_ENV), + '%s/eddn_%s_config' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) +) +shutil.copy( + 'contrib/systemd/start-eddn-service', + '%s/start-eddn-%s-service' % ( START_SCRIPT_BIN, setup_env.EDDN_ENV ) +) + +# Ensure the service log file archiving script is in place +print """ +****************************************************************************** +Ensuring the service log file archiving script is in place +""" +shutil.copy( + 'contrib/eddn-logs-archive', + START_SCRIPT_BIN +) + +# Ensure the latest monitor files are in place +old_umask = os.umask(022) +print """ +****************************************************************************** +Ensuring %s exists... +""" % ( SHARE_EDDN_FILES ) +old_cwd = os.getcwd() +if not os.path.isdir(SHARE_EDDN_FILES): + # We're still using Python 2.7, so no pathlib + os.chdir('/') + for pc in SHARE_EDDN_FILES[1:].split('/'): + try: + os.mkdir(pc) + + except OSError: + pass + + os.chdir(pc) + + if not os.path.isdir(SHARE_EDDN_FILES): + print "%s can't be created, aborting!!!" % (SHARE_EDDN_FILES) + exit(-1) + +os.chdir(old_cwd) +print """ +****************************************************************************** +Ensuring latest monitor files are in place... +""" +# Copy the monitor (Web page) files +try: + shutil.rmtree('%s/monitor' % ( SHARE_EDDN_FILES )) +except OSError: + pass +shutil.copytree('contrib/monitor', '%s/monitor' % ( SHARE_EDDN_FILES )) +# And a copy of the schemas too +print """ +****************************************************************************** +Ensuring latest schema files are in place for web access... +""" +try: + shutil.rmtree('%s/schemas' % ( SHARE_EDDN_FILES )) +except OSError: + pass +shutil.copytree('schemas', '%s/schemas' % ( SHARE_EDDN_FILES )) + +# You still need to make an override config file +if not os.path.isfile('%s/config.json' % ( SHARE_EDDN_FILES )): + shutil.copy('docs/config-EXAMPLE.json', SHARE_EDDN_FILES) + print """ +****************************************************************************** +There was no config.json file in place, so docs/config-EXAMPLE.json was +copied into: + + %s + +Please review, edit and rename this file to 'config.json' so that this +software will actually work. +See docs/Running-this-software.md for guidance. +****************************************************************************** +""" % ( SHARE_EDDN_FILES ) +os.umask(old_umask) diff --git a/src/eddn/Gateway.py b/src/eddn/Gateway.py index 1531156..63d10d7 100644 --- a/src/eddn/Gateway.py +++ b/src/eddn/Gateway.py @@ -159,9 +159,8 @@ def parse_and_error_handle(data): return "FAIL: " + str(validationResults.messages) -@app.post('/upload/') +@app.route('/upload/', method=['OPTIONS', 'POST']) def upload(): - response.set_header("Access-Control-Allow-Origin", "*") try: # Body may or may not be compressed. message_body = get_decompressed_message() @@ -182,7 +181,7 @@ def upload(): return parse_and_error_handle(message_body) -@app.get('/health_check/') +@app.route('/health_check/', method=['OPTIONS', 'GET']) def health_check(): """ This should only be used by the gateway monitoring script. It is used @@ -192,9 +191,8 @@ def health_check(): return Settings.EDDN_VERSION -@app.get('/stats/') +@app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): - response.set_header("Access-Control-Allow-Origin", "*") stats = statsCollector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) @@ -209,9 +207,29 @@ class MalformedUploadError(Exception): pass +class EnableCors(object): + name = 'enable_cors' + api = 2 + + def apply(self, fn, context): + def _enable_cors(*args, **kwargs): + # set CORS headers + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' + response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + + if request.method != 'OPTIONS': + # actual request; reply with the actual response + return fn(*args, **kwargs) + + return _enable_cors + + def main(): loadConfig() configure() + + app.install(EnableCors()) app.run( host=Settings.GATEWAY_HTTP_BIND_ADDRESS, port=Settings.GATEWAY_HTTP_PORT, diff --git a/src/eddn/Monitor.py b/src/eddn/Monitor.py index 0d9cc3d..4c188d4 100644 --- a/src/eddn/Monitor.py +++ b/src/eddn/Monitor.py @@ -13,11 +13,12 @@ import collections import zmq.green as zmq import re -from bottle import get, request, response, run as bottle_run from eddn.conf.Settings import Settings, loadConfig from gevent import monkey monkey.patch_all() +from bottle import Bottle, get, request, response, run +app = Bottle() # This import must be done post-monkey-patching! if Settings.RELAY_DUPLICATE_MAX_MINUTES: @@ -31,12 +32,12 @@ def date(__format): return d.strftime(__format) -@get('/ping') +@app.route('/ping', method=['OPTIONS', 'GET']) def ping(): return 'pong' -@get('/getTotalSoftwares/') +@app.route('/getTotalSoftwares/', method=['OPTIONS', 'GET']) def getTotalSoftwares(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -62,7 +63,7 @@ def getTotalSoftwares(): return simplejson.dumps(softwares) -@get('/getSoftwares/') +@app.route('/getSoftwares/', method=['OPTIONS', 'GET']) def getSoftwares(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -91,7 +92,7 @@ def getSoftwares(): return simplejson.dumps(softwares) -@get('/getTotalSchemas/') +@app.route('/getTotalSchemas/', method=['OPTIONS', 'GET']) def getTotalSchemas(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -113,7 +114,7 @@ def getTotalSchemas(): return simplejson.dumps(schemas) -@get('/getSchemas/') +@app.route('/getSchemas/', method=['OPTIONS', 'GET']) def getSchemas(): response.set_header("Access-Control-Allow-Origin", "*") db = mariadb.connect(user=Settings.MONITOR_DB['user'], password=Settings.MONITOR_DB['password'], database=Settings.MONITOR_DB['database']) @@ -211,11 +212,36 @@ class Monitor(Thread): gevent.spawn(monitor_worker, inboundMessage) +class EnableCors(object): + """Enable CORS responses.""" + + name = 'enable_cors' + api = 2 + + def apply(self, fn, context): + """ + Apply a CORS handler. + + Ref: + """ + def _enable_cors(*args, **kwargs): + """Set CORS Headers.""" + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' + response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + + if request.method != 'OPTIONS': + # actual request; reply with the actual response + return fn(*args, **kwargs) + + return _enable_cors + def main(): loadConfig() m = Monitor() m.start() - bottle_run( + app.install(EnableCors()) + app.run( host=Settings.MONITOR_HTTP_BIND_ADDRESS, port=Settings.MONITOR_HTTP_PORT, server='gevent', diff --git a/src/eddn/Relay.py b/src/eddn/Relay.py index f69b495..10629d4 100644 --- a/src/eddn/Relay.py +++ b/src/eddn/Relay.py @@ -18,11 +18,12 @@ import simplejson import hashlib import uuid import zmq.green as zmq -from bottle import get, response, run as bottle_run from eddn.conf.Settings import Settings, loadConfig from gevent import monkey monkey.patch_all() +from bottle import Bottle, get, request, response, run +app = Bottle() # This import must be done post-monkey-patching! from eddn.core.StatsCollector import StatsCollector @@ -36,9 +37,8 @@ if Settings.RELAY_DUPLICATE_MAX_MINUTES: duplicateMessages.start() -@get('/stats/') +@app.route('/stats/', method=['OPTIONS', 'GET']) def stats(): - response.set_header("Access-Control-Allow-Origin", "*") stats = statsCollector.getSummary() stats["version"] = Settings.EDDN_VERSION return simplejson.dumps(stats) @@ -145,11 +145,38 @@ class Relay(Thread): gevent.spawn(relay_worker, inboundMessage) +class EnableCors(object): + """Enable CORS responses.""" + + name = 'enable_cors' + api = 2 + + def apply(self, fn, context): + """ + Apply a CORS handler. + + Ref: + """ + def _enable_cors(*args, **kwargs): + """Set CORS Headers.""" + response.headers['Access-Control-Allow-Origin'] = '*' + response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' + response.headers['Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' + + if request.method != 'OPTIONS': + # actual request; reply with the actual response + return fn(*args, **kwargs) + + return _enable_cors + + def main(): loadConfig() r = Relay() r.start() - bottle_run( + + app.install(EnableCors()) + app.run( host=Settings.RELAY_HTTP_BIND_ADDRESS, port=Settings.RELAY_HTTP_PORT, server='gevent',