Merge remote-tracking branch 'upstream/master'

This commit is contained in:
Jonathan Harris 2015-07-02 00:09:43 +01:00
commit 5fb1465de4
9 changed files with 380 additions and 141 deletions

View File

@ -10,8 +10,8 @@ EDDN Status: http://eddn-gateway.elite-markets.net/ | http://eddn.ed-td.space/
| -------------------------------------------------------------------------------------- | ------------- | --------------- | --------------- |
| [**E:D Market Connector**](https://github.com/Marginal/EDMarketConnector) | Otis B. | No | **Yes** (v1/v2) |
| [**edce-client**](https://github.com/Andargor/edce-client/) | Andargor | No | **Yes** (v1/v2) |
| [**EDDB**](http://eddb.io/) | themroc | **Yes** (v1) | No |
| [**ED-TD**](http://ed-tb.space/) | AnthorNet | **Yes** (v1/v2) | **Yes** (v2) |
| [**EDDB**](http://eddb.io/) | themroc | **Yes** (v1/v2) | No |
| [**ED-TD**](http://ed-td.space/) | AnthorNet | **Yes** (v1/v2) | **Yes** (v2) |
| [**EliteOCR**](https://forums.frontier.co.uk/showthread.php?t=68771) | seeebek | No | **Yes** (v1) |
| [**Maddavo's Market Data**](http://www.davek.com.au/td/) | maddavo | **Yes** (v1/v2) | **Yes** (v1/v2) |
| [**Maddavo's Market Data**](http://www.davek.com.au/td/) | maddavo | **Yes** (v1/v2) | **Yes** (v1/v2) |
| [**RegulatedNoise DJ-version**](https://forums.frontier.co.uk/showthread.php?t=137732) | Duke | **Yes** (v1) | **Yes** (v1) |

View File

@ -39,7 +39,7 @@
<ul class="nav navbar-nav">
<li><a href="#gateways">Gateways</a></li>
<li><a href="#relays">Relays</a></li>
<li><a href="#softwares">Softwares</a></li>
<li><a href="#software">Software</a></li>
<li><a href="#uploaders">Uploaders</a></li>
<li><a href="#schemas">Schemas</a></li>
</ul>
@ -100,6 +100,12 @@
<td class="invalid_5min stat">-</td>
<td class="invalid_60min stat">-</td>
</tr>
<tr>
<th>Outdated messages</th>
<td class="outdated_1min stat">-</td>
<td class="outdated_5min stat">-</td>
<td class="outdated_60min stat">-</td>
</tr>
<tr>
<th>Messages passed to relay</th>
<td class="outbound_1min stat">-</td>
@ -223,10 +229,10 @@
</div>
</section>
<section id="softwares" class="container">
<section id="software" class="container">
<div class="row">
<div class="col-md-6">
<h2>Softwares</h2>
<h2>Software</h2>
</div>
</div>
@ -260,7 +266,10 @@
</div>
<div class="row">
<div class="col-md-4 col-md-offset-8">
<div class="col-md-8">
Total hits calculated since May 5, 2015
</div>
<div class="col-md-4">
<p class="text-muted small">
Last updated: <strong><span class="update_timestamp">N/A</span></strong>.
</p>
@ -310,7 +319,10 @@
</div>
<div class="row">
<div class="col-md-4 col-md-offset-8">
<div class="col-md-8">
Total hits calculated since May 5, 2015
</div>
<div class="col-md-4">
<p class="text-muted small">
Last updated: <strong><span class="update_timestamp">N/A</span></strong>.
</p>
@ -326,7 +338,7 @@
<section id="schemas" class="container">
<div class="row">
<div class="col-md-6">
<h2>Schemas</h2>
<h2>Schemas <a href="./schemas.html"><span class="glyphicon glyphicon-search" aria-hidden="true"></span></a></h2>
</div>
</div>
@ -360,7 +372,10 @@
</div>
<div class="row">
<div class="col-md-4 col-md-offset-8">
<div class="col-md-8">
Total hits calculated since May 5, 2015
</div>
<div class="col-md-4">
<p class="text-muted small">
Last updated: <strong><span class="update_timestamp">N/A</span></strong>.
</p>
@ -392,4 +407,4 @@
<script src="http://code.highcharts.com/4.1.5/modules/drilldown.js"></script>
<script src="./js/eddn.js"></script>
</body>
</html>
</html>

View File

@ -76,7 +76,7 @@ var doUpdateSoftwares = function()
dataType: "json",
url: monitorEndPoint + 'getTotalSoftwares/',
success: function(softwaresTotalData){
var chart = $('#softwares .chart').highcharts(),
var chart = $('#software .chart').highcharts(),
series = chart.get('softwares');
// Count total by software, all versions included
@ -99,13 +99,13 @@ var doUpdateSoftwares = function()
tmp.sort(function(a,b) { return b.total - a.total; });
softwaresTotal = tmp;
$('#softwares .table tbody').empty();
$('#software .table tbody').empty();
// Prepare drilldowns
$.each(softwaresTotalData, function(software, hits){
softwareSplit = software.split(' | ');
$('#softwares .table tbody').append(
$('#software .table tbody').append(
newTr = $('<tr>').attr('data-type', 'drilldown').attr('data-parent', softwareSplit[0]).attr('data-name', software).on('mouseover', function(){
chart.get('software-' + makeSlug(software)).setState('hover');
chart.tooltip.refresh(chart.get('software-' + makeSlug(software)));
@ -144,7 +144,7 @@ var doUpdateSoftwares = function()
// Add main softwares
$.each(softwaresTotal, function(key, values){
$('#softwares .table tbody').append(
$('#software .table tbody').append(
newTr = $('<tr>').attr('data-type', 'parent').attr('data-name', values.name).on('click', function(event){
event.stopImmediatePropagation();
currentSoftware = $(this).attr('data-name');
@ -153,22 +153,22 @@ var doUpdateSoftwares = function()
{
currentDrillDown = currentSoftware;
$('#softwares .table thead th:eq(0)').html('<span class="glyphicon glyphicon-remove" aria-hidden="true"></span>')
$('#software .table thead th:eq(0)').html('<span class="glyphicon glyphicon-remove" aria-hidden="true"></span>')
.css('cursor','pointer')
.on('click', function(){
currentDrillDown = false;
chart.showDrillUpButton();
$('#softwares .table thead th:eq(0)').html('');
$('#softwares .table thead th:eq(1)').html('');
$('#softwares .table tbody tr[data-type=parent]').show();
$('#softwares .table tbody tr[data-type=drilldown]').hide();
$('#software .table thead th:eq(0)').html('');
$('#software .table thead th:eq(1)').html('');
$('#software .table tbody tr[data-type=parent]').show();
$('#software .table tbody tr[data-type=drilldown]').hide();
drillDownSoftware = !drillDownSoftware;
doUpdateSoftwares();
chart.drillUp();
});
$('#softwares .table thead th:eq(1)').html(currentSoftware);
$('#softwares .table tbody tr[data-type=parent]').hide();
$('#softwares .table tbody tr[data-type=drilldown][data-parent="' + currentSoftware + '"]').show();
$('#software .table thead th:eq(1)').html(currentSoftware);
$('#software .table tbody tr[data-type=parent]').hide();
$('#software .table tbody tr[data-type=drilldown][data-parent="' + currentSoftware + '"]').show();
var currentData = [];
@ -191,7 +191,7 @@ var doUpdateSoftwares = function()
if(chart.drillUpButton)
chart.drillUpButton = chart.drillUpButton.destroy();
$('#softwares .table tbody tr[data-type=drilldown][data-parent="' + currentSoftware + '"]').each(function(){
$('#software .table tbody tr[data-type=drilldown][data-parent="' + currentSoftware + '"]').each(function(){
$(this).find('.square').css('background', chart.get('software-' + makeSlug($(this).attr('data-name'))).color);
});
}
@ -237,18 +237,18 @@ var doUpdateSoftwares = function()
});
if(drillDownSoftware)
$('#softwares .table tbody tr[data-type=drilldown][data-parent="' + currentDrillDown + '"]').each(function(){
$('#software .table tbody tr[data-type=drilldown][data-parent="' + currentDrillDown + '"]').each(function(){
$(this).find('.square').css('background', chart.get('software-' + makeSlug($(this).attr('data-name'))).color);
});
chart.redraw();
$('#softwares').find(".stat").removeClass("warning").each(function() {
$('#software').find(".stat").removeClass("warning").each(function() {
if ($(this).html() == "0")
$(this).addClass("warning");
});
$('#softwares').find(".update_timestamp").html(d.toString("yyyy-MM-dd HH:mm:ss"));
$('#software').find(".update_timestamp").html(d.toString("yyyy-MM-dd HH:mm:ss"));
}
});
}
@ -451,6 +451,10 @@ var showStats = function(type, currentItem){
el.find(".invalid_1min").html((currentItemStats["invalid"] || {})['1min'] || 0);
el.find(".invalid_5min").html((currentItemStats["invalid"] || {})['5min'] || 0);
el.find(".invalid_60min").html((currentItemStats["invalid"] || {})['60min'] || 0);
el.find(".outdated_1min").html((currentItemStats["outdated"] || {})['1min'] || 0);
el.find(".outdated_5min").html((currentItemStats["outdated"] || {})['5min'] || 0);
el.find(".outdated_60min").html((currentItemStats["outdated"] || {})['60min'] || 0);
}
if(type == 'relays')
@ -525,8 +529,9 @@ var start = function(){
credits: { enabled: false },
exporting: { enabled: false },
series: [
{id: 'inbound', data: [], name: 'Messages received', zIndex: 300},
{id: 'invalid', data: [], name: 'Invalid messages', zIndex: 1},
{id: 'inbound', data: [], name: 'Messages received', zIndex: 300},
{id: 'invalid', data: [], name: 'Invalid messages', zIndex: 1},
{id: 'outdated', data: [], name: 'Outdated messages', zIndex: 1},
{id: 'outbound', data: [], name: 'Messages passed to relay', zIndex: 200}
]
}).hide();
@ -590,7 +595,7 @@ var start = function(){
}, updateInterval);
// Grab software from monitor
$('#softwares .chart').highcharts({
$('#software .chart').highcharts({
chart: {
type: 'pie', animation: Highcharts.svg
},

View File

@ -0,0 +1,312 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>EDDN Status - Schemas</title>
<!-- Bootstrap -->
<link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet" />
<link href="./css/eddn.css" rel="stylesheet" />
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
<script src="https://code.jquery.com/jquery-1.11.3.min.js"></script>
<script src="./js/date.js"></script>
</head>
<body data-spy="scroll" data-target="#header-nav" data-offset="70">
<header>
<nav class="navbar navbar-default navbar-fixed-top">
<div class="container">
<div class="navbar-header">
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#header-nav">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="">EDDN Status - Schemas</a>
</div>
<div class="collapse navbar-collapse" id="header-nav">
<ul class="nav navbar-nav">
<li><a href="./#schemas">Back to EDDN Status</a></li>
</ul>
</div>
</div>
</nav>
</header>
<section id="schemas" class="container">
<div class="row">
<div class="col-md-12">
<h2>Schemas (Last 60 days)</h2>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-body">
<div class="row">
<div class="col-md-12 charts" style="height: 500px;">
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<section id="schemasStacked" class="container">
<div class="row">
<div class="col-md-12">
<h2>Schemas in percent (Last 60 days)</h2>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="panel panel-default">
<div class="panel-body">
<div class="row">
<div class="col-md-12 charts" style="height: 500px;">
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<footer>
<nav class="navbar navbar-default">
<div class="container">
<p class="navbar-text">
The <em>Elite: Dangerous Data Network</em> is volunteer-run and not affiliated with <a href="https://frontier.co.uk/" target="_blank">Frontier Developments</a>.
|
Charts by <a href="http://www.highcharts.com/" target="_blank">HighCharts</a>
</p>
</div>
</nav>
</footer>
<!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
<!-- Include all compiled plugins (below), or include individual files as needed -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
<script src="http://code.highcharts.com/4.1.5/highcharts.js"></script>
<script type="text/javascript">
makeSlug = function(str) {
var slugcontent_hyphens = str.replace(/\s/g,'-');
var finishedslug = slugcontent_hyphens.replace(/[^a-zA-Z0-9\-]/g,'');
return finishedslug.toLowerCase();
}
$(document).ready(function(){
var monitorEndPoint = 'http://eddn-monitor.ed-td.space:9091/';
var chartContainer = $('#schemas .charts');
chartContainer.highcharts({
chart: {
type: 'spline', animation: Highcharts.svg,
events: {
load: function(){ setTimeout(function(){$(window).trigger('resize');}, 250); }
},
marginRight: 10
},
title: { text: '', style: {display: 'none'} },
xAxis: {
type: 'datetime',
//tickPixelInterval: 24 * 3600 * 1000
},
yAxis: {
title: {text: ''},
plotLines: [{value: 0, width: 1, color: '#808080'}],
min: 0
},
//tooltip: { enabled: false },
credits: { enabled: false },
exporting: { enabled: false },
series: [
{id: 'duplicate-message', data: [], name: 'Messages duplicate', zIndex: 1}
]
});
var chartStackedContainer = $('#schemasStacked .charts');
chartStackedContainer.highcharts({
chart: {
type: 'area', animation: Highcharts.svg,
events: {
load: function(){ setTimeout(function(){$(window).trigger('resize');}, 250); }
},
marginRight: 10
},
colors: ['#000000', '#7cb5ec', '#2b908f', '#f7a35c', '#f15c80', '#e4d354', '#2b908f', '#f45b5b', '#91e8e1'],
title: { text: '', style: {display: 'none'} },
xAxis: {
type: 'datetime',
//tickPixelInterval: 24 * 3600 * 1000
},
yAxis: {
title: {text: ''},
plotLines: [{value: 0, width: 1, color: '#808080'}],
min: 0
},
plotOptions: {
area: {
stacking: 'percent',
lineColor: '#ffffff',
lineWidth: 1,
marker: {
lineWidth: 1,
lineColor: '#ffffff'
}
}
},
tooltip: {
pointFormat: '<span style="color:{series.color}">{series.name}</span>: <b>{point.percentage:.1f}%</b> ({point.y:,.0f})<br/>',
shared: true
},
credits: { enabled: false },
exporting: { enabled: false },
series: [
{id: 'duplicate-message', data: [], name: 'Messages duplicate', zIndex: 1}
]
});
var dYesterday = new (function(d){ d.setDate(d.getDate()-1); return d})(new Date),
d60 = new (function(d){ d.setDate(d.getDate()-60); return d})(new Date),
day60 = d60.getUTCFullYear() + '-' + ("0" + (d60.getUTCMonth() + 1)).slice(-2) + '-' + ("0" + (d60.getUTCDate())).slice(-2),
yesterday = dYesterday.getUTCFullYear() + '-' + ("0" + (dYesterday.getUTCMonth() + 1)).slice(-2) + '-' + ("0" + (dYesterday.getUTCDate())).slice(-2);
$.ajax({
dataType: "json",
url: monitorEndPoint + 'getSchemas/?dateStart=' + day60 + '&dateEnd = ' + yesterday,
success: function(schemas){
var chart = chartContainer.highcharts();
var chartStacked = chartStackedContainer.highcharts();
$.each(schemas, function(date, schemasCount){
// Convert date for highcharts
date = date.split('-')
date = Date.UTC(date[0], date[1] - 1, date[2]);
$.each(schemasCount, function(schema, hits){
// IF TEST CONTINUE
if(schema.substr(schema.length - 4) == 'test')
return;
// Check series exists
if(!chart.get(makeSlug(schema)))
{
// Format name
if(makeSlug(schema) == 'httpschemaselite-marketsneteddncommodity1')
name = 'Commodity v1';
else if(makeSlug(schema) == 'httpschemaselite-marketsneteddncommodity2')
name = 'Commodity v2';
else if(makeSlug(schema) == 'httpschemaselite-marketsneteddnshipyard1')
name = 'Shipyard v1';
else
name = schema;
chart.addSeries({
id: makeSlug(schema),
name: name,
data: []
});
if(name == 'Commodity v2')
chart.addSeries({
id: makeSlug(schema + '(Projected)'),
name: name + ' (Projected * 55.0287)',
data: []
});
}
// Add data
serie = chart.get(makeSlug(schema));
serie.addPoint({x: date, y: parseInt(hits)}, false);
// Add projected
if(makeSlug(schema) == 'httpschemaselite-marketsneteddncommodity2')
{
serie = chart.get(makeSlug(schema + '(Projected)'));
serie.addPoint({x: date, y: Math.round(parseInt(hits)*55.0287)}, false);
}
// Check series exists
if(!chartStacked.get(makeSlug(schema)))
{
// Format name
if(makeSlug(schema) == 'httpschemaselite-marketsneteddncommodity1')
name = 'Commodity v1';
else if(makeSlug(schema) == 'httpschemaselite-marketsneteddncommodity2')
name = 'Commodity v2';
else if(makeSlug(schema) == 'httpschemaselite-marketsneteddnshipyard1')
name = 'Shipyard v1';
else
name = schema;
if(name == 'Commodity v2' && !chartStacked.get(makeSlug(schema + '(Projected)')))
chartStacked.addSeries({
id: makeSlug(schema + '(Projected)'),
name: name + ' (Projected * 55.0287)',
data: []
});
else
{
if(name != 'Commodity v2')
chartStacked.addSeries({
id: makeSlug(schema),
name: name,
data: []
});
}
}
// Add projected
if(makeSlug(schema) == 'httpschemaselite-marketsneteddncommodity2')
{
serie = chartStacked.get(makeSlug(schema + '(Projected)'));
serie.addPoint({x: date, y: Math.round(parseInt(hits)*55.0287)}, false);
}
// Add data
else
{
serie = chartStacked.get(makeSlug(schema));
serie.addPoint({x: date, y: parseInt(hits)}, false);
}
});
});
chart.redraw();
chartStacked.redraw();
}
});
});
</script>
</body>
</html>

View File

@ -1,102 +0,0 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "http://schemas.elite-markets.net/eddn/commodity/1#",
"type": "object",
"additionalProperties": false,
"properties": {
"$schemaRef": {
"type": "string",
"additionalProperties": false
},
"header": {
"type": "object",
"additionalProperties": true,
"properties": {
"uploaderID": {
"type": "string",
"additionalProperties": false
},
"softwareName": {
"type": "string",
"additionalProperties": false
},
"softwareVersion": {
"type": "string",
"additionalProperties": false
},
"gatewayTimestamp": {
"type": "string",
"format": "date-time",
"description": "Timestamp upon receipt at the gateway. If present, this property will be overwritten by the gateway; submitters are not intended to populate this property.",
"additionalProperties": false
}
}
},
"message": {
"type": "object",
"additionalProperties": true,
"properties": {
"systemName": {
"type": "string",
"additionalProperties": false
},
"stationName": {
"type": "string",
"additionalProperties": false
},
"itemName": {
"type": "string",
"additionalProperties": false
},
"buyPrice": {
"type": "integer",
"description": "Price to buy from the market",
"additionalProperties": false
},
"stationStock": {
"type": "integer",
"additionalProperties": false
},
"supplyLevel": {
"$ref": "#/definitions/levelType"
},
"sellPrice": {
"type": "integer",
"description": "Price to sell to the market",
"additionalProperties": false
},
"demand": {
"type": "integer",
"additionalProperties": false
},
"demandLevel": {
"$ref": "#/definitions/levelType"
},
"timestamp": {
"type": "string",
"format": "date-time",
"additionalProperties": false
}
},
"required": [
"systemName",
"stationName",
"itemName",
"stationStock",
"sellPrice",
"demand",
"timestamp"
]
}
},
"required": [
"$schemaRef",
"header",
"message"
],
"definitions" : {
"levelType" : {
"enum": ["Low", "Med", "High"]
}
}
}

View File

@ -45,7 +45,7 @@ def configure():
sender.bind(binding)
for schemaRef, schemaFile in Settings.GATEWAY_JSON_SCHEMAS.iteritems():
validator.addSchemaResource(schemaRef, resource_string(__name__, schemaFile))
validator.addSchemaResource(schemaRef, resource_string('eddn.Gateway', schemaFile))
def push_message(string_message, topic):
@ -130,6 +130,12 @@ def parse_and_error_handle(data):
logger.error("Error to %s: %s" % (get_remote_address(), exc.message))
return str(exc)
# Here we check if an outdated schema has been passed
if parsed_message["$schemaRef"] in Settings.GATEWAY_OUTDATED_SCHEMAS:
response.status = '426 Upgrade Required' # Bottle (and underlying httplib) don't know this one
statsCollector.tally("outdated")
return "FAIL: The schema you have used is no longer supported. Please check for an updated version of your application."
validationResults = validator.validate(parsed_message)
if validationResults.severity <= ValidationSeverity.WARN:
@ -206,7 +212,6 @@ class MalformedUploadError(Exception):
def main():
loadConfig()
configure()
run(host='0.0.0.0', port=Settings.GATEWAY_HTTP_PORT, server='gevent')

View File

@ -38,13 +38,18 @@ class _Settings(object):
GATEWAY_IP_KEY_SALT = None
GATEWAY_JSON_SCHEMAS = {
"http://schemas.elite-markets.net/eddn/commodity/1": "schemas/commodity-v0.1.json",
"http://schemas.elite-markets.net/eddn/commodity/1/test": "schemas/commodity-v0.1.json",
"http://schemas.elite-markets.net/eddn/commodity/2": "schemas/commodity-v2.0.json",
"http://schemas.elite-markets.net/eddn/commodity/2/test": "schemas/commodity-v2.0.json"
"http://schemas.elite-markets.net/eddn/commodity/2/test": "schemas/commodity-v2.0.json",
"http://schemas.elite-markets.net/eddn/shipyard/1": "schemas/shipyard-v1.0.json",
"http://schemas.elite-markets.net/eddn/shipyard/1/test": "schemas/shipyard-v1.0.json"
}
GATEWAY_OUTDATED_SCHEMAS = [
"http://schemas.elite-markets.net/eddn/commodity/1",
"http://schemas.elite-markets.net/eddn/commodity/1/test"
]
###############################################################################
# Monitor settings
###############################################################################

View File

@ -1,3 +1,2 @@
# coding: utf8
__version__ = "0.4"
# This should be a version number as understood by setuptools
__version__ = "0.5"