mirror of
https://github.com/EDCD/EDMarketConnector.git
synced 2025-04-08 05:20:03 +03:00
More rigorous validation of data from the Companion API.
This commit is contained in:
Jonathan Harris
parent
f39c369322
commit
0fb1f51f33
27
bpc.py
27
bpc.py
@ -7,7 +7,11 @@ import numbers
|
||||
import time
|
||||
|
||||
from config import config
|
||||
from companion import categorymap, commoditymap, bracketmap
|
||||
|
||||
bracketmap = { 0: '',
|
||||
1: 'Low',
|
||||
2: 'Med',
|
||||
3: 'High', }
|
||||
|
||||
def export(data, csv=False):
|
||||
|
||||
@ -26,16 +30,15 @@ def export(data, csv=False):
|
||||
h.write(header)
|
||||
|
||||
for commodity in data['lastStarport']['commodities']:
|
||||
if isinstance(commodity.get('demandBracket'), numbers.Integral) and commodity.get('categoryname') and categorymap.get(commodity['categoryname'], True):
|
||||
h.write(('%s;%s;%s;%s;%s;%s;%s;%s;%s;\n' % (
|
||||
rowheader,
|
||||
commoditymap.get(commodity['name'].strip(), commodity['name'].strip()),
|
||||
commodity.get('sellPrice') and int(commodity['sellPrice']) or '',
|
||||
commodity.get('buyPrice') and int(commodity['buyPrice']) or '',
|
||||
int(commodity.get('demand', 0)) if commodity.get('demandBracket') else '',
|
||||
bracketmap.get(commodity.get('demandBracket'), ''),
|
||||
int(commodity.get('stock', 0)) if commodity.get('stockBracket') else '',
|
||||
bracketmap.get(commodity.get('stockBracket'), ''),
|
||||
timestamp)).encode('utf-8'))
|
||||
h.write(('%s;%s;%s;%s;%s;%s;%s;%s;%s;\n' % (
|
||||
rowheader,
|
||||
commodity['name'],
|
||||
commodity['sellPrice'] or '',
|
||||
commodity['buyPrice'] or '',
|
||||
int(commodity['demand']) if commodity['demandBracket'] else '',
|
||||
bracketmap[commodity['demandBracket']],
|
||||
int(commodity['stock']) if commodity['stockBracket'] else '',
|
||||
bracketmap[commodity['stockBracket']],
|
||||
timestamp)).encode('utf-8'))
|
||||
|
||||
h.close()
|
||||
|
||||
57
companion.py
57
companion.py
@ -5,6 +5,7 @@ import json
|
||||
import requests
|
||||
from collections import defaultdict
|
||||
from cookielib import LWPCookieJar
|
||||
import numbers
|
||||
import os
|
||||
from os.path import dirname, join
|
||||
from requests.packages import urllib3
|
||||
@ -37,10 +38,6 @@ commoditymap= { 'Agricultural Medicines': 'Agri-Medicines',
|
||||
'S A P8 Core Container': 'SAP 8 Core Container',
|
||||
'Terrain Enrichment Systems': 'Land Enrichment Systems', }
|
||||
|
||||
bracketmap = { 1: 'Low',
|
||||
2: 'Med',
|
||||
3: 'High', }
|
||||
|
||||
|
||||
class ServerError(Exception):
|
||||
def __str__(self):
|
||||
@ -130,10 +127,11 @@ class Session:
|
||||
|
||||
r.raise_for_status()
|
||||
try:
|
||||
return json.loads(r.text)
|
||||
data = json.loads(r.text)
|
||||
except:
|
||||
self.dump(r)
|
||||
raise ServerError()
|
||||
return self.fixup(data)
|
||||
|
||||
def close(self):
|
||||
self.state = Session.STATE_NONE
|
||||
@ -144,8 +142,57 @@ class Session:
|
||||
pass
|
||||
self.session = None
|
||||
|
||||
|
||||
# Fixup anomalies in the recieved commodity data
|
||||
def fixup(self, data):
|
||||
commodities = data['lastStarport']['commodities']
|
||||
i=0
|
||||
while i<len(commodities):
|
||||
commodity = commodities[i]
|
||||
|
||||
# Check all required numeric fields are present and are numeric
|
||||
# Catches "demandBracket": "" for some phantom commodites in ED 1.3
|
||||
for thing in ['buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket']:
|
||||
if not isinstance(commodity.get(thing), numbers.Number):
|
||||
if __debug__: print 'Invalid "%s":"%s" (%s) for "%s"' % (thing, commodity.get(thing), type(commodity.get(thing)), commodity.get('name', ''))
|
||||
break
|
||||
else:
|
||||
if not categorymap.get(commodity['categoryname'], True): # Check marketable
|
||||
pass
|
||||
elif not commodity.get('categoryname', '').strip():
|
||||
if __debug__: print 'Missing "categoryname" for "%s"' % commodity.get('name', '')
|
||||
elif not commodity.get('name', '').strip():
|
||||
if __debug__: print 'Missing "name" for a commodity in "%s"' % commodity.get('categoryname', '')
|
||||
elif not commodity['demandBracket'] in range(4):
|
||||
if __debug__: print 'Invalid "demandBracket":"%s" for "%s"' % (commodity['demandBracket'], commodity['name'])
|
||||
elif not commodity['stockBracket'] in range(4):
|
||||
if __debug__: print 'Invalid "stockBracket":"%s" for "%s"' % (commodity['stockBracket'], commodity['name'])
|
||||
else:
|
||||
# Rewrite text fields
|
||||
commodity['categoryname'] = categorymap.get(commodity['categoryname'].strip(),
|
||||
commodity['categoryname'].strip())
|
||||
commodity['name'] = commoditymap.get(commodity['name'].strip(),
|
||||
commodity['name'].strip())
|
||||
|
||||
# Force demand and stock to zero if their corresponding bracket is zero
|
||||
# Fixes spurious "demand": 1 in ED 1.3
|
||||
if not commodity['demandBracket']:
|
||||
commodity['demand'] = 0
|
||||
if not commodity['stockBracket']:
|
||||
commodity['stock'] = 0
|
||||
|
||||
# We're good
|
||||
i+=1
|
||||
continue
|
||||
|
||||
# Skip the commodity
|
||||
commodities.pop(i)
|
||||
|
||||
return data
|
||||
|
||||
def dump(self, r):
|
||||
if __debug__:
|
||||
print 'Status\t%s' % r.status_code
|
||||
print 'URL\t%s' % r.url
|
||||
print 'Headers\t%s' % r.headers
|
||||
print ('Content:\n%s' % r.text).encode('utf-8')
|
||||
|
||||
49
eddn.py
49
eddn.py
@ -9,11 +9,14 @@ from sys import platform
|
||||
import time
|
||||
|
||||
from config import applongname, appversion, config
|
||||
from companion import categorymap, commoditymap, bracketmap
|
||||
|
||||
upload = 'http://eddn-gateway.elite-markets.net:8080/upload/'
|
||||
schema = 'http://schemas.elite-markets.net/eddn/commodity/1'
|
||||
|
||||
bracketmap = { 1: 'Low',
|
||||
2: 'Med',
|
||||
3: 'High', }
|
||||
|
||||
def export(data, callback):
|
||||
|
||||
callback('Sending data to EDDN...')
|
||||
@ -36,26 +39,30 @@ def export(data, callback):
|
||||
for commodity in commodities:
|
||||
i = i+1
|
||||
callback('Sending %d/%d' % (i, len(commodities)))
|
||||
if isinstance(commodity.get('demandBracket'), numbers.Integral) and commodity.get('categoryname') and categorymap.get(commodity['categoryname'], True):
|
||||
msg = { '$schemaRef': schema,
|
||||
'header': header,
|
||||
'message': {
|
||||
'systemName': systemName,
|
||||
'stationName': stationName,
|
||||
'itemName': commoditymap.get(commodity['name'].strip(), commodity['name'].strip()),
|
||||
'buyPrice': int(commodity.get('buyPrice', 0)),
|
||||
'stationStock': commodity.get('stockBracket') and int(commodity.get('stock', 0)) or 0,
|
||||
'sellPrice': int(commodity.get('sellPrice', 0)),
|
||||
'demand': commodity.get('demandBracket') and int(commodity.get('demand', 0)) or 0,
|
||||
'timestamp': timestamp,
|
||||
}
|
||||
}
|
||||
if commodity.get('stockBracket'):
|
||||
msg['message']['supplyLevel'] = bracketmap.get(commodity['stockBracket'])
|
||||
if commodity.get('demandBracket'):
|
||||
msg['message']['demandLevel'] = bracketmap.get(commodity['demandBracket'])
|
||||
data = { '$schemaRef': schema,
|
||||
'header': header,
|
||||
'message': {
|
||||
'systemName': systemName,
|
||||
'stationName': stationName,
|
||||
'itemName': commodity['name'],
|
||||
'buyPrice': commodity['buyPrice'],
|
||||
'stationStock': int(commodity['stock']),
|
||||
'sellPrice': commodity['sellPrice'],
|
||||
'demand': int(commodity['demand']),
|
||||
'timestamp': timestamp,
|
||||
}
|
||||
}
|
||||
if commodity['stockBracket']:
|
||||
data['message']['supplyLevel'] = bracketmap[commodity['stockBracket']]
|
||||
if commodity['demandBracket']:
|
||||
data['message']['demandLevel'] = bracketmap[commodity['demandBracket']]
|
||||
|
||||
r = session.post(upload, data=json.dumps(msg))
|
||||
r.raise_for_status()
|
||||
r = session.post(upload, data=json.dumps(data))
|
||||
if __debug__ and r.status_code != requests.codes.ok:
|
||||
print 'Status\t%s' % r.status_code
|
||||
print 'URL\t%s' % r.url
|
||||
print 'Headers\t%s' % r.headers
|
||||
print ('Content:\n%s' % r.text).encode('utf-8')
|
||||
r.raise_for_status()
|
||||
|
||||
session.close()
|
||||
|
||||
29
td.py
29
td.py
@ -4,13 +4,21 @@ from os.path import join
|
||||
from collections import defaultdict
|
||||
import codecs
|
||||
import numbers
|
||||
from operator import itemgetter
|
||||
from platform import system
|
||||
from sys import platform
|
||||
import time
|
||||
|
||||
from config import applongname, appversion, config
|
||||
from companion import categorymap, commoditymap, bracketmap
|
||||
|
||||
demandbracketmap = { 0: '?',
|
||||
1: 'L',
|
||||
2: 'M',
|
||||
3: 'H', }
|
||||
stockbracketmap = { 0: '-',
|
||||
1: 'L',
|
||||
2: 'M',
|
||||
3: 'H', }
|
||||
|
||||
def export(data):
|
||||
|
||||
@ -27,21 +35,20 @@ def export(data):
|
||||
# sort commodities by category
|
||||
bycategory = defaultdict(list)
|
||||
for commodity in data['lastStarport']['commodities']:
|
||||
if isinstance(commodity.get('demandBracket'), numbers.Integral) and commodity.get('categoryname') and categorymap.get(commodity['categoryname'], True):
|
||||
bycategory[categorymap.get(commodity['categoryname'], commodity['categoryname'])].append(commodity)
|
||||
bycategory[commodity['categoryname']].append(commodity)
|
||||
|
||||
for category in sorted(bycategory):
|
||||
h.write(' + %s\n' % category)
|
||||
# corrections to commodity names can change the sort order
|
||||
for commodity in sorted(bycategory[category], key=lambda x:commoditymap.get(x['name'].strip(),x['name'])):
|
||||
for commodity in sorted(bycategory[category], key=itemgetter('name')):
|
||||
h.write(' %-23s %7d %7d %9s%c %8s%c %s\n' % (
|
||||
commoditymap.get(commodity['name'].strip(), commodity['name'].strip()),
|
||||
commodity.get('sellPrice', 0),
|
||||
commodity.get('buyPrice', 0),
|
||||
int(commodity.get('demand', 0)) if commodity.get('demandBracket') else '',
|
||||
bracketmap.get(commodity.get('demandBracket'), '?')[0],
|
||||
int(commodity.get('stock', 0)) if commodity.get('stockBracket') else '',
|
||||
bracketmap.get(commodity.get('stockBracket'), '-')[0],
|
||||
commodity['name'],
|
||||
commodity['sellPrice'],
|
||||
commodity['buyPrice'],
|
||||
int(commodity['demand']) if commodity['demandBracket'] else '',
|
||||
demandbracketmap[commodity['demandBracket']],
|
||||
int(commodity['stock']) if commodity['stockBracket'] else '',
|
||||
stockbracketmap[commodity['stockBracket']],
|
||||
timestamp))
|
||||
|
||||
h.close()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user