1
0
mirror of https://github.com/EDCD/EDMarketConnector.git synced 2025-04-21 11:27:38 +03:00

Merge pull request from A-UNDERSCORE-D/enhancement/cleanup-companion

Cleanup companion.py
This commit is contained in:
Athanasius 2020-07-09 16:58:22 +01:00 committed by GitHub
commit cd0352d506
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -4,35 +4,41 @@ from builtins import object
import base64
import csv
import requests
from http.cookiejar import LWPCookieJar # No longer needed but retained in case plugins use it
# TODO: see https://github.com/EDCD/EDMarketConnector/issues/569
from http.cookiejar import LWPCookieJar # No longer needed but retained in case plugins use it
from email.utils import parsedate
import hashlib
import json
import numbers
import os
from os.path import dirname, isfile, join
from os.path import join
import random
import sys
import time
from traceback import print_exc
import urllib.parse
import webbrowser
import zlib
from config import appname, appversion, config
from protocol import protocolhandler
from typing import TYPE_CHECKING
if TYPE_CHECKING:
_ = lambda x: x # noqa # to make flake8 stop complaining that the hacked in _ method doesnt exist
holdoff = 60 # be nice
timeout = 10 # requests timeout
auth_timeout = 30 # timeout for initial auth
holdoff = 60 # be nice
timeout = 10 # requests timeout
auth_timeout = 30 # timeout for initial auth
# Currently the "Elite Dangerous Market Connector (EDCD/Athanasius)" one in
# Athanasius' Frontier account
CLIENT_ID = os.getenv('CLIENT_ID') or 'fb88d428-9110-475f-a3d2-dc151c2b9c7a' # Obtain from https://auth.frontierstore.net/client/signup
# Obtain from https://auth.frontierstore.net/client/signup
CLIENT_ID = os.getenv('CLIENT_ID') or 'fb88d428-9110-475f-a3d2-dc151c2b9c7a'
SERVER_AUTH = 'https://auth.frontierstore.net'
URL_AUTH = '/auth'
URL_TOKEN = '/token'
URL_TOKEN = '/token'
USER_AGENT = 'EDCD-{}-{}'.format(appname, appversion)
SERVER_LIVE = 'https://companion.orerve.net'
SERVER_BETA = 'https://pts-companion.orerve.net'
@ -47,7 +53,7 @@ category_map = {
'Narcotics' : 'Legal Drugs',
'Slaves' : 'Slavery',
'Waste ' : 'Waste',
'NonMarketable' : False, # Don't appear in the in-game market so don't report
'NonMarketable' : False, # Don't appear in the in-game market so don't report
}
commodity_map = {}
@ -105,51 +111,65 @@ ship_map = {
# In practice these arrays aren't very sparse so just convert them to lists with any 'gaps' holding None.
def listify(thing):
if thing is None:
return [] # data is not present
return [] # data is not present
elif isinstance(thing, list):
return list(thing) # array is not sparse
return list(thing) # array is not sparse
elif isinstance(thing, dict):
retval = []
for k,v in thing.items():
for k, v in thing.items():
idx = int(k)
if idx >= len(retval):
retval.extend([None] * (idx - len(retval)))
retval.append(v)
else:
retval[idx] = v
return retval
else:
assert False, thing # we expect an array or a sparse array
return list(thing) # hope for the best
raise ValueError("expected an array or sparse array, got {!r}".format(thing))
class ServerError(Exception):
def __init__(self, *args):
self.args = args if args else (_('Error: Frontier server is down'),) # Raised when cannot contact the Companion API server
# Raised when cannot contact the Companion API server
self.args = args if args else (_('Error: Frontier server is down'),)
class ServerLagging(Exception):
def __init__(self, *args):
self.args = args if args else (_('Error: Frontier server is lagging'),) # Raised when Companion API server is returning old data, e.g. when the servers are too busy
# Raised when Companion API server is returning old data, e.g. when the servers are too busy
self.args = args if args else (_('Error: Frontier server is lagging'),)
class SKUError(Exception):
def __init__(self, *args):
self.args = args if args else (_('Error: Frontier server SKU problem'),) # Raised when the Companion API server thinks that the user has not purchased E:D. i.e. doesn't have the correct 'SKU'
# Raised when the Companion API server thinks that the user has not purchased E:D
# i.e. doesn't have the correct 'SKU'
self.args = args if args else (_('Error: Frontier server SKU problem'),)
class CredentialsError(Exception):
def __init__(self, *args):
self.args = args if args else (_('Error: Invalid Credentials'),)
class CmdrError(Exception):
def __init__(self, *args):
self.args = args if args else (_('Error: Wrong Cmdr'),) # Raised when the user has multiple accounts and the username/password setting is not for the account they're currently playing OR the user has reset their Cmdr and the Companion API server is still returning data for the old Cmdr
# Raised when the user has multiple accounts and the username/password setting is not for
# the account they're currently playing OR the user has reset their Cmdr and the Companion API
# server is still returning data for the old Cmdr
self.args = args if args else (_('Error: Wrong Cmdr'),)
class Auth(object):
def __init__(self, cmdr):
self.cmdr = cmdr
self.session = requests.Session()
self.session.headers['User-Agent'] = 'EDCD-%s-%s' % (appname, appversion)
self.session.headers['User-Agent'] = USER_AGENT
self.verifier = self.state = None
def refresh(self):
@ -166,21 +186,25 @@ class Auth(object):
'client_id': CLIENT_ID,
'refresh_token': tokens[idx],
}
r = self.session.post(SERVER_AUTH + URL_TOKEN, data=data, timeout=auth_timeout)
if r.status_code == requests.codes.ok:
data = r.json()
tokens[idx] = data.get('refresh_token', '')
config.set('fdev_apikeys', tokens)
config.save() # Save settings now for use by command-line app
config.save() # Save settings now for use by command-line app
return data.get('access_token')
else:
print('Auth\tCan\'t refresh token for %s' % self.cmdr)
print('Auth\tCan\'t refresh token for {}'.format(self.cmdr))
self.dump(r)
except:
print('Auth\tCan\'t refresh token for %s' % self.cmdr)
except Exception:
print('Auth\tCan\'t refresh token for {}'.format(self.cmdr))
print_exc()
else:
print('Auth\tNo token for %s' % self.cmdr)
print('Auth\tNo token for {}'.format(self.cmdr))
# New request
print('Auth\tNew authorization request')
@ -189,29 +213,35 @@ class Auth(object):
s = random.SystemRandom().getrandbits(8 * 32)
self.state = self.base64URLEncode(s.to_bytes(32, byteorder='big'))
# Won't work under IE: https://blogs.msdn.microsoft.com/ieinternals/2011/07/13/understanding-protocols/
webbrowser.open('%s%s?response_type=code&audience=frontier&scope=capi&client_id=%s&code_challenge=%s&code_challenge_method=S256&state=%s&redirect_uri=%s' % (SERVER_AUTH, URL_AUTH, CLIENT_ID, self.base64URLEncode(hashlib.sha256(self.verifier).digest()), self.state, protocolhandler.redirect))
webbrowser.open(
'{server_auth}{url_auth}?response_type=code&audience=frontier&scope=capi&client_id={client_id}&code_challenge={challenge}&code_challenge_method=S256&state={state}&redirect_uri={redirect}'.format( # noqa: E501 # I cant make this any shorter
server_auth=SERVER_AUTH,
url_auth=URL_AUTH,
client_id=CLIENT_ID,
challenge=self.base64URLEncode(hashlib.sha256(self.verifier).digest()),
state=self.state,
redirect=protocolhandler.redirect
)
)
def authorize(self, payload):
# Handle OAuth authorization code callback. Returns access token if successful, otherwise raises CredentialsError
if not '?' in payload:
print('Auth\tMalformed response "%s"' % payload)
raise CredentialsError() # Not well formed
# Handle OAuth authorization code callback.
# Returns access token if successful, otherwise raises CredentialsError
if '?' not in payload:
print('Auth\tMalformed response {!r}'.format(payload))
raise CredentialsError('malformed payload') # Not well formed
data = urllib.parse.parse_qs(payload[payload.index('?')+1:])
data = urllib.parse.parse_qs(payload[(payload.index('?') + 1):])
if not self.state or not data.get('state') or data['state'][0] != self.state:
print('Auth\tUnexpected response "%s"' % payload)
raise CredentialsError() # Unexpected reply
print('Auth\tUnexpected response {!r}'.format(payload))
raise CredentialsError('Unexpected response from authorization {!r}'.format(payload)) # Unexpected reply
if not data.get('code'):
print('Auth\tNegative response "%s"' % payload)
if data.get('error_description'):
raise CredentialsError('Error: %s' % data['error_description'][0])
elif data.get('error'):
raise CredentialsError('Error: %s' % data['error'][0])
elif data.get('message'):
raise CredentialsError('Error: %s' % data['message'][0])
else:
raise CredentialsError()
print('Auth\tNegative response {!r}'.format(payload))
error = next(
(data[k] for k in ('error_description', 'error', 'message') if k in data), ('<unknown error>',)
)
raise CredentialsError('Error: {!r}'.format(error)[0])
try:
r = None
@ -222,55 +252,53 @@ class Auth(object):
'code': data['code'][0],
'redirect_uri': protocolhandler.redirect,
}
r = self.session.post(SERVER_AUTH + URL_TOKEN, data=data, timeout=auth_timeout)
data = r.json()
if r.status_code == requests.codes.ok:
print('Auth\tNew token for %s' % self.cmdr)
print('Auth\tNew token for {}'.format(self.cmdr))
cmdrs = config.get('cmdrs')
idx = cmdrs.index(self.cmdr)
tokens = config.get('fdev_apikeys') or []
tokens = tokens + [''] * (len(cmdrs) - len(tokens))
tokens[idx] = data.get('refresh_token', '')
config.set('fdev_apikeys', tokens)
config.save() # Save settings now for use by command-line app
return data.get('access_token')
except:
print('Auth\tCan\'t get token for %s' % self.cmdr)
print_exc()
if r: self.dump(r)
raise CredentialsError()
config.save() # Save settings now for use by command-line app
print('Auth\tCan\'t get token for %s' % self.cmdr)
return data.get('access_token')
except Exception as e:
print('Auth\tCan\'t get token for {}'.format(self.cmdr))
print_exc()
if r:
self.dump(r)
raise CredentialsError('unable to get token') from e
print('Auth\tCan\'t get token for {}'.format(self.cmdr))
self.dump(r)
if data.get('error_description'):
raise CredentialsError('Error: %s' % data['error_description'])
elif data.get('error'):
raise CredentialsError('Error: %s' % data['error'])
elif data.get('message'):
raise CredentialsError('Error: %s' % data['message'])
else:
raise CredentialsError()
error = next((data[k] for k in ('error_description', 'error', 'message') if k in data), ('<unknown error>',))
raise CredentialsError('Error: {!r}'.format(error)[0])
@staticmethod
def invalidate(cmdr):
print('Auth\tInvalidated token for %s' % cmdr)
print('Auth\tInvalidated token for {}'.format(cmdr))
cmdrs = config.get('cmdrs')
idx = cmdrs.index(cmdr)
tokens = config.get('fdev_apikeys') or []
tokens = tokens + [''] * (len(cmdrs) - len(tokens))
tokens[idx] = ''
config.set('fdev_apikeys', tokens)
config.save() # Save settings now for use by command-line app
config.save() # Save settings now for use by command-line app
def dump(self, r):
print('Auth\t' + r.url, r.status_code, r.reason and r.reason or 'None', r.text)
print('Auth\t' + r.url, r.status_code, r.reason if r.reason else 'None', r.text)
def base64URLEncode(self, text):
return base64.urlsafe_b64encode(text).decode().replace('=', '')
class Session(object):
STATE_INIT, STATE_AUTH, STATE_OK = list(range(3))
def __init__(self):
@ -278,27 +306,26 @@ class Session(object):
self.credentials = None
self.session = None
self.auth = None
self.retrying = False # Avoid infinite loop when successful auth / unsuccessful query
# yuck suppress InsecurePlatformWarning under Python < 2.7.9 which lacks SNI support
if sys.version_info < (2,7,9):
from requests.packages import urllib3
urllib3.disable_warnings()
self.retrying = False # Avoid infinite loop when successful auth / unsuccessful query
def login(self, cmdr=None, is_beta=None):
# Returns True if login succeeded, False if re-authorization initiated.
if not CLIENT_ID:
raise CredentialsError()
raise CredentialsError('cannot login without a valid Client ID')
if not cmdr or is_beta is None:
# Use existing credentials
if not self.credentials:
raise CredentialsError() # Shouldn't happen
raise CredentialsError('Missing credentials') # Shouldn't happen
elif self.state == Session.STATE_OK:
return True # already logged in
return True # already logged in
else:
credentials = {'cmdr': cmdr, 'beta': is_beta}
if self.credentials == credentials and self.state == Session.STATE_OK:
return True # already logged in
return True # already logged in
else:
# changed account or retrying login during auth
self.close()
@ -307,11 +334,13 @@ class Session(object):
self.server = self.credentials['beta'] and SERVER_BETA or SERVER_LIVE
self.state = Session.STATE_INIT
self.auth = Auth(self.credentials['cmdr'])
access_token = self.auth.refresh()
if access_token:
self.auth = None
self.start(access_token)
return True
else:
self.state = Session.STATE_AUTH
return False
@ -320,33 +349,40 @@ class Session(object):
# Callback from protocol handler
def auth_callback(self):
if self.state != Session.STATE_AUTH:
raise CredentialsError() # Shouldn't be getting a callback
# Shouldn't be getting a callback
raise CredentialsError('Got an auth callback while not doing auth')
try:
self.start(self.auth.authorize(protocolhandler.lastpayload))
self.auth = None
except:
self.state = Session.STATE_INIT # Will try to authorize again on next login or query
except Exception:
self.state = Session.STATE_INIT # Will try to authorize again on next login or query
self.auth = None
raise # Bad thing happened
raise # Bad thing happened
def start(self, access_token):
self.session = requests.Session()
self.session.headers['Authorization'] = 'Bearer %s' % access_token
self.session.headers['User-Agent'] = 'EDCD-%s-%s' % (appname, appversion)
self.session.headers['Authorization'] = 'Bearer {}'.format(access_token)
self.session.headers['User-Agent'] = USER_AGENT
self.state = Session.STATE_OK
def query(self, endpoint):
if self.state == Session.STATE_INIT:
if self.login():
return self.query(endpoint)
elif self.state == Session.STATE_AUTH:
raise CredentialsError()
raise CredentialsError('cannot make a query when unauthorized')
try:
r = self.session.get(self.server + endpoint, timeout=timeout)
except:
if __debug__: print_exc()
raise ServerError()
except Exception as e:
if __debug__:
print_exc()
raise ServerError('unable to get endpoint {}'.format(endpoint)) from e
if r.url.startswith(SERVER_AUTH):
# Redirected back to Auth server - force full re-authentication
@ -355,33 +391,39 @@ class Session(object):
self.retrying = False
self.login()
raise CredentialsError()
elif 500 <= r.status_code < 600:
# Server error. Typically 500 "Internal Server Error" if server is down
self.dump(r)
raise ServerError()
raise ServerError('Received error {} from server'.format(r.status_code))
try:
r.raise_for_status() # Typically 403 "Forbidden" on token expiry
data = r.json() # May also fail here if token expired since response is empty
except:
r.raise_for_status() # Typically 403 "Forbidden" on token expiry
data = r.json() # May also fail here if token expired since response is empty
except (requests.HTTPError, ValueError) as e:
print_exc()
self.dump(r)
self.close()
if self.retrying: # Refresh just succeeded but this query failed! Force full re-authentication
self.invalidate()
self.retrying = False
self.login()
raise CredentialsError()
raise CredentialsError('query failed after refresh') from e
elif self.login(): # Maybe our token expired. Re-authorize in any case
self.retrying = True
return self.query(endpoint)
else:
self.retrying = False
raise CredentialsError()
raise CredentialsError('HTTP error or invalid JSON') from e
self.retrying = False
if 'timestamp' not in data:
data['timestamp'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', parsedate(r.headers['Date']))
return data
def profile(self):
@ -391,20 +433,26 @@ class Session(object):
data = self.query(URL_QUERY)
if data['commander'].get('docked'):
services = data['lastStarport'].get('services', {})
last_starport_name = data['lastStarport']['name']
last_starport_id = int(data['lastStarport']['id'])
if services.get('commodities'):
marketdata = self.query(URL_MARKET)
if (data['lastStarport']['name'] != marketdata['name'] or
int(data['lastStarport']['id']) != int(marketdata['id'])):
if (last_starport_name != marketdata['name'] or last_starport_id != int(marketdata['id'])):
raise ServerLagging()
else:
data['lastStarport'].update(marketdata)
if services.get('outfitting') or services.get('shipyard'):
shipdata = self.query(URL_SHIPYARD)
if (data['lastStarport']['name'] != shipdata['name'] or
int(data['lastStarport']['id']) != int(shipdata['id'])):
if (last_starport_name != shipdata['name'] or last_starport_id != int(shipdata['id'])):
raise ServerLagging()
else:
data['lastStarport'].update(shipdata)
return data
def close(self):
@ -412,8 +460,11 @@ class Session(object):
if self.session:
try:
self.session.close()
except:
if __debug__: print_exc()
except Exception:
if __debug__:
print_exc()
self.session = None
def invalidate(self):
@ -424,15 +475,15 @@ class Session(object):
def dump(self, r):
print('cAPI\t' + r.url, r.status_code, r.reason and r.reason or 'None', r.text)
# Returns a shallow copy of the received data suitable for export to older tools - English commodity names and anomalies fixed up
# Returns a shallow copy of the received data suitable for export to older tools
# English commodity names and anomalies fixed up
def fixup(data):
if not commodity_map:
# Lazily populate
for f in ['commodity.csv', 'rare_commodity.csv']:
for f in ('commodity.csv', 'rare_commodity.csv'):
with open(join(config.respath, f), 'r') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
commodity_map[row['symbol']] = (row['category'], row['name'])
@ -440,30 +491,53 @@ def fixup(data):
for commodity in data['lastStarport'].get('commodities') or []:
# Check all required numeric fields are present and are numeric
# Catches "demandBracket": "" for some phantom commodites in ED 1.3 - https://github.com/Marginal/EDMarketConnector/issues/2
# Catches "demandBracket": "" for some phantom commodites in
# ED 1.3 - https://github.com/Marginal/EDMarketConnector/issues/2
#
# But also see https://github.com/Marginal/EDMarketConnector/issues/32
for thing in ['buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket']:
for thing in ('buyPrice', 'sellPrice', 'demand', 'demandBracket', 'stock', 'stockBracket'):
if not isinstance(commodity.get(thing), numbers.Number):
if __debug__: print('Invalid "%s":"%s" (%s) for "%s"' % (thing, commodity.get(thing), type(commodity.get(thing)), commodity.get('name', '')))
if __debug__:
print(
'Invalid {!r}:{!r} ({}) for {!r}'.format(
thing,
commodity.get(thing),
type(commodity.get(thing)),
commodity.get('name', '')
)
)
break
else:
if not category_map.get(commodity['categoryname'], True): # Check not marketable i.e. Limpets
# Check not marketable i.e. Limpets
if not category_map.get(commodity['categoryname'], True):
pass
elif commodity['demandBracket'] == 0 and commodity['stockBracket'] == 0: # Check not normally stocked e.g. Salvage
# Check not normally stocked e.g. Salvage
elif commodity['demandBracket'] == 0 and commodity['stockBracket'] == 0:
pass
elif commodity.get('legality'): # Check not prohibited
elif commodity.get('legality'): # Check not prohibited
pass
elif not commodity.get('categoryname'):
if __debug__: print('Missing "categoryname" for "%s"' % commodity.get('name', ''))
if __debug__:
print('Missing "categoryname" for {!r}'.format(commodity.get('name', '')))
elif not commodity.get('name'):
if __debug__: print('Missing "name" for a commodity in "%s"' % commodity.get('categoryname', ''))
if __debug__:
print('Missing "name" for a commodity in {!r}'.format(commodity.get('categoryname', '')))
elif not commodity['demandBracket'] in range(4):
if __debug__: print('Invalid "demandBracket":"%s" for "%s"' % (commodity['demandBracket'], commodity['name']))
if __debug__:
print('Invalid "demandBracket":{!r} for {!r}'.format(commodity['demandBracket'], commodity['name']))
elif not commodity['stockBracket'] in range(4):
if __debug__: print('Invalid "stockBracket":"%s" for "%s"' % (commodity['stockBracket'], commodity['name']))
if __debug__:
print('Invalid "stockBracket":{!r} for {!r}'.format(commodity['stockBracket'], commodity['name']))
else:
# Rewrite text fields
new = dict(commodity) # shallow copy
new = dict(commodity) # shallow copy
if commodity['name'] in commodity_map:
(new['categoryname'], new['name']) = commodity_map[commodity['name']]
elif commodity['categoryname'] in category_map:
@ -480,30 +554,36 @@ def fixup(data):
commodities.append(new)
# return a shallow copy
datacopy = dict(data)
datacopy['lastStarport'] = dict(data['lastStarport'])
datacopy = data.copy()
datacopy['lastStarport'] = data['lastStarport'].copy()
datacopy['lastStarport']['commodities'] = commodities
return datacopy
# Return a subset of the received data describing the current ship
def ship(data):
def filter_ship(d):
filtered = {}
for k, v in d.items():
if v == []:
pass # just skip empty fields for brevity
elif k in ['alive', 'cargo', 'cockpitBreached', 'health', 'oxygenRemaining', 'rebuilds', 'starsystem', 'station']:
pass # noisy
elif k in ['locDescription', 'locName'] or k.endswith('LocDescription') or k.endswith('LocName'):
pass # also noisy, and redundant
elif k in ['dir', 'LessIsGood']:
pass # dir is not ASCII - remove to simplify handling
pass # just skip empty fields for brevity
elif k in ('alive', 'cargo', 'cockpitBreached', 'health', 'oxygenRemaining',
'rebuilds', 'starsystem', 'station'):
pass # noisy
elif k in ('locDescription', 'locName') or k.endswith('LocDescription') or k.endswith('LocName'):
pass # also noisy, and redundant
elif k in ('dir', 'LessIsGood'):
pass # dir is not ASCII - remove to simplify handling
elif hasattr(v, 'items'):
filtered[k] = filter_ship(v)
else:
filtered[k] = v
return filtered
# subset of "ship" that's not noisy
@ -515,11 +595,13 @@ def ship_file_name(ship_name, ship_type):
name = str(ship_name or ship_map.get(ship_type.lower(), ship_type)).strip()
if name.endswith('.'):
name = name[:-1]
if name.lower() in ['con', 'prn', 'aux', 'nul',
if name.lower() in ('con', 'prn', 'aux', 'nul',
'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9',
'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9']:
'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9'):
name = name + '_'
return name.translate({ ord(x): u'_' for x in ['\0', '<', '>', ':', '"', '/', '\\', '|', '?', '*'] })
return name.translate({ord(x): u'_' for x in ('\0', '<', '>', ':', '"', '/', '\\', '|', '?', '*')})
# singleton