mirror of
https://github.com/norohind/jubilant-system.git
synced 2025-06-04 09:20:57 +03:00
discover mode done
This commit is contained in:
parent
ad355db369
commit
c045007ad0
GPG Key ID:
01C3BECC26FB59E1
2
doc.txt
2
doc.txt
@ -6,7 +6,7 @@ What to track?
|
||||
functionality:
|
||||
ASAP check for new squads
|
||||
Occasionally iterate over all squads (by id)
|
||||
update/insert new data to DB (if nothing changed, then just write empty history record with inserted_timestamp and squad_id)
|
||||
update/insert new data to DB
|
||||
alert if new data accord to triggers
|
||||
|
||||
DB tables
|
||||
|
||||
226
main.py
226
main.py
@ -1,11 +1,8 @@
|
||||
import json
|
||||
import requests
|
||||
import sqlite3
|
||||
import time
|
||||
import datetime
|
||||
|
||||
import utils
|
||||
from EDMCLogging import get_main_logger
|
||||
import sql_requests
|
||||
|
||||
logger = get_main_logger()
|
||||
db = sqlite3.connect('squads.sqlite')
|
||||
@ -13,166 +10,79 @@ db = sqlite3.connect('squads.sqlite')
|
||||
with open('sql_schema.sql', 'r', encoding='utf-8') as schema_file:
|
||||
db.executescript(''.join(schema_file.readlines()))
|
||||
|
||||
ruTag = 32
|
||||
BASE_URL = 'https://api.orerve.net/2.0/website/squadron/'
|
||||
INFO_ENDPOINT = 'info'
|
||||
NEWS_ENDPOINT = 'news/list'
|
||||
ruTag: id = 32
|
||||
|
||||
"""
|
||||
Two modes:
|
||||
1. Discover new squads
|
||||
get last_known_id
|
||||
tries = 0
|
||||
failed: list
|
||||
while True
|
||||
if tries = 2
|
||||
break
|
||||
|
||||
def update_squad_news(squad_id: int, db_conn: sqlite3.Connection) -> bool:
|
||||
"""Update news for squad with specified ID
|
||||
|
||||
:param squad_id: id of squad to insert news
|
||||
:param db_conn: connection to sqlite DB
|
||||
:return: True if squad exists, False if not
|
||||
:rtype: bool
|
||||
"""
|
||||
|
||||
"""
|
||||
How it should works?
|
||||
Request news
|
||||
if squad doesn't exists
|
||||
return False
|
||||
|
||||
else
|
||||
insert all news even if it already exist in DB
|
||||
return True
|
||||
"""
|
||||
news_request: requests.Response = utils.authed_request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id})
|
||||
if news_request.status_code != 200: # must not happen
|
||||
logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}')
|
||||
# we will not break it, let next code break it by itself
|
||||
|
||||
squad_news: dict = news_request.json()['squadron']
|
||||
|
||||
if 'id' not in squad_news.keys(): # squadron doesn't FDEV
|
||||
return False
|
||||
|
||||
else: # squadron exists FDEV
|
||||
del squad_news['id']
|
||||
|
||||
for type_of_news_key in squad_news:
|
||||
one_type_of_news: list = squad_news[type_of_news_key]
|
||||
|
||||
news: dict
|
||||
for news in one_type_of_news:
|
||||
with db_conn:
|
||||
db_conn.execute(
|
||||
sql_requests.insert_news,
|
||||
(
|
||||
squad_id,
|
||||
type_of_news_key,
|
||||
news.get('id'),
|
||||
news.get('date'),
|
||||
news.get('category'),
|
||||
news.get('activity'),
|
||||
news.get('season'),
|
||||
news.get('bookmark'),
|
||||
news.get('motd'),
|
||||
news.get('author'),
|
||||
news.get('cmdr_id'),
|
||||
news.get('user_id')
|
||||
)
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def update_squad_info(squad_id: int, db_conn: sqlite3.Connection) -> bool:
|
||||
"""Update/insert information about squadron with specified id in our DB
|
||||
|
||||
:param squad_id: id of squad to update/insert
|
||||
:param db_conn: connection to sqlite DB
|
||||
:return: True if squad exists, False if not
|
||||
:rtype: bool
|
||||
"""
|
||||
|
||||
"""
|
||||
How it should works?
|
||||
Request squad's info
|
||||
if squad exists FDEV
|
||||
insert info in DB
|
||||
request news, insert to DB
|
||||
return True
|
||||
|
||||
if squad doesn't exists FDEV
|
||||
if squad in DB and isn't deleted in our DB
|
||||
write to squads_states record with all null except ID (it will mean that squad was deleted)
|
||||
id_to_try = last_known_id + 1
|
||||
update squad info with id_to_try and suppressing absence
|
||||
|
||||
if success
|
||||
process triggers
|
||||
tries = 0
|
||||
for failed_squad in failed
|
||||
delete(failed_squad)
|
||||
failed_squad = list()
|
||||
|
||||
|
||||
return False
|
||||
*Should we return something more then just a bool, may be a message to notify_discord?
|
||||
"""
|
||||
|
||||
squad_request: requests.Response = utils.authed_request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id})
|
||||
|
||||
if squad_request.status_code == 200: # squad exists FDEV
|
||||
squad_request_json: dict = squad_request.json()['squadron']
|
||||
with db_conn:
|
||||
db_conn.execute(
|
||||
sql_requests.insert_squad_states,
|
||||
(
|
||||
squad_id,
|
||||
squad_request_json['name'],
|
||||
squad_request_json['tag'],
|
||||
utils.fdev2people(squad_request_json['ownerName']),
|
||||
squad_request_json['ownerId'],
|
||||
squad_request_json['platform'],
|
||||
squad_request_json['created'],
|
||||
squad_request_json['created_ts'],
|
||||
squad_request_json['acceptingNewMembers'],
|
||||
squad_request_json['powerId'],
|
||||
squad_request_json['powerName'],
|
||||
squad_request_json['superpowerId'],
|
||||
squad_request_json['superpowerName'],
|
||||
squad_request_json['factionId'],
|
||||
squad_request_json['factionName'],
|
||||
json.dumps(squad_request_json['userTags']),
|
||||
squad_request_json['memberCount'],
|
||||
squad_request_json['pendingCount'],
|
||||
squad_request_json['full'],
|
||||
squad_request_json['publicComms'],
|
||||
squad_request_json['publicCommsOverride'],
|
||||
squad_request_json['publicCommsAvailable'],
|
||||
squad_request_json['current_season_trade_score'],
|
||||
squad_request_json['previous_season_trade_score'],
|
||||
squad_request_json['current_season_combat_score'],
|
||||
squad_request_json['previous_season_combat_score'],
|
||||
squad_request_json['current_season_exploration_score'],
|
||||
squad_request_json['previous_season_exploration_score'],
|
||||
squad_request_json['current_season_cqc_score'],
|
||||
squad_request_json['previous_season_cqc_score'],
|
||||
squad_request_json['current_season_bgs_score'],
|
||||
squad_request_json['previous_season_bgs_score'],
|
||||
squad_request_json['current_season_powerplay_score'],
|
||||
squad_request_json['previous_season_powerplay_score'],
|
||||
squad_request_json['current_season_aegis_score'],
|
||||
squad_request_json['previous_season_aegis_score']
|
||||
)
|
||||
)
|
||||
|
||||
update_squad_news(squad_id, db_conn)
|
||||
return True
|
||||
|
||||
elif squad_request.status_code == 404: # squad doesn't exists FDEV
|
||||
if db_conn.execute(
|
||||
sql_requests.check_if_squad_exists_in_db,
|
||||
(squad_id,)).fetchone()[0] > 0: # we have it in DB
|
||||
|
||||
if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] == 0:
|
||||
# we don't have it deleted in DB
|
||||
with db_conn:
|
||||
db_conn.execute(sql_requests.properly_delete_squad, (squad_id,))
|
||||
|
||||
return False # squadron stop their existing or never exists... it doesn't exists anyway
|
||||
|
||||
else: # any other codes (except 418, that one handles in authed_request), never should happen
|
||||
logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}')
|
||||
raise utils.FAPIUnknownStatusCode
|
||||
else (fail)
|
||||
failed.append(id_to_try)
|
||||
tries = tries + 1
|
||||
|
||||
sleep(3)
|
||||
|
||||
|
||||
2. Update exists
|
||||
get oldest updated existing squad
|
||||
update it
|
||||
if squad still exists
|
||||
process triggers
|
||||
"""
|
||||
|
||||
|
||||
def discover_triggers(squad_info: dict):
|
||||
print(squad_info.get('name'), squad_info.get('tag'), squad_info.get('ownerName'))
|
||||
|
||||
|
||||
def discover():
|
||||
id_to_try = utils.get_last_known_id(db)
|
||||
tries: int = 0
|
||||
failed: list = list()
|
||||
tries_limit: int = 5000
|
||||
|
||||
while True:
|
||||
id_to_try = id_to_try + 1
|
||||
logger.debug(f'Starting discover loop iteration, tries: {tries} of {tries_limit}, id to try {id_to_try}, '
|
||||
f'failed list: {failed}')
|
||||
|
||||
if tries == tries_limit:
|
||||
break
|
||||
|
||||
squad_info = utils.update_squad_info(id_to_try, db, suppress_absence=True)
|
||||
|
||||
if isinstance(squad_info, dict): # success
|
||||
logger.debug(f'Success discover for {id_to_try} ID')
|
||||
discover_triggers(squad_info)
|
||||
tries = 0 # reset tries counter
|
||||
for failed_squad in failed: # since we found an exists squad, then all previous failed wasn't exists
|
||||
utils.properly_delete_squadron(failed_squad, db)
|
||||
|
||||
failed = list()
|
||||
|
||||
else: # should be only False
|
||||
logger.debug(f'Fail on discovery for {id_to_try} ID')
|
||||
failed.append(id_to_try)
|
||||
tries = tries + 1
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
|
||||
update_squad_info(47999, db)
|
||||
discover()
|
||||
|
||||
@ -60,4 +60,9 @@ properly_delete_squad: str = """insert into squads_states (squad_id) values (?);
|
||||
|
||||
check_if_we_already_deleted_squad_in_db: str = """select count(*)
|
||||
from squads_states
|
||||
where squad_id = ? and tag is null"""
|
||||
where squad_id = ? and tag is null"""
|
||||
|
||||
select_last_known_id: str = """select squad_id
|
||||
from squads_states
|
||||
order by squad_id desc
|
||||
limit 1;"""
|
||||
206
utils.py
206
utils.py
@ -1,8 +1,16 @@
|
||||
import sqlite3
|
||||
from typing import Union
|
||||
import requests
|
||||
import json
|
||||
from EDMCLogging import get_main_logger
|
||||
import sql_requests
|
||||
|
||||
logger = get_main_logger()
|
||||
|
||||
BASE_URL = 'https://api.orerve.net/2.0/website/squadron/'
|
||||
INFO_ENDPOINT = 'info'
|
||||
NEWS_ENDPOINT = 'news/list'
|
||||
|
||||
|
||||
class FAPIDownForMaintenance(Exception):
|
||||
pass
|
||||
@ -16,7 +24,7 @@ def authed_request(url: str, method: str = 'get', **kwargs) -> requests.Response
|
||||
"""Makes request to any url with valid bearer token"""
|
||||
bearer: str = _get_bearer()
|
||||
|
||||
logger.debug(f'Requesting {method.upper()} {url!r}')
|
||||
logger.debug(f'Requesting {method.upper()} {url!r}, kwargs: {kwargs}')
|
||||
|
||||
fapiRequest: requests.Response = requests.request(
|
||||
method=method,
|
||||
@ -84,3 +92,199 @@ def notify_discord(message: str) -> None:
|
||||
logger.debug('Sending successful')
|
||||
return
|
||||
|
||||
|
||||
def _update_squad_news(squad_id: int, db_conn: sqlite3.Connection) -> Union[bool, str]:
|
||||
"""Update news for squad with specified ID
|
||||
|
||||
:param squad_id: id of squad to insert news
|
||||
:param db_conn: connection to sqlite DB
|
||||
:return: motd if squad exists, False if not
|
||||
:rtype: bool, str
|
||||
"""
|
||||
|
||||
"""
|
||||
How it should works?
|
||||
Request news
|
||||
if squad doesn't exists
|
||||
return False
|
||||
|
||||
else
|
||||
insert all news even if it already exist in DB
|
||||
return motd
|
||||
"""
|
||||
news_request: requests.Response = authed_request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id})
|
||||
if news_request.status_code != 200: # must not happen
|
||||
logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}')
|
||||
# we will not break it, let next code break it by itself
|
||||
|
||||
squad_news: dict = news_request.json()['squadron']
|
||||
|
||||
if 'id' not in squad_news.keys(): # squadron doesn't FDEV
|
||||
return False
|
||||
|
||||
else: # squadron exists FDEV
|
||||
del squad_news['id']
|
||||
|
||||
for type_of_news_key in squad_news:
|
||||
one_type_of_news: list = squad_news[type_of_news_key]
|
||||
|
||||
news: dict
|
||||
for news in one_type_of_news:
|
||||
with db_conn:
|
||||
db_conn.execute(
|
||||
sql_requests.insert_news,
|
||||
(
|
||||
squad_id,
|
||||
type_of_news_key,
|
||||
news.get('id'),
|
||||
news.get('date'),
|
||||
news.get('category'),
|
||||
news.get('activity'),
|
||||
news.get('season'),
|
||||
news.get('bookmark'),
|
||||
news.get('motd'),
|
||||
news.get('author'),
|
||||
news.get('cmdr_id'),
|
||||
news.get('user_id')
|
||||
)
|
||||
)
|
||||
|
||||
return next(iter(squad_news['public_statements']), dict()).get('motd', '')
|
||||
|
||||
|
||||
def update_squad_info(squad_id: int, db_conn: sqlite3.Connection, suppress_absence: bool = False) -> Union[bool, dict]:
|
||||
"""Update/insert information about squadron with specified id in our DB
|
||||
|
||||
:param squad_id: id of squad to update/insert
|
||||
:param db_conn: connection to sqlite DB
|
||||
:param suppress_absence: if we shouldn't mark squad as deleted if we didn't found it by FDEV
|
||||
:return: squad dict if squad exists, False if not
|
||||
:rtype: bool, dict
|
||||
"""
|
||||
|
||||
"""
|
||||
How it should works?
|
||||
Request squad's info
|
||||
|
||||
if squad is properly deleted in our DB
|
||||
return False
|
||||
|
||||
if squad exists FDEV
|
||||
insert info in DB
|
||||
request news, insert to DB
|
||||
return squad dict
|
||||
|
||||
if squad doesn't exists FDEV
|
||||
if squad in DB
|
||||
if isn't deleted in our DB
|
||||
write to squads_states record with all null except ID (it will mean that squad was deleted)
|
||||
else if not suppress_absence
|
||||
write to squads_states record with all null except ID (it will mean that squad was deleted)
|
||||
|
||||
|
||||
|
||||
return False
|
||||
*Should we return something more then just a bool, may be a message to notify_discord?
|
||||
"""
|
||||
|
||||
if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] != 0:
|
||||
# we have it as properly deleted in our DB
|
||||
logger.debug(f'squad {squad_id} is marked as deleted in our DB, returning False')
|
||||
return False
|
||||
|
||||
squad_request: requests.Response = authed_request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id})
|
||||
|
||||
if squad_request.status_code == 200: # squad exists FDEV
|
||||
squad_request_json: dict = squad_request.json()['squadron']
|
||||
squad_request_json['ownerName'] = fdev2people(squad_request_json['ownerName']) # normalize value
|
||||
|
||||
with db_conn:
|
||||
db_conn.execute(
|
||||
sql_requests.insert_squad_states,
|
||||
(
|
||||
squad_id,
|
||||
squad_request_json['name'],
|
||||
squad_request_json['tag'],
|
||||
squad_request_json['ownerName'],
|
||||
squad_request_json['ownerId'],
|
||||
squad_request_json['platform'],
|
||||
squad_request_json['created'],
|
||||
squad_request_json['created_ts'],
|
||||
squad_request_json['acceptingNewMembers'],
|
||||
squad_request_json['powerId'],
|
||||
squad_request_json['powerName'],
|
||||
squad_request_json['superpowerId'],
|
||||
squad_request_json['superpowerName'],
|
||||
squad_request_json['factionId'],
|
||||
squad_request_json['factionName'],
|
||||
json.dumps(squad_request_json['userTags']),
|
||||
squad_request_json['memberCount'],
|
||||
squad_request_json['pendingCount'],
|
||||
squad_request_json['full'],
|
||||
squad_request_json['publicComms'],
|
||||
squad_request_json['publicCommsOverride'],
|
||||
squad_request_json['publicCommsAvailable'],
|
||||
squad_request_json['current_season_trade_score'],
|
||||
squad_request_json['previous_season_trade_score'],
|
||||
squad_request_json['current_season_combat_score'],
|
||||
squad_request_json['previous_season_combat_score'],
|
||||
squad_request_json['current_season_exploration_score'],
|
||||
squad_request_json['previous_season_exploration_score'],
|
||||
squad_request_json['current_season_cqc_score'],
|
||||
squad_request_json['previous_season_cqc_score'],
|
||||
squad_request_json['current_season_bgs_score'],
|
||||
squad_request_json['previous_season_bgs_score'],
|
||||
squad_request_json['current_season_powerplay_score'],
|
||||
squad_request_json['previous_season_powerplay_score'],
|
||||
squad_request_json['current_season_aegis_score'],
|
||||
squad_request_json['previous_season_aegis_score']
|
||||
)
|
||||
)
|
||||
|
||||
motd: str = _update_squad_news(squad_id, db_conn) # yeah, it can return bool but never should does it
|
||||
squad_request_json.update(motd=motd)
|
||||
|
||||
return squad_request_json
|
||||
|
||||
elif squad_request.status_code == 404: # squad doesn't exists FDEV
|
||||
if db_conn.execute(
|
||||
sql_requests.check_if_squad_exists_in_db,
|
||||
(squad_id,)).fetchone()[0] > 0: # we have it in DB
|
||||
|
||||
if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] == 0:
|
||||
# we don't have it deleted in DB, let's fix it
|
||||
delete_squadron(squad_id, db_conn)
|
||||
|
||||
elif not suppress_absence:
|
||||
# we don't have it in DB at all but let's mark it as deleted to avoid requests to FDEV about it in future
|
||||
delete_squadron(squad_id, db_conn)
|
||||
|
||||
return False # squadron stop their existing or never exists... it doesn't exists anyway
|
||||
|
||||
else: # any other codes (except 418, that one handles in authed_request), never should happen
|
||||
logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}')
|
||||
raise FAPIUnknownStatusCode
|
||||
|
||||
|
||||
def properly_delete_squadron(squad_id: int, db_conn: sqlite3.Connection) -> None:
|
||||
"""Properly deletes squadron from our DB
|
||||
|
||||
:param squad_id: squad id to delete
|
||||
:param db_conn: connection to DB
|
||||
:return:
|
||||
"""
|
||||
logger.debug(f'Properly deleting {squad_id}')
|
||||
|
||||
with db_conn:
|
||||
db_conn.execute(sql_requests.properly_delete_squad, (squad_id,))
|
||||
|
||||
|
||||
def get_last_known_id(db_conn: sqlite3.Connection) -> int:
|
||||
sql_request_result = db_conn.execute(sql_requests.select_last_known_id).fetchone()
|
||||
if sql_request_result is None:
|
||||
logger.debug(f"Can't get last know id from DB, defaulting to 0")
|
||||
return 0
|
||||
|
||||
else:
|
||||
logger.debug(f'last know id from DB: {sql_request_result[0]}')
|
||||
return sql_request_result[0]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user