diff --git a/doc.txt b/doc.txt index 721e162..1c7cafa 100644 --- a/doc.txt +++ b/doc.txt @@ -6,7 +6,7 @@ What to track? functionality: ASAP check for new squads Occasionally iterate over all squads (by id) -update/insert new data to DB (if nothing changed, then just write empty history record with inserted_timestamp and squad_id) +update/insert new data to DB alert if new data accord to triggers DB tables diff --git a/main.py b/main.py index cfd9205..a7a1cbf 100644 --- a/main.py +++ b/main.py @@ -1,11 +1,8 @@ -import json -import requests import sqlite3 import time -import datetime + import utils from EDMCLogging import get_main_logger -import sql_requests logger = get_main_logger() db = sqlite3.connect('squads.sqlite') @@ -13,166 +10,79 @@ db = sqlite3.connect('squads.sqlite') with open('sql_schema.sql', 'r', encoding='utf-8') as schema_file: db.executescript(''.join(schema_file.readlines())) -ruTag = 32 -BASE_URL = 'https://api.orerve.net/2.0/website/squadron/' -INFO_ENDPOINT = 'info' -NEWS_ENDPOINT = 'news/list' +ruTag: id = 32 +""" +Two modes: +1. Discover new squads + get last_known_id + tries = 0 + failed: list + while True + if tries = 2 + break -def update_squad_news(squad_id: int, db_conn: sqlite3.Connection) -> bool: - """Update news for squad with specified ID - - :param squad_id: id of squad to insert news - :param db_conn: connection to sqlite DB - :return: True if squad exists, False if not - :rtype: bool - """ - - """ - How it should works? - Request news - if squad doesn't exists - return False - - else - insert all news even if it already exist in DB - return True - """ - news_request: requests.Response = utils.authed_request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id}) - if news_request.status_code != 200: # must not happen - logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}') - # we will not break it, let next code break it by itself - - squad_news: dict = news_request.json()['squadron'] - - if 'id' not in squad_news.keys(): # squadron doesn't FDEV - return False - - else: # squadron exists FDEV - del squad_news['id'] - - for type_of_news_key in squad_news: - one_type_of_news: list = squad_news[type_of_news_key] - - news: dict - for news in one_type_of_news: - with db_conn: - db_conn.execute( - sql_requests.insert_news, - ( - squad_id, - type_of_news_key, - news.get('id'), - news.get('date'), - news.get('category'), - news.get('activity'), - news.get('season'), - news.get('bookmark'), - news.get('motd'), - news.get('author'), - news.get('cmdr_id'), - news.get('user_id') - ) - ) - - return True - - -def update_squad_info(squad_id: int, db_conn: sqlite3.Connection) -> bool: - """Update/insert information about squadron with specified id in our DB - - :param squad_id: id of squad to update/insert - :param db_conn: connection to sqlite DB - :return: True if squad exists, False if not - :rtype: bool - """ - - """ - How it should works? - Request squad's info - if squad exists FDEV - insert info in DB - request news, insert to DB - return True - - if squad doesn't exists FDEV - if squad in DB and isn't deleted in our DB - write to squads_states record with all null except ID (it will mean that squad was deleted) + id_to_try = last_known_id + 1 + update squad info with id_to_try and suppressing absence + + if success + process triggers + tries = 0 + for failed_squad in failed + delete(failed_squad) + failed_squad = list() + - return False - *Should we return something more then just a bool, may be a message to notify_discord? - """ - - squad_request: requests.Response = utils.authed_request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id}) - - if squad_request.status_code == 200: # squad exists FDEV - squad_request_json: dict = squad_request.json()['squadron'] - with db_conn: - db_conn.execute( - sql_requests.insert_squad_states, - ( - squad_id, - squad_request_json['name'], - squad_request_json['tag'], - utils.fdev2people(squad_request_json['ownerName']), - squad_request_json['ownerId'], - squad_request_json['platform'], - squad_request_json['created'], - squad_request_json['created_ts'], - squad_request_json['acceptingNewMembers'], - squad_request_json['powerId'], - squad_request_json['powerName'], - squad_request_json['superpowerId'], - squad_request_json['superpowerName'], - squad_request_json['factionId'], - squad_request_json['factionName'], - json.dumps(squad_request_json['userTags']), - squad_request_json['memberCount'], - squad_request_json['pendingCount'], - squad_request_json['full'], - squad_request_json['publicComms'], - squad_request_json['publicCommsOverride'], - squad_request_json['publicCommsAvailable'], - squad_request_json['current_season_trade_score'], - squad_request_json['previous_season_trade_score'], - squad_request_json['current_season_combat_score'], - squad_request_json['previous_season_combat_score'], - squad_request_json['current_season_exploration_score'], - squad_request_json['previous_season_exploration_score'], - squad_request_json['current_season_cqc_score'], - squad_request_json['previous_season_cqc_score'], - squad_request_json['current_season_bgs_score'], - squad_request_json['previous_season_bgs_score'], - squad_request_json['current_season_powerplay_score'], - squad_request_json['previous_season_powerplay_score'], - squad_request_json['current_season_aegis_score'], - squad_request_json['previous_season_aegis_score'] - ) - ) - - update_squad_news(squad_id, db_conn) - return True - - elif squad_request.status_code == 404: # squad doesn't exists FDEV - if db_conn.execute( - sql_requests.check_if_squad_exists_in_db, - (squad_id,)).fetchone()[0] > 0: # we have it in DB - - if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] == 0: - # we don't have it deleted in DB - with db_conn: - db_conn.execute(sql_requests.properly_delete_squad, (squad_id,)) - - return False # squadron stop their existing or never exists... it doesn't exists anyway - - else: # any other codes (except 418, that one handles in authed_request), never should happen - logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}') - raise utils.FAPIUnknownStatusCode + else (fail) + failed.append(id_to_try) + tries = tries + 1 + + sleep(3) + + +2. Update exists + get oldest updated existing squad + update it + if squad still exists + process triggers +""" +def discover_triggers(squad_info: dict): + print(squad_info.get('name'), squad_info.get('tag'), squad_info.get('ownerName')) +def discover(): + id_to_try = utils.get_last_known_id(db) + tries: int = 0 + failed: list = list() + tries_limit: int = 5000 + + while True: + id_to_try = id_to_try + 1 + logger.debug(f'Starting discover loop iteration, tries: {tries} of {tries_limit}, id to try {id_to_try}, ' + f'failed list: {failed}') + + if tries == tries_limit: + break + + squad_info = utils.update_squad_info(id_to_try, db, suppress_absence=True) + + if isinstance(squad_info, dict): # success + logger.debug(f'Success discover for {id_to_try} ID') + discover_triggers(squad_info) + tries = 0 # reset tries counter + for failed_squad in failed: # since we found an exists squad, then all previous failed wasn't exists + utils.properly_delete_squadron(failed_squad, db) + + failed = list() + + else: # should be only False + logger.debug(f'Fail on discovery for {id_to_try} ID') + failed.append(id_to_try) + tries = tries + 1 + + time.sleep(2) - -update_squad_info(47999, db) +discover() diff --git a/sql_requests.py b/sql_requests.py index 50790ec..b2270c5 100644 --- a/sql_requests.py +++ b/sql_requests.py @@ -60,4 +60,9 @@ properly_delete_squad: str = """insert into squads_states (squad_id) values (?); check_if_we_already_deleted_squad_in_db: str = """select count(*) from squads_states -where squad_id = ? and tag is null""" \ No newline at end of file +where squad_id = ? and tag is null""" + +select_last_known_id: str = """select squad_id +from squads_states +order by squad_id desc +limit 1;""" \ No newline at end of file diff --git a/utils.py b/utils.py index ad57672..eb26036 100644 --- a/utils.py +++ b/utils.py @@ -1,8 +1,16 @@ +import sqlite3 +from typing import Union import requests +import json from EDMCLogging import get_main_logger +import sql_requests logger = get_main_logger() +BASE_URL = 'https://api.orerve.net/2.0/website/squadron/' +INFO_ENDPOINT = 'info' +NEWS_ENDPOINT = 'news/list' + class FAPIDownForMaintenance(Exception): pass @@ -16,7 +24,7 @@ def authed_request(url: str, method: str = 'get', **kwargs) -> requests.Response """Makes request to any url with valid bearer token""" bearer: str = _get_bearer() - logger.debug(f'Requesting {method.upper()} {url!r}') + logger.debug(f'Requesting {method.upper()} {url!r}, kwargs: {kwargs}') fapiRequest: requests.Response = requests.request( method=method, @@ -84,3 +92,199 @@ def notify_discord(message: str) -> None: logger.debug('Sending successful') return + +def _update_squad_news(squad_id: int, db_conn: sqlite3.Connection) -> Union[bool, str]: + """Update news for squad with specified ID + + :param squad_id: id of squad to insert news + :param db_conn: connection to sqlite DB + :return: motd if squad exists, False if not + :rtype: bool, str + """ + + """ + How it should works? + Request news + if squad doesn't exists + return False + + else + insert all news even if it already exist in DB + return motd + """ + news_request: requests.Response = authed_request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id}) + if news_request.status_code != 200: # must not happen + logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}') + # we will not break it, let next code break it by itself + + squad_news: dict = news_request.json()['squadron'] + + if 'id' not in squad_news.keys(): # squadron doesn't FDEV + return False + + else: # squadron exists FDEV + del squad_news['id'] + + for type_of_news_key in squad_news: + one_type_of_news: list = squad_news[type_of_news_key] + + news: dict + for news in one_type_of_news: + with db_conn: + db_conn.execute( + sql_requests.insert_news, + ( + squad_id, + type_of_news_key, + news.get('id'), + news.get('date'), + news.get('category'), + news.get('activity'), + news.get('season'), + news.get('bookmark'), + news.get('motd'), + news.get('author'), + news.get('cmdr_id'), + news.get('user_id') + ) + ) + + return next(iter(squad_news['public_statements']), dict()).get('motd', '') + + +def update_squad_info(squad_id: int, db_conn: sqlite3.Connection, suppress_absence: bool = False) -> Union[bool, dict]: + """Update/insert information about squadron with specified id in our DB + + :param squad_id: id of squad to update/insert + :param db_conn: connection to sqlite DB + :param suppress_absence: if we shouldn't mark squad as deleted if we didn't found it by FDEV + :return: squad dict if squad exists, False if not + :rtype: bool, dict + """ + + """ + How it should works? + Request squad's info + + if squad is properly deleted in our DB + return False + + if squad exists FDEV + insert info in DB + request news, insert to DB + return squad dict + + if squad doesn't exists FDEV + if squad in DB + if isn't deleted in our DB + write to squads_states record with all null except ID (it will mean that squad was deleted) + else if not suppress_absence + write to squads_states record with all null except ID (it will mean that squad was deleted) + + + + return False + *Should we return something more then just a bool, may be a message to notify_discord? + """ + + if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] != 0: + # we have it as properly deleted in our DB + logger.debug(f'squad {squad_id} is marked as deleted in our DB, returning False') + return False + + squad_request: requests.Response = authed_request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id}) + + if squad_request.status_code == 200: # squad exists FDEV + squad_request_json: dict = squad_request.json()['squadron'] + squad_request_json['ownerName'] = fdev2people(squad_request_json['ownerName']) # normalize value + + with db_conn: + db_conn.execute( + sql_requests.insert_squad_states, + ( + squad_id, + squad_request_json['name'], + squad_request_json['tag'], + squad_request_json['ownerName'], + squad_request_json['ownerId'], + squad_request_json['platform'], + squad_request_json['created'], + squad_request_json['created_ts'], + squad_request_json['acceptingNewMembers'], + squad_request_json['powerId'], + squad_request_json['powerName'], + squad_request_json['superpowerId'], + squad_request_json['superpowerName'], + squad_request_json['factionId'], + squad_request_json['factionName'], + json.dumps(squad_request_json['userTags']), + squad_request_json['memberCount'], + squad_request_json['pendingCount'], + squad_request_json['full'], + squad_request_json['publicComms'], + squad_request_json['publicCommsOverride'], + squad_request_json['publicCommsAvailable'], + squad_request_json['current_season_trade_score'], + squad_request_json['previous_season_trade_score'], + squad_request_json['current_season_combat_score'], + squad_request_json['previous_season_combat_score'], + squad_request_json['current_season_exploration_score'], + squad_request_json['previous_season_exploration_score'], + squad_request_json['current_season_cqc_score'], + squad_request_json['previous_season_cqc_score'], + squad_request_json['current_season_bgs_score'], + squad_request_json['previous_season_bgs_score'], + squad_request_json['current_season_powerplay_score'], + squad_request_json['previous_season_powerplay_score'], + squad_request_json['current_season_aegis_score'], + squad_request_json['previous_season_aegis_score'] + ) + ) + + motd: str = _update_squad_news(squad_id, db_conn) # yeah, it can return bool but never should does it + squad_request_json.update(motd=motd) + + return squad_request_json + + elif squad_request.status_code == 404: # squad doesn't exists FDEV + if db_conn.execute( + sql_requests.check_if_squad_exists_in_db, + (squad_id,)).fetchone()[0] > 0: # we have it in DB + + if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] == 0: + # we don't have it deleted in DB, let's fix it + delete_squadron(squad_id, db_conn) + + elif not suppress_absence: + # we don't have it in DB at all but let's mark it as deleted to avoid requests to FDEV about it in future + delete_squadron(squad_id, db_conn) + + return False # squadron stop their existing or never exists... it doesn't exists anyway + + else: # any other codes (except 418, that one handles in authed_request), never should happen + logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}') + raise FAPIUnknownStatusCode + + +def properly_delete_squadron(squad_id: int, db_conn: sqlite3.Connection) -> None: + """Properly deletes squadron from our DB + + :param squad_id: squad id to delete + :param db_conn: connection to DB + :return: + """ + logger.debug(f'Properly deleting {squad_id}') + + with db_conn: + db_conn.execute(sql_requests.properly_delete_squad, (squad_id,)) + + +def get_last_known_id(db_conn: sqlite3.Connection) -> int: + sql_request_result = db_conn.execute(sql_requests.select_last_known_id).fetchone() + if sql_request_result is None: + logger.debug(f"Can't get last know id from DB, defaulting to 0") + return 0 + + else: + logger.debug(f'last know id from DB: {sql_request_result[0]}') + return sql_request_result[0]