diff --git a/DB.py b/DB.py index 12c3fa3..52bc79b 100644 --- a/DB.py +++ b/DB.py @@ -3,6 +3,36 @@ from SQLRequests import SQLRequests db = sqlite3.connect('jubilant-system.sqlite') db.row_factory = lambda c, r: dict(zip([col[0] for col in c.description], r)) +db.executescript(SQLRequests.schema) + + +class settings: + @staticmethod + def set(key: str, value: str | int): + if isinstance(value, int): + with db: + db.execute( + SQLRequests.settings_set_int, + {'int_value': value, 'key': key} + ) + + else: + with db: + db.execute( + SQLRequests.settings_set_str, + {'str_value': value, 'key': key} + ) + + +def enable_triggers() -> None: + settings.set('disable_triggers', 0) + + +def disable_triggers() -> None: + settings.set('disable_triggers', 1) + + +enable_triggers() def allocate_operation_id(squad_id: int) -> int: @@ -32,15 +62,16 @@ def insert_info_news(news_dict: dict | None, info_dict: dict) -> int: return operation_id -def delete_squadron(squad_id: int, suppress_deleted=True) -> None: +def delete_squadron(squad_id: int, suppress_deleted=True) -> int | None: """ - A function to make record in squadrons_deleted table + A function to make record in squadrons_deleted table and returns operation_id with squad_deletion + or None if squad wasn't deleted :param squad_id: squad_id to mark as deleted :param suppress_deleted: if IntegrityError exception due to this squad already exists in squadrons_deleted table should be suppressed - :return: + :return: operation_id or None """ try: @@ -48,6 +79,33 @@ def delete_squadron(squad_id: int, suppress_deleted=True) -> None: operation_id = allocate_operation_id(squad_id) db.execute(SQLRequests.delete_squadron, {'squad_id': squad_id, 'operation_id': operation_id}) + return operation_id + except sqlite3.IntegrityError as e: if not suppress_deleted: raise e + + +def build_squadrons_current_data() -> None: + db.executescript(SQLRequests.build_squadrons_current_data) + + +def last_known_squadron() -> int: + if (res := db.execute(SQLRequests.last_known_squadron).fetchone()) is None: + return 0 + + else: + return res['squad_id'] + + +def get_backupdate_squad_ids(limit: int) -> list[int]: + return [squad_row['squad_id'] for squad_row in db.execute(SQLRequests.select_new_squadrons_backupdate, {'limit': limit}).fetchall()] + + +def get_squads_for_update(limit: int) -> list[int]: + return [squad_row['squad_id'] for squad_row in db.execute(SQLRequests.get_squads_for_update, {'limit': limit}).fetchall()] + + +def ensure_squadrons_current_data_exists() -> None: + if db.execute(SQLRequests.ensure_squadrons_current_state_exists).fetchone()['count'] == 0: + build_squadrons_current_data() diff --git a/DataMigration.py b/DataMigration.py index b2551e1..a816f38 100644 --- a/DataMigration.py +++ b/DataMigration.py @@ -166,6 +166,7 @@ new_db.executescript(QUERIES.NEW.SCHEMA) news: dict[int, list[dict]] = defaultdict(list) news_cache_timer = time() for one_news in old_db.execute(QUERIES.OLD.ALL_NEWS_RECORDS): + one_news['inserted_timestamp'] = datetime.strptime(one_news['inserted_timestamp'], '%Y-%m-%d %H:%M:%S') news[one_news['squad_id']].append(one_news) print(f'news cached for {time() - news_cache_timer} s') @@ -204,14 +205,14 @@ for row in old_db.execute(QUERIES.OLD.ALL_RECORDS): high_bound = parsed_timestamp + delta for one_squad_news in news[squad_id]: - if low_bound < datetime.strptime(one_squad_news['inserted_timestamp'], '%Y-%m-%d %H:%M:%S') < high_bound: + if low_bound < one_squad_news['inserted_timestamp'] < high_bound: one_squad_news['operation_id'] = operation_id new_db.execute(QUERIES.NEW.INSERT_NEWS, one_squad_news) break - if iterations_counter % 1000 == 0: + if iterations_counter % 100000 == 0: new_db.commit() - print(f'Iterations: {iterations_counter}; avg iteration time: {(time() - loop_timer)/iterations_counter} s; avg local iter time {(time() - loop_timer_secondary)/1000} s') + print(f'Iterations: {iterations_counter}; avg iteration time: {(time() - loop_timer)/iterations_counter} s; avg local iter time {(time() - loop_timer_secondary)/100000} s') loop_timer_secondary = time() iterations_counter += 1 diff --git a/FAPI/Queries.py b/FAPI/Queries.py index bca9fe5..abf1cb8 100644 --- a/FAPI/Queries.py +++ b/FAPI/Queries.py @@ -41,7 +41,6 @@ def get_squad_info(squad_id: int) -> dict | None: """ """ - How it should works? Request squad's info if squad exists FDEV @@ -50,12 +49,6 @@ def get_squad_info(squad_id: int) -> dict | None: if squad doesn't exists FDEV return None """ - """ - if DB.db.execute(SQLRequests.squad_deleted, {'squad_id': squad_id}) == 1: - # we have it deleted in our DB - logger.debug(f'squad {squad_id} is marked as deleted in our DB, returning False') - return None - """ squad_request: requests.Response = request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id}) @@ -65,22 +58,9 @@ def get_squad_info(squad_id: int) -> dict | None: squad_request_json['userTags'] = json.dumps(squad_request_json['userTags']) squad_request_json = perform_info_mapping(squad_request_json) - """ - with DB.db: - operation_id = DB.allocate_operation_id(squad_id) - squad_request_json['operation_id'] = operation_id - DB.db.execute(SQLRequests.insert_info, squad_request_json) - """ - return squad_request_json elif squad_request.status_code == 404: # squad doesn't exists FDEV - """ - if not suppress_absence: - with DB.db: - operation_id = DB.allocate_operation_id(squad_id) - DB.db.execute(SQLRequests.delete_squadron, {'operation_id': operation_id, 'squad_id': squad_id}) - """ return None else: # any other codes (except 418, that one handles in authed_request), never should happen diff --git a/FAPI/__init__.py b/FAPI/__init__.py index cae44b2..a32b9b2 100644 --- a/FAPI/__init__.py +++ b/FAPI/__init__.py @@ -1 +1,24 @@ from . import Queries +import DB + + +def update_squad(squad_id: int, suppress_absence=False) -> None | int: + """ + Updates specified squad and returns operation_id or None if squad not found + :param squad_id: + :param suppress_absence: + :return: + """ + squad_info = Queries.get_squad_info(squad_id) + operation_id = None + if squad_info is None: + # Squad not found FDEV + if not suppress_absence: + operation_id = DB.delete_squadron(squad_id) + + else: + # Then we got valid squad_info dict + news_info = Queries.get_squad_news(squad_id) + operation_id = DB.insert_info_news(news_info, squad_info) + + return operation_id diff --git a/SQLRequests.py b/SQLRequests.py index 94bca1c..dd9e380 100644 --- a/SQLRequests.py +++ b/SQLRequests.py @@ -15,3 +15,9 @@ class SQLRequests: insert_info = read_request('insert_info') delete_squadron = read_request('delete_squadron') insert_news = read_request('insert_news') + settings_set_int = read_request('settings_set_int') + settings_set_str = read_request('settings_set_str') + last_known_squadron = read_request('latest_known_squadron') + select_new_squadrons_backupdate = read_request('select_new_squadrons_backupdate') + get_squads_for_update = read_request('get_squads_for_update') + ensure_squadrons_current_state_exists = read_request('ensure_squadrons_current_state_exists') diff --git a/main.py b/main.py index e3d4d4b..0391a7a 100644 --- a/main.py +++ b/main.py @@ -1,49 +1,226 @@ -""" -Workflow to discover new squadrons (db operations): -1. Get next id -2. Query info endpoint -3. If squadron exists: - Insert squad_id into operations_info - get operation_id - Insert into squadrons_historical_data data from info endpoint - Insert into squadrons_news_historical data from news endpoint +from loguru import logger -else: - ignore, don't insert squad_id to squadrons_deleted - -Workflow to update existing squadron: -1. Get most early updated squad from squadrons_current_data -2. Insert squad_id into operations_info -3. Get operation_id -4. Request info endpoint - if squad exists: - query news endpoint - insert data from info and news queries to appropriate historical tables - - else: - insert squad_id, operations_id to squadrons_deleted -""" +import time +import traceback import FAPI +import signal import DB +import sys +import inspect + +logger.remove() +logger.add(sys.stderr, level="DEBUG") + +shutting_down: bool = False +can_be_shutdown: bool = False -def update_squad(squad_id: int, suppress_absence=False) -> None: - squad_info = FAPI.Queries.get_squad_info(squad_id) - if squad_info is None: - # Squad not found FDEV - if not suppress_absence: - DB.delete_squadron(squad_id) +def shutdown_callback(sig: int, frame) -> None: + logger.info(f'Planning shutdown by {sig} signal') + try: + frame_info = inspect.getframeinfo(frame) + func = frame_info.function + code_line = frame_info.code_context[0] + logger.info(f'Currently at {func}:{frame_info.lineno}: {code_line!r}\n{traceback.print_tb(frame)}') - else: - # Then we got valid squad_info dict - news_info = FAPI.Queries.get_squad_news(squad_id) - print(DB.insert_info_news(news_info, squad_info)) + except Exception as e: + logger.info(f"Can't detect where we are because {e}") + + global shutting_down + shutting_down = True + + if can_be_shutdown: + logger.info('Can be shutdown') + exit(0) + + +def discover(back_count: int = 0): + """Discover new squads + :param back_count: int how many squads back we should check, it is helpful to recheck newly created squads + :return: + """ + + id_to_try = DB.last_known_squadron() + tries: int = 0 + failed: list = list() + TRIES_LIMIT_RETROSPECTIVELY: int = 5000 + TRIES_LIMIT_ON_THE_TIME: int = 5 + + def smart_tries_limit(_squad_id: int) -> int: + + if _squad_id < 65000: + return TRIES_LIMIT_RETROSPECTIVELY + + else: + return TRIES_LIMIT_ON_THE_TIME + + """ + tries_limit, probably, should be something more smart because on retrospectively scan we can + have large spaces of dead squadrons but when we are discovering on real time, large value of tries_limit + will just waste our time and, probable, confuses FDEV + *Outdated but it still can be more smart* + """ + + if back_count != 0: + logger.debug(f'back_count = {back_count}') + + squad_id: list + for squad_id in DB.get_backupdate_squad_ids(back_count): + squad_id: int = squad_id[0] + logger.debug(f'Back updating {squad_id}') + FAPI.update_squad(squad_id) + + while True: + + if shutting_down: + return + + id_to_try = id_to_try + 1 + # logger.debug(f'Starting discover loop iteration, tries: {tries} of {tries_limit}, id to try {id_to_try}, ' + # f'failed list: {failed}') + + if tries == smart_tries_limit(id_to_try): + break + + squad_operation_id = FAPI.update_squad(id_to_try, suppress_absence=True) + + if squad_operation_id is not None: # success + logger.debug(f'Success discover for {id_to_try} ID') + tries = 0 # reset tries counter + + for failed_squad in failed: # since we found an exists squad, then all previous failed wasn't exists + DB.delete_squadron(failed_squad) + + failed = list() + + else: # fail, should be only False + logger.debug(f'Fail on discovery for {id_to_try} ID') + failed.append(id_to_try) + tries = tries + 1 + + +def update(squad_id: int = None, amount_to_update: int = 1): + """ + + :param squad_id: update specified squad, updates only that squad + :param amount_to_update: update specified amount, ignores when squad_id specified + :return: + """ + + if isinstance(squad_id, int): + logger.debug(f'Going to update one specified squadron: {squad_id} ID') + FAPI.update_squad(squad_id, suppress_absence=True) + # suppress_absence is required because if we're manually updating squad with some high id it may just don't exist yet + return + + logger.debug(f'Going to update {amount_to_update} squadrons') + + squads_id_to_update: list[int] = DB.get_squads_for_update(amount_to_update) + + for id_to_update in squads_id_to_update: # if db is empty, then loop will not happen + + if shutting_down: + return + + logger.info(f'Updating {id_to_update} ID') + FAPI.update_squad(id_to_update) def main(): - update_squad(2530) - update_squad(47999) + DB.ensure_squadrons_current_data_exists() + global can_be_shutdown + signal.signal(signal.SIGTERM, shutdown_callback) + signal.signal(signal.SIGINT, shutdown_callback) + + def help_cli() -> str: + return """Possible arguments: + main.py discover + main.py update + main.py update amount + main.py update id + main.py daemon""" + + logger.debug(f'argv: {sys.argv}') + + if len(sys.argv) == 1: + print(help_cli()) + exit(1) + + elif len(sys.argv) == 2: + if sys.argv[1] == 'discover': + # main.py discover + logger.info(f'Entering discover mode') + discover() + exit(0) + + elif sys.argv[1] == 'update': + # main.py update + logger.info(f'Entering common update mode') + update() + exit(0) + + elif sys.argv[1] == 'daemon': + # main.py daemon + logger.info('Entering daemon mode') + while True: + can_be_shutdown = False + + update(amount_to_update=500) + if shutting_down: + exit(0) + + logger.info('Updated, sleeping') + can_be_shutdown = True + time.sleep(30 * 60) + can_be_shutdown = False + logger.info('Discovering') + + discover(back_count=20) + if shutting_down: + exit(0) + + logger.info('Discovered, sleeping') + can_be_shutdown = True + time.sleep(30 * 60) + + else: + print(help_cli()) + exit(1) + + elif len(sys.argv) == 4: + if sys.argv[1] == 'update': + if sys.argv[2] == 'amount': + # main.py update amount + + try: + amount: int = int(sys.argv[3]) + logger.info(f'Entering update amount mode, amount: {amount}') + update(amount_to_update=amount) + exit(0) + + except ValueError: + print('Amount must be integer') + exit(1) + + elif sys.argv[2] == 'id': + # main.py update id + try: + id_for_update: int = int(sys.argv[3]) + logger.info(f'Entering update specified squad: {id_for_update} ID') + update(squad_id=id_for_update) + exit(0) + + except ValueError: + print('ID must be integer') + exit(1) + + else: + logger.info(f'Unknown argument {sys.argv[2]}') + + else: + print(help_cli()) + exit(1) if __name__ == '__main__': diff --git a/sql/ensure_squadrons_current_state_exists.sql b/sql/ensure_squadrons_current_state_exists.sql new file mode 100644 index 0000000..41264c0 --- /dev/null +++ b/sql/ensure_squadrons_current_state_exists.sql @@ -0,0 +1 @@ +select count(*) as count from squadrons_current_data; \ No newline at end of file diff --git a/sql/get_squads_for_update.sql b/sql/get_squads_for_update.sql new file mode 100644 index 0000000..73db9a4 --- /dev/null +++ b/sql/get_squads_for_update.sql @@ -0,0 +1,3 @@ +select squad_id from squadrons_current_data +order by updated +limit :limit \ No newline at end of file diff --git a/sql/latest_known_squadron.sql b/sql/latest_known_squadron.sql new file mode 100644 index 0000000..d31a1be --- /dev/null +++ b/sql/latest_known_squadron.sql @@ -0,0 +1 @@ +select max(squad_id) as squad_id from squadrons_current_data; \ No newline at end of file diff --git a/sql/schema.sql b/sql/schema.sql index fac98e3..dad386b 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -230,6 +230,8 @@ begin new.current_season_aegis_score, new.previous_season_aegis_score, (select timestamp from operations_info where operations_info.operation_id = new.operation_id)) on conflict do update set + updated = (select timestamp from operations_info where operations_info.operation_id = new.operation_id), + operation_id = new.operation_id, name = new.name, tag = new.tag, owner_id = new.owner_id, @@ -299,6 +301,8 @@ begin new.operation_id, (select timestamp from operations_info where operations_info.operation_id = new.operation_id) ) on conflict do update set + updated = (select timestamp from operations_info where operations_info.operation_id = new.operation_id), + operation_id = new.operation_id, motd = new.motd, author = new.author, cmdr_id = new.cmdr_id, diff --git a/sql/select_new_squadrons_backupdate.sql b/sql/select_new_squadrons_backupdate.sql new file mode 100644 index 0000000..a7cab0c --- /dev/null +++ b/sql/select_new_squadrons_backupdate.sql @@ -0,0 +1,10 @@ +select * from + ( + select distinct squad_id + from squadrons_historical_data + inner join operations_info oi + on oi.operation_id = squadrons_historical_data.operation_id + except select squad_id from squadrons_deleted + ) +order by squad_id desc +limit :count; \ No newline at end of file diff --git a/sql/settings_set_int.sql b/sql/settings_set_int.sql new file mode 100644 index 0000000..d14a7fc --- /dev/null +++ b/sql/settings_set_int.sql @@ -0,0 +1,6 @@ +insert into settings ( + key, int_value + ) values ( + :key, + :int_value + ) on conflict do update set int_value = :int_value where key = :key; \ No newline at end of file diff --git a/sql/settings_set_str.sql b/sql/settings_set_str.sql new file mode 100644 index 0000000..3fdc32f --- /dev/null +++ b/sql/settings_set_str.sql @@ -0,0 +1,6 @@ +insert into settings ( + key, txt_value + ) values ( + :key, + :txt_value + ) on conflict do update set txt_value = :txt_value where key = :key; \ No newline at end of file