Done updating and discovering

DB.py

@@ -3,6 +3,36 @@ from SQLRequests import SQLRequests
 
 db = sqlite3.connect('jubilant-system.sqlite')
 db.row_factory = lambda c, r: dict(zip([col[0] for col in c.description], r))
+db.executescript(SQLRequests.schema)
+
+
+class settings:
+    @staticmethod
+    def set(key: str, value: str | int):
+        if isinstance(value, int):
+            with db:
+                db.execute(
+                    SQLRequests.settings_set_int,
+                    {'int_value': value, 'key': key}
+                )
+
+        else:
+            with db:
+                db.execute(
+                    SQLRequests.settings_set_str,
+                    {'str_value': value, 'key': key}
+                )
+
+
+def enable_triggers() -> None:
+    settings.set('disable_triggers', 0)
+
+
+def disable_triggers() -> None:
+    settings.set('disable_triggers', 1)
+
+
+enable_triggers()
 
 
 def allocate_operation_id(squad_id: int) -> int:
@@ -32,15 +62,16 @@ def insert_info_news(news_dict: dict | None, info_dict: dict) -> int:
     return operation_id
 
 
-def delete_squadron(squad_id: int, suppress_deleted=True) -> None:
+def delete_squadron(squad_id: int, suppress_deleted=True) -> int | None:
     """
-    A function to make record in squadrons_deleted table
+    A function to make record in squadrons_deleted table and returns operation_id with squad_deletion
+    or None if squad wasn't deleted
 
     :param squad_id: squad_id to mark as deleted
     :param suppress_deleted: if IntegrityError exception due to this squad already
     exists in squadrons_deleted table should be suppressed
 
-    :return:
+    :return: operation_id or None
     """
 
     try:
@@ -48,6 +79,33 @@ def delete_squadron(squad_id: int, suppress_deleted=True) -> None:
             operation_id = allocate_operation_id(squad_id)
             db.execute(SQLRequests.delete_squadron, {'squad_id': squad_id, 'operation_id': operation_id})
 
+        return operation_id
+
     except sqlite3.IntegrityError as e:
         if not suppress_deleted:
             raise e
+
+
+def build_squadrons_current_data() -> None:
+    db.executescript(SQLRequests.build_squadrons_current_data)
+
+
+def last_known_squadron() -> int:
+    if (res := db.execute(SQLRequests.last_known_squadron).fetchone()) is None:
+        return 0
+
+    else:
+        return res['squad_id']
+
+
+def get_backupdate_squad_ids(limit: int) -> list[int]:
+    return [squad_row['squad_id'] for squad_row in db.execute(SQLRequests.select_new_squadrons_backupdate, {'limit': limit}).fetchall()]
+
+
+def get_squads_for_update(limit: int) -> list[int]:
+    return [squad_row['squad_id'] for squad_row in db.execute(SQLRequests.get_squads_for_update, {'limit': limit}).fetchall()]
+
+
+def ensure_squadrons_current_data_exists() -> None:
+    if db.execute(SQLRequests.ensure_squadrons_current_state_exists).fetchone()['count'] == 0:
+        build_squadrons_current_data()

DataMigration.py

@@ -166,6 +166,7 @@ new_db.executescript(QUERIES.NEW.SCHEMA)
 news: dict[int, list[dict]] = defaultdict(list)
 news_cache_timer = time()
 for one_news in old_db.execute(QUERIES.OLD.ALL_NEWS_RECORDS):
+    one_news['inserted_timestamp'] = datetime.strptime(one_news['inserted_timestamp'], '%Y-%m-%d %H:%M:%S')
     news[one_news['squad_id']].append(one_news)
 
 print(f'news cached for {time() - news_cache_timer} s')
@@ -204,14 +205,14 @@ for row in old_db.execute(QUERIES.OLD.ALL_RECORDS):
         high_bound = parsed_timestamp + delta
 
         for one_squad_news in news[squad_id]:
-            if low_bound < datetime.strptime(one_squad_news['inserted_timestamp'], '%Y-%m-%d %H:%M:%S') < high_bound:
+            if low_bound < one_squad_news['inserted_timestamp'] < high_bound:
                 one_squad_news['operation_id'] = operation_id
                 new_db.execute(QUERIES.NEW.INSERT_NEWS, one_squad_news)
                 break
 
-    if iterations_counter % 1000 == 0:
+    if iterations_counter % 100000 == 0:
         new_db.commit()
-        print(f'Iterations: {iterations_counter}; avg iteration time: {(time() - loop_timer)/iterations_counter} s; avg local iter time {(time() - loop_timer_secondary)/1000} s')
+        print(f'Iterations: {iterations_counter}; avg iteration time: {(time() - loop_timer)/iterations_counter} s; avg local iter time {(time() - loop_timer_secondary)/100000} s')
         loop_timer_secondary = time()
 
     iterations_counter += 1

FAPI/Queries.py

@@ -41,7 +41,6 @@ def get_squad_info(squad_id: int) -> dict | None:
     """
 
     """
-    How it should works? 
     Request squad's info
     
     if squad exists FDEV
@@ -50,12 +49,6 @@ def get_squad_info(squad_id: int) -> dict | None:
     if squad doesn't exists FDEV        
        return None
     """
-    """
-    if DB.db.execute(SQLRequests.squad_deleted, {'squad_id': squad_id}) == 1:
-        # we have it deleted in our DB
-        logger.debug(f'squad {squad_id} is marked as deleted in our DB, returning False')
-        return None
-    """
 
     squad_request: requests.Response = request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id})
 
@@ -65,22 +58,9 @@ def get_squad_info(squad_id: int) -> dict | None:
         squad_request_json['userTags'] = json.dumps(squad_request_json['userTags'])
         squad_request_json = perform_info_mapping(squad_request_json)
 
-        """
-        with DB.db:
-            operation_id = DB.allocate_operation_id(squad_id)
-            squad_request_json['operation_id'] = operation_id
-            DB.db.execute(SQLRequests.insert_info, squad_request_json)
-         """
-
         return squad_request_json
 
     elif squad_request.status_code == 404:  # squad doesn't exists FDEV
-        """
-        if not suppress_absence:
-            with DB.db:
-                operation_id = DB.allocate_operation_id(squad_id)
-                DB.db.execute(SQLRequests.delete_squadron, {'operation_id': operation_id, 'squad_id': squad_id})
-        """
         return None
 
     else:  # any other codes (except 418, that one handles in authed_request), never should happen

FAPI/__init__.py

@@ -1 +1,24 @@
 from . import Queries
+import DB
+
+
+def update_squad(squad_id: int, suppress_absence=False) -> None | int:
+    """
+    Updates specified squad and returns operation_id or None if squad not found
+    :param squad_id:
+    :param suppress_absence:
+    :return:
+    """
+    squad_info = Queries.get_squad_info(squad_id)
+    operation_id = None
+    if squad_info is None:
+        # Squad not found FDEV
+        if not suppress_absence:
+            operation_id = DB.delete_squadron(squad_id)
+
+    else:
+        # Then we got valid squad_info dict
+        news_info = Queries.get_squad_news(squad_id)
+        operation_id = DB.insert_info_news(news_info, squad_info)
+
+    return operation_id

SQLRequests.py

@@ -15,3 +15,9 @@ class SQLRequests:
     insert_info = read_request('insert_info')
     delete_squadron = read_request('delete_squadron')
     insert_news = read_request('insert_news')
+    settings_set_int = read_request('settings_set_int')
+    settings_set_str = read_request('settings_set_str')
+    last_known_squadron = read_request('latest_known_squadron')
+    select_new_squadrons_backupdate = read_request('select_new_squadrons_backupdate')
+    get_squads_for_update = read_request('get_squads_for_update')
+    ensure_squadrons_current_state_exists = read_request('ensure_squadrons_current_state_exists')

main.py

@@ -1,49 +1,226 @@
-"""
-Workflow to discover new squadrons (db operations):
-1. Get next id
-2. Query info endpoint
-3. If squadron exists:
-    Insert squad_id into operations_info
-    get operation_id
-    Insert into squadrons_historical_data data from info endpoint
-    Insert into squadrons_news_historical data from news endpoint
+from loguru import logger
 
-else:
-    ignore, don't insert squad_id to squadrons_deleted
-
-Workflow to update existing squadron:
-1. Get most early updated squad from squadrons_current_data
-2. Insert squad_id into operations_info
-3. Get operation_id
-4. Request info endpoint
-    if squad exists:
-        query news endpoint
-        insert data from info and news queries to appropriate historical tables
-
-    else:
-        insert squad_id, operations_id to squadrons_deleted
-"""
+import time
+import traceback
 
 import FAPI
+import signal
 import DB
+import sys
+import inspect
+
+logger.remove()
+logger.add(sys.stderr, level="DEBUG")
+
+shutting_down: bool = False
+can_be_shutdown: bool = False
 
 
-def update_squad(squad_id: int, suppress_absence=False) -> None:
-    squad_info = FAPI.Queries.get_squad_info(squad_id)
-    if squad_info is None:
-        # Squad not found FDEV
-        if not suppress_absence:
-            DB.delete_squadron(squad_id)
+def shutdown_callback(sig: int, frame) -> None:
+    logger.info(f'Planning shutdown by {sig} signal')
+    try:
+        frame_info = inspect.getframeinfo(frame)
+        func = frame_info.function
+        code_line = frame_info.code_context[0]
+        logger.info(f'Currently at {func}:{frame_info.lineno}: {code_line!r}\n{traceback.print_tb(frame)}')
 
-    else:
-        # Then we got valid squad_info dict
-        news_info = FAPI.Queries.get_squad_news(squad_id)
-        print(DB.insert_info_news(news_info, squad_info))
+    except Exception as e:
+        logger.info(f"Can't detect where we are because {e}")
+
+    global shutting_down
+    shutting_down = True
+
+    if can_be_shutdown:
+        logger.info('Can be shutdown')
+        exit(0)
+
+
+def discover(back_count: int = 0):
+    """Discover new squads
+    :param back_count: int how many squads back we should check, it is helpful to recheck newly created squads
+    :return:
+    """
+
+    id_to_try = DB.last_known_squadron()
+    tries: int = 0
+    failed: list = list()
+    TRIES_LIMIT_RETROSPECTIVELY: int = 5000
+    TRIES_LIMIT_ON_THE_TIME: int = 5
+
+    def smart_tries_limit(_squad_id: int) -> int:
+
+        if _squad_id < 65000:
+            return TRIES_LIMIT_RETROSPECTIVELY
+
+        else:
+            return TRIES_LIMIT_ON_THE_TIME
+
+    """
+    tries_limit, probably, should be something more smart because on retrospectively scan we can
+    have large spaces of dead squadrons but when we are discovering on real time, large value of tries_limit
+    will just waste our time and, probable, confuses FDEV 
+    *Outdated but it still can be more smart*
+    """
+
+    if back_count != 0:
+        logger.debug(f'back_count = {back_count}')
+
+        squad_id: list
+        for squad_id in DB.get_backupdate_squad_ids(back_count):
+            squad_id: int = squad_id[0]
+            logger.debug(f'Back updating {squad_id}')
+            FAPI.update_squad(squad_id)
+
+    while True:
+
+        if shutting_down:
+            return
+
+        id_to_try = id_to_try + 1
+        # logger.debug(f'Starting discover loop iteration, tries: {tries} of {tries_limit}, id to try {id_to_try}, '
+        #             f'failed list: {failed}')
+
+        if tries == smart_tries_limit(id_to_try):
+            break
+
+        squad_operation_id = FAPI.update_squad(id_to_try, suppress_absence=True)
+
+        if squad_operation_id is not None:  # success
+            logger.debug(f'Success discover for {id_to_try} ID')
+            tries = 0  # reset tries counter
+
+            for failed_squad in failed:  # since we found an exists squad, then all previous failed wasn't exists
+                DB.delete_squadron(failed_squad)
+
+            failed = list()
+
+        else:  # fail, should be only False
+            logger.debug(f'Fail on discovery for {id_to_try} ID')
+            failed.append(id_to_try)
+            tries = tries + 1
+
+
+def update(squad_id: int = None, amount_to_update: int = 1):
+    """
+
+    :param squad_id: update specified squad, updates only that squad
+    :param amount_to_update: update specified amount, ignores when squad_id specified
+    :return:
+    """
+
+    if isinstance(squad_id, int):
+        logger.debug(f'Going to update one specified squadron: {squad_id} ID')
+        FAPI.update_squad(squad_id, suppress_absence=True)
+        # suppress_absence is required because if we're manually updating squad with some high id it may just don't exist yet
+        return
+
+    logger.debug(f'Going to update {amount_to_update} squadrons')
+
+    squads_id_to_update: list[int] = DB.get_squads_for_update(amount_to_update)
+
+    for id_to_update in squads_id_to_update:  # if db is empty, then loop will not happen
+
+        if shutting_down:
+            return
+
+        logger.info(f'Updating {id_to_update} ID')
+        FAPI.update_squad(id_to_update)
 
 
 def main():
-    update_squad(2530)
-    update_squad(47999)
+    DB.ensure_squadrons_current_data_exists()
+    global can_be_shutdown
+    signal.signal(signal.SIGTERM, shutdown_callback)
+    signal.signal(signal.SIGINT, shutdown_callback)
+
+    def help_cli() -> str:
+        return """Possible arguments:
+    main.py discover
+    main.py update
+    main.py update amount <amount: int>
+    main.py update id <id: int>
+    main.py daemon"""
+
+    logger.debug(f'argv: {sys.argv}')
+
+    if len(sys.argv) == 1:
+        print(help_cli())
+        exit(1)
+
+    elif len(sys.argv) == 2:
+        if sys.argv[1] == 'discover':
+            # main.py discover
+            logger.info(f'Entering discover mode')
+            discover()
+            exit(0)
+
+        elif sys.argv[1] == 'update':
+            # main.py update
+            logger.info(f'Entering common update mode')
+            update()
+            exit(0)
+
+        elif sys.argv[1] == 'daemon':
+            # main.py daemon
+            logger.info('Entering daemon mode')
+            while True:
+                can_be_shutdown = False
+
+                update(amount_to_update=500)
+                if shutting_down:
+                    exit(0)
+
+                logger.info('Updated, sleeping')
+                can_be_shutdown = True
+                time.sleep(30 * 60)
+                can_be_shutdown = False
+                logger.info('Discovering')
+
+                discover(back_count=20)
+                if shutting_down:
+                    exit(0)
+
+                logger.info('Discovered, sleeping')
+                can_be_shutdown = True
+                time.sleep(30 * 60)
+
+        else:
+            print(help_cli())
+            exit(1)
+
+    elif len(sys.argv) == 4:
+        if sys.argv[1] == 'update':
+            if sys.argv[2] == 'amount':
+                # main.py update amount <amount: int>
+
+                try:
+                    amount: int = int(sys.argv[3])
+                    logger.info(f'Entering update amount mode, amount: {amount}')
+                    update(amount_to_update=amount)
+                    exit(0)
+
+                except ValueError:
+                    print('Amount must be integer')
+                    exit(1)
+
+            elif sys.argv[2] == 'id':
+                # main.py update id <id: int>
+                try:
+                    id_for_update: int = int(sys.argv[3])
+                    logger.info(f'Entering update specified squad: {id_for_update} ID')
+                    update(squad_id=id_for_update)
+                    exit(0)
+
+                except ValueError:
+                    print('ID must be integer')
+                    exit(1)
+
+            else:
+                logger.info(f'Unknown argument {sys.argv[2]}')
+
+    else:
+        print(help_cli())
+        exit(1)
 
 
 if __name__ == '__main__':

sql/ensure_squadrons_current_state_exists.sql

@@ -0,0 +1 @@
+select count(*) as count from squadrons_current_data;

sql/get_squads_for_update.sql

@@ -0,0 +1,3 @@
+select squad_id from squadrons_current_data
+order by updated
+limit :limit

sql/latest_known_squadron.sql

@@ -0,0 +1 @@
+select max(squad_id) as squad_id from squadrons_current_data;

sql/schema.sql

@@ -230,6 +230,8 @@ begin
     new.current_season_aegis_score,
     new.previous_season_aegis_score,
     (select timestamp from operations_info where operations_info.operation_id = new.operation_id)) on conflict do update set
+        updated = (select timestamp from operations_info where operations_info.operation_id = new.operation_id),
+        operation_id = new.operation_id,
         name = new.name,
         tag = new.tag,
         owner_id = new.owner_id,
@@ -299,6 +301,8 @@ begin
      new.operation_id,
      (select timestamp from operations_info where operations_info.operation_id = new.operation_id)
      ) on conflict do update set
+    updated = (select timestamp from operations_info where operations_info.operation_id = new.operation_id),
+    operation_id = new.operation_id,
     motd = new.motd,
     author = new.author,
     cmdr_id = new.cmdr_id,

sql/select_new_squadrons_backupdate.sql

@@ -0,0 +1,10 @@
+select * from
+              (
+                  select distinct squad_id
+                  from squadrons_historical_data
+                      inner join operations_info oi
+                          on oi.operation_id = squadrons_historical_data.operation_id
+                  except select squad_id from squadrons_deleted
+                  )
+order by squad_id desc
+limit :count;

sql/settings_set_int.sql

@@ -0,0 +1,6 @@
+insert into settings (
+                      key, int_value
+                      ) values (
+                                :key,
+                                :int_value
+                                ) on conflict do update set int_value = :int_value where key = :key;

sql/settings_set_str.sql

@@ -0,0 +1,6 @@
+insert into settings (
+                      key, txt_value
+                      ) values (
+                                :key,
+                                :txt_value
+                                ) on conflict do update set txt_value = :txt_value where key = :key;