discover mode done

doc.txt

@@ -6,7 +6,7 @@ What to track?
 functionality:
 ASAP check for new squads
 Occasionally iterate over all squads (by id)
-update/insert new data to DB (if nothing changed, then just write empty history record with inserted_timestamp and squad_id)
+update/insert new data to DB
 alert if new data accord to triggers
 
 DB tables

main.py

@@ -1,11 +1,8 @@
-import json
-import requests
 import sqlite3
 import time
-import datetime
+
 import utils
 from EDMCLogging import get_main_logger
-import sql_requests
 
 logger = get_main_logger()
 db = sqlite3.connect('squads.sqlite')
@@ -13,166 +10,79 @@ db = sqlite3.connect('squads.sqlite')
 with open('sql_schema.sql', 'r', encoding='utf-8') as schema_file:
     db.executescript(''.join(schema_file.readlines()))
 
-ruTag = 32
+ruTag: id = 32
-BASE_URL = 'https://api.orerve.net/2.0/website/squadron/'
-INFO_ENDPOINT = 'info'
-NEWS_ENDPOINT = 'news/list'
 
+"""
+Two modes:
+1. Discover new squads
+    get last_known_id
+    tries = 0
+    failed: list
+    while True
+        if tries = 2
+            break
 
-def update_squad_news(squad_id: int, db_conn: sqlite3.Connection) -> bool:
+        id_to_try = last_known_id + 1
-    """Update news for squad with specified ID
+        update squad info with id_to_try and suppressing absence 
-
+        
-    :param squad_id: id of squad to insert news
+        if success
-    :param db_conn: connection to sqlite DB
+            process triggers
-    :return: True if squad exists, False if not
+            tries = 0
-    :rtype: bool
+            for failed_squad in failed
-    """
+                delete(failed_squad)
-
+                failed_squad = list()
-    """
+        
-    How it should works?
-    Request news
-    if squad doesn't exists
-        return False
-    
-    else
-        insert all news even if it already exist in DB
-        return True
-    """
-    news_request: requests.Response = utils.authed_request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id})
-    if news_request.status_code != 200:  # must not happen
-        logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}')
-        # we will not break it, let next code break it by itself
-
-    squad_news: dict = news_request.json()['squadron']
-
-    if 'id' not in squad_news.keys():  # squadron doesn't FDEV
-        return False
-
-    else:  # squadron exists FDEV
-        del squad_news['id']
-
-        for type_of_news_key in squad_news:
-            one_type_of_news: list = squad_news[type_of_news_key]
-
-            news: dict
-            for news in one_type_of_news:
-                with db_conn:
-                    db_conn.execute(
-                        sql_requests.insert_news,
-                        (
-                            squad_id,
-                            type_of_news_key,
-                            news.get('id'),
-                            news.get('date'),
-                            news.get('category'),
-                            news.get('activity'),
-                            news.get('season'),
-                            news.get('bookmark'),
-                            news.get('motd'),
-                            news.get('author'),
-                            news.get('cmdr_id'),
-                            news.get('user_id')
-                        )
-                    )
-
-        return True
-
-
-def update_squad_info(squad_id: int, db_conn: sqlite3.Connection) -> bool:
-    """Update/insert information about squadron with specified id in our DB
-
-    :param squad_id: id of squad to update/insert
-    :param db_conn: connection to sqlite DB
-    :return: True if squad exists, False if not
-    :rtype: bool
-    """
-
-    """
-    How it should works?
-    Request squad's info
-    if squad exists FDEV
-        insert info in DB
-        request news, insert to DB
-        return True
-    
-    if squad doesn't exists FDEV
-        if squad in DB and isn't deleted in our DB
-            write to squads_states record with all null except ID (it will mean that squad was deleted)
             
-       return False
+        else (fail)
-    *Should we return something more then just a bool, may be a message to notify_discord?
+            failed.append(id_to_try)
-    """
+            tries = tries + 1
-
+            
-    squad_request: requests.Response = utils.authed_request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id})
+        sleep(3)
-
+            
-    if squad_request.status_code == 200:  # squad exists FDEV
+            
-        squad_request_json: dict = squad_request.json()['squadron']
+2. Update exists
-        with db_conn:
+    get oldest updated existing squad
-            db_conn.execute(
+    update it
-                sql_requests.insert_squad_states,
+    if squad still exists
-                (
+        process triggers
-                    squad_id,
+"""
-                    squad_request_json['name'],
-                    squad_request_json['tag'],
-                    utils.fdev2people(squad_request_json['ownerName']),
-                    squad_request_json['ownerId'],
-                    squad_request_json['platform'],
-                    squad_request_json['created'],
-                    squad_request_json['created_ts'],
-                    squad_request_json['acceptingNewMembers'],
-                    squad_request_json['powerId'],
-                    squad_request_json['powerName'],
-                    squad_request_json['superpowerId'],
-                    squad_request_json['superpowerName'],
-                    squad_request_json['factionId'],
-                    squad_request_json['factionName'],
-                    json.dumps(squad_request_json['userTags']),
-                    squad_request_json['memberCount'],
-                    squad_request_json['pendingCount'],
-                    squad_request_json['full'],
-                    squad_request_json['publicComms'],
-                    squad_request_json['publicCommsOverride'],
-                    squad_request_json['publicCommsAvailable'],
-                    squad_request_json['current_season_trade_score'],
-                    squad_request_json['previous_season_trade_score'],
-                    squad_request_json['current_season_combat_score'],
-                    squad_request_json['previous_season_combat_score'],
-                    squad_request_json['current_season_exploration_score'],
-                    squad_request_json['previous_season_exploration_score'],
-                    squad_request_json['current_season_cqc_score'],
-                    squad_request_json['previous_season_cqc_score'],
-                    squad_request_json['current_season_bgs_score'],
-                    squad_request_json['previous_season_bgs_score'],
-                    squad_request_json['current_season_powerplay_score'],
-                    squad_request_json['previous_season_powerplay_score'],
-                    squad_request_json['current_season_aegis_score'],
-                    squad_request_json['previous_season_aegis_score']
-                )
-            )
-
-            update_squad_news(squad_id, db_conn)
-            return True
-
-    elif squad_request.status_code == 404:  # squad doesn't exists FDEV
-        if db_conn.execute(
-                sql_requests.check_if_squad_exists_in_db,
-                (squad_id,)).fetchone()[0] > 0:  # we have it in DB
-
-            if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] == 0:
-                # we don't have it deleted in DB
-                with db_conn:
-                    db_conn.execute(sql_requests.properly_delete_squad, (squad_id,))
-
-        return False  # squadron stop their existing or never exists... it doesn't exists anyway
-
-    else:  # any other codes (except 418, that one handles in authed_request), never should happen
-        logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}')
-        raise utils.FAPIUnknownStatusCode
 
 
+def discover_triggers(squad_info: dict):
+    print(squad_info.get('name'), squad_info.get('tag'), squad_info.get('ownerName'))
 
 
+def discover():
+    id_to_try = utils.get_last_known_id(db)
+    tries: int = 0
+    failed: list = list()
+    tries_limit: int = 5000
+
+    while True:
+        id_to_try = id_to_try + 1
+        logger.debug(f'Starting discover loop iteration, tries: {tries} of {tries_limit}, id to try {id_to_try}, '
+                     f'failed list: {failed}')
+
+        if tries == tries_limit:
+            break
+
+        squad_info = utils.update_squad_info(id_to_try, db, suppress_absence=True)
+
+        if isinstance(squad_info, dict):  # success
+            logger.debug(f'Success discover for {id_to_try} ID')
+            discover_triggers(squad_info)
+            tries = 0  # reset tries counter
+            for failed_squad in failed:  # since we found an exists squad, then all previous failed wasn't exists
+                utils.properly_delete_squadron(failed_squad, db)
+
+            failed = list()
+
+        else:  # should be only False
+            logger.debug(f'Fail on discovery for {id_to_try} ID')
+            failed.append(id_to_try)
+            tries = tries + 1
+
+        time.sleep(2)
 
 
-
+discover()
-update_squad_info(47999, db)

sql_requests.py

@@ -60,4 +60,9 @@ properly_delete_squad: str = """insert into squads_states (squad_id) values (?);
 
 check_if_we_already_deleted_squad_in_db: str = """select count(*) 
 from squads_states 
-where squad_id = ? and tag is null"""
+where squad_id = ? and tag is null"""
+
+select_last_known_id: str = """select squad_id 
+from squads_states 
+order by squad_id desc
+limit 1;"""

utils.py

@@ -1,8 +1,16 @@
+import sqlite3
+from typing import Union
 import requests
+import json
 from EDMCLogging import get_main_logger
+import sql_requests
 
 logger = get_main_logger()
 
+BASE_URL = 'https://api.orerve.net/2.0/website/squadron/'
+INFO_ENDPOINT = 'info'
+NEWS_ENDPOINT = 'news/list'
+
 
 class FAPIDownForMaintenance(Exception):
     pass
@@ -16,7 +24,7 @@ def authed_request(url: str, method: str = 'get', **kwargs) -> requests.Response
     """Makes request to any url with valid bearer token"""
     bearer: str = _get_bearer()
 
-    logger.debug(f'Requesting {method.upper()} {url!r}')
+    logger.debug(f'Requesting {method.upper()} {url!r}, kwargs: {kwargs}')
 
     fapiRequest: requests.Response = requests.request(
         method=method,
@@ -84,3 +92,199 @@ def notify_discord(message: str) -> None:
     logger.debug('Sending successful')
     return
 
+
+def _update_squad_news(squad_id: int, db_conn: sqlite3.Connection) -> Union[bool, str]:
+    """Update news for squad with specified ID
+
+    :param squad_id: id of squad to insert news
+    :param db_conn: connection to sqlite DB
+    :return: motd if squad exists, False if not
+    :rtype: bool, str
+    """
+
+    """
+    How it should works?
+    Request news
+    if squad doesn't exists
+        return False
+    
+    else
+        insert all news even if it already exist in DB
+        return motd
+    """
+    news_request: requests.Response = authed_request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id})
+    if news_request.status_code != 200:  # must not happen
+        logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}')
+        # we will not break it, let next code break it by itself
+
+    squad_news: dict = news_request.json()['squadron']
+
+    if 'id' not in squad_news.keys():  # squadron doesn't FDEV
+        return False
+
+    else:  # squadron exists FDEV
+        del squad_news['id']
+
+        for type_of_news_key in squad_news:
+            one_type_of_news: list = squad_news[type_of_news_key]
+
+            news: dict
+            for news in one_type_of_news:
+                with db_conn:
+                    db_conn.execute(
+                        sql_requests.insert_news,
+                        (
+                            squad_id,
+                            type_of_news_key,
+                            news.get('id'),
+                            news.get('date'),
+                            news.get('category'),
+                            news.get('activity'),
+                            news.get('season'),
+                            news.get('bookmark'),
+                            news.get('motd'),
+                            news.get('author'),
+                            news.get('cmdr_id'),
+                            news.get('user_id')
+                        )
+                    )
+
+        return next(iter(squad_news['public_statements']), dict()).get('motd', '')
+
+
+def update_squad_info(squad_id: int, db_conn: sqlite3.Connection, suppress_absence: bool = False) -> Union[bool, dict]:
+    """Update/insert information about squadron with specified id in our DB
+
+    :param squad_id: id of squad to update/insert
+    :param db_conn: connection to sqlite DB
+    :param suppress_absence: if we shouldn't mark squad as deleted if we didn't found it by FDEV
+    :return: squad dict if squad exists, False if not
+    :rtype: bool, dict
+    """
+
+    """
+    How it should works?
+    Request squad's info
+    
+    if squad is properly deleted in our DB
+        return False
+
+    if squad exists FDEV
+        insert info in DB
+        request news, insert to DB
+        return squad dict
+    
+    if squad doesn't exists FDEV
+        if squad in DB
+            if isn't deleted in our DB
+                write to squads_states record with all null except ID (it will mean that squad was deleted)
+        else if not suppress_absence
+            write to squads_states record with all null except ID (it will mean that squad was deleted)
+            
+            
+            
+       return False
+    *Should we return something more then just a bool, may be a message to notify_discord?
+    """
+
+    if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] != 0:
+        # we have it as properly deleted in our DB
+        logger.debug(f'squad {squad_id} is marked as deleted in our DB, returning False')
+        return False
+
+    squad_request: requests.Response = authed_request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id})
+
+    if squad_request.status_code == 200:  # squad exists FDEV
+        squad_request_json: dict = squad_request.json()['squadron']
+        squad_request_json['ownerName'] = fdev2people(squad_request_json['ownerName'])  # normalize value
+
+        with db_conn:
+            db_conn.execute(
+                sql_requests.insert_squad_states,
+                (
+                    squad_id,
+                    squad_request_json['name'],
+                    squad_request_json['tag'],
+                    squad_request_json['ownerName'],
+                    squad_request_json['ownerId'],
+                    squad_request_json['platform'],
+                    squad_request_json['created'],
+                    squad_request_json['created_ts'],
+                    squad_request_json['acceptingNewMembers'],
+                    squad_request_json['powerId'],
+                    squad_request_json['powerName'],
+                    squad_request_json['superpowerId'],
+                    squad_request_json['superpowerName'],
+                    squad_request_json['factionId'],
+                    squad_request_json['factionName'],
+                    json.dumps(squad_request_json['userTags']),
+                    squad_request_json['memberCount'],
+                    squad_request_json['pendingCount'],
+                    squad_request_json['full'],
+                    squad_request_json['publicComms'],
+                    squad_request_json['publicCommsOverride'],
+                    squad_request_json['publicCommsAvailable'],
+                    squad_request_json['current_season_trade_score'],
+                    squad_request_json['previous_season_trade_score'],
+                    squad_request_json['current_season_combat_score'],
+                    squad_request_json['previous_season_combat_score'],
+                    squad_request_json['current_season_exploration_score'],
+                    squad_request_json['previous_season_exploration_score'],
+                    squad_request_json['current_season_cqc_score'],
+                    squad_request_json['previous_season_cqc_score'],
+                    squad_request_json['current_season_bgs_score'],
+                    squad_request_json['previous_season_bgs_score'],
+                    squad_request_json['current_season_powerplay_score'],
+                    squad_request_json['previous_season_powerplay_score'],
+                    squad_request_json['current_season_aegis_score'],
+                    squad_request_json['previous_season_aegis_score']
+                )
+            )
+
+            motd: str = _update_squad_news(squad_id, db_conn)  # yeah, it can return bool but never should does it
+            squad_request_json.update(motd=motd)
+
+            return squad_request_json
+
+    elif squad_request.status_code == 404:  # squad doesn't exists FDEV
+        if db_conn.execute(
+                sql_requests.check_if_squad_exists_in_db,
+                (squad_id,)).fetchone()[0] > 0:  # we have it in DB
+
+            if db_conn.execute(sql_requests.check_if_we_already_deleted_squad_in_db, (squad_id,)).fetchone()[0] == 0:
+                # we don't have it deleted in DB, let's fix it
+                delete_squadron(squad_id, db_conn)
+
+        elif not suppress_absence:
+            # we don't have it in DB at all but let's mark it as deleted to avoid requests to FDEV about it in future
+            delete_squadron(squad_id, db_conn)
+
+        return False  # squadron stop their existing or never exists... it doesn't exists anyway
+
+    else:  # any other codes (except 418, that one handles in authed_request), never should happen
+        logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}')
+        raise FAPIUnknownStatusCode
+
+
+def properly_delete_squadron(squad_id: int, db_conn: sqlite3.Connection) -> None:
+    """Properly deletes squadron from our DB
+
+    :param squad_id: squad id to delete
+    :param db_conn: connection to DB
+    :return:
+    """
+    logger.debug(f'Properly deleting {squad_id}')
+
+    with db_conn:
+        db_conn.execute(sql_requests.properly_delete_squad, (squad_id,))
+
+
+def get_last_known_id(db_conn: sqlite3.Connection) -> int:
+    sql_request_result = db_conn.execute(sql_requests.select_last_known_id).fetchone()
+    if sql_request_result is None:
+        logger.debug(f"Can't get last know id from DB, defaulting to 0")
+        return 0
+
+    else:
+        logger.debug(f'last know id from DB: {sql_request_result[0]}')
+        return sql_request_result[0]