Basic functional implemented: can update squadron with specified id

DB.py

@@ -0,0 +1,53 @@
+import sqlite3
+from SQLRequests import SQLRequests
+
+db = sqlite3.connect('jubilant-system.sqlite')
+db.row_factory = lambda c, r: dict(zip([col[0] for col in c.description], r))
+
+
+def allocate_operation_id(squad_id: int) -> int:
+    return db.execute(SQLRequests.create_operation_id, {'squad_id': squad_id}).fetchone()['operation_id']
+
+
+def insert_info_news(news_dict: dict | None, info_dict: dict) -> int:
+    """
+    Saved both news and info endpoint's data
+
+    :param news_dict:
+    :param info_dict:
+    :return:
+    """
+
+    with db:
+        operation_id = allocate_operation_id(info_dict['squad_id'])
+        info_dict['operation_id'] = operation_id
+
+        db.execute(SQLRequests.insert_info, info_dict)
+
+        if news_dict is not None:
+            news_dict['type_of_news'] = 'public_statements'
+            news_dict['operation_id'] = operation_id
+            db.execute(SQLRequests.insert_news, news_dict)
+
+    return operation_id
+
+
+def delete_squadron(squad_id: int, suppress_deleted=True) -> None:
+    """
+    A function to make record in squadrons_deleted table
+
+    :param squad_id: squad_id to mark as deleted
+    :param suppress_deleted: if IntegrityError exception due to this squad already
+    exists in squadrons_deleted table should be suppressed
+
+    :return:
+    """
+
+    try:
+        with db:
+            operation_id = allocate_operation_id(squad_id)
+            db.execute(SQLRequests.delete_squadron, {'squad_id': squad_id, 'operation_id': operation_id})
+
+    except sqlite3.IntegrityError as e:
+        if not suppress_deleted:
+            raise e

FAPI/BearerManager.py

@@ -0,0 +1,39 @@
+from enum import Enum
+from loguru import logger
+import os
+import requests
+
+
+class BearerManager:
+    class Endpoints(Enum):
+        RANDOM = '/random_token'
+
+    def __init__(self, demb_capi_auth: str, base_address: str):
+
+        self.base_address = base_address
+        self.demb_capi_auth = demb_capi_auth
+
+    def get_random_bearer(self) -> str:
+        """Gets bearer token from capi.demb.design (companion-api project)
+
+        :return: bearer token as str
+        """
+
+        bearer_request: requests.Response = self._request(self.Endpoints.RANDOM)
+
+        try:
+            bearer: str = bearer_request.json()['access_token']
+
+        except Exception as e:
+            logger.exception(f'Unable to parse capi.demb.design answer\nrequested: {bearer_request.url!r}\n'
+                             f'code: {bearer_request.status_code!r}\nresponse: {bearer_request.content!r}', exc_info=e)
+            raise e
+
+        return bearer
+
+    def _request(self, _endpoint: Endpoints) -> requests.Response:
+        endpoint = self.base_address + _endpoint.value
+        return requests.get(url=endpoint, headers={'auth': self.demb_capi_auth})
+
+
+bearer_manager = BearerManager(os.environ['DEMB_CAPI_AUTH'], 'https://capi.demb.design')

FAPI/Converters.py

@@ -0,0 +1,3 @@
+def dehexify(hex_str: str) -> str:
+    """Converts string with hex chars to string"""
+    return bytes.fromhex(hex_str).decode('utf-8')

FAPI/Exceptions.py

@@ -0,0 +1,6 @@
+class FAPIDownForMaintenance(Exception):
+    pass
+
+
+class FAPIUnknownStatusCode(Exception):
+    pass

FAPI/Mappings.py

@@ -0,0 +1,45 @@
+info_request_mapping = {
+    "id": "squad_id",
+    "name": "name",
+    "tag": "tag",
+    "ownerId": "owner_id",
+    "ownerName": "owner_name",
+    "platform": "platform",
+    "created": "created",
+    "created_ts": "created_ts",
+    "acceptingNewMembers": "accepting_new_members",
+    "powerId": "power_id",
+    "powerName": "power_name",
+    "superpowerId": "superpower_id",
+    "superpowerName": "superpower_name",
+    "factionId": "faction_id",
+    "factionName": "faction_name",
+    "deleteAfter": "delete_after",
+    "userTags": "user_tags",
+    "memberCount": "member_count",
+    "onlineCount": "online_count",
+    "pendingCount": "pending_count",
+    "publicComms": "public_comms",
+    "publicCommsOverride": "public_comms_override",
+    "publicCommsAvailable": "public_comms_available"
+}
+
+news_request_mapping = {
+    'id': 'news_id'
+}
+
+
+def perform_mapping(mapping: dict, dict_to_map: dict) -> dict:
+    for key in (list(dict_to_map.keys())):
+        if key in mapping:
+            dict_to_map[mapping[key]] = dict_to_map.pop(key)
+
+    return dict_to_map
+
+
+def perform_info_mapping(info_data: dict) -> dict:
+    return perform_mapping(info_request_mapping, info_data)
+
+
+def perform_news_mapping(news_data: dict) -> dict:
+    return perform_mapping(news_request_mapping, news_data)

FAPI/Queries.py

@@ -0,0 +1,88 @@
+from .Mappings import perform_info_mapping, perform_news_mapping
+from .Exceptions import *
+from .Requester import *
+from loguru import logger
+from . import Converters
+import json
+
+
+def get_squad_news(squad_id) -> dict | None:
+    """
+    Returns news of squadron with specified id or None if squadron doesn't exists or there is no news
+
+    :param squad_id:
+    :return:
+    """
+
+    news_request: requests.Response = request(BASE_URL + NEWS_ENDPOINT, params={'squadronId': squad_id})
+    if news_request.status_code != 200:  # must not happen
+        logger.warning(f'Got not 200 status code on requesting news, content: {news_request.content}, '
+                       f'code: {news_request.status_code}')
+
+    squad_news: dict = news_request.json()['squadron']
+
+    if isinstance(squad_news, list):  # check squadron 2517 for example 0_0
+        # squadron have no public statements
+        return None
+
+    elif 'id' not in squad_news.keys():  # squadron doesn't FDEV
+        return None
+
+    else:
+        if 'public_statements' in squad_news.keys() and len(squad_news['public_statements']) > 0:
+            return perform_news_mapping(squad_news['public_statements'][0])
+
+
+def get_squad_info(squad_id: int) -> dict | None:
+    """Returns information about squadron with specified id or None if squadrons doesn't exists on FDEV side
+
+    :param squad_id: id of squad to update/insert
+    :return: dict with state of squadron or None if squad not found
+    """
+
+    """
+    How it should works? 
+    Request squad's info
+    
+    if squad exists FDEV
+        return squadron state
+    
+    if squad doesn't exists FDEV        
+       return None
+    """
+    """
+    if DB.db.execute(SQLRequests.squad_deleted, {'squad_id': squad_id}) == 1:
+        # we have it deleted in our DB
+        logger.debug(f'squad {squad_id} is marked as deleted in our DB, returning False')
+        return None
+    """
+
+    squad_request: requests.Response = request(BASE_URL + INFO_ENDPOINT, params={'squadronId': squad_id})
+
+    if squad_request.status_code == 200:  # squad exists FDEV
+        squad_request_json: dict = squad_request.json()['squadron']
+        squad_request_json['ownerName'] = Converters.dehexify(squad_request_json['ownerName'])  # normalize value
+        squad_request_json['userTags'] = json.dumps(squad_request_json['userTags'])
+        squad_request_json = perform_info_mapping(squad_request_json)
+
+        """
+        with DB.db:
+            operation_id = DB.allocate_operation_id(squad_id)
+            squad_request_json['operation_id'] = operation_id
+            DB.db.execute(SQLRequests.insert_info, squad_request_json)
+         """
+
+        return squad_request_json
+
+    elif squad_request.status_code == 404:  # squad doesn't exists FDEV
+        """
+        if not suppress_absence:
+            with DB.db:
+                operation_id = DB.allocate_operation_id(squad_id)
+                DB.db.execute(SQLRequests.delete_squadron, {'operation_id': operation_id, 'squad_id': squad_id})
+        """
+        return None
+
+    else:  # any other codes (except 418, that one handles in authed_request), never should happen
+        logger.warning(f'Unknown squad info status_code: {squad_request.status_code}, content: {squad_request.content}')
+        raise FAPIUnknownStatusCode(f'Status code: {squad_request.status_code}, content: {squad_request.content}')

FAPI/Requester.py

@@ -0,0 +1,30 @@
+import requests
+from .BearerManager import bearer_manager
+from loguru import logger
+from . import Exceptions
+
+
+"""
+Functions to perform queries to FDEV
+"""
+
+
+BASE_URL = 'https://api.orerve.net/2.0/website/squadron/'
+INFO_ENDPOINT = 'info'
+NEWS_ENDPOINT = 'news/list'
+
+
+def request(url: str, method: str = 'get', **kwargs) -> requests.Response:
+    _request: requests.Response = requests.request(
+        method=method,
+        url=url,
+        headers={'Authorization': f'Bearer {bearer_manager.get_random_bearer()}'},
+        **kwargs
+    )
+
+    if _request.status_code == 418:  # FAPI is on maintenance
+        logger.warning(f'{method.upper()} {_request.url} returned 418, content dump:\n{_request.content}')
+
+        raise Exceptions.FAPIDownForMaintenance
+
+    return _request

FAPI/__init__.py

@@ -0,0 +1 @@
+from . import Queries

SQLRequests.py

@@ -0,0 +1,17 @@
+from os import path
+
+
+def read_request(name: str) -> str:
+    req_path = path.join('sql', name+'.sql')
+    with open(req_path, mode='r') as file:
+        return ''.join(file.readlines())
+
+
+class SQLRequests:
+    build_squadrons_current_data = read_request('build_squadrons_current_data')
+    schema = read_request('schema')
+    squad_deleted = read_request('squad_deleted')
+    create_operation_id = read_request('create_operation_id')
+    insert_info = read_request('insert_info')
+    delete_squadron = read_request('delete_squadron')
+    insert_news = read_request('insert_news')

main.py

@@ -22,4 +22,29 @@ Workflow to update existing squadron:
 
     else:
         insert squad_id, operations_id to squadrons_deleted
-"""
+"""
+
+import FAPI
+import DB
+
+
+def update_squad(squad_id: int, suppress_absence=False) -> None:
+    squad_info = FAPI.Queries.get_squad_info(squad_id)
+    if squad_info is None:
+        # Squad not found FDEV
+        if not suppress_absence:
+            DB.delete_squadron(squad_id)
+
+    else:
+        # Then we got valid squad_info dict
+        news_info = FAPI.Queries.get_squad_news(squad_id)
+        print(DB.insert_info_news(news_info, squad_info))
+
+
+def main():
+    update_squad(2530)
+    update_squad(47999)
+
+
+if __name__ == '__main__':
+    main()

sql/create_operation_id.sql

@@ -0,0 +1 @@
+insert into operations_info (squad_id) values (:squad_id) RETURNING operation_id;

sql/delete_squadron.sql

@@ -0,0 +1 @@
+insert into squadrons_deleted (operation_id, squad_id) VALUES (:operation_id, :squad_id);

sql/insert_info.sql

@@ -0,0 +1,43 @@
+INSERT INTO squadrons_historical_data values (
+:operation_id,
+:name,
+:tag,
+:owner_id,
+:owner_name,
+:platform,
+:created,
+:created_ts,
+:accepting_new_members,
+:power_id,
+:power_name,
+:superpower_id,
+:superpower_name,
+:faction_id,
+:faction_name,
+:delete_after,
+:credits_balance,
+:credits_in,
+:credits_out,
+:user_tags,
+:member_count,
+:online_count,
+:pending_count,
+:full,
+:public_comms,
+:public_comms_override,
+:public_comms_available,
+:current_season_trade_score,
+:previous_season_trade_score,
+:current_season_combat_score,
+:previous_season_combat_score,
+:current_season_exploration_score,
+:previous_season_exploration_score,
+:current_season_cqc_score,
+:previous_season_cqc_score,
+:current_season_bgs_score,
+:previous_season_bgs_score,
+:current_season_powerplay_score,
+:previous_season_powerplay_score,
+:current_season_aegis_score,
+:previous_season_aegis_score
+);

sql/insert_news.sql

@@ -0,0 +1,22 @@
+insert into squadrons_news_historical (
+    operation_id,
+    type_of_news,
+    news_id,
+    "date",
+    category,
+    motd,
+    author,
+    cmdr_id,
+    user_id
+) values
+(
+    :operation_id,
+    :type_of_news,
+    :news_id,
+    :date,
+    :category,
+    :motd,
+    :author,
+    :cmdr_id,
+    :user_id
+);

sql/squad_deleted.sql

@@ -0,0 +1 @@
+select count(*) from squadrons_deleted where squad_id = :squad_id;