ED/SquadsActivityMonitor
1
0
Fork 0
mirror of https://github.com/norohind/SquadsActivityMonitor.git synced 2025-04-12 13:00:02 +03:00
Code Issues Projects Releases Wiki Activity

Add continues pull from sqlite to postgres

Browse Source
This commit is contained in:
norohind 2021-11-28 14:43:52 +03:00
parent 145cb8a19b
commit 803b6cce7c
Signed by: norohind
GPG Key ID: 01C3BECC26FB59E1
1 changed files with 44 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
sqlite2postgres.py
View File

@ -30,21 +30,52 @@ with pg_conn:
with pg_conn.cursor() as cursor:
cursor.execute(postgres_sql_requests.schema_create)
with pg_conn:
with pg_conn.cursor() as cursor:
start = time.time()
insert_statement = ''
sqlite_content = sqlite_conn.execute('select * from squads_stats_states;').fetchall()
# for row in sqlite_content:
# # cursor.execute(insert_pg, row)
# insert_statement += cursor.mogrify(insert_pg, row).decode('utf-8')
def initial_pull_sqlite_to_postgres() -> None:
"""
Function for initial transferring data from sqlite to postgres
:return:
"""
with pg_conn:
with pg_conn.cursor() as cursor:
start = time.time()
insert_statement = ''
sqlite_content = sqlite_conn.execute('select * from squads_stats_states;').fetchall()
# for row in sqlite_content:
# # cursor.execute(insert_pg, row)
# insert_statement += cursor.mogrify(insert_pg, row).decode('utf-8')
# print((time.time() - start) * 100, 'ms for creating insert_statement')
# print('inserting')
cursor.executemany(insert_pg, sqlite_content)
# 149619.26856040955 ms total
print((time.time() - start) * 100, 'ms total')
def continues_pull_sqlite_to_postgres() -> None:
"""
Function to synchronize data from sqlite to postgres table
:return:
"""
with pg_conn:
with pg_conn.cursor() as cursor:
cursor: psycopg2.extensions.cursor
cursor.execute('select action_id from squads_stats_states order by action_id desc limit 1;')
last_action_id_postgres: int = cursor.fetchone()[0]
new_records = sqlite_conn.execute('select * from squads_stats_states where action_id > :action_id;',
{'action_id': last_action_id_postgres}).fetchall()
cursor.executemany(insert_pg, new_records)
continues_pull_sqlite_to_postgres()
# print((time.time() - start) * 100, 'ms for creating insert_statement')
# print('inserting')
cursor.executemany(insert_pg, sqlite_content)
# 149619.26856040955 ms total
print((time.time() - start) * 100, 'ms total')
pg_conn.close()
sqlite_conn.close()