Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trying to make the SQL connections more reliable with threads.

Browse Source
This commit is contained in:
Scott Idem
2024-10-08 18:45:53 -04:00
parent 50955aff3a
commit 30c39d58dd
1 changed files with 30 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
app/db_sql.py
View File

@@ -7,6 +7,7 @@ from app.log import log, logging, logger_reset
from sqlalchemy import create_engine, text, Time
from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError
from sqlalchemy.pool import NullPool
# from multiprocessing import Pool
db_uri = settings.SQLALCHEMY_DB_URI
@@ -18,12 +19,12 @@ db_uri = settings.SQLALCHEMY_DB_URI
connection_string = db_uri
# engine = create_engine(url=connection_string, poolclass=NullPool, echo=False, isolation_level='READ COMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) # 2024-10-08
engine = create_engine(
url=connection_string,
echo=False,
pool_use_lifo=True,
pool_pre_ping=True,
isolation_level='READ COMMITTED',
connect_args={'connect_timeout': settings.DB['connect_timeout']}
url = connection_string,
echo = False,
pool_use_lifo = True,
pool_pre_ping = True,
isolation_level = 'READ COMMITTED',
connect_args = {'connect_timeout': settings.DB['connect_timeout']}
)
# engine = create_engine(url=connection_string, pool_size=5, max_overflow=15, timeout=settings.DB['connect_timeout'], pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']})
# NOTE: The default isolation_level is 'REPEATABLE READ'. This can sometimes not show updated data.
@@ -31,6 +32,22 @@ engine = create_engine(
# NOTE: Switching from READ COMMITTED to READ UNCOMMITTED (or REPEATABLE READ). Testing started 2024-04-23
# levels: "REPEATABLE READ" "READ COMMITTED" "READ UNCOMMITTED" "SERIALIZABLE"
# def run_in_process(some_data_record):
# with engine.connect() as conn:
# conn.execute(text("..."))
# def initializer():
# """ensure the parent proc's database connections are not touched
# in the new connection pool"""
# engine.dispose(close=False)
# with Pool(10, initializer=initializer) as p:
# p.map(run_in_process, data)
log.info('DB SQL trying to connect...')
db = None
try:
@@ -74,12 +91,12 @@ def sql_connect(
# new_engine = create_engine(url=settings.SQLALCHEMY_DB_URI, poolclass=NullPool, echo=False, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']})
new_engine = create_engine(
url=settings.SQLALCHEMY_DB_URI,
echo=False,
pool_use_lifo=True,
pool_pre_ping=True,
isolation_level='READ COMMITTED',
connect_args={'connect_timeout': settings.DB['connect_timeout']}
url = settings.SQLALCHEMY_DB_URI,
echo = False,
pool_use_lifo = True,
pool_pre_ping = True,
isolation_level = 'READ COMMITTED',
connect_args = {'connect_timeout': settings.DB['connect_timeout']}
)
# new_engine = create_engine(url=settings.SQLALCHEMY_DB_URI, pool_size=5, max_overflow=15, timeout=settings.DB['connect_timeout'], pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']})
# new_engine = create_engine(url=settings.SQLALCHEMY_DB_URI, pool_size=5, max_overflow=15, pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']})
@@ -1268,7 +1285,7 @@ def redis_lookup_id_random(
table_name: str,
check_int_id: bool = False,
log_lvl: int = logging.INFO, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
minutes: int = 8, # Expire the Redis key after 8 minutes
minutes: int = 30, # Expire the Redis key after 8 minutes
reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key
) -> str|int|bool|None:
log.setLevel(log_lvl)