Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(sql): modularize Redis and ID lookup functions

Browse Source
This commit is contained in:
Scott Idem
2026-01-06 17:32:22 -05:00
parent a6a5162385
commit 56fe7ed953
2 changed files with 153 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
app/db_sql.py
View File

@@ -23,6 +23,12 @@ from app.lib_sql_search import (
sql_where_qry_part as _sql_where_qry_part
)
from app.lib_redis_helpers import (
redis_lookup_id_random as _redis_lookup_id_random,
get_id_random as _get_id_random,
reset_redis as _reset_redis
)
db_uri = settings.SQLALCHEMY_DB_URI
@@ -1342,115 +1348,7 @@ def redis_lookup_id_random(
reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key
) -> str|int|bool|None:
log.setLevel(log_lvl)
if isinstance(record_id_random, str) and len(record_id_random) >= 11 and len(record_id_random) <= 22: pass
elif isinstance(record_id_random, int):
record_id = record_id_random
if check_int_id:
log.info(f'Checking the int ID if exists. Table Name: {table_name} ID: {record_id}')
if get_id_random_result := get_id_random(
record_id = record_id,
table_name = table_name,
):
log.info(f'The int ID exists. Returning the int ID. ID Random: {get_id_random_result}')
return record_id
else:
log.info(f'The int ID does not exists. Returning False. Table Name: {table_name} ID: {record_id}')
return False
else:
log.debug(f'Not checking if the int ID exists. Returning the int ID. ID: {record_id}')
return record_id
elif record_id_random is None:
log.info(f'No record ID was passed. Returning None')
return None
else:
log.error(f'Unexpected data type or string format: {type(record_id_random)} Expected type is a string 11 or 22 characters long.')
return False
if record_id_random and table_name:
# WARNING: The record_id_random string length should be checked just in case?
if len(record_id_random) < 11:
log.error(f'The length of id_random is too short: {record_id_random} ({len(record_id_random)} chars)')
return False
elif len(record_id_random) > 22:
log.error(f'The length of id_random is too long: {record_id_random} ({len(record_id_random)} chars)')
return False
else:
pass
elif record_id_random:
log.error(f'Missing table_name to select from for id_random "{record_id_random}"')
return False
elif table_name:
log.error(f'Missing id_random to select from table "{table_name}"')
return False
else:
log.error('Missing table_name and record_id_random')
return False
# r = redis.Redis(host='localhost', port=6379, db=7, password=None, decode_responses=True)
r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True)
# key_name = 'record_id:'+record_id_random
key_name = f'{table_name}:{record_id_random}'
record_id = r.get(key_name)
# log.debug(f'Record ID found: {record_id}')
# WARNING WARNING WARNING
# WARNING WARNING WARNING
# WARNING WARNING WARNING
# log.info(f'Looking up ID in Redis is being partially bypassed. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
# record_id = None # Uncomment this line to bypass the Redis lookup... trust?
# WARNING WARNING WARNING
# WARNING WARNING WARNING
# WARNING WARNING WARNING
# To help prevent corrupt data from being stored in Redis for too long, we will set the record_id to None 25% of the time. This will force the function to look up the record_id in the database. This is a temporary solution. Why is the data from SQL not correct in the first place??? -2024-10-08
if record_id and random.randint(1, reset_rate) == 1:
log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
record_id = None
if record_id:
r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
return int(record_id)
elif table_name:
data = { 'id_random': record_id_random }
sql = f"""
SELECT id
FROM `{table_name}` AS `table`
WHERE `table`.id_random = :id_random;
"""
if select_results := sql_select(sql=sql, data=data):
log.debug(f'SQL: SELECT result: {select_results}')
# log.debug(type(select_results))
if isinstance(select_results, dict):
log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""")
if record_id := select_results.get('id'):
r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
return int(record_id)
else:
log.error('The SQL result was not what was expected. The ID field was not found.')
return False
else:
log.error(f'SQL: More than one record may have been found in the table "{table_name}". There may be a duplicate id_random value in this table. This should not happen!')
log.error(select_results)
# Try again...
log.warning(f'SQL: ID Random "{record_id_random}" was not found in table "{table_name}". Trying again...')
new_result = redis_lookup_id_random(record_id_random=record_id_random, table_name=table_name)
return new_result
# return False
else:
log.warning(f'SQL: ID Random "{record_id_random}" was not found in table "{table_name}". Returning None.')
return None
log.error('We should not be here. Something unexpected happened.')
return False
return _redis_lookup_id_random(record_id_random, table_name, check_int_id, log_lvl, minutes, reset_rate)
# ### END ### API DB SQL ### redis_lookup_id_random() ###
@@ -1463,48 +1361,13 @@ def get_id_random(
log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
) -> str|bool|None:
log.setLevel(log_lvl)
data = { 'id': record_id }
sql = f"""
SELECT id_random
FROM `{table_name}` AS `table`
WHERE `table`.id = :id;
"""
if select_results := sql_select(sql=sql, data=data):
log.debug(select_results)
log.debug(type(select_results))
if isinstance(select_results, dict):
# log.info(f"""Record ID found: {select_results['id_random']}""") # DOES UNCOMMENTING THIS BREAK STUFF???
if record_id_random := select_results.get('id_random'):
return str(record_id_random)
else:
log.error('The SQL result was not what was expected.')
return False
elif isinstance(select_results, list):
log.exception('More than one record may have been found. There may be a duplicate id. This should never happen.')
log.error(select_results)
return False
else:
log.exception(f'Got an unexpected result while trying to look up the ID. Is the table name correct? {table_name} Is the record ID valid? {record_id}')
log.error(select_results)
return False
elif select_results is None:
log.warning(f'No results with: Table Name: {table_name} ID: {record_id}')
log.debug(select_results)
return None
else: # False or something else not True
log.error(f'Something went wrong while trying to look up the ID. Is the table name correct? {table_name} Is the record ID valid? {record_id}')
log.error(select_results)
return False
return _get_id_random(record_id, table_name, log_lvl)
# ### END ### API DB SQL ### get_id_random() ###
@logger_reset
def reset_redis():
r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True)
r.flushdb()
return True
return _reset_redis()
# ### BEGIN ### API DB SQL ### lookup_id_random_pop() ###

144
app/lib_redis_helpers.py Normal file
View File

@@ -0,0 +1,144 @@
"""
Redis-based ID resolution and caching helpers for Aether.
"""
import datetime
import random
import redis
import logging
from app.config import settings
log = logging.getLogger(__name__)
def redis_lookup_id_random(
record_id_random: int|str,
table_name: str,
check_int_id: bool = False,
log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
minutes: int = 30, # Expire the Redis key after 8 minutes
reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key
) -> str|int|bool|None:
"""
Looks up a record ID in Redis, falling back to SQL if not found.
Resolves 'id_random' (URL-safe string) to internal integer 'id'.
"""
from app.db_sql import sql_select, get_id_random
log.setLevel(log_lvl)
if isinstance(record_id_random, str) and len(record_id_random) >= 11 and len(record_id_random) <= 22: pass
elif isinstance(record_id_random, int):
record_id = record_id_random
if check_int_id:
log.info(f'Checking the int ID if exists. Table Name: {table_name} ID: {record_id}')
if get_id_random_result := get_id_random(
record_id = record_id,
table_name = table_name,
):
log.info(f'The int ID exists. Returning the int ID. ID Random: {get_id_random_result}')
return record_id
else:
log.info(f'The int ID does not exists. Returning False. Table Name: {table_name} ID: {record_id}')
return False
else:
log.debug(f'Not checking if the int ID exists. Returning the int ID. ID: {record_id}')
return record_id
elif record_id_random is None:
log.info(f'No record ID was passed. Returning None')
return None
else:
log.error(f'Unexpected data type or string format: {type(record_id_random)} Expected type is a string 11 or 22 characters long.')
return False
if record_id_random and table_name:
if len(record_id_random) < 11:
log.error(f'The length of id_random is too short: {record_id_random} ({len(record_id_random)} chars)')
return False
elif len(record_id_random) > 22:
log.error(f'The length of id_random is too long: {record_id_random} ({len(record_id_random)} chars)')
return False
elif record_id_random:
log.error(f'Missing table_name to select from for id_random "{record_id_random}"')
return False
elif table_name:
log.error(f'Missing id_random to select from table "{table_name}"')
return False
else:
log.error('Missing table_name and record_id_random')
return False
r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True)
key_name = f'{table_name}:{record_id_random}'
record_id = r.get(key_name)
if record_id and random.randint(1, reset_rate) == 1:
log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
record_id = None
if record_id:
r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
return int(record_id)
elif table_name:
data = { 'id_random': record_id_random }
sql = f"SELECT id FROM `{table_name}` AS `table` WHERE `table`.id_random = :id_random;"
if select_results := sql_select(sql=sql, data=data):
log.debug(f'SQL: SELECT result: {select_results}')
if isinstance(select_results, dict):
log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""")
if rid := select_results.get('id'):
r.setex(key_name, datetime.timedelta(minutes=minutes), value=rid)
return int(rid)
else:
log.error('The SQL result was not what was expected. The ID field was not found.')
return False
else:
log.error(f'SQL: More than one record found in "{table_name}". Duplicate id_random!')
return redis_lookup_id_random(record_id_random=record_id_random, table_name=table_name)
else:
log.warning(f'SQL: ID Random "{record_id_random}" not found in "{table_name}". Returning None.')
return None
log.error('Unexpected state in redis_lookup_id_random.')
return False
def get_id_random(
record_id: int,
table_name: str,
log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
) -> str|bool|None:
"""
Looks up the 'id_random' for a given internal integer ID.
"""
from app.db_sql import sql_select
log.setLevel(log_lvl)
data = { 'id': record_id }
sql = f"SELECT id_random FROM `{table_name}` AS `table` WHERE `table`.id = :id;"
if select_results := sql_select(sql=sql, data=data):
if isinstance(select_results, dict):
if record_id_random := select_results.get('id_random'):
return str(record_id_random)
else:
log.error('The SQL result was not what was expected.')
return False
elif isinstance(select_results, list):
log.exception('More than one record may have been found. Duplicate ID!')
return False
else:
log.exception(f'Got an unexpected result while trying to look up the ID.')
return False
elif select_results is None:
return None
else:
return False
def reset_redis():
"""Flushes the Redis database used for ID caching."""
r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True)
r.flushdb()
return True