Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(redis): implement bidirectional ID caching and extensive E2E benchmarks

Browse Source
This commit is contained in:
Scott Idem
2026-02-10 18:08:04 -05:00
parent 17ae70992f
commit 3e6ea108cf
3 changed files with 124 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
app/lib_redis_helpers.py
View File

@@ -77,17 +77,11 @@ def redis_lookup_id_random(
return False
key_name = f'{table_name}:{record_id_random}'
rev_key_prefix = f'rev:{table_name}:'
# Use the global redis client instead of creating a new one every time
record_id = redis_client.get(key_name)
# ### SECTION ### THE "RESET RATE" WORKAROUND (DEPRECATED) ###
# This was used to force a SQL lookup occasionally to correct stale data.
# We are disabling this for now to see if the recent logic improvements fixed the root cause.
# if record_id and random.randint(1, reset_rate) == 1:
# log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={redis_client.ttl(key_name)} seconds')
# record_id = None
if record_id:
redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={redis_client.ttl(key_name)} seconds')
@@ -101,7 +95,9 @@ def redis_lookup_id_random(
if isinstance(select_results, dict):
log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""")
if record_id := select_results.get('id'):
# Populating BOTH directions in Redis
redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
redis_client.setex(f'{rev_key_prefix}{record_id}', datetime.timedelta(minutes=minutes), value=record_id_random)
return int(record_id)
else:
log.error('The SQL result was not what was expected. The ID field was not found.')
@@ -121,9 +117,11 @@ def get_id_random(
record_id: int,
table_name: str,
log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
minutes: int = 30, # Expire the Redis key after 30 minutes
) -> str|bool|None:
"""
Looks up the 'id_random' for a given internal integer ID.
Uses Redis caching for performance.
"""
from app.db_sql import sql_select, get_last_sql_error
log.setLevel(log_lvl)
@@ -132,6 +130,13 @@ def get_id_random(
if not table_name or table_name.startswith('lu_') or table_name.startswith('v_lu_'):
return None
# Check Redis cache first (using 'rev:' prefix for integer -> string mappings)
key_name = f'rev:{table_name}:{record_id}'
if cached_val := redis_client.get(key_name):
# Extend TTL on hit
redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=cached_val)
return str(cached_val)
data = { 'id': record_id }
sql = f"SELECT id_random FROM `{table_name}` AS `table` WHERE `table`.id = :id;"
@@ -148,6 +153,9 @@ def get_id_random(
if select_results:
if isinstance(select_results, dict):
if record_id_random := select_results.get('id_random'):
# Populating BOTH directions in Redis
redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id_random)
redis_client.setex(f'{table_name}:{record_id_random}', datetime.timedelta(minutes=minutes), value=record_id)
return str(record_id_random)
else:
log.error('The SQL result was not what was expected.')