Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(redis): consolidate ID resolution and remove probabilistic refresh hack

Browse Source
This commit is contained in:
Scott Idem
2026-02-10 17:45:31 -05:00
parent 6d5633dc86
commit 17ae70992f
2 changed files with 23 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
app/lib_id_resolver.py
View File

@@ -1,130 +0,0 @@
"""
Centralized ID random to integer ID resolution.
"""
import logging
import datetime
import random
import redis
from app.config import settings
from app.db_connection import db
log = logging.getLogger(__name__)
def redis_lookup_id_random(
record_id_random: int|str,
table_name: str,
check_int_id: bool = False,
log_lvl: int = logging.WARNING,
minutes: int = 30,
reset_rate: int = 10,
) -> str|int|bool|None:
"""
Looks up a record ID in Redis, falling back to SQL if not found.
Resolves 'id_random' (URL-safe string) to internal integer 'id'.
"""
from app.db_sql import sql_select, get_id_random
log.setLevel(log_lvl)
if isinstance(record_id_random, str) and 11 <= len(record_id_random) <= 22:
pass
elif isinstance(record_id_random, int):
if check_int_id:
if get_id_random(record_id=record_id_random, table_name=table_name):
return record_id_random
return False
return record_id_random
elif record_id_random is None:
return None
else:
log.error(f'Unexpected data type: {type(record_id_random)}. Expected string (11-22 chars) or int.')
return False
if not table_name:
log.error(f'Missing table_name for id_random lookup: {record_id_random}')
return False
r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True)
key_name = f'{table_name}:{record_id_random}'
record_id = r.get(key_name)
# Periodic cache refresh
if record_id and random.randint(1, reset_rate) == 1:
log.warning(f'Redis: Randomly (1/{reset_rate}) refreshing cache for Key="{key_name}"')
record_id = None
if record_id:
r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
return int(record_id)
else:
data = { 'id_random': record_id_random }
sql = f"SELECT id FROM `{table_name}` WHERE id_random = :id_random;"
if select_results := sql_select(sql=sql, data=data):
if isinstance(select_results, dict):
if rid := select_results.get('id'):
r.setex(key_name, datetime.timedelta(minutes=minutes), value=rid)
return int(rid)
log.error('SQL result missing ID field.')
return False
else:
log.error(f'SQL: Duplicate id_random found in "{table_name}". Retrying...')
return redis_lookup_id_random(record_id_random=record_id_random, table_name=table_name)
else:
log.warning(f'SQL: ID Random "{record_id_random}" not found in "{table_name}".')
return None
def lookup_id_random_pop(
obj_data: dict,
log_lvl: int = logging.WARNING,
):
"""
Resolves any *_id_random fields in a dict to their integer IDs and removes the random keys.
"""
log.setLevel(log_lvl)
# List of common prefix patterns to resolve
id_patterns = [
'account', 'activity_log', 'address', 'archive', 'contact', 'cont_edu_cert',
'cont_edu_cert_person', 'event', 'event_abstract', 'event_badge',
'event_badge_template', 'event_exhibit', 'event_file', 'event_location',
'event_person', 'event_person_profile', 'event_presentation',
'event_presenter', 'event_registration', 'event_session', 'event_track',
'grant', 'hosted_file', 'journal', 'journal_entry', 'membership_group',
'membership_person_group', 'membership_person', 'membership_type',
'membership_person_type', 'order', 'order_line', 'order_cart',
'order_cart_line', 'organization', 'page', 'person', 'post', 'product',
'sponsorship', 'sponsorship_cfg', 'site', 'user'
]
for prefix in id_patterns:
key = f'{prefix}_id_random'
if key in obj_data:
table = prefix
if prefix == 'address_location': table = 'address'
if prefix in ['contact_1', 'contact_2']: table = 'contact'
if prefix == 'event_id_random_only': table = 'event'
resolved_id = redis_lookup_id_random(record_id_random=obj_data[key], table_name=table)
obj_data[f'{prefix}_id'] = resolved_id
obj_data.pop(key)
# Handle polymorphic link fields
polymorphic = [
('for_type', 'for_id_random', 'for_id'),
('link_to_type', 'link_to_id_random', 'link_to_id'),
('object_type', 'object_id_random', 'object_id'),
('to_object_type', 'to_object_id_random', 'to_object_id'),
('from_object_type', 'from_object_id_random', 'from_object_id')
]
for type_key, rand_key, id_key in polymorphic:
if type_key in obj_data and rand_key in obj_data:
obj_data[id_key] = redis_lookup_id_random(
record_id_random=obj_data[rand_key],
table_name=obj_data[type_key]
)
obj_data.pop(rand_key)
return obj_data

33
app/lib_redis_helpers.py
View File

@@ -10,13 +10,23 @@ from app.config import settings
log = logging.getLogger(__name__)
# --- Global Redis Client ---
# Using a single client instance with internal connection pooling is more efficient.
redis_client = redis.Redis(
host=settings.REDIS['server'],
port=settings.REDIS['port'],
db=7,
password=None,
decode_responses=True
)
def redis_lookup_id_random(
record_id_random: int|str,
table_name: str,
check_int_id: bool = False,
log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
minutes: int = 30, # Expire the Redis key after 8 minutes
reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key
minutes: int = 30, # Expire the Redis key after 30 minutes
reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key (DEPRECATED)
) -> str|int|bool|None:
"""
Looks up a record ID in Redis, falling back to SQL if not found.
@@ -66,18 +76,21 @@ def redis_lookup_id_random(
log.error('Missing table_name and record_id_random')
return False
r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True)
key_name = f'{table_name}:{record_id_random}'
record_id = r.get(key_name)
# Use the global redis client instead of creating a new one every time
record_id = redis_client.get(key_name)
if record_id and random.randint(1, reset_rate) == 1:
log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
record_id = None
# ### SECTION ### THE "RESET RATE" WORKAROUND (DEPRECATED) ###
# This was used to force a SQL lookup occasionally to correct stale data.
# We are disabling this for now to see if the recent logic improvements fixed the root cause.
# if record_id and random.randint(1, reset_rate) == 1:
# log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={redis_client.ttl(key_name)} seconds')
# record_id = None
if record_id:
r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds')
redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={redis_client.ttl(key_name)} seconds')
return int(record_id)
elif table_name:
data = { 'id_random': record_id_random }
@@ -88,7 +101,7 @@ def redis_lookup_id_random(
if isinstance(select_results, dict):
log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""")
if record_id := select_results.get('id'):
r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id)
return int(record_id)
else:
log.error('The SQL result was not what was expected. The ID field was not found.')