diff --git a/app/lib_id_resolver.py b/app/lib_id_resolver.py deleted file mode 100644 index 9933c0b..0000000 --- a/app/lib_id_resolver.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -Centralized ID random to integer ID resolution. -""" -import logging -import datetime -import random -import redis -from app.config import settings -from app.db_connection import db - -log = logging.getLogger(__name__) - -def redis_lookup_id_random( - record_id_random: int|str, - table_name: str, - check_int_id: bool = False, - log_lvl: int = logging.WARNING, - minutes: int = 30, - reset_rate: int = 10, - ) -> str|int|bool|None: - """ - Looks up a record ID in Redis, falling back to SQL if not found. - Resolves 'id_random' (URL-safe string) to internal integer 'id'. - """ - from app.db_sql import sql_select, get_id_random - - log.setLevel(log_lvl) - - if isinstance(record_id_random, str) and 11 <= len(record_id_random) <= 22: - pass - elif isinstance(record_id_random, int): - if check_int_id: - if get_id_random(record_id=record_id_random, table_name=table_name): - return record_id_random - return False - return record_id_random - elif record_id_random is None: - return None - else: - log.error(f'Unexpected data type: {type(record_id_random)}. Expected string (11-22 chars) or int.') - return False - - if not table_name: - log.error(f'Missing table_name for id_random lookup: {record_id_random}') - return False - - r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True) - key_name = f'{table_name}:{record_id_random}' - - record_id = r.get(key_name) - - # Periodic cache refresh - if record_id and random.randint(1, reset_rate) == 1: - log.warning(f'Redis: Randomly (1/{reset_rate}) refreshing cache for Key="{key_name}"') - record_id = None - - if record_id: - r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) - return int(record_id) - else: - data = { 'id_random': record_id_random } - sql = f"SELECT id FROM `{table_name}` WHERE id_random = :id_random;" - - if select_results := sql_select(sql=sql, data=data): - if isinstance(select_results, dict): - if rid := select_results.get('id'): - r.setex(key_name, datetime.timedelta(minutes=minutes), value=rid) - return int(rid) - log.error('SQL result missing ID field.') - return False - else: - log.error(f'SQL: Duplicate id_random found in "{table_name}". Retrying...') - return redis_lookup_id_random(record_id_random=record_id_random, table_name=table_name) - else: - log.warning(f'SQL: ID Random "{record_id_random}" not found in "{table_name}".') - return None - -def lookup_id_random_pop( - obj_data: dict, - log_lvl: int = logging.WARNING, - ): - """ - Resolves any *_id_random fields in a dict to their integer IDs and removes the random keys. - """ - log.setLevel(log_lvl) - - # List of common prefix patterns to resolve - id_patterns = [ - 'account', 'activity_log', 'address', 'archive', 'contact', 'cont_edu_cert', - 'cont_edu_cert_person', 'event', 'event_abstract', 'event_badge', - 'event_badge_template', 'event_exhibit', 'event_file', 'event_location', - 'event_person', 'event_person_profile', 'event_presentation', - 'event_presenter', 'event_registration', 'event_session', 'event_track', - 'grant', 'hosted_file', 'journal', 'journal_entry', 'membership_group', - 'membership_person_group', 'membership_person', 'membership_type', - 'membership_person_type', 'order', 'order_line', 'order_cart', - 'order_cart_line', 'organization', 'page', 'person', 'post', 'product', - 'sponsorship', 'sponsorship_cfg', 'site', 'user' - ] - - for prefix in id_patterns: - key = f'{prefix}_id_random' - if key in obj_data: - table = prefix - if prefix == 'address_location': table = 'address' - if prefix in ['contact_1', 'contact_2']: table = 'contact' - if prefix == 'event_id_random_only': table = 'event' - - resolved_id = redis_lookup_id_random(record_id_random=obj_data[key], table_name=table) - obj_data[f'{prefix}_id'] = resolved_id - obj_data.pop(key) - - # Handle polymorphic link fields - polymorphic = [ - ('for_type', 'for_id_random', 'for_id'), - ('link_to_type', 'link_to_id_random', 'link_to_id'), - ('object_type', 'object_id_random', 'object_id'), - ('to_object_type', 'to_object_id_random', 'to_object_id'), - ('from_object_type', 'from_object_id_random', 'from_object_id') - ] - - for type_key, rand_key, id_key in polymorphic: - if type_key in obj_data and rand_key in obj_data: - obj_data[id_key] = redis_lookup_id_random( - record_id_random=obj_data[rand_key], - table_name=obj_data[type_key] - ) - obj_data.pop(rand_key) - - return obj_data \ No newline at end of file diff --git a/app/lib_redis_helpers.py b/app/lib_redis_helpers.py index ed6eb64..8542540 100644 --- a/app/lib_redis_helpers.py +++ b/app/lib_redis_helpers.py @@ -10,13 +10,23 @@ from app.config import settings log = logging.getLogger(__name__) +# --- Global Redis Client --- +# Using a single client instance with internal connection pooling is more efficient. +redis_client = redis.Redis( + host=settings.REDIS['server'], + port=settings.REDIS['port'], + db=7, + password=None, + decode_responses=True +) + def redis_lookup_id_random( record_id_random: int|str, table_name: str, check_int_id: bool = False, log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - minutes: int = 30, # Expire the Redis key after 8 minutes - reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key + minutes: int = 30, # Expire the Redis key after 30 minutes + reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key (DEPRECATED) ) -> str|int|bool|None: """ Looks up a record ID in Redis, falling back to SQL if not found. @@ -66,18 +76,21 @@ def redis_lookup_id_random( log.error('Missing table_name and record_id_random') return False - r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True) key_name = f'{table_name}:{record_id_random}' - record_id = r.get(key_name) + # Use the global redis client instead of creating a new one every time + record_id = redis_client.get(key_name) - if record_id and random.randint(1, reset_rate) == 1: - log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') - record_id = None + # ### SECTION ### THE "RESET RATE" WORKAROUND (DEPRECATED) ### + # This was used to force a SQL lookup occasionally to correct stale data. + # We are disabling this for now to see if the recent logic improvements fixed the root cause. + # if record_id and random.randint(1, reset_rate) == 1: + # log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={redis_client.ttl(key_name)} seconds') + # record_id = None if record_id: - r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) - log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') + redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) + log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={redis_client.ttl(key_name)} seconds') return int(record_id) elif table_name: data = { 'id_random': record_id_random } @@ -88,7 +101,7 @@ def redis_lookup_id_random( if isinstance(select_results, dict): log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""") if record_id := select_results.get('id'): - r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) + redis_client.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) return int(record_id) else: log.error('The SQL result was not what was expected. The ID field was not found.')