diff --git a/app/db_sql.py b/app/db_sql.py index 4b9a09f..a6d66e3 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -23,6 +23,12 @@ from app.lib_sql_search import ( sql_where_qry_part as _sql_where_qry_part ) +from app.lib_redis_helpers import ( + redis_lookup_id_random as _redis_lookup_id_random, + get_id_random as _get_id_random, + reset_redis as _reset_redis +) + db_uri = settings.SQLALCHEMY_DB_URI @@ -1342,115 +1348,7 @@ def redis_lookup_id_random( reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key ) -> str|int|bool|None: log.setLevel(log_lvl) - - if isinstance(record_id_random, str) and len(record_id_random) >= 11 and len(record_id_random) <= 22: pass - elif isinstance(record_id_random, int): - record_id = record_id_random - if check_int_id: - log.info(f'Checking the int ID if exists. Table Name: {table_name} ID: {record_id}') - if get_id_random_result := get_id_random( - record_id = record_id, - table_name = table_name, - ): - log.info(f'The int ID exists. Returning the int ID. ID Random: {get_id_random_result}') - return record_id - else: - log.info(f'The int ID does not exists. Returning False. Table Name: {table_name} ID: {record_id}') - return False - else: - log.debug(f'Not checking if the int ID exists. Returning the int ID. ID: {record_id}') - return record_id - elif record_id_random is None: - log.info(f'No record ID was passed. Returning None') - return None - else: - log.error(f'Unexpected data type or string format: {type(record_id_random)} Expected type is a string 11 or 22 characters long.') - return False - - if record_id_random and table_name: - # WARNING: The record_id_random string length should be checked just in case? - if len(record_id_random) < 11: - log.error(f'The length of id_random is too short: {record_id_random} ({len(record_id_random)} chars)') - return False - elif len(record_id_random) > 22: - log.error(f'The length of id_random is too long: {record_id_random} ({len(record_id_random)} chars)') - return False - else: - pass - elif record_id_random: - log.error(f'Missing table_name to select from for id_random "{record_id_random}"') - return False - elif table_name: - log.error(f'Missing id_random to select from table "{table_name}"') - return False - else: - log.error('Missing table_name and record_id_random') - return False - - # r = redis.Redis(host='localhost', port=6379, db=7, password=None, decode_responses=True) - r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True) - - # key_name = 'record_id:'+record_id_random - key_name = f'{table_name}:{record_id_random}' - - record_id = r.get(key_name) - # log.debug(f'Record ID found: {record_id}') - - # WARNING WARNING WARNING - # WARNING WARNING WARNING - # WARNING WARNING WARNING - # log.info(f'Looking up ID in Redis is being partially bypassed. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') - # record_id = None # Uncomment this line to bypass the Redis lookup... trust? - # WARNING WARNING WARNING - # WARNING WARNING WARNING - # WARNING WARNING WARNING - - # To help prevent corrupt data from being stored in Redis for too long, we will set the record_id to None 25% of the time. This will force the function to look up the record_id in the database. This is a temporary solution. Why is the data from SQL not correct in the first place??? -2024-10-08 - if record_id and random.randint(1, reset_rate) == 1: - log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') - record_id = None - - if record_id: - r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) - log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') - - return int(record_id) - elif table_name: - data = { 'id_random': record_id_random } - sql = f""" - SELECT id - FROM `{table_name}` AS `table` - WHERE `table`.id_random = :id_random; - """ - - if select_results := sql_select(sql=sql, data=data): - log.debug(f'SQL: SELECT result: {select_results}') - # log.debug(type(select_results)) - if isinstance(select_results, dict): - log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""") - if record_id := select_results.get('id'): - r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) - - return int(record_id) - else: - log.error('The SQL result was not what was expected. The ID field was not found.') - return False - else: - log.error(f'SQL: More than one record may have been found in the table "{table_name}". There may be a duplicate id_random value in this table. This should not happen!') - log.error(select_results) - - # Try again... - log.warning(f'SQL: ID Random "{record_id_random}" was not found in table "{table_name}". Trying again...') - new_result = redis_lookup_id_random(record_id_random=record_id_random, table_name=table_name) - return new_result - - # return False - else: - log.warning(f'SQL: ID Random "{record_id_random}" was not found in table "{table_name}". Returning None.') - return None - - log.error('We should not be here. Something unexpected happened.') - return False + return _redis_lookup_id_random(record_id_random, table_name, check_int_id, log_lvl, minutes, reset_rate) # ### END ### API DB SQL ### redis_lookup_id_random() ### @@ -1463,48 +1361,13 @@ def get_id_random( log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL ) -> str|bool|None: log.setLevel(log_lvl) - - data = { 'id': record_id } - sql = f""" - SELECT id_random - FROM `{table_name}` AS `table` - WHERE `table`.id = :id; - """ - - if select_results := sql_select(sql=sql, data=data): - log.debug(select_results) - log.debug(type(select_results)) - if isinstance(select_results, dict): - # log.info(f"""Record ID found: {select_results['id_random']}""") # DOES UNCOMMENTING THIS BREAK STUFF??? - if record_id_random := select_results.get('id_random'): - return str(record_id_random) - else: - log.error('The SQL result was not what was expected.') - return False - elif isinstance(select_results, list): - log.exception('More than one record may have been found. There may be a duplicate id. This should never happen.') - log.error(select_results) - return False - else: - log.exception(f'Got an unexpected result while trying to look up the ID. Is the table name correct? {table_name} Is the record ID valid? {record_id}') - log.error(select_results) - return False - elif select_results is None: - log.warning(f'No results with: Table Name: {table_name} ID: {record_id}') - log.debug(select_results) - return None - else: # False or something else not True - log.error(f'Something went wrong while trying to look up the ID. Is the table name correct? {table_name} Is the record ID valid? {record_id}') - log.error(select_results) - return False + return _get_id_random(record_id, table_name, log_lvl) # ### END ### API DB SQL ### get_id_random() ### @logger_reset def reset_redis(): - r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True) - r.flushdb() - return True + return _reset_redis() # ### BEGIN ### API DB SQL ### lookup_id_random_pop() ### diff --git a/app/lib_redis_helpers.py b/app/lib_redis_helpers.py new file mode 100644 index 0000000..e8828b4 --- /dev/null +++ b/app/lib_redis_helpers.py @@ -0,0 +1,144 @@ +""" +Redis-based ID resolution and caching helpers for Aether. +""" +import datetime +import random +import redis +import logging + +from app.config import settings + +log = logging.getLogger(__name__) + +def redis_lookup_id_random( + record_id_random: int|str, + table_name: str, + check_int_id: bool = False, + log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + minutes: int = 30, # Expire the Redis key after 8 minutes + reset_rate: int = 10, # 1 in 10 chance of resetting the Redis key + ) -> str|int|bool|None: + """ + Looks up a record ID in Redis, falling back to SQL if not found. + Resolves 'id_random' (URL-safe string) to internal integer 'id'. + """ + from app.db_sql import sql_select, get_id_random + log.setLevel(log_lvl) + + if isinstance(record_id_random, str) and len(record_id_random) >= 11 and len(record_id_random) <= 22: pass + elif isinstance(record_id_random, int): + record_id = record_id_random + if check_int_id: + log.info(f'Checking the int ID if exists. Table Name: {table_name} ID: {record_id}') + if get_id_random_result := get_id_random( + record_id = record_id, + table_name = table_name, + ): + log.info(f'The int ID exists. Returning the int ID. ID Random: {get_id_random_result}') + return record_id + else: + log.info(f'The int ID does not exists. Returning False. Table Name: {table_name} ID: {record_id}') + return False + else: + log.debug(f'Not checking if the int ID exists. Returning the int ID. ID: {record_id}') + return record_id + elif record_id_random is None: + log.info(f'No record ID was passed. Returning None') + return None + else: + log.error(f'Unexpected data type or string format: {type(record_id_random)} Expected type is a string 11 or 22 characters long.') + return False + + if record_id_random and table_name: + if len(record_id_random) < 11: + log.error(f'The length of id_random is too short: {record_id_random} ({len(record_id_random)} chars)') + return False + elif len(record_id_random) > 22: + log.error(f'The length of id_random is too long: {record_id_random} ({len(record_id_random)} chars)') + return False + elif record_id_random: + log.error(f'Missing table_name to select from for id_random "{record_id_random}"') + return False + elif table_name: + log.error(f'Missing id_random to select from table "{table_name}"') + return False + else: + log.error('Missing table_name and record_id_random') + return False + + r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True) + key_name = f'{table_name}:{record_id_random}' + + record_id = r.get(key_name) + + if record_id and random.randint(1, reset_rate) == 1: + log.warning(f'Redis: Randomly (1/{reset_rate}) setting record_id to None. Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') + record_id = None + + if record_id: + r.setex(key_name, datetime.timedelta(minutes=minutes), value=record_id) + log.info(f'Redis: Entry found for: Key="{key_name}" value="{record_id}" TTL={r.ttl(key_name)} seconds') + return int(record_id) + elif table_name: + data = { 'id_random': record_id_random } + sql = f"SELECT id FROM `{table_name}` AS `table` WHERE `table`.id_random = :id_random;" + + if select_results := sql_select(sql=sql, data=data): + log.debug(f'SQL: SELECT result: {select_results}') + if isinstance(select_results, dict): + log.info(f"""SQL: Found ID Random for: {str(record_id_random)} = {str(select_results.get('id'))}""") + if rid := select_results.get('id'): + r.setex(key_name, datetime.timedelta(minutes=minutes), value=rid) + return int(rid) + else: + log.error('The SQL result was not what was expected. The ID field was not found.') + return False + else: + log.error(f'SQL: More than one record found in "{table_name}". Duplicate id_random!') + return redis_lookup_id_random(record_id_random=record_id_random, table_name=table_name) + else: + log.warning(f'SQL: ID Random "{record_id_random}" not found in "{table_name}". Returning None.') + return None + + log.error('Unexpected state in redis_lookup_id_random.') + return False + + +def get_id_random( + record_id: int, + table_name: str, + log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + ) -> str|bool|None: + """ + Looks up the 'id_random' for a given internal integer ID. + """ + from app.db_sql import sql_select + log.setLevel(log_lvl) + + data = { 'id': record_id } + sql = f"SELECT id_random FROM `{table_name}` AS `table` WHERE `table`.id = :id;" + + if select_results := sql_select(sql=sql, data=data): + if isinstance(select_results, dict): + if record_id_random := select_results.get('id_random'): + return str(record_id_random) + else: + log.error('The SQL result was not what was expected.') + return False + elif isinstance(select_results, list): + log.exception('More than one record may have been found. Duplicate ID!') + return False + else: + log.exception(f'Got an unexpected result while trying to look up the ID.') + return False + elif select_results is None: + return None + else: + return False + + +def reset_redis(): + """Flushes the Redis database used for ID caching.""" + r = redis.Redis(host=settings.REDIS['server'], port=settings.REDIS['port'], db=7, password=None, decode_responses=True) + r.flushdb() + return True \ No newline at end of file