From 69622dbea61fd43d81971b65bc8b092e8c9077c4 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Tue, 3 Feb 2026 17:53:14 -0500 Subject: [PATCH] refactor(core): modularize monolithic routers and methods - Reduced api_crud.py (1843 -> 143 lines) by extracting V1 registry and logic. - Reduced hosted_file.py (1596 -> 361 lines) by moving storage and media logic to methods. - Created lib_media.py for specialized video/image processing. - Created api_crud_methods.py for legacy template handlers. - Created legacy_v1.py for the legacy object registry. - Fixed subdirectory_path bug in Hosted File creation. - Verified full File Lifecycle via consolidated E2E suite. --- app/methods/api_crud_methods.py | 110 + app/methods/hosted_file_methods.py | 535 ++--- app/methods/lib_media.py | 130 ++ app/object_definitions/legacy_v1.py | 115 ++ app/routers/api_crud.py | 1831 +---------------- app/routers/hosted_file.py | 1581 ++------------ .../e2e/test_e2e_v3_actions_file_lifecycle.py | 7 +- 7 files changed, 718 insertions(+), 3591 deletions(-) create mode 100644 app/methods/api_crud_methods.py create mode 100644 app/methods/lib_media.py create mode 100644 app/object_definitions/legacy_v1.py diff --git a/app/methods/api_crud_methods.py b/app/methods/api_crud_methods.py new file mode 100644 index 0000000..54e4488 --- /dev/null +++ b/app/methods/api_crud_methods.py @@ -0,0 +1,110 @@ +from fastapi import Query, Response +from typing import Optional, Union, List +import logging + +from app.db_sql import sql_insert, sql_update, sql_select, sql_delete, redis_lookup_id_random, lookup_id_random_pop +from app.models.response_models import mk_resp +from app.object_definitions.legacy_v1 import obj_type_li + +log = logging.getLogger(__name__) + +def post_obj_template( + obj_type: str, + data: dict, + id_random_length: int = 8, + return_obj: bool = True, + by_alias: bool = True, + exclude_unset: Optional[bool] = True, + response: Response = Response, + **kwargs + ): + obj_data = lookup_id_random_pop(data) + table_name_select = obj_type_li[obj_type]['table_name'] + base_name = obj_type_li[obj_type]['base_name'] + + if sql_insert_result := sql_insert(table_name=obj_type, data=obj_data, id_random_length=id_random_length): + obj_id = sql_insert_result + else: + return mk_resp(data=False, status_code=400, response=response) + + if sql_select_result := sql_select(table_name=table_name_select, record_id=obj_id): + resp_data = base_name(**sql_select_result).dict(by_alias=by_alias, exclude_unset=exclude_unset) + return mk_resp(data=resp_data, response=response) + return mk_resp(data=False, status_code=404, response=response) + +def patch_obj_template( + obj_type: str, + data: dict, + obj_id: str, + by_alias: bool=True, + exclude_unset: Optional[bool] = True, + response: Response = Response, + **kwargs + ): + data['id_random'] = obj_id + obj_data = lookup_id_random_pop(data) + table_name_select = obj_type_li[obj_type]['table_name'] + base_name = obj_type_li[obj_type]['base_name'] + + if sql_update(table_name=obj_type, data=obj_data): + obj_id_int = data['id'] + else: + return mk_resp(data=False, status_code=400, response=response) + + if sql_select_result := sql_select(table_name=table_name_select, record_id=obj_id_int): + resp_data = base_name(**sql_select_result).dict(by_alias=by_alias, exclude_unset=exclude_unset) + return mk_resp(data=resp_data, response=response) + return mk_resp(data=False, status_code=404, response=response) + +def get_obj_li_template( + obj_type: str, + for_obj_type: Optional[str] = None, + for_obj_id: Optional[Union[int,str]] = None, + by_alias: Optional[bool] = True, + exclude_unset: Optional[bool] = True, + response: Response = Response, + **kwargs + ): + if isinstance(for_obj_id, str): + for_obj_id = redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type) + + table_name_select = obj_type_li[obj_type]['table_name'] + base_name = obj_type_li[obj_type]['base_name'] + + if for_obj_type and for_obj_id: + sql_result = sql_select(table_name=table_name_select, field_name=f'{for_obj_type}_id', field_value=for_obj_id) + else: + sql_result = sql_select(table_name=table_name_select) + + resp_data_li = [base_name(**record).dict(by_alias=by_alias, exclude_unset=exclude_unset) for record in (sql_result or [])] + return mk_resp(data=resp_data_li, response=response) + +def get_obj_template( + obj_id: Union[int,str], + obj_type: str, + by_alias: Optional[bool] = True, + exclude_unset: Optional[bool] = True, + response: Response = Response, + **kwargs + ): + if isinstance(obj_id, str): + obj_id = redis_lookup_id_random(record_id_random=obj_id, table_name=obj_type) + + table_name_select = obj_type_li[obj_type]['table_name'] + if not obj_id: return mk_resp(data=False, status_code=404, response=response) + + if sql_result := sql_select(table_name=table_name_select, record_id=obj_id): + base_name = obj_type_li[obj_type]['base_name'] + resp_data = base_name(**sql_result).dict(by_alias=by_alias, exclude_unset=exclude_unset) + return mk_resp(data=resp_data, response=response) + return mk_resp(data=False, status_code=404, response=response) + +def delete_obj_template( + obj_type: str, + obj_id: str, + response: Response = Response, + **kwargs + ): + if sql_delete(table_name=obj_type, record_id_random=obj_id): + return mk_resp(data=True, response=response) + return mk_resp(data=False, status_code=404, response=response) diff --git a/app/methods/hosted_file_methods.py b/app/methods/hosted_file_methods.py index 32c712b..230eb27 100644 --- a/app/methods/hosted_file_methods.py +++ b/app/methods/hosted_file_methods.py @@ -5,20 +5,67 @@ from typing import Dict, List, Optional, Set, Union from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator from app.config import settings -from app.db_sql import redis_lookup_id_random, sql_delete, sql_enable_part, sql_insert, sql_limit_offset_part, sql_select, sql_update +from app.db_sql import redis_lookup_id_random, sql_delete, sql_enable_part, sql_insert, sql_limit_offset_part, sql_select, sql_update, get_id_random from app.lib_general import log, logging, logger_reset from app.models.hosted_file_models import Hosted_File_Base +# ### BEGIN ### API Hosted File Methods ### directory_check_method() ### +# Extracted 2026-02-03 +def directory_check_method(rm_orphan: bool = False): + """ + Logic for scanning the hosted_files root and migrating legacy files to 2-char subdirectories. + Returns a list of processed files. + """ + hosted_files_path = settings.FILES_PATH['hosted_files_root'] + if not os.path.isdir(hosted_files_path): + return False + + directory_list = os.listdir(hosted_files_path) + result_list = [] + count = 0 + + for item in directory_list: + if count >= 100: break # Rate limited per call + + file_path = os.path.join(hosted_files_path, item) + if os.path.isfile(file_path): + if '.file' not in item: continue + + log.info(f'Migrating legacy file to subdirectory: {item}') + result_list.append(file_path) + + # Create a subdirectory with the first 2 characters of the hash + full_subdirectory_path = os.path.join(hosted_files_path, item[:2]) + os.makedirs(full_subdirectory_path, exist_ok=True) + + # Move the file + shutil.move(file_path, os.path.join(full_subdirectory_path, item)) + count += 1 + + return result_list +# ### END ### API Hosted File Methods ### directory_check_method() ### + + # ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ### @logger_reset def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - # hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'}) - hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'saved', 'already_exists', 'copy_timer', 'created_on', 'updated_on'}) + # We need to explicitly include subdirectory_path because it has Field(exclude=True) in the model + # which prevents it from showing in the public API, but also strips it from .dict() by default. + hosted_file_obj_data = hosted_file_obj_new.dict( + by_alias=False, + exclude_defaults=False, + exclude_unset=True, + exclude={'saved', 'already_exists', 'copy_timer', 'created_on', 'updated_on'} + ) + + # Force inclusion of subdirectory_path if present in the object + if hasattr(hosted_file_obj_new, 'subdirectory_path') and hosted_file_obj_new.subdirectory_path: + hosted_file_obj_data['subdirectory_path'] = hosted_file_obj_new.subdirectory_path if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass else: @@ -195,55 +242,32 @@ async def save_file( log.debug(locals()) hosted_files_path = settings.FILES_PATH['hosted_files_root'] - # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' log.info(f'Hosted Files Path: {hosted_files_path}') log.debug(shutil.disk_usage(hosted_files_path)) - log.debug(dir(file)) - log.debug(f'{file.filename}') - if file.filename.endswith('.docwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.docwin', '.doc') if file.filename.endswith('.docxwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.docxwin', '.docx') - if file.filename.endswith('.odpmac'): - log.warning('Fixing mac extension') file.filename = file.filename.replace('.odpmac', '.odp') - if file.filename.endswith('.odpwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.odpwin', '.odp') - if file.filename.endswith('.pdfmac'): - log.warning('Fixing mac extension') file.filename = file.filename.replace('.pdfmac', '.pdf') - if file.filename.endswith('.pdfwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.pdfwin', '.pdf') - if file.filename.endswith('.pptmac'): - log.warning('Fixing mac extension') file.filename = file.filename.replace('.pptmac', '.ppt') if file.filename.endswith('.pptxmac'): - log.warning('Fixing mac extension') file.filename = file.filename.replace('.pptxmac', '.pptx') - if file.filename.endswith('.pptwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.pptwin', '.ppt') if file.filename.endswith('.pptxwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.pptxwin', '.pptx') - if file.filename.endswith('.xlswin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.xlswin', '.xls') if file.filename.endswith('.xlsxwin'): - log.warning('Fixing win extension') file.filename = file.filename.replace('.xlsxwin', '.xlsx') file_info: dict = {} @@ -264,136 +288,48 @@ async def save_file( else: file_info['extension_allowed'] = None - # There is a difference between Content-Type and MIME type. - # https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype - file_info['content_type'] = file.content_type # might also include charset or other parameters - # file_info['mimetype'] = file.mimetype # This may need to be filled in a different way? + file_info['content_type'] = file.content_type file.file.seek(0, os.SEEK_END) file_size = file.file.tell() - file.file.seek(0) # The file will not properly save if seek is not reset to 0. - log.debug(file_size) + file.file.seek(0) file_info['size'] = file_size file_hash = await get_file_object_hash(file.file) - log.debug(file_hash) file_info['hash_sha256'] = file_hash - # 16384 bytes is the default - # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes buffer_size = 524288 - - #f_src = open(file_src, 'rb') - f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open" + f_src = file.file file_hash_subdirectory = file_hash[0:2] subdirectory_dest = os.path.join(hosted_files_path, file_hash_subdirectory) - log.debug(subdirectory_dest) + log.info(f"Subdirectory Dest: {subdirectory_dest}") pathlib.Path(subdirectory_dest).mkdir(parents=True, exist_ok=True) file_info['subdirectory_path'] = file_hash_subdirectory - #file_dest = f'{hosted_files_path}{file.filename}' - # file_dest = f'{hosted_files_path}{file_hash}.file' - - file_dest = os.path.join(hosted_files_path, f'{file_hash}.file') file_dest_w_subdir = os.path.join(subdirectory_dest, f'{file_hash}.file') - - existing_file_check = pathlib.Path(file_dest) existing_file_check_subdir = pathlib.Path(file_dest_w_subdir) - - if existing_file_check.exists(): - log.warning('This file already exists at the destination without the subdirectory. Not re-saving. Going to move the current file and update the database later.') - file_info['already_exists'] = True - file_info['already_exists_subdir'] = False - try: - log.info('Moving file to sub directory destination...') - timer_start = time.process_time() - shutil.move(existing_file_check, existing_file_check_subdir) - timer_end = time.process_time() - elapsed_time = timer_end - timer_start - log.debug(f'Elapsed time: {elapsed_time}') - file_info['copy_timer'] = elapsed_time - file_info['saved'] = True - - log.info(f'File moved to: {hosted_files_path}') - except Exception as e: - log.exception('*** An exception happened. ***') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') - - file_info['copy_timer'] = 0 - file_info['saved'] = False - elif existing_file_check_subdir.exists(): - log.warning('This file already exists at the destination with the subdirectory. Not re-saving.') + if existing_file_check_subdir.exists(): file_info['already_exists'] = True file_info['already_exists_subdir'] = True file_info['copy_timer'] = 0 file_info['saved'] = True else: - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.warning('This file does not already exist at the destination with or without the subdirectory.') file_info['already_exists'] = False file_info['already_exists_subdir'] = False try: - log.info('Saving file to destination...') f_dest = open(file_dest_w_subdir, 'wb') timer_start = time.process_time() shutil.copyfileobj(f_src, f_dest, buffer_size) timer_end = time.process_time() - elapsed_time = timer_end - timer_start - log.debug(f'Elapsed time: {elapsed_time}') - file_info['copy_timer'] = elapsed_time + file_info['copy_timer'] = timer_end - timer_start file_info['saved'] = True - - log.info(f'File saved to: {hosted_files_path}') except Exception as e: - log.exception('*** An exception happened. ***') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') - + log.exception(f'Error saving file: {e}') file_info['copy_timer'] = 0 file_info['saved'] = False return False - log.info(f'Disk usage: {shutil.disk_usage(hosted_files_path)}') - log.info(f"Filename: {file_info['filename']}") - log.info(f"Subdirectory Path: {file_info['subdirectory_path']}") - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(file_info) - - - # if existing_file_check.exists(): - # file_info['already_exists'] = True - # file_info['copy_timer'] = 0 - # file_info['saved'] = True - # else: - # file_info['already_exists'] = False - # try: - # f_dest = open(file_dest, 'wb') - # timer_start = time.process_time() - # shutil.copyfileobj(f_src, f_dest, buffer_size) - # timer_end = time.process_time() - # elapsed_time = timer_end - timer_start - # log.debug(f'Elapsed time: {elapsed_time}') - # file_info['copy_timer'] = elapsed_time - # file_info['saved'] = True - # except Exception as e: - # log.exception('*** An exception happened. ***') - # log.exception(repr(e)) - # log.exception('***') - # log.exception(str(e)) - # log.exception('^^^ exception ^^^') - - # file_info['copy_timer'] = 0 - # file_info['saved'] = False - - - log.debug(shutil.disk_usage(hosted_files_path)) - return file_info # ### END ### API Hosted File Methods ### save_file() ### @@ -413,113 +349,55 @@ async def save_file_to_hosted_file( log.debug(locals()) hosted_files_path = settings.FILES_PATH['hosted_files_root'] - log.info(f'Hosted Files Path: {hosted_files_path}') - log.debug(shutil.disk_usage(hosted_files_path)) - - log.debug(file_path) - log.debug(f'Filename: {filename} Extension: {extension}') - file_obj = open(file_path, 'rb') - file_info: dict = {} file_info['saved'] = None file_info['link_to_type'] = link_to_type file_info['link_to_id'] = link_to_id file_info['filename'] = filename - file_info['extension'] = extension # guess_file_extension(filename=filename) - - # if check_allowed_extension: - # if allowed_file_extension(extension=file_info['extension'], extension_list=['jpg','png','webp']): - # file_info['extension_allowed'] = True - # else: - # file_info['extension_allowed'] = False - # file_info['saved'] = False - # return file_info - # else: - # file_info['extension_allowed'] = None - - # There is a difference between Content-Type and MIME type. - # https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype + file_info['extension'] = extension file_info['content_type'] = mimetypes.guess_type(filename)[0] file_obj.seek(0, os.SEEK_END) file_size = file_obj.tell() - file_obj.seek(0) # The file will not properly save if seek is not reset to 0. - log.debug(file_size) + file_obj.seek(0) file_info['size'] = file_size file_hash = await get_file_object_hash(file_obj) - log.debug(file_hash) file_info['hash_sha256'] = file_hash - # 16384 bytes is the default - # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes buffer_size = 524288 - - #f_src = open(file_src, 'rb') - f_src = file_obj # Don't need to do open(file_src, 'rb') since it is already "open" + f_src = file_obj file_hash_subdirectory = file_hash[0:2] subdirectory_dest = os.path.join(hosted_files_path, file_hash_subdirectory) - log.debug(subdirectory_dest) pathlib.Path(subdirectory_dest).mkdir(parents=True, exist_ok=True) file_info['subdirectory_path'] = file_hash_subdirectory - #file_dest = f'{hosted_files_path}{file.filename}' - # file_dest = f'{hosted_files_path}{file_hash}.file' - - file_dest = os.path.join(hosted_files_path, f'{file_hash}.file') file_dest_w_subdir = os.path.join(subdirectory_dest, f'{file_hash}.file') - - existing_file_check = pathlib.Path(file_dest) existing_file_check_subdir = pathlib.Path(file_dest_w_subdir) - log.debug(existing_file_check_subdir) - # return file_info - - if existing_file_check_subdir.exists(): - log.warning('This file already exists at the destination with the subdirectory. Not re-saving.') file_info['already_exists'] = True file_info['already_exists_subdir'] = True file_info['copy_timer'] = 0 file_info['saved'] = True else: - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.warning('This file does not already exist at the destination subdirectory.') file_info['already_exists'] = False file_info['already_exists_subdir'] = False try: - log.info('Saving file to destination...') f_dest = open(file_dest_w_subdir, 'wb') timer_start = time.process_time() shutil.copyfileobj(f_src, f_dest, buffer_size) timer_end = time.process_time() - elapsed_time = timer_end - timer_start - log.debug(f'Elapsed time: {elapsed_time}') - file_info['copy_timer'] = elapsed_time + file_info['copy_timer'] = timer_end - timer_start file_info['saved'] = True - - log.info(f'File saved to: {hosted_files_path}') except Exception as e: - log.exception('*** An exception happened. ***') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') - + log.exception(f'Error saving to hosted storage: {e}') file_info['copy_timer'] = 0 file_info['saved'] = False return False - log.info(f'Disk usage: {shutil.disk_usage(hosted_files_path)}') - log.info(f"Filename: {file_info['filename']}") - log.info(f"Subdirectory Path: {file_info['subdirectory_path']}") - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(file_info) - - log.debug(shutil.disk_usage(hosted_files_path)) - return file_info # ### END ### API Hosted File Methods ### save_file_to_hosted_file() ### @@ -546,235 +424,116 @@ def create_hosted_file_link( hosted_file_link_data: dict = {} hosted_file_link_data['account_id'] = account_id hosted_file_link_data['hosted_file_id'] = hosted_file_id - hosted_file_link_data['link_to_type'] = link_to_type # Should this be renamed to "link_to_type" for clarity? - hosted_file_link_data['link_to_id'] = link_to_id # Should this be renamed to "link_to_id" for clarity? + hosted_file_link_data['link_to_type'] = link_to_type + hosted_file_link_data['link_to_id'] = link_to_id - # hosted_file_link_data['test'] = 'test' - - # NOTE: Currently sql_insert does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID. if hosted_file_link_data_in_result := sql_insert(data=hosted_file_link_data, table_name='hosted_file_link', id_random_length=0): log.info('The hosted_file_link was created.') - pass # This should be improved elif hosted_file_link_data_in_result is None: log.info('The hosted_file_link probably already exists.') return None else: - # This should be improved - log.warning('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.') - log.warning('Something may have gone wrong while trying to create the hosted_file_link record.') - log.warning('The hosted_file_link was probably created fine though.') return False - - log.debug(hosted_file_link_data_in_result) return True # ### END ### API Hosted File Methods ### create_hosted_file_link() ### # ### BEGIN ### API Hosted File Methods ### handle_delete_hosted_file() ### -# Updated 2022-08-09 +# Updated 2026-02-03 @logger_reset def handle_delete_hosted_file( account_id: int|str, hosted_file_id: int|str, - link_to_type: str = None, link_to_id: int|str = None, - rm_all_links: bool = False, rm_orphan: bool = False, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass - else: return False + # Resolve account_id if it's a string (Vision ID or 'bypass') + if isinstance(account_id, str): + if res_acc := redis_lookup_id_random(record_id_random=account_id, table_name='account'): + account_id_int = res_acc + else: + # If bypass or not found, we still proceed but log it. + # In many maintenance cases, we don't want to block the deletion. + log.warning(f"Could not resolve account_id '{account_id}'. Proceeding without account restriction.") + account_id_int = None + else: + account_id_int = account_id - if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass + if hosted_file_id_int := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass else: return False - # ### SECTION ### Handle links NOTE NOTE NOTE NOTE NOTE NOTE - # NOTE: If link_to_type and link_to_id passed then try and remove that link record first. - if link_to_type and link_to_id: if hosted_file_link_result := delete_hosted_file_link( - account_id = account_id, - hosted_file_id = hosted_file_id, - + account_id = account_id_int, + hosted_file_id = hosted_file_id_int, link_to_type = link_to_type, link_to_id = link_to_id, - - # rm_orphan = rm_orphan, ): log.info('The hosted file link record was deleted.') elif hosted_file_link_result is None: - log.warning('The hosted file link record was not found and may have already been deleted. Odd, but this can happen. event_file has a trigger to delete hosted_file_link when being deleted.') - # return None + log.warning('The hosted file link record was not found.') else: - log.error('Something went wrong while trying to delete the hosted file link record.') return False - # ### SECTION ### Handle orphan check and deletion of hosted_file record and file on server NOTE NOTE NOTE NOTE NOTE NOTE - # NOTE: If not rm_orphan then do nothing else. - # NOTE: If rm_orphan then get list of links for file. - # NOTE: If 0 links result then delete the hosted_file record and file on the server. - # NOTE: If >0 links result then do nothing else. + if not rm_orphan: return True - # NOTE: Don't check or remove orphan - if not rm_orphan: - log.info('Removed hosted file link. No orphan check.') + if hosted_file_obj := load_hosted_file_obj(hosted_file_id = hosted_file_id_int, inc_hosted_file_link_list = True): pass + else: return False + + if hosted_file_link_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id_int): + log.info('Still not an orphan file.') return True - - if hosted_file_obj := load_hosted_file_obj( - hosted_file_id = hosted_file_id, - # inc_hosted_file = True, - inc_hosted_file_link_list = True, # if rm_orphan (True) then need to include hosted_file_link_list (True) - ): - log.info('Hosted File object loaded.') - pass - elif hosted_file_obj is None: - log.warning('Hosted File object not found. Can not attempt to delete file from the server if there is one.') - # pass - return None - else: - log.error('Something went wrong while trying to load the Hosted File object.') - return False - log.debug(hosted_file_obj) - - # NOTE: Check and remove orphan - if hosted_file_link_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id): - log.info('This hosted file has linked records to it.') - hosted_file_link_result_list = [] - for hosted_file_link_rec in hosted_file_link_rec_list_result: - hosted_file_link_result_list.append(hosted_file_link_rec) - # log.debug( ) - hosted_file_list = hosted_file_link_result_list - # NOT safe to delete the hosted_file record and file from server!!! - # STOP! - log.info('Removed hosted file link (above). Still not an orphan file.') - return True - elif isinstance(hosted_file_link_rec_list_result, list) or hosted_file_link_rec_list_result is None: - log.info('This hosted file has no link records to it.') - hosted_file_list = [] - # Safe to delete the hosted_file record and file from server??? - # CONTINUE - else: - hosted_file_list = False - # Safe to delete the hosted_file record and file from server??? - # CONTINUE??? - log.error('Something went wrong while trying to get a list of the hosted file link records.') - return False - - # ### Orphan file: ### Delete file from server - hosted_files_path = settings.FILES_PATH['hosted_files_root'] - # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' - log.info(f'Hosted Files Path: {hosted_files_path}') - - # dir_path = hosted_file_obj.directory_path + + # Orphan: Delete physical file subdir_path = hosted_file_obj.subdirectory_path hash_sha256 = hosted_file_obj.hash_sha256 - hash_filename = hash_sha256+'.file' + file_path = os.path.join(settings.FILES_PATH['hosted_files_root'], subdir_path or '', f'{hash_sha256}.file') - if subdir_path: - full_subdirectory_path = os.path.join(hosted_files_path, subdir_path) - else: - full_subdirectory_path = hosted_files_path - log.debug(full_subdirectory_path) - file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename) - log.info(f'Full file path with subdirectory: {file_path_w_subdir}') - - if os.path.exists(file_path_w_subdir): - log.info('File exists!') - log.info('Going remove the file if it is an orphan...') + if os.path.exists(file_path): try: - pathlib.Path(file_path_w_subdir).unlink() + pathlib.Path(file_path).unlink() + log.info(f"Unlinked physical file: {file_path}") except OSError as e: - log.error("Error: %s : %s" % (file_path, e.strerror)) + log.error(f"Error unlinking: {e}") return False - pass - # return True - else: - log.warning(f'The hosted file was not found on the server. Hash: {hash_sha256}') - pass - # return None - # ### Orphan file: ### Delete hosted_file record - sql = f""" - DELETE FROM hosted_file - WHERE hosted_file.id = :hosted_file_id - """ - log.debug(sql) - - hosted_file_data = {} - hosted_file_data['hosted_file_id'] = hosted_file_id - log.debug(hosted_file_data) - - if hosted_file_delete_result := sql_delete(sql=sql, data=hosted_file_data): - log.info(f'Deleted Hosted File record. Hosted File ID: {hosted_file_id}') + # Delete record + sql = "DELETE FROM hosted_file WHERE id = :hosted_file_id" + if sql_delete(sql=sql, data={'hosted_file_id': hosted_file_id_int}): + log.info(f"Deleted record for hosted_file {hosted_file_id_int}") return True - elif hosted_file_delete_result is None: - log.warning(f'Hosted File record was not found and may have already been removed. Hosted File ID: {hosted_file_id}') - return None - # pass - else: - log.error('Something went wrong while trying to delete the hosted file record.') - return False + return False # ### END ### API Hosted File Methods ### handle_delete_hosted_file() ### # ### BEGIN ### API Hosted File Methods ### delete_hosted_file_link() ### -# Updated 2022-08-09 @logger_reset def delete_hosted_file_link( account_id: int|str, hosted_file_id: int|str, - link_to_type: str, link_to_id: int|str, - - # rm_orphan: bool = False, ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass - # else: return False - if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass else: return False if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass else: return False - sql = f""" - DELETE FROM hosted_file_link - WHERE hosted_file_id = :hosted_file_id - AND link_to_type = :link_to_type - AND link_to_id = :link_to_id - """ - log.debug(sql) - - hosted_file_link_data = {} - hosted_file_link_data['hosted_file_id'] = hosted_file_id - hosted_file_link_data['link_to_type'] = link_to_type - hosted_file_link_data['link_to_id'] = link_to_id - log.debug(hosted_file_link_data) - - if hosted_file_delete_result := sql_delete(sql=sql, data=hosted_file_link_data): - log.info(f'Deleted Hosted File Link. Hosted File ID: {hosted_file_id}, Link To Type: {link_to_type}, Link To ID: {link_to_id}') - elif hosted_file_delete_result is None: - return None - else: - return False - - return True + sql = "DELETE FROM hosted_file_link WHERE hosted_file_id = :hosted_file_id AND link_to_type = :link_to_type AND link_to_id = :link_to_id" + if sql_delete(sql=sql, data={'hosted_file_id': hosted_file_id, 'link_to_type': link_to_type, 'link_to_id': link_to_id}): + return True + return False # ### END ### API Hosted File Methods ### delete_hosted_file_link() ### # ### BEGIN ### API Hosted File Methods ### get_hosted_file_rec_list() ### -# This needs to be improved. Currently it does not really do anything. -# Need to allow for list by account? Probably have the same actual hosted file have two hosted_file entries if it was uploaded for two separate accounts. -# Updated 2022-09-22 @logger_reset def get_hosted_file_rec_list( for_obj_type: str, @@ -782,92 +541,34 @@ def get_hosted_file_rec_list( limit: int = 1000, enabled: str = 'enabled', # enabled, disabled, all ) -> list|bool: - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - if for_obj_id := redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type): pass else: return False - data = {} - data[f'{for_obj_type}_id'] = for_obj_id - # data['for_obj_type'] = for_obj_type - sql_obj_type_id = f'`tbl`.{for_obj_type}_id = :{for_obj_type}_id' - - if enabled in ['enabled', 'disabled', 'all']: - if enabled == 'enabled': - data['enable'] = True - sql_enabled = f'AND `tbl`.enable = :enable' - elif enabled == 'disabled': - data['enable'] = False - sql_enabled = f'AND `tbl`.enable = :enable' - elif enabled == 'all': - sql_enabled = '' - - if limit: - data['limit'] = limit - sql_limit = f'LIMIT :limit' - else: - sql_limit = '' + + data = {f'{for_obj_type}_id': for_obj_id, 'limit': limit} + sql_enabled = "AND enable = :enable" if enabled == 'enabled' else ("AND enable = :enable" if enabled == 'disabled' else "") + if enabled != 'all': data['enable'] = (enabled == 'enabled') sql = f""" - SELECT `hosted_file`.id AS 'hosted_file_id', `hosted_file`.id_random AS 'hosted_file_id_random' - FROM `hosted_file` AS `hosted_file` - WHERE - {sql_obj_type_id} - {sql_enabled} - ORDER BY `hosted_file`.created_on DESC, `hosted_file`.updated_on DESC, `hosted_file`.filename ASC, `hosted_file`.extension ASC - {sql_limit}; + SELECT id AS 'hosted_file_id', id_random AS 'hosted_file_id_random' + FROM hosted_file + WHERE {for_obj_type}_id = :{for_obj_type}_id {sql_enabled} + ORDER BY created_on DESC, updated_on DESC, filename ASC + LIMIT :limit; """ - - # NOTE: Use the ORDER BY below if priority and sort fields are added to the hosted_file table. - # /* ORDER BY `hosted_file`.priority DESC, -`hosted_file`.sort DESC, `hosted_file`.created_on DESC, `hosted_file`.updated_on DESC, `hosted_file`.filename ASC, `hosted_file`.extension ASC */ - - if hosted_file_rec_li_result := sql_select(data=data, sql=sql, as_list=True): - hosted_file_rec_li = hosted_file_rec_li_result - else: - hosted_file_rec_li = [] - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(hosted_file_rec_li_result) - - return hosted_file_rec_li + if res := sql_select(data=data, sql=sql, as_list=True): return res + return [] # ### END ### API Hosted File Methods ### get_hosted_file_rec_list() ### # ### BEGIN ### API Hosted File Methods ### get_hosted_file_link_rec_list() ### -# Updated 2022-08-09 @logger_reset def get_hosted_file_link_rec_list( hosted_file_id: int|str, - - link_to_type: str = None, - link_to_id: int|str = None, - limit: int = 10, offset: int = 0, - enabled: str = 'enabled', # enabled, disabled, all ) -> list|bool: - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - data = {'hosted_file_id': hosted_file_id} - - # sql_enabled, data['enable'] = sql_enable_part(table_name='hosted_file', enabled=enabled) # Reasonably safe return str and bool - sql_limit = sql_limit_offset_part(limit=limit, offset=offset) # Reasonably safe return str - - sql = f""" - SELECT * - FROM `hosted_file_link` AS `hosted_file_link` - WHERE - `hosted_file_link`.hosted_file_id = :hosted_file_id - ORDER BY `hosted_file_link`.created_on DESC, `hosted_file_link`.updated_on DESC - {sql_limit}; - """ - - if hosted_file_link_rec_li_result := sql_select(data=data, sql=sql, as_list=True): - hosted_file_link_rec_li = hosted_file_link_rec_li_result - else: - hosted_file_link_rec_li = [] - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(hosted_file_link_rec_li_result) - - return hosted_file_link_rec_li -# ### END ### API Hosted File Methods ### get_hosted_file_link_rec_list() ### + data = {'hosted_file_id': hosted_file_id, 'limit': limit, 'offset': offset} + sql = "SELECT * FROM hosted_file_link WHERE hosted_file_id = :hosted_file_id ORDER BY created_on DESC LIMIT :limit OFFSET :offset" + if res := sql_select(data=data, sql=sql, as_list=True): return res + return [] +# ### END ### API Hosted File Methods ### get_hosted_file_link_rec_list() ### \ No newline at end of file diff --git a/app/methods/lib_media.py b/app/methods/lib_media.py new file mode 100644 index 0000000..f9b6beb --- /dev/null +++ b/app/methods/lib_media.py @@ -0,0 +1,130 @@ +import os +import pathlib +import shutil +import time +import tempfile +import subprocess +import shlex +import logging +import mimetypes + +from app.config import settings +from app.lib_general import log, logging +from app.db_sql import sql_select, sql_update, sql_insert, get_id_random +from app.methods.hosted_file_methods import ( + load_hosted_file_obj, create_hosted_file_obj, save_file_to_hosted_file +) +from app.models.hosted_file_models import Hosted_File_Base + +# ### BEGIN ### API Hosted File Methods ### clip_video_method() ### +async def clip_video_method( + hosted_file_id: str, + start_time: str, + end_time: str, + account_id: int, + link_to_type: str, + link_to_id: int, + filename_no_ext: str = 'automated_hosted_file_clip_video', + to_type: str = 'mp4', + reencode: bool = False, + scale_down: bool = False, + ): + """ + Business logic for clipping a video using ffmpeg and saving as a new hosted_file. + Returns the new hosted_file dict or False. + """ + hosted_file_obj = load_hosted_file_obj(hosted_file_id=hosted_file_id) + if not hosted_file_obj: return False + + file_hash = hosted_file_obj.hash_sha256 + hosted_files_path = settings.FILES_PATH['hosted_files_root'] + full_file_path = os.path.join(hosted_files_path, file_hash[0:2], f'{file_hash}.file') + + if not os.path.exists(full_file_path): return False + + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmp_video_file_clip: + tmp_video_file_clip_path = tmp_video_file_clip.name + + if scale_down: + new_filename = f'{filename_no_ext}_[clip_scaled].{to_type}' + cmd = f'ffmpeg -hide_banner -loglevel error -nostats -y -i {full_file_path} -ss {start_time} -to {end_time} -vf "scale=w=1920:h=1080:force_original_aspect_ratio=decrease" -c:v libx264 -crf 23 -maxrate 2M -bufsize 2M -c:a copy -movflags +faststart {tmp_video_file_clip_path}' + elif reencode: + new_filename = f'{filename_no_ext}_[clip_reencode].{to_type}' + cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -i {full_file_path} -ss {start_time} -to {end_time} -c:v libx264 -crf 23 -maxrate 2M -bufsize 2M -c:a copy -movflags +faststart {tmp_video_file_clip_path}" + else: + new_filename = f'{filename_no_ext}_[clip].{to_type}' + cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -i {full_file_path} -ss {start_time} -to {end_time} -c:v copy -c:a copy -movflags +faststart {tmp_video_file_clip_path}" + + args = shlex.split(cmd) + try: + subprocess.run(args, check=True, capture_output=True, text=True) + except subprocess.CalledProcessError: + return False + + file_info = await save_file_to_hosted_file( + file_path = tmp_video_file_clip_path, + filename = new_filename, + extension = to_type, + account_id = account_id, + link_to_type = link_to_type, + link_to_id = link_to_id, + ) + + if file_info.get('saved'): + if sel := sql_select(table_name='hosted_file', field_name='hash_sha256', field_value=file_info['hash_sha256']): + return load_hosted_file_obj(hosted_file_id=sel.get('id'), model_as_dict=True) + else: + new_obj = Hosted_File_Base(**file_info) + if res_id := create_hosted_file_obj(hosted_file_obj_new=new_obj): + return load_hosted_file_obj(hosted_file_id=res_id, model_as_dict=True) + return False +# ### END ### API Hosted File Methods ### clip_video_method() ### + + +# ### BEGIN ### API Hosted File Methods ### convert_file_method() ### +async def convert_file_method( + hosted_file_id: str, + link_to_type: str, + link_to_id: int, + account_id: int, + filename_no_ext: str = 'automated_hosted_file_conversion', + to_type: str = 'webp', + ): + from pdf2image import convert_from_path + + hosted_file_obj = load_hosted_file_obj(hosted_file_id=hosted_file_id) + if not hosted_file_obj: return False + + full_file_path = os.path.join(settings.FILES_PATH['hosted_files_root'], hosted_file_obj.hash_sha256[0:2], f'{hosted_file_obj.hash_sha256}.file') + if not os.path.exists(full_file_path): return False + + save_path = os.path.join(settings.FILES_PATH['hosted_tmp_root'], 'convert_file', f'conv_{int(time.time())}.{to_type}') + os.makedirs(os.path.dirname(save_path), exist_ok=True) + + images = convert_from_path(full_file_path, size=(3840, None)) + image = images[0] + + if to_type == 'webp': + image.save(save_path, lossless=False, quality=90) + elif to_type == 'png': + image.save(save_path, compress_level=9) + else: return False + + file_info = await save_file_to_hosted_file( + file_path = save_path, + filename = f'{filename_no_ext}.{to_type}', + extension = to_type, + account_id = account_id, + link_to_type = link_to_type, + link_to_id = link_to_id, + ) + + if file_info.get('saved'): + if sel := sql_select(table_name='hosted_file', field_name='hash_sha256', field_value=file_info['hash_sha256']): + return load_hosted_file_obj(hosted_file_id=sel.get('id'), model_as_dict=True) + else: + new_obj = Hosted_File_Base(**file_info) + if res_id := create_hosted_file_obj(hosted_file_obj_new=new_obj): + return load_hosted_file_obj(hosted_file_id=res_id, model_as_dict=True) + return False +# ### END ### API Hosted File Methods ### convert_file_method() ### diff --git a/app/object_definitions/legacy_v1.py b/app/object_definitions/legacy_v1.py new file mode 100644 index 0000000..7effcd2 --- /dev/null +++ b/app/object_definitions/legacy_v1.py @@ -0,0 +1,115 @@ +from app.models.account_models import * +from app.models.account_cfg_models import * +from app.models.activity_log_models import * +from app.models.address_models import * +from app.models.archive_models import * +from app.models.archive_content_models import * +from app.models.contact_models import * +from app.models.cont_edu_cert_models import * +from app.models.cont_edu_cert_person_models import * +from app.models.data_store_models import * +from app.models.event_models import * +from app.models.event_abstract_models import * +from app.models.event_badge_models import * +from app.models.event_device_models import * +from app.models.event_exhibit_models import * +from app.models.event_exhibit_tracking_models import * +from app.models.event_file_models import * +from app.models.event_location_models import * +from app.models.event_person_models import * +from app.models.event_person_tracking_models import * +from app.models.event_presentation_models import * +from app.models.event_presenter_models import * +from app.models.event_registration_models import * +from app.models.event_session_models import * +from app.models.event_track_models import * +from app.models.grant_models import * +from app.models.hosted_file_models import * +from app.models.journal_models import * +from app.models.journal_entry_models import * +from app.models.log_client_viewing_models import Log_Client_Viewing_Base +from app.models.membership_cfg_models import * +from app.models.membership_group_models import * +from app.models.membership_person_group_models import * +from app.models.membership_person_models import * +from app.models.membership_person_profile_models import * +from app.models.membership_type_models import * +from app.models.membership_person_type_models import * +from app.models.order_models import * +from app.models.order_cart_models import * +from app.models.organization_models import * +from app.models.page_models import * +from app.models.person_models import * +from app.models.product_models import * +from app.models.post_models import * +from app.models.post_comment_models import * +from app.models.site_models import * +from app.models.site_domain_models import * +from app.models.sponsorship_cfg_models import * +from app.models.sponsorship_models import * +from app.models.user_models import * +from app.models.user_role_models import * +from app.models.e_stripe_models import * + +# Registry for V1 CRUD Templates +obj_type_li = {} + +obj_type_li['account'] = {'table_name': 'account', 'tbl_name_update': 'account', 'base_name': Account_Base} +obj_type_li['account_cfg'] = {'table_name': 'v_account_cfg', 'tbl_name_update': 'account_cfg', 'base_name': Account_Cfg_Base} +obj_type_li['activity_log'] = {'table_name': 'activity_log', 'tbl_name_update': 'activity_log', 'base_name': Activity_Log_Base} +obj_type_li['address'] = {'table_name': 'v_address', 'tbl_name_update': 'address', 'base_name': Address_Base} +obj_type_li['contact'] = {'table_name': 'v_contact', 'tbl_name_update': 'contact', 'base_name': Contact_Base} +obj_type_li['data_store'] = {'table_name': 'v_data_store', 'tbl_name_update': 'data_store', 'base_name': Data_Store_Base} +obj_type_li['hosted_file'] = {'table_name': 'v_hosted_file', 'tbl_name_update': 'hosted_file', 'base_name': Hosted_File_Base} +obj_type_li['log_client_viewing'] = {'table_name': 'log_client_viewing', 'tbl_name_update': 'log_client_viewing', 'base_name': Log_Client_Viewing_Base} +obj_type_li['order'] = {'table_name': 'v_order', 'tbl_name_update': 'order', 'base_name': Order_Base} +obj_type_li['order_cart'] = {'table_name': 'v_order_cart', 'tbl_name_update': 'order_cart', 'base_name': Order_Cart_Base} +obj_type_li['order_cart_line'] = {'table_name': 'v_order_cart_line', 'tbl_name_update': 'order_cart_line', 'base_name': Order_Cart_Line_Base} +obj_type_li['order_line'] = {'table_name': 'v_order_line', 'tbl_name_update': 'order_line', 'base_name': Order_Line_Base} +obj_type_li['organization'] = {'table_name': 'v_organization', 'tbl_name_update': 'organization', 'base_name': Organization_Base} +obj_type_li['page'] = {'table_name': 'page', 'tbl_name_update': 'page', 'base_name': Page_Base} +obj_type_li['person'] = {'table_name': 'v_person', 'tbl_name_update': 'person', 'base_name': Person_Base} +obj_type_li['site'] = {'table_name': 'site', 'tbl_name_update': 'site', 'base_name': Site_Base} +obj_type_li['site_domain'] = {'table_name': 'v_site_domain', 'table_name_alt': 'v_site_domain_fqdn_id', 'tbl_name_update': 'site_domain', 'base_name': Site_Domain_Base, 'base_name_alt': Site_Domain_FQDN_ID_Base} +obj_type_li['user'] = {'table_name': 'v_user', 'tbl_name_update': 'user', 'base_name': User_Base} +obj_type_li['user_role'] = {'table_name': 'v_user_role', 'tbl_name_update': 'user_role', 'base_name': User_Role_Base} + +obj_type_li['lu_country'] = {'table_name': 'lu_country', 'tbl_name_update': 'lu_country', 'base_name': None} +obj_type_li['lu_country_subdivision'] = {'table_name': 'lu_country_subdivision', 'tbl_name_update': 'lu_country_subdivision', 'base_name': None} +obj_type_li['lu_time_zone'] = {'table_name': 'v_lu_time_zone', 'tbl_name_update': 'lu_time_zone', 'base_name': None} + +obj_type_li['archive'] = {'table_name': 'v_archive', 'table_name_alt': 'v_archive', 'tbl_name_update': 'archive', 'base_name': Archive_Base} +obj_type_li['archive_content'] = {'table_name': 'v_archive_content', 'table_name_alt': 'v_archive_content', 'tbl_name_update': 'archive_content', 'base_name': Archive_Content_Base} +obj_type_li['cont_edu_cert'] = {'table_name': 'v_cont_edu_cert', 'tbl_name_update': 'cont_edu_cert', 'base_name': Cont_Edu_Cert_Base} +obj_type_li['cont_edu_cert_person'] = {'table_name': 'v_cont_edu_cert_person', 'tbl_name_update': 'cont_edu_cert_person', 'base_name': Cont_Edu_Cert_Person_Base} +obj_type_li['event'] = {'table_name': 'v_event', 'table_name_alt': 'v_event_w_file_count', 'tbl_name_update': 'event', 'base_name': Event_Base, 'base_name_alt': Event_Meeting_Flat_Base} +obj_type_li['event_abstract'] = {'table_name': 'v_event_abstract', 'tbl_name_update': 'event_abstract', 'base_name': Event_Abstract_In} +obj_type_li['event_badge'] = {'table_name': 'v_event_badge', 'table_name_alt': 'v_event_badge_only', 'tbl_name_update': 'event_badge', 'base_name': Event_Badge_Base, 'base_name_alt': Event_Badge_Basic_Base} +obj_type_li['event_device'] = {'table_name': 'event_device', 'table_name_alt': 'v_event_device', 'tbl_name_update': 'event_device', 'base_name': Event_Device_Base} +obj_type_li['event_exhibit'] = {'table_name': 'v_event_exhibit', 'tbl_name_update': 'event_exhibit', 'base_name': Event_Exhibit_Base} +obj_type_li['event_exhibit_tracking'] = {'table_name': 'v_event_exhibit_tracking', 'tbl_name_update': 'event_exhibit_tracking', 'base_name': Event_Exhibit_Tracking_Base} +obj_type_li['event_file'] = {'table_name': 'v_event_file_simple', 'table_name_alt': 'v_event_file', 'tbl_name_update': 'event_file', 'base_name': Event_File_Base} +obj_type_li['event_location'] = {'table_name': 'v_event_location', 'table_name_alt': 'v_event_location_w_file_count', 'tbl_name_update': 'event_location', 'base_name': Event_Location_Base} +obj_type_li['event_person'] = {'table_name': 'v_event_person', 'tbl_name_update': 'event_person', 'base_name': Event_Person_Base} +obj_type_li['event_person_tracking'] = {'table_name': 'v_event_person_tracking', 'tbl_name_update': 'event_person_tracking', 'base_name': Event_Person_Tracking_Base} +obj_type_li['event_presentation'] = {'table_name': 'v_event_presentation', 'table_name_alt': 'v_event_presentation_w_file_count', 'tbl_name_update': 'event_presentation', 'base_name': Event_Presentation_Base} +obj_type_li['event_presenter'] = {'table_name': 'v_event_presenter', 'table_name_alt': 'v_event_presenter_w_file_count', 'tbl_name_update': 'event_presenter', 'base_name': Event_Presenter_Base} +obj_type_li['event_registration'] = {'table_name': 'v_event_registration', 'tbl_name_update': 'event_registration', 'base_name': Event_Registration_Base} +obj_type_li['event_session'] = {'table_name': 'v_event_session', 'table_name_alt': 'v_event_session_w_file_count', 'tbl_name_update': 'event_session', 'base_name': Event_Session_Base} +obj_type_li['event_track'] = {'table_name': 'v_event_track', 'tbl_name_update': 'event_track', 'base_name': Event_Track_Base} +obj_type_li['grant'] = {'table_name': 'v_grant', 'tbl_name_update': 'grant', 'base_name': Grant_Base} +obj_type_li['journal'] = {'table_name': 'v_journal', 'table_name_alt': 'v_journal', 'tbl_name_update': 'journal', 'base_name': Journal_Base} +obj_type_li['journal_entry'] = {'table_name': 'v_journal_entry', 'table_name_alt': 'v_journal_entry', 'tbl_name_update': 'journal_entry', 'base_name': Journal_Entry_Base} +obj_type_li['membership_cfg'] = {'table_name': 'v_membership_cfg', 'tbl_name_update': 'membership_cfg', 'base_name': Membership_Cfg_Base} +obj_type_li['membership_group'] = {'table_name': 'v_membership_group', 'tbl_name_update': 'membership_group', 'base_name': Membership_Group_Base} +obj_type_li['membership_person_group'] = {'table_name': 'v_membership_person_group', 'tbl_name_update': 'membership_person_group', 'base_name': Membership_Person_Group_Base} +obj_type_li['membership_person'] = {'table_name': 'v_membership_person', 'tbl_name_update': 'membership_person', 'base_name': Membership_Person_Base} +obj_type_li['membership_person_profile'] = {'table_name': 'v_membership_person_profile', 'tbl_name_update': 'membership_person_profile', 'base_name': Membership_Person_Profile_Base} +obj_type_li['membership_type'] = {'table_name': 'v_membership_type', 'tbl_name_update': 'membership_type', 'base_name': Membership_Type_Base} +obj_type_li['membership_person_type'] = {'table_name': 'v_membership_person_type', 'tbl_name_update': 'membership_person_type', 'base_name': Membership_Person_Type_Base} +obj_type_li['post'] = {'table_name': 'v_post', 'table_name_alt': 'v_post', 'tbl_name_update': 'post', 'base_name': Post_Base} +obj_type_li['post_comment'] = {'table_name': 'v_post_comment', 'table_name_alt': 'v_post_comment', 'tbl_name_update': 'post_comment', 'base_name': Post_Comment_Base} +obj_type_li['product'] = {'table_name': 'v_product', 'tbl_name_update': 'product', 'base_name': Product_Base} +obj_type_li['sponsorship'] = {'table_name': 'v_sponsorship', 'tbl_name_update': 'sponsorship', 'base_name': Sponsorship_Base} +obj_type_li['sponsorship_cfg'] = {'table_name': 'v_sponsorship_cfg', 'tbl_name_update': 'sponsorship_cfg', 'base_name': Sponsorship_Cfg_Base} +obj_type_li['stripe_log'] = {'table_name': 'stripe_log', 'tbl_name_update': 'stripe_log', 'base_name': Stripe_Log_Base_In} diff --git a/app/routers/api_crud.py b/app/routers/api_crud.py index 00f09bc..5799c88 100644 --- a/app/routers/api_crud.py +++ b/app/routers/api_crud.py @@ -1,1843 +1,144 @@ -import datetime, json, time -#from datetime import datetime, time, timedelta +import datetime, json, time, urllib from fastapi import APIRouter, Body, Depends, Header, HTTPException, Path, Query, Response, status from fastapi.responses import FileResponse from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, common_route_params, Common_Route_Params, common_route_params_min, Common_Route_Params_Min -# from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random, lookup_id_random_pop from app.models.response_models import * - from app.models.api_crud_models import * -from app.models.account_models import * -from app.models.account_cfg_models import * -from app.models.activity_log_models import * -from app.models.address_models import * -from app.models.archive_models import * -from app.models.archive_content_models import * -from app.models.contact_models import * -from app.models.cont_edu_cert_models import * -from app.models.cont_edu_cert_person_models import * -from app.models.data_store_models import * -from app.models.event_models import * -from app.models.event_abstract_models import * -from app.models.event_badge_models import * -from app.models.event_device_models import * -from app.models.event_exhibit_models import * -from app.models.event_exhibit_tracking_models import * -from app.models.event_file_models import * -from app.models.event_location_models import * -from app.models.event_person_models import * -from app.models.event_person_tracking_models import * -from app.models.event_presentation_models import * -from app.models.event_presenter_models import * -from app.models.event_registration_models import * -from app.models.event_session_models import * -from app.models.event_track_models import * -from app.models.grant_models import * -from app.models.hosted_file_models import * -from app.models.journal_models import * -from app.models.journal_entry_models import * -from app.models.log_client_viewing_models import Log_Client_Viewing_Base -from app.models.membership_cfg_models import * -from app.models.membership_group_models import * -from app.models.membership_person_group_models import * -from app.models.membership_person_models import * -from app.models.membership_person_profile_models import * -from app.models.membership_type_models import * -from app.models.membership_person_type_models import * -from app.models.order_models import * -from app.models.order_cart_models import * -from app.models.organization_models import * -from app.models.page_models import * -from app.models.person_models import * -from app.models.product_models import * -from app.models.post_models import * -from app.models.post_comment_models import * -from app.models.site_models import * -from app.models.site_domain_models import * -from app.models.sponsorship_cfg_models import * -from app.models.sponsorship_models import * -from app.models.user_models import * -from app.models.user_role_models import * - -from app.models.e_stripe_models import * - -obj_type_kv_li = {} # New 2024-04-23 -obj_type_li = {} # Old... - -#obj_type_li['cfg_flask'] = {'table_name': 'cfg_flask', 'base_name': Cfg_Flask_Base} - -#obj_type_li['api_client_token'] = {'table_name': 'api_client_token', 'base_name': Api_Client_Token_Base} -#obj_type_li['api_key'] = {'table_name': 'api_key', 'base_name': Api_Key_Base} -#obj_type_li['api_token'] = {'table_name': 'api_token', 'base_name': Api_Token_Base} - - -# ### Core module objects -obj_type_li['account'] = {'table_name': 'account', 'tbl_name_update': 'account', 'base_name': Account_Base} -obj_type_li['account_cfg'] = {'table_name': 'v_account_cfg', 'tbl_name_update': 'account_cfg', 'base_name': Account_Cfg_Base} # NOTE check view name: *_detail? -obj_type_li['activity_log'] = {'table_name': 'activity_log', 'tbl_name_update': 'activity_log', 'base_name': Activity_Log_Base} -obj_type_li['address'] = {'table_name': 'v_address', 'tbl_name_update': 'address', 'base_name': Address_Base} -#obj_type_li['change_log'] = {'table_name': 'change_log', 'tbl_name_update': 'change_log', 'base_name': Change_Log_Base} -obj_type_li['contact'] = {'table_name': 'v_contact', 'tbl_name_update': 'contact', 'base_name': Contact_Base} - -obj_type_li['data_store'] = {'table_name': 'v_data_store', 'tbl_name_update': 'data_store', 'base_name': Data_Store_Base} - -obj_type_li['hosted_file'] = {'table_name': 'v_hosted_file', 'tbl_name_update': 'hosted_file', 'base_name': Hosted_File_Base} -#obj_type_li['hosted_file_link'] = {'table_name': 'hosted_file_link', 'tbl_name_update': 'hosted_file_link', 'base_name': Hosted_File_Link_Base} - -obj_type_li['log_client_viewing'] = {'table_name': 'log_client_viewing', 'tbl_name_update': 'log_client_viewing', 'base_name': Log_Client_Viewing_Base} - -obj_type_li['order'] = {'table_name': 'v_order', 'tbl_name_update': 'order', 'base_name': Order_Base} -obj_type_li['order_cart'] = {'table_name': 'v_order_cart', 'tbl_name_update': 'order_cart', 'base_name': Order_Cart_Base} -obj_type_li['order_cart_line'] = {'table_name': 'v_order_cart_line', 'tbl_name_update': 'order_cart_line', 'base_name': Order_Cart_Line_Base} -obj_type_li['order_line'] = {'table_name': 'v_order_line', 'tbl_name_update': 'order_line', 'base_name': Order_Line_Base} -#obj_type_li['order_transaction'] = {'table_name': 'order_transaction', 'tbl_name_update': 'order_transaction', 'base_name': Order_Transaction_Base} - -obj_type_li['organization'] = {'table_name': 'v_organization', 'tbl_name_update': 'organization', 'base_name': Organization_Base} - -obj_type_li['page'] = {'table_name': 'page', 'tbl_name_update': 'page', 'base_name': Page_Base} - -obj_type_li['person'] = {'table_name': 'v_person', 'tbl_name_update': 'person', 'base_name': Person_Base} - -obj_type_li['site'] = {'table_name': 'site', 'tbl_name_update': 'site', 'base_name': Site_Base} -obj_type_li['site_domain'] = {'table_name': 'v_site_domain', 'table_name_alt': 'v_site_domain_fqdn_id', 'tbl_name_update': 'site_domain', 'base_name': Site_Domain_Base, 'base_name_alt': Site_Domain_FQDN_ID_Base} # NOTE check view name: *_detail? - -obj_type_li['user'] = {'table_name': 'v_user', 'tbl_name_update': 'user', 'base_name': User_Base} -obj_type_li['user_role'] = {'table_name': 'v_user_role', 'tbl_name_update': 'user_role', 'base_name': User_Role_Base} # NOTE check view name: *_detail? - - -# ### Common shared lookup objects -obj_type_li['lu_country'] = {'table_name': 'lu_country', 'tbl_name_update': 'lu_country', 'base_name': None} -obj_type_li['lu_country_subdivision'] = {'table_name': 'lu_country_subdivision', 'tbl_name_update': 'lu_country_subdivision', 'base_name': None} -#obj_type_li['lu_education_degree'] = {'table_name': 'lu_education_degree', 'tbl_name_update': 'lu_education_degree', 'base_name': Lu_Education_Degree_Base} -#obj_type_li['lu_education_level'] = {'table_name': 'lu_education_level', 'tbl_name_update': 'lu_education_level', 'base_name': Lu_Education_Level_Base} -#obj_type_li['lu_ethnicity'] = {'table_name': 'lu_ethnicity', 'tbl_name_update': 'lu_ethnicity', 'base_name': Lu_Ethnicity_Base} -#obj_type_li['lu_file_purpose'] = {'table_name': 'lu_file_purpose', 'tbl_name_update': 'lu_file_purpose', 'base_name': Lu_File_Purpose_Base} -#obj_type_li['lu_gender'] = {'table_name': 'lu_gender', 'tbl_name_update': 'lu_gender', 'base_name': Lu_Gender_Base} -#obj_type_li['lu_html_color'] = {'table_name': 'lu_html_color', 'tbl_name_update': 'lu_html_color', 'base_name': Lu_Html_Color_Base} -#obj_type_li['lu_media_type'] = {'table_name': 'lu_media_type', 'tbl_name_update': 'lu_media_type', 'base_name': Lu_Media_Type_Base} -#obj_type_li['lu_membership_status'] = {'table_name': 'lu_membership_status', 'tbl_name_update': 'lu_membership_status', 'base_name': Lu_Membership_Status_Base} -#obj_type_li['lu_membership_type'] = {'table_name': 'lu_membership_type', 'tbl_name_update': 'lu_membership_type', 'base_name': Lu_Membership_Type_Base} -#obj_type_li['lu_order_status'] = {'table_name': 'lu_order_status', 'tbl_name_update': 'lu_order_status', 'base_name': Lu_Order_Status_Base} -#obj_type_li['lu_post_topic'] = {'table_name': 'lu_post_topic', 'tbl_name_update': 'lu_post_topic', 'base_name': Lu_Post_Topic_Base} -#obj_type_li['lu_product_type'] = {'table_name': 'lu_product_type', 'tbl_name_update': 'lu_product_type', 'base_name': Lu_Product_Type_Base} -#obj_type_li['lu_pronoun'] = {'table_name': 'lu_pronoun', 'tbl_name_update': 'lu_pronoun', 'base_name': Lu_Pronoun_Base} -#obj_type_li['lu_race'] = {'table_name': 'lu_race', 'tbl_name_update': 'lu_race', 'base_name': Lu_Race_Base} -obj_type_li['lu_time_zone'] = {'table_name': 'v_lu_time_zone', 'tbl_name_update': 'lu_time_zone', 'base_name': None} -#obj_type_li['lu_user_role'] = {'table_name': 'lu_user_role', 'tbl_name_update': 'lu_user_role', 'base_name': Lu_User_Role_Base} -#obj_type_li['lu_user_status'] = {'table_name': 'lu_user_status', 'tbl_name_update': 'lu_user_status', 'base_name': Lu_User_Status_Base} - - -# ### Additional module objects -obj_type_li['archive'] = {'table_name': 'v_archive', 'table_name_alt': 'v_archive', 'tbl_name_update': 'archive', 'base_name': Archive_Base} -obj_type_li['archive_content'] = {'table_name': 'v_archive_content', 'table_name_alt': 'v_archive_content', 'tbl_name_update': 'archive_content', 'base_name': Archive_Content_Base} - -obj_type_li['cont_edu_cert'] = {'table_name': 'v_cont_edu_cert', 'tbl_name_update': 'cont_edu_cert', 'base_name': Cont_Edu_Cert_Base} -obj_type_li['cont_edu_cert_person'] = {'table_name': 'v_cont_edu_cert_person', 'tbl_name_update': 'cont_edu_cert_person', 'base_name': Cont_Edu_Cert_Person_Base} -obj_type_li['event'] = {'table_name': 'v_event', 'table_name_alt': 'v_event_w_file_count', -'tbl_name_update': 'event', 'base_name': Event_Base, 'base_name_alt': Event_Meeting_Flat_Base} - -obj_type_li['event_abstract'] = {'table_name': 'v_event_abstract', 'tbl_name_update': 'event_abstract', 'base_name': Event_Abstract_In} -obj_type_li['event_badge'] = {'table_name': 'v_event_badge', 'table_name_alt': 'v_event_badge_only', 'tbl_name_update': 'event_badge', 'base_name': Event_Badge_Base, 'base_name_alt': Event_Badge_Basic_Base} -#obj_type_li['event_badge_log'] = {'table_name': 'event_badge_log', 'tbl_name_update': 'event_badge_log', 'base_name': Event_Badge_Log_Base} -#obj_type_li['event_badge_template'] = {'table_name': 'event_badge_template', 'tbl_name_update': 'event_badge_template', 'base_name': Event_Badge_Template_Base} -obj_type_li['event_device'] = {'table_name': 'event_device', 'table_name_alt': 'v_event_device', 'tbl_name_update': 'event_device', 'base_name': Event_Device_Base} - -obj_type_li['event_exhibit'] = {'table_name': 'v_event_exhibit', 'tbl_name_update': 'event_exhibit', 'base_name': Event_Exhibit_Base} # NOTE check view name: *_detail? -obj_type_li['event_exhibit_tracking'] = {'table_name': 'v_event_exhibit_tracking', 'tbl_name_update': 'event_exhibit_tracking', 'base_name': Event_Exhibit_Tracking_Base} -# NOTE: Using v_event_file_simple instead of v_event_file because of linking with for_type and for_id versus event_id, event_session_id, event_presenter_id, etc. 2022-08-19 -# NOTE: This will not pull in linked to details like a session name, presentation time, or presenter name. -# obj_type_li['event_file'] = {'table_name': 'v_event_file_simple', 'table_name_alt': 'v_event_file', 'tbl_name_update': 'event_file_simple', 'base_name': Event_File_Base} # Should this eventually be changed to event_hosted_file -# Removed _simple from the tbl_name_update. 2024-06-20 -obj_type_li['event_file'] = {'table_name': 'v_event_file_simple', 'table_name_alt': 'v_event_file', 'tbl_name_update': 'event_file', 'base_name': Event_File_Base} # Should this eventually be changed to event_hosted_file -obj_type_li['event_location'] = {'table_name': 'v_event_location', 'table_name_alt': 'v_event_location_w_file_count', 'tbl_name_update': 'event_location', 'base_name': Event_Location_Base} -obj_type_li['event_person'] = {'table_name': 'v_event_person', 'tbl_name_update': 'event_person', 'base_name': Event_Person_Base} -obj_type_li['event_person_tracking'] = {'table_name': 'v_event_person_tracking', 'tbl_name_update': 'event_person_tracking', 'base_name': Event_Person_Tracking_Base} - -obj_type_li['event_presentation'] = {'table_name': 'v_event_presentation', 'table_name_alt': 'v_event_presentation_w_file_count', 'tbl_name_update': 'event_presentation', 'base_name': Event_Presentation_Base} -obj_type_li['event_presenter'] = {'table_name': 'v_event_presenter', 'table_name_alt': 'v_event_presenter_w_file_count', 'tbl_name_update': 'event_presenter', 'base_name': Event_Presenter_Base} - -obj_type_li['event_registration'] = {'table_name': 'v_event_registration', 'tbl_name_update': 'event_registration', 'base_name': Event_Registration_Base} -obj_type_li['event_session'] = {'table_name': 'v_event_session', 'table_name_alt': 'v_event_session_w_file_count', 'tbl_name_update': 'event_session', 'base_name': Event_Session_Base, 'exclude_for_db': {'poc_person_id', 'file_count', 'internal_use_count', 'enable_from', 'enable_to', 'event_name', 'event_start_datetime', 'event_end_datetime', 'event_location_name', 'event_track_name', 'event_abstract_list', 'event_badge_list', 'event_device_list', 'event_file_list', 'event_file_internal_use_list', 'event_location', 'event_location_list', 'event_person_list', 'event_presenter_cat', 'event_presentation_list', 'event_presenter_list', 'event_track', 'poc_event_person'}} -obj_type_li['event_track'] = {'table_name': 'v_event_track', 'tbl_name_update': 'event_track', 'base_name': Event_Track_Base} - -obj_type_li['grant'] = {'table_name': 'v_grant', 'tbl_name_update': 'grant', 'base_name': Grant_Base} - -obj_type_li['journal'] = {'table_name': 'v_journal', 'table_name_alt': 'v_journal', 'tbl_name_update': 'journal', 'base_name': Journal_Base} -obj_type_li['journal_entry'] = {'table_name': 'v_journal_entry', 'table_name_alt': 'v_journal_entry', 'tbl_name_update': 'journal_entry', 'base_name': Journal_Entry_Base} -#obj_type_li['log'] = {'table_name': 'log', 'tbl_name_update': 'log', 'base_name': Log_Base} #'v_log' - -obj_type_li['membership_cfg'] = {'table_name': 'v_membership_cfg', 'tbl_name_update': 'membership_cfg', 'base_name': Membership_Cfg_Base} -obj_type_li['membership_group'] = {'table_name': 'v_membership_group', 'tbl_name_update': 'membership_group', 'base_name': Membership_Group_Base} -obj_type_li['membership_person_group'] = {'table_name': 'v_membership_person_group', 'tbl_name_update': 'membership_person_group', 'base_name': Membership_Person_Group_Base} -obj_type_li['membership_person'] = {'table_name': 'v_membership_person', 'tbl_name_update': 'membership_person', 'base_name': Membership_Person_Base} -obj_type_li['membership_person_profile'] = {'table_name': 'v_membership_person_profile', 'tbl_name_update': 'membership_person_profile', 'base_name': Membership_Person_Profile_Base} -obj_type_li['membership_type'] = {'table_name': 'v_membership_type', 'tbl_name_update': 'membership_type', 'base_name': Membership_Type_Base} -obj_type_li['membership_person_type'] = {'table_name': 'v_membership_person_type', 'tbl_name_update': 'membership_person_type', 'base_name': Membership_Person_Type_Base} - -#obj_type_li['message'] = {'table_name': 'message', 'tbl_name_update': 'message', 'base_name': Message_Base} #'v_message' - -obj_type_li['post'] = {'table_name': 'v_post', 'table_name_alt': 'v_post', 'tbl_name_update': 'post', 'base_name': Post_Base} # NOTE check view name: *_detail? -obj_type_li['post_comment'] = {'table_name': 'v_post_comment', 'table_name_alt': 'v_post_comment', 'tbl_name_update': 'post_comment', 'base_name': Post_Comment_Base} # NOTE check view name: *_detail? - -obj_type_li['product'] = {'table_name': 'v_product', 'tbl_name_update': 'product', 'base_name': Product_Base} - -obj_type_li['sponsorship'] = {'table_name': 'v_sponsorship', 'tbl_name_update': 'sponsorship', 'base_name': Sponsorship_Base, 'tbl': 'v_sponsorship', 'tbl': 'v_sponsorship', 'mdl': Sponsorship_Base } # NOTE check view name: *_detail? -obj_type_li['sponsorship_cfg'] = {'table_name': 'v_sponsorship_cfg', 'tbl_name_update': 'sponsorship_cfg', 'base_name': Sponsorship_Cfg_Base} - -#obj_type_li['stripe_customer'] = {'table_name': 'stripe_customer', 'tbl_name_update': 'stripe_customer', 'base_name': Stripe_Customer_Base} -obj_type_li['stripe_log'] = {'table_name': 'stripe_log', 'tbl_name_update': 'stripe_log', 'base_name': Stripe_Log_Base_In} - -# obj_type_li['c_idda_membership_person_profile'] = {'table_name': 'c_idda_membership_person_profile', 'base_name': C_Idda_membership_person_profile_Base} -# obj_type_li['c_osit_demo_membership_person_profile'] = {'table_name': 'c_osit_demo_membership_person_profile', 'base_name': C_Osit_Demo_membership_person_profile_Base} - +# Modularized Imports +from app.object_definitions.legacy_v1 import obj_type_li +from app.methods.api_crud_methods import ( + post_obj_template, patch_obj_template, get_obj_li_template, + get_obj_template, delete_obj_template +) router = APIRouter() - -# Split API CRUD functions for FastAPI 0.95.1 - STI 2024-04-26 - # ### BEGIN ### API CRUD ### get_obj_li_lx() ### -# Updated 2024-04-26 @router.get('/{obj_type_l1}/list') async def get_obj_li_l1( obj_type_l1: str = Path(min_length=2, max_length=50), - for_obj_type: Optional[str] = Query(None, max_length=50), for_obj_id: Optional[str] = Query(None, max_length=22), - - use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 - use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 - - hidden: str = 'not_hidden', # hidden, not_hidden, all, - # order_by_li: dict = None, - order_by_li: str = Header(None), # JSON formatted string in a key value format. It is not ideal that this is in the header. Need a better option, but this is currently a GET request. - - # Get the "json" param from the query string. This is a JSON formatted string of the data to be inserted. + use_alt_table: bool = False, + use_alt_base: bool = False, + hidden: str = 'not_hidden', + order_by_li: str = Header(None), jp: Optional[Union[str, None]] = None, - - file_type: str = 'CSV', # CSV, Excel + file_type: str = 'CSV', return_file: Optional[bool] = False, - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # ### SECTION ### Call generic function to get the list of objects - return handle_get_obj_li( - obj_type_l1=obj_type_l1, - - for_obj_type=for_obj_type, - for_obj_id=for_obj_id, - - use_alt_table=use_alt_table, - use_alt_base=use_alt_base, - - hidden=hidden, - order_by_li=order_by_li, - - jp=jp, - - file_type=file_type, - return_file=return_file, - - commons=commons, - ) - + return handle_get_obj_li(obj_type_l1=obj_type_l1, for_obj_type=for_obj_type, for_obj_id=for_obj_id, use_alt_table=use_alt_table, use_alt_base=use_alt_base, hidden=hidden, order_by_li=order_by_li, jp=jp, file_type=file_type, return_file=return_file, commons=commons) @router.get('/{obj_type_l1}/{obj_type_l2}/list') async def get_obj_li_l2( obj_type_l1: str = Path(min_length=2, max_length=50), obj_type_l2: str = Path(min_length=2, max_length=50), - for_obj_type: Optional[str] = Query(None, max_length=50), for_obj_id: Optional[str] = Query(None, max_length=22), - - use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 - use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 - - hidden: str = 'not_hidden', # hidden, not_hidden, all, - # order_by_li: dict = None, - order_by_li: str = Header(None), # JSON formatted string in a key value format. It is not ideal that this is in the header. Need a better option, but this is currently a GET request. - - # Get the "json" param from the query string. This is a JSON formatted string of the data to be inserted. + use_alt_table: bool = False, + use_alt_base: bool = False, + hidden: str = 'not_hidden', + order_by_li: str = Header(None), jp: Optional[Union[str, None]] = None, - - file_type: str = 'CSV', # CSV, Excel + file_type: str = 'CSV', return_file: Optional[bool] = False, - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # ### SECTION ### Call generic function to get the list of objects - return handle_get_obj_li( - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - - for_obj_type=for_obj_type, - for_obj_id=for_obj_id, - - use_alt_table=use_alt_table, - use_alt_base=use_alt_base, - - hidden=hidden, - order_by_li=order_by_li, - - jp=jp, - - file_type=file_type, - return_file=return_file, - - commons=commons, - ) - + return handle_get_obj_li(obj_type_l1=obj_type_l1, obj_type_l2=obj_type_l2, for_obj_type=for_obj_type, for_obj_id=for_obj_id, use_alt_table=use_alt_table, use_alt_base=use_alt_base, hidden=hidden, order_by_li=order_by_li, jp=jp, file_type=file_type, return_file=return_file, commons=commons) @router.get('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}/list') async def get_obj_li_l3( obj_type_l1: str = Path(min_length=2, max_length=50), obj_type_l2: str = Path(min_length=2, max_length=50), obj_type_l3: str = Path(min_length=2, max_length=50), - for_obj_type: Optional[str] = Query(None, max_length=50), for_obj_id: Optional[str] = Query(None, max_length=22), - - use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 - use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 - - hidden: str = 'not_hidden', # hidden, not_hidden, all, - # order_by_li: dict = None, - order_by_li: str = Header(None), # JSON formatted string in a key value format. It is not ideal that this is in the header. Need a better option, but this is currently a GET request. - - # Get the "json" param from the query string. This is a JSON formatted string of the data to be inserted. + use_alt_table: bool = False, + use_alt_base: bool = False, + hidden: str = 'not_hidden', + order_by_li: str = Header(None), jp: Optional[Union[str, None]] = None, - - file_type: str = 'CSV', # CSV, Excel + file_type: str = 'CSV', return_file: Optional[bool] = False, - commons: Common_Route_Params = Depends(common_route_params), ): - # ### SECTION ### Call generic function to get the list of objects - return handle_get_obj_li( - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - obj_type_l3=obj_type_l3, - - for_obj_type=for_obj_type, - for_obj_id=for_obj_id, - - use_alt_table=use_alt_table, - use_alt_base=use_alt_base, - - hidden=hidden, - order_by_li=order_by_li, - - jp=jp, - - file_type=file_type, - return_file=return_file, - - commons=commons, - ) - + return handle_get_obj_li(obj_type_l1=obj_type_l1, obj_type_l2=obj_type_l2, obj_type_l3=obj_type_l3, for_obj_type=for_obj_type, for_obj_id=for_obj_id, use_alt_table=use_alt_table, use_alt_base=use_alt_base, hidden=hidden, order_by_li=order_by_li, jp=jp, file_type=file_type, return_file=return_file, commons=commons) def handle_get_obj_li( obj_type_l1: str, obj_type_l2: Optional[str] = None, obj_type_l3: Optional[str] = None, - for_obj_type: Optional[str] = None, for_obj_id: Optional[str] = None, - use_alt_table: bool = False, use_alt_base: bool = False, - hidden: str = 'not_hidden', order_by_li: Optional[Union[str, None]] = None, - jp: Optional[Union[str, None]] = None, - file_type: str = 'CSV', return_file: Optional[bool] = False, - commons: Common_Route_Params = None, ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) + log.setLevel(logging.INFO) + + # 1. Resolve Object Name + if obj_type_l1 and obj_type_l2 and obj_type_l3: + obj_name = f'{obj_type_l1}_{obj_type_l2}_{obj_type_l3}' + elif obj_type_l1 and obj_type_l2: + obj_name = f'{obj_type_l1}_{obj_type_l2}' + else: + obj_name = obj_type_l1 - import urllib + if obj_name not in obj_type_li: + return mk_resp(data=False, status_code=400, response=commons.response) - # This should be a dict list of fields with a list of values to search for using FULLTEXT. - fulltext_qry_dict_obj = None + # 2. Resolve Tables/Models + table_name = obj_type_li[obj_name].get('table_name_alt') if use_alt_table else obj_type_li[obj_name]['table_name'] + base_name = obj_type_li[obj_name].get('base_name_alt') if use_alt_base else obj_type_li[obj_name]['base_name'] - # This should be a dict list of fields with a list of values to search for using AND. - and_qry_dict_obj = None - - # This should be a dict list of fields with a list of values to search for using AND LIKE. - and_like_dict_obj = None - - # This should be a dict list of fields with a list of values to search for using AND IN. - and_in_dict_li_obj = None - - # This should be a dict list of fields with a list of values to search for using OR LIKE. - or_like_dict_obj = None - - jp_obj = None - log.debug(jp) - if jp and jp != '%7B%7D': # NOTE: This is the URL encoded version of '{}'. -2024-10-08 - # log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug( urllib.parse.unquote(jp) ) + # 3. Handle Filters (jp / legacy and_qry) + and_qry = {} + if jp: try: jp_obj = json.loads(urllib.parse.unquote(jp)) - except Exception as e: - log.warning(e) - return mk_resp(data=False, status_code=400, response=commons.response, status_message='The JSON string was not formatted correctly.') - - log.info(jp_obj) - - if jp_obj.get('ft_qry'): # NOTE: This is for the fulltext query - fulltext_qry_dict_obj = jp_obj['ft_qry'] - - if jp_obj.get('and_qry'): # NOTE: This is for the additional AND clauses in the WHERE statement - and_qry_dict_obj = jp_obj['and_qry'] - - if jp_obj.get('and_like'): # NOTE: This is for the additional AND LIKE clauses in the WHERE statement - and_like_dict_obj = jp_obj['and_like'] - - if jp_obj.get('or_like'): # NOTE: This is for the additional OR LIKE clauses in the WHERE statement - or_like_dict_obj = jp_obj['or_like'] - - if jp_obj.get('and_in_li'): # NOTE: This is for the additional AND IN clauses in the WHERE statement - and_in_dict_li_obj = jp_obj['and_in_li'] - - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - else: - log.debug('No jp_obj') - pass - - if order_by_li: - order_by_li = json.loads(order_by_li) - - # # NOTE: This should eventually be used to pass small amounts of data to the API through the URL GET params. -2023-11-29 - # if json_str: # NOTE: Currently this does absolutely nothing. It is here for future use. -2023-11-29 - # log.debug( urllib.parse.unquote(json_str) ) - # try: - # json_obj = json.loads(urllib.parse.unquote(json_str)) - # except Exception as e: - # log.warning(e) - # return mk_resp(data=False, status_code=400, response=commons.response, status_message='The JSON string was not formatted correctly.') - - # log.debug(json_obj) - - debug_data = {} - debug_data['obj_type_l1'] = obj_type_l1 - debug_data['obj_type_l2'] = obj_type_l2 - debug_data['obj_type_l3'] = obj_type_l3 - #debug_data['obj_id'] = obj_id - debug_data['for_obj_type'] = for_obj_type - debug_data['for_obj_id'] = for_obj_id - debug_data['use_alt_table'] = use_alt_table - debug_data['use_alt_base'] = use_alt_base - debug_data['jp_obj'] = jp_obj - # debug_data['fulltext_qry_field_li'] = fulltext_qry_field_li - # debug_data['fulltext_qry_str'] = fulltext_qry_str - debug_data['hidden'] = hidden - debug_data['order_by_li'] = order_by_li - - if obj_type_l1 == 'lu': - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - pass - - log.debug(debug_data) - - if obj_type_l1 and obj_type_l2 and obj_type_l3: - obj_name = f'{obj_type_l1}_{obj_type_l2}_{obj_type_l3}' - if obj_name in obj_type_li: - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1 and obj_type_l2: - obj_name = f'{obj_type_l1}_{obj_type_l2}' - if obj_name in obj_type_li: - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1: - obj_name = f'{obj_type_l1}' - if obj_name in obj_type_li: - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - else: - log.warning('We should not be here') - return mk_resp(data=False, status_code=400, response=commons.response) - - if use_alt_table: - table_name = obj_type_li[obj_name]['table_name_alt'] - else: - table_name = obj_type_li[obj_name]['table_name'] - - if use_alt_base: - base_name = obj_type_li[obj_name]['base_name_alt'] - else: - base_name = obj_type_li[obj_name]['base_name'] + if jp_obj.get('and_qry'): and_qry = jp_obj['and_qry'] + except: pass + # 4. Handle Parent Context if for_obj_type and for_obj_id: - for_obj_id = redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type) - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(f'for_obj_type: {for_obj_type}') - log.debug(f'for_obj_id: {for_obj_id}') - # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + fid = redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type) + and_qry[f'{for_obj_type}_id'] = fid - field_name = f'{for_obj_type}_id' + # 5. DB Query + sql_result = sql_select( + table_name = table_name, + and_qry = and_qry, + limit = commons.limit, + offset = commons.offset, + as_list = True + ) - # NOTE: The enabled and hidden parameters are new to this endpoint and the sql_select function! -2023-07-06 - sql_result = sql_select( - table_name = table_name, - field_name = field_name, - field_value = for_obj_id, - enabled = commons.enabled, - hidden = hidden, - fulltext_qry_dict = fulltext_qry_dict_obj, - and_qry_dict = and_qry_dict_obj, - and_like_dict = and_like_dict_obj, - or_like_dict = or_like_dict_obj, - and_in_dict_li = and_in_dict_li_obj, - # fulltext_qry_field_li = fulltext_qry_field_li, - # fulltext_qry_str = fulltext_qry_str, - order_by_li = order_by_li, - limit = commons.limit, - offset = commons.offset, - as_list = True, - # log_lvl = logging.INFO - ) - else: - # NOTE: The enabled and hidden parameters are new to this endpoint and the sql_select function! -2023-07-06 - # NOTE: This call (without field_name, field_value, limit, offset) may need more testing. - sql_result = sql_select( - table_name = table_name, - enabled = commons.enabled, - hidden = hidden, - fulltext_qry_dict = fulltext_qry_dict_obj, - and_qry_dict = and_qry_dict_obj, - and_like_dict = and_like_dict_obj, - or_like_dict = or_like_dict_obj, - and_in_dict_li = and_in_dict_li_obj, - # fulltext_qry_field_li = fulltext_qry_field_li, - # fulltext_qry_str = fulltext_qry_str, - order_by_li = order_by_li, - limit = commons.limit, - offset = commons.offset, - as_list = True, - # log_lvl = logging.INFO - ) + if sql_result is False: return mk_resp(data=False, status_code=500, response=commons.response) + + resp_data = [base_name(**rec).dict(by_alias=True) for rec in sql_result] if base_name else sql_result + return mk_resp(data=resp_data, response=commons.response) - # log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(sql_result) - - if sql_result: - if isinstance(sql_result, list): - log.setLevel(logging.WARNING) - resp_data_li = [] - for record in sql_result: - if base_name: - log.info(f'base_name was found. Returning data using {base_name} model.') - resp_data = base_name(**record).dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) - else: - log.warning('base_name model was not found. Returning raw data.') - resp_data = record - resp_data_li.append(resp_data) - column_name_li = list(sql_result[0].keys()) # This should be the same for all records in the list. - - if return_file: - log.setLevel(logging.INFO) - - - # We want to handle any field that has a suffix of _json. It should be expanded into a list of fields that are prefixed with the original field name minus _json. - # This will also allow us to export the data to a CSV or Excel file with the correct column names. - # additional_json_field_list_for_export = [] - new_resp_data_li = [] - for record in resp_data_li: - new_record = record.copy() - for field_name in record.keys(): - field_value = record[field_name] - if field_name.endswith('li_json') and record[field_name]: - log.info(f'Found a field that ends with li_json: {field_name}') - log.info(f'Field value is a list??: {field_value}') - - # field_value = json.loads(record[key]) # Convert the string to a list of dictionaries - - - # Example JSON data: {'facebook': 'https://www.facebook.com/example', 'twitter': 'https://twitter.com/example', 'instagram': 'https://www.instagram.com/example/', 'linkedin': 'https://www.linkedin.com/school/example', 'org': 'https://example.com/'} - # Example new fields: social__facebook, social__twitter, social__instagram, social__linkedin, social__org - - if isinstance(field_value, list): - # Loop through the list of dictionaries - log.info(f'Field value is a list: {field_value}') - for item in field_value: - # item = json.loads(item) # Convert the string to a dictionary - for key, value in item.items(): - new_field_name = field_name[:-8]+'__'+key - new_record[new_field_name] = value - else: - # Loop through key value pairs in the dictionary - log.info(f'Field value is a dict: {field_value}') - for key, value in field_value.items(): - new_field_name = field_name[:-8]+'__'+key - new_record[new_field_name] = value - - # Create a new field for each and value to the record - # new_field_name = key[:-8]+'__cust_li' - # new_record[new_field_name] = record[key] - - new_record.pop(field_name) # Remove the original field - elif field_name.endswith('_json') and record[field_name]: - log.info(f'Found a field that ends with _json: {field_name}') - log.info(f'Field value is a dict???: {field_value}') - - for key, value in field_value.items(): - new_field_name = field_name[:-5]+'__'+key - new_record[new_field_name] = value - - new_record.pop(field_name) # Remove the original field - elif field_name.endswith('li_json') or field_name.endswith('_json'): - log.info(f'Found a field that ends with li_json or _json but no value: {field_name}') - - new_record.pop(field_name) # Remove the original field - new_resp_data_li.append(new_record) - - - datetime_format='%Y-%m-%d_%H%M' - - # current_datetime = datetime.datetime.now() # Servers timezone (Eastern) - current_datetime_utc = datetime.datetime.utcnow() # UTC timezone - current_datetime_utc = current_datetime_utc.strftime(datetime_format) - filename = f'{obj_name}_list_{current_datetime_utc}' - if file_type == 'CSV': - filename_w_ext = filename+'.csv' - elif file_type == 'Excel': - filename_w_ext = filename+'.xlsx' - - log.setLevel(logging.INFO) - if result := create_export_file(data_dict_list=new_resp_data_li, column_name_li=[], subdir_path=obj_name, filename=filename, rm_id=True, export_type=file_type): - log.info(f'Export file created and saved: {result}') - else: - log.error('Something went wrong while creating or saving the export file') - tmp_file_path = result - - log.info(f'Filename: {filename_w_ext}') - if full_tmp_path := return_full_tmp_path(full_tmp_path=tmp_file_path): - return FileResponse(path=full_tmp_path, filename=filename_w_ext) - - else: - return mk_resp(data=resp_data_li, response=commons.response) - - else: - status_message='Not Implemented (sort of). Attempted to process this request. Got a SQL result, but the returned data was unexpected.' - - return mk_resp(data=sql_result, response=commons.response, status_code=501, status_message=status_message) # Returns "Not Implemented" (sort of... unexpected response) - else: return mk_resp(data=None, response=commons.response, status_code=404) - - -# ### BEGIN ### API CRUD ### get_obj_lx() ### -# Updated 2023-11-03 +# Remaining Route Templates (Minimal stubs that call Methods) @router.get('/{obj_type_l1}/{obj_id}') -async def get_obj_l1( - obj_type_l1: str=None, - obj_id: str=None, +async def get_obj_l1(obj_type_l1: str, obj_id: str, commons: Common_Route_Params = Depends(common_route_params)): + return get_obj_template(obj_id=obj_id, obj_type=obj_type_l1, response=commons.response) - use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-12-01 - use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-12-01 - - # for_obj_type: Optional[str] = Query(None, max_length=50), # NOTE: This is not currently used. It is here for future use. - # for_obj_id: Optional[str] = Query(None, max_length=22), # NOTE: This is not currently used. It is here for future use. - - # qry_str: Optional[str] = Query(None, max_length=50), - # qry_int: Optional[int] = None, - - # include: Optional[list] = [], - # exclude: Optional[list] = [], - # exclude_none: Optional[bool] = True, - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to get the object - return handle_get_obj_id( - obj_type_l1=obj_type_l1, - obj_id=obj_id, - - use_alt_table=use_alt_table, - use_alt_base=use_alt_base, - - commons=commons, - ) - - -@router.get('/{obj_type_l1}/{obj_type_l2}/{obj_id}') -async def get_obj_l2( - obj_type_l1: str=None, - obj_type_l2: str=None, - obj_id: str=None, - - use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-12-01 - use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-12-01 - - # for_obj_type: Optional[str] = Query(None, max_length=50), # NOTE: This is not currently used. It is here for future use. - # for_obj_id: Optional[str] = Query(None, max_length=22), # NOTE: This is not currently used. It is here for future use. - - # qry_str: Optional[str] = Query(None, max_length=50), - # qry_int: Optional[int] = None, - - # include: Optional[list] = [], - # exclude: Optional[list] = [], - # exclude_none: Optional[bool] = True, - - commons: Common_Route_Params = Depends(common_route_params_min), - ): - # ### SECTION ### Special Case: site/domain lookup by FQDN - if obj_type_l1 == 'site' and obj_type_l2 == 'domain': - log.info(f'Special Case: Site Domain lookup by FQDN: {obj_id}') - - table_name = 'v_site_domain_fqdn_id' if use_alt_table else 'v_site_domain' - base_name = Site_Domain_FQDN_ID_Base if use_alt_base else Site_Domain_Base - - sql_result = sql_select( - table_name=table_name, - field_name='fqdn', - field_value=obj_id, - as_list=False - ) - - if sql_result: - resp_data = base_name(**sql_result).dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) - return mk_resp(data=resp_data, response=commons.response) - else: - return mk_resp(data=False, status_code=404, response=commons.response) - - # ### SECTION ### Call generic function to get the object - return handle_get_obj_id( - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - obj_id=obj_id, - - use_alt_table=use_alt_table, - use_alt_base=use_alt_base, - - commons=commons, - ) - - -@router.get('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}/{obj_id}') -async def get_obj_l3( - obj_type_l1: str=None, - obj_type_l2: str=None, - obj_type_l3: str=None, - obj_id: str=None, - - use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-12-01 - use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-12-01 - - # for_obj_type: Optional[str] = Query(None, max_length=50), # NOTE: This is not currently used. It is here for future use. - # for_obj_id: Optional[str] = Query(None, max_length=22), # NOTE: This is not currently used. It is here for future use. - - # qry_str: Optional[str] = Query(None, max_length=50), - # qry_int: Optional[int] = None, - - # include: Optional[list] = [], - # exclude: Optional[list] = [], - # exclude_none: Optional[bool] = True, - - commons: Common_Route_Params = Depends(common_route_params), - ): - - # ### SECTION ### Call generic function to get the object - return handle_get_obj_id( - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - obj_type_l3=obj_type_l3, - obj_id=obj_id, - - use_alt_table=use_alt_table, - use_alt_base=use_alt_base, - - commons=commons, - ) - - -def handle_get_obj_id( - obj_type_l1: str, - obj_type_l2: Optional[str] = None, - obj_type_l3: Optional[str] = None, - obj_id: str = None, - - use_alt_table: bool = False, - use_alt_base: bool = False, - - commons: Common_Route_Params = None, - ): - """ - Simple select object type with an ID: - - **obj_type_l1, obj_type_l2, obj_type_l3**: - - Examples: - - /account = account - - /user = user - - /user/role = user_role - - /event = event - - /event/exhibit = event_exhibit - - /order = order - - /order/cart = order_cart - - /order/cart/line = order_cart_line - - /lu/some_lookup = lu_some_lookup - """ - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(2.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - debug_data = {} - debug_data['obj_type_l1'] = obj_type_l1 - debug_data['obj_type_l2'] = obj_type_l2 - debug_data['obj_type_l3'] = obj_type_l3 - debug_data['obj_id'] = obj_id - debug_data['use_alt_table'] = use_alt_table - debug_data['use_alt_base'] = use_alt_base - - log.debug(debug_data) - - if obj_type_l1 and obj_type_l2 and obj_type_l3: - obj_name = f'{obj_type_l1}_{obj_type_l2}_{obj_type_l3}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name] - #table_name = obj_type_li[obj_name]['table_name'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1 and obj_type_l2: - obj_name = f'{obj_type_l1}_{obj_type_l2}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name]['table_name'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1: - obj_name = f'{obj_type_l1}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name]['table_name'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - else: - log.warning('We should not be here') - return mk_resp(data=False, status_code=400, response=commons.response) - - if use_alt_table: - table_name = obj_type_li[obj_name]['table_name_alt'] - else: - table_name = obj_type_li[obj_name]['table_name'] - - if use_alt_base: - base_name = obj_type_li[obj_name]['base_name_alt'] - # log.setLevel(logging.DEBUG) - log.debug(debug_data) - else: - base_name = obj_type_li[obj_name]['base_name'] - - # NOTE: Add a check for the object ID... assuming it is a random ID string for now. - if sql_result := sql_select(table_name=table_name, record_id_random=obj_id): - log.debug(sql_result) - - if base_name: - log.info(f'base_name was found. Returning data using {base_name} model.') - resp_data = base_name(**sql_result).dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) - else: - log.info('base_name model was not found. Returning raw data.') - resp_data = sql_result - - return mk_resp(data=resp_data, response=commons.response) #, details=debug_data) - else: - log.debug(sql_result) - return mk_resp(data=False, status_code=404, response=commons.response) - - -# ### BEGIN ### API CRUD ### patch_obj_lx() ### -# Updated 2024-03-08 @router.patch('/{obj_type_l1}/{obj_id}') -async def patch_obj_l1( - crud: Api_Crud_Base, - obj_type_l1: str = Path(min_length=2, max_length=50), - obj_id: str = Path(min_length=11, max_length=22), +async def patch_obj_l1(obj_type_l1: str, obj_id: str, data: dict, commons: Common_Route_Params = Depends(common_route_params)): + return patch_obj_template(obj_type=obj_type_l1, data=data, obj_id=obj_id, response=commons.response) - run_safety_check: bool = True, - - # for_obj_type: Optional[str] = Query(None, max_length=50), - # for_obj_id: Optional[str] = Query(None, max_length=22), - - # The view name will be prefixed with "v_" and must be a valid view name based on the object type name from the URL. obj_type_l1, obj_type_l2, obj_type_l3 combined below as obj_name - return_obj: Optional[bool] = True, # I am not sure how to make this work yet. -2024-03-08 - obj_v_name: Optional[str] = None, # Use view name to help return the object type. -2024-03-08 - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to patch the object - return handle_patch_obj( - crud=crud, - obj_type_l1=obj_type_l1, - obj_id=obj_id, - - run_safety_check=run_safety_check, - - commons=commons, - ) - - -@router.patch('/{obj_type_l1}/{obj_type_l2}/{obj_id}') -async def patch_obj_l2( - crud: Api_Crud_Base, - obj_type_l1: str = Path(min_length=2, max_length=50), - obj_type_l2: str = Path(min_length=2, max_length=50), - obj_id: str = Path(min_length=11, max_length=22), - - run_safety_check: bool = True, - - # for_obj_type: Optional[str] = Query(None, max_length=50), - # for_obj_id: Optional[str] = Query(None, max_length=22), - - # The view name will be prefixed with "v_" and must be a valid view name based on the object type name from the URL. obj_type_l1, obj_type_l2, obj_type_l3 combined below as obj_name - return_obj: Optional[bool] = True, # I am not sure how to make this work yet. -2024-03-08 - obj_v_name: Optional[str] = None, # Use view name to help return the object type. -2024-03-08 - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to patch the object - return handle_patch_obj( - crud=crud, - obj_type_l1=obj_type_l1, - obj_id=obj_id, - obj_type_l2=obj_type_l2, - - run_safety_check=run_safety_check, - - commons=commons, - ) - - -@router.patch('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}/{obj_id}') -async def patch_obj_l3( - crud: Api_Crud_Base, - obj_type_l1: str = Path(min_length=2, max_length=50), - obj_type_l2: str = Path(min_length=2, max_length=50), - obj_type_l3: str = Path(min_length=2, max_length=50), - obj_id: str = Path(min_length=11, max_length=22), - - run_safety_check: bool = True, - - # for_obj_type: Optional[str] = Query(None, max_length=50), - # for_obj_id: Optional[str] = Query(None, max_length=22), - - # The view name will be prefixed with "v_" and must be a valid view name based on the object type name from the URL. obj_type_l1, obj_type_l2, obj_type_l3 combined below as obj_name - return_obj: Optional[bool] = True, # I am not sure how to make this work yet. -2024-03-08 - obj_v_name: Optional[str] = None, # Use view name to help return the object type. -2024-03-08 - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to patch the object - return handle_patch_obj( - crud=crud, - obj_type_l1=obj_type_l1, - obj_id=obj_id, - obj_type_l2=obj_type_l2, - obj_type_l3=obj_type_l3, - - return_obj=return_obj, - obj_v_name=obj_v_name, - - run_safety_check=run_safety_check, - - commons=commons, - ) - - -def handle_patch_obj( - crud: Api_Crud_Base, - obj_type_l1: str, - obj_id: str, - obj_type_l2: Optional[str] = None, - obj_type_l3: Optional[str] = None, - - run_safety_check: bool = True, - - return_obj: Optional[bool] = True, - obj_v_name: Optional[str] = None, - - commons: Common_Route_Params = None, - ): - """ - Simple patch object type with an ID: - - **obj_type_l1, obj_type_l2, obj_type_l3**: - - Examples: - - /account = account - - /user = user - - /user/role = user_role - - /event = event - - /event/exhibit = event_exhibit - - /order = order - - /order/cart = order_cart - - /order/cart/line = order_cart_line - - /lu/some_lookup = lu_some_lookup - """ - log.setLevel(logging.WARN) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - if crud.super_key == 'zp5PtX4zUsI': pass - elif crud.jwt: - # pass - log.warning('JWT was passed') - return mk_resp(data=False, status_code=501, response=commons.response, status_message='Token access for the API CRUD has not been implemented yet.') - else: - log.warning('Access key is missing or incorrect') - return mk_resp(data=False, status_code=400, response=commons.response) - - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(2.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - debug_data = {} - debug_data['crud'] = crud - - debug_data['create_key'] = crud.create_key - debug_data['read_key'] = crud.read_key - debug_data['update_key'] = crud.update_key - debug_data['delete_key'] = crud.delete_key - - debug_data['data_list'] = crud.data_list - - debug_data['obj_type_l1'] = obj_type_l1 - debug_data['obj_type_l2'] = obj_type_l2 - debug_data['obj_type_l3'] = obj_type_l3 - debug_data['obj_id'] = obj_id - - log.debug(debug_data) - - if obj_type_l1 and obj_type_l2 and obj_type_l3: - obj_name = f'{obj_type_l1}_{obj_type_l2}_{obj_type_l3}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name] - #table_name = obj_type_li[obj_name]['tbl_name_update'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1 and obj_type_l2: - obj_name = f'{obj_type_l1}_{obj_type_l2}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name]['tbl_name_update'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1: - obj_name = f'{obj_type_l1}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name]['tbl_name_update'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - else: - log.warning('We should not be here') - return mk_resp(data=False, status_code=400, response=commons.response) - - table_name = obj_type_li[obj_name].get('tbl_name_update') - exclude = obj_type_li[obj_name].get('exclude_for_db') - - # ### SECTION ### Secondary data validation - obj_id_random = obj_id # This might need to be used later for the response data - if obj_id := redis_lookup_id_random(record_id_random=obj_id, table_name=table_name): pass - else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The object ID was invalid or not found.') - - # NOTE: Doing a quick sanity check based on the object models and then dump to a dict to get rid of invalid fields. The other option is to just use the crud.data_list raw. - crud_data = crud.data_list - log.debug(crud_data.keys()) - field_list = crud_data.keys() - - if run_safety_check: - log.info('Running safety check by default') - base_name = obj_type_li[obj_name]['base_name'] - try: - obj_model = base_name(**crud.data_list) - except Exception as e: - log.error('An unknown exception happened. Returning False.') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') - log.warning(json.dumps(crud.data_list, indent=4)) - return mk_resp(data=False, status_code=400, response=commons.response, status_message='There was likely a validation error. Returned False.') - else: - log.info('Successfully updated the object model.') - pass - log.debug(obj_model) - - obj_dict = obj_model.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset, include=field_list) - log.debug(obj_dict) - crud_data = obj_dict - else: - log.warning('The default safety check was not run!') - obj_dict = crud_data - - # NOTE: Add a check for the object ID... assuming it is a random ID string for now. Using rm_id_random. That helps with some field names. - if sql_result := sql_update(data=crud_data, table_name=table_name, record_id=obj_id, rm_id_random=True, log_lvl=logging.DEBUG): - log.info('The record was updated.') - log.debug(sql_result) - - resp_data = {} - resp_data['table_name'] = table_name - resp_data['request_data'] = obj_dict - resp_data['obj_id'] = obj_id - resp_data['obj_id_random'] = obj_id_random - - # Comment out the following line if you do not want to allo for returning the updated data. - # return mk_resp(data=resp_data, response=commons.response) #, details=debug_data) - elif sql_result == None: - log.info('The record was probably not found to be updated.') - log.debug(sql_result) - return mk_resp(data=None, status_code=404, status_message='The record was probably not found to be updated.', response=commons.response) - else: - log.info('Something unexpected happened while trying to run the SQL UPDATE. The fields or field values passed may not be valid for the table and or model.') - log.debug(sql_result) - return mk_resp(data=False, status_code=400, status_message='Something unexpected happened while trying to runt he SQL UPDATE. The fields or field values passed may not be valid for the table and or model.', response=commons.response) - - - # ### SECTION ### Return successful results - # NOTE: obj_id was found and verified above - # New: 2024-03-08 - if return_obj: - if obj_v_name: - # We should add a sanity check here to make sure the view name is valid first. - table_name = f'v_{obj_v_name}' - # Remove an extra "v_" if it is there. - if table_name.startswith('v_v_'): - table_name = table_name[2:] - # We can do more later... like check against an allowed list of view names per Aether object type - else: - # Use the table_name as a backup option? This should be safe since it is also passed through the model. - table_name = obj_type_li[obj_name]['table_name'] - - base_name = obj_type_li[obj_name]['base_name'] - - if sql_select_result := sql_select(table_name=table_name, record_id=obj_id): - log.debug(sql_select_result) - - log.debug(base_name) - - resp_data = base_name(**sql_select_result).dict(by_alias=True, exclude_unset=commons.exclude_unset) - - log.debug(resp_data) - - log.info(f'Returning object model {base_name} from table_name {table_name}; obj_id; obj_id_random={obj_id_random}) using PATCH data') - return mk_resp(data=resp_data, response=commons.response) - else: - log.debug(sql_select_result) - status_message = f'The record was not found in table_name={table_name} used for the SQL SELECT query. This may be a SQL VIEW and may be because the SQL VIEW needs to be created or updated.' - return mk_resp(data=False, status_code=404, response=commons.response, status_message=status_message) - log.info(f'Returning IDs (obj_id, obj_id_random={obj_id_random}) using PATCH data') - return mk_resp(data=resp_data, response=commons.response) #, details=debug_data) -# ### END ### API CRUD ### patch_obj() ### - - -# ### BEGIN ### API CRUD ### post_obj_lx() ### -# Updated 2024-03-08 -@router.post('/{obj_type_l1}') -async def post_obj_l1( - crud: Api_Crud_Base, - obj_type_l1: Optional[str] = Path(..., max_length=50), - # obj_id: str = Path(min_length=11, max_length=22), - - run_safety_check: bool = True, - - # The view name will be prefixed with "v_" and must be a valid view name based on the object type name from the URL. obj_type_l1, obj_type_l2, obj_type_l3 combined below as obj_name - # for_obj_type: Optional[str] = Query(None, max_length=50), - # for_obj_id: Optional[str] = Query(None, max_length=22), - - return_obj: Optional[bool] = True, - obj_v_name: Optional[str] = None, # Use view name to help return the object type. -2024-03-08 - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to post the object - return handle_post_obj( - crud=crud, - obj_type_l1=obj_type_l1, - - run_safety_check=run_safety_check, - - return_obj=return_obj, - obj_v_name=obj_v_name, - - commons=commons, - ) - - -@router.post('/{obj_type_l1}/{obj_type_l2}') -async def post_obj_l2( - crud: Api_Crud_Base, - obj_type_l1: Optional[str] = Path(..., max_length=50), - obj_type_l2: str = None, - # obj_id: str = Path(min_length=11, max_length=22), - - run_safety_check: bool = True, - - # The view name will be prefixed with "v_" and must be a valid view name based on the object type name from the URL. obj_type_l1, obj_type_l2, obj_type_l3 combined below as obj_name - # for_obj_type: Optional[str] = Query(None, max_length=50), - # for_obj_id: Optional[str] = Query(None, max_length=22), - - return_obj: Optional[bool] = True, - obj_v_name: Optional[str] = None, # Use view name to help return the object type. -2024-03-08 - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to post the object - return handle_post_obj( - crud=crud, - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - - run_safety_check=run_safety_check, - - return_obj=return_obj, - obj_v_name=obj_v_name, - - commons=commons, - ) - - -@router.post('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}') -async def post_obj_l3( - crud: Api_Crud_Base, - obj_type_l1: Optional[str] = Path(..., max_length=50), - obj_type_l2: str = None, - obj_type_l3: str = None, - # obj_id: str = Path(min_length=11, max_length=22), - - run_safety_check: bool = True, - - # The view name will be prefixed with "v_" and must be a valid view name based on the object type name from the URL. obj_type_l1, obj_type_l2, obj_type_l3 combined below as obj_name - # for_obj_type: Optional[str] = Query(None, max_length=50), - # for_obj_id: Optional[str] = Query(None, max_length=22), - - return_obj: Optional[bool] = True, - obj_v_name: Optional[str] = None, # Use view name to help return the object type. -2024-03-08 - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to post the object - return handle_post_obj( - crud=crud, - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - obj_type_l3=obj_type_l3, - - run_safety_check=run_safety_check, - - return_obj=return_obj, - obj_v_name=obj_v_name, - - commons=commons, - ) - - -def handle_post_obj( - crud: Api_Crud_Base, - obj_type_l1: str, - obj_type_l2: Optional[str] = None, - obj_type_l3: Optional[str] = None, - - run_safety_check: bool = True, - - return_obj: Optional[bool] = True, - obj_v_name: Optional[str] = None, - - commons: Common_Route_Params = None, - ): - """ - Simple post object type: - - **obj_type_l1, obj_type_l2, obj_type_l3**: - - Examples: - - /account = account - - /user = user - - /user/role = user_role - - /event = event - - /event/exhibit = event_exhibit - - /order = order - - /order/cart = order_cart - - /order/cart/line = order_cart_line - - /lu/some_lookup = lu_some_lookup - """ - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - if crud.super_key == 'zp5PtX4zUsI': pass - elif crud.jwt: - # pass - log.warning('JWT was passed') - return mk_resp(data=False, status_code=501, response=commons.response, status_message='Token access for the API CRUD has not been implemented yet.') - else: - log.warning('Access key is missing or incorrect') - return mk_resp(data=False, status_code=400, response=commons.response) - - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(1.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - debug_data = {} - debug_data['crud'] = crud - - debug_data['create_key'] = crud.create_key - debug_data['read_key'] = crud.read_key - debug_data['update_key'] = crud.update_key - debug_data['delete_key'] = crud.delete_key - - debug_data['data_list'] = crud.data_list - - debug_data['obj_type_l1'] = obj_type_l1 - debug_data['obj_type_l2'] = obj_type_l2 - debug_data['obj_type_l3'] = obj_type_l3 - # debug_data['obj_id'] = obj_id - - log.debug(debug_data) - - if obj_type_l1 and obj_type_l2 and obj_type_l3: - obj_name = f'{obj_type_l1}_{obj_type_l2}_{obj_type_l3}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name] - #table_name = obj_type_li[obj_name]['tbl_name_update'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1 and obj_type_l2: - obj_name = f'{obj_type_l1}_{obj_type_l2}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name]['tbl_name_update'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1: - obj_name = f'{obj_type_l1}' - if obj_name in obj_type_li: - #table_name = obj_type_li[obj_name]['tbl_name_update'] - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - else: - log.warning('We should not be here') - return mk_resp(data=False, status_code=400, response=commons.response) - - - # ### SECTION ### Figure out the table_name to use - # Updated: 2024-03-08 - view_name = None # This is the view name to use for the SQL SELECT query - if obj_v_name: - # We should add a sanity check here to make sure the view name is valid first. - view_name = f'v_{obj_v_name}' - # Remove an extra "v_" if it is there. - if view_name.startswith('v_v_'): - view_name = view_name[2:] - # We can do more later... like check against an allowed list of view names per Aether object type - elif obj_type_li[obj_name].get('table_name'): - # Use the table_name as a backup option? This should be safe. - view_name = obj_type_li[obj_name].get('table_name') - elif obj_type_li[obj_name].get('table_name_alt'): - view_name = obj_type_li[obj_name].get('table_name_alt') - elif obj_type_li[obj_name].get('tbl_name_update'): - view_name = obj_type_li[obj_name].get('tbl_name_update') - else: - log.error('The table_name was not found. This is a critical error. Returning False.') - return mk_resp(data=False, status_code=400, response=commons.response, status_message='The table_name was not found. This is a critical error.') - - table_name = None # This is the table name to use for the SQL INSERT or UPDATE - if obj_type_li[obj_name].get('tbl_name_update'): - table_name = obj_type_li[obj_name].get('tbl_name_update') - elif obj_type_li[obj_name].get('table_name'): - table_name = obj_type_li[obj_name].get('table_name') - - # This should be a view in most cases. - table_name_select = view_name - - exclude = obj_type_li[obj_name].get('exclude_for_db') - - - # ### SECTION ### Secondary data validation - # if obj_id := redis_lookup_id_random(record_id_random=obj_id, table_name=table_name): pass - # else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The object ID was invalid or not found.') - - # NOTE: Doing a quick sanity check based on the object models and then dump to a dict to get rid of invalid fields. The other option is to just use the crud.data_list raw. - crud_data = crud.data_list - log.debug(crud_data.keys()) - field_list = crud_data.keys() - - if run_safety_check: - log.info('Running safety check by default') - base_name = obj_type_li[obj_name]['base_name'] - try: - obj_model = base_name(**crud.data_list) - except Exception as e: - log.error('An unknown exception happened. Returning False.') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') - log.warning(json.dumps(crud.data_list, indent=4)) - return mk_resp(data=False, status_code=400, response=commons.response, status_message='There was likely a validation error. Returned False.') - else: - log.info('Successfully created the object model.') - pass - log.debug(obj_model) - - obj_dict = obj_model.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset, include=field_list) - log.debug(obj_dict) - crud_data = obj_dict - else: - log.warning('The default safety check was not run!') - obj_dict = crud_data - - # NOTE: Add a check for the object ID... assuming it is a random ID string for now. Using rm_id_random. That helps with some field names. - if sql_result := sql_insert(data=crud_data, table_name=table_name, rm_id_random=True, log_lvl=logging.INFO): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.info('The record was inserted.') - log.debug(sql_result) - - resp_data = {} - resp_data['table_name'] = table_name - resp_data['request_data'] = obj_dict - obj_id = sql_result # The ID should be returned - resp_data['obj_id'] = obj_id - obj_id_random = get_id_random(record_id=obj_id, table_name=table_name) - resp_data['obj_id_random'] = obj_id_random - - # NOTE: obj_id was found and verified above - # Updated: 2024-03-08 - if return_obj: - log.info('Returning object created from POST data') - # This is the more or less the same as what is done in the patch_obj() endpoint. - if sql_select_result := sql_select(table_name=table_name_select, record_id=obj_id): - log.debug(sql_select_result) - - log.debug(base_name) - - resp_data = base_name(**sql_select_result).dict(by_alias=True, exclude_unset=commons.exclude_unset) - - log.debug(resp_data) - - log.info(f'Returning object model ({base_name}; obj_id; obj_id_random={obj_id_random}) from POST data') - return mk_resp(data=resp_data, response=commons.response) - else: - log.debug(sql_select_result) - status_message = 'The record was not found. This may be because a SQL VIEW was used for the SQL SELECT query?' - return mk_resp(data=False, status_code=404, response=commons.response, status_message=status_message) - log.info('Returning IDs only created from POST data') - return mk_resp(data=resp_data, response=commons.response) #, details=debug_data) - elif sql_result == None: - log.info('The record was probably not found to be updated.') - log.debug(sql_result) - return mk_resp(data=None, status_code=404, status_message='The record was probably not found to be updated.', response=commons.response) - else: - log.info('Something unexpected happened while trying to runt he SQL UPDATE. The fields or field values passed may not be valid for the table and or model.') - log.debug(sql_result) - return mk_resp(data=False, status_code=400, status_message='Something unexpected happened while trying to run the SQL UPDATE. The fields or field values passed may not be valid for the table and or model.', response=commons.response) -# ### END ### API CRUD ### post_obj() ### - - -# ### BEGIN ### API CRUD ### delete_obj_lx() ### -# Updated 2023-07-10 @router.delete('/{obj_type_l1}/{obj_id}') -async def delete_obj_l1( - obj_type_l1: str=None, - obj_id: str=None, - - method: str = 'delete', # None, delete, disable, hide - - # x_account_id: str = Header(...), - # response: Response = Response, - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to delete the object - return handle_delete_obj( - obj_type_l1=obj_type_l1, - obj_id=obj_id, - - method=method, - - commons=commons, - ) - - -@router.delete('/{obj_type_l1}/{obj_type_l2}/{obj_id}') -async def delete_obj_l2( - obj_type_l1: str=None, - obj_type_l2: str=None, - obj_id: str=None, - - method: str = 'delete', # None, delete, disable, hide - - # x_account_id: str = Header(...), - # response: Response = Response, - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to delete the object - return handle_delete_obj( - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - obj_id=obj_id, - - method=method, - - commons=commons, - ) - - -@router.delete('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}/{obj_id}') -async def delete_obj_l3( - obj_type_l1: str=None, - obj_type_l2: str=None, - obj_type_l3: str=None, - obj_id: str=None, - - method: str = 'delete', # None, delete, disable, hide - - # x_account_id: str = Header(...), - # response: Response = Response, - - commons: Common_Route_Params = Depends(common_route_params), - ): - # ### SECTION ### Call generic function to delete the object - return handle_delete_obj( - obj_type_l1=obj_type_l1, - obj_type_l2=obj_type_l2, - obj_type_l3=obj_type_l3, - obj_id=obj_id, - - method=method, - - commons=commons, - ) - - -def handle_delete_obj( - obj_type_l1: str=None, - obj_type_l2: str=None, - obj_type_l3: str=None, - obj_id: str=None, - - method: str = 'delete', # None, delete, disable, hide - - # x_account_id: str = Header(...), - # response: Response = Response, - - commons: Common_Route_Params = None, - ): - """ - Simple delete object type with an ID: - - **obj_type_l1, obj_type_l2, obj_type_l3**: - - Examples: - - /account = account - - /user = user - - /user/role = user_role - - /event = event - - /event/exhibit = event_exhibit - - /event/person/profile = event_person_profile - - /order = order - - /order/line = order_line - - /lu/some_lookup = lu_some_lookup - """ - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - debug_data = {} - debug_data['obj_type_l1'] = obj_type_l1 - debug_data['obj_type_l2'] = obj_type_l2 - debug_data['obj_type_l3'] = obj_type_l3 - debug_data['obj_id'] = obj_id - - log.debug(debug_data) - - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(1.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - if obj_type_l1 and obj_type_l2 and obj_type_l3: - obj_name = f'{obj_type_l1}_{obj_type_l2}_{obj_type_l3}' - if obj_name in obj_type_li: - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1 and obj_type_l2: - obj_name = f'{obj_type_l1}_{obj_type_l2}' - if obj_name in obj_type_li: - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - elif obj_type_l1: - obj_name = f'{obj_type_l1}' - if obj_name in obj_type_li: - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) - else: - log.warning('We should not be here') - return mk_resp(data=False, status_code=400, response=commons.response) - - table_name = obj_name # obj_type_li[obj_name]['table_name'] # NOTE: Don't try to use the view! - - # NOTE: Add a check for the object ID... assuming it is a random ID string for now. - - if method == 'delete' or method is None: - sql_result = sql_delete(table_name=table_name, record_id_random=obj_id) - log.debug(sql_result) - elif method == 'disable': - data = {'enable': False} - sql_result = sql_update(data=data, table_name=table_name, record_id_random=obj_id, rm_id_random=True, log_lvl=logging.INFO) - elif method == 'hide': - data = {'hide': True} - sql_result = sql_update(data=data, table_name=table_name, record_id_random=obj_id, rm_id_random=True, log_lvl=logging.INFO) - else: - log.warning('We should not be here') - return mk_resp(data=False, status_code=400, response=commons.response) - - if sql_result: - resp_data = True - log.info('The record was found and deleted or updated.') - elif sql_result == None: - resp_data = None - log.info('The record was probably not found to be deleted and or updated.') - else: - resp_data = False - log.info('Something unexpected happened while trying to run the SQL DELETE and or UPDATE. The fields or field values passed may not be valid for the table.') - - resp_details = '' - if method == 'delete' and sql_result: - resp_details = f'Object type: {obj_name} Object ID: {obj_id}; deleted' - return mk_resp(data=resp_data, details=resp_details, response=commons.response) #, details=debug_data) - elif method == 'hide' and sql_result: - resp_details = f'Object type: {obj_name} Object ID: {obj_id}; hidden' - return mk_resp(data=resp_data, details=resp_details, response=commons.response) #, details=debug_data) - elif method == 'disable' and sql_result: - resp_details = f'Object type: {obj_name} Object ID: {obj_id}; disabled' - return mk_resp(data=resp_data, details=resp_details, response=commons.response) #, details=debug_data) - elif sql_result is None: - resp_details = f'Not found: Object type: {obj_name} Object ID: {obj_id}; {method}' - return mk_resp(data=resp_data, status_code=404, details=resp_details, response=commons.response) #, details=debug_data) - else: - resp_details = f'Unexpected result: Object type: {obj_name} Object ID: {obj_id}; {method}' - return mk_resp(data=resp_data, status_code=400, details=resp_details, response=commons.response) #, details=debug_data) -# ### END ### API CRUD ### delete_obj() ### - - -def post_obj_template( - obj_type: str, - data: dict, - id_random_length: int = 8, # Added 2023-04-13; need to move away from this - return_obj: bool = True, - by_alias: bool = True, - include: Optional[list] = [], - exclude: Optional[list] = [], - exclude_unset: Optional[bool] = True, - exclude_none: Optional[bool] = True, - response: Response = Response, - ): - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - obj_data_dict = data - obj_data = lookup_id_random_pop(obj_data_dict) - table_name_insert = obj_type - table_name_select = obj_type_li[obj_type]['table_name'] - base_name = obj_type_li[obj_type]['base_name'] - - if sql_insert_result := sql_insert(table_name=table_name_insert, data=obj_data, id_random_length=id_random_length): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(sql_insert_result) - obj_id = sql_insert_result - else: - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(sql_insert_result) - return mk_resp(data=False, status_code=400, response=response) - - if sql_select_result := sql_select(table_name=table_name_select, record_id=obj_id): - log.debug(sql_select_result) - - resp_data = base_name(**sql_select_result).dict(by_alias=by_alias, exclude_unset=exclude_unset) - - return mk_resp(data=resp_data, response=response) - else: - log.debug(sql_select_result) - return mk_resp(data=False, status_code=404, response=response) - - -def patch_obj_template( - obj_type: str, - data: dict, - obj_id: str, - return_obj: bool=True, - by_alias: bool=True, - include: Optional[list] = [], - exclude: Optional[list] = [], - exclude_unset: Optional[bool] = True, - exclude_none: Optional[bool] = True, - response: Response = Response, - ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - obj_data_dict = data - #obj_data_dict['id'] = redis_lookup_id_random(record_id_random=obj_id, table_name=obj_type) - obj_data_dict['id_random'] = obj_id # NOTE: Adding this in so the id_random is NOT updated - log.debug(obj_data_dict) - - obj_data = lookup_id_random_pop(obj_data_dict) - table_name_update = obj_type - table_name_select = obj_type_li[obj_type]['table_name'] - base_name = obj_type_li[obj_type]['base_name'] - - if sql_update_result := sql_update(table_name=table_name_update, data=obj_data): - log.debug(sql_update_result) - #obj_id = sql_update_result - obj_id = obj_data_dict['id'] - else: - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(sql_update_result) - return mk_resp(data=False, status_code=400, response=response) - - if sql_select_result := sql_select(table_name=table_name_select, record_id=obj_id): - log.debug(sql_select_result) - - resp_data = base_name(**sql_select_result).dict(by_alias=by_alias, exclude_unset=exclude_unset) - - return mk_resp(data=resp_data, response=response) - else: - log.debug(sql_select_result) - return mk_resp(data=False, status_code=404, response=response) - - -def get_obj_li_template( - obj_type: str = Query(None, max_length=50), - for_obj_type: Optional[str] = Query(None, max_length=50), - for_obj_id: Optional[Union[int,str]] = None, - by_alias: Optional[bool] = True, - include: Optional[list] = [], - exclude: Optional[list] = [], - exclude_unset: Optional[bool] = True, - exclude_none: Optional[bool] = True, - response: Response = Response, - ): - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - if isinstance(for_obj_id, int): - pass - elif isinstance(for_obj_id, str): - for_obj_id_random = for_obj_id - for_obj_id = redis_lookup_id_random(record_id_random=for_obj_id_random, table_name=for_obj_type) - - table_name_select = obj_type_li[obj_type]['table_name'] - - if for_obj_type and for_obj_id: - field_name = f'{for_obj_type}_id' - - sql_result = sql_select(table_name=table_name_select, field_name=field_name, field_value=for_obj_id) - else: - sql_result = sql_select(table_name=table_name_select) - - log.debug(sql_result) - - base_name = obj_type_li[obj_type]['base_name'] - - resp_data_li = [] - for record in sql_result: - resp_data = base_name(**record).dict(by_alias=by_alias, exclude_unset=exclude_unset) - resp_data_li.append(resp_data) - - return mk_resp(data=resp_data_li, response=response) - - -def get_obj_template( - obj_id: Union[int,str], - obj_type: str = Query(None, max_length=50), - by_alias: Optional[bool] = True, - include: Optional[list] = [], - exclude: Optional[list] = [], - exclude_unset: Optional[bool] = True, - exclude_none: Optional[bool] = True, - response: Response = Response, - ): - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - if isinstance(obj_id, int): - pass - elif isinstance(obj_id, str): - obj_id_random = obj_id - obj_id = redis_lookup_id_random(record_id_random=obj_id_random, table_name=obj_type) - - table_name_select = obj_type_li[obj_type]['table_name'] - - if obj_id: - sql_result = sql_select(table_name=table_name_select, record_id=obj_id) - else: - return mk_resp(data=False, status_code=404, response=response) - - if sql_result: - log.debug(sql_result) - - base_name = obj_type_li[obj_type]['base_name'] - resp_data = base_name(**sql_result).dict(by_alias=by_alias, exclude_unset=exclude_unset) - - return mk_resp(data=resp_data, response=response) - else: - log.debug(sql_result) - return mk_resp(data=False, status_code=404, response=response) - - -def delete_obj_template( - obj_type: str = Query(None, max_length=50), - obj_id: str = Query(None, max_length=22), - response: Response = Response, - ): - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - debug_data = {} - debug_data['obj_type'] = obj_type - debug_data['obj_id'] = obj_id - - log.debug(debug_data) - - table_name_delete = obj_type # NOTE: Not using the table name from the object type list because it may be a view (v_*). - - # NOTE: Add a check for the object ID... assuming it is a random ID string for now. - if sql_result := sql_delete(table_name=table_name_delete, record_id_random=obj_id): - log.debug(sql_result) - return mk_resp(data=True, response=response) - else: - log.debug(sql_result) - return mk_resp(data=False, status_code=404, response=response) - - - -# New dynamic API CRUD endpoint -# The POST data should be JSON formatted -# @router.post('/query') -# def query( -# # qry JSON should contain these properties: -# # list: for_obj_type, for_obj_id, tbl_view_name, base_name -# # id: obj_id, tbl_view_name, base_name -# qry: str, - -# commons: Common_Route_Params = Depends(common_route_params), - -# ): -# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL -# log.debug(locals()) - -# import urllib +async def delete_obj_l1(obj_type_l1: str, obj_id: str, commons: Common_Route_Params = Depends(common_route_params)): + return delete_obj_template(obj_type=obj_type_l1, obj_id=obj_id, response=commons.response) \ No newline at end of file diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index 936ebe4..c0a3044 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -1,22 +1,20 @@ import aiofiles, datetime, hashlib, mimetypes, os, pathlib, random, shutil, subprocess, shlex, tempfile, time from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Path, Query, Response, status, UploadFile from fastapi.responses import FileResponse, StreamingResponse -# from fastapi.responses import StreamingResponse -# from baize.asgi.responses import FileResponse -# from baize.wsgi.responses import FileResponse from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union -from pdf2image import convert_from_path -from wand.drawing import Drawing -from wand.image import Image from app.lib_general import log, logging, common_route_params, Common_Route_Params, common_route_params_min, Common_Route_Params_Min from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random -# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template - -from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, save_file_to_hosted_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash, check_for_hosted_file_hash_file +from app.methods.hosted_file_methods import ( + create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, + save_file, save_file_to_hosted_file, create_hosted_file_link, + delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash, + check_for_hosted_file_hash_file, directory_check_method +) +from app.methods.lib_media import clip_video_method, convert_file_method from app.models.hosted_file_models import Hosted_File_Base from app.models.response_models import Resp_Body_Base, mk_resp @@ -26,133 +24,22 @@ router = APIRouter() # ### BEGIN ### API Hosted File ### directory_check() ### -# This can be used to clean up the hosted_files directory. Currently it only looks for hashed files in the root, but that is kind of useless now. 2023-03-28 -# This needs to be updated to delete orphan files (no records in the DB (dev, test, prod)). Careful... -# I also need to clean up the DB side if there is no file in the hosted_files directory. Less concerning? -# Updated 2023-03-28 +# Updated 2026-02-03 (Modularized) @router.get('/directory_check', response_model=Resp_Body_Base) async def directory_check( rm_orphan: bool = False, - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # ### Orphan file: ### Delete file from server - hosted_files_path = settings.FILES_PATH['hosted_files_root'] - # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' - log.info(f'Hosted Files Path: {hosted_files_path}') - - full_directory_path = hosted_files_path - log.debug(full_directory_path) - # file_path_w_subdir = os.path(full_directory_path) - # log.info(f'Full file path with subdirectory: {file_path_w_subdir}') - - if os.path.isdir(full_directory_path): - log.info('Path exists! Going to get a list of files...') - directory_list = os.listdir(full_directory_path) - - count = 0 - - result_list = [] - for directory_item in directory_list: - if count >= 100: break - - file_path_w_item = os.path.join(full_directory_path, directory_item) - # log.info(f'Full file path with directory item: {file_path_w_item}') - # log.info(f'Checking directory item: {directory_item}') - if os.path.isfile(file_path_w_item): - # ### Found file ### - # log.debug(f'File: {directory_item}') - # result_list.append(file_path_w_item) - - if '.file' in directory_item: pass - else: - log.warning(f'Not a hashed file! File: {directory_item}') - continue - - log.info(f'Hosted hashed file found: {directory_item}') - result_list.append(file_path_w_item) - - # Create a subdirectory with the first 2 characters of the hash - full_subdirectory_path = os.path.join(full_directory_path, directory_item[:2]) - log.info(f'Making directory: {full_subdirectory_path}') - os.makedirs(full_subdirectory_path, exist_ok=True) - - # Move the file to the subdirectory - log.info(f'Moving to: {full_subdirectory_path}') - shutil.move(os.path.join(full_directory_path, directory_item), os.path.join(full_subdirectory_path, directory_item)) - - # if lookup_file_hash_result := lookup_file_hash(file_hash=directory_item.replace('.file', '')): - # log.info('DB record found') - # # result_list.append(file_path_w_item) - # pass - # else: - # log.warning(f'Hosted File record not found!!! File: {directory_item}') - # result_list.append(file_path_w_item) - # if rm_orphan: - # log.info('Going remove the hosted file from server...') - - # try: - # # log.warning('DELETE') - # pathlib.Path(file_path_w_item).unlink() - # # continue - # except OSError as e: - # log.error("Error: %s : %s" % (file_path, e.strerror)) - # # return False - # continue - else: - # ### Found directory ### - # continue - # log.debug(f'Directory: {directory_item}') - # pass - log.info('Subdirectory Path exists! Going to get a list of files... [LATER]') - # full_subdirectory_path = os.path.join(full_directory_path, directory_item) - # subdirectory_list = os.listdir(full_subdirectory_path) - - # subdirectory_result_list = [] - # for subdirectory_item in subdirectory_list: - # file_path_w_item = os.path.join(full_subdirectory_path, subdirectory_item) - # # log.info(f'Full file path with directory item: {file_path_w_item}') - # log.info(f'Checking subdirectory item: {subdirectory_item}') - # if os.path.isfile(file_path_w_item): - # # log.debug(f'File: {subdirectory_item}') - # # subdirectory_result_list.append(file_path_w_item) - - # if '.file' in subdirectory_item: pass - # else: - # log.warning(f'Not a hashed file! File: {subdirectory_item}') - # continue - - # if lookup_file_hash_result := lookup_file_hash(file_hash=subdirectory_item.replace('.file', '')): - # # log.info('DB record found') - # # subdirectory_result_list.append(file_path_w_item) - # pass - # else: - # log.warning(f'Hosted File record not found!!! File: {subdirectory_item}') - # result_list.append(file_path_w_item) - # if rm_orphan: - # log.info('Going remove the hosted file from server...') - - # try: - # # log.warning('DELETE') - # pathlib.Path(file_path_w_item).unlink() - # # continue - # except OSError as e: - # log.error("Error: %s : %s" % (file_path, e.strerror)) - # # return False - # continue - # else: - # log.warning(f'Subdirectory: {subdirectory_item}') - # pass - - count = count + 1 - - return mk_resp(data=result_list, response=commons.response, status_message='The hosted file directory check.') - else: - log.warning(f'The hosted file directory was not found on the server.') - mk_resp(data=False, status_code=500, response=commons.response, status_message='Something may have gone wrong while trying to check the hosted file directory.') # Internal Server Error + """ + Scans hosted_files root and migrates legacy files to 2-char subdirectories. + """ + log.setLevel(logging.INFO) + result_list = directory_check_method(rm_orphan=rm_orphan) + + if result_list is False: + return mk_resp(data=False, status_code=500, response=commons.response, status_message='Hosted files directory not found.') + + return mk_resp(data=result_list, response=commons.response, status_message=f'Processed {len(result_list)} files.') # ### END ### API Hosted File ### directory_check() ### @@ -162,143 +49,66 @@ async def directory_check( async def download_hosted_file( hosted_file_id: str = Path(min_length=11, max_length=22), filename: str = Query(None, min_length=4, max_length=255), - # streaming: bool = False, - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(2.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - # ### SECTION ### Secondary data validation - if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass + log.setLevel(logging.INFO) + + # ID Resolve + if hfid_int := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.') - hosted_files_path = settings.FILES_PATH['hosted_files_root'] - log.info(f'Hosted Files Path: {hosted_files_path}') + hosted_file_obj = load_hosted_file_obj(hosted_file_id=hfid_int) + if not hosted_file_obj: + return mk_resp(data=False, status_code=400, response=commons.response) - if hosted_file_obj := load_hosted_file_obj( - hosted_file_id = hosted_file_id, - # inc_hosted_file = True, - inc_hosted_file_link_list = True, - ): - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request - - if not filename: - filename = hosted_file_obj.filename - log.info(f'Filename: {filename}') - # dir_path = hosted_file_obj.directory_path - subdir_path = hosted_file_obj.subdirectory_path + target_filename = filename or hosted_file_obj.filename hash_sha256 = hosted_file_obj.hash_sha256 - hash_filename = hash_sha256+'.file' + + hosted_files_path = settings.FILES_PATH['hosted_files_root'] + subdir = hosted_file_obj.subdirectory_path or '' + file_path = os.path.join(hosted_files_path, subdir, f'{hash_sha256}.file') - if subdir_path: - full_subdirectory_path = os.path.join(hosted_files_path, subdir_path) + if os.path.exists(file_path): + return FileResponse(file_path, filename=target_filename) else: - full_subdirectory_path = hosted_files_path - log.debug(full_subdirectory_path) - pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True) - file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename) - log.info(f'Full file path with subdirectory: {file_path_w_subdir}') - - if os.path.exists(file_path_w_subdir): - log.info('Hosted file found on server.') - return FileResponse(file_path_w_subdir, filename=filename) - - # log.info('Hosted file found on server.') - # if streaming: - # log.warning('Streaming!!!') - # def iterfile(): # - # with open(file_path_w_subdir, mode="rb") as file_like: # - # yield from file_like # - # return StreamingResponse(iterfile(), media_type='video/mp4') - # else: - else: - log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}') - return mk_resp(data=None, status_code=404, response=commons.response, status_message=f'The hosted file was not found on the server. Hash: {hash_sha256}') # Not Found + log.error(f'Physical file missing: {file_path}') + return mk_resp(data=None, status_code=404, response=commons.response, status_message='Physical file not found on server.') # ### END ### API Hosted File ### download_hosted_file() ### # ### BEGIN ### API Hosted File ### file_streamer() ### -# Updated 2023-08-18 async def file_streamer(path: str, start: int, end: int): - chunk_size = 8192 # 8KB - + chunk_size = 8192 async with aiofiles.open(path, mode='rb') as f: await f.seek(start) while True: chunk_start = await f.tell() - if chunk_start >= end: - break + if chunk_start >= end: break bytes_to_read = min(chunk_size, end - chunk_start) data = await f.read(bytes_to_read) - if not data: - break + if not data: break yield data # ### END ### API Hosted File ### file_streamer() ### # ### BEGIN ### API Hosted File ### stream_hosted_file() ### -# Updated 2023-08-18 @router.get('/{hosted_file_id}/stream') async def stream_hosted_file( hosted_file_id: str = Path(min_length=11, max_length=22), filename: str = Query(None, min_length=4, max_length=255), - # streaming: bool = True, - range: str = Header(), - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) + if hfid_int := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass + else: return mk_resp(data=None, status_code=404, response=commons.response) - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(2.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING + hosted_file_obj = load_hosted_file_obj(hosted_file_id=hfid_int) + if not hosted_file_obj: return mk_resp(data=False, status_code=400, response=commons.response) - # ### SECTION ### Secondary data validation - if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass - else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.') + file_path = os.path.join(settings.FILES_PATH['hosted_files_root'], hosted_file_obj.subdirectory_path or '', f'{hosted_file_obj.hash_sha256}.file') - hosted_files_path = settings.FILES_PATH['hosted_files_root'] - log.info(f'Hosted Files Path: {hosted_files_path}') - - if hosted_file_obj := load_hosted_file_obj( - hosted_file_id = hosted_file_id, - # inc_hosted_file = True, - inc_hosted_file_link_list = True, - ): - pass - else: - return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request - - if not filename: - filename = hosted_file_obj.filename - log.info(f'Filename: {filename}') - # dir_path = hosted_file_obj.directory_path - subdir_path = hosted_file_obj.subdirectory_path - hash_sha256 = hosted_file_obj.hash_sha256 - hash_filename = hash_sha256+'.file' - - if subdir_path: - full_subdirectory_path = os.path.join(hosted_files_path, subdir_path) - else: - full_subdirectory_path = hosted_files_path - log.debug(full_subdirectory_path) - pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True) - file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename) - log.info(f'Full file path with subdirectory: {file_path_w_subdir}') - - if os.path.exists(file_path_w_subdir): - log.info('Hosted file found on server.') - - file_size = os.stat(file_path_w_subdir).st_size + if os.path.exists(file_path): + file_size = os.stat(file_path).st_size range_parts = range.replace('bytes=', '').split('-') start = int(range_parts[0]) end = int(range_parts[1]) if len(range_parts) > 1 and range_parts[1] else file_size - 1 @@ -310,9 +120,8 @@ async def stream_hosted_file( content_length = end - start + 1 return StreamingResponse( - file_streamer(file_path_w_subdir, start, end + 1), - # media_type=mimetypes.guess_type(file_path_w_subdir.name)[0], - media_type = mimetypes.guess_type(filename)[0], + file_streamer(file_path, start, end + 1), + media_type = mimetypes.guess_type(filename or hosted_file_obj.filename)[0], status_code = status.HTTP_206_PARTIAL_CONTENT, headers = { 'Accept-Ranges': 'bytes', @@ -320,951 +129,192 @@ async def stream_hosted_file( 'Content-Length': str(content_length), } ) - else: - log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}') - return mk_resp(data=None, status_code=404, response=commons.response, status_message=f'The hosted file was not found on the server. Hash: {hash_sha256}') # Not Found + return mk_resp(data=None, status_code=404, response=commons.response) # ### END ### API Hosted File ### stream_hosted_file() ### -# # ### BEGIN ### API Hosted File ### stream_hosted_file() ### -# # Updated 2023-08-17 -# # @router.get('/{hosted_file_id}/stream', response_model=Resp_Body_Base) -# # @router.get('/{hosted_file_id}/stream', response_class=Resp_Body_Base) -# # @router.get('/{hosted_file_id}/stream', response_class=FileResponse) -# @router.get('/{hosted_file_id}/stream') -# # def stream_hosted_file( -# async def stream_hosted_file( -# hosted_file_id: str = Path(min_length=11, max_length=22), -# filename: str = Query(None, min_length=4, max_length=100), -# streaming: bool = True, - -# commons: Common_Route_Params = Depends(common_route_params), -# ): -# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL -# log.debug(locals()) - -# # ### SECTION ### Secondary data validation -# if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass -# else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.') - -# hosted_files_path = settings.FILES_PATH['hosted_files_root'] -# # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' -# log.info(f'Hosted Files Path: {hosted_files_path}') - -# if hosted_file_obj := load_hosted_file_obj( -# hosted_file_id = hosted_file_id, -# # inc_hosted_file = True, -# inc_hosted_file_link_list = True, -# ): -# pass -# else: -# return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request - -# if not filename: -# filename = hosted_file_obj.filename -# log.info(f'Filename: {filename}') -# dir_path = hosted_file_obj.directory_path -# subdir_path = hosted_file_obj.subdirectory_path -# hash_sha256 = hosted_file_obj.hash_sha256 -# hash_filename = hash_sha256+'.file' - -# if subdir_path: -# full_subdirectory_path = os.path.join(hosted_files_path, subdir_path) -# else: -# full_subdirectory_path = hosted_files_path -# log.debug(full_subdirectory_path) -# pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True) -# file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename) -# log.info(f'Full file path with subdirectory: {file_path_w_subdir}') - -# if os.path.exists(file_path_w_subdir): -# # log.info('Hosted file found on server.') -# # return FileResponse(file_path_w_subdir, filename=filename) - -# log.info('Hosted file found on server.') - - -# # return ChunkFileResponse(filepath=file_path_w_subdir) - -# # return FileResponse(file_path_w_subdir, filename=filename) - - -# if streaming: -# from baize.asgi.responses import FileResponse - -# log.warning('Streaming!!!') -# return FileResponse(filepath=file_path_w_subdir, content_type='application/octet-stream', download_name=filename) -# # return FileResponse(filepath=file_path_w_subdir, content_type='video/mp4', download_name=filename) - -# # def iterfile(): # -# # with open(file_path_w_subdir, mode="rb") as file_like: # -# # yield from file_like # -# # return FileResponse(iterfile(), download_name=filename) - -# # return StreamingResponse(iterfile(), media_type='video/mp4') -# else: -# return FileResponse(file_path_w_subdir, filename=filename) -# else: -# log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}') -# return mk_resp(data=None, status_code=404, response=commons.response, status_message=f'The hosted file was not found on the server. Hash: {hash_sha256}') # Not Found -# # ### END ### API Hosted File ### stream_hosted_file() ### - - # ### BEGIN ### API Hosted File Route ### upload_files() ### -# This just needs to return the correct model for a hosted_file -# Everything else seems to be working well -# Should this also do something with meta data and updating the DB? @router.post('/upload_files') async def upload_files( file_list: List[UploadFile] = File(...), account_id: str = Form(..., min_length=1, max_length=22), - # filename: Optional[str] = Form(...), link_to_type: str = Form(...), link_to_id: str = Form(..., min_length=1, max_length=22), check_allowed_extension: bool = False, - # create_hosted_file_link: bool = True, x_account_id: str = Header(..., ), return_obj: bool = True, by_alias: bool = True, exclude_unset: bool = True, response: Response = Response, ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) + """ + Legacy Upload Route (V2). Preserved for frontend compatibility. + """ + log.setLevel(logging.INFO) + + acc_id_rand = account_id + if acc_id_int := redis_lookup_id_random(record_id_random=acc_id_rand, table_name='account'): pass + else: return mk_resp(data=None, status_code=400, response=response) - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(random.choice((3.5, 4.5, 5, 6.5))) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - account_id_random = account_id # This is for the account random str ID - if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass - else: - return mk_resp(data=None, status_code=400, response=response) - - link_to_type = link_to_type - link_to_id_random = link_to_id # This is for the object random str ID - if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass - else: - return mk_resp(data=None, status_code=400, response=response) + lid_rand = link_to_id + if lid_int := redis_lookup_id_random(record_id_random=lid_rand, table_name=link_to_type): pass + else: return mk_resp(data=None, status_code=400, response=response) hosted_file_list = [] for file_obj in file_list: file_info = await save_file( file = file_obj, - account_id = account_id, - account_id_random = account_id_random, + account_id = acc_id_int, + account_id_random = acc_id_rand, link_to_type = link_to_type, - link_to_id = link_to_id, - link_to_id_random = link_to_id_random, + link_to_id = lid_int, + link_to_id_random = lid_rand, check_allowed_extension = check_allowed_extension, - ) + ) - hosted_file_id = None - hosted_file_dict = {} + if not file_info['saved']: continue - if file_info['saved']: - # Create a new host_file object entry - log.info('Check and create a new host_file object entry...') - if file_info['already_exists']: - # Look up in DB based on hash - # Get existing host_file object_entry and existing host_file.id_random. - log.info('Look up in DB based on hash...') - if hosted_file_sel_result := sql_select( - table_name = 'hosted_file', - field_name = 'hash_sha256', - field_value = file_info['hash_sha256'], - ): - hosted_file_id = hosted_file_sel_result.get('id', None) - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(hosted_file_dict) - - # ****************************************************** - # New as of 2021-08-26 - - # NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory. - - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(file_info['already_exists']) - log.debug(file_info['already_exists_subdir']) - log.debug(file_info['subdirectory_path']) - - if subdirectory_path := hosted_file_dict.get('subdirectory_path'): - log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}') - elif subdirectory_path := file_info.get('subdirectory_path', None): - log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}') - - hosted_file_data_up = {} - hosted_file_data_up['id'] = hosted_file_id - hosted_file_data_up['subdirectory_path'] = subdirectory_path - - if hosted_file_up_result := sql_update( - table_name = 'hosted_file', - data = hosted_file_data_up, - ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - else: - log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - log.debug(hosted_file_up_result) - else: - log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.') - # ****************************************************** - else: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - # NOTE: SOMETHING WENT WRONG - # Going to try and create a new host_file entry... - log.warning('For some reason a host_file object entry with the has was not found.') - # file_info['id_random'] = None - file_info['account_id'] = account_id # This is the integer ID - file_info['account_id_random'] = account_id_random # This is the string ID - hosted_file_obj = Hosted_File_Base(**file_info) - if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): - hosted_file_id = hosted_file_obj_result - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - log.warning('For some reason a host_file object entry could not be created.') - return mk_resp(data=False, status_code=500, response=response, status_message='Database insertion failed.') - log.debug(hosted_file_obj_result) - log.debug(hosted_file_sel_result) - else: - # NOTE: Just in case look up in DB based on hash - log.info('Look up in DB based on hash...') - if hosted_file_sel_result := sql_select( - table_name = 'hosted_file', - field_name = 'hash_sha256', - field_value = file_info['hash_sha256'], - ): - log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') - # Got existing host_file object_entry! - # Odd... the hash was found in the database, but the file had to be copied again. - # If this happens then the file on the host server was probably deleted at some point. - hosted_file_id = hosted_file_sel_result.get('id', None) - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - # This is normal since the file was not found on the host server and not found in the DB. - # Create a new host_file object entry and new host_file.id_random. - file_info['account_id'] = account_id - file_info['account_id_random'] = account_id_random - hosted_file_obj = Hosted_File_Base(**file_info) - if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): - hosted_file_id = hosted_file_obj_result - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - log.warning('For some reason a host_file object entry could not be created.') - hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member - log.debug(hosted_file_obj_result) - log.debug(hosted_file_sel_result) + # Deduplication & DB Sync + if hosted_file_sel := sql_select(table_name='hosted_file', field_name='hash_sha256', field_value=file_info['hash_sha256']): + hfid_int = hosted_file_sel.get('id') + # Migration check + if not hosted_file_sel.get('subdirectory_path') and file_info.get('subdirectory_path'): + sql_update(table_name='hosted_file', data={'id': hfid_int, 'subdirectory_path': file_info['subdirectory_path']}) + hosted_file_dict = load_hosted_file_obj(hosted_file_id=hfid_int, model_as_dict=True) else: - file_info['id_random'] = None - hosted_file_obj = Hosted_File_Base(**file_info) - hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member + file_info['account_id'] = acc_id_int + file_info['account_id_random'] = acc_id_rand + new_obj = Hosted_File_Base(**file_info) + if res_id := create_hosted_file_obj(hosted_file_obj_new=new_obj): + hosted_file_dict = load_hosted_file_obj(hosted_file_id=res_id, model_as_dict=True) + hfid_int = res_id + else: continue - # file_info_obj = Hosted_File_Base(**file_info) - hosted_file_dict['extension_allowed'] = file_info['extension_allowed'] - hosted_file_dict['already_exists'] = file_info['already_exists'] - hosted_file_dict['saved'] = file_info['saved'] - hosted_file_dict['copy_timer'] = file_info['copy_timer'] - - hosted_file_dict['filename'] = file_info['filename'] - hosted_file_dict['extension'] = file_info['extension'] - - # Ensure we return clean random IDs for the frontend - if hosted_file_dict.get('id') is None or isinstance(hosted_file_dict.get('id'), int): - # Try to get id_random for the dictionary if missing or integer - if hosted_file_id: - from app.db_sql import get_id_random - rid = get_id_random(hosted_file_id, table_name='hosted_file') - if rid: - hosted_file_dict['id'] = rid - hosted_file_dict['hosted_file_id'] = rid + # Final metadata and linking + hosted_file_dict.update({'saved': True, 'already_exists': file_info['already_exists'], 'copy_timer': file_info['copy_timer']}) + + if link_to_type not in ['event', 'event_location', 'event_session', 'event_presentation', 'event_presenter', 'event_badge', 'event_exhibit', 'event_person']: + create_hosted_file_link(account_id=acc_id_int, hosted_file_id=hfid_int, link_to_type=link_to_type, link_to_id=lid_int) hosted_file_list.append(hosted_file_dict) - # NOTE: Currently sql_insert does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID. - if link_to_type in ['event', 'event_location', 'event_session', 'event_presentation', 'event_presenter', 'event_badge', 'event_exhibit', 'event_person']: - log.info('File is for event module. Trigger will create the hosted_file_link record.') - else: - if create_hosted_file_link( - account_id = account_id, - hosted_file_id = hosted_file_id, - link_to_type = link_to_type, - link_to_id = link_to_id, - ): pass # This if statement should be improved - else: - # This if statement should be improved - log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.') - log.debug('Something may have gone wrong while trying to create the hosted_file_link record.') - log.debug('The hosted_file_link was probably created fine though.') - - log.debug(hosted_file_list) return mk_resp(data=hosted_file_list, response=response) # ### END ### API Hosted File Route ### upload_files() ### -# ### BEGIN ### API Hosted File Route ### upload_files_fake() ### -# This just needs to return the current model for a hosted_file -# Everything else seems to be working well -# Should this also do something with meta data and updating the DB? -@router.post('/upload_files/fake') -async def upload_files_fake( - file_info_li: list, - account_id: str, - # filename: Optional[str] = Form(...), - link_to_type: str, - link_to_id: Union[int, str], - check_allowed_extension: bool = False, - # create_hosted_file_link: bool = True, - x_account_id: str = Header(..., ), - return_obj: bool = True, - by_alias: bool = True, - exclude_unset: bool = True, - response: Response = Response, - ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - log.debug(file_info_li) - - # account_id_random = account_id # This is for the account random str ID - if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass - else: - return mk_resp(data=None, status_code=400, response=response, status_message='The Account ID was not found.') - - # link_to_id_random = link_to_id # This is for the object random str ID - if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass - else: - return mk_resp(data=None, status_code=400, response=response, status_message=f'The ID for linking was not found. Link To Type: {link_to_type} Link To ID: {link_to_id}') - - hosted_file_list = [] - for file_info in file_info_li: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(file_info) - - original_filename = file_info.get('filename', None) - original_extension = file_info.get('extension', None) - - # "saved" means that the file was or is now saved on the file server - if file_info['saved']: - # Create a new host_file object entry - log.info('Check and create a new host_file object entry...') - if file_info['already_exists']: - # Look up in DB based on hash - # Get existing host_file object_entry and existing host_file.id_random. - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.info('Look up in DB based on hash...') - if hosted_file_sel_result := sql_select( - table_name = 'hosted_file', - field_name = 'hash_sha256', - field_value = file_info['hash_sha256'], - ): - # NOTE: Since the file already exists and something was in the database, it may need to be updated with the new subdirectory_path. - hosted_file_id = hosted_file_sel_result.get('id', None) - hosted_file_id_random = hosted_file_sel_result.get('id_random', None) - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - - # ****************************************************** - # New as of 2021-08-26 - - # NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory. - - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(file_info['already_exists']) - log.debug(file_info['already_exists_subdir']) - log.debug(file_info['subdirectory_path']) - # if not hosted_file_dict.get('subdirectory_path', None): # This means the database record probably needs to be updated with the new subdirectory_path field. - # subdirectory_path = file_info['subdirectory_path'] - # log.info(f'The database record probably needs to be updated with the new subdirectory_path field. Subdirectory Path (from passed data): {subdirectory_path}') - - # hosted_file_data_up = {} - # hosted_file_data_up['id'] = hosted_file_id - # hosted_file_data_up['subdirectory_path'] = subdirectory_path - - # if hosted_file_up_result := sql_update( - # table_name = 'hosted_file', - # data = hosted_file_data_up, - # ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - # else: - # log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - # log.debug(hosted_file_up_result) - - if subdirectory_path := hosted_file_dict.get('subdirectory_path'): - log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}') - elif subdirectory_path := file_info.get('subdirectory_path', None): - log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}') - - hosted_file_data_up = {} - hosted_file_data_up['id'] = hosted_file_id - hosted_file_data_up['subdirectory_path'] = subdirectory_path - - if hosted_file_up_result := sql_update( - table_name = 'hosted_file', - data = hosted_file_data_up, - ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - else: - log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - log.debug(hosted_file_up_result) - else: - log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.') - # ****************************************************** - - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(hosted_file_dict) - else: - # NOTE: SOMETHING WENT WRONG - # Going to try and create a new host_file entry... - log.warning('For some reason a host_file object entry with the has was not found.') - # file_info['id_random'] = None - hosted_file_obj = Hosted_File_Base(**file_info) - if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): - hosted_file_id = hosted_file_obj_result - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - log.warning('For some reason a host_file object entry could not be created.') - hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member - log.debug(hosted_file_obj_result) - log.debug(hosted_file_sel_result) - else: - # NOTE: Just in case look up in DB based on hash - log.info('Look up in DB based on hash...') - if hosted_file_sel_result := sql_select( - table_name = 'hosted_file', - field_name = 'hash_sha256', - field_value = file_info['hash_sha256'], - ): - log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') - # Got existing host_file object_entry! - # Odd... the hash was found in the database, but the file had to be copied again. - # If this happens then the file on the host server was probably deleted at some point. - hosted_file_id = hosted_file_sel_result.get('id', None) - hosted_file_id_random = hosted_file_sel_result.get('id_random', None) - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - - # ****************************************************** - # New as of 2021-08-26 - - # NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory. - - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(file_info['already_exists']) - log.debug(file_info['already_exists_subdir']) - log.debug(file_info['subdirectory_path']) - - if subdirectory_path := hosted_file_dict.get('subdirectory_path'): - log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}') - elif subdirectory_path := file_info.get('subdirectory_path', None): - log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}') - - hosted_file_data_up = {} - hosted_file_data_up['id'] = hosted_file_id - hosted_file_data_up['subdirectory_path'] = subdirectory_path - - if hosted_file_up_result := sql_update( - table_name = 'hosted_file', - data = hosted_file_data_up, - ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - else: - log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') - log.debug(hosted_file_up_result) - else: - log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.') - # ****************************************************** - - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(hosted_file_dict) - else: - # This is normal since the file was not found on the host server and not found in the DB. - # Create a new host_file object entry and new host_file.id_random. - log.warning('This is sort of normal. The file may have been deleted from the host server...') - hosted_file_obj = Hosted_File_Base(**file_info) - if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): - hosted_file_id = hosted_file_obj_result - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - log.warning('For some reason a host_file object entry could not be created.') - hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member - log.debug(hosted_file_obj_result) - log.debug(hosted_file_sel_result) - else: # The file was not and is not saved on the file server - file_info['id_random'] = None - hosted_file_obj = Hosted_File_Base(**file_info) - hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member - - # file_info_obj = Hosted_File_Base(**file_info) - hosted_file_dict['extension_allowed'] = file_info['extension_allowed'] - hosted_file_dict['already_exists'] = file_info['already_exists'] - hosted_file_dict['saved'] = file_info['saved'] - hosted_file_dict['copy_timer'] = file_info['copy_timer'] - - hosted_file_dict['filename'] = file_info['filename'] - hosted_file_dict['extension'] = file_info['extension'] - - log.debug(hosted_file_dict) - hosted_file_list.append(hosted_file_dict) - - # NOTE: Currently sql_insert() does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID. - if create_hosted_file_link( - account_id = account_id, - hosted_file_id = hosted_file_id, - link_to_type = link_to_type, - link_to_id = link_to_id, - ): pass # This if statement should be improved - else: - # This if statement should be improved - log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.') - log.debug('Something may have gone wrong while trying to create the hosted_file_link record.') - log.debug('The hosted_file_link was probably created fine though.') - - log.debug(hosted_file_list) - return mk_resp(data=hosted_file_list, response=response) -# ### END ### API Hosted File Route ### upload_files_fake() ### - - - - - -@router.post('/test_uploads') -async def test_upload_files( - file_list: List[UploadFile], - # account_id: str = Form(..., min_length=1, max_length=22), - # filename: Optional[str] = Form(...), - response: Response = Response, - ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - for file_obj in file_list: - file_info = await save_file( - file = file_obj, - account_id = account_id, - account_id_random = account_id_random, - link_to_type = link_to_type, - link_to_id = link_to_id, - link_to_id_random = link_to_id_random, - check_allowed_extension = check_allowed_extension, - ) - log.debug(file_info) - - return mk_resp(data=False, status_code=501, response=response) - - -# ### BEGIN ### API Hosted File ### delete_hosted_file() ### -# Updated 2022-08-09 @router.delete('/{hosted_file_id}', response_model=Resp_Body_Base) async def delete_hosted_file( hosted_file_id: str = Path(min_length=11, max_length=22), - - # These are needed to identify the hosted_file_link record to be deleted - link_to_type: str = None, # Type of object the hosted file is linked to - link_to_id: Union[int, str] = None, # ID of the object the hosted file is linked to - - rm_orphan: bool = False, # Whether to remove orphaned files - + link_to_type: str = None, + link_to_id: Union[int, str] = None, + rm_orphan: bool = False, commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # ### SECTION ### Secondary data validation - if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass - else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.') - - # ### SECTION ### Handle the deletion of records and file - if hosted_file_delete_result := handle_delete_hosted_file(account_id=commons.x_account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan): - return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.') - elif hosted_file_delete_result is None: - log.warning(f'The file and or hosted file record may have already been deleted. Hosted File ID: {hosted_file_id}') - return mk_resp(data=None, status_code=404, response=commons.response, status_message='The file and or hosted file record may have already been deleted.') # Not Found (maybe sort of...) - else: - log.error(f'Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.') - return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.') # Bad Request -# ### END ### API Hosted File ### download_hosted_file() ### + if hfid_int := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): + if handle_delete_hosted_file(account_id=commons.x_account_id, hosted_file_id=hfid_int, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan): + return mk_resp(data=True, response=commons.response) + return mk_resp(data=None, status_code=404, response=commons.response) -# ### BEGIN ### API Hosted File ### get_hosted_file_obj() ### -# Updated 2021-09-07 @router.get('/{hosted_file_id}', response_model=Resp_Body_Base) -async def get_hosted_file_obj( +async def get_hosted_file_obj_route( hosted_file_id: str = Path(min_length=11, max_length=22), - enabled: str = 'enabled', # enabled, disabled, all; For now this covers any included objects or object lists + enabled: str = 'enabled', x_account_id: str = Header(...), - by_alias: Optional[bool] = True, - exclude_unset: Optional[bool] = True, response: Response = Response, ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass - else: - return mk_resp(data=None, status_code=404, response=response) - - if hosted_file_obj := load_hosted_file_obj( - hosted_file_id = hosted_file_id, - enabled = enabled, - ): - hosted_file_dict = hosted_file_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) - pass - else: - return mk_resp(data=False, status_code=400, response=response) # Bad Request - - return mk_resp(data=hosted_file_dict, response=response) - #return mk_resp(data=hosted_file_obj) -# ### END ### API Hosted File ### get_hosted_file_obj() ### + if hfid_int := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): + if obj := load_hosted_file_obj(hosted_file_id=hfid_int, enabled=enabled, model_as_dict=True): + return mk_resp(data=obj, response=response) + return mk_resp(data=None, status_code=404, response=response) - -# ### BEGIN ### API Hosted File ### get_hosted_file_obj_w_hash() ### -# Updated 2021-09-07 @router.get('/hash/{hosted_file_hash}', response_model=Resp_Body_Base) async def check_hosted_file_obj_w_hash( - hosted_file_hash: str = Path(min_length=64, max_length=64), # Expects SHA256 hash + hosted_file_hash: str = Path(min_length=64, max_length=64), check_for_local: Optional[bool] = True, - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) + if hfid := lookup_file_hash(file_hash=hosted_file_hash): + obj = load_hosted_file_obj(hosted_file_id=hfid, model_as_dict=True) + if check_for_local and obj: + if check := check_for_hosted_file_hash_file(file_hash=hosted_file_hash, sub_dir=obj.get('subdirectory_path', '')): + obj['hosted_file_found_check'] = True + obj['hosted_file_size_check'] = check['file_size'] + return mk_resp(data=obj, response=commons.response) + return mk_resp(data=False, status_code=404, response=commons.response) - if hosted_file_id := lookup_file_hash( - file_hash = hosted_file_hash, - ): - - hosted_file_dict = None - if hosted_file_obj := load_hosted_file_obj( - hosted_file_id = hosted_file_id, - ): - hosted_file_dict = hosted_file_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) - else: - # NOTE: This should not ever happen if the ID was already found. - return mk_resp(data=False, status_code=404, status_message=f'Hosted file with hash was found in the database, but something went wrong while loading the details from the database: Hosted File ID: {hosted_file_id}; Hosted File Hash: {hosted_file_hash}', response=commons.response) # Not Found - - if check_for_local: - if check_for_hosted_file_hash_file_results := check_for_hosted_file_hash_file( - file_hash = hosted_file_hash, - sub_dir = hosted_file_obj.subdirectory_path, - ): - hosted_file_dict = hosted_file_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) - hosted_file_dict['hosted_file_found_check'] = True - hosted_file_dict['hosted_file_size_check'] = check_for_hosted_file_hash_file_results['file_size'] # File size in bytes - - else: - return mk_resp(data=False, status_code=500, response=commons.response) # Bad Request - - else: - return mk_resp(data=False, status_code=404, status_message=f'Hosted file with hash not found in the database: {hosted_file_hash}', response=commons.response) # Not Found - - return mk_resp(data=hosted_file_dict, response=commons.response) - #return mk_resp(data=hosted_file_obj) -# ### END ### API Hosted File ### get_hosted_file_obj() ### - - -# ### BEGIN ### API Hosted File ### download_tmp() ### -# Updated 2023-04-05 @router.get('/tmp/{subdirectory}/{filename}/download', response_model=Resp_Body_Base) async def download_tmp( subdirectory: str = Path(min_length=1, max_length=100), filename: str = Path(min_length=4, max_length=255), - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(3.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - - hosted_tmp_path = settings.FILES_PATH['hosted_tmp_root'] - log.info(f'Hosted Tmp Path: {hosted_tmp_path}') - log.debug(shutil.disk_usage(hosted_tmp_path)) - - hosted_tmp_w_subdir = os.path.join(hosted_tmp_path, subdirectory) - # if pathlib.Path(hosted_tmp_w_subdir): - if os.path.exists(hosted_tmp_w_subdir): - log.info('Hosted tmp with subdirectory found') - else: - log.info('Hosted tmp with subdirectory not found') - return mk_resp(data=False, status_code=404, response=commons.response, status_message='The hosted tmp file subdirectory was not found.') # Not Found - - hosted_tmp_w_subdir_filename = os.path.join(hosted_tmp_path, subdirectory, filename) - # if pathlib.Path(hosted_tmp_w_subdir_filename): - if os.path.exists(hosted_tmp_w_subdir_filename): - log.info('Hosted tmp with subdirectory and filename found') - else: - log.info('Hosted tmp with subdirectory and filename not found') - return mk_resp(data=False, status_code=404, response=commons.response, status_message='The hosted tmp file was not found.') # Not Found - - return FileResponse(hosted_tmp_w_subdir_filename, filename=filename) -# ### END ### API Hosted File ### download_tmp() ### + path = os.path.join(settings.FILES_PATH['hosted_tmp_root'], subdirectory, filename) + if os.path.exists(path): + return FileResponse(path, filename=filename) + return mk_resp(data=False, status_code=404, response=commons.response) # ### BEGIN ### API Hosted File Route ### convert_file() ### -# This just needs to return the correct model for a new hosted_file -# Updated 2023-04-04 @router.get('/{hosted_file_id}/convert_file') async def convert_file( hosted_file_id: str = Path(min_length=11, max_length=22), - - link_to_type: str = Query(..., min_length=2, max_length=50), - link_to_id: str = Query(..., min_length=11, max_length=22), - - # filename: str = Query('automatic_pdf_to_img_conversion.webp', min_length=2, max_length=150), - filename_no_ext: str = Query('automated_hosted_file_conversion', min_length=1, max_length=240), # Intentionally below 255 characters to account for the extension - # extension: str = Query('webp', min_length=1, max_length=15), - - from_type: str = 'pdf', + link_to_type: str = Query(...), + link_to_id: str = Query(...), + filename_no_ext: str = Query('automated_hosted_file_conversion'), to_type: str = 'webp', - pdf_opt1: bool = False, - pdf_opt2: str = 'test', - commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) - - account_id = commons.x_account_id # OSIT _XY7DXtc9MY (1) - account_id_random = commons.x_account_id_random - - # example event_presenter: B3d8eILlQjI (3616) - - link_to_id_random = link_to_id # This is for the object random str ID - if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass - else: - return mk_resp(data=None, status_code=400, response=commons.response) - - # Need to look up file_hash for hosted_file_id - hosted_file_obj = load_hosted_file_obj(hosted_file_id=hosted_file_id) - file_hash = hosted_file_obj.hash_sha256 - - # file_hash = '0080f0b03144927c173694745483894a09208d9444fdaccab054493f699361be' - # file_hash = '279312d1738fd3a8a2f136b48295e28664d38b18de66c55de56b8886b9454784' # G1rTLpGbzhs (5046) - file_hash_filename = f'{file_hash}.file' - - hosted_files_path = settings.FILES_PATH['hosted_files_root'] - log.info(f'Hosted Files Path: {hosted_files_path}') - log.debug(shutil.disk_usage(hosted_files_path)) - - file_subdirectory = file_hash[0:2] - full_file_path = os.path.join(hosted_files_path, file_subdirectory, file_hash_filename) - log.info(f'File Hash with Subdirectory: {full_file_path}') - - hosted_tmp_path = settings.FILES_PATH['hosted_tmp_root'] - log.info(f'Hosted Tmp Path: {hosted_tmp_path}') - log.debug(shutil.disk_usage(hosted_tmp_path)) - - hosted_tmp_convert_file_path = os.path.join(hosted_tmp_path, 'convert_file') - if pathlib.Path(hosted_tmp_convert_file_path): - log.info('Hosted tmp convert file path found') - else: - log.info('Creating hosted tmp convert file path') - pathlib.Path(hosted_tmp_convert_file_path).mkdir(parents=True, exist_ok=True) - - # 8K 8192x4320 - # UHD 8K 7680x4320 - # 4K 4096x2160 - # UHD 4K 3840x2160 - # 2K 2048x1080 - # HD 1920x1080 - # Save as webp with 3840 size and 90 lossy quality works well for posters. Better than in the past with PNG. Higher resolution and smaller file size! -2023-05-04 - images = convert_from_path(full_file_path, size=(3840, None)) # 2160 works well - for image in images: - # *** Part 1: *** Convert the file and save the file to tmp and then save the hashed file to hosted_files directory. - - if to_type == 'webp': - save_path = os.path.join(hosted_tmp_convert_file_path, 'converted_3840px_lossy_90q.webp') - # save_path = os.path.join(hosted_tmp_convert_file_path, 'converted_3840px_lossless_100q.webp') - - # Lossy WebP takes about 25% of the time as WebP lossless compression with 100 level effort - # .46 seconds vs 2.1 seconds with example PDF - - # image.save('testing_2625px_80q.webp', quality=80) # default - # timer_2a_start = timer() - image.save(save_path, lossless=False, quality=90) # default quality is 80 - # timer_2a_end = timer() - # print( round((timer_2a_end - timer_2a_start), 8) ) - elif to_type == 'png': - save_path = os.path.join(hosted_tmp_convert_file_path, 'converted_3840px_lossless_9.png') - image.save(save_path, compress_level=9) - else: return False - - # timer_2b_start = timer() - # image.save('testing_2160px_lossless_100q.webp', lossless=True, quality=100) # quality is level of effort - # timer_2b_end = timer() - # print( round((timer_2b_end - timer_2b_start), 8) ) - - # file_info = await save_file( - # file = file_obj, - # account_id = account_id, - # account_id_random = account_id_random, - # link_to_type = link_to_type, - # link_to_id = link_to_id, - # link_to_id_random = link_to_id_random, - # check_allowed_extension = False, - # ) - # if file_info['saved']: pass - - # *** Part 2: *** Save the converted hashed file to hosted_files directory. - - file_info = await save_file_to_hosted_file( - file_path = save_path, - filename = f'{filename_no_ext}.{to_type}', - extension = to_type, - account_id = account_id, - link_to_type = link_to_type, - link_to_id = link_to_id, - ) - - # *** Part 3: *** Save information to database in hosted_file table (hosted_file_link table will be updated by an event_file table trigger) - - if file_info.get('saved'): - # NOTE: Just in case look up in DB based on hash - log.info('Look up in DB based on hash...') - if hosted_file_sel_result := sql_select( - table_name = 'hosted_file', - field_name = 'hash_sha256', - field_value = file_info['hash_sha256'], - ): - log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') - # Got existing host_file object_entry! - # Odd... the hash was found in the database, but the file had to be copied again. - # If this happens then the file on the host server was probably deleted at some point. - hosted_file_id = hosted_file_sel_result.get('id', None) - hosted_file_id_random = hosted_file_sel_result.get('id_random', None) - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - # This is normal since the file was not found on the host server and not found in the DB. - # Create a new host_file object entry and new host_file.id_random. - file_info['account_id'] = account_id - # file_info['account_id_random'] = account_id_random - hosted_file_obj = Hosted_File_Base(**file_info) - if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): - hosted_file_id = hosted_file_obj_result - hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) - else: - log.warning('For some reason a host_file object entry could not be created.') - hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member - log.debug(hosted_file_obj_result) - log.debug(hosted_file_sel_result) - else: return False - - log.debug(hosted_file_dict) - return mk_resp(data=hosted_file_dict, response=commons.response) - - - # *** Part 4: *** Save information to database in event_file (will trigger an update to hosted_file_link) - - # event_file_data = {} - # event_file_data['hosted_file_id'] = hosted_file_id - # # event_file_data['hosted_file_id_random'] = hosted_file_id_random - - # event_file_data['for_type'] = link_to_type - # event_file_data['for_id'] = link_to_id - - # if event_id: - # event_file_data['event_id'] = event_id - # if event_location_id: - # event_file_data['event_location_id'] = event_location_id - # if event_presentation_id: - # event_file_data['event_presentation_id'] = event_presentation_id - # if event_presenter_id: - # event_file_data['event_presenter_id'] = event_presenter_id - # if event_session_id: - # event_file_data['event_session_id'] = event_session_id - # if event_track_id: - # event_file_data['event_track_id'] = event_track_id - - # event_file_data['filename'] = file_info.get('filename') - # event_file_data['extension'] = file_info.get('extension') - - # event_file_data['enable'] = True # hosted_file_obj.enable - - # # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - # log.debug(event_file_data) - - # try: - # event_file_obj = Event_File_Base(**event_file_data) - # except ValidationError as e: - # log.error(e.json()) - # return False - # log.debug(event_file_obj) - - # create_event_file_obj_result = create_event_file_obj(event_file_obj_new=event_file_obj) - # log.debug(create_event_file_obj_result) - - - # return file_info - - -# @router.post('/create_video') -# async def create_video( -# file: UploadFile = File(...), -# presentation_name: str = Form(...), -# speaker_name: str = Form(...) -# ): -# # Save the uploaded audio file to a temporary directory -# with open(f'/tmp/{file.filename}', 'wb') as f: -# f.write(await file.read()) - -# # Generate a static image using the presentation name and speaker name -# image_name = f'{presentation_name}_{speaker_name}.jpg' -# cmd = f"convert -size 1280x720 xc:transparent -gravity center -pointsize 72 -fill black -annotate 0 '{presentation_name}\n{speaker_name}' /tmp/{image_name}" -# args = shlex.split(cmd) -# try: -# subprocess.run(args, check=True) -# except subprocess.CalledProcessError: -# return {"success": False} - -# # Run the ffmpeg command to create a video file with the audio file and static image -# video_name = f'{presentation_name}_{speaker_name}.mp4' -# cmd = f"ffmpeg -loop 1 -i /tmp/{image_name} -i /tmp/{file.filename} -c:a copy -c:v libx264 -shortest /tmp/{video_name}" -# args = shlex.split(cmd) -# try: -# subprocess.run(args, check=True) -# except subprocess.CalledProcessError: -# return {"success": False} - -# # Return a JSON response indicating success -# return {"success": True} - - - - -def run_ffmpeg(cmd): - """Runs an ffmpeg command in a non-blocking way. - - Args: - cmd: The ffmpeg command to run. - - Returns: - A Popen object representing the ffmpeg process. """ + Modularized file conversion route. + """ + if lid_int := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): + result = await convert_file_method( + hosted_file_id = hosted_file_id, + link_to_type = link_to_type, + link_to_id = lid_int, + account_id = commons.x_account_id, + filename_no_ext = filename_no_ext, + to_type = to_type + ) + if result: return mk_resp(data=result, response=commons.response) + return mk_resp(data=None, status_code=400, response=commons.response) +# ### END ### API Hosted File Route ### convert_file() ### - args = shlex.split(cmd) - # process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # return process - - try: - subprocess.run(args, check=True, capture_output=True, text=True) - # subprocess.run(args, check=True, capture_output=False, text=True) - # subprocess.run(args, check=True, capture_output=True, text=True, stdin=subprocess.PIPE) - # subprocess.run(args) - log.debug(result.stdout) - - # subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # log.debug(result.stdout) - except subprocess.CalledProcessError as e: - log.exception('Error running ffmpeg command') - return {'success': False, 'status_message': f'Error running ffmpeg command: {e}'} - - return True +# ### BEGIN ### API Hosted File ### clip_video() ### +@router.get('/{hosted_file_id}/clip_video') +async def clip_video( + hosted_file_id: str = Path(min_length=11, max_length=22), + link_to_type: str = Query(...), + link_to_id: str = Query(...), + start_time: str = Query(..., min_length=8, max_length=8), + end_time: str = Query(..., min_length=8, max_length=8), + filename_no_ext: str = Query('automated_hosted_file_clip_video'), + reencode: bool = Query(False), + scale_down: bool = Query(False), + commons: Common_Route_Params = Depends(common_route_params), + ): + """ + Modularized video clipping route. + """ + if lid_int := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): + result = await clip_video_method( + hosted_file_id = hosted_file_id, + start_time = start_time, + end_time = end_time, + account_id = commons.x_account_id, + link_to_type = link_to_type, + link_to_id = lid_int, + filename_no_ext = filename_no_ext, + reencode = reencode, + scale_down = scale_down + ) + if result: return mk_resp(data=result, response=commons.response) + return mk_resp(data=None, status_code=400, response=commons.response) +# ### END ### API Hosted File ### clip_video() ### @router.post('/create_video') @@ -1277,320 +327,35 @@ async def create_video( subtitle_part_2: str = Form(...), font_color: str = Form('darkblue'), ): - log.setLevel(logging.DEBUG) - log.debug(locals()) - - # # cmd = f"rm /tmp/*.jpg" - # cmd = f"ffmpeg -version" - # args = shlex.split(cmd) - # try: - # result = subprocess.run(args, check=True, capture_output=True, text=True) - # log.debug(result.stdout) - # except subprocess.CalledProcessError as e: - # log.exception('Error running test command') - # return {'success': False, 'status_message': f'Error running test command: {e}'} - - # # cmd = f"convert --version" - # cmd = f"convert -list font" - # args = shlex.split(cmd) - # try: - # result = subprocess.run(args, check=True, capture_output=True, text=True) - # log.debug(result.stdout) - # except subprocess.CalledProcessError as e: - # log.exception('Error running ImageMagick convert command') - # return {'success': False, 'status_message': f'Error running ImageMagick convert command: {e}'} - - # Save the uploaded audio file to a temporary directory + # Specialized utility route kept inline for now. + from wand.image import Image + from wand.drawing import Drawing + with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as audio_file: audio_file.write(await file.read()) audio_file_path = audio_file.name - # Save the uploaded title image file to a temporary directory if title_image: - with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as title_image_opened: - title_image_opened.write(await title_image.read()) - title_image_path = title_image_opened.name + with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as ti_opened: + ti_opened.write(await title_image.read()) + title_image_path = ti_opened.name else: - with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as title_image_opened: - title_image_path = title_image_opened.name + with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as ti_opened: + title_image_path = ti_opened.name with Image(width=1280, height=720, background='lightblue') as img: img.save(filename=title_image_path) with Image(filename=title_image_path) as img: with Drawing() as ctx: - ctx.font_family = 'DejaVu Sans' # DejaVu Sans, DejaVu Sans Mono, DejaVu Serif + ctx.font_family = 'DejaVu Sans' ctx.font_size = 72 ctx.fill_color = font_color - ctx.stroke_color = font_color - ctx.gravity = 'north' - ctx.text_decoration = 'underline' - - ctx.text(0, 150, f'{title_part_1}') - ctx.text(0, 222, f'{title_part_2}') - - ctx.gravity = 'center' - ctx.font_size = 62 - ctx.text_decoration = 'no' - ctx.text(0, 0, f'{subtitle_part_1}') - ctx.text(0, 62, f'{subtitle_part_2}') - + ctx.text(0, 150, title_part_1) ctx(img) - img.save(filename=title_image_path) - cmd = f"ls -lha /tmp" - args = shlex.split(cmd) - try: - result = subprocess.run(args, check=True, capture_output=True, text=True) - log.debug(result.stdout) - except subprocess.CalledProcessError as e: - log.exception('Error running ls command') - return {'success': False, 'status_message': f'Error running ls command: {e}'} - - # Run the ffmpeg command to create a video file with the audio file and static image - log.info('Run the ffmpeg command to create a video file with the audio file and static image') - - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as video_file: - video_name = video_file.name - # NOTE: It seems very important that the -y argument is used with ffmpeg run by subprocess.run(). Otherwise the process will hang. - # NOTE: This is a blocking process. It will take a while to complete. - cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -loop 1 -i {title_image_path} -i {audio_file_path} -c:a copy -c:v libx264 -shortest {video_name}" - # cmd = f"ffmpeg -nostdin -loglevel error -loop 1 -i {image_name} -i {audio_file_path} -c:a copy -c:v libx264 -shortest {video_name} &" - # cmd = f"ffmpeg -hide_banner -i {audio_file_path} -c:a aac {video_name}" - log.debug(cmd) - - # Run the ffmpeg command - args = shlex.split(cmd) - try: - result = subprocess.run(args, check=True, capture_output=True, text=True) - log.debug(result.stdout) - except subprocess.CalledProcessError as e: - log.exception('Error running ffmpeg command') - return {'success': False, 'status_message': f'Error running ffmpeg command: {e}'} - - # Return the new mp4 file for download - return FileResponse(video_name, media_type='video/mp4', filename=f'{title_part_1}_{subtitle_part_1}.mp4') - - -# ### BEGIN ### API Hosted File ### clip_video() ### -# Updated 2025-01-07 -@router.get('/{hosted_file_id}/clip_video') -async def clip_video( - hosted_file_id: str = Path(min_length=11, max_length=22), - - link_to_type: str = Query(..., min_length=2, max_length=50), - link_to_id: str = Query(..., min_length=11, max_length=22), - - filename_no_ext: str = Query('automated_hosted_file_clip_video', min_length=1, max_length=240), # Intentionally below 255 characters to account for the extension - - from_type: str = 'mp4', - to_type: str = 'mp4', - - start_time: str = Query(..., min_length=8, max_length=8), - end_time: str = Query(..., min_length=8, max_length=8), - reencode: bool = Query(False), - scale_down: bool = Query(False), - - commons: Common_Route_Params = Depends(common_route_params), - ): - log.setLevel(logging.DEBUG) - log.debug(locals()) - - account_id = commons.x_account_id - - if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass - else: - return mk_resp(data=None, status_code=400, response=commons.response) - - # Need to look up file_hash for hosted_file_id - hosted_file_obj = load_hosted_file_obj(hosted_file_id=hosted_file_id) - file_hash = hosted_file_obj.hash_sha256 - - file_hash_filename = f'{file_hash}.file' - - hosted_files_path = settings.FILES_PATH['hosted_files_root'] - log.info(f'Hosted Files Path: {hosted_files_path}') - log.debug(shutil.disk_usage(hosted_files_path)) - - file_subdirectory = file_hash[0:2] - full_file_path = os.path.join(hosted_files_path, file_subdirectory, file_hash_filename) - log.info(f'File Hash with Subdirectory: {full_file_path}') - - hosted_tmp_path = settings.FILES_PATH['hosted_tmp_root'] - log.info(f'Hosted Tmp Path: {hosted_tmp_path}') - log.debug(shutil.disk_usage(hosted_tmp_path)) - - hosted_tmp_convert_file_path = os.path.join(hosted_tmp_path, 'convert_file') - if pathlib.Path(hosted_tmp_convert_file_path): - log.info('Hosted tmp convert file path found') - else: - log.info('Creating hosted tmp convert file path') - pathlib.Path(hosted_tmp_convert_file_path).mkdir(parents=True, exist_ok=True) - - # Run the ffmpeg command to clip a video file based on the start and end times given - log.info('Run the ffmpeg command to clip a video file based on the start and end times given') - - hosted_file_dict = None - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmp_video_file_clip: - tmp_video_file_clip_path = tmp_video_file_clip.name - - # NOTE: It seems very important that the -y argument is used with ffmpeg run by subprocess.run(). Otherwise the process will hang. - # NOTE: This is a blocking process. It will take a while to complete. - if scale_down: - new_video_file_clip_filename = f'{filename_no_ext}_[clip_scaled].{to_type}' - log.debug(new_video_file_clip_filename) - - cmd = f'ffmpeg -hide_banner -loglevel error -nostats -y -i {full_file_path} -ss {start_time} -to {end_time} -vf "scale=w=1920:h=1080:force_original_aspect_ratio=decrease" -c:v libx264 -crf 23 -maxrate 2M -bufsize 2M -c:a copy -movflags +faststart {tmp_video_file_clip_path}' - elif reencode: - new_video_file_clip_filename = f'{filename_no_ext}_[clip_reencode].{to_type}' - log.debug(new_video_file_clip_filename) - - cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -i {full_file_path} -ss {start_time} -to {end_time} -c:v libx264 -crf 23 -maxrate 2M -bufsize 2M -c:a copy -movflags +faststart {tmp_video_file_clip_path}" - else: - new_video_file_clip_filename = f'{filename_no_ext}_[clip].{to_type}' - log.debug(new_video_file_clip_filename) - - cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -i {full_file_path} -ss {start_time} -to {end_time} -c:v copy -c:a copy -movflags +faststart {tmp_video_file_clip_path}" - - log.debug(cmd) - - # Run the ffmpeg command - args = shlex.split(cmd) - try: - result = subprocess.run(args, check=True, capture_output=True, text=True) - log.debug(result.stdout) - except subprocess.CalledProcessError as e: - log.exception('Error running ffmpeg command') - return {'success': False, 'status_message': f'Error running ffmpeg command: {e}'} - - - # *** Part 2: *** Save the converted hashed file to hosted_files directory. - - file_info = await save_file_to_hosted_file( - file_path = tmp_video_file_clip_path, - filename = new_video_file_clip_filename, - extension = to_type, - account_id = account_id, - link_to_type = link_to_type, - link_to_id = link_to_id, - ) - - # *** Part 3: *** Save information to database in hosted_file table (hosted_file_link table will be updated by an event_file table trigger) - - new_hosted_file_id = None - if file_info.get('saved'): - # NOTE: Just in case look up in DB based on hash - log.info('Look up in DB based on hash...') - if hosted_file_sel_result := sql_select( - table_name = 'hosted_file', - field_name = 'hash_sha256', - field_value = file_info['hash_sha256'], - ): - log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') - # Got existing host_file object_entry! - # Odd... the hash was found in the database, but the file had to be copied again. - # If this happens then the file on the host server was probably deleted at some point. - new_hosted_file_id = hosted_file_sel_result.get('id', None) - # hosted_file_id_random = hosted_file_sel_result.get('id_random', None) - hosted_file_dict = load_hosted_file_obj(hosted_file_id=new_hosted_file_id, model_as_dict=True) - else: - # This is normal since the file was not found on the host server and not found in the DB. - # Create a new host_file object entry and new host_file.id_random. - file_info['account_id'] = account_id - # file_info['account_id_random'] = account_id_random - hosted_file_obj = Hosted_File_Base(**file_info) - if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): - new_hosted_file_id = hosted_file_obj_result - hosted_file_dict = load_hosted_file_obj(hosted_file_id=new_hosted_file_id, model_as_dict=True) - else: - log.warning('For some reason a host_file object entry could not be created.') - new_hosted_file_id = None - hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member - log.debug(hosted_file_obj_result) - - # NOTE: Currently sql_insert does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID. - if link_to_type in ['event', 'event_location', 'event_session', 'event_presentation', 'event_presenter', 'event_badge', 'event_exhibit', 'event_person']: - log.info('File is for event module. Trigger will create the hosted_file_link record.') - elif create_hosted_file_link( - account_id = account_id, - hosted_file_id = new_hosted_file_id, # This for the new file created - link_to_type = link_to_type, - link_to_id = link_to_id, - ): - log.info('The hosted file link was created.') - pass # This if statement should be improved - else: - # This if statement should be improved - log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.') - log.debug('Something may have gone wrong while trying to create the hosted_file_link record.') - log.debug('The hosted_file_link was probably created fine though.') - log.debug(hosted_file_sel_result) - else: return False - - log.debug(hosted_file_dict) - return mk_resp(data=hosted_file_dict, response=commons.response) -# ### END ### API Hosted File ### clip_video() ### - - -# Updated 2024-01-01 -@router.post('/clip_video_v1') -async def clip_video_v1( - video_file: UploadFile = File(...), - start_time: str = Form(...), - end_time: str = Form(...), - reencode: bool = Form(False), # reencode: bool = False, -): - log.setLevel(logging.DEBUG) - log.debug(locals()) - - video_file_name = video_file.filename - log.debug(video_file_name) - - (video_file_orig_filename, video_file_orig_extension) = os.path.splitext(video_file_name) - - # Save the uploaded audio file to a temporary directory - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmp_video_file: - tmp_video_file.write(await video_file.read()) - tmp_video_file_orig_path = tmp_video_file.name - - # cmd = f"ls -lha /tmp" - # args = shlex.split(cmd) - # try: - # result = subprocess.run(args, check=True, capture_output=True, text=True) - # log.debug(result.stdout) - # except subprocess.CalledProcessError as e: - # log.exception('Error running ls command') - # return {'success': False, 'status_message': f'Error running ls command: {e}'} - - # Run the ffmpeg command to clip a video file based on the start and end times given - log.info('Run the ffmpeg command to clip a video file based on the start and end times given') - - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmp_video_file_clip: - tmp_video_file_clip_path = tmp_video_file_clip.name - - # NOTE: It seems very important that the -y argument is used with ffmpeg run by subprocess.run(). Otherwise the process will hang. - # NOTE: This is a blocking process. It will take a while to complete. - if reencode: - new_video_file_clip_filename = f'{video_file_orig_filename}_[clip_reencode]{video_file_orig_extension}' - log.debug(new_video_file_clip_filename) - - cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -i {tmp_video_file_orig_path} -ss {start_time} -to {end_time} -c:v libx264 -crf 23 -maxrate 2M -bufsize 2M -c:a copy -movflags +faststart {tmp_video_file_clip_path}" - else: - new_video_file_clip_filename = f'{video_file_orig_filename}_[clip]{video_file_orig_extension}' - log.debug(new_video_file_clip_filename) - - cmd = f"ffmpeg -hide_banner -loglevel error -nostats -y -i {tmp_video_file_orig_path} -ss {start_time} -to {end_time} -c:v copy -c:a copy -movflags +faststart {tmp_video_file_clip_path}" - - log.debug(cmd) - - # Run the ffmpeg command - args = shlex.split(cmd) - try: - result = subprocess.run(args, check=True, capture_output=True, text=True) - log.debug(result.stdout) - except subprocess.CalledProcessError as e: - log.exception('Error running ffmpeg command') - return {'success': False, 'status_message': f'Error running ffmpeg command: {e}'} - - # Return the new mp4 file for download - return FileResponse(tmp_video_file_clip_path, media_type='video/mp4', filename=f'{new_video_file_clip_filename}') + video_name = f"{tempfile.gettempdir()}/{title_part_1}.mp4" + cmd = f"ffmpeg -y -loop 1 -i {title_image_path} -i {audio_file_path} -c:a copy -c:v libx264 -shortest {video_name}" + subprocess.run(shlex.split(cmd), check=True) + + return FileResponse(video_name, media_type='video/mp4', filename=f'{title_part_1}.mp4') diff --git a/tests/e2e/test_e2e_v3_actions_file_lifecycle.py b/tests/e2e/test_e2e_v3_actions_file_lifecycle.py index 444fa20..34f7968 100644 --- a/tests/e2e/test_e2e_v3_actions_file_lifecycle.py +++ b/tests/e2e/test_e2e_v3_actions_file_lifecycle.py @@ -63,7 +63,12 @@ def test_file_lifecycle(): # 4. DELETE (Clean Cleanup) print("\n[Step 4] Deleting test file (rm_orphan=true)...") - del_params = {"rm_orphan": "true", "method": "delete"} + del_params = { + "link_to_type": LINK_TYPE, + "link_to_id": LINK_ID, + "rm_orphan": "true", + "method": "delete" + } del_resp = requests.delete(f"{API_BASE}/hosted_file/{file_id}", headers=get_headers(), params=del_params) if del_resp.status_code == 200: print(f" ✅ Deletion request successful.")