from __future__ import annotations import datetime, hashlib, os, pathlib, shutil, time #from datetime import datetime, time, timedelta from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random # from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.models.hosted_file_models import Hosted_File_Base from app.models.response_models import mk_resp router = APIRouter() # ### BEGIN ### API Hosted File Route ### upload_files() ### # This just needs to return the currect model for a hosted_file # Everything else seems to be working well # Should this also do something with meta data and updating the DB? @router.post('/upload_files/') async def upload_files( file_list: List[UploadFile] = File(...), account_id: str = Form(..., min_length=1, max_length=22), # filename: Optional[str] = Form(...), for_object_type: str = Form(...), for_object_id: str = Form(..., min_length=1, max_length=22), check_allowed_extension: bool = False, x_account_id: str = Header(..., ), return_obj: bool = True, by_alias: bool = True, exclude_unset: bool = True, ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id_random = account_id # This is for the account random str ID if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass else: return mk_resp(data=None, status_code=400) for_object_type = for_object_type for_object_id_random = for_object_id # This is for the object random str ID if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass else: return mk_resp(data=None, status_code=400) file_info_list = [] for file_obj in file_list: file_info = await save_file( file = file_obj, account_id = account_id, account_id_random = account_id_random, for_object_type = for_object_type, for_object_id = for_object_id, for_object_id_random = for_object_id_random, check_allowed_extension = check_allowed_extension, ) if file_info['saved']: # Create a new host_file object entry if file_info['already_exists']: # Look up in DB based on hash # Get existing host_file object_entry and existing host_file.id_random. if hosted_file_sel_result := sql_select( table_name = 'hosted_file', field_name = 'hash_sha256', filed_value = file_info['hash_sha256'], ): file_info['id_random'] = hosted_file_result.get('id_random', None) hosted_file_obj = Hosted_File_Base(**file_info) else: # SOMETHING WENT WRONG # Going to try and create a new host_file entry... log.warning('For some reason a host_file object entry with the has was not found.') # file_info['id_random'] = None hosted_file_dict = Hosted_File_Base(**file_info).dict(by_alias=False, exclude_defaults=False, exclude_unset=True) if hosted_file_in_result := sql_insert(data=hosted_file_dict, table_name='hosted_file'): hosted_file.id_random = hosted_file_result.get('id_random', None) else: log.warning('For some reason a host_file object entry could not be created.') hosted_file.id_random = None log.debug(hosted_file_in_result) log.debug(hosted_file_sel_result) else: # Just in case look up in DB based on hash if hosted_file_sel_result := sql_select( table_name = 'hosted_file', field_name = 'hash_sha256', filed_value = file_info['hash_sha256'], ): log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') # Got existing host_file object_entry! # Odd... the hash was found in the database, but the file had to be copied again. # If this happens then the file on the host server was probably deleted at some point. file_info['id_random'] = hosted_file_result.get('id_random', None) hosted_file_obj = Hosted_File_Base(**file_info) else: # This is normal since the file was not found on the host server and not found in the DB. # Create a new host_file object entry and new host_file.id_random. hosted_file_dict = Hosted_File_Base(**file_info).dict(by_alias=False, exclude_defaults=False, exclude_unset=True) if hosted_file_in_result := sql_insert(data=hosted_file_dict, table_name='hosted_file'): file_info['id_random'] = hosted_file_result.get('id_random', None) else: log.warning('For some reason a host_file object entry could not be created.') file_info['id_random'] = None log.debug(hosted_file_in_result) log.debug(hosted_file_sel_result) else: file_info['id_random'] = None # file_info_obj = Hosted_File_Base(**file_info) file_info_list.append(file_info_obj) log.debug(file_info_list) return mk_resp(data=file_info_list) # ### END ### API Hosted File Route ### upload_files() ### # ### BEGIN ### API Hosted File Route ### save_file() ### async def save_file( file: UploadFile, account_id: int, account_id_random: str, for_object_type: str, for_object_id: int, for_object_id_random: str, check_allowed_extension: bool = False, ): # log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) hosted_file_path = '/home/scott/tmp/hosted_file_dev/' log.debug(shutil.disk_usage(hosted_file_path)) log.debug(dir(file)) log.debug(f'{file.filename}') file_info = {} file_info['saved'] = None file_info['for_object_type'] = for_object_type file_info['for_object_id'] = for_object_id file_info['for_object_id_random'] = for_object_id_random file_info['filename'] = file.filename file_info['extension'] = guess_file_extension(filename=file.filename) if check_allowed_extension: if allowed_file_extension: file_info['extension_allowed'] = True else: file_info['extension_allowed'] = False file_info['saved'] = False return file_info else: file_info['extension_allowed'] = None # There is a difference between Content-Type and MIME type. # https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype file_info['content_type'] = file.content_type # might also include charset or other parameters # file_info['mimetype'] = file.mimetype # This may need to be filled in a different way? file.file.seek(0, os.SEEK_END) file_size = file.file.tell() file.file.seek(0) # The file will not properly save if seek is not reset to 0. log.debug(file_size) file_info['size'] = file_size file_hash = await get_file_object_hash(file.file) log.debug(file_hash) file_info['hash_sha256'] = file_hash # 16384 bytes is the default # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes buffer_size = 524288 #f_src = open(file_src, 'rb') f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open" #file_dest = f'{hosted_file_path}{file.filename}' file_dest = f'{hosted_file_path}{file_hash}.file' existing_file_check = pathlib.Path(file_dest) if existing_file_check.exists(): file_info['already_exists'] = True file_info['copy_timer'] = 0 else: file_info['already_exists'] = False f_dest = open(file_dest, 'wb') timer_start = time.process_time() shutil.copyfileobj(f_src, f_dest, buffer_size) timer_end = time.process_time() elapsed_time = timer_end - timer_start log.debug(f'Elapsed time: {elapsed_time}') file_info['copy_timer'] = elapsed_time file_info['saved'] = True log.debug(shutil.disk_usage(hosted_file_path)) return file_info # ### END ### API Hosted File Route ### save_file() ### # ### BEGIN ### API Hosted File Route ### get_file_object_hash() ### async def get_file_object_hash(file_object:File): #log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # 4096 bytes is the current block size on my workstation and Linode server # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes block_size = 131072 hash_value = hashlib.sha256() timer_start = time.process_time() for chunk in iter(lambda: file_object.read(block_size), b""): hash_value.update(chunk) file_hash = hash_value.hexdigest() file_object.seek(0) # The file will not properly save if seek is not reset to 0. timer_end = time.process_time() elapsed_time = timer_end - timer_start log.debug(f'Elapsed time: {elapsed_time}') return file_hash # ### END ### API Hosted File Route ### get_file_object_hash() ### # ### BEGIN ### API Hosted File Route ### guess_file_extension() ### def guess_file_extension(filename:str): return filename.rsplit('.', 1)[1].lower() # ### END ### API Hosted File Route ### guess_file_extension() ### # ### BEGIN ### API Hosted File Route ### allowed_file_extension() ### def allowed_file_extension(extension:str): return extension.lower() in app.config['ALLOWED_EXTENSIONS'] # ### END ### API Hosted File Route ### allowed_file_extension() ### # ### BEGIN ### API Hosted File Route ### save_file() ### async def hosted_file_link( account_id: int, hosted_file_id: str, for_object_type: str, for_object_id: int, for_object_id_random: str, ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) hosted_file_link_data = {} hosted_file_link_data['account_id'] = account_id hosted_file_link_data['hosted_file_id'] = hosted_file_id hosted_file_link_data['object_type'] = for_object_type hosted_file_link_data['object_id'] = for_object_id if response['data']['id'] == True: #print('Tried to insert a new hosted_file record, but there is a duplicate.') # There was likely a record with the same hash value. table_name = 'hosted_file' field_name = 'hash_sha256' field_value = data['hash_sha256'] select_hosted_file_response = select_record(table_name=table_name, field_name=field_name, field_value=field_value) if select_hosted_file_response: hosted_file_link_data['hosted_file_id'] = select_hosted_file_response['id'] response['data']['id'] = select_hosted_file_response['id'] response['data']['id_random'] = select_hosted_file_response['id_random'] else: return False else: print('Inserted new host_file record.') pass table_name = 'hosted_file_link' hosted_file_link_response = sql_insert_for_rest(data=hosted_file_link_data, table_name=table_name, sql=None, model=None, resource_ref=True) # ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ### def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'user', 'created_on', 'updated_on'}) if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass else: return False log.debug(hosted_file_obj_in_result) hosted_file_id = hosted_file_obj_in_result log.debug(f'Returning the new hosted_file_id: {hosted_file_id}') return hosted_file_id # ### END ### API Hosted File Methods ### create_hosted_file_obj() ###