from __future__ import annotations import datetime, hashlib, os, pathlib, shutil, time #from datetime import datetime, time, timedelta from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random # from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.models.hosted_file_models import Hosted_File_Base from app.models.response_models import mk_resp router = APIRouter() # This just needs to return the currect model for a hosted_file # Everything else seems to be working well # Should this also do something with meta data and updating the DB? @router.post('/upload_files/') async def create_upload_files( file_list: List[UploadFile] = File(...), account_id: str = Form(..., min_length=1, max_length=22), # filename: Optional[str] = Form(...), for_object_type: str = Form(...), for_object_id: str = Form(..., min_length=1, max_length=22), x_account_id: Optional[str] = Header(..., ), return_obj: Optional[bool] = True, by_alias: Optional[bool] = True, exclude_unset: Optional[bool] = True, ): account_id_random = account_id # This is for the account random str ID if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass else: return mk_resp(data=None, status_code=400) for_object_type = for_object_type for_object_id_random = for_object_id # This is for the object random str ID if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass else: return mk_resp(data=None, status_code=400) # file_info_li = await save_file_li(file_list, for_object_type, for_object_id, for_object_id_random) file_info_list = [] for file_obj in file_list: file_info = await save_file(file=file_obj, account_id=account_id, account_id_random=account_id_random, for_object_type=for_object_type, for_object_id=for_object_id, for_object_id_random=for_object_id_random) file_info_list.append(file_info) log.debug(file_info_list) return mk_resp(data=file_info_list) async def save_file( file: UploadFile, account_id: int, account_id_random: str, for_object_type: str, for_object_id: int, for_object_id_random: str, ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) hosted_file_path = '/home/scott/tmp/hosted_file_dev/' log.debug(shutil.disk_usage(hosted_file_path)) log.debug(dir(file)) log.debug(f'{file.filename}') file_info = {} file_info['for_object_type'] = for_object_type file_info['for_object_id'] = for_object_id file_info['for_object_id_random'] = for_object_id_random file_info['filename'] = file.filename file_info['extension'] = guess_file_extension(filename=file.filename) # There is a difference between Content-Type and MIME type. # https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype file_info['content_type'] = file.content_type # might also include charset or other parameters # file_info['mimetype'] = file.mimetype # This may need to be filled in a different way? file.file.seek(0, os.SEEK_END) file_size = file.file.tell() file.file.seek(0) # The file will not properly save if seek is not reset to 0. log.debug(file_size) file_info['size'] = file_size file_hash = await get_file_object_hash(file.file) log.debug(file_hash) file_info['hash_sha256'] = file_hash # 16384 bytes is the default # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes buffer_size = 524288 #f_src = open(file_src, 'rb') f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open" #file_dest = f'{hosted_file_path}{file.filename}' file_dest = f'{hosted_file_path}{file_hash}.file' existing_file_check = pathlib.Path(file_dest) if existing_file_check.exists(): file_info['already_exists'] = True file_info['copy_timer'] = 0 else: file_info['already_exists'] = False f_dest = open(file_dest, 'wb') timer_start = time.process_time() shutil.copyfileobj(f_src, f_dest, buffer_size) timer_end = time.process_time() elapsed_time = timer_end - timer_start log.debug(f'Elapsed time: {elapsed_time}') file_info['copy_timer'] = elapsed_time log.debug(shutil.disk_usage(hosted_file_path)) return file_info async def get_file_object_hash(file_object:File): #log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # 4096 bytes is the current block size on my workstation and Linode server # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes block_size = 131072 hash_value = hashlib.sha256() timer_start = time.process_time() for chunk in iter(lambda: file_object.read(block_size), b""): hash_value.update(chunk) file_hash = hash_value.hexdigest() file_object.seek(0) # The file will not properly save if seek is not reset to 0. timer_end = time.process_time() elapsed_time = timer_end - timer_start log.debug(f'Elapsed time: {elapsed_time}') return file_hash # def allowed_file_extension(filename): # return False # return '.' in filename and filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS'] def guess_file_extension(filename:str): return filename.rsplit('.', 1)[1].lower() # def copyLargeFile(src, dest, buffer_size=16000): # with open(src, 'rb') as fsrc: # with open(dest, 'wb') as fdest: # shutil.copyfileobj(fsrc, fdest, buffer_size)