From eb23d16ad5714816dabc3ec3b7cbb042072b0f86 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Mon, 14 Jun 2021 16:04:37 -0400 Subject: [PATCH] Unplanned work for file uploads --- app/routers/account.py | 4 +- app/routers/api_crud.py | 2 +- app/routers/event.py | 4 +- app/routers/event_person.py | 4 +- app/routers/hosted_file.py | 213 ++++++++++++++++++++++++++---------- app/routers/person.py | 4 +- app/routers/user.py | 4 +- 7 files changed, 164 insertions(+), 71 deletions(-) diff --git a/app/routers/account.py b/app/routers/account.py index 6b95adf..ef2235d 100644 --- a/app/routers/account.py +++ b/app/routers/account.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template @@ -13,7 +13,7 @@ from app.methods.account_methods import load_account_obj from app.methods.account_cfg_methods import load_account_cfg_obj from app.models.account_models import Account_Base -from app.models.response_models import * +from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() diff --git a/app/routers/api_crud.py b/app/routers/api_crud.py index 43adae0..9b02875 100644 --- a/app/routers/api_crud.py +++ b/app/routers/api_crud.py @@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import * from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random, lookup_id_random_pop from app.models.response_models import * diff --git a/app/routers/event.py b/app/routers/event.py index 65e0966..4cca8a2 100644 --- a/app/routers/event.py +++ b/app/routers/event.py @@ -5,14 +5,14 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import * from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.event_methods import load_event_obj, update_event_obj from app.models.event_models import Event_Base -from app.models.response_models import * +from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() diff --git a/app/routers/event_person.py b/app/routers/event_person.py index 69ff77b..68e66e6 100644 --- a/app/routers/event_person.py +++ b/app/routers/event_person.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template @@ -16,7 +16,7 @@ from app.methods.user_methods import create_user_obj, load_user_obj, update_user from app.models.event_person_models import Event_Person_New_Base, Event_Person_Base from app.models.person_models import Person_Base -from app.models.response_models import * +from app.models.response_models import Resp_Body_Base, mk_resp from app.models.user_models import User_New_Base, User_Base diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index edefa1f..4d391db 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -1,87 +1,84 @@ -import datetime, shutil, time +from __future__ import annotations +import datetime, hashlib, os, pathlib, shutil, time #from datetime import datetime, time, timedelta from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union -from app.lib_general import * -from ..log import * +from app.lib_general import log, logging from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random -from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template +# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.models.hosted_file_models import Hosted_File_Base -from app.models.response_models import * +from app.models.response_models import mk_resp router = APIRouter() # This just needs to return the currect model for a hosted_file -# Everything else seems to be workign well +# Everything else seems to be working well # Should this also do something with meta data and updating the DB? @router.post('/upload_files/') async def create_upload_files( - files: List[UploadFile] = File(...), + file_list: List[UploadFile] = File(...), account_id: str = Form(..., min_length=1, max_length=22), - filename: Optional[str] = Form(...), - object_type: Optional[str] = Form(...), - object_id: Optional[str] = Form(..., min_length=1, max_length=22), + # filename: Optional[str] = Form(...), + for_object_type: str = Form(...), + for_object_id: str = Form(..., min_length=1, max_length=22), x_account_id: Optional[str] = Header(..., ), return_obj: Optional[bool] = True, by_alias: Optional[bool] = True, exclude_unset: Optional[bool] = True, ): - data = {} - data['account_id'] = account_id - #data['filename'] = filename - data['object_type'] = object_type - data['object_id'] = object_id - data['file_li'] = [] + account_id_random = account_id # This is for the account random str ID + if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass + else: + return mk_resp(data=None, status_code=404) - log.debug(await save_file_li(files)) + for_object_type = for_object_type + for_object_id_random = for_object_id # This is for the object random str ID + if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass + else: + return mk_resp(data=None, status_code=404) - response_li = [] - for file in files: - file_info = {} - file_info['filename'] = file.filename - file_info['content_type'] = file.content_type + # file_info_li = await save_file_li(file_list, for_object_type, for_object_id, for_object_id_random) - file.file.seek(0, os.SEEK_END) - file_size = file.file.tell() - file.file.seek(0) # The file will not properly save if seek is not reset to 0. - file_info['size'] = file_size - log.debug(file_size) + file_info_list = [] + for file_obj in file_list: + file_info = await save_file(file=file_obj, account_id=account_id, account_id_random=account_id_random, for_object_type=for_object_type, for_object_id=for_object_id, for_object_id_random=for_object_id_random) + file_info_list.append(file_info) - data['file_li'].append(file_info) - - return mk_resp(data=data) - - #response = { 'response_data': response_data, 'response_li':response_li } - #response['filenames'] = [file.filename for file in files] - #return response - #return {'filenames': [file.filename for file in files]} + log.debug(file_info_list) + return mk_resp(data=file_info_list) # This is not needed. Just for reference of the mk_resp() function -@router.post('/', response_model=Resp_Body_Base) -async def post_upload_file( - account_id: str = Query(..., min_length=1, max_length=22), +# @router.post('/', response_model=Resp_Body_Base) +# async def post_upload_file( +# account_id: str = Query(..., min_length=1, max_length=22), +# ): +# if return_obj: +# if order_cart_obj := load_order_cart_obj(order_cart_id=order_cart_id, inc_order_cart_line_li=inc_order_cart_line_li, inc_order_cart_cfg=inc_order_cart_cfg): +# data = order_cart_obj.dict(by_alias=True, exclude_unset=False) +# return mk_resp(data=data) +# else: +# return mk_resp(data=False, status_code=404) # Not Found +# else: +# return mk_resp(data=True) + + +async def save_file_li( + file_li: List[UploadFile], # = File(...), + account_id: int, + account_id_random: str, + for_object_type: str, + for_object_id: int, + for_object_id_random: str, ): - if return_obj: - if order_cart_obj := load_order_cart_obj(order_cart_id=order_cart_id, inc_order_cart_line_li=inc_order_cart_line_li, inc_order_cart_cfg=inc_order_cart_cfg): - data = order_cart_obj.dict(by_alias=True, exclude_unset=False) - return mk_resp(data=data) - else: - return mk_resp(data=False, status_code=404) # Not Found - else: - return mk_resp(data=True) - - - -async def save_file_li(file_li: List[UploadFile] = File(...)): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -89,18 +86,32 @@ async def save_file_li(file_li: List[UploadFile] = File(...)): log.debug(shutil.disk_usage(hosted_file_path)) - result = [] - for file in file_li: + file_info_li = [] + for file in file_li.g: + log.debug(file) log.debug(f'{file.filename}') + file_info = {} + file_info['for_object_type'] = for_object_type + file_info['for_object_id'] = for_object_id + file_info['for_object_id_random'] = for_object_id_random + file_info['filename'] = file.filename + + # There is a difference between Content-Type and MIME type. + # https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype + file_info['content_type'] = file.content_type # might also include charset or other parameters + file_info['mimetype'] = file.mimetype + file.file.seek(0, os.SEEK_END) file_size = file.file.tell() file.file.seek(0) # The file will not properly save if seek is not reset to 0. #file_info['size'] = request_file_size log.debug(file_size) + file_info['size'] = file_size file_hash = await get_file_object_hash(file.file) log.debug(file_hash) + file_info['hash_sha256'] = file_hash # 16384 bytes is the default # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes @@ -112,6 +123,89 @@ async def save_file_li(file_li: List[UploadFile] = File(...)): #file_dest = f'{hosted_file_path}{file.filename}' file_dest = f'{hosted_file_path}{file_hash}.file' + + existing_file_check = pathlib.Path(file_dest) + + if existing_file_check.exists(): + file_info['already_exists'] = True + file_info['copy_timer'] = 0 + else: + file_info['already_exists'] = False + f_dest = open(file_dest, 'wb') + + timer_start = time.process_time() + shutil.copyfileobj(f_src, f_dest, buffer_size) + timer_end = time.process_time() + elapsed_time = timer_end - timer_start + log.debug(f'Elapsed time: {elapsed_time}') + file_info['copy_timer'] = elapsed_time + # result.append({ 'filename': file.filename }) + file_info_li.append(file_info) + + log.debug(shutil.disk_usage(hosted_file_path)) + + return file_info_li + + +async def save_file( + file: UploadFile, + account_id: int, + account_id_random: str, + for_object_type: str, + for_object_id: int, + for_object_id_random: str, + ): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + hosted_file_path = '/home/scott/tmp/hosted_file_dev/' + + log.debug(shutil.disk_usage(hosted_file_path)) + + log.debug(dir(file)) + log.debug(f'{file.filename}') + + file_info = {} + file_info['for_object_type'] = for_object_type + file_info['for_object_id'] = for_object_id + file_info['for_object_id_random'] = for_object_id_random + file_info['filename'] = file.filename + file_info['extension'] = guess_file_extension(filename=file.filename) + + # There is a difference between Content-Type and MIME type. + # https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype + file_info['content_type'] = file.content_type # might also include charset or other parameters + # file_info['mimetype'] = file.mimetype # This may need to be filled in a different way? + + file.file.seek(0, os.SEEK_END) + file_size = file.file.tell() + file.file.seek(0) # The file will not properly save if seek is not reset to 0. + #file_info['size'] = request_file_size + log.debug(file_size) + file_info['size'] = file_size + + file_hash = await get_file_object_hash(file.file) + log.debug(file_hash) + file_info['hash_sha256'] = file_hash + + # 16384 bytes is the default + # 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes + buffer_size = 524288 + + #file_src = file.filename + #f_src = open(file_src, 'rb') + f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open" + + #file_dest = f'{hosted_file_path}{file.filename}' + file_dest = f'{hosted_file_path}{file_hash}.file' + + existing_file_check = pathlib.Path(file_dest) + + if existing_file_check.exists(): + file_info['already_exists'] = True + file_info['copy_timer'] = 0 + else: + file_info['already_exists'] = False f_dest = open(file_dest, 'wb') timer_start = time.process_time() @@ -119,15 +213,14 @@ async def save_file_li(file_li: List[UploadFile] = File(...)): timer_end = time.process_time() elapsed_time = timer_end - timer_start log.debug(f'Elapsed time: {elapsed_time}') - - result.append({ 'filename': file.filename }) + file_info['copy_timer'] = elapsed_time log.debug(shutil.disk_usage(hosted_file_path)) - return result + return file_info -async def get_file_object_hash(file_object): +async def get_file_object_hash(file_object:File): #log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -143,7 +236,7 @@ async def get_file_object_hash(file_object): file_object.seek(0) # The file will not properly save if seek is not reset to 0. timer_end = time.process_time() elapsed_time = timer_end - timer_start - #log.debug(f'Elapsed time: {elapsed_time}') + log.debug(f'Elapsed time: {elapsed_time}') return file_hash @@ -153,8 +246,8 @@ async def get_file_object_hash(file_object): # return '.' in filename and filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS'] -# def guess_file_extension(filename): -# return filename.rsplit('.', 1)[1].lower() +def guess_file_extension(filename:str): + return filename.rsplit('.', 1)[1].lower() # def copyLargeFile(src, dest, buffer_size=16000): # with open(src, 'rb') as fsrc: diff --git a/app/routers/person.py b/app/routers/person.py index 2c2fd60..89f49b7 100644 --- a/app/routers/person.py +++ b/app/routers/person.py @@ -5,14 +5,14 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.person_methods import load_person_obj, update_person_obj from app.models.person_models import Person_Base -from app.models.response_models import * +from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() diff --git a/app/routers/user.py b/app/routers/user.py index dc10f0a..729b2d4 100644 --- a/app/routers/user.py +++ b/app/routers/user.py @@ -5,14 +5,14 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, secure_hash_string, verify_secure_hash_string from app.config import settings -from app.db_sql import * +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.user_methods import load_user_obj from app.models.common_field_schema import default_num_bytes -from app.models.response_models import * +from app.models.response_models import Resp_Body_Base, mk_resp from app.models.user_models import User_Base, User_New_Base, User_Out_Base