From ada4a75ecb87764ab78899611ed268e2c1981350 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Sat, 22 May 2021 03:58:29 -0400 Subject: [PATCH] Working on file uploads. Finally! --- app/main.py | 10 ++- app/routers/hosted_file.py | 156 +++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 app/routers/hosted_file.py diff --git a/app/main.py b/app/main.py index 6bd5208..0a154e7 100644 --- a/app/main.py +++ b/app/main.py @@ -18,7 +18,7 @@ from .lib_general import * from .log import * # Import the routers here first: -from .routers import api_crud, api, account, address, archive, archive_content, contact, event, event_exhibit, event_registration, event_session, flask_cfg, journal, journal_entry, lookup, membership, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals +from .routers import api_crud, api, account, address, archive, archive_content, contact, event, event_exhibit, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, lookup, membership, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals from .db_sql import db @@ -142,6 +142,14 @@ app.include_router( #dependencies=[Depends(get_account_header)], #responses={404: {'description': 'Not found'}}, ) +app.include_router( + hosted_file.router, + prefix='/hosted_file', + tags=['Hosted File'], + #dependencies=[Depends(get_token_header)], + #dependencies=[Depends(get_account_header)], + #responses={404: {'description': 'Not found'}}, +) app.include_router( journal.router, prefix='/journal', diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py new file mode 100644 index 0000000..ea235ca --- /dev/null +++ b/app/routers/hosted_file.py @@ -0,0 +1,156 @@ +import datetime, shutil, time +#from datetime import datetime, time, timedelta +from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile +from pydantic import BaseModel, EmailStr, Field +from typing import Dict, List, Optional, Set, Union + +from ..lib_general import * +from ..log import * +from app.config import settings +from app.db_sql import * + +from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template + +from ..models.hosted_file_model import Hosted_File_Base +from ..models.response_model import * + + +router = APIRouter() + + +@router.post('/upload_files/') +async def create_upload_files(files: List[UploadFile] = File(...), field_1: str = Form(...), field_2: str = Form(...)): + + response_data = {} + response_data['field_1'] = field_1 + response_data['field_2'] = field_2 + + log.debug(await save_file_li(files)) + + response_li = [] + for file in files: + response_detailed = {} + response_detailed['filename'] = file.filename + response_detailed['content_type'] = file.content_type + response_li.append(response_detailed) + + response = { 'response_data': response_data, 'response_li':response_li } + #response['filenames'] = [file.filename for file in files] + return response + return {'filenames': [file.filename for file in files]} + + +@router.post("/upload_files_form/") +async def create_upload_files_form( + file: bytes = File(...), fileb: UploadFile = File(...), token: str = Form(...) +): + return { + "file_size": len(file), + "token": token, + "fileb_content_type": fileb.content_type, + } + + +@router.post('/', response_model=Resp_Body_Base) +async def post_upload_file( + account_id: str = Query(..., min_length=1, max_length=22), + hosted_file_obj: Hosted_File_Base = None, + x_account_id: Optional[str] = Header(..., ), + return_obj: Optional[bool] = True, + by_alias: Optional[bool] = True, + exclude_unset: Optional[bool] = True, + ): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + log.debug(order_cart_obj) + + order_cart_obj_up_result = update_order_cart_obj(order_cart_obj=order_cart_obj, repl_order_cart_line_li=repl_order_cart_line_li) + if isinstance(order_cart_obj_up_result, int): + log.info(f'Order cart update and the result was an int: {order_cart_obj_up_result}') + pass + elif isinstance(order_cart_obj_up_result, bool) and order_cart_obj_up_result: + log.info(f'Order cart update and the result was an bool: {order_cart_obj_up_result}') + pass + elif isinstance(order_cart_obj_up_result, bool) and not order_cart_obj_up_result: + log.error(f'Order cart update and the result was an bool: {order_cart_obj_up_result}') + return mk_resp(data=False, status_code=500) # Internal Server Error + + if return_obj: + if order_cart_obj := load_order_cart_obj(order_cart_id=order_cart_id, inc_order_cart_line_li=inc_order_cart_line_li, inc_order_cart_cfg=inc_order_cart_cfg): + data = order_cart_obj.dict(by_alias=True, exclude_unset=False) + return mk_resp(data=data) + else: + return mk_resp(data=False, status_code=404) # Not Found + else: + return mk_resp(data=True) + + +async def save_file(file: UploadFile = File(...)): + with open(file.filename, 'wb') as buffer: + shutil.copyfileobj(file.file, buffer) + + return {'filename': file.filename} + + +async def save_file_li(file_li: List[UploadFile] = File(...)): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + hosted_file_path = '/home/scott/tmp/hosted_file_dev/' + + log.debug(shutil.disk_usage(hosted_file_path)) + + result = [] + for file in file_li: + log.debug(f'{file.filename} | {file.spool_max_size}') + + file_hash = await get_file_object_hash(file.file) + log.debug(file_hash) + + buffer_size = 16000 + + #file_src = file.filename + #f_src = open(file_src, 'rb') + f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open" + + #file_dest = f'{hosted_file_path}{file.filename}' + file_dest = f'{hosted_file_path}{file_hash}.file' + f_dest = open(file_dest, 'wb') + + shutil.copyfileobj(f_src, f_dest, buffer_size) + + #with open(file.filename, 'wb') as file_dest: + # shutil.copyfileobj(file.file, file_dest, buffer_size) + + result.append({ 'filename': file.filename }) + + log.debug(shutil.disk_usage(hosted_file_path)) + + return result + + +async def get_file_object_hash(file_object): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + # 4096 bytes is the current block size on my workstation and Linode server + block_size = 131072 # 4096 8192 16384 32768 65536 bytes + hash_md5 = hashlib.sha256() + + timer_start = time.process_time() + for chunk in iter(lambda: file_object.read(block_size), b""): + hash_md5.update(chunk) + file_hash = hash_md5.hexdigest() + file_object.seek(0) # The file will not properly save if seek is not reset to 0. + timer_end = time.process_time() + elapsed_time = timer_end - timer_start + log.debug(f'Elapsed time: {elapsed_time}') + + return file_hash + + +# def copyLargeFile(src, dest, buffer_size=16000): +# with open(src, 'rb') as fsrc: +# with open(dest, 'wb') as fdest: +# shutil.copyfileobj(fsrc, fdest, buffer_size) \ No newline at end of file