From b9742cfcd80975e49baa473f14a844d3a653c17c Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Wed, 25 Mar 2026 13:05:09 -0400 Subject: [PATCH] feat(routers): migrate hosted_file hash lookup to V3 actions Ported the legacy '/hosted_file/hash/{hash}' endpoint to the V3 actions router. The new endpoint '/v3/action/hosted_file/hash/{hosted_file_hash}' supports: - ID Vision: returns random string IDs instead of internal integers - Local Check: verifies physical file existence on disk (check_for_local=True) - Deduplication: enables frontend to check for existing files before upload Also added PROJECT document for AE Hosted Files migration tracking. Co-Authored-By: Claude Sonnet 4.6 --- app/routers/api_v3_actions_hosted_file.py | 35 ++++- .../PROJECT__AE_hosted_files_uploads_util.md | 124 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 documentation/PROJECT__AE_hosted_files_uploads_util.md diff --git a/app/routers/api_v3_actions_hosted_file.py b/app/routers/api_v3_actions_hosted_file.py index fbc3dbf..db986bf 100644 --- a/app/routers/api_v3_actions_hosted_file.py +++ b/app/routers/api_v3_actions_hosted_file.py @@ -15,7 +15,8 @@ from app.config import settings from app.db_sql import redis_lookup_id_random, sql_select, sql_update, sql_delete, get_id_random from app.methods.hosted_file_methods import ( create_hosted_file_obj, load_hosted_file_obj, save_file, - create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list + create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, + lookup_file_hash, check_for_hosted_file_hash_file ) from app.methods.lib_media import convert_file_method from app.methods.lib_media import clip_video_method @@ -354,6 +355,38 @@ async def download_file_by_hash_action( return FileResponse(full_file_path, filename=target_filename, media_type=media_type) +@router.get('/hash/{hosted_file_hash}', response_model=Resp_Body_Base) +async def check_hosted_file_obj_w_hash_action( + response: Response, + hosted_file_hash: str = Path(min_length=64, max_length=64), + check_for_local: Optional[bool] = Query(True), + account: AccountContext = Depends(get_account_context_optional), + delay: DelayParams = Depends(), + ): + """ + Look up a hosted_file record by its hash (Deduplication Check). + Optionally verifies physical file existence on disk. + """ + if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s) + + if hfid := lookup_file_hash(file_hash=hosted_file_hash): + obj_model = load_hosted_file_obj(hosted_file_id=hfid, model_as_dict=False) + if not obj_model: + return mk_resp(data=False, status_code=404, response=response, status_message="Record found but data could not be loaded.") + + if check_for_local: + # We use the model directly to access subdirectory_path even if it's excluded from dicts + sub_dir = getattr(obj_model, 'subdirectory_path', '') or '' + if check := check_for_hosted_file_hash_file(file_hash=hosted_file_hash, sub_dir=sub_dir): + obj_model.hosted_file_found_check = True + obj_model.hosted_file_size_check = check['file_size'] + + # mk_resp will handle model->dict conversion with proper ID Vision mapping + return mk_resp(data=obj_model) + + return mk_resp(data=False, status_code=404, response=response, status_message="No record found for this hash.") + + @router.delete('/{hosted_file_id}', response_model=Resp_Body_Base) async def delete_file_action( hosted_file_id: str = Path(min_length=11, max_length=22), diff --git a/documentation/PROJECT__AE_hosted_files_uploads_util.md b/documentation/PROJECT__AE_hosted_files_uploads_util.md new file mode 100644 index 0000000..f2f7b1b --- /dev/null +++ b/documentation/PROJECT__AE_hosted_files_uploads_util.md @@ -0,0 +1,124 @@ +# PROJECT: AE Hosted Files — Upload Util & V3 Actions Migration + +**Status:** In Progress +**Date:** 2026-03-25 +**Affected systems:** Frontend (aether_app_sveltekit), Backend (aether_api_fastapi) + +--- + +## Background + +The legacy `hosted_file.router` (registered at prefix `/hosted_file`) was commented out +in `app/routers/registry.py` as part of the V3 migration: + +```python +# app.include_router(hosted_file.router, prefix='/hosted_file', tags=['Hosted File']) +app.include_router(api_v3_actions_hosted_file.router, prefix='/v3/action/hosted_file', ...) +``` + +This broke several frontend features that were still calling the old endpoints. +Three endpoints have been fixed on the frontend side (already committed and pushed). +One endpoint still needs a backend fix. + +--- + +## Endpoints: Status Summary + +### FIXED (frontend updated to call new V3 path) + +| Old endpoint | New endpoint | Frontend file | +|---|---|---| +| `POST /hosted_file/upload_files` | `POST /v3/action/hosted_file/upload` | `src/lib/ae_core/ae_comp__hosted_files_upload.svelte`, `src/routes/events/ae_comp__event_files_upload.svelte` | +| `GET /hosted_file/{id}/clip_video` | `GET /v3/action/hosted_file/{id}/clip_video` | `src/lib/ae_core/ae_comp__hosted_files_clip_video.svelte` | + +### NEEDS BACKEND ACTION — Hash Lookup Endpoint + +**Missing endpoint:** `GET /hosted_file/hash/{hosted_file_hash}` + +This endpoint existed in the legacy `hosted_file.py` router (line 233) and has **not** been +ported to `api_v3_actions_hosted_file.py`. + +**What it does:** +1. Looks up a `hosted_file` record by its `hash_sha256` field +2. Optionally checks that the physical file actually exists on disk (`check_for_local=true`) +3. Returns the full hosted_file object with two extra flags: + - `hosted_file_found_check: true` — file record exists AND physical file confirmed on disk + - `hosted_file_size_check: ` — file size from disk + +**Legacy implementation (hosted_file.py:233):** +```python +@router.get('/hash/{hosted_file_hash}', response_model=Resp_Body_Base) +async def check_hosted_file_obj_w_hash( + hosted_file_hash: str = Path(min_length=64, max_length=64), + check_for_local: Optional[bool] = True, + commons: Common_Route_Params = Depends(common_route_params), + ): + if hfid := lookup_file_hash(file_hash=hosted_file_hash): + obj = load_hosted_file_obj(hosted_file_id=hfid, model_as_dict=True) + if check_for_local and obj: + if check := check_for_hosted_file_hash_file(file_hash=hosted_file_hash, sub_dir=obj.get('subdirectory_path', '')): + obj['hosted_file_found_check'] = True + obj['hosted_file_size_check'] = check['file_size'] + return mk_resp(data=obj, response=commons.response) + return mk_resp(data=False, status_code=404, response=commons.response) +``` + +**Where it's called on the frontend:** +- `src/lib/ae_core/core__check_hosted_file_obj_w_hash.ts` — thin wrapper, calls `GET /hosted_file/hash/{hash}` +- `src/lib/elements/element_input_file.svelte` — calls this before uploading (dedup check) +- `src/lib/elements/element_input_files_tbl.svelte` — same (dedup check in the table file input) +- Exported via `src/lib/ae_core/ae_core_functions.ts` as `core_func.check_hosted_file_obj_w_hash` + +**Current impact:** The 404 causes a null return. The frontend checks +`result && result.hosted_file_found_check` — so if null, it silently skips the dedup check +and proceeds to upload anyway. Uploads still work, but duplicate files may be created rather +than reusing existing records. + +**Requested fix (backend):** +Port this endpoint to `api_v3_actions_hosted_file.py` as: + +``` +GET /v3/action/hosted_file/hash/{hosted_file_hash} +``` + +Parameters and response shape should match the legacy implementation exactly. +The `check_for_local` query param (default `True`) must be preserved — the frontend +passes `check_for_local=true` and expects `hosted_file_found_check` in the response. + +**After backend deploys the new endpoint**, the frontend needs one line changed in +`src/lib/ae_core/core__check_hosted_file_obj_w_hash.ts`: +```ts +// Before: +const endpoint = `/hosted_file/hash/${hosted_file_hash}`; +// After: +const endpoint = `/v3/action/hosted_file/hash/${hosted_file_hash}`; +``` + +--- + +## Other Legacy Endpoints — Audit Notes + +The following were also in `hosted_file.py` but appear to either have V3 equivalents already +or are not currently called by the frontend. Backend should confirm: + +| Legacy endpoint | V3 equivalent | Notes | +|---|---|---| +| `GET /hosted_file/{id}/download` | `GET /v3/action/hosted_file/{id}/download` | Exists in V3 router | +| `DELETE /hosted_file/{id}` | `DELETE /v3/action/hosted_file/{id}` | Exists in V3 router | +| `GET /hosted_file/{id}/convert_file` | `GET /v3/action/hosted_file/{id}/convert_file` | Exists in V3 router | +| `GET /hosted_file/{id}/stream` | Unknown | Not confirmed in V3 router — verify | +| `GET /hosted_file/directory_check` | Unknown | Admin/dev utility — verify if still needed | +| `GET /hosted_file/hash/{hash}/download` (via V3) | `GET /v3/action/hosted_file/hash/{sha256}/download` | Exists in V3 router (hash-based download) | +| `GET /hosted_file/tmp/{subdir}/{filename}/download` | Unknown | Temp file download — verify if still needed | +| `POST /hosted_file/create_video` | Unknown | Verify if still needed | + +--- + +## Coordinator Notes + +- Frontend commits fixing upload and clip_video are on branch `ae_app_3x_llm` + (commits `a5a806e2` and `362136e6`) +- Once the backend adds the hash lookup endpoint, the frontend one-line fix in + `core__check_hosted_file_obj_w_hash.ts` can be committed alongside it +- The `check_for_local` flag is important — it verifies the physical file exists on disk, + not just the DB record. Don't drop it in the V3 port.