Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Security: Implement recursion depth limits and field allowlists for Advanced Search; add reference SQL exports.

Browse Source
This commit is contained in:
Scott Idem
2026-01-02 19:38:37 -05:00
parent 5a4c82e4cb
commit 09ec231303
5 changed files with 211 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
app/db_sql.py
View File

@@ -1,7 +1,8 @@
import datetime, json, pytz, random, redis, secrets
from typing import Any, Optional
from typing import Any, List, Optional
from timeit import default_timer as timer
from fastapi import HTTPException
from app.config import settings
from app.log import log, logging, logger_reset
@@ -616,6 +617,7 @@ def sql_select(
or_like_dict: dict|None = None,
and_in_dict_li: dict|None = None,
search_query: Any|None = None, # NEW 2026-01-02 (SearchQuery model)
searchable_fields: List[str]|None = None, # NEW 2026-01-03
fulltext_qry_field_li: list|None = None, # ['field_name_1', 'field_name_2']
fulltext_qry_str: str|None = None, # 'search string'
order_by_li: dict|None = None, # {"the_field_name": "DESC"}
@@ -721,7 +723,7 @@ def sql_select(
sql_search_qry = ''
if search_query:
log.info('Creating partial SQL string for complex SearchQuery.')
sql_search_qry, data_search = sql_search_qry_part(search_query)
sql_search_qry, data_search = sql_search_qry_part(search_query, searchable_fields=searchable_fields)
data = {**data, **data_search}
sql = text(
@@ -828,7 +830,7 @@ def sql_select(
sql_search_qry = ''
if search_query:
log.info('Creating partial SQL string for complex SearchQuery.')
sql_search_qry, data_search = sql_search_qry_part(search_query)
sql_search_qry, data_search = sql_search_qry_part(search_query, searchable_fields=searchable_fields)
data = {**data, **data_search}
# # NOTE: Version 3 of the fulltext search
@@ -2110,16 +2112,19 @@ def sql_limit_offset_part(limit: int, offset: int = 0) -> bool|str:
@logger_reset
def sql_search_qry_part(
search_query: Any, # SearchQuery model instance
searchable_fields: List[str]|None = None, # List of allowed fields
max_depth: int = 5, # Maximum recursion depth
) -> tuple[str, dict]:
"""
Recursively builds a SQL WHERE clause from a SearchQuery model.
Uses unique parameter names to prevent collisions.
Enforces security via field allowlist and recursion depth limits.
"""
log.setLevel(logging.INFO)
log.debug(locals())
data = {}
param_counter = [0] # List used as a reference for unique parameter names
param_counter = [0]
def get_param_name():
param_counter[0] += 1
@@ -2138,7 +2143,10 @@ def sql_search_qry_part(
"is_not_null": "IS NOT NULL"
}
def process_node(query_node) -> str:
def process_node(query_node, current_depth: int) -> str:
if current_depth > max_depth:
raise HTTPException(status_code=400, detail=f"Search query too complex (max depth {max_depth} reached).")
clauses = []
# Process 'query_string' (Standardized Full-Text Search)
@@ -2156,7 +2164,7 @@ def sql_search_qry_part(
and_clauses.append(clause)
data.update(item_data)
else: # Nested SearchQuery
and_clauses.append(f"({process_node(item)})")
and_clauses.append(f"({process_node(item, current_depth + 1)})")
if and_clauses:
clauses.append(f"({' AND '.join(and_clauses)})")
@@ -2169,13 +2177,17 @@ def sql_search_qry_part(
or_clauses.append(clause)
data.update(item_data)
else: # Nested SearchQuery
or_clauses.append(f"({process_node(item)})")
or_clauses.append(f"({process_node(item, current_depth + 1)})")
if or_clauses:
clauses.append(f"({' OR '.join(or_clauses)})")
return ' AND '.join(clauses)
def process_filter(f) -> tuple[str, dict]:
# Field Validation: Check against allowlist
if searchable_fields is not None and f.field not in searchable_fields:
raise HTTPException(status_code=400, detail=f"Searching on field '{f.field}' is not permitted.")
sql_op = operator_map.get(f.op.lower())
if not sql_op:
raise ValueError(f"Unsupported search operator: {f.op}")
@@ -2185,7 +2197,6 @@ def sql_search_qry_part(
clause = f"`{f.field}` {sql_op}"
elif f.op.lower() == 'in':
p_name = get_param_name()
# IN operator requires a tuple or list
clause = f"`{f.field}` IN (:{p_name})"
filter_data[p_name] = f.value
else:
@@ -2196,7 +2207,7 @@ def sql_search_qry_part(
return clause, filter_data
# Initial processing
sql_where = process_node(search_query)
sql_where = process_node(search_query, 1)
if sql_where:
return f"AND ({sql_where})", data
return "", {}

13
app/object_definitions/journals.py
View File

@@ -18,6 +18,12 @@ journal_obj_li = {
'journal_id_random',
'title', 'description',
],
# V3 Search Security:
'searchable_fields': [
'journal_id_random', 'name', 'short_name', 'summary', 'outline',
'description', 'type_code', 'tags', 'billable', 'enable', 'hide',
'priority', 'group', 'created_on', 'updated_on'
],
},
'journal_entry': {
'tbl': 'journal_entry',
@@ -34,5 +40,12 @@ journal_obj_li = {
'exp_default': [
'journal_entry_id_random',
],
# V3 Search Security:
'searchable_fields': [
'journal_entry_id_random', 'journal_id_random', 'name', 'short_name',
'summary', 'content', 'type_code', 'topic_code', 'category_code',
'tags', 'location', 'billable', 'enable', 'hide', 'priority', 'group',
'created_on', 'updated_on'
],
},
}

5
app/routers/api_crud_v3.py
View File

@@ -269,6 +269,9 @@ async def search_obj_li(
if not table_name or not base_name:
return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{obj_name}' (view: {view}) is incomplete.")
# Get searchable fields for this object type
searchable_fields = obj_cfg.get('searchable_fields')
if for_obj_type and for_obj_id:
# Resolve parentage context for search
resolved_for_obj_id = redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type)
@@ -282,6 +285,7 @@ async def search_obj_li(
enabled=status_filter.enabled,
hidden=status_filter.hidden,
search_query=search_query,
searchable_fields=searchable_fields,
order_by_li=order_by_li,
limit=pagination.limit,
offset=pagination.offset,
@@ -293,6 +297,7 @@ async def search_obj_li(
enabled=status_filter.enabled,
hidden=status_filter.hidden,
search_query=search_query,
searchable_fields=searchable_fields,
order_by_li=order_by_li,
limit=pagination.limit,
offset=pagination.offset,

121
documentation/aether_dev__hosted_file__2026-01-03_00_03_33.sql Normal file
View File

@@ -0,0 +1,121 @@
-- phpMyAdmin SQL Dump
-- version 5.2.3
-- https://www.phpmyadmin.net/
--
-- Host: localhost
-- Generation Time: Jan 03, 2026 at 12:03 AM
-- Server version: 12.1.2-MariaDB-log
-- PHP Version: 8.4.15
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
START TRANSACTION;
SET time_zone = "+00:00";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8mb4 */;
--
-- Database: `aether_dev`
--
-- --------------------------------------------------------
--
-- Table structure for table `hosted_file`
--
CREATE TABLE `hosted_file` (
`id` int(11) NOT NULL,
`id_random` varchar(22) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`hash_sha256` varchar(64) DEFAULT NULL,
`account_id` varchar(22) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
`title` varchar(500) DEFAULT NULL,
`description` text DEFAULT NULL,
`version` int(11) DEFAULT NULL,
`directory_path` varchar(1000) DEFAULT NULL,
`subdirectory_path` varchar(1000) DEFAULT NULL COMMENT 'new as of 2021-08-26',
`filename` varchar(255) DEFAULT NULL,
`extension` varchar(10) DEFAULT NULL,
`content_type` varchar(100) DEFAULT NULL,
`mimetype` varchar(100) DEFAULT NULL,
`size` int(11) UNSIGNED DEFAULT NULL COMMENT 'bytes',
`cloud_storage` text DEFAULT NULL,
`owner_user_id` int(11) DEFAULT NULL,
`group_user_id` int(11) DEFAULT NULL,
`package_name` varchar(500) DEFAULT NULL,
`hide` tinyint(1) DEFAULT 0,
`priority` tinyint(1) DEFAULT NULL,
`sort` int(11) DEFAULT NULL,
`group` varchar(100) DEFAULT NULL,
`enable` tinyint(1) DEFAULT 1,
`notes` text DEFAULT NULL,
`created_on` timestamp NOT NULL DEFAULT current_timestamp(),
`updated_on` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp()
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--
-- Triggers `hosted_file`
--
DELIMITER $$
CREATE TRIGGER `before_insert_hosted_file` BEFORE INSERT ON `hosted_file` FOR EACH ROW BEGIN
BEGIN
IF (NEW.id_random IS NULL OR NEW.id_random = '' OR LENGTH(NEW.id_random) < 6)
THEN
/* SET NEW.id_random = gen_rand_pattern('3C-2N-2N-2N'); */
/* SET NEW.id_random = gen_rand_pattern('4C-2N-2N-2N'); */
SET NEW.id_random = gen_rand_str(11, 'URL');
END IF;
END;
END
$$
DELIMITER ;
DELIMITER $$
CREATE TRIGGER `before_update_hosted_file` BEFORE UPDATE ON `hosted_file` FOR EACH ROW BEGIN
BEGIN
IF (NEW.id_random IS NULL OR NEW.id_random = '' OR LENGTH(NEW.id_random) < 6)
THEN
/* SET NEW.id_random = gen_rand_pattern('3C-2N-2N-2N'); */
/* SET NEW.id_random = gen_rand_pattern('4C-2N-2N-2N'); */
SET NEW.id_random = gen_rand_str(11, 'URL');
END IF;
END;
END
$$
DELIMITER ;
--
-- Indexes for dumped tables
--
--
-- Indexes for table `hosted_file`
--
ALTER TABLE `hosted_file`
ADD PRIMARY KEY (`id`),
ADD UNIQUE KEY `id_random` (`id_random`),
ADD UNIQUE KEY `hash_sha256` (`hash_sha256`),
ADD KEY `account_id` (`account_id`),
ADD KEY `created_on` (`created_on`),
ADD KEY `updated_on` (`updated_on`),
ADD KEY `filename` (`filename`),
ADD KEY `extension` (`extension`),
ADD KEY `hide` (`hide`),
ADD KEY `enable` (`enable`);
--
-- AUTO_INCREMENT for dumped tables
--
--
-- AUTO_INCREMENT for table `hosted_file`
--
ALTER TABLE `hosted_file`
MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
COMMIT;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;

52
documentation/aether_dev__hosted_file_link__2026-01-03_00_03_53.sql Normal file
View File

@@ -0,0 +1,52 @@
-- phpMyAdmin SQL Dump
-- version 5.2.3
-- https://www.phpmyadmin.net/
--
-- Host: localhost
-- Generation Time: Jan 03, 2026 at 12:03 AM
-- Server version: 12.1.2-MariaDB-log
-- PHP Version: 8.4.15
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
START TRANSACTION;
SET time_zone = "+00:00";
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8mb4 */;
--
-- Database: `aether_dev`
--
-- --------------------------------------------------------
--
-- Table structure for table `hosted_file_link`
--
CREATE TABLE `hosted_file_link` (
`account_id` int(11) DEFAULT NULL COMMENT 'is this needed?',
`hosted_file_id` int(11) NOT NULL,
`link_to_type` varchar(50) NOT NULL COMMENT 'change to for_object_type',
`link_to_id` int(11) NOT NULL COMMENT 'change to for_object_id',
`created_on` timestamp NOT NULL DEFAULT current_timestamp(),
`updated_on` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp()
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='All hosted files that are actively linked or in use';
--
-- Indexes for dumped tables
--
--
-- Indexes for table `hosted_file_link`
--
ALTER TABLE `hosted_file_link`
ADD PRIMARY KEY (`hosted_file_id`,`link_to_type`,`link_to_id`) USING BTREE;
COMMIT;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;