From 09ec2313030767270186d8f9c253b10d21226d71 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Fri, 2 Jan 2026 19:38:37 -0500 Subject: [PATCH] Security: Implement recursion depth limits and field allowlists for Advanced Search; add reference SQL exports. --- app/db_sql.py | 29 +++-- app/object_definitions/journals.py | 13 ++ app/routers/api_crud_v3.py | 5 + ..._dev__hosted_file__2026-01-03_00_03_33.sql | 121 ++++++++++++++++++ ..._hosted_file_link__2026-01-03_00_03_53.sql | 52 ++++++++ 5 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 documentation/aether_dev__hosted_file__2026-01-03_00_03_33.sql create mode 100644 documentation/aether_dev__hosted_file_link__2026-01-03_00_03_53.sql diff --git a/app/db_sql.py b/app/db_sql.py index c95ac74..bdb89d7 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -1,7 +1,8 @@ import datetime, json, pytz, random, redis, secrets -from typing import Any, Optional +from typing import Any, List, Optional from timeit import default_timer as timer +from fastapi import HTTPException from app.config import settings from app.log import log, logging, logger_reset @@ -616,6 +617,7 @@ def sql_select( or_like_dict: dict|None = None, and_in_dict_li: dict|None = None, search_query: Any|None = None, # NEW 2026-01-02 (SearchQuery model) + searchable_fields: List[str]|None = None, # NEW 2026-01-03 fulltext_qry_field_li: list|None = None, # ['field_name_1', 'field_name_2'] fulltext_qry_str: str|None = None, # 'search string' order_by_li: dict|None = None, # {"the_field_name": "DESC"} @@ -721,7 +723,7 @@ def sql_select( sql_search_qry = '' if search_query: log.info('Creating partial SQL string for complex SearchQuery.') - sql_search_qry, data_search = sql_search_qry_part(search_query) + sql_search_qry, data_search = sql_search_qry_part(search_query, searchable_fields=searchable_fields) data = {**data, **data_search} sql = text( @@ -828,7 +830,7 @@ def sql_select( sql_search_qry = '' if search_query: log.info('Creating partial SQL string for complex SearchQuery.') - sql_search_qry, data_search = sql_search_qry_part(search_query) + sql_search_qry, data_search = sql_search_qry_part(search_query, searchable_fields=searchable_fields) data = {**data, **data_search} # # NOTE: Version 3 of the fulltext search @@ -2110,16 +2112,19 @@ def sql_limit_offset_part(limit: int, offset: int = 0) -> bool|str: @logger_reset def sql_search_qry_part( search_query: Any, # SearchQuery model instance + searchable_fields: List[str]|None = None, # List of allowed fields + max_depth: int = 5, # Maximum recursion depth ) -> tuple[str, dict]: """ Recursively builds a SQL WHERE clause from a SearchQuery model. Uses unique parameter names to prevent collisions. + Enforces security via field allowlist and recursion depth limits. """ log.setLevel(logging.INFO) log.debug(locals()) data = {} - param_counter = [0] # List used as a reference for unique parameter names + param_counter = [0] def get_param_name(): param_counter[0] += 1 @@ -2138,7 +2143,10 @@ def sql_search_qry_part( "is_not_null": "IS NOT NULL" } - def process_node(query_node) -> str: + def process_node(query_node, current_depth: int) -> str: + if current_depth > max_depth: + raise HTTPException(status_code=400, detail=f"Search query too complex (max depth {max_depth} reached).") + clauses = [] # Process 'query_string' (Standardized Full-Text Search) @@ -2156,7 +2164,7 @@ def sql_search_qry_part( and_clauses.append(clause) data.update(item_data) else: # Nested SearchQuery - and_clauses.append(f"({process_node(item)})") + and_clauses.append(f"({process_node(item, current_depth + 1)})") if and_clauses: clauses.append(f"({' AND '.join(and_clauses)})") @@ -2169,13 +2177,17 @@ def sql_search_qry_part( or_clauses.append(clause) data.update(item_data) else: # Nested SearchQuery - or_clauses.append(f"({process_node(item)})") + or_clauses.append(f"({process_node(item, current_depth + 1)})") if or_clauses: clauses.append(f"({' OR '.join(or_clauses)})") return ' AND '.join(clauses) def process_filter(f) -> tuple[str, dict]: + # Field Validation: Check against allowlist + if searchable_fields is not None and f.field not in searchable_fields: + raise HTTPException(status_code=400, detail=f"Searching on field '{f.field}' is not permitted.") + sql_op = operator_map.get(f.op.lower()) if not sql_op: raise ValueError(f"Unsupported search operator: {f.op}") @@ -2185,7 +2197,6 @@ def sql_search_qry_part( clause = f"`{f.field}` {sql_op}" elif f.op.lower() == 'in': p_name = get_param_name() - # IN operator requires a tuple or list clause = f"`{f.field}` IN (:{p_name})" filter_data[p_name] = f.value else: @@ -2196,7 +2207,7 @@ def sql_search_qry_part( return clause, filter_data # Initial processing - sql_where = process_node(search_query) + sql_where = process_node(search_query, 1) if sql_where: return f"AND ({sql_where})", data return "", {} diff --git a/app/object_definitions/journals.py b/app/object_definitions/journals.py index fd2abdc..5f9e812 100644 --- a/app/object_definitions/journals.py +++ b/app/object_definitions/journals.py @@ -18,6 +18,12 @@ journal_obj_li = { 'journal_id_random', 'title', 'description', ], + # V3 Search Security: + 'searchable_fields': [ + 'journal_id_random', 'name', 'short_name', 'summary', 'outline', + 'description', 'type_code', 'tags', 'billable', 'enable', 'hide', + 'priority', 'group', 'created_on', 'updated_on' + ], }, 'journal_entry': { 'tbl': 'journal_entry', @@ -34,5 +40,12 @@ journal_obj_li = { 'exp_default': [ 'journal_entry_id_random', ], + # V3 Search Security: + 'searchable_fields': [ + 'journal_entry_id_random', 'journal_id_random', 'name', 'short_name', + 'summary', 'content', 'type_code', 'topic_code', 'category_code', + 'tags', 'location', 'billable', 'enable', 'hide', 'priority', 'group', + 'created_on', 'updated_on' + ], }, } diff --git a/app/routers/api_crud_v3.py b/app/routers/api_crud_v3.py index 9ff9f5c..7c8bcb9 100644 --- a/app/routers/api_crud_v3.py +++ b/app/routers/api_crud_v3.py @@ -269,6 +269,9 @@ async def search_obj_li( if not table_name or not base_name: return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{obj_name}' (view: {view}) is incomplete.") + # Get searchable fields for this object type + searchable_fields = obj_cfg.get('searchable_fields') + if for_obj_type and for_obj_id: # Resolve parentage context for search resolved_for_obj_id = redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type) @@ -282,6 +285,7 @@ async def search_obj_li( enabled=status_filter.enabled, hidden=status_filter.hidden, search_query=search_query, + searchable_fields=searchable_fields, order_by_li=order_by_li, limit=pagination.limit, offset=pagination.offset, @@ -293,6 +297,7 @@ async def search_obj_li( enabled=status_filter.enabled, hidden=status_filter.hidden, search_query=search_query, + searchable_fields=searchable_fields, order_by_li=order_by_li, limit=pagination.limit, offset=pagination.offset, diff --git a/documentation/aether_dev__hosted_file__2026-01-03_00_03_33.sql b/documentation/aether_dev__hosted_file__2026-01-03_00_03_33.sql new file mode 100644 index 0000000..6049715 --- /dev/null +++ b/documentation/aether_dev__hosted_file__2026-01-03_00_03_33.sql @@ -0,0 +1,121 @@ +-- phpMyAdmin SQL Dump +-- version 5.2.3 +-- https://www.phpmyadmin.net/ +-- +-- Host: localhost +-- Generation Time: Jan 03, 2026 at 12:03 AM +-- Server version: 12.1.2-MariaDB-log +-- PHP Version: 8.4.15 + +SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; +START TRANSACTION; +SET time_zone = "+00:00"; + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; + +-- +-- Database: `aether_dev` +-- + +-- -------------------------------------------------------- + +-- +-- Table structure for table `hosted_file` +-- + +CREATE TABLE `hosted_file` ( + `id` int(11) NOT NULL, + `id_random` varchar(22) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `hash_sha256` varchar(64) DEFAULT NULL, + `account_id` varchar(22) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `title` varchar(500) DEFAULT NULL, + `description` text DEFAULT NULL, + `version` int(11) DEFAULT NULL, + `directory_path` varchar(1000) DEFAULT NULL, + `subdirectory_path` varchar(1000) DEFAULT NULL COMMENT 'new as of 2021-08-26', + `filename` varchar(255) DEFAULT NULL, + `extension` varchar(10) DEFAULT NULL, + `content_type` varchar(100) DEFAULT NULL, + `mimetype` varchar(100) DEFAULT NULL, + `size` int(11) UNSIGNED DEFAULT NULL COMMENT 'bytes', + `cloud_storage` text DEFAULT NULL, + `owner_user_id` int(11) DEFAULT NULL, + `group_user_id` int(11) DEFAULT NULL, + `package_name` varchar(500) DEFAULT NULL, + `hide` tinyint(1) DEFAULT 0, + `priority` tinyint(1) DEFAULT NULL, + `sort` int(11) DEFAULT NULL, + `group` varchar(100) DEFAULT NULL, + `enable` tinyint(1) DEFAULT 1, + `notes` text DEFAULT NULL, + `created_on` timestamp NOT NULL DEFAULT current_timestamp(), + `updated_on` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp() +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- +-- Triggers `hosted_file` +-- +DELIMITER $$ +CREATE TRIGGER `before_insert_hosted_file` BEFORE INSERT ON `hosted_file` FOR EACH ROW BEGIN + BEGIN + IF (NEW.id_random IS NULL OR NEW.id_random = '' OR LENGTH(NEW.id_random) < 6) + THEN + /* SET NEW.id_random = gen_rand_pattern('3C-2N-2N-2N'); */ + /* SET NEW.id_random = gen_rand_pattern('4C-2N-2N-2N'); */ + SET NEW.id_random = gen_rand_str(11, 'URL'); + END IF; + END; +END +$$ +DELIMITER ; +DELIMITER $$ +CREATE TRIGGER `before_update_hosted_file` BEFORE UPDATE ON `hosted_file` FOR EACH ROW BEGIN + BEGIN + IF (NEW.id_random IS NULL OR NEW.id_random = '' OR LENGTH(NEW.id_random) < 6) + THEN + /* SET NEW.id_random = gen_rand_pattern('3C-2N-2N-2N'); */ + /* SET NEW.id_random = gen_rand_pattern('4C-2N-2N-2N'); */ + SET NEW.id_random = gen_rand_str(11, 'URL'); + END IF; + END; +END +$$ +DELIMITER ; + +-- +-- Indexes for dumped tables +-- + +-- +-- Indexes for table `hosted_file` +-- +ALTER TABLE `hosted_file` + ADD PRIMARY KEY (`id`), + ADD UNIQUE KEY `id_random` (`id_random`), + ADD UNIQUE KEY `hash_sha256` (`hash_sha256`), + ADD KEY `account_id` (`account_id`), + ADD KEY `created_on` (`created_on`), + ADD KEY `updated_on` (`updated_on`), + ADD KEY `filename` (`filename`), + ADD KEY `extension` (`extension`), + ADD KEY `hide` (`hide`), + ADD KEY `enable` (`enable`); + +-- +-- AUTO_INCREMENT for dumped tables +-- + +-- +-- AUTO_INCREMENT for table `hosted_file` +-- +ALTER TABLE `hosted_file` + MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; +COMMIT; + +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; diff --git a/documentation/aether_dev__hosted_file_link__2026-01-03_00_03_53.sql b/documentation/aether_dev__hosted_file_link__2026-01-03_00_03_53.sql new file mode 100644 index 0000000..0844c2a --- /dev/null +++ b/documentation/aether_dev__hosted_file_link__2026-01-03_00_03_53.sql @@ -0,0 +1,52 @@ +-- phpMyAdmin SQL Dump +-- version 5.2.3 +-- https://www.phpmyadmin.net/ +-- +-- Host: localhost +-- Generation Time: Jan 03, 2026 at 12:03 AM +-- Server version: 12.1.2-MariaDB-log +-- PHP Version: 8.4.15 + +SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; +START TRANSACTION; +SET time_zone = "+00:00"; + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; + +-- +-- Database: `aether_dev` +-- + +-- -------------------------------------------------------- + +-- +-- Table structure for table `hosted_file_link` +-- + +CREATE TABLE `hosted_file_link` ( + `account_id` int(11) DEFAULT NULL COMMENT 'is this needed?', + `hosted_file_id` int(11) NOT NULL, + `link_to_type` varchar(50) NOT NULL COMMENT 'change to for_object_type', + `link_to_id` int(11) NOT NULL COMMENT 'change to for_object_id', + `created_on` timestamp NOT NULL DEFAULT current_timestamp(), + `updated_on` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp() +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='All hosted files that are actively linked or in use'; + +-- +-- Indexes for dumped tables +-- + +-- +-- Indexes for table `hosted_file_link` +-- +ALTER TABLE `hosted_file_link` + ADD PRIMARY KEY (`hosted_file_id`,`link_to_type`,`link_to_id`) USING BTREE; +COMMIT; + +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;