From 860cf80a4ed385d0f041305d157134bca03a93c8 Mon Sep 17 00:00:00 2001
From: Scott Idem <stidem@gmail.com>
Date: Wed, 28 Jan 2026 12:15:01 -0500
Subject: [PATCH] Documentation Standardisation & Unit Test Stabilization

- Overhauled README.md to serve as a unified system index and WIP tracker.
- Standardized documentation filenames (ARCH__, GUIDE__, PLAN__) for better discoverability.
- Archived completed project plans and scopes.
- Fixed regressions in unit tests (errors, models, email) caused by V3 architectural updates.
- Ensured unit tests remain non-destructive and environment-independent via mocking.
---
 README.md                                     | 104 +++++++++++++++++-
 ...D_AGENT_ARCH.md => ARCH__UNIFIED_AGENT.md} |   0
 ..._CORE_ARCHITECTURE.md => ARCH__V3_CORE.md} |   0
 ...EARNINGS.md => ARCH__V3_CRUD_LEARNINGS.md} |   0
 ...S.md => ARCH__V3_DEVELOPMENT_STANDARDS.md} |   0
 ...ode_samples.md => FRONTEND_API_SAMPLES.md} |   0
 ..._MANUAL.md => GUIDE__DEPLOYMENT_MANUAL.md} |   0
 ...T_GUIDE.md => GUIDE__LOCAL_DEVELOPMENT.md} |   0
 ...API_GUIDE.md => GUIDE__V3_FRONTEND_API.md} |   0
 ...RDENING.md => PLAN__SECURITY_HARDENING.md} |   0
 .../PLAN__COMMON_PARAMETERS_REFACTORING.md}   |   0
 .../PLAN__V3_HOSTED_FILE_MIGRATION.md}        |   0
 .../PLAN__V3_SCHEMA_DISCOVERY.md}             |   0
 .../SCOPE__MCP_FIELD_MANAGER.md}              |   0
 tests/unit/test_unit_email_safe.py            |  22 ++--
 tests/unit/test_unit_errors.py                |   6 +-
 tests/unit/test_unit_models.py                |  14 +--
 17 files changed, 118 insertions(+), 28 deletions(-)
 rename documentation/{UNIFIED_AGENT_ARCH.md => ARCH__UNIFIED_AGENT.md} (100%)
 rename documentation/{V3_CORE_ARCHITECTURE.md => ARCH__V3_CORE.md} (100%)
 rename documentation/{V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md => ARCH__V3_CRUD_LEARNINGS.md} (100%)
 rename documentation/{V3_DEVELOPMENT_STANDARDS.md => ARCH__V3_DEVELOPMENT_STANDARDS.md} (100%)
 rename documentation/{Aether_Svelte_API_TS_code_samples.md => FRONTEND_API_SAMPLES.md} (100%)
 rename documentation/{DEPLOYMENT_GUIDE_MANUAL.md => GUIDE__DEPLOYMENT_MANUAL.md} (100%)
 rename documentation/{LOCAL_DEVELOPMENT_GUIDE.md => GUIDE__LOCAL_DEVELOPMENT.md} (100%)
 rename documentation/{V3_FRONTEND_API_GUIDE.md => GUIDE__V3_FRONTEND_API.md} (100%)
 rename documentation/{PROJECT_SECURITY_HARDENING.md => PLAN__SECURITY_HARDENING.md} (100%)
 rename documentation/{COMMON_PARAMETERS_REFACTORING_PROPOSAL.md => archive/PLAN__COMMON_PARAMETERS_REFACTORING.md} (100%)
 rename documentation/{V3_HOSTED_FILE_MIGRATION_PLAN.md => archive/PLAN__V3_HOSTED_FILE_MIGRATION.md} (100%)
 rename documentation/{V3_SCHEMA_DISCOVERY_PLAN.md => archive/PLAN__V3_SCHEMA_DISCOVERY.md} (100%)
 rename documentation/{MCP_FIELD_MANAGER_SCOPE.md => archive/SCOPE__MCP_FIELD_MANAGER.md} (100%)

diff --git a/README.md b/README.md
index 7348122..9381abe 100755
--- a/README.md
+++ b/README.md
@@ -1,2 +1,102 @@
-# Aether API Python FastAPI
-The Aether API was created and is being developed by Scott Idem using the Python FastAPI framework.
+# Aether API (FastAPI)
+
+The **Aether API** is a high-performance, multi-tenant backend infrastructure built using the Python **FastAPI** framework. It serves as the central data and logic hub for the Aether Platform, supporting both legacy applications and modern V3/V4 standards.
+
+---
+
+## 🏗️ Architecture Overview
+
+The API is currently in a transitional state between legacy (V1/V2) patterns and the modern **V3 CRUD Architecture**.
+
+### **V3 CRUD (Modern)**
+- **Path:** `/v3/crud/`
+- **Core Principles:**
+    - **`id_random` Primary:** All public communication uses URL-safe string IDs. Internal integer IDs are hidden.
+    - **Nested URL Structure:** Enforces parent-child relationships (e.g., `/v3/crud/journal/{id}/entry/`).
+    - **Granular Dependencies:** Uses specialized FastAPI dependencies for Account Context, Pagination, Filtering, and Serialization.
+    - **Advanced Search:** POST-based search with recursive logic and standardized operators.
+    - **Schema Discovery:** Dynamic introspection of database and Pydantic models via `/v3/crud/{obj_type}/schema`.
+
+### **V3 Actions**
+- **Path:** `/v3/action/`
+- Handles complex binary operations (uploads, streaming downloads) separately from standard metadata CRUD.
+
+### **Legacy API (V1/V2)**
+- **Path:** `/`, `/api/`, `/crud/`, `/v2/crud/`
+- Maintained for backward compatibility but currently being systematically audited and deprecated.
+- **Deprecation System:** Accessing legacy routes triggers a `!!! DEPRECATED ROUTE ACCESSED` warning in logs.
+
+---
+
+## 🛠️ Core Technologies
+- **Framework:** FastAPI (v0.95.1)
+- **Database:** MariaDB (Remote Master) + SQLAlchemy (v1.4.52)
+- **Caching/ID Resolution:** Redis
+- **Security:** JWT (JSON Web Tokens) + API Key Machine Authorization
+- **Logging:** Structured logging with module-level isolation and rotation.
+
+---
+
+## 🚀 Getting Started
+
+### **Local Development**
+1.  **Environment:** Requires Python 3.9+.
+2.  **Setup:**
+    ```bash
+    virtualenv environment
+    source environment/bin/activate
+    pip install -r admin/requirements.txt
+    ```
+3.  **Run:**
+    ```bash
+    uvicorn app.main:app --host 0.0.0.0 --port 5005 --reload
+    ```
+- **Documentation:** [GUIDE__LOCAL_DEVELOPMENT.md](documentation/GUIDE__LOCAL_DEVELOPMENT.md)
+
+### **Deployment**
+- The API is typically deployed via **Docker Compose** within the `aether_container_env`.
+- **Manual Deployment:** [GUIDE__DEPLOYMENT_MANUAL.md](documentation/GUIDE__DEPLOYMENT_MANUAL.md)
+
+---
+
+## 📂 Documentation Index
+
+### **Architecture & Standards**
+- [V3 Core Architecture](documentation/ARCH__V3_CORE.md): Modular structure and boot sequence.
+- [V3 Development Standards](documentation/ARCH__V3_DEVELOPMENT_STANDARDS.md): ID Vision, inheritance, and naming rules.
+- [Unified Agent Arch](documentation/ARCH__UNIFIED_AGENT.md): Vision for cross-stack AI agent awareness.
+
+### **Integration Guides**
+- [V3 Frontend API Guide](documentation/GUIDE__V3_FRONTEND_API.md): How to use the V3 CRUD and Search endpoints.
+- [Frontend Code Samples](documentation/FRONTEND_API_SAMPLES.md): TypeScript snippets for common API calls.
+
+### **Security**
+- [Project Security Hardening](documentation/PLAN__SECURITY_HARDENING.md): Path towards cryptographic JWT verification.
+
+---
+
+## 🧪 Testing Suite
+The project maintains an exhaustive test suite under the `tests/` directory.
+- **Unit Tests:** `tests/unit/` (Mocked logic).
+- **Integration Tests:** `tests/integration/` (Local DB/Redis connectivity).
+- **E2E Tests:** `tests/e2e/` (Network-based API validation).
+- **Documentation:** [tests/README.md](tests/README.md)
+
+---
+
+## 🚧 Current Status & Work in Progress
+
+### **Active Workstreams**
+- **[Backend] API Deprecation:** Systematic pruning of orphaned routers and methods (ID: 111523094).
+- **[ID Vision]:** Phase 2 complete. String-ID standardization extended to Page, Post, Person, Journal, Contact, and User models.
+- **[V3 Migration]:** Transitioning Launcher file caching to V3 CRUD (ID: 173518010).
+
+### **Known Bugs / Issues**
+- **Badge Rendering:**Corrupted numeric `id` fields in `event_badge_template` table causing template load failures in Svelte 5 views.
+- **Websockets:** Legacy `websockets.py` and `websockets_redis.py` require unification and stability improvements.
+- **Intermittent Timeouts:** Some E2E tests occasionally reproduce 403s/Timeouts on nested GET calls (investigating).
+
+---
+
+## 📜 Release Snapshot
+Current Baseline: **`release/2026-01-28-v3_prod-snapshot`** (Stable v4.9.0).
\ No newline at end of file
diff --git a/documentation/UNIFIED_AGENT_ARCH.md b/documentation/ARCH__UNIFIED_AGENT.md
similarity index 100%
rename from documentation/UNIFIED_AGENT_ARCH.md
rename to documentation/ARCH__UNIFIED_AGENT.md
diff --git a/documentation/V3_CORE_ARCHITECTURE.md b/documentation/ARCH__V3_CORE.md
similarity index 100%
rename from documentation/V3_CORE_ARCHITECTURE.md
rename to documentation/ARCH__V3_CORE.md
diff --git a/documentation/V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md b/documentation/ARCH__V3_CRUD_LEARNINGS.md
similarity index 100%
rename from documentation/V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md
rename to documentation/ARCH__V3_CRUD_LEARNINGS.md
diff --git a/documentation/V3_DEVELOPMENT_STANDARDS.md b/documentation/ARCH__V3_DEVELOPMENT_STANDARDS.md
similarity index 100%
rename from documentation/V3_DEVELOPMENT_STANDARDS.md
rename to documentation/ARCH__V3_DEVELOPMENT_STANDARDS.md
diff --git a/documentation/Aether_Svelte_API_TS_code_samples.md b/documentation/FRONTEND_API_SAMPLES.md
similarity index 100%
rename from documentation/Aether_Svelte_API_TS_code_samples.md
rename to documentation/FRONTEND_API_SAMPLES.md
diff --git a/documentation/DEPLOYMENT_GUIDE_MANUAL.md b/documentation/GUIDE__DEPLOYMENT_MANUAL.md
similarity index 100%
rename from documentation/DEPLOYMENT_GUIDE_MANUAL.md
rename to documentation/GUIDE__DEPLOYMENT_MANUAL.md
diff --git a/documentation/LOCAL_DEVELOPMENT_GUIDE.md b/documentation/GUIDE__LOCAL_DEVELOPMENT.md
similarity index 100%
rename from documentation/LOCAL_DEVELOPMENT_GUIDE.md
rename to documentation/GUIDE__LOCAL_DEVELOPMENT.md
diff --git a/documentation/V3_FRONTEND_API_GUIDE.md b/documentation/GUIDE__V3_FRONTEND_API.md
similarity index 100%
rename from documentation/V3_FRONTEND_API_GUIDE.md
rename to documentation/GUIDE__V3_FRONTEND_API.md
diff --git a/documentation/PROJECT_SECURITY_HARDENING.md b/documentation/PLAN__SECURITY_HARDENING.md
similarity index 100%
rename from documentation/PROJECT_SECURITY_HARDENING.md
rename to documentation/PLAN__SECURITY_HARDENING.md
diff --git a/documentation/COMMON_PARAMETERS_REFACTORING_PROPOSAL.md b/documentation/archive/PLAN__COMMON_PARAMETERS_REFACTORING.md
similarity index 100%
rename from documentation/COMMON_PARAMETERS_REFACTORING_PROPOSAL.md
rename to documentation/archive/PLAN__COMMON_PARAMETERS_REFACTORING.md
diff --git a/documentation/V3_HOSTED_FILE_MIGRATION_PLAN.md b/documentation/archive/PLAN__V3_HOSTED_FILE_MIGRATION.md
similarity index 100%
rename from documentation/V3_HOSTED_FILE_MIGRATION_PLAN.md
rename to documentation/archive/PLAN__V3_HOSTED_FILE_MIGRATION.md
diff --git a/documentation/V3_SCHEMA_DISCOVERY_PLAN.md b/documentation/archive/PLAN__V3_SCHEMA_DISCOVERY.md
similarity index 100%
rename from documentation/V3_SCHEMA_DISCOVERY_PLAN.md
rename to documentation/archive/PLAN__V3_SCHEMA_DISCOVERY.md
diff --git a/documentation/MCP_FIELD_MANAGER_SCOPE.md b/documentation/archive/SCOPE__MCP_FIELD_MANAGER.md
similarity index 100%
rename from documentation/MCP_FIELD_MANAGER_SCOPE.md
rename to documentation/archive/SCOPE__MCP_FIELD_MANAGER.md
diff --git a/tests/unit/test_unit_email_safe.py b/tests/unit/test_unit_email_safe.py
index 1251aee..5d4d692 100644
--- a/tests/unit/test_unit_email_safe.py
+++ b/tests/unit/test_unit_email_safe.py
@@ -7,6 +7,13 @@ from unittest.mock import MagicMock, patch
 # Add current directory to path
 sys.path.append(os.getcwd())
 
+# Mock app.config BEFORE imports
+mock_config = MagicMock()
+mock_settings = MagicMock()
+mock_settings.SMTP = {}
+mock_config.settings = mock_settings
+sys.modules['app.config'] = mock_config
+
 # Mock html2text before importing app.lib_email
 sys.modules['html2text'] = MagicMock()
 
@@ -19,12 +26,7 @@ from app.config import settings
 class TestEmail(unittest.TestCase):
     def test_send_email_missing_settings(self):
         print("\nTesting send_email with missing SMTP settings...")
-        # Backup original settings
-        original_smtp = getattr(settings, 'SMTP', {})
-        
-        # Clear settings
-        settings.SMTP = {}
-        
+        # settings is already mocked to have an empty SMTP dict
         result = send_email(
             from_email="test@example.com",
             to_email="test@example.com",
@@ -33,15 +35,11 @@ class TestEmail(unittest.TestCase):
             test=True
         )
         
-        # Restore settings
-        settings.SMTP = original_smtp
-        
         print(f"Result (should be False): {result}")
         self.assertFalse(result)
 
     def test_send_email_invalid_port(self):
         print("\nTesting send_email with invalid port...")
-        original_smtp = getattr(settings, 'SMTP', {})
         
         settings.SMTP = {
             'server': 'smtp.example.com',
@@ -58,10 +56,8 @@ class TestEmail(unittest.TestCase):
             test=True
         )
         
-        settings.SMTP = original_smtp
-        
         print(f"Result (should be False): {result}")
         self.assertFalse(result)
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()
\ No newline at end of file
diff --git a/tests/unit/test_unit_errors.py b/tests/unit/test_unit_errors.py
index 6e67fb7..c009757 100644
--- a/tests/unit/test_unit_errors.py
+++ b/tests/unit/test_unit_errors.py
@@ -27,14 +27,16 @@ def test_error_formatting():
     formatted = format_db_error(raw)
     print(f"Raw: {raw}")
     print(f"Formatted: {formatted}")
-    if formatted == "Duplicate entry 'abc' for key 'id_random'":
+    
+    if formatted.category == "database_duplicate" and formatted.code == 1062:
         print("✅ Error formatting works.")
     else:
         print("❌ Error formatting FAILED.")
 
 def test_null_error_handling():
     print("\n--- Testing Null Error Handling ---")
-    if format_db_error(None) == "":
+    formatted = format_db_error(None)
+    if formatted.category == "unknown":
         print("✅ Null error handled correctly.")
     else:
         print("❌ Null error check FAILED.")
diff --git a/tests/unit/test_unit_models.py b/tests/unit/test_unit_models.py
index e9356fd..b96f52f 100644
--- a/tests/unit/test_unit_models.py
+++ b/tests/unit/test_unit_models.py
@@ -1,6 +1,5 @@
 import sys
 import os
-from typing import ClassVar
 from unittest.mock import MagicMock
 
 # --- Environment Setup ---
@@ -23,7 +22,7 @@ mock_lib_general = MagicMock()
 mock_lib_general.log = MagicMock()
 mock_lib_general.logging = MagicMock()
 sys.modules['app.lib_general'] = mock_lib_general
-sys.modules['app.log'] = MagicMock() # Ensure app.log is also mocked if needed separately
+
 mock_db_sql = MagicMock()
 mock_db_sql.redis_lookup_id_random.return_value = 1
 mock_db_sql.get_id_random.return_value = "mock_id"
@@ -44,13 +43,7 @@ except Exception as e:
 def test_person_null_given_name():
     """Test that given_name=None is converted to empty string."""
     try:
-        # construct() bypasses validation, so we use the constructor
-        # We provide dummy values for other likely required fields
-        p = Person_Base.construct(given_name=None)
-        # Note: In Pydantic V1 validators run on __init__. 
-        # Since we mocked the environment, we'll test the validator function directly if init fails.
-        
-        from app.models.person_models import Person_Base
+        # In Pydantic V1, we test the validator classmethod directly if instantiation is too complex with mocks
         val = Person_Base.given_name_validator(None)
         if val == "":
             print("✅ given_name validator: None -> '' (Success)")
@@ -62,7 +55,6 @@ def test_person_null_given_name():
 def test_person_null_allow_auth_key():
     """Test that allow_auth_key=None is converted to True."""
     try:
-        from app.models.person_models import Person_Base
         val = Person_Base.allow_auth_key_validator(None)
         if val is True:
             print("✅ allow_auth_key validator: None -> True (Success)")
@@ -73,4 +65,4 @@ def test_person_null_allow_auth_key():
 
 if __name__ == "__main__":
     test_person_null_given_name()
-    test_person_null_allow_auth_key()
+    test_person_null_allow_auth_key()
\ No newline at end of file