Tests: Reorganize test suite into functional subdirectories

- Categorized scripts into tests/unit/, tests/integration/, tests/e2e/, and tests/tools/.
- Adopted consistent naming prefixes (test_unit_*, test_int_*, test_e2e_*, tool_*).
- Renamed conftest_mock.py to mock_config_helper.py for clarity.
- Updated test_int_boot_diagnosis.py with sys.path setup for root-level execution.

tests/unit/test_unit_email.py

@@ -0,0 +1,102 @@
+import unittest
+from unittest.mock import MagicMock, patch
+import sys
+import os
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+# --- Mocking Dependencies BEFORE Import ---
+
+# 1. Mock html2text
+mock_html2text = MagicMock()
+mock_html2text.html2text.return_value = "Mock Text Content"
+sys.modules['html2text'] = mock_html2text
+
+# 2. Mock app.config
+# We need to create a mock module and assign it to sys.modules['app.config']
+mock_config = MagicMock()
+# Create a Mock Settings object with an SMTP attribute
+mock_settings = MagicMock()
+mock_settings.SMTP = {}
+mock_config.settings = mock_settings
+sys.modules['app.config'] = mock_config
+
+# 3. Mock app.log with a functional decorator
+mock_log = MagicMock()
+
+def simple_decorator(func):
+    """Pass-through decorator."""
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+    return wrapper
+
+mock_log.logger_reset = simple_decorator
+sys.modules['app.log'] = mock_log
+
+# Now we can import the function to test
+from app.lib_email import send_email
+
+class TestEmailConfiguration(unittest.TestCase):
+    
+    @patch('app.lib_email.smtplib')
+    def test_send_email_missing_server_config(self, mock_smtplib):
+        """Test that send_email returns False gracefully if server/port are missing."""
+        print("\n--- Testing send_email with missing server/port ---")
+        
+        # Setup the mock settings for this test
+        sys.modules['app.config'].settings.SMTP = {
+            'server': '',
+            'port': '',
+            'username': 'user',
+            'password': 'password'
+        }
+        
+        result = send_email(
+            from_email="test@example.com",
+            to_email="recipient@example.com",
+            subject="Test",
+            body_html="<p>Body</p>",
+            test=True
+        )
+        
+        print(f"Result: {result}")
+        self.assertFalse(result)
+        # Verify SMTP_SSL was NOT called
+        mock_smtplib.SMTP_SSL.assert_not_called()
+
+    @patch('app.lib_email.smtplib')
+    def test_send_email_valid_config(self, mock_smtplib):
+        """Test that send_email attempts to connect if config is valid."""
+        print("\n--- Testing send_email with VALID config ---")
+        
+        # Setup the mock settings for this test
+        sys.modules['app.config'].settings.SMTP = {
+            'server': 'smtp.example.com',
+            'port': '465',
+            'username': 'user',
+            'password': 'password'
+        }
+        
+        # Mock successful connection context manager
+        mock_server_instance = MagicMock()
+        mock_smtplib.SMTP_SSL.return_value.__enter__.return_value = mock_server_instance
+        
+        result = send_email(
+            from_email="test@example.com",
+            to_email="recipient@example.com",
+            subject="Test",
+            body_html="<p>Body</p>",
+            test=False # Attempt real send (mocked)
+        )
+        
+        print(f"Result: {result}")
+        self.assertTrue(result)
+        
+        # Verify connection and login were called
+        mock_smtplib.SMTP_SSL.assert_called_with('smtp.example.com', 465, context=unittest.mock.ANY)
+        mock_server_instance.login.assert_called_with('user', 'password')
+        mock_server_instance.send_message.assert_called()
+
+if __name__ == '__main__':
+    unittest.main()

tests/unit/test_unit_email_safe.py

@@ -0,0 +1,67 @@
+import sys
+import os
+import logging
+import unittest
+from unittest.mock import MagicMock, patch
+
+# Add current directory to path
+sys.path.append(os.getcwd())
+
+# Mock html2text before importing app.lib_email
+sys.modules['html2text'] = MagicMock()
+
+# Configure logging
+logging.basicConfig(level=logging.DEBUG)
+
+from app.lib_email import send_email
+from app.config import settings
+
+class TestEmail(unittest.TestCase):
+    def test_send_email_missing_settings(self):
+        print("\nTesting send_email with missing SMTP settings...")
+        # Backup original settings
+        original_smtp = getattr(settings, 'SMTP', {})
+        
+        # Clear settings
+        settings.SMTP = {}
+        
+        result = send_email(
+            from_email="test@example.com",
+            to_email="test@example.com",
+            subject="Test Email",
+            body_html="<p>Test</p>",
+            test=True
+        )
+        
+        # Restore settings
+        settings.SMTP = original_smtp
+        
+        print(f"Result (should be False): {result}")
+        self.assertFalse(result)
+
+    def test_send_email_invalid_port(self):
+        print("\nTesting send_email with invalid port...")
+        original_smtp = getattr(settings, 'SMTP', {})
+        
+        settings.SMTP = {
+            'server': 'smtp.example.com',
+            'port': 'invalid',
+            'username': 'user',
+            'password': 'pass'
+        }
+        
+        result = send_email(
+            from_email="test@example.com",
+            to_email="test@example.com",
+            subject="Test Email",
+            body_html="<p>Test</p>",
+            test=True
+        )
+        
+        settings.SMTP = original_smtp
+        
+        print(f"Result (should be False): {result}")
+        self.assertFalse(result)
+
+if __name__ == "__main__":
+    unittest.main()

tests/unit/test_unit_errors.py

@@ -0,0 +1,44 @@
+import sys
+import os
+import asyncio
+from unittest.mock import MagicMock, AsyncMock
+
+# --- Environment Setup ---
+sys.modules['redis'] = MagicMock()
+sys.modules['sqlalchemy'] = MagicMock()
+sys.modules['sqlalchemy.text'] = MagicMock()
+sys.modules['app.config'] = MagicMock()
+sys.modules['app.log'] = MagicMock()
+sys.modules['app.lib_general'] = MagicMock()
+
+# Mock app.db_sql
+mock_db_sql = MagicMock()
+mock_db_sql.get_last_sql_error.return_value = '(pymysql.err.IntegrityError) (1062, "Duplicate entry \'test-id\' for key \'id_random\'" )'
+sys.modules['app.db_sql'] = mock_db_sql
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+from app.lib_api_crud_v3 import format_db_error
+
+def test_error_formatting():
+    print("\n--- Testing Error Formatting ---")
+    raw = '(pymysql.err.IntegrityError) (1062, "Duplicate entry \'abc\' for key \'id_random\'" )'
+    formatted = format_db_error(raw)
+    print(f"Raw: {raw}")
+    print(f"Formatted: {formatted}")
+    if formatted == "Duplicate entry 'abc' for key 'id_random'":
+        print("✅ Error formatting works.")
+    else:
+        print("❌ Error formatting FAILED.")
+
+def test_null_error_handling():
+    print("\n--- Testing Null Error Handling ---")
+    if format_db_error(None) == "":
+        print("✅ Null error handled correctly.")
+    else:
+        print("❌ Null error check FAILED.")
+
+if __name__ == "__main__":
+    test_error_formatting()
+    test_null_error_handling()

tests/unit/test_unit_filtering.py

@@ -0,0 +1,57 @@
+import sys
+import os
+from unittest.mock import MagicMock
+
+# --- Environment Setup ---
+sys.modules['redis'] = MagicMock()
+sys.modules['sqlalchemy'] = MagicMock()
+sys.modules['app.config'] = MagicMock()
+sys.modules['html2text'] = MagicMock()
+sys.modules['app.log'] = MagicMock()
+sys.modules['app.lib_general'] = MagicMock()
+sys.modules['app.db_sql'] = MagicMock()
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+# --- Imports ---
+try:
+    from app.models.journal_models import Journal_Base
+    from app.models.journal_entry_models import Journal_Entry_Base
+    print("✅ Journal models imported.")
+except Exception as e:
+    print(f"❌ Failed to import models: {e}")
+    sys.exit(1)
+
+# --- Tests ---
+def test_journal_exclusion_list():
+    """Verify Journal_Base has the correct excluded fields."""
+    expected = [
+        'person_external_id', 'person_given_name', 'person_family_name', 
+        'person_full_name', 'person_primary_email', 'person_passcode',
+        'journal_entry_count', 'file_count', 'file_count_all'
+    ]
+    if hasattr(Journal_Base, 'fields_to_exclude_from_db'):
+        actual = Journal_Base.fields_to_exclude_from_db
+        missing = [f for f in expected if f not in actual]
+        if not missing:
+            print("✅ Journal_Base: All view-fields correctly marked for exclusion.")
+        else:
+            print(f"❌ Journal_Base: Missing exclusions: {missing}")
+    else:
+        print("❌ Journal_Base: fields_to_exclude_from_db attribute is missing.")
+
+def test_journal_entry_exclusion_list():
+    """Verify Journal_Entry_Base has the correct excluded fields."""
+    if hasattr(Journal_Entry_Base, 'fields_to_exclude_from_db'):
+        actual = Journal_Entry_Base.fields_to_exclude_from_db
+        if 'file_count' in actual:
+            print("✅ Journal_Entry_Base: 'file_count' correctly marked for exclusion.")
+        else:
+            print("❌ Journal_Entry_Base: 'file_count' missing from exclusions.")
+    else:
+        print("❌ Journal_Entry_Base: fields_to_exclude_from_db attribute is missing.")
+
+if __name__ == "__main__":
+    test_journal_exclusion_list()
+    test_journal_entry_exclusion_list()

tests/unit/test_unit_models.py

@@ -0,0 +1,76 @@
+import sys
+import os
+from typing import ClassVar
+from unittest.mock import MagicMock
+
+# --- Environment Setup ---
+# Mocking heavy dependencies to allow running in restricted environments
+sys.modules['redis'] = MagicMock()
+sys.modules['sqlalchemy'] = MagicMock()
+sys.modules['sqlalchemy.exc'] = MagicMock()
+sys.modules['sqlalchemy.pool'] = MagicMock()
+sys.modules['fastapi'] = MagicMock()
+sys.modules['app.config'] = MagicMock()
+sys.modules['html2text'] = MagicMock()
+sys.modules['app.lib_email'] = MagicMock()
+sys.modules['app.lib_export'] = MagicMock()
+sys.modules['app.lib_jwt'] = MagicMock()
+sys.modules['app.lib_hash'] = MagicMock()
+sys.modules['app.log'] = MagicMock()
+
+# Mock app.lib_general (needed for log/logging)
+mock_lib_general = MagicMock()
+mock_lib_general.log = MagicMock()
+mock_lib_general.logging = MagicMock()
+sys.modules['app.lib_general'] = mock_lib_general
+sys.modules['app.log'] = MagicMock() # Ensure app.log is also mocked if needed separately
+mock_db_sql = MagicMock()
+mock_db_sql.redis_lookup_id_random.return_value = 1
+mock_db_sql.get_id_random.return_value = "mock_id"
+sys.modules['app.db_sql'] = mock_db_sql
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+# --- Imports ---
+try:
+    from app.models.person_models import Person_Base
+    print("✅ Person_Base model imported.")
+except Exception as e:
+    print(f"❌ Failed to import models: {e}")
+    sys.exit(1)
+
+# --- Tests ---
+def test_person_null_given_name():
+    """Test that given_name=None is converted to empty string."""
+    try:
+        # construct() bypasses validation, so we use the constructor
+        # We provide dummy values for other likely required fields
+        p = Person_Base.construct(given_name=None)
+        # Note: In Pydantic V1 validators run on __init__. 
+        # Since we mocked the environment, we'll test the validator function directly if init fails.
+        
+        from app.models.person_models import Person_Base
+        val = Person_Base.given_name_validator(None)
+        if val == "":
+            print("✅ given_name validator: None -> '' (Success)")
+        else:
+            print(f"❌ given_name validator: Expected '', got {val!r}")
+    except Exception as e:
+        print(f"❌ test_person_null_given_name failed: {e}")
+
+def test_person_null_allow_auth_key():
+    """Test that allow_auth_key=None is converted to True."""
+    try:
+        from app.models.person_models import Person_Base
+        val = Person_Base.allow_auth_key_validator(None)
+        if val is True:
+            print("✅ allow_auth_key validator: None -> True (Success)")
+        else:
+            print(f"❌ allow_auth_key validator: Expected True, got {val!r}")
+    except Exception as e:
+        print(f"❌ test_person_null_allow_auth_key failed: {e}")
+
+if __name__ == "__main__":
+    test_person_null_given_name()
+    test_person_null_allow_auth_key()

tests/unit/test_unit_payload_sanitization.py

@@ -0,0 +1,104 @@
+import sys
+import os
+from unittest.mock import MagicMock
+
+# --- Environment Setup ---
+sys.modules['redis'] = MagicMock()
+sys.modules['sqlalchemy'] = MagicMock()
+sys.modules['app.config'] = MagicMock()
+sys.modules['html2text'] = MagicMock()
+sys.modules['app.log'] = MagicMock()
+sys.modules['app.lib_general'] = MagicMock()
+
+# Mock app.db_sql
+mock_db_sql = MagicMock()
+# Mock ID resolution: abc -> 123
+mock_db_sql.redis_lookup_id_random.side_effect = lambda record_id_random, table_name: 123 if record_id_random == 'abc' else None
+sys.modules['app.db_sql'] = mock_db_sql
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+from app.lib_api_crud_v3 import sanitize_payload
+from pydantic import BaseModel, Field, ValidationError
+from typing import Optional, List, ClassVar
+
+class MockModel(BaseModel):
+    id: Optional[int]
+    name: str = Field(None, min_length=3)
+    account_id: Optional[int]
+    
+    fields_to_exclude_from_db: ClassVar[List[str]] = ['computed_field']
+
+def test_permissive_update():
+    print("--- Testing Permissive Update (ignore_extra=True) ---")
+    payload = {
+        "name": "Test",
+        "extra_field": "Should be removed",
+        "computed_field": "Should be removed"
+    }
+    sanitize_payload(payload, MockModel, ignore_extra=True)
+    print(f"Sanitized Payload: {payload}")
+    
+    assert "extra_field" not in payload
+    assert "computed_field" not in payload
+    assert payload["name"] == "Test"
+    print("✅ Permissive update stripping works.")
+
+def test_strict_update():
+    print("\n--- Testing Strict Update (ignore_extra=False) ---")
+    payload = {
+        "name": "Test",
+        "extra_field": "Should be removed",
+        "computed_field": "Should be removed"
+    }
+    sanitize_payload(payload, MockModel, ignore_extra=False)
+    print(f"Sanitized Payload: {payload}")
+    
+    assert "extra_field" in payload
+    assert "computed_field" not in payload
+    print("✅ Strict update correctly preserves unknown fields (waiting for DB error) but strips excluded fields.")
+
+def test_id_resolution():
+    print("\n--- Testing ID Resolution ---")
+    payload = {
+        "name": "Test",
+        "account_id_random": "abc"
+    }
+    sanitize_payload(payload, MockModel)
+    print(f"Sanitized Payload: {payload}")
+    
+    assert payload.get("account_id") == 123
+    assert "account_id_random" not in payload
+    print("✅ ID resolution (account_id_random -> account_id) works.")
+
+def test_structured_validation_errors():
+    print("\n--- Testing Structured Validation Errors ---")
+    payload = {
+        "name": "a" # Too short
+    }
+    try:
+        MockModel(**payload)
+    except ValidationError as e:
+        structured_errors = {err['loc'][-1]: err['msg'] for err in e.errors()}
+        print(f"Structured Errors: {structured_errors}")
+        assert "name" in structured_errors
+        # Pydantic 1.x error message
+        assert "at least 3 characters" in structured_errors["name"]
+        print("✅ Structured validation errors work.")
+
+if __name__ == "__main__":
+    try:
+        test_permissive_update()
+        test_strict_update()
+        test_id_resolution()
+        test_structured_validation_errors()
+        print("\n🎉 All local logic tests passed!")
+    except AssertionError as e:
+        print(f"\n❌ Test failed: {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n💥 An error occurred: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)

tests/unit/test_unit_router_stripping.py

@@ -0,0 +1,68 @@
+import sys
+import os
+import asyncio
+from unittest.mock import MagicMock, AsyncMock
+
+# --- Environment Setup ---
+sys.modules['redis'] = MagicMock()
+sys.modules['sqlalchemy'] = MagicMock()
+sys.modules['sqlalchemy.text'] = MagicMock()
+sys.modules['app.config'] = MagicMock()
+sys.modules['app.log'] = MagicMock()
+sys.modules['app.lib_general'] = MagicMock()
+
+# Mock app.db_sql
+mock_db_sql = MagicMock()
+sys.modules['app.db_sql'] = mock_db_sql
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+# Mock the FastAPI response/request
+mock_request = AsyncMock()
+mock_response = MagicMock()
+
+async def test_router_filtering():
+    print("\n--- Testing Router Filtering Logic ---")
+    
+    # We'll simulate the filtering logic from the router directly
+    # since importing the full router requires heavy FastAPI setup.
+    
+    # Input data with virtual fields
+    raw_data = {
+        "given_name": "Test",
+        "account_id_random": "abc-123",
+        "person_id_random": "p-456",
+        "person_full_name": "Test Person", # View field
+        "id_random": "keep-me"
+    }
+    
+    # Logic from create_object/patch_obj
+    data_to_insert = raw_data.copy()
+    
+    # 1. Filter _id_random
+    keys_to_remove = [k for k in data_to_insert.keys() if k.endswith('_id_random') and k != 'id_random']
+    for k in keys_to_remove:
+        del data_to_insert[k]
+    
+    # 2. Filter model-specific (Manual simulation)
+    excluded = ['person_full_name']
+    for k in excluded:
+        if k in data_to_insert:
+            del data_to_insert[k]
+            
+    print(f"Original keys: {list(raw_data.keys())}")
+    print(f"Filtered keys: {list(data_to_insert.keys())}")
+    
+    if 'account_id_random' not in data_to_insert and 'person_full_name' not in data_to_insert:
+        print("✅ Router filtering correctly removed virtual/view fields.")
+    else:
+        print("❌ Router filtering FAILED to remove some fields.")
+    
+    if 'id_random' in data_to_insert:
+        print("✅ Router filtering correctly kept 'id_random'.")
+    else:
+        print("❌ Router filtering accidentally removed 'id_random'.")
+
+if __name__ == "__main__":
+    asyncio.run(test_router_filtering())

tests/unit/test_unit_schema_logic.py

@@ -0,0 +1,44 @@
+import sys
+import os
+import json
+from unittest.mock import MagicMock
+
+# Add current directory to path
+sys.path.append(os.getcwd())
+
+# 1. Mock EVERYTHING before importing the target
+mock_config = MagicMock()
+mock_config.settings = MagicMock()
+sys.modules["app.config"] = mock_config
+
+mock_db = MagicMock()
+sys.modules["app.db_sql"] = MagicMock(db=mock_db)
+
+# 2. Mock DESCRIBE result
+mock_row = [
+    ("created_on", "timestamp", "NO", "", "current_timestamp()", "")
+]
+mock_db.execute.return_value.fetchall.return_value = mock_row
+
+# 3. Import and test
+from app.lib_schema_v3 import get_object_schema_info
+
+def test_isolated_logic():
+    # We need to mock obj_type_kv_li as well
+    import app.lib_schema_v3
+    app.lib_schema_v3.obj_type_kv_li = {"journal": {"tbl": "journal", "mdl": MagicMock()}}
+    
+    print("Testing isolated logic for get_object_schema_info...")
+    info = get_object_schema_info("journal")
+    
+    col = info["database"]["columns"][0]
+    print(f"Resulting column info: {json.dumps(col, indent=2)}")
+    
+    assert col["field"] == "created_on"
+    assert col["db_type"] == "timestamp"
+    assert col["required"] is True
+    assert col["db_default"] == "current_timestamp()"
+    print("\nIsolated test passed! The logic in lib_schema_v3 is correct.")
+
+if __name__ == "__main__":
+    test_isolated_logic()