feat: add Novi AMS → Mailman 3 cron-based mirror sync bridge (IDAA)

Implements a full proof-of-concept for syncing IDAA's Novi AMS membership groups to Mailman 3 mailing lists via a cron-triggered reconciliation approach. Key changes: - methods: rewrote sync engine around confirmed Novi API shape — group-based member fetch (/groups/{guid}/members + /customers/{uuid}), respects Active=false and UnsubscribeFromEmails=true flags - methods: mirror_novi_group_to_mailman_list() diffs Novi group against Mailman roster and subscribes/unsubscribes accordingly (full mirror) - methods: mirror_all_configured_mappings() iterates novi_mailman_sync config array in IDAA site cfg_json — this is the cron target - router: replaced old /sync endpoint with POST /sync (all mappings) and POST /sync/group/{guid} (single mapping); removed webhook endpoint (sync is cron-based, not event-driven) - router: added GET/POST/DELETE endpoints for list member inspection and manual subscribe/unsubscribe - tests: two new e2e scripts covering connection checks and full member lifecycle; old webhook integration test archived Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 16:36:32 -04:00
parent 29579fd9f1
commit 6b25cf9c6d
6 changed files with 711 additions and 275 deletions
--- a/tests/README.md
+++ b/tests/README.md
@@ -28,6 +28,8 @@ These consolidated scripts are the primary verification tool for the V3 API.
 | `test_e2e_v3_demo_parity.py` | **Demo Parity + Nested Create Regression**: Vision ID check for Badge, Exhibit, Tracking; nested create lifecycle (POST+DELETE) for `journal/journal_entry` and `event/event_session`; alias resolution. **Run after any model or nested-router change.** |
 | `test_e2e_v3_action_event_file.py` | **Event Actions**: Specialized atomic upload and linking for event files. |
 | `test_e2e_v3_action_zoom.py` | **Zoom Integration**: Verifies OAuth and ticket sync logic for Zoom Events. |
+| `test_e2e_v3_action_novi_mailman.py` | **Novi-Mailman Bridge — Connections**: Verifies Novi AMS and Mailman 3 API credentials are valid (IDAA). Run first before the lists test. |
+| `test_e2e_v3_action_novi_mailman_lists.py` | **Novi-Mailman Bridge — List Operations**: Full member lifecycle — read roster, subscribe, verify, unsubscribe — against `mm3@idaa.org`, `mm3@dgrzone.com`, `mm3@oneskyit.com`. |
 | `test_e2e_v3_action_event_exhibit_tracking_export.py` | **Exhibit Leads Export**: Auth/permission guards, CSV column structure, XLSX bytes, and `return_file` mode for the V3 tracking export action. |
 | `test_e2e_v3_accounts.py` | CRUD verification for the core Account object. |
 | `test_e2e_v3_schema.py` | Network verification of the V3 metadata discovery endpoint. |
@@ -53,6 +55,7 @@ Tests exist to be used — run the relevant suite whenever you touch backend cod
 | Search / filter changes | `test_e2e_v3_search_engine.py` |
 | Auth / account context changes | `test_e2e_v3_security_audit.py`, `test_e2e_v3_auth_security.py` |
 | File upload / download changes | `test_e2e_v3_actions_file_lifecycle.py` |
+| Novi-Mailman bridge changes | `test_e2e_v3_action_novi_mailman.py`, `test_e2e_v3_action_novi_mailman_lists.py` |
 | Event exhibit tracking export changes | `test_e2e_v3_action_event_exhibit_tracking_export.py` |
 | Any backend change before frontend hand-off | All of the above |

@@ -136,4 +139,9 @@ These are IDs for records that we can use for testing. Please do not delete them

 ### Events Module (IDAA Recovery Meetings)
 * Aether test/demo Event: '1Pkd025vvxU' (36) "IDAA Recovery Meeting Test"
-* Aether test/demo Event: 'gIZgAjISkf8' (43) "IDAA Recovery Meeting Test"
+* Aether test/demo Event: 'gIZgAjISkf8' (43) "IDAA Recovery Meeting Test"
+
+### IDAA and Novi AMS
+Scott Idem (test 1)
+* Novi (customer) UUID: "1dadf11c-b74b-4582-8a0a-7ec738a033dc"
+* Novi Email: "stidem+test1@gmail.com"
--- a/tests/archive/test_novi_webhook_ARCHIVED.py
+++ b/tests/archive/test_novi_webhook_ARCHIVED.py
@@ -0,0 +1,51 @@
+"""
+ARCHIVED 2026-03-17
+The webhook endpoint (/webhook/novi) was removed — sync is cron-based.
+If Novi webhook support is added in future, restore the endpoint in
+api_v3_actions_e_novi_mailman.py and move this file back to tests/e2e/.
+The webhook secret is stored as novi_webhook_secret in IDAA site cfg_json.
+
+Original: Integration test — send a signed Novi webhook payload to the API.
+"""
+
+import os
+import json
+import hmac
+import hashlib
+import requests
+
+BASE_URL = os.environ.get('AE_API_BASE', 'https://dev-api.oneskyit.com')
+ENDPOINT = f"{BASE_URL}/v3/action/e_novi_mailman/webhook/novi"
+SECRET = os.environ.get('NOVI_WEBHOOK_SECRET', 'test-secret')
+
+payload = {
+    "EventType": "MembershipActivated",
+    "Member": {
+        "Email": "test+webhook@example.com",
+        "FirstName": "Test",
+        "LastName": "Webhook",
+        "MembershipStatus": "Active"
+    }
+}
+
+body = json.dumps(payload).encode('utf-8')
+signature = hmac.new(SECRET.encode('utf-8'), body, hashlib.sha256).hexdigest()
+
+headers = {
+    'Content-Type': 'application/json',
+    'X-Novi-Signature': signature
+}
+
+print('Posting to', ENDPOINT)
+resp = requests.post(ENDPOINT, headers=headers, data=body, timeout=30)
+print('Status:', resp.status_code)
+try:
+    print('JSON:', resp.json())
+except Exception:
+    print('Body:', resp.text)
+
+if resp.status_code == 200:
+    print('\u2705 PASS: webhook accepted')
+else:
+    print('\u274C FAIL: webhook rejected')
+    raise SystemExit(1)
--- a/tests/e2e/test_e2e_v3_action_novi_mailman.py
+++ b/tests/e2e/test_e2e_v3_action_novi_mailman.py
@@ -0,0 +1,99 @@
+"""
+E2E tests for the Novi-Mailman Bridge — API connection checks.
+
+Verifies that both the Novi AMS API and Mailman 3 REST API are reachable
+and that the credentials stored in IDAA site cfg_json are valid.
+
+Run from project root:
+    ./environment/bin/python3 tests/e2e/test_e2e_v3_action_novi_mailman.py
+
+Environment:
+    AE_API_BASE  — override the target API base URL (default: https://dev-api.oneskyit.com)
+
+Related tests:
+    test_e2e_v3_action_novi_mailman_lists.py  — member read/subscribe/unsubscribe lifecycle
+    test_e2e_v3_action_novi_mailman_sync.py   — full Novi → Mailman mirror sync (TODO)
+
+Credential storage:
+    All Novi and Mailman credentials live in IDAA site cfg_json (id_random='58_gJESdlUh').
+    See project memory for key names.
+"""
+
+import os
+import sys
+import time
+import requests
+
+BASE_URL    = os.environ.get('AE_API_BASE', 'https://dev-api.oneskyit.com')
+ACTION_BASE = f"{BASE_URL}/v3/action/e_novi_mailman"
+API_KEY     = "PMM4n50teUCaOMMTN8qOJA"
+
+AUTH_HEADERS = {
+    "X-Aether-API-Key": API_KEY,
+    "x-no-account-id":  "bypass",
+}
+
+pass_count = 0
+fail_count = 0
+
+
+def print_result(label: str, success: bool, message: str = ""):
+    global pass_count, fail_count
+    status = "✅ PASS" if success else "❌ FAIL"
+    msg = f"   {message}" if message else ""
+    print(f"  [{status}] {label}{msg}")
+    if success:
+        pass_count += 1
+    else:
+        fail_count += 1
+
+
+def test_novi_connection():
+    print("\n[1] Novi API Connection")
+    try:
+        resp = requests.get(f"{ACTION_BASE}/test_connection/novi", headers=AUTH_HEADERS, timeout=15)
+        data = resp.json().get('data', {})
+        if resp.status_code == 200 and data.get('ok'):
+            print_result("Novi credentials valid", True)
+        elif resp.status_code == 401:
+            print_result("Novi credentials valid", False, f"401 — {data.get('error', resp.text[:120])}")
+        else:
+            print_result("Novi credentials valid", False, f"HTTP {resp.status_code}: {resp.text[:120]}")
+    except Exception as e:
+        print_result("Novi credentials valid", False, f"Exception: {e}")
+
+
+def test_mailman_connection():
+    print("\n[2] Mailman 3 API Connection")
+    try:
+        resp = requests.get(f"{ACTION_BASE}/test_connection/mailman", headers=AUTH_HEADERS, timeout=15)
+        data = resp.json().get('data', {})
+        if resp.status_code == 200 and data.get('ok'):
+            print_result("Mailman credentials valid", True, f"version={data.get('version', 'unknown')}")
+        elif resp.status_code == 401:
+            print_result("Mailman credentials valid", False,
+                         f"401 — {data.get('error', resp.text[:120])}  "
+                         f"(add mailman_* keys to IDAA cfg_json)")
+        else:
+            print_result("Mailman credentials valid", False, f"HTTP {resp.status_code}: {resp.text[:120]}")
+    except Exception as e:
+        print_result("Mailman credentials valid", False, f"Exception: {e}")
+
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print(" Novi-Mailman Bridge — Connection Tests")
+    print(f" Target: {ACTION_BASE}")
+    print("=" * 60)
+
+    t_start = time.time()
+
+    test_novi_connection()
+    test_mailman_connection()
+
+    elapsed = time.time() - t_start
+    print(f"\n{'=' * 60}")
+    print(f" Results: {pass_count} passed, {fail_count} failed  ({elapsed:.2f}s)")
+    print("=" * 60)
+
+    sys.exit(0 if fail_count == 0 else 1)
--- a/tests/e2e/test_e2e_v3_action_novi_mailman_lists.py
+++ b/tests/e2e/test_e2e_v3_action_novi_mailman_lists.py
@@ -0,0 +1,166 @@
+"""
+E2E tests for the Novi-Mailman Bridge — Mailman list member operations.
+
+Covers the full member lifecycle:
+  1. Read current members of all TEST_LISTS
+  2. Subscribe TEST_EMAIL to the primary test list (TEST_LISTS[0])
+  3. Verify the address appears in the member roster
+  4. Unsubscribe TEST_EMAIL (cleanup)
+  5. Verify the address is removed
+
+Run from project root:
+    ./environment/bin/python3 tests/e2e/test_e2e_v3_action_novi_mailman_lists.py
+
+Configuration:
+    TEST_LISTS[0]  — list used for the subscribe/unsubscribe lifecycle test
+    TEST_EMAIL     — address used as the test subscriber (safe to add/remove)
+    TEST_LISTS[1:] — additional lists read-only (member count + roster check)
+
+Notes:
+    - Uses dot-notation for list IDs in URL paths (mm3@idaa.org → mm3.idaa.org)
+      because @ is a special character in URL paths.
+    - Mailman pre-confirms subscriptions (no confirmation email sent to TEST_EMAIL).
+    - Run test_e2e_v3_action_novi_mailman.py first to confirm both APIs are reachable.
+"""
+
+import sys
+import time
+import requests
+
+BASE_URL    = "https://dev-api.oneskyit.com/v3/action/e_novi_mailman"
+API_KEY     = "PMM4n50teUCaOMMTN8qOJA"
+TEST_EMAIL  = "scott.idem+mm3api@gmail.com"
+TEST_NAME   = "Scott Idem (MM3 API Test)"
+
+TEST_LISTS = [
+    "mm3@idaa.org",
+    "mm3@dgrzone.com",
+    "mm3@oneskyit.com",
+]
+
+HEADERS = {
+    "X-Aether-API-Key": API_KEY,
+    "x-no-account-id":  "bypass",
+}
+
+pass_count = 0
+fail_count = 0
+
+
+def print_result(label: str, success: bool, message: str = ""):
+    global pass_count, fail_count
+    status = "✅ PASS" if success else "❌ FAIL"
+    msg = f"   {message}" if message else ""
+    print(f"  [{status}] {label}{msg}")
+    if success:
+        pass_count += 1
+    else:
+        fail_count += 1
+
+
+def get_members(list_id: str) -> list | None:
+    """Return member list for a given list_id, or None on failure."""
+    list_id_dot = list_id.replace('@', '.')
+    resp = requests.get(f"{BASE_URL}/mailman/lists/{list_id_dot}/members",
+                        headers=HEADERS, timeout=15)
+    if resp.status_code == 200 and resp.json().get('meta', {}).get('success'):
+        return resp.json()['data'].get('members', [])
+    return None
+
+
+def subscribe(list_id: str, email: str, display_name: str = '') -> bool:
+    list_id_dot = list_id.replace('@', '.')
+    resp = requests.post(
+        f"{BASE_URL}/mailman/lists/{list_id_dot}/subscribe",
+        headers=HEADERS,
+        params={"email": email, "display_name": display_name},
+        timeout=15,
+    )
+    return resp.status_code == 200 and resp.json().get('meta', {}).get('success')
+
+
+def unsubscribe(list_id: str, email: str) -> bool:
+    list_id_dot = list_id.replace('@', '.')
+    resp = requests.delete(
+        f"{BASE_URL}/mailman/lists/{list_id_dot}/subscribe",
+        headers=HEADERS,
+        params={"email": email},
+        timeout=15,
+    )
+    return resp.status_code == 200 and resp.json().get('meta', {}).get('success')
+
+
+# ── Tests ─────────────────────────────────────────────────────────────────
+
+def test_read_members():
+    print("\n[1] Read Members")
+    for list_id in TEST_LISTS:
+        members = get_members(list_id)
+        if members is not None:
+            emails = [m['email'] for m in members]
+            print_result(f"Read {list_id}", True, f"{len(members)} member(s): {', '.join(emails) or 'none'}")
+        else:
+            print_result(f"Read {list_id}", False, "request failed")
+
+
+def test_subscribe():
+    print(f"\n[2] Subscribe {TEST_EMAIL}")
+    list_id = TEST_LISTS[0]  # mm3@idaa.org
+    ok = subscribe(list_id, TEST_EMAIL, TEST_NAME)
+    print_result(f"Subscribe to {list_id}", ok)
+
+
+def test_verify_subscription():
+    print(f"\n[3] Verify {TEST_EMAIL} appears in member list")
+    list_id = TEST_LISTS[0]
+    members = get_members(list_id)
+    if members is None:
+        print_result(f"Verify subscription in {list_id}", False, "could not fetch members")
+        return
+    found = any(m['email'] == TEST_EMAIL for m in members)
+    print_result(f"Found {TEST_EMAIL} in {list_id}", found,
+                 "" if found else "not found after subscribe")
+
+
+def test_unsubscribe():
+    print(f"\n[4] Unsubscribe {TEST_EMAIL} (cleanup)")
+    list_id = TEST_LISTS[0]
+    ok = unsubscribe(list_id, TEST_EMAIL)
+    print_result(f"Unsubscribe from {list_id}", ok)
+
+
+def test_verify_unsubscription():
+    print(f"\n[5] Verify {TEST_EMAIL} removed from member list")
+    list_id = TEST_LISTS[0]
+    members = get_members(list_id)
+    if members is None:
+        print_result(f"Verify removal from {list_id}", False, "could not fetch members")
+        return
+    still_present = any(m['email'] == TEST_EMAIL for m in members)
+    print_result(f"{TEST_EMAIL} removed from {list_id}", not still_present,
+                 "still present after unsubscribe" if still_present else "")
+
+
+# ── Main ──────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print(" Novi-Mailman Bridge — List Member Operations E2E")
+    print(f" Target: {BASE_URL}")
+    print(f" Test email: {TEST_EMAIL}")
+    print("=" * 60)
+
+    t_start = time.time()
+
+    test_read_members()
+    test_subscribe()
+    test_verify_subscription()
+    test_unsubscribe()
+    test_verify_unsubscription()
+
+    elapsed = time.time() - t_start
+    print(f"\n{'=' * 60}")
+    print(f" Results: {pass_count} passed, {fail_count} failed  ({elapsed:.2f}s)")
+    print("=" * 60)
+
+    sys.exit(0 if fail_count == 0 else 1)