feat: add Novi AMS → Mailman 3 cron-based mirror sync bridge (IDAA)

Implements a full proof-of-concept for syncing IDAA's Novi AMS membership groups to Mailman 3 mailing lists via a cron-triggered reconciliation approach. Key changes: - methods: rewrote sync engine around confirmed Novi API shape — group-based member fetch (/groups/{guid}/members + /customers/{uuid}), respects Active=false and UnsubscribeFromEmails=true flags - methods: mirror_novi_group_to_mailman_list() diffs Novi group against Mailman roster and subscribes/unsubscribes accordingly (full mirror) - methods: mirror_all_configured_mappings() iterates novi_mailman_sync config array in IDAA site cfg_json — this is the cron target - router: replaced old /sync endpoint with POST /sync (all mappings) and POST /sync/group/{guid} (single mapping); removed webhook endpoint (sync is cron-based, not event-driven) - router: added GET/POST/DELETE endpoints for list member inspection and manual subscribe/unsubscribe - tests: two new e2e scripts covering connection checks and full member lifecycle; old webhook integration test archived Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 16:36:32 -04:00
parent 29579fd9f1
commit 6b25cf9c6d
6 changed files with 711 additions and 275 deletions
--- a/app/routers/api_v3_actions_e_novi_mailman.py
+++ b/app/routers/api_v3_actions_e_novi_mailman.py
@@ -1,17 +1,20 @@
-from fastapi import APIRouter, Body, Depends, Query
-from typing import Optional
 import asyncio

-from app.lib_general import log
+from fastapi import APIRouter, Depends, Query
+from typing import Optional
+
 from app.lib_general_v3 import AccountContext, get_account_context, DelayParams
 from app.models.response_models import Resp_Body_Base, mk_resp
 from app.methods.e_novi_mailman_methods import (
    test_novi_connection,
    test_mailman_connection,
    get_mailman_lists,
+    get_mailman_list_members,
+    subscribe_member_to_list,
+    unsubscribe_member_from_list,
    get_novi_members,
-    sync_novi_to_mailman,
-    handle_novi_webhook,
+    mirror_novi_group_to_mailman_list,
+    mirror_all_configured_mappings,
 )

 router = APIRouter()
@@ -24,7 +27,7 @@ async def test_novi(
        account: AccountContext = Depends(get_account_context),
        delay: DelayParams = Depends(),
        ):
-    """Verify Novi AMS API credentials stored in data_store."""
+    """Verify Novi AMS API credentials from IDAA site cfg_json."""
    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
    result = test_novi_connection()
    if result.get('ok'):
@@ -37,7 +40,7 @@ async def test_mailman(
        account: AccountContext = Depends(get_account_context),
        delay: DelayParams = Depends(),
        ):
-    """Verify Mailman 3 REST API credentials stored in data_store."""
+    """Verify Mailman 3 REST API credentials from IDAA site cfg_json."""
    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
    result = test_mailman_connection()
    if result.get('ok'):
@@ -47,6 +50,53 @@ async def test_mailman(

 # ── Inspection / Preview ──────────────────────────────────────────────────

+@router.get('/mailman/lists/{list_id}/members', response_model=Resp_Body_Base)
+async def list_mailman_list_members(
+        list_id: str,
+        count: int = Query(100, ge=1, le=500),
+        page: int = Query(1, ge=1),
+        account: AccountContext = Depends(get_account_context),
+        delay: DelayParams = Depends(),
+        ):
+    """Return members of a specific Mailman 3 list."""
+    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
+    data = get_mailman_list_members(list_id=list_id, count=count, page=page)
+    if data is not None:
+        return mk_resp(data={"count": len(data), "members": data})
+    return mk_resp(data=False, status_code=500, status_message=f"Failed to fetch members for list '{list_id}'.")
+
+
+@router.post('/mailman/lists/{list_id}/subscribe', response_model=Resp_Body_Base)
+async def mailman_subscribe(
+        list_id: str,
+        email: str = Query(..., description="Email address to subscribe"),
+        display_name: str = Query('', description="Optional display name"),
+        account: AccountContext = Depends(get_account_context),
+        delay: DelayParams = Depends(),
+        ):
+    """Subscribe an email address to a Mailman 3 list."""
+    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
+    ok = subscribe_member_to_list(list_id=list_id, email=email, display_name=display_name)
+    if ok:
+        return mk_resp(data={"list_id": list_id, "email": email}, status_message="Subscribed successfully.")
+    return mk_resp(data=False, status_code=500, status_message=f"Failed to subscribe {email} to {list_id}.")
+
+
+@router.delete('/mailman/lists/{list_id}/subscribe', response_model=Resp_Body_Base)
+async def mailman_unsubscribe(
+        list_id: str,
+        email: str = Query(..., description="Email address to unsubscribe"),
+        account: AccountContext = Depends(get_account_context),
+        delay: DelayParams = Depends(),
+        ):
+    """Unsubscribe an email address from a Mailman 3 list."""
+    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
+    ok = unsubscribe_member_from_list(list_id=list_id, email=email)
+    if ok:
+        return mk_resp(data={"list_id": list_id, "email": email}, status_message="Unsubscribed successfully.")
+    return mk_resp(data=False, status_code=500, status_message=f"Failed to unsubscribe {email} from {list_id}.")
+
+
@router.get('/mailman/lists', response_model=Resp_Body_Base)
 async def list_mailman_lists(
        account: AccountContext = Depends(get_account_context),
@@ -68,7 +118,7 @@ async def list_novi_members(
        account: AccountContext = Depends(get_account_context),
        delay: DelayParams = Depends(),
        ):
-    """Fetch a page of Novi AMS members. Useful for inspecting data before a full sync."""
+    """Fetch a page of Novi AMS members."""
    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
    data = get_novi_members(status_filter=status_filter, page_size=page_size, offset=offset)
    if data is not None:
@@ -79,38 +129,36 @@ async def list_novi_members(
 # ── Sync ──────────────────────────────────────────────────────────────────

@router.post('/sync', response_model=Resp_Body_Base)
-async def sync_full(
-        list_id: str = Query(..., description="Target Mailman list ID, e.g. 'members@yourdomain.org'"),
-        active_status: str = Query('Active', description="Novi membership status that maps to 'subscribed'"),
+async def sync_all_mappings(
        account: AccountContext = Depends(get_account_context),
        delay: DelayParams = Depends(),
        ):
    """
-    Full sync: pull all Novi members and reconcile Mailman subscription state.
-    Active members are subscribed; all others are unsubscribed.
-    This is the manual / scheduled trigger — the webhook handles real-time updates.
+    Run all Novi → Mailman mirror syncs configured in novi_mailman_sync (IDAA cfg_json).
+    This is the cron target — call on a schedule to keep all lists in sync.
    """
    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
-    result = sync_novi_to_mailman(list_id=list_id, active_status=active_status)
-    if result:
-        return mk_resp(data=result, status_message="Novi → Mailman sync complete.")
-    return mk_resp(data=False, status_code=500, status_message="Sync failed.")
+    results = mirror_all_configured_mappings()
+    if results is not None:
+        return mk_resp(data=results, status_message=f"Mirror sync complete. {len(results)} mapping(s) processed.")
+    return mk_resp(data=False, status_code=500, status_message="Mirror sync failed.")


-# ── Webhook ───────────────────────────────────────────────────────────────
-
-@router.post('/webhook/novi', response_model=Resp_Body_Base, include_in_schema=True)
-async def novi_membership_webhook(
-        payload: dict = Body(...),
+@router.post('/sync/group/{novi_group_guid}', response_model=Resp_Body_Base)
+async def sync_single_group(
+        novi_group_guid: str,
+        mailman_list_id: str = Query(..., description="Target Mailman list, e.g. 'mm3@idaa.org'"),
+        account: AccountContext = Depends(get_account_context),
+        delay: DelayParams = Depends(),
        ):
    """
-    Receives Novi AMS membership webhook events and immediately updates
-    the corresponding Mailman subscription — no auth required (Novi pushes to this endpoint).
-
-    TODO: Add HMAC signature verification once Novi webhook secret is configured.
+    Mirror a single Novi group to a specific Mailman list.
+    Useful for testing or forcing a refresh of one mapping.
    """
-    log.info(f"Novi webhook received: EventType={payload.get('EventType')} Email={payload.get('Member', {}).get('Email')}")
-    result = handle_novi_webhook(payload)
+    if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s)
+    result = mirror_novi_group_to_mailman_list(novi_group_guid, mailman_list_id)
    if result:
-        return mk_resp(data=result)
-    return mk_resp(data=False, status_code=400, status_message="Webhook payload could not be processed.")
+        return mk_resp(data=result, status_message="Mirror sync complete.")
+    return mk_resp(data=False, status_code=500, status_message="Mirror sync failed.")
+
+