OSIT-AE-API-FastAPI/app/methods/e_novi_mailman_methods.py

import json, requests
from typing import Dict, List, Optional
from app.db_sql import sql_select
from app.lib_general import log, logging, logger_reset

# ---------------------------------------------------------------------------
# Novi-Mailman Bridge — IDAA
#
# Credentials live in site.cfg_json for the IDAA site (id_random='58_gJESdlUh').
# Novi keys already present:
#   novi_api_root_url  — e.g. "https://www.idaa.org/api"
#   novi_idaa_api_key  — Base64 API key (Basic auth)
#
# Keys that must be added to cfg_json before Mailman or webhooks can work:
#   mailman_base_url     — e.g. "http://lists.idaa.org:8001"
#   mailman_username     — Mailman REST admin user (usually "restadmin")
#   mailman_password     — Mailman REST admin password
#   mailman_list_id      — Target list, e.g. "members@idaa.org"
#   novi_webhook_secret  — Shared secret for HMAC-SHA256 webhook validation
# ---------------------------------------------------------------------------

IDAA_SITE_ID_RANDOM = '58_gJESdlUh'


# ── Config Helper ─────────────────────────────────────────────────────────

@logger_reset
def _load_idaa_cfg() -> Optional[Dict]:
    """
    Load IDAA site cfg_json.  Returns the parsed dict, or None on failure.
    """
    from app.methods.site_methods import load_site_obj
    site = load_site_obj(site_id=IDAA_SITE_ID_RANDOM, model_as_dict=True)
    if not site:
        log.error("Could not load IDAA site record (id_random='%s').", IDAA_SITE_ID_RANDOM)
        return None
    cfg = site.get('cfg_json')
    if isinstance(cfg, str):
        try:
            cfg = json.loads(cfg)
        except Exception as e:
            log.error("Failed to parse IDAA cfg_json: %s", e)
            return None
    if not isinstance(cfg, dict):
        log.error("IDAA cfg_json is not a dict after parsing.")
        return None
    return cfg


# ── Novi AMS Methods ──────────────────────────────────────────────────────

@logger_reset
def test_novi_connection() -> Dict:
    """
    Verify Novi AMS API credentials from IDAA site cfg_json.
    Uses the first group GUID in novi_idaa_group_guid_li as a lightweight auth probe.
    Returns {'ok': True, 'member_count': N} on success.
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return {"ok": False, "error": "Could not load IDAA site config."}

    base_url = cfg.get('novi_api_root_url', '').rstrip('/')
    api_key  = cfg.get('novi_idaa_api_key', '')

    if not base_url or not api_key:
        return {"ok": False, "error": "novi_api_root_url or novi_idaa_api_key missing from cfg_json."}

    group_guid_li = cfg.get('novi_idaa_group_guid_li') or []
    if not group_guid_li:
        return {"ok": False, "error": "novi_idaa_group_guid_li missing from cfg_json."}

    headers = {"Authorization": f"Basic {api_key}", "Accept": "application/json"}
    try:
        # Use first group as a lightweight auth probe (pageSize=1)
        guid = group_guid_li[0]
        resp = requests.get(f"{base_url}/groups/{guid}/members",
                            headers=headers, params={"pageSize": 1}, timeout=10)
        if resp.status_code == 200:
            return {"ok": True, "probe_group": guid}
        return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
    except Exception as e:
        log.exception("Novi connection test failed: %s", e)
        return {"ok": False, "error": str(e)}


@logger_reset
def get_novi_members(status_filter: Optional[str] = None, page_size: int = 500, offset: int = 0) -> Optional[List[Dict]]:
    """
    Fetch member records from Novi AMS.

    Novi has no flat member-list endpoint.  Members are fetched per group from
    novi_idaa_group_guid_li, deduped by UniqueID, then each member's full record
    (including Email) is fetched via GET /customers/{uuid}.

    status_filter and pagination (page_size/offset) are not supported at the
    Novi API level for this approach — all group members are returned.
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return None

    base_url      = cfg.get('novi_api_root_url', '').rstrip('/')
    api_key       = cfg.get('novi_idaa_api_key', '')
    group_guid_li = cfg.get('novi_idaa_group_guid_li') or []
    headers       = {"Authorization": f"Basic {api_key}", "Accept": "application/json"}

    if not group_guid_li:
        log.error("novi_idaa_group_guid_li missing from cfg_json.")
        return None

    # Step 1: collect unique member UUIDs across all configured groups
    seen_uuids: set = set()
    uuid_list: List[str] = []
    for guid in group_guid_li:
        try:
            resp = requests.get(f"{base_url}/groups/{guid}/members",
                                headers=headers, params={"pageSize": page_size}, timeout=30)
            if resp.status_code != 200:
                log.error("Novi group %s fetch error: %s", guid, resp.status_code)
                continue
            for entry in resp.json():
                uid = entry.get('UniqueID')
                if uid and uid not in seen_uuids:
                    seen_uuids.add(uid)
                    uuid_list.append(uid)
        except Exception as e:
            log.exception("Failed to fetch Novi group %s: %s", guid, e)

    log.info("Novi: %d unique members across %d group(s).", len(uuid_list), len(group_guid_li))

    # Step 2: fetch full customer record (including Email) for each UUID
    members: List[Dict] = []
    for uid in uuid_list:
        try:
            resp = requests.get(f"{base_url}/customers/{uid}", headers=headers, timeout=10)
            if resp.status_code == 200:
                members.append(resp.json())
            else:
                log.warning("Novi customer %s fetch error: %s", uid, resp.status_code)
        except Exception as e:
            log.exception("Failed to fetch Novi customer %s: %s", uid, e)

    return members


# ── Mailman 3 Methods ─────────────────────────────────────────────────────

@logger_reset
def test_mailman_connection() -> Dict:
    """
    Verify Mailman 3 REST API credentials from IDAA site cfg_json.
    Returns {'ok': True, 'version': '...'} on success.
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return {"ok": False, "error": "Could not load IDAA site config."}

    base_url = cfg.get('mailman_base_url', '').rstrip('/')
    username = cfg.get('mailman_username', 'restadmin')
    password = cfg.get('mailman_password', '')

    if not base_url or not password:
        return {"ok": False, "error": "mailman_base_url or mailman_password missing from cfg_json."}

    try:
        resp = requests.get(f"{base_url}/3.1/system/versions", auth=(username, password), timeout=10)
        if resp.status_code == 200:
            return {"ok": True, "version": resp.json().get('mailman_version')}
        return {"ok": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}"}
    except Exception as e:
        log.exception("Mailman connection test failed: %s", e)
        return {"ok": False, "error": str(e)}


@logger_reset
def get_mailman_list_members(list_id: str, count: int = 100, page: int = 1) -> Optional[List[Dict]]:
    """
    Return members of a specific Mailman 3 list.

    Args:
        list_id: fqdn_listname e.g. 'mm3@idaa.org' or dot-notation 'mm3.idaa.org'
        count:   page size (Mailman default 20, max typically 100)
        page:    1-based page number
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return None

    base_url    = cfg.get('mailman_base_url', '').rstrip('/')
    auth        = (cfg.get('mailman_username', 'restadmin'), cfg.get('mailman_password', ''))
    list_id_dot = list_id.replace('@', '.')

    try:
        resp = requests.get(
            f"{base_url}/3.1/lists/{list_id_dot}/roster/member",
            auth=auth,
            params={"count": count, "page": page},
            timeout=10,
        )
        if resp.status_code != 200:
            log.error("Mailman member list fetch failed for %s: %s", list_id, resp.status_code)
            return None
        data = resp.json()
        return data.get('entries', [])
    except Exception as e:
        log.exception("Failed to fetch members for list %s: %s", list_id, e)
        return None


@logger_reset
def get_mailman_lists() -> Optional[List[Dict]]:
    """Return all mailing lists from this Mailman 3 instance."""
    cfg = _load_idaa_cfg()
    if not cfg:
        return None

    base_url = cfg.get('mailman_base_url', '').rstrip('/')
    auth     = (cfg.get('mailman_username', 'restadmin'), cfg.get('mailman_password', ''))

    try:
        resp = requests.get(f"{base_url}/3.1/lists", auth=auth, timeout=10)
        if resp.status_code != 200:
            log.error("Mailman list fetch failed: %s", resp.status_code)
            return None
        return resp.json().get('entries', [])
    except Exception as e:
        log.exception("Failed to fetch Mailman lists: %s", e)
        return None


@logger_reset
def subscribe_member_to_list(list_id: str, email: str, display_name: str = '') -> bool:
    """
    Subscribe an email address to a Mailman 3 list (pre-confirmed, no welcome email).
    Returns True on success or already-subscribed, False on error.
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return False

    base_url = cfg.get('mailman_base_url', '').rstrip('/')
    auth     = (cfg.get('mailman_username', 'restadmin'), cfg.get('mailman_password', ''))

    payload = {
        "list_id":              list_id.replace('@', '.'),
        "subscriber":           email,
        "display_name":         display_name,
        "pre_verified":         True,
        "pre_confirmed":        True,
        "pre_approved":         True,
        "send_welcome_message": False,
    }

    try:
        resp = requests.post(f"{base_url}/3.1/members", auth=auth, json=payload, timeout=10)
        if resp.status_code in (200, 201):
            log.info("Subscribed %s to %s", email, list_id)
            return True
        if resp.status_code == 409:
            log.debug("%s already subscribed to %s — skipping.", email, list_id)
            return True
        log.error("Subscribe failed for %s: %s - %s", email, resp.status_code, resp.text[:200])
        return False
    except Exception as e:
        log.exception("Error subscribing %s to %s: %s", email, list_id, e)
        return False


@logger_reset
def unsubscribe_member_from_list(list_id: str, email: str) -> bool:
    """
    Unsubscribe an email address from a Mailman 3 list.
    Returns True on success or not-found, False on error.
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return False

    base_url    = cfg.get('mailman_base_url', '').rstrip('/')
    auth        = (cfg.get('mailman_username', 'restadmin'), cfg.get('mailman_password', ''))
    list_id_dot = list_id.replace('@', '.')

    try:
        resp = requests.delete(
            f"{base_url}/3.1/lists/{list_id_dot}/member/{email}",
            auth=auth,
            timeout=10,
        )
        if resp.status_code in (200, 204):
            log.info("Unsubscribed %s from %s", email, list_id)
            return True
        if resp.status_code == 404:
            log.debug("%s not found in %s — skipping.", email, list_id)
            return True
        log.error("Unsubscribe failed for %s: %s - %s", email, resp.status_code, resp.text[:200])
        return False
    except Exception as e:
        log.exception("Error unsubscribing %s from %s: %s", email, list_id, e)
        return False


# ── Mirror Sync Engine ────────────────────────────────────────────────────

@logger_reset
def mirror_novi_group_to_mailman_list(novi_group_guid: str, mailman_list_id: str) -> Optional[Dict]:
    """
    Mirror a single Novi group to a Mailman 3 list.

    - Fetches all members of `novi_group_guid` from Novi.
    - For each member, fetches their customer record to get Email, name, and
      membership flags.  Members with `Active=False` or `UnsubscribeFromEmails=True`
      are excluded from the target set.
    - Fetches current members of `mailman_list_id`.
    - Subscribes addresses in Novi but not in Mailman.
    - Unsubscribes addresses in Mailman but not in Novi (mirror / full reconcile).
    - Returns a result dict with counts.
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return None

    base_url = cfg.get('novi_api_root_url', '').rstrip('/')
    api_key  = cfg.get('novi_idaa_api_key', '')
    headers  = {"Authorization": f"Basic {api_key}", "Accept": "application/json"}

    log.info("Mirror sync: Novi group %s → Mailman list %s", novi_group_guid, mailman_list_id)

    # ── Step 1: Novi group → UUIDs ────────────────────────────────────────
    try:
        resp = requests.get(f"{base_url}/groups/{novi_group_guid}/members",
                            headers=headers, params={"pageSize": 500}, timeout=30)
        if resp.status_code != 200:
            log.error("Novi group fetch failed (%s): %s", novi_group_guid, resp.status_code)
            return None
        uuid_list = [m['UniqueID'] for m in resp.json() if m.get('UniqueID')]
    except Exception as e:
        log.exception("Failed to fetch Novi group %s: %s", novi_group_guid, e)
        return None

    log.info("Novi group %s has %d member(s).", novi_group_guid, len(uuid_list))

    # ── Step 2: UUID → customer record → email ────────────────────────────
    # email.lower() → display_name
    novi_members: Dict[str, str] = {}
    skipped_inactive = 0
    skipped_unsub    = 0
    skipped_no_email = 0

    for uid in uuid_list:
        try:
            r = requests.get(f"{base_url}/customers/{uid}", headers=headers, timeout=10)
            if r.status_code != 200:
                log.warning("Novi customer %s fetch failed: %s", uid, r.status_code)
                continue
            c = r.json()
            if not c.get('Active', False):
                skipped_inactive += 1
                continue
            if c.get('UnsubscribeFromEmails', False):
                skipped_unsub += 1
                continue
            email = (c.get('Email') or '').strip()
            if not email:
                skipped_no_email += 1
                continue
            display = f"{c.get('FirstName', '')} {c.get('LastName', '')}".strip()
            novi_members[email.lower()] = display
        except Exception as e:
            log.exception("Failed to fetch Novi customer %s: %s", uid, e)

    log.info("Novi active/subscribed members with email: %d (skipped: inactive=%d unsub=%d no_email=%d)",
             len(novi_members), skipped_inactive, skipped_unsub, skipped_no_email)

    # ── Step 3: Current Mailman members ───────────────────────────────────
    mailman_entries = get_mailman_list_members(mailman_list_id)
    if mailman_entries is None:
        log.error("Could not fetch current Mailman members for %s — aborting.", mailman_list_id)
        return None
    mailman_emails = {m['email'].lower() for m in mailman_entries}

    # ── Step 4: Diff ──────────────────────────────────────────────────────
    novi_email_set  = set(novi_members.keys())
    to_subscribe    = novi_email_set - mailman_emails
    to_unsubscribe  = mailman_emails - novi_email_set

    log.info("Diff — to subscribe: %d, to unsubscribe: %d", len(to_subscribe), len(to_unsubscribe))

    results = {
        "novi_group_guid":    novi_group_guid,
        "mailman_list_id":    mailman_list_id,
        "novi_count":         len(novi_email_set),
        "mailman_count_before": len(mailman_emails),
        "subscribed":         0,
        "unsubscribed":       0,
        "errors":             0,
        "skipped_inactive":   skipped_inactive,
        "skipped_unsub":      skipped_unsub,
        "skipped_no_email":   skipped_no_email,
    }

    # ── Step 5: Apply ─────────────────────────────────────────────────────
    for email in to_subscribe:
        ok = subscribe_member_to_list(mailman_list_id, email, novi_members[email])
        if ok: results['subscribed'] += 1
        else:  results['errors'] += 1

    for email in to_unsubscribe:
        ok = unsubscribe_member_from_list(mailman_list_id, email)
        if ok: results['unsubscribed'] += 1
        else:  results['errors'] += 1

    log.info("Mirror sync complete: %s", results)
    return results


@logger_reset
def mirror_all_configured_mappings() -> Optional[List[Dict]]:
    """
    Run mirror_novi_group_to_mailman_list for every entry in
    cfg_json['novi_mailman_sync'].

    Expected cfg_json shape:
        "novi_mailman_sync": [
            {"novi_group_guid": "...", "mailman_list_id": "members@idaa.org"},
            ...
        ]
    """
    cfg = _load_idaa_cfg()
    if not cfg:
        return None

    sync_map = cfg.get('novi_mailman_sync') or []
    if not sync_map:
        log.warning("novi_mailman_sync not configured in IDAA cfg_json.")
        return []

    results = []
    for mapping in sync_map:
        guid    = mapping.get('novi_group_guid', '').strip()
        list_id = mapping.get('mailman_list_id', '').strip()
        if not guid or not list_id:
            log.warning("Skipping incomplete novi_mailman_sync entry: %s", mapping)
            continue
        result = mirror_novi_group_to_mailman_list(guid, list_id)
        results.append(result or {"novi_group_guid": guid, "mailman_list_id": list_id, "error": "sync failed"})

    return results