Prepare project for public release

wlc_monitor.py

@@ -0,0 +1,669 @@
+#!/usr/bin/env python3
+"""SNMP health check for the Catalyst 9800 WLC, with email alerting.
+
+One invocation = one poll. State is kept in a JSON file so the next run can
+diff against it; drive it from a systemd timer (see wlc-monitor.timer).
+
+Why poll instead of receiving traps/syslog: a poll that FAILS is itself a
+signal. Traps go silent when the controller dies, and silence is
+indistinguishable from "everything is fine". This design alerts on absence.
+
+    wlc_monitor.py --config config.ini            # one poll, send alerts
+    wlc_monitor.py --config config.ini --dry-run  # poll, print, send nothing
+    wlc_monitor.py --config config.ini --test-email
+    wlc_monitor.py --config config.ini --show      # dump current state
+
+Requires: pysnmp  (pip install pysnmp)
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import configparser
+import json
+import os
+import signal
+import smtplib
+import socket
+import ssl
+import sys
+import threading
+import warnings
+from datetime import datetime, timezone
+from email.message import EmailMessage
+from pathlib import Path
+
+warnings.filterwarnings("ignore", message=".*CFB has been moved.*")
+
+try:
+    from pysnmp.hlapi.v3arch.asyncio import (
+        SnmpEngine, UsmUserData, UdpTransportTarget, ContextData,
+        ObjectType, ObjectIdentity, get_cmd, bulk_cmd,
+        usmHMACSHAAuthProtocol, usmHMAC128SHA224AuthProtocol,
+        usmHMAC192SHA256AuthProtocol, usmHMAC256SHA384AuthProtocol,
+        usmHMAC384SHA512AuthProtocol, usmAesCfb128Protocol,
+    )
+except ImportError:
+    sys.exit("error: pysnmp not installed.  pip install pysnmp")
+
+AUTH_PROTOCOLS = {
+    "sha": usmHMACSHAAuthProtocol,
+    "sha1": usmHMACSHAAuthProtocol,
+    "sha224": usmHMAC128SHA224AuthProtocol,
+    "sha256": usmHMAC192SHA256AuthProtocol,
+    "sha384": usmHMAC256SHA384AuthProtocol,
+    "sha512": usmHMAC384SHA512AuthProtocol,
+}
+
+# ---------------------------------------------------------------- OIDs
+# Scalars.
+OID_SYS_DESCR = "1.3.6.1.2.1.1.1.0"
+OID_SYS_UPTIME = "1.3.6.1.2.1.1.3.0"
+OID_SYS_NAME = "1.3.6.1.2.1.1.5.0"
+# CISCO-CONFIG-MAN-MIB: sysUpTime at which running/startup last changed.
+OID_RUN_LAST_CHANGED = "1.3.6.1.4.1.9.9.43.1.1.1.0"
+OID_RUN_LAST_SAVED = "1.3.6.1.4.1.9.9.43.1.1.2.0"
+
+# CISCO-LWAPP-AP-MIB cLApEntry columns. Column meanings were verified against
+# `show ap uptime` on this controller rather than taken from a MIB reference:
+#   .5 = AP name   .6 = AP up time   .7 = association (CAPWAP) up time
+# Only APs currently joined appear in this table, so presence == joined.
+AP_TABLE = "1.3.6.1.4.1.9.9.513.1.1.1.1"
+AP_COL_NAME = f"{AP_TABLE}.5"
+AP_COL_UPTIME = f"{AP_TABLE}.6"
+AP_COL_ASSOC_UPTIME = f"{AP_TABLE}.7"
+
+# TimeTicks are 32-bit hundredths of a second and wrap at ~497 days. A counter
+# that "decreased" may simply have wrapped, so distinguish the two by how far
+# it moved FORWARD (mod 2^32) versus how much wall-clock time actually passed
+# between polls. Five minutes of real time cannot produce months of ticks.
+TICKS_MAX = 2 ** 32
+WRAP_SLACK_S = 600          # tolerate clock skew / a missed poll or two
+WRAP_FACTOR = 3             # and be generous about timer jitter
+
+
+def ticks_to_human(t: int) -> str:
+    s = int(t) // 100
+    d, s = divmod(s, 86400)
+    h, s = divmod(s, 3600)
+    m, s = divmod(s, 60)
+    if d:
+        return f"{d}d {h}h {m}m"
+    if h:
+        return f"{h}h {m}m"
+    return f"{m}m {s}s"
+
+
+def counter_reset(old: int, new: int, elapsed_s: float | None = None) -> bool:
+    """True if `new` is a genuine reset of `old` rather than a 32-bit wrap.
+
+    A wrapped counter has still moved forward by only the elapsed time; a reset
+    counter has effectively jumped backwards by nearly its whole range. Compare
+    the forward distance (mod 2^32) against the time that really passed.
+    """
+    if new >= old:
+        return False
+    forward = (new - old) % TICKS_MAX
+    if elapsed_s is None:
+        # No timing reference (first diff after an upgrade, say). Only a value
+        # very close to the ceiling is plausibly a wrap.
+        return not (old > 4.2e9 and new < 1e7)
+    budget = (elapsed_s * WRAP_FACTOR + WRAP_SLACK_S) * 100
+    return forward > budget
+
+
+# ---------------------------------------------------------------- SNMP
+class Poller:
+    def __init__(self, host, port, user, auth, priv, auth_protocol,
+                 timeout, retries):
+        self.host, self.port = host, port
+        self.user, self.auth, self.priv = user, auth, priv
+        self.auth_protocol = auth_protocol
+        self.timeout, self.retries = timeout, retries
+        self.engine = SnmpEngine()
+
+    def _creds(self):
+        return UsmUserData(self.user, self.auth, self.priv,
+                           authProtocol=AUTH_PROTOCOLS[self.auth_protocol],
+                           privProtocol=usmAesCfb128Protocol)
+
+    async def _target(self):
+        return await UdpTransportTarget.create(
+            (self.host, self.port), timeout=self.timeout, retries=self.retries)
+
+    async def get(self, tgt, *oids):
+        ei, es, _ix, vbs = await get_cmd(
+            self.engine, self._creds(), tgt, ContextData(),
+            *[ObjectType(ObjectIdentity(o)) for o in oids], lookupMib=False)
+        if ei:
+            raise RuntimeError(str(ei))
+        if es:
+            raise RuntimeError(es.prettyPrint())
+        return [vb[1] for vb in vbs]
+
+    async def walk(self, tgt, base):
+        """Return {oid_suffix: value} for one table column."""
+        out, objs = {}, [ObjectType(ObjectIdentity(base))]
+        while True:
+            ei, es, _ix, rows = await bulk_cmd(
+                self.engine, self._creds(), tgt, ContextData(), 0, 25,
+                *objs, lookupMib=False)
+            if ei:
+                raise RuntimeError(str(ei))
+            if es:
+                raise RuntimeError(es.prettyPrint())
+            if not rows:
+                break
+            done = False
+            for vb in rows:
+                oid, val = str(vb[0]), vb[1]
+                if not oid.startswith(base + "."):
+                    done = True
+                    break
+                out[oid[len(base) + 1:]] = val
+            if done:
+                break
+            objs = [ObjectType(ObjectIdentity(str(rows[-1][0])))]
+        return out
+
+    async def poll(self) -> dict:
+        tgt = await self._target()
+        descr, uptime, name, changed, saved = await self.get(
+            tgt, OID_SYS_DESCR, OID_SYS_UPTIME, OID_SYS_NAME,
+            OID_RUN_LAST_CHANGED, OID_RUN_LAST_SAVED)
+
+        names = await self.walk(tgt, AP_COL_NAME)
+        ups = await self.walk(tgt, AP_COL_UPTIME)
+        assoc = await self.walk(tgt, AP_COL_ASSOC_UPTIME)
+
+        aps = {}
+        for idx, nm in names.items():
+            missing = [
+                column for column, values in (
+                    ("AP uptime", ups), ("association uptime", assoc)
+                ) if idx not in values
+            ]
+            if missing:
+                raise RuntimeError(
+                    f"incomplete SNMP AP row {idx}: missing {', '.join(missing)}")
+            aps[str(nm)] = {
+                "index": idx,
+                "ap_uptime": int(ups[idx]),
+                "assoc_uptime": int(assoc[idx]),
+            }
+        return {
+            "ok": True,
+            "sys_name": str(name),
+            "sys_descr": str(descr)[:120],
+            "sys_uptime": int(uptime),
+            "run_last_changed": int(changed),
+            "run_last_saved": int(saved),
+            "aps": aps,
+        }
+
+
+# ---------------------------------------------------------------- diff
+def elapsed_since(prev: dict) -> float | None:
+    """Wall-clock seconds since the previous poll, if we can tell."""
+    ts = prev.get("last_poll")
+    if not ts:
+        return None
+    try:
+        then = datetime.fromisoformat(ts)
+    except (TypeError, ValueError):
+        return None
+    if then.tzinfo is None:
+        return None
+    delta = (datetime.now(timezone.utc) - then).total_seconds()
+    return delta if delta >= 0 else None
+
+
+def build_alerts(prev: dict, cur: dict, cfg) -> list[tuple[str, str, str]]:
+    """Return [(severity, title, detail)]. prev may be {} on first run."""
+    alerts = []
+    first_run = not prev.get("aps") and not prev.get("ok")
+    el = elapsed_since(prev)
+
+    # --- reachability transitions -------------------------------------
+    if cur["ok"] and prev.get("consecutive_failures", 0) >= cfg.fail_threshold:
+        alerts.append((
+            "RECOVERY", f"{cur['sys_name']}: controller reachable again",
+            f"SNMP polling recovered after "
+            f"{prev.get('consecutive_failures')} consecutive failures.\n"
+            f"Controller uptime is now {ticks_to_human(cur['sys_uptime'])}."))
+
+    if not cur["ok"]:
+        fails = cur.get("consecutive_failures", 1)
+        if fails == cfg.fail_threshold:      # alert once, on crossing
+            alerts.append((
+                "CRITICAL", "WLC unreachable over SNMP",
+                f"{fails} consecutive failed polls of {cfg.host}.\n"
+                f"Last error: {cur.get('error')}\n\n"
+                "The controller may be down, rebooting, or unreachable "
+                "from this host."))
+        return alerts
+
+    if first_run:
+        return alerts        # nothing to diff against; stay quiet
+
+    # --- controller reboot --------------------------------------------
+    if counter_reset(prev.get("sys_uptime", 0), cur["sys_uptime"], el):
+        alerts.append((
+            "CRITICAL", f"{cur['sys_name']} rebooted",
+            f"Controller uptime went from "
+            f"{ticks_to_human(prev['sys_uptime'])} to "
+            f"{ticks_to_human(cur['sys_uptime'])}.\n"
+            f"{cur['sys_descr']}"))
+
+    # --- config changes -----------------------------------------------
+    # Only compare when we actually have a previous reading. Treating a missing
+    # baseline as 0 would report a config change after every failed poll.
+    prev_changed = prev.get("run_last_changed")
+    if prev_changed is not None and cur["run_last_changed"] > prev_changed and \
+            not counter_reset(prev.get("sys_uptime", 0), cur["sys_uptime"], el):
+        alerts.append((
+            "NOTICE", f"{cur['sys_name']} running-config changed",
+            f"running-config last changed at controller uptime "
+            f"{ticks_to_human(cur['run_last_changed'])} "
+            f"(previous poll saw {ticks_to_human(prev_changed)}).\n\n"
+            "SNMP reports THAT the config changed, not what changed. "
+            "Check `show archive config differences` or the session logs."))
+
+    # unsaved config that stays unsaved
+    if cur["run_last_changed"] > cur["run_last_saved"]:
+        unsaved_for = cur["sys_uptime"] - cur["run_last_changed"]
+        overdue = unsaved_for > cfg.unsaved_ticks
+        if overdue and not prev.get("_unsaved_alerted", False):
+            alerts.append((
+                "NOTICE", f"{cur['sys_name']} has unsaved config",
+                f"running-config has differed from startup-config for "
+                f"{ticks_to_human(unsaved_for)}.\n"
+                "A reboot would lose those changes. Run `write memory`."))
+        cur["_unsaved_alerted"] = overdue
+    else:
+        cur["_unsaved_alerted"] = False
+
+    # --- AP state ------------------------------------------------------
+    prev_aps, cur_aps = prev.get("aps", {}), cur["aps"]
+
+    for nm in sorted(set(prev_aps) - set(cur_aps)):
+        alerts.append((
+            "CRITICAL", f"AP down: {nm}",
+            f"{nm} is no longer registered to {cur['sys_name']}.\n"
+            f"At the previous poll it had been joined for "
+            f"{ticks_to_human(prev_aps[nm]['assoc_uptime'])}.\n\n"
+            f"APs still joined: {len(cur_aps)}"))
+
+    for nm in sorted(set(cur_aps) - set(prev_aps)):
+        alerts.append((
+            "RECOVERY", f"AP joined: {nm}",
+            f"{nm} registered to {cur['sys_name']}.\n"
+            f"AP has been up {ticks_to_human(cur_aps[nm]['ap_uptime'])}, "
+            f"joined {ticks_to_human(cur_aps[nm]['assoc_uptime'])} ago.\n\n"
+            f"APs joined: {len(cur_aps)}"))
+
+    for nm in sorted(set(cur_aps) & set(prev_aps)):
+        p, c = prev_aps[nm], cur_aps[nm]
+        if counter_reset(p["ap_uptime"], c["ap_uptime"], el):
+            alerts.append((
+                "WARNING", f"AP rebooted: {nm}",
+                f"{nm} uptime went from {ticks_to_human(p['ap_uptime'])} "
+                f"to {ticks_to_human(c['ap_uptime'])}.\n"
+                "The access point restarted (power, watchdog, or upgrade)."))
+        elif counter_reset(p["assoc_uptime"], c["assoc_uptime"], el):
+            # Tunnel flapped without the AP itself rebooting -- this is the
+            # case a naive up/down check silently misses.
+            alerts.append((
+                "WARNING", f"AP re-joined controller: {nm}",
+                f"{nm} association uptime reset from "
+                f"{ticks_to_human(p['assoc_uptime'])} to "
+                f"{ticks_to_human(c['assoc_uptime'])}, but AP uptime "
+                f"({ticks_to_human(c['ap_uptime'])}) did not reset.\n\n"
+                "The CAPWAP tunnel dropped and re-established without the AP "
+                "rebooting -- typically a network path or controller issue "
+                "rather than an AP fault."))
+    return alerts
+
+
+# ---------------------------------------------------------------- email
+def send_email(cfg, subject: str, body: str) -> None:
+    msg = EmailMessage()
+    msg["From"] = cfg.mail_from
+    msg["To"] = ", ".join(cfg.mail_to)
+    msg["Subject"] = subject
+    msg["X-WLC-Monitor"] = socket.gethostname()
+    msg.set_content(body)
+    context = ssl.create_default_context()
+    if cfg.smtp_security == "ssl":
+        server = smtplib.SMTP_SSL(
+            cfg.smtp_host, cfg.smtp_port, timeout=15, context=context)
+    else:
+        server = smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=15)
+    with server as s:
+        if cfg.smtp_security == "starttls":
+            s.starttls(context=context)
+        if cfg.smtp_user:
+            s.login(cfg.smtp_user, cfg.smtp_password)
+        s.send_message(msg)
+
+
+def format_body(detail: str, cur: dict, cfg) -> str:
+    now = datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
+    lines = [detail, "", "-" * 58,
+             f"controller : {cfg.host}",
+             f"detected   : {now}",
+             f"monitored  : {socket.gethostname()}"]
+    if cur.get("ok"):
+        lines += [f"uptime     : {ticks_to_human(cur['sys_uptime'])}",
+                  f"APs joined : {len(cur['aps'])}"]
+        for nm, ap in sorted(cur["aps"].items()):
+            lines.append(f"             {nm}  up {ticks_to_human(ap['ap_uptime'])}, "
+                         f"joined {ticks_to_human(ap['assoc_uptime'])}")
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------- config
+class Cfg:
+    def __init__(self, path: Path):
+        cp = configparser.ConfigParser()
+        # The file is optional: in a container every value can come from the
+        # environment, so credentials never have to be baked into an image.
+        found = cp.read(path) if path else []
+        if path and not found and not os.environ.get("WLC_HOST"):
+            sys.exit(f"error: cannot read config {path} and WLC_HOST is unset")
+        for section in ("wlc", "mail", "monitor"):
+            if not cp.has_section(section):
+                cp.add_section(section)
+        w, m, g = cp["wlc"], cp["mail"], cp["monitor"]
+
+        def s(env, sect, key, default=None):
+            """Environment wins over the file, file wins over the default."""
+            v = os.environ.get(env)
+            if v is not None and v != "":
+                return v
+            return sect.get(key, fallback=default)
+
+        def i(env, sect, key, default):
+            v = s(env, sect, key)
+            if v in (None, ""):
+                return default
+            try:
+                return int(v)
+            except ValueError:
+                sys.exit(f"error: {env or key} must be an integer, got {v!r}")
+
+        def secret(env, sect, key):
+            """Read a secret from ENV, ENV_FILE, or the config file."""
+            direct = os.environ.get(env)
+            file_name = os.environ.get(f"{env}_FILE")
+            if direct and file_name:
+                sys.exit(f"error: set only one of {env} and {env}_FILE")
+            if file_name:
+                try:
+                    return Path(file_name).read_text().strip()
+                except OSError as e:
+                    sys.exit(f"error: cannot read {env}_FILE {file_name!r}: {e}")
+            if direct:
+                return direct
+            return sect.get(key, fallback=None)
+
+        self.host = s("WLC_HOST", w, "host")
+        self.port = i("WLC_PORT", w, "port", 161)
+        self.user = s("WLC_SNMP_USER", w, "snmp_user")
+        self.auth = secret("WLC_SNMP_AUTH", w, "snmp_auth")
+        self.priv = secret("WLC_SNMP_PRIV", w, "snmp_priv")
+        self.auth_protocol = (
+            s("WLC_SNMP_AUTH_PROTOCOL", w, "auth_protocol", "sha")
+            or "sha"
+        ).lower()
+        self.timeout = i("WLC_TIMEOUT", w, "timeout", 5)
+        self.retries = i("WLC_RETRIES", w, "retries", 1)
+
+        self.smtp_host = s("MAIL_SMTP_HOST", m, "smtp_host")
+        self.smtp_port = i("MAIL_SMTP_PORT", m, "smtp_port", 25)
+        self.smtp_security = (
+            s("MAIL_SMTP_SECURITY", m, "smtp_security", "none") or "none"
+        ).lower()
+        self.smtp_user = s("MAIL_SMTP_USER", m, "smtp_user")
+        self.smtp_password = secret("MAIL_SMTP_PASSWORD", m, "smtp_password")
+        self.mail_from = s("MAIL_FROM", m, "from")
+        to = s("MAIL_TO", m, "to") or ""
+        self.mail_to = [a.strip() for a in to.split(",") if a.strip()]
+        self.subject_prefix = s("MAIL_SUBJECT_PREFIX", m, "subject_prefix", "[WLC]")
+
+        self.state_file = Path(
+            s("MONITOR_STATE_FILE", g, "state_file", "/data/state.json")).expanduser()
+        self.fail_threshold = i("MONITOR_FAIL_THRESHOLD", g, "fail_threshold", 2)
+        self.unsaved_minutes = i(
+            "MONITOR_UNSAVED_MINUTES", g, "unsaved_minutes", 60)
+        self.unsaved_ticks = self.unsaved_minutes * 60 * 100
+        self.interval = i("MONITOR_INTERVAL", g, "interval", 300)
+
+        missing = [n for n, v in (
+            ("host", self.host), ("snmp_user", self.user),
+            ("snmp_auth", self.auth), ("snmp_priv", self.priv),
+            ("smtp_host", self.smtp_host), ("mail_from", self.mail_from)
+        ) if not v]
+        if missing:
+            sys.exit(f"error: missing required settings: {', '.join(missing)}")
+        if not self.mail_to:
+            sys.exit("error: no recipients configured (MAIL_TO / [mail] to)")
+        if self.auth_protocol not in AUTH_PROTOCOLS:
+            choices = ", ".join(sorted(AUTH_PROTOCOLS))
+            sys.exit(f"error: unsupported SNMP auth protocol "
+                     f"{self.auth_protocol!r}; choose {choices}")
+        if self.smtp_security not in {"none", "starttls", "ssl"}:
+            sys.exit("error: MAIL_SMTP_SECURITY must be none, starttls, or ssl")
+        if bool(self.smtp_user) != bool(self.smtp_password):
+            sys.exit("error: SMTP username and password must be set together")
+        if self.smtp_user and self.smtp_security == "none":
+            sys.exit("error: SMTP authentication requires starttls or ssl")
+
+        for name, value, minimum, maximum in (
+            ("WLC_PORT", self.port, 1, 65535),
+            ("MAIL_SMTP_PORT", self.smtp_port, 1, 65535),
+            ("WLC_TIMEOUT", self.timeout, 1, None),
+            ("WLC_RETRIES", self.retries, 0, None),
+            ("MONITOR_FAIL_THRESHOLD", self.fail_threshold, 1, None),
+            ("MONITOR_UNSAVED_MINUTES", self.unsaved_minutes, 0, None),
+            ("MONITOR_INTERVAL", self.interval, 1, None),
+        ):
+            if value < minimum or (maximum is not None and value > maximum):
+                expected = f"{minimum}..{maximum}" if maximum else f">= {minimum}"
+                sys.exit(f"error: {name} must be {expected}, got {value}")
+
+        if found and path and path.stat().st_mode & 0o007:
+            print(f"warning: {path} is world-accessible; it holds SNMP "
+                  f"credentials. Use mode 600, or 640 with a dedicated group.",
+                  file=sys.stderr)
+
+
+def load_state(p: Path) -> dict:
+    try:
+        state = json.loads(p.read_text())
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+    return state if isinstance(state, dict) else {}
+
+
+def save_state(p: Path, state: dict) -> None:
+    p.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    tmp = p.with_suffix(p.suffix + ".tmp")
+    tmp.write_text(json.dumps(state, indent=2))
+    tmp.chmod(0o600)
+    tmp.replace(p)          # atomic; a killed run can't leave half a state file
+
+
+# ---------------------------------------------------------------- loop
+def run_loop(cfg, a) -> int:
+    """Poll forever. Used when running as a container instead of a timer."""
+    stop = threading.Event()
+
+    def handle(signum, _frame):
+        # Docker sends SIGTERM and waits ~10s before SIGKILL. Exit promptly
+        # and cleanly so a stop/restart never truncates the state file.
+        print(f"[{signal.Signals(signum).name}] shutting down", flush=True)
+        stop.set()
+
+    signal.signal(signal.SIGTERM, handle)
+    signal.signal(signal.SIGINT, handle)
+
+    print(f"polling {cfg.host} every {cfg.interval}s; "
+          f"state={cfg.state_file}; alerts -> {', '.join(cfg.mail_to)}",
+          flush=True)
+
+    while not stop.is_set():
+        started = datetime.now(timezone.utc)
+        try:
+            run_once(cfg, a)
+        except Exception as e:
+            # Never let one bad iteration kill the monitor; a crashed monitor
+            # is silent, and silence is the failure mode we are avoiding.
+            print(f"poll iteration failed: {type(e).__name__}: {e}",
+                  file=sys.stderr, flush=True)
+        # Subtract the work time so the cadence stays honest under slow polls.
+        elapsed = (datetime.now(timezone.utc) - started).total_seconds()
+        stop.wait(max(1.0, cfg.interval - elapsed))
+    return 0
+
+
+# ---------------------------------------------------------------- main
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(
+        description="SNMP health check + email alerts for a Catalyst 9800.")
+    ap.add_argument("--config", default=None,
+                    help="path to config.ini. Optional: environment variables "
+                         "can supply every setting (see README). Defaults to "
+                         "config.ini beside this script if it exists.")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="poll and print alerts, send no mail, still save state")
+    ap.add_argument("--no-save", action="store_true", help="do not update state file")
+    ap.add_argument("--test-email", action="store_true", help="send one test message and exit")
+    ap.add_argument("--show", action="store_true", help="print the current poll as JSON and exit")
+    ap.add_argument("--loop", action="store_true",
+                    help="poll forever every --interval seconds (container mode)")
+    ap.add_argument("--healthcheck", action="store_true",
+                    help="exit 0 if the last poll is recent, 1 otherwise")
+    ap.add_argument("--interval", type=int, default=None,
+                    help="seconds between polls in --loop mode "
+                         "(default: MONITOR_INTERVAL or 300)")
+    a = ap.parse_args(argv)
+
+    if a.config:
+        cfg_path = Path(a.config)
+    else:
+        beside = Path(__file__).with_name("config.ini")
+        cfg_path = beside if beside.exists() else None
+    cfg = Cfg(cfg_path)
+    if a.interval is not None:
+        if a.interval < 1:
+            ap.error("--interval must be at least 1 second")
+        cfg.interval = a.interval
+
+    if a.healthcheck:
+        # Deliberately reports on the LOOP, not on the controller. An
+        # unreachable WLC is a working monitor doing its job -- restarting the
+        # container for that would throw away the state that suppresses
+        # duplicate alerts. Only a stalled poll loop is unhealthy.
+        st = load_state(cfg.state_file)
+        age = elapsed_since(st)
+        if age is None:
+            print("no state yet")
+            return 1
+        limit = cfg.interval * 3 + 60
+        print(f"last poll {int(age)}s ago (limit {limit}s)")
+        return 0 if age <= limit else 1
+
+    if a.test_email:
+        try:
+            send_email(cfg, f"{cfg.subject_prefix} test message",
+                       "This is a test from wlc_monitor.py.\n"
+                       f"Sent via {cfg.smtp_host}:{cfg.smtp_port} from "
+                       f"{socket.gethostname()}.\n\n"
+                       "If you are reading this, the alert path works.")
+        except Exception as e:
+            print(f"FAILED to send: {type(e).__name__}: {e}", file=sys.stderr)
+            return 1
+        print(f"test message sent to {', '.join(cfg.mail_to)}")
+        return 0
+
+    if a.loop:
+        return run_loop(cfg, a)
+    return run_once(cfg, a)
+
+
+def run_once(cfg, a) -> int:
+    prev = load_state(cfg.state_file)
+    pending = [
+        item for item in prev.get("_pending_emails", [])
+        if isinstance(item, dict) and {"subject", "body"} <= item.keys()
+    ]
+    poller = Poller(cfg.host, cfg.port, cfg.user, cfg.auth, cfg.priv,
+                    cfg.auth_protocol, cfg.timeout, cfg.retries)
+    try:
+        cur = asyncio.run(poller.poll())
+        cur["consecutive_failures"] = 0
+    except Exception as e:
+        # Carry the last known good readings forward. Without this, a failed
+        # poll erases the baseline and the next success looks like a change.
+        cur = {"ok": False, "error": f"{type(e).__name__}: {e}",
+               "consecutive_failures": prev.get("consecutive_failures", 0) + 1,
+               "sys_name": prev.get("sys_name", cfg.host),
+               "aps": prev.get("aps", {})}
+        for k in ("sys_uptime", "sys_descr", "run_last_changed",
+                  "run_last_saved", "_unsaved_alerted"):
+            if k in prev:
+                cur[k] = prev[k]
+
+    if a.show:
+        print(json.dumps(cur, indent=2, default=str))
+        return 0 if cur["ok"] else 1
+
+    alerts = build_alerts(prev, cur, cfg)
+    messages = pending + [
+        {
+            "subject": f"{cfg.subject_prefix} {severity}: {title}",
+            "body": format_body(detail, cur, cfg),
+        }
+        for severity, title, detail in alerts
+    ]
+
+    rc = 0
+    remaining = list(pending) if a.dry_run else []
+    for index, message in enumerate(messages):
+        subject, body = message["subject"], message["body"]
+        if a.dry_run:
+            print(f"\n=== would send: {subject} ===\n{body}")
+        else:
+            try:
+                send_email(cfg, subject, body)
+                print(f"sent: {subject}")
+            except Exception as e:
+                print(f"FAILED to send {subject!r}: {type(e).__name__}: {e}",
+                      file=sys.stderr)
+                # Preserve FIFO ordering: a later recovery must never arrive
+                # before the outage message whose delivery just failed.
+                remaining.extend(messages[index:])
+                rc = 1
+                break
+
+    if not messages:
+        state = "ok" if cur["ok"] else f"UNREACHABLE (x{cur['consecutive_failures']})"
+        print(f"no change; {state}; "
+              f"APs joined: {len(cur.get('aps', {}))}")
+
+    # Advance the comparison baseline even when delivery fails, but retain a
+    # small durable outbox so transition alerts are retried on the next poll.
+    if remaining:
+        cur["_pending_emails"] = remaining
+    else:
+        cur.pop("_pending_emails", None)
+    if not a.no_save:
+        cur["last_poll"] = datetime.now(timezone.utc).isoformat()
+        save_state(cfg.state_file, cur)
+    return rc
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())