669 lines
28 KiB
Python
Executable file
669 lines
28 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""SNMP health check for the Catalyst 9800 WLC, with email alerting.
|
|
|
|
One invocation = one poll. State is kept in a JSON file so the next run can
|
|
diff against it; drive it from a systemd timer (see wlc-monitor.timer).
|
|
|
|
Why poll instead of receiving traps/syslog: a poll that FAILS is itself a
|
|
signal. Traps go silent when the controller dies, and silence is
|
|
indistinguishable from "everything is fine". This design alerts on absence.
|
|
|
|
wlc_monitor.py --config config.ini # one poll, send alerts
|
|
wlc_monitor.py --config config.ini --dry-run # poll, print, send nothing
|
|
wlc_monitor.py --config config.ini --test-email
|
|
wlc_monitor.py --config config.ini --show # dump current state
|
|
|
|
Requires: pysnmp (pip install pysnmp)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import configparser
|
|
import json
|
|
import os
|
|
import signal
|
|
import smtplib
|
|
import socket
|
|
import ssl
|
|
import sys
|
|
import threading
|
|
import warnings
|
|
from datetime import datetime, timezone
|
|
from email.message import EmailMessage
|
|
from pathlib import Path
|
|
|
|
warnings.filterwarnings("ignore", message=".*CFB has been moved.*")
|
|
|
|
try:
|
|
from pysnmp.hlapi.v3arch.asyncio import (
|
|
SnmpEngine, UsmUserData, UdpTransportTarget, ContextData,
|
|
ObjectType, ObjectIdentity, get_cmd, bulk_cmd,
|
|
usmHMACSHAAuthProtocol, usmHMAC128SHA224AuthProtocol,
|
|
usmHMAC192SHA256AuthProtocol, usmHMAC256SHA384AuthProtocol,
|
|
usmHMAC384SHA512AuthProtocol, usmAesCfb128Protocol,
|
|
)
|
|
except ImportError:
|
|
sys.exit("error: pysnmp not installed. pip install pysnmp")
|
|
|
|
AUTH_PROTOCOLS = {
|
|
"sha": usmHMACSHAAuthProtocol,
|
|
"sha1": usmHMACSHAAuthProtocol,
|
|
"sha224": usmHMAC128SHA224AuthProtocol,
|
|
"sha256": usmHMAC192SHA256AuthProtocol,
|
|
"sha384": usmHMAC256SHA384AuthProtocol,
|
|
"sha512": usmHMAC384SHA512AuthProtocol,
|
|
}
|
|
|
|
# ---------------------------------------------------------------- OIDs
|
|
# Scalars.
|
|
OID_SYS_DESCR = "1.3.6.1.2.1.1.1.0"
|
|
OID_SYS_UPTIME = "1.3.6.1.2.1.1.3.0"
|
|
OID_SYS_NAME = "1.3.6.1.2.1.1.5.0"
|
|
# CISCO-CONFIG-MAN-MIB: sysUpTime at which running/startup last changed.
|
|
OID_RUN_LAST_CHANGED = "1.3.6.1.4.1.9.9.43.1.1.1.0"
|
|
OID_RUN_LAST_SAVED = "1.3.6.1.4.1.9.9.43.1.1.2.0"
|
|
|
|
# CISCO-LWAPP-AP-MIB cLApEntry columns. Column meanings were verified against
|
|
# `show ap uptime` on this controller rather than taken from a MIB reference:
|
|
# .5 = AP name .6 = AP up time .7 = association (CAPWAP) up time
|
|
# Only APs currently joined appear in this table, so presence == joined.
|
|
AP_TABLE = "1.3.6.1.4.1.9.9.513.1.1.1.1"
|
|
AP_COL_NAME = f"{AP_TABLE}.5"
|
|
AP_COL_UPTIME = f"{AP_TABLE}.6"
|
|
AP_COL_ASSOC_UPTIME = f"{AP_TABLE}.7"
|
|
|
|
# TimeTicks are 32-bit hundredths of a second and wrap at ~497 days. A counter
|
|
# that "decreased" may simply have wrapped, so distinguish the two by how far
|
|
# it moved FORWARD (mod 2^32) versus how much wall-clock time actually passed
|
|
# between polls. Five minutes of real time cannot produce months of ticks.
|
|
TICKS_MAX = 2 ** 32
|
|
WRAP_SLACK_S = 600 # tolerate clock skew / a missed poll or two
|
|
WRAP_FACTOR = 3 # and be generous about timer jitter
|
|
|
|
|
|
def ticks_to_human(t: int) -> str:
|
|
s = int(t) // 100
|
|
d, s = divmod(s, 86400)
|
|
h, s = divmod(s, 3600)
|
|
m, s = divmod(s, 60)
|
|
if d:
|
|
return f"{d}d {h}h {m}m"
|
|
if h:
|
|
return f"{h}h {m}m"
|
|
return f"{m}m {s}s"
|
|
|
|
|
|
def counter_reset(old: int, new: int, elapsed_s: float | None = None) -> bool:
|
|
"""True if `new` is a genuine reset of `old` rather than a 32-bit wrap.
|
|
|
|
A wrapped counter has still moved forward by only the elapsed time; a reset
|
|
counter has effectively jumped backwards by nearly its whole range. Compare
|
|
the forward distance (mod 2^32) against the time that really passed.
|
|
"""
|
|
if new >= old:
|
|
return False
|
|
forward = (new - old) % TICKS_MAX
|
|
if elapsed_s is None:
|
|
# No timing reference (first diff after an upgrade, say). Only a value
|
|
# very close to the ceiling is plausibly a wrap.
|
|
return not (old > 4.2e9 and new < 1e7)
|
|
budget = (elapsed_s * WRAP_FACTOR + WRAP_SLACK_S) * 100
|
|
return forward > budget
|
|
|
|
|
|
# ---------------------------------------------------------------- SNMP
|
|
class Poller:
|
|
def __init__(self, host, port, user, auth, priv, auth_protocol,
|
|
timeout, retries):
|
|
self.host, self.port = host, port
|
|
self.user, self.auth, self.priv = user, auth, priv
|
|
self.auth_protocol = auth_protocol
|
|
self.timeout, self.retries = timeout, retries
|
|
self.engine = SnmpEngine()
|
|
|
|
def _creds(self):
|
|
return UsmUserData(self.user, self.auth, self.priv,
|
|
authProtocol=AUTH_PROTOCOLS[self.auth_protocol],
|
|
privProtocol=usmAesCfb128Protocol)
|
|
|
|
async def _target(self):
|
|
return await UdpTransportTarget.create(
|
|
(self.host, self.port), timeout=self.timeout, retries=self.retries)
|
|
|
|
async def get(self, tgt, *oids):
|
|
ei, es, _ix, vbs = await get_cmd(
|
|
self.engine, self._creds(), tgt, ContextData(),
|
|
*[ObjectType(ObjectIdentity(o)) for o in oids], lookupMib=False)
|
|
if ei:
|
|
raise RuntimeError(str(ei))
|
|
if es:
|
|
raise RuntimeError(es.prettyPrint())
|
|
return [vb[1] for vb in vbs]
|
|
|
|
async def walk(self, tgt, base):
|
|
"""Return {oid_suffix: value} for one table column."""
|
|
out, objs = {}, [ObjectType(ObjectIdentity(base))]
|
|
while True:
|
|
ei, es, _ix, rows = await bulk_cmd(
|
|
self.engine, self._creds(), tgt, ContextData(), 0, 25,
|
|
*objs, lookupMib=False)
|
|
if ei:
|
|
raise RuntimeError(str(ei))
|
|
if es:
|
|
raise RuntimeError(es.prettyPrint())
|
|
if not rows:
|
|
break
|
|
done = False
|
|
for vb in rows:
|
|
oid, val = str(vb[0]), vb[1]
|
|
if not oid.startswith(base + "."):
|
|
done = True
|
|
break
|
|
out[oid[len(base) + 1:]] = val
|
|
if done:
|
|
break
|
|
objs = [ObjectType(ObjectIdentity(str(rows[-1][0])))]
|
|
return out
|
|
|
|
async def poll(self) -> dict:
|
|
tgt = await self._target()
|
|
descr, uptime, name, changed, saved = await self.get(
|
|
tgt, OID_SYS_DESCR, OID_SYS_UPTIME, OID_SYS_NAME,
|
|
OID_RUN_LAST_CHANGED, OID_RUN_LAST_SAVED)
|
|
|
|
names = await self.walk(tgt, AP_COL_NAME)
|
|
ups = await self.walk(tgt, AP_COL_UPTIME)
|
|
assoc = await self.walk(tgt, AP_COL_ASSOC_UPTIME)
|
|
|
|
aps = {}
|
|
for idx, nm in names.items():
|
|
missing = [
|
|
column for column, values in (
|
|
("AP uptime", ups), ("association uptime", assoc)
|
|
) if idx not in values
|
|
]
|
|
if missing:
|
|
raise RuntimeError(
|
|
f"incomplete SNMP AP row {idx}: missing {', '.join(missing)}")
|
|
aps[str(nm)] = {
|
|
"index": idx,
|
|
"ap_uptime": int(ups[idx]),
|
|
"assoc_uptime": int(assoc[idx]),
|
|
}
|
|
return {
|
|
"ok": True,
|
|
"sys_name": str(name),
|
|
"sys_descr": str(descr)[:120],
|
|
"sys_uptime": int(uptime),
|
|
"run_last_changed": int(changed),
|
|
"run_last_saved": int(saved),
|
|
"aps": aps,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------- diff
|
|
def elapsed_since(prev: dict) -> float | None:
|
|
"""Wall-clock seconds since the previous poll, if we can tell."""
|
|
ts = prev.get("last_poll")
|
|
if not ts:
|
|
return None
|
|
try:
|
|
then = datetime.fromisoformat(ts)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if then.tzinfo is None:
|
|
return None
|
|
delta = (datetime.now(timezone.utc) - then).total_seconds()
|
|
return delta if delta >= 0 else None
|
|
|
|
|
|
def build_alerts(prev: dict, cur: dict, cfg) -> list[tuple[str, str, str]]:
|
|
"""Return [(severity, title, detail)]. prev may be {} on first run."""
|
|
alerts = []
|
|
first_run = not prev.get("aps") and not prev.get("ok")
|
|
el = elapsed_since(prev)
|
|
|
|
# --- reachability transitions -------------------------------------
|
|
if cur["ok"] and prev.get("consecutive_failures", 0) >= cfg.fail_threshold:
|
|
alerts.append((
|
|
"RECOVERY", f"{cur['sys_name']}: controller reachable again",
|
|
f"SNMP polling recovered after "
|
|
f"{prev.get('consecutive_failures')} consecutive failures.\n"
|
|
f"Controller uptime is now {ticks_to_human(cur['sys_uptime'])}."))
|
|
|
|
if not cur["ok"]:
|
|
fails = cur.get("consecutive_failures", 1)
|
|
if fails == cfg.fail_threshold: # alert once, on crossing
|
|
alerts.append((
|
|
"CRITICAL", "WLC unreachable over SNMP",
|
|
f"{fails} consecutive failed polls of {cfg.host}.\n"
|
|
f"Last error: {cur.get('error')}\n\n"
|
|
"The controller may be down, rebooting, or unreachable "
|
|
"from this host."))
|
|
return alerts
|
|
|
|
if first_run:
|
|
return alerts # nothing to diff against; stay quiet
|
|
|
|
# --- controller reboot --------------------------------------------
|
|
if counter_reset(prev.get("sys_uptime", 0), cur["sys_uptime"], el):
|
|
alerts.append((
|
|
"CRITICAL", f"{cur['sys_name']} rebooted",
|
|
f"Controller uptime went from "
|
|
f"{ticks_to_human(prev['sys_uptime'])} to "
|
|
f"{ticks_to_human(cur['sys_uptime'])}.\n"
|
|
f"{cur['sys_descr']}"))
|
|
|
|
# --- config changes -----------------------------------------------
|
|
# Only compare when we actually have a previous reading. Treating a missing
|
|
# baseline as 0 would report a config change after every failed poll.
|
|
prev_changed = prev.get("run_last_changed")
|
|
if prev_changed is not None and cur["run_last_changed"] > prev_changed and \
|
|
not counter_reset(prev.get("sys_uptime", 0), cur["sys_uptime"], el):
|
|
alerts.append((
|
|
"NOTICE", f"{cur['sys_name']} running-config changed",
|
|
f"running-config last changed at controller uptime "
|
|
f"{ticks_to_human(cur['run_last_changed'])} "
|
|
f"(previous poll saw {ticks_to_human(prev_changed)}).\n\n"
|
|
"SNMP reports THAT the config changed, not what changed. "
|
|
"Check `show archive config differences` or the session logs."))
|
|
|
|
# unsaved config that stays unsaved
|
|
if cur["run_last_changed"] > cur["run_last_saved"]:
|
|
unsaved_for = cur["sys_uptime"] - cur["run_last_changed"]
|
|
overdue = unsaved_for > cfg.unsaved_ticks
|
|
if overdue and not prev.get("_unsaved_alerted", False):
|
|
alerts.append((
|
|
"NOTICE", f"{cur['sys_name']} has unsaved config",
|
|
f"running-config has differed from startup-config for "
|
|
f"{ticks_to_human(unsaved_for)}.\n"
|
|
"A reboot would lose those changes. Run `write memory`."))
|
|
cur["_unsaved_alerted"] = overdue
|
|
else:
|
|
cur["_unsaved_alerted"] = False
|
|
|
|
# --- AP state ------------------------------------------------------
|
|
prev_aps, cur_aps = prev.get("aps", {}), cur["aps"]
|
|
|
|
for nm in sorted(set(prev_aps) - set(cur_aps)):
|
|
alerts.append((
|
|
"CRITICAL", f"AP down: {nm}",
|
|
f"{nm} is no longer registered to {cur['sys_name']}.\n"
|
|
f"At the previous poll it had been joined for "
|
|
f"{ticks_to_human(prev_aps[nm]['assoc_uptime'])}.\n\n"
|
|
f"APs still joined: {len(cur_aps)}"))
|
|
|
|
for nm in sorted(set(cur_aps) - set(prev_aps)):
|
|
alerts.append((
|
|
"RECOVERY", f"AP joined: {nm}",
|
|
f"{nm} registered to {cur['sys_name']}.\n"
|
|
f"AP has been up {ticks_to_human(cur_aps[nm]['ap_uptime'])}, "
|
|
f"joined {ticks_to_human(cur_aps[nm]['assoc_uptime'])} ago.\n\n"
|
|
f"APs joined: {len(cur_aps)}"))
|
|
|
|
for nm in sorted(set(cur_aps) & set(prev_aps)):
|
|
p, c = prev_aps[nm], cur_aps[nm]
|
|
if counter_reset(p["ap_uptime"], c["ap_uptime"], el):
|
|
alerts.append((
|
|
"WARNING", f"AP rebooted: {nm}",
|
|
f"{nm} uptime went from {ticks_to_human(p['ap_uptime'])} "
|
|
f"to {ticks_to_human(c['ap_uptime'])}.\n"
|
|
"The access point restarted (power, watchdog, or upgrade)."))
|
|
elif counter_reset(p["assoc_uptime"], c["assoc_uptime"], el):
|
|
# Tunnel flapped without the AP itself rebooting -- this is the
|
|
# case a naive up/down check silently misses.
|
|
alerts.append((
|
|
"WARNING", f"AP re-joined controller: {nm}",
|
|
f"{nm} association uptime reset from "
|
|
f"{ticks_to_human(p['assoc_uptime'])} to "
|
|
f"{ticks_to_human(c['assoc_uptime'])}, but AP uptime "
|
|
f"({ticks_to_human(c['ap_uptime'])}) did not reset.\n\n"
|
|
"The CAPWAP tunnel dropped and re-established without the AP "
|
|
"rebooting -- typically a network path or controller issue "
|
|
"rather than an AP fault."))
|
|
return alerts
|
|
|
|
|
|
# ---------------------------------------------------------------- email
|
|
def send_email(cfg, subject: str, body: str) -> None:
|
|
msg = EmailMessage()
|
|
msg["From"] = cfg.mail_from
|
|
msg["To"] = ", ".join(cfg.mail_to)
|
|
msg["Subject"] = subject
|
|
msg["X-WLC-Monitor"] = socket.gethostname()
|
|
msg.set_content(body)
|
|
context = ssl.create_default_context()
|
|
if cfg.smtp_security == "ssl":
|
|
server = smtplib.SMTP_SSL(
|
|
cfg.smtp_host, cfg.smtp_port, timeout=15, context=context)
|
|
else:
|
|
server = smtplib.SMTP(cfg.smtp_host, cfg.smtp_port, timeout=15)
|
|
with server as s:
|
|
if cfg.smtp_security == "starttls":
|
|
s.starttls(context=context)
|
|
if cfg.smtp_user:
|
|
s.login(cfg.smtp_user, cfg.smtp_password)
|
|
s.send_message(msg)
|
|
|
|
|
|
def format_body(detail: str, cur: dict, cfg) -> str:
|
|
now = datetime.now(timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
lines = [detail, "", "-" * 58,
|
|
f"controller : {cfg.host}",
|
|
f"detected : {now}",
|
|
f"monitored : {socket.gethostname()}"]
|
|
if cur.get("ok"):
|
|
lines += [f"uptime : {ticks_to_human(cur['sys_uptime'])}",
|
|
f"APs joined : {len(cur['aps'])}"]
|
|
for nm, ap in sorted(cur["aps"].items()):
|
|
lines.append(f" {nm} up {ticks_to_human(ap['ap_uptime'])}, "
|
|
f"joined {ticks_to_human(ap['assoc_uptime'])}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------- config
|
|
class Cfg:
|
|
def __init__(self, path: Path):
|
|
cp = configparser.ConfigParser()
|
|
# The file is optional: in a container every value can come from the
|
|
# environment, so credentials never have to be baked into an image.
|
|
found = cp.read(path) if path else []
|
|
if path and not found and not os.environ.get("WLC_HOST"):
|
|
sys.exit(f"error: cannot read config {path} and WLC_HOST is unset")
|
|
for section in ("wlc", "mail", "monitor"):
|
|
if not cp.has_section(section):
|
|
cp.add_section(section)
|
|
w, m, g = cp["wlc"], cp["mail"], cp["monitor"]
|
|
|
|
def s(env, sect, key, default=None):
|
|
"""Environment wins over the file, file wins over the default."""
|
|
v = os.environ.get(env)
|
|
if v is not None and v != "":
|
|
return v
|
|
return sect.get(key, fallback=default)
|
|
|
|
def i(env, sect, key, default):
|
|
v = s(env, sect, key)
|
|
if v in (None, ""):
|
|
return default
|
|
try:
|
|
return int(v)
|
|
except ValueError:
|
|
sys.exit(f"error: {env or key} must be an integer, got {v!r}")
|
|
|
|
def secret(env, sect, key):
|
|
"""Read a secret from ENV, ENV_FILE, or the config file."""
|
|
direct = os.environ.get(env)
|
|
file_name = os.environ.get(f"{env}_FILE")
|
|
if direct and file_name:
|
|
sys.exit(f"error: set only one of {env} and {env}_FILE")
|
|
if file_name:
|
|
try:
|
|
return Path(file_name).read_text().strip()
|
|
except OSError as e:
|
|
sys.exit(f"error: cannot read {env}_FILE {file_name!r}: {e}")
|
|
if direct:
|
|
return direct
|
|
return sect.get(key, fallback=None)
|
|
|
|
self.host = s("WLC_HOST", w, "host")
|
|
self.port = i("WLC_PORT", w, "port", 161)
|
|
self.user = s("WLC_SNMP_USER", w, "snmp_user")
|
|
self.auth = secret("WLC_SNMP_AUTH", w, "snmp_auth")
|
|
self.priv = secret("WLC_SNMP_PRIV", w, "snmp_priv")
|
|
self.auth_protocol = (
|
|
s("WLC_SNMP_AUTH_PROTOCOL", w, "auth_protocol", "sha")
|
|
or "sha"
|
|
).lower()
|
|
self.timeout = i("WLC_TIMEOUT", w, "timeout", 5)
|
|
self.retries = i("WLC_RETRIES", w, "retries", 1)
|
|
|
|
self.smtp_host = s("MAIL_SMTP_HOST", m, "smtp_host")
|
|
self.smtp_port = i("MAIL_SMTP_PORT", m, "smtp_port", 25)
|
|
self.smtp_security = (
|
|
s("MAIL_SMTP_SECURITY", m, "smtp_security", "none") or "none"
|
|
).lower()
|
|
self.smtp_user = s("MAIL_SMTP_USER", m, "smtp_user")
|
|
self.smtp_password = secret("MAIL_SMTP_PASSWORD", m, "smtp_password")
|
|
self.mail_from = s("MAIL_FROM", m, "from")
|
|
to = s("MAIL_TO", m, "to") or ""
|
|
self.mail_to = [a.strip() for a in to.split(",") if a.strip()]
|
|
self.subject_prefix = s("MAIL_SUBJECT_PREFIX", m, "subject_prefix", "[WLC]")
|
|
|
|
self.state_file = Path(
|
|
s("MONITOR_STATE_FILE", g, "state_file", "/data/state.json")).expanduser()
|
|
self.fail_threshold = i("MONITOR_FAIL_THRESHOLD", g, "fail_threshold", 2)
|
|
self.unsaved_minutes = i(
|
|
"MONITOR_UNSAVED_MINUTES", g, "unsaved_minutes", 60)
|
|
self.unsaved_ticks = self.unsaved_minutes * 60 * 100
|
|
self.interval = i("MONITOR_INTERVAL", g, "interval", 300)
|
|
|
|
missing = [n for n, v in (
|
|
("host", self.host), ("snmp_user", self.user),
|
|
("snmp_auth", self.auth), ("snmp_priv", self.priv),
|
|
("smtp_host", self.smtp_host), ("mail_from", self.mail_from)
|
|
) if not v]
|
|
if missing:
|
|
sys.exit(f"error: missing required settings: {', '.join(missing)}")
|
|
if not self.mail_to:
|
|
sys.exit("error: no recipients configured (MAIL_TO / [mail] to)")
|
|
if self.auth_protocol not in AUTH_PROTOCOLS:
|
|
choices = ", ".join(sorted(AUTH_PROTOCOLS))
|
|
sys.exit(f"error: unsupported SNMP auth protocol "
|
|
f"{self.auth_protocol!r}; choose {choices}")
|
|
if self.smtp_security not in {"none", "starttls", "ssl"}:
|
|
sys.exit("error: MAIL_SMTP_SECURITY must be none, starttls, or ssl")
|
|
if bool(self.smtp_user) != bool(self.smtp_password):
|
|
sys.exit("error: SMTP username and password must be set together")
|
|
if self.smtp_user and self.smtp_security == "none":
|
|
sys.exit("error: SMTP authentication requires starttls or ssl")
|
|
|
|
for name, value, minimum, maximum in (
|
|
("WLC_PORT", self.port, 1, 65535),
|
|
("MAIL_SMTP_PORT", self.smtp_port, 1, 65535),
|
|
("WLC_TIMEOUT", self.timeout, 1, None),
|
|
("WLC_RETRIES", self.retries, 0, None),
|
|
("MONITOR_FAIL_THRESHOLD", self.fail_threshold, 1, None),
|
|
("MONITOR_UNSAVED_MINUTES", self.unsaved_minutes, 0, None),
|
|
("MONITOR_INTERVAL", self.interval, 1, None),
|
|
):
|
|
if value < minimum or (maximum is not None and value > maximum):
|
|
expected = f"{minimum}..{maximum}" if maximum else f">= {minimum}"
|
|
sys.exit(f"error: {name} must be {expected}, got {value}")
|
|
|
|
if found and path and path.stat().st_mode & 0o007:
|
|
print(f"warning: {path} is world-accessible; it holds SNMP "
|
|
f"credentials. Use mode 600, or 640 with a dedicated group.",
|
|
file=sys.stderr)
|
|
|
|
|
|
def load_state(p: Path) -> dict:
|
|
try:
|
|
state = json.loads(p.read_text())
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
return {}
|
|
return state if isinstance(state, dict) else {}
|
|
|
|
|
|
def save_state(p: Path, state: dict) -> None:
|
|
p.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
tmp = p.with_suffix(p.suffix + ".tmp")
|
|
tmp.write_text(json.dumps(state, indent=2))
|
|
tmp.chmod(0o600)
|
|
tmp.replace(p) # atomic; a killed run can't leave half a state file
|
|
|
|
|
|
# ---------------------------------------------------------------- loop
|
|
def run_loop(cfg, a) -> int:
|
|
"""Poll forever. Used when running as a container instead of a timer."""
|
|
stop = threading.Event()
|
|
|
|
def handle(signum, _frame):
|
|
# Docker sends SIGTERM and waits ~10s before SIGKILL. Exit promptly
|
|
# and cleanly so a stop/restart never truncates the state file.
|
|
print(f"[{signal.Signals(signum).name}] shutting down", flush=True)
|
|
stop.set()
|
|
|
|
signal.signal(signal.SIGTERM, handle)
|
|
signal.signal(signal.SIGINT, handle)
|
|
|
|
print(f"polling {cfg.host} every {cfg.interval}s; "
|
|
f"state={cfg.state_file}; alerts -> {', '.join(cfg.mail_to)}",
|
|
flush=True)
|
|
|
|
while not stop.is_set():
|
|
started = datetime.now(timezone.utc)
|
|
try:
|
|
run_once(cfg, a)
|
|
except Exception as e:
|
|
# Never let one bad iteration kill the monitor; a crashed monitor
|
|
# is silent, and silence is the failure mode we are avoiding.
|
|
print(f"poll iteration failed: {type(e).__name__}: {e}",
|
|
file=sys.stderr, flush=True)
|
|
# Subtract the work time so the cadence stays honest under slow polls.
|
|
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
|
stop.wait(max(1.0, cfg.interval - elapsed))
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------- main
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(
|
|
description="SNMP health check + email alerts for a Catalyst 9800.")
|
|
ap.add_argument("--config", default=None,
|
|
help="path to config.ini. Optional: environment variables "
|
|
"can supply every setting (see README). Defaults to "
|
|
"config.ini beside this script if it exists.")
|
|
ap.add_argument("--dry-run", action="store_true",
|
|
help="poll and print alerts, send no mail, still save state")
|
|
ap.add_argument("--no-save", action="store_true", help="do not update state file")
|
|
ap.add_argument("--test-email", action="store_true", help="send one test message and exit")
|
|
ap.add_argument("--show", action="store_true", help="print the current poll as JSON and exit")
|
|
ap.add_argument("--loop", action="store_true",
|
|
help="poll forever every --interval seconds (container mode)")
|
|
ap.add_argument("--healthcheck", action="store_true",
|
|
help="exit 0 if the last poll is recent, 1 otherwise")
|
|
ap.add_argument("--interval", type=int, default=None,
|
|
help="seconds between polls in --loop mode "
|
|
"(default: MONITOR_INTERVAL or 300)")
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.config:
|
|
cfg_path = Path(a.config)
|
|
else:
|
|
beside = Path(__file__).with_name("config.ini")
|
|
cfg_path = beside if beside.exists() else None
|
|
cfg = Cfg(cfg_path)
|
|
if a.interval is not None:
|
|
if a.interval < 1:
|
|
ap.error("--interval must be at least 1 second")
|
|
cfg.interval = a.interval
|
|
|
|
if a.healthcheck:
|
|
# Deliberately reports on the LOOP, not on the controller. An
|
|
# unreachable WLC is a working monitor doing its job -- restarting the
|
|
# container for that would throw away the state that suppresses
|
|
# duplicate alerts. Only a stalled poll loop is unhealthy.
|
|
st = load_state(cfg.state_file)
|
|
age = elapsed_since(st)
|
|
if age is None:
|
|
print("no state yet")
|
|
return 1
|
|
limit = cfg.interval * 3 + 60
|
|
print(f"last poll {int(age)}s ago (limit {limit}s)")
|
|
return 0 if age <= limit else 1
|
|
|
|
if a.test_email:
|
|
try:
|
|
send_email(cfg, f"{cfg.subject_prefix} test message",
|
|
"This is a test from wlc_monitor.py.\n"
|
|
f"Sent via {cfg.smtp_host}:{cfg.smtp_port} from "
|
|
f"{socket.gethostname()}.\n\n"
|
|
"If you are reading this, the alert path works.")
|
|
except Exception as e:
|
|
print(f"FAILED to send: {type(e).__name__}: {e}", file=sys.stderr)
|
|
return 1
|
|
print(f"test message sent to {', '.join(cfg.mail_to)}")
|
|
return 0
|
|
|
|
if a.loop:
|
|
return run_loop(cfg, a)
|
|
return run_once(cfg, a)
|
|
|
|
|
|
def run_once(cfg, a) -> int:
|
|
prev = load_state(cfg.state_file)
|
|
pending = [
|
|
item for item in prev.get("_pending_emails", [])
|
|
if isinstance(item, dict) and {"subject", "body"} <= item.keys()
|
|
]
|
|
poller = Poller(cfg.host, cfg.port, cfg.user, cfg.auth, cfg.priv,
|
|
cfg.auth_protocol, cfg.timeout, cfg.retries)
|
|
try:
|
|
cur = asyncio.run(poller.poll())
|
|
cur["consecutive_failures"] = 0
|
|
except Exception as e:
|
|
# Carry the last known good readings forward. Without this, a failed
|
|
# poll erases the baseline and the next success looks like a change.
|
|
cur = {"ok": False, "error": f"{type(e).__name__}: {e}",
|
|
"consecutive_failures": prev.get("consecutive_failures", 0) + 1,
|
|
"sys_name": prev.get("sys_name", cfg.host),
|
|
"aps": prev.get("aps", {})}
|
|
for k in ("sys_uptime", "sys_descr", "run_last_changed",
|
|
"run_last_saved", "_unsaved_alerted"):
|
|
if k in prev:
|
|
cur[k] = prev[k]
|
|
|
|
if a.show:
|
|
print(json.dumps(cur, indent=2, default=str))
|
|
return 0 if cur["ok"] else 1
|
|
|
|
alerts = build_alerts(prev, cur, cfg)
|
|
messages = pending + [
|
|
{
|
|
"subject": f"{cfg.subject_prefix} {severity}: {title}",
|
|
"body": format_body(detail, cur, cfg),
|
|
}
|
|
for severity, title, detail in alerts
|
|
]
|
|
|
|
rc = 0
|
|
remaining = list(pending) if a.dry_run else []
|
|
for index, message in enumerate(messages):
|
|
subject, body = message["subject"], message["body"]
|
|
if a.dry_run:
|
|
print(f"\n=== would send: {subject} ===\n{body}")
|
|
else:
|
|
try:
|
|
send_email(cfg, subject, body)
|
|
print(f"sent: {subject}")
|
|
except Exception as e:
|
|
print(f"FAILED to send {subject!r}: {type(e).__name__}: {e}",
|
|
file=sys.stderr)
|
|
# Preserve FIFO ordering: a later recovery must never arrive
|
|
# before the outage message whose delivery just failed.
|
|
remaining.extend(messages[index:])
|
|
rc = 1
|
|
break
|
|
|
|
if not messages:
|
|
state = "ok" if cur["ok"] else f"UNREACHABLE (x{cur['consecutive_failures']})"
|
|
print(f"no change; {state}; "
|
|
f"APs joined: {len(cur.get('aps', {}))}")
|
|
|
|
# Advance the comparison baseline even when delivery fails, but retain a
|
|
# small durable outbox so transition alerts are retried on the next poll.
|
|
if remaining:
|
|
cur["_pending_emails"] = remaining
|
|
else:
|
|
cur.pop("_pending_emails", None)
|
|
if not a.no_save:
|
|
cur["last_poll"] = datetime.now(timezone.utc).isoformat()
|
|
save_state(cfg.state_file, cur)
|
|
return rc
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|