feat: add node agent system with heartbeat collection, installer generation, and traffic flow telemetry
CI / backend (push) Failing after 3s
CI / frontend (push) Failing after 28s

Add NodeAgent and TrafficFlow models to track agent status and network flows, implement /agents/heartbeat endpoint to receive interface counters, conntrack flows, firewall status, and nftables ruleset hash from agents, add nexafabric-agent.py Python script to collect host telemetry including VM/LXC interface hints via tap/fwbr regex matching, conntrack flow parsing with protocol/state/byte counters, and pve-firewall status checks,
This commit is contained in:
2026-07-09 14:13:47 +02:00
parent 88badf1f22
commit e67174a4ae
8 changed files with 677 additions and 7 deletions
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import os
import platform
import re
import socket
import ssl
import subprocess
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
VERSION = "0.1.0"
VM_INTERFACE_RE = re.compile(r"(?:tap|fwbr|fwln|fwpr)(\d+)")
def read_text(path: str) -> str | None:
try:
return Path(path).read_text(encoding="utf-8").strip()
except OSError:
return None
def run_command(args: list[str], timeout: int = 5) -> tuple[int, str]:
try:
result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False)
return result.returncode, result.stdout.strip()
except (OSError, subprocess.SubprocessError):
return 127, ""
def collect_interfaces() -> list[dict[str, Any]]:
interfaces = []
for item in Path("/sys/class/net").iterdir() if Path("/sys/class/net").exists() else []:
name = item.name
if name == "lo":
continue
vm_match = VM_INTERFACE_RE.search(name)
interfaces.append(
{
"name": name,
"vmid": vm_match.group(1) if vm_match else None,
"operstate": read_text(f"/sys/class/net/{name}/operstate") or "unknown",
"rx_bytes": int(read_text(f"/sys/class/net/{name}/statistics/rx_bytes") or 0),
"tx_bytes": int(read_text(f"/sys/class/net/{name}/statistics/tx_bytes") or 0),
"rx_packets": int(read_text(f"/sys/class/net/{name}/statistics/rx_packets") or 0),
"tx_packets": int(read_text(f"/sys/class/net/{name}/statistics/tx_packets") or 0),
}
)
return interfaces
def parse_conntrack_line(line: str) -> dict[str, Any] | None:
parts = line.split()
if len(parts) < 5 or parts[0] not in {"tcp", "udp", "icmp"}:
return None
protocol = parts[0]
state = None
if protocol == "tcp" and len(parts) > 3 and "=" not in parts[3]:
state = parts[3]
values: dict[str, list[str]] = {}
for part in parts:
if "=" not in part:
continue
key, value = part.split("=", 1)
values.setdefault(key, []).append(value)
src_values = values.get("src", [])
dst_values = values.get("dst", [])
if not src_values or not dst_values:
return None
packet_values = [int(value) for value in values.get("packets", []) if value.isdigit()]
byte_values = [int(value) for value in values.get("bytes", []) if value.isdigit()]
sport = values.get("sport", [None])[0]
dport = values.get("dport", [None])[0]
return {
"source_ip": src_values[0],
"destination_ip": dst_values[0],
"protocol": protocol,
"source_port": int(sport) if sport and sport.isdigit() else None,
"destination_port": int(dport) if dport and dport.isdigit() else None,
"packets": sum(packet_values),
"bytes": sum(byte_values),
"state": state,
}
def collect_flows(limit: int = 500) -> list[dict[str, Any]]:
code, output = run_command(["conntrack", "-L", "-o", "extended"], timeout=10)
if code != 0 or not output:
return []
flows = []
seen = set()
for line in output.splitlines():
flow = parse_conntrack_line(line)
if not flow:
continue
key = (
flow["source_ip"],
flow["destination_ip"],
flow["protocol"],
flow.get("source_port"),
flow.get("destination_port"),
)
if key in seen:
continue
seen.add(key)
flows.append(flow)
if len(flows) >= limit:
break
return flows
def collect_conntrack() -> dict[str, Any]:
code, output = run_command(["conntrack", "-C"])
if code == 0 and output.isdigit():
return {"count": int(output), "source": "conntrack"}
for path in ("/proc/net/nf_conntrack", "/proc/net/ip_conntrack"):
try:
with open(path, "r", encoding="utf-8", errors="ignore") as handle:
return {"count": sum(1 for _ in handle), "source": path}
except OSError:
continue
return {"count": None, "source": "unavailable"}
def collect_firewall() -> dict[str, Any]:
status = {}
code, output = run_command(["systemctl", "is-active", "pve-firewall"])
status["pve_firewall"] = output if code == 0 else "unknown"
code, output = run_command(["nft", "-j", "list", "ruleset"], timeout=10)
if code == 0 and output:
status["nft_ruleset_sha256"] = hashlib.sha256(output.encode("utf-8")).hexdigest()
else:
status["nft_ruleset_sha256"] = None
return status
def collect_payload(config: dict[str, Any]) -> dict[str, Any]:
uptime = read_text("/proc/uptime")
return {
"version": VERSION,
"node_name": config.get("node_name"),
"collected_at": datetime.now(timezone.utc).isoformat(),
"hostname": socket.gethostname(),
"kernel": platform.release(),
"uptime_seconds": float(uptime.split()[0]) if uptime else None,
"loadavg": list(os.getloadavg()) if hasattr(os, "getloadavg") else [],
"interfaces": collect_interfaces(),
"flows": collect_flows(int(config.get("flow_limit", 500))),
"conntrack": collect_conntrack(),
"firewall": collect_firewall(),
"extra": {"platform": platform.platform()},
}
def post_heartbeat(config: dict[str, Any], payload: dict[str, Any]) -> None:
api_url = str(config["api_url"]).rstrip("/")
data = json.dumps(payload).encode("utf-8")
request = urllib.request.Request(
f"{api_url}/agents/heartbeat",
data=data,
headers={
"Authorization": f"Bearer {config['token']}",
"Content-Type": "application/json",
"User-Agent": f"nexafabric-agent/{VERSION}",
},
method="POST",
)
context = None
if not bool(config.get("verify_tls", True)):
context = ssl._create_unverified_context()
with urllib.request.urlopen(request, timeout=15, context=context) as response:
response.read()
def load_config(path: str) -> dict[str, Any]:
return json.loads(Path(path).read_text(encoding="utf-8"))
def main() -> int:
parser = argparse.ArgumentParser(description="NexaFabric Proxmox node telemetry agent")
parser.add_argument("--config", default="/etc/nexafabric-agent/config.json")
parser.add_argument("--once", action="store_true")
args = parser.parse_args()
config = load_config(args.config)
interval = int(config.get("interval_seconds", 30))
while True:
payload = collect_payload(config)
try:
post_heartbeat(config, payload)
print(f"heartbeat ok: {payload['collected_at']}", flush=True)
except (OSError, urllib.error.URLError, urllib.error.HTTPError) as exc:
print(f"heartbeat failed: {exc}", flush=True)
if args.once:
return 0
time.sleep(interval)
if __name__ == "__main__":
raise SystemExit(main())
+255 -3
View File
@@ -1,10 +1,11 @@
from datetime import datetime
from datetime import datetime, timedelta
import csv
import io
from ipaddress import ip_address, ip_interface, ip_network
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import StreamingResponse
from fastapi import APIRouter, Depends, Header, HTTPException, Request
from fastapi.responses import FileResponse, PlainTextResponse, StreamingResponse
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -12,6 +13,7 @@ from sqlalchemy.orm import Session
from app.api.deps import CurrentUser
from app.api.v1 import auth
from app.core.security import hash_password
from app.core.security import create_token, decode_token
from app.db.session import get_db
from app.models.domain import (
AuditLog,
@@ -20,6 +22,7 @@ from app.models.domain import (
Job,
Network,
Node,
NodeAgent,
Policy,
Project,
Role,
@@ -29,11 +32,13 @@ from app.models.domain import (
SystemSetting,
Subnet,
Tenant,
TrafficFlow,
User,
Workload,
)
from app.schemas.domain import (
AuditLogRead,
AgentHeartbeat,
ClusterCreate,
ClusterRead,
ClusterUpdate,
@@ -45,6 +50,7 @@ from app.schemas.domain import (
NetworkCreate,
NetworkRead,
NodeRead,
NodeWithAgentRead,
PolicyCreate,
PolicyRead,
ProjectCreate,
@@ -233,6 +239,13 @@ def endpoint_values(db: Session, cluster: Cluster, ref: str) -> tuple[list[str |
return [], [f"Endpoint {ref} is not yet resolvable to a Proxmox firewall matcher."]
def flow_int(value: object, default: int = 0) -> int:
try:
return int(value) if value not in (None, "") else default
except (TypeError, ValueError):
return default
def proxmox_action(action: str) -> str:
return {"allow": "ACCEPT", "deny": "DROP", "reject": "REJECT"}.get(action, "ACCEPT")
@@ -489,6 +502,12 @@ def delete_cluster(cluster_id: str, user: CurrentUser, db: Session = Depends(get
db.delete(subnet)
for network in db.scalars(select(Network).where(Network.id.in_(network_ids))).all():
db.delete(network)
node_ids = [row[0] for row in db.execute(select(Node.id).where(Node.cluster_id == cluster.id)).all()]
if node_ids:
for agent in db.scalars(select(NodeAgent).where(NodeAgent.node_id.in_(node_ids))).all():
db.delete(agent)
for flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id.in_(node_ids))).all():
db.delete(flow)
for workload in db.scalars(select(Workload).where(Workload.cluster_id == cluster.id)).all():
db.delete(workload)
for node in db.scalars(select(Node).where(Node.cluster_id == cluster.id)).all():
@@ -618,6 +637,207 @@ def nodes(_: CurrentUser, db: Session = Depends(get_db)) -> list[Node]:
return db.scalars(select(Node).order_by(Node.name)).all()
def node_agent_payload(node: Node, agent: NodeAgent | None) -> dict:
return {
"id": node.id,
"cluster_id": node.cluster_id,
"name": node.name,
"status": node.status,
"cpu_count": node.cpu_count,
"memory_mb": node.memory_mb,
"agent": {
"node_id": agent.node_id,
"status": agent.status,
"version": agent.version,
"last_seen_at": agent.last_seen_at,
"install_count": agent.install_count,
"last_payload": agent.last_payload,
}
if agent
else None,
}
@api_router.get("/nodes/agents", response_model=list[NodeWithAgentRead])
def nodes_with_agents(_: CurrentUser, db: Session = Depends(get_db)) -> list[dict]:
agents = {agent.node_id: agent for agent in db.scalars(select(NodeAgent)).all()}
return [
node_agent_payload(node, agents.get(node.id))
for node in db.scalars(select(Node).order_by(Node.name)).all()
]
def agent_install_script(base_url: str, token: str, node: Node) -> str:
return f"""#!/bin/sh
set -eu
NEXAFABRIC_URL="${{NEXAFABRIC_URL:-{base_url.rstrip("/")}}}"
INSTALL_DIR="${{INSTALL_DIR:-/opt/nexafabric-agent}}"
CONFIG_DIR="${{CONFIG_DIR:-/etc/nexafabric-agent}}"
SERVICE_NAME="${{SERVICE_NAME:-nexafabric-agent}}"
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 is required. Install python3 on this Proxmox node and run this script again." >&2
exit 1
fi
if ! command -v conntrack >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
apt-get update
apt-get install -y conntrack
fi
mkdir -p "$INSTALL_DIR" "$CONFIG_DIR"
curl -fsSL "$NEXAFABRIC_URL/api/v1/agents/download/nexafabric-agent.py" -o "$INSTALL_DIR/nexafabric-agent.py"
chmod 0755 "$INSTALL_DIR/nexafabric-agent.py"
cat > "$CONFIG_DIR/config.json" <<'JSON'
{{
"api_url": "{base_url.rstrip("/")}/api/v1",
"token": "{token}",
"node_id": "{node.id}",
"node_name": "{node.name}",
"interval_seconds": 30,
"flow_limit": 500,
"verify_tls": true
}}
JSON
chmod 0600 "$CONFIG_DIR/config.json"
cat > "/etc/systemd/system/$SERVICE_NAME.service" <<EOF
[Unit]
Description=NexaFabric Node Agent
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=/usr/bin/python3 $INSTALL_DIR/nexafabric-agent.py --config $CONFIG_DIR/config.json
Restart=always
RestartSec=10
User=root
[Install]
WantedBy=multi-user.target
EOF
if command -v systemctl >/dev/null 2>&1; then
systemctl daemon-reload
systemctl enable --now "$SERVICE_NAME.service"
systemctl status "$SERVICE_NAME.service" --no-pager || true
else
echo "systemctl not found. Run manually: python3 $INSTALL_DIR/nexafabric-agent.py --config $CONFIG_DIR/config.json"
fi
"""
@api_router.get("/nodes/{node_id}/agent/install", response_class=PlainTextResponse)
def node_agent_install(node_id: str, request: Request, user: CurrentUser, db: Session = Depends(get_db)) -> str:
node = db.get(Node, node_id)
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent = db.get(NodeAgent, node.id)
if not agent:
agent = NodeAgent(node_id=node.id, status="pending_install", install_count=0)
db.add(agent)
agent.install_count = (agent.install_count or 0) + 1
token = create_token(node.id, "agent", timedelta(days=365), {"node_name": node.name})
write_audit(db, action="agent.install.generated", object_type="node", object_id=node.id, user_id=user.id)
base_url = str(request.base_url).rstrip("/")
return agent_install_script(base_url, token, node)
@api_router.get("/nodes/{node_id}/agent/install-info")
def node_agent_install_info(node_id: str, request: Request, user: CurrentUser, db: Session = Depends(get_db)) -> dict[str, str]:
node = db.get(Node, node_id)
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent = db.get(NodeAgent, node.id)
if not agent:
agent = NodeAgent(node_id=node.id, status="pending_install", install_count=0)
db.add(agent)
agent.install_count = (agent.install_count or 0) + 1
install_token = create_token(node.id, "agent-install", timedelta(days=7), {"node_name": node.name})
write_audit(db, action="agent.install.generated", object_type="node", object_id=node.id, user_id=user.id)
base_url = str(request.base_url).rstrip("/")
install_url = f"{base_url}/api/v1/agents/install/{node.id}?token={install_token}"
return {
"node_id": node.id,
"install_url": install_url,
"command": f"curl -fsSL '{install_url}' | sh",
}
@api_router.get("/agents/install/{node_id}", response_class=PlainTextResponse)
def public_node_agent_install(node_id: str, token: str, request: Request, db: Session = Depends(get_db)) -> str:
try:
claims = decode_token(token)
except Exception as exc:
raise HTTPException(status_code=401, detail="Invalid install token") from exc
if claims.get("typ") != "agent-install" or claims.get("sub") != node_id:
raise HTTPException(status_code=403, detail="Install token does not match this node")
node = db.get(Node, node_id)
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent_token = create_token(node.id, "agent", timedelta(days=365), {"node_name": node.name})
base_url = str(request.base_url).rstrip("/")
return agent_install_script(base_url, agent_token, node)
@api_router.get("/agents/download/nexafabric-agent.py")
def download_node_agent() -> FileResponse:
path = Path(__file__).resolve().parents[2] / "agent_assets" / "nexafabric-agent.py"
return FileResponse(path, media_type="text/x-python", filename="nexafabric-agent.py")
@api_router.post("/agents/heartbeat")
def agent_heartbeat(payload: AgentHeartbeat, authorization: str | None = Header(default=None), db: Session = Depends(get_db)) -> dict:
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="Missing agent token")
token = authorization.split(" ", 1)[1]
try:
claims = decode_token(token)
except Exception as exc:
raise HTTPException(status_code=401, detail="Invalid agent token") from exc
if claims.get("typ") != "agent":
raise HTTPException(status_code=403, detail="Token is not an agent token")
node = db.get(Node, claims.get("sub"))
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent = db.get(NodeAgent, node.id)
if not agent:
agent = NodeAgent(node_id=node.id, install_count=0)
db.add(agent)
agent.status = "online"
agent.version = payload.version
agent.last_seen_at = datetime.utcnow()
agent.last_payload = payload.model_dump()
for old_flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id == node.id)).all():
db.delete(old_flow)
for raw_flow in payload.flows[:1000]:
source_ip = str(raw_flow.get("source_ip") or "")
destination_ip = str(raw_flow.get("destination_ip") or "")
if not source_ip or not destination_ip:
continue
observed_at = payload.collected_at or datetime.utcnow()
db.add(
TrafficFlow(
node_id=node.id,
source_ip=source_ip,
destination_ip=destination_ip,
protocol=str(raw_flow.get("protocol") or "unknown"),
source_port=flow_int(raw_flow.get("source_port"), 0) or None,
destination_port=flow_int(raw_flow.get("destination_port"), 0) or None,
bytes=flow_int(raw_flow.get("bytes")),
packets=flow_int(raw_flow.get("packets")),
state=str(raw_flow.get("state") or "") or None,
observed_at=observed_at.replace(tzinfo=None) if observed_at.tzinfo else observed_at,
raw=raw_flow,
)
)
commit_or_400(db)
return {"status": "ok", "node_id": node.id}
@api_router.get("/vms", response_model=list[WorkloadRead])
def workloads(_: CurrentUser, db: Session = Depends(get_db)) -> list[Workload]:
return db.scalars(select(Workload).order_by(Workload.name)).all()
@@ -632,7 +852,39 @@ def workload_insights(workload_id: str, _: CurrentUser, db: Session = Depends(ge
select(Policy).where((Policy.project_id == workload.project_id) | (Policy.project_id.is_(None))).order_by(Policy.name)
).all()
assigned_ips = db.scalars(select(IpAddress).where(IpAddress.workload_id == workload.id).order_by(IpAddress.address)).all()
workload_ips = [address.address for address in assigned_ips]
ip_owners = {
address.address: db.get(Workload, address.workload_id)
for address in db.scalars(select(IpAddress).where(IpAddress.workload_id.is_not(None))).all()
}
traffic = []
if workload_ips:
flows = db.scalars(
select(TrafficFlow)
.where((TrafficFlow.source_ip.in_(workload_ips)) | (TrafficFlow.destination_ip.in_(workload_ips)))
.order_by(TrafficFlow.updated_at.desc())
.limit(50)
).all()
for flow in flows:
source_owner = ip_owners.get(flow.source_ip)
destination_owner = ip_owners.get(flow.destination_ip)
traffic.append(
{
"source": source_owner.name if source_owner else "external",
"destination": destination_owner.name if destination_owner else "external",
"source_ip": flow.source_ip,
"destination_ip": flow.destination_ip,
"protocol": flow.protocol,
"port": flow.destination_port,
"source_port": flow.source_port,
"bytes": flow.bytes,
"packets": flow.packets,
"state": flow.state,
"decision": "observed",
"observed_at": flow.observed_at.isoformat() if flow.observed_at else None,
"ip_addresses": [flow.source_ip, flow.destination_ip],
}
)
audit_mode_notes = [
f"{policy.name} is in audit mode; matching traffic is logged without enforcement."
for policy in policies
+30 -1
View File
@@ -2,7 +2,7 @@ from datetime import datetime
from enum import StrEnum
from uuid import uuid4
from sqlalchemy import JSON, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy import JSON, BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db.session import Base
@@ -129,6 +129,18 @@ class Node(Base, TimestampMixin):
cluster: Mapped[Cluster] = relationship()
class NodeAgent(Base, TimestampMixin):
__tablename__ = "node_agents"
node_id: Mapped[str] = mapped_column(ForeignKey("nodes.id"), primary_key=True)
status: Mapped[str] = mapped_column(String(100), default="not_installed")
version: Mapped[str | None] = mapped_column(String(50))
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime)
last_payload: Mapped[dict | None] = mapped_column(JSON)
install_count: Mapped[int] = mapped_column(Integer, default=0)
node: Mapped[Node] = relationship()
class Workload(Base, TimestampMixin):
__tablename__ = "workloads"
@@ -183,6 +195,23 @@ class IpAddress(Base, TimestampMixin):
note: Mapped[str | None] = mapped_column(Text)
class TrafficFlow(Base, TimestampMixin):
__tablename__ = "traffic_flows"
id: Mapped[str] = mapped_column(String, primary_key=True, default=new_id)
node_id: Mapped[str] = mapped_column(ForeignKey("nodes.id"), index=True)
source_ip: Mapped[str] = mapped_column(String(100), index=True)
destination_ip: Mapped[str] = mapped_column(String(100), index=True)
protocol: Mapped[str] = mapped_column(String(20), default="unknown")
source_port: Mapped[int | None] = mapped_column(Integer)
destination_port: Mapped[int | None] = mapped_column(Integer)
bytes: Mapped[int] = mapped_column(BigInteger, default=0)
packets: Mapped[int] = mapped_column(BigInteger, default=0)
state: Mapped[str | None] = mapped_column(String(100))
observed_at: Mapped[datetime | None] = mapped_column(DateTime)
raw: Mapped[dict | None] = mapped_column(JSON)
class SecurityGroup(Base, TimestampMixin):
__tablename__ = "security_groups"
+28
View File
@@ -181,6 +181,34 @@ class NodeRead(OrmModel):
memory_mb: int
class NodeAgentRead(BaseModel):
node_id: str
status: str
version: str | None = None
last_seen_at: datetime | None = None
install_count: int = 0
last_payload: dict[str, Any] | None = None
class NodeWithAgentRead(NodeRead):
agent: NodeAgentRead | None = None
class AgentHeartbeat(BaseModel):
version: str
node_name: str | None = None
collected_at: datetime | None = None
hostname: str | None = None
kernel: str | None = None
uptime_seconds: float | None = None
loadavg: list[float] = []
interfaces: list[dict[str, Any]] = []
flows: list[dict[str, Any]] = []
conntrack: dict[str, Any] = {}
firewall: dict[str, Any] = {}
extra: dict[str, Any] = {}
class WorkloadRead(OrmModel):
id: str
cluster_id: str