nessi/NexaFabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add node agent system with heartbeat collection, installer generation, and traffic flow telemetry
CI / backend (push) Failing after 3s
Details
CI / frontend (push) Failing after 28s
Details

Browse Source 
Add NodeAgent and TrafficFlow models to track agent status and network flows, implement /agents/heartbeat endpoint to receive interface counters, conntrack flows, firewall status, and nftables ruleset hash from agents, add nexafabric-agent.py Python script to collect host telemetry including VM/LXC interface hints via tap/fwbr regex matching, conntrack flow parsing with protocol/state/byte counters, and pve-firewall status checks,
This commit is contained in:
nessi
2026-07-09 14:13:47 +02:00
parent 88badf1f22
commit e67174a4ae
8 changed files with 677 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/v1/router.py
+255 -3
View File
@@ -1,10 +1,11 @@
from datetime import datetime
from datetime import datetime, timedelta
import csv
import io
from ipaddress import ip_address, ip_interface, ip_network
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import StreamingResponse
from fastapi import APIRouter, Depends, Header, HTTPException, Request
from fastapi.responses import FileResponse, PlainTextResponse, StreamingResponse
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@@ -12,6 +13,7 @@ from sqlalchemy.orm import Session
from app.api.deps import CurrentUser
from app.api.v1 import auth
from app.core.security import hash_password
from app.core.security import create_token, decode_token
from app.db.session import get_db
from app.models.domain import (
AuditLog,
@@ -20,6 +22,7 @@ from app.models.domain import (
Job,
Network,
Node,
NodeAgent,
Policy,
Project,
Role,
@@ -29,11 +32,13 @@ from app.models.domain import (
SystemSetting,
Subnet,
Tenant,
TrafficFlow,
User,
Workload,
)
from app.schemas.domain import (
AuditLogRead,
AgentHeartbeat,
ClusterCreate,
ClusterRead,
ClusterUpdate,
@@ -45,6 +50,7 @@ from app.schemas.domain import (
NetworkCreate,
NetworkRead,
NodeRead,
NodeWithAgentRead,
PolicyCreate,
PolicyRead,
ProjectCreate,
@@ -233,6 +239,13 @@ def endpoint_values(db: Session, cluster: Cluster, ref: str) -> tuple[list[str |
return [], [f"Endpoint {ref} is not yet resolvable to a Proxmox firewall matcher."]
def flow_int(value: object, default: int = 0) -> int:
try:
return int(value) if value not in (None, "") else default
except (TypeError, ValueError):
return default
def proxmox_action(action: str) -> str:
return {"allow": "ACCEPT", "deny": "DROP", "reject": "REJECT"}.get(action, "ACCEPT")
@@ -489,6 +502,12 @@ def delete_cluster(cluster_id: str, user: CurrentUser, db: Session = Depends(get
db.delete(subnet)
for network in db.scalars(select(Network).where(Network.id.in_(network_ids))).all():
db.delete(network)
node_ids = [row[0] for row in db.execute(select(Node.id).where(Node.cluster_id == cluster.id)).all()]
if node_ids:
for agent in db.scalars(select(NodeAgent).where(NodeAgent.node_id.in_(node_ids))).all():
db.delete(agent)
for flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id.in_(node_ids))).all():
db.delete(flow)
for workload in db.scalars(select(Workload).where(Workload.cluster_id == cluster.id)).all():
db.delete(workload)
for node in db.scalars(select(Node).where(Node.cluster_id == cluster.id)).all():
@@ -618,6 +637,207 @@ def nodes(_: CurrentUser, db: Session = Depends(get_db)) -> list[Node]:
return db.scalars(select(Node).order_by(Node.name)).all()
def node_agent_payload(node: Node, agent: NodeAgent | None) -> dict:
return {
"id": node.id,
"cluster_id": node.cluster_id,
"name": node.name,
"status": node.status,
"cpu_count": node.cpu_count,
"memory_mb": node.memory_mb,
"agent": {
"node_id": agent.node_id,
"status": agent.status,
"version": agent.version,
"last_seen_at": agent.last_seen_at,
"install_count": agent.install_count,
"last_payload": agent.last_payload,
}
if agent
else None,
}
@api_router.get("/nodes/agents", response_model=list[NodeWithAgentRead])
def nodes_with_agents(_: CurrentUser, db: Session = Depends(get_db)) -> list[dict]:
agents = {agent.node_id: agent for agent in db.scalars(select(NodeAgent)).all()}
return [
node_agent_payload(node, agents.get(node.id))
for node in db.scalars(select(Node).order_by(Node.name)).all()
]
def agent_install_script(base_url: str, token: str, node: Node) -> str:
return f"""#!/bin/sh
set -eu
NEXAFABRIC_URL="${{NEXAFABRIC_URL:-{base_url.rstrip("/")}}}"
INSTALL_DIR="${{INSTALL_DIR:-/opt/nexafabric-agent}}"
CONFIG_DIR="${{CONFIG_DIR:-/etc/nexafabric-agent}}"
SERVICE_NAME="${{SERVICE_NAME:-nexafabric-agent}}"
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 is required. Install python3 on this Proxmox node and run this script again." >&2
exit 1
fi
if ! command -v conntrack >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
apt-get update
apt-get install -y conntrack
fi
mkdir -p "$INSTALL_DIR" "$CONFIG_DIR"
curl -fsSL "$NEXAFABRIC_URL/api/v1/agents/download/nexafabric-agent.py" -o "$INSTALL_DIR/nexafabric-agent.py"
chmod 0755 "$INSTALL_DIR/nexafabric-agent.py"
cat > "$CONFIG_DIR/config.json" <<'JSON'
{{
"api_url": "{base_url.rstrip("/")}/api/v1",
"token": "{token}",
"node_id": "{node.id}",
"node_name": "{node.name}",
"interval_seconds": 30,
"flow_limit": 500,
"verify_tls": true
}}
JSON
chmod 0600 "$CONFIG_DIR/config.json"
cat > "/etc/systemd/system/$SERVICE_NAME.service" <<EOF
[Unit]
Description=NexaFabric Node Agent
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=/usr/bin/python3 $INSTALL_DIR/nexafabric-agent.py --config $CONFIG_DIR/config.json
Restart=always
RestartSec=10
User=root
[Install]
WantedBy=multi-user.target
EOF
if command -v systemctl >/dev/null 2>&1; then
systemctl daemon-reload
systemctl enable --now "$SERVICE_NAME.service"
systemctl status "$SERVICE_NAME.service" --no-pager || true
else
echo "systemctl not found. Run manually: python3 $INSTALL_DIR/nexafabric-agent.py --config $CONFIG_DIR/config.json"
fi
"""
@api_router.get("/nodes/{node_id}/agent/install", response_class=PlainTextResponse)
def node_agent_install(node_id: str, request: Request, user: CurrentUser, db: Session = Depends(get_db)) -> str:
node = db.get(Node, node_id)
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent = db.get(NodeAgent, node.id)
if not agent:
agent = NodeAgent(node_id=node.id, status="pending_install", install_count=0)
db.add(agent)
agent.install_count = (agent.install_count or 0) + 1
token = create_token(node.id, "agent", timedelta(days=365), {"node_name": node.name})
write_audit(db, action="agent.install.generated", object_type="node", object_id=node.id, user_id=user.id)
base_url = str(request.base_url).rstrip("/")
return agent_install_script(base_url, token, node)
@api_router.get("/nodes/{node_id}/agent/install-info")
def node_agent_install_info(node_id: str, request: Request, user: CurrentUser, db: Session = Depends(get_db)) -> dict[str, str]:
node = db.get(Node, node_id)
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent = db.get(NodeAgent, node.id)
if not agent:
agent = NodeAgent(node_id=node.id, status="pending_install", install_count=0)
db.add(agent)
agent.install_count = (agent.install_count or 0) + 1
install_token = create_token(node.id, "agent-install", timedelta(days=7), {"node_name": node.name})
write_audit(db, action="agent.install.generated", object_type="node", object_id=node.id, user_id=user.id)
base_url = str(request.base_url).rstrip("/")
install_url = f"{base_url}/api/v1/agents/install/{node.id}?token={install_token}"
return {
"node_id": node.id,
"install_url": install_url,
"command": f"curl -fsSL '{install_url}' | sh",
}
@api_router.get("/agents/install/{node_id}", response_class=PlainTextResponse)
def public_node_agent_install(node_id: str, token: str, request: Request, db: Session = Depends(get_db)) -> str:
try:
claims = decode_token(token)
except Exception as exc:
raise HTTPException(status_code=401, detail="Invalid install token") from exc
if claims.get("typ") != "agent-install" or claims.get("sub") != node_id:
raise HTTPException(status_code=403, detail="Install token does not match this node")
node = db.get(Node, node_id)
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent_token = create_token(node.id, "agent", timedelta(days=365), {"node_name": node.name})
base_url = str(request.base_url).rstrip("/")
return agent_install_script(base_url, agent_token, node)
@api_router.get("/agents/download/nexafabric-agent.py")
def download_node_agent() -> FileResponse:
path = Path(__file__).resolve().parents[2] / "agent_assets" / "nexafabric-agent.py"
return FileResponse(path, media_type="text/x-python", filename="nexafabric-agent.py")
@api_router.post("/agents/heartbeat")
def agent_heartbeat(payload: AgentHeartbeat, authorization: str | None = Header(default=None), db: Session = Depends(get_db)) -> dict:
if not authorization or not authorization.lower().startswith("bearer "):
raise HTTPException(status_code=401, detail="Missing agent token")
token = authorization.split(" ", 1)[1]
try:
claims = decode_token(token)
except Exception as exc:
raise HTTPException(status_code=401, detail="Invalid agent token") from exc
if claims.get("typ") != "agent":
raise HTTPException(status_code=403, detail="Token is not an agent token")
node = db.get(Node, claims.get("sub"))
if not node:
raise HTTPException(status_code=404, detail="Node not found")
agent = db.get(NodeAgent, node.id)
if not agent:
agent = NodeAgent(node_id=node.id, install_count=0)
db.add(agent)
agent.status = "online"
agent.version = payload.version
agent.last_seen_at = datetime.utcnow()
agent.last_payload = payload.model_dump()
for old_flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id == node.id)).all():
db.delete(old_flow)
for raw_flow in payload.flows[:1000]:
source_ip = str(raw_flow.get("source_ip") or "")
destination_ip = str(raw_flow.get("destination_ip") or "")
if not source_ip or not destination_ip:
continue
observed_at = payload.collected_at or datetime.utcnow()
db.add(
TrafficFlow(
node_id=node.id,
source_ip=source_ip,
destination_ip=destination_ip,
protocol=str(raw_flow.get("protocol") or "unknown"),
source_port=flow_int(raw_flow.get("source_port"), 0) or None,
destination_port=flow_int(raw_flow.get("destination_port"), 0) or None,
bytes=flow_int(raw_flow.get("bytes")),
packets=flow_int(raw_flow.get("packets")),
state=str(raw_flow.get("state") or "") or None,
observed_at=observed_at.replace(tzinfo=None) if observed_at.tzinfo else observed_at,
raw=raw_flow,
)
)
commit_or_400(db)
return {"status": "ok", "node_id": node.id}
@api_router.get("/vms", response_model=list[WorkloadRead])
def workloads(_: CurrentUser, db: Session = Depends(get_db)) -> list[Workload]:
return db.scalars(select(Workload).order_by(Workload.name)).all()
@@ -632,7 +852,39 @@ def workload_insights(workload_id: str, _: CurrentUser, db: Session = Depends(ge
select(Policy).where((Policy.project_id == workload.project_id) | (Policy.project_id.is_(None))).order_by(Policy.name)
).all()
assigned_ips = db.scalars(select(IpAddress).where(IpAddress.workload_id == workload.id).order_by(IpAddress.address)).all()
workload_ips = [address.address for address in assigned_ips]
ip_owners = {
address.address: db.get(Workload, address.workload_id)
for address in db.scalars(select(IpAddress).where(IpAddress.workload_id.is_not(None))).all()
}
traffic = []
if workload_ips:
flows = db.scalars(
select(TrafficFlow)
.where((TrafficFlow.source_ip.in_(workload_ips)) | (TrafficFlow.destination_ip.in_(workload_ips)))
.order_by(TrafficFlow.updated_at.desc())
.limit(50)
).all()
for flow in flows:
source_owner = ip_owners.get(flow.source_ip)
destination_owner = ip_owners.get(flow.destination_ip)
traffic.append(
{
"source": source_owner.name if source_owner else "external",
"destination": destination_owner.name if destination_owner else "external",
"source_ip": flow.source_ip,
"destination_ip": flow.destination_ip,
"protocol": flow.protocol,
"port": flow.destination_port,
"source_port": flow.source_port,
"bytes": flow.bytes,
"packets": flow.packets,
"state": flow.state,
"decision": "observed",
"observed_at": flow.observed_at.isoformat() if flow.observed_at else None,
"ip_addresses": [flow.source_ip, flow.destination_ip],
}
)
audit_mode_notes = [
f"{policy.name} is in audit mode; matching traffic is logged without enforcement."
for policy in policies