feat: add node agent system with heartbeat collection, installer generation, and traffic flow telemetry
Add NodeAgent and TrafficFlow models to track agent status and network flows, implement /agents/heartbeat endpoint to receive interface counters, conntrack flows, firewall status, and nftables ruleset hash from agents, add nexafabric-agent.py Python script to collect host telemetry including VM/LXC interface hints via tap/fwbr regex matching, conntrack flow parsing with protocol/state/byte counters, and pve-firewall status checks,
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
VERSION = "0.1.0"
|
||||
VM_INTERFACE_RE = re.compile(r"(?:tap|fwbr|fwln|fwpr)(\d+)")
|
||||
|
||||
|
||||
def read_text(path: str) -> str | None:
|
||||
try:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def run_command(args: list[str], timeout: int = 5) -> tuple[int, str]:
|
||||
try:
|
||||
result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, check=False)
|
||||
return result.returncode, result.stdout.strip()
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return 127, ""
|
||||
|
||||
|
||||
def collect_interfaces() -> list[dict[str, Any]]:
|
||||
interfaces = []
|
||||
for item in Path("/sys/class/net").iterdir() if Path("/sys/class/net").exists() else []:
|
||||
name = item.name
|
||||
if name == "lo":
|
||||
continue
|
||||
vm_match = VM_INTERFACE_RE.search(name)
|
||||
interfaces.append(
|
||||
{
|
||||
"name": name,
|
||||
"vmid": vm_match.group(1) if vm_match else None,
|
||||
"operstate": read_text(f"/sys/class/net/{name}/operstate") or "unknown",
|
||||
"rx_bytes": int(read_text(f"/sys/class/net/{name}/statistics/rx_bytes") or 0),
|
||||
"tx_bytes": int(read_text(f"/sys/class/net/{name}/statistics/tx_bytes") or 0),
|
||||
"rx_packets": int(read_text(f"/sys/class/net/{name}/statistics/rx_packets") or 0),
|
||||
"tx_packets": int(read_text(f"/sys/class/net/{name}/statistics/tx_packets") or 0),
|
||||
}
|
||||
)
|
||||
return interfaces
|
||||
|
||||
|
||||
def parse_conntrack_line(line: str) -> dict[str, Any] | None:
|
||||
parts = line.split()
|
||||
if len(parts) < 5 or parts[0] not in {"tcp", "udp", "icmp"}:
|
||||
return None
|
||||
protocol = parts[0]
|
||||
state = None
|
||||
if protocol == "tcp" and len(parts) > 3 and "=" not in parts[3]:
|
||||
state = parts[3]
|
||||
|
||||
values: dict[str, list[str]] = {}
|
||||
for part in parts:
|
||||
if "=" not in part:
|
||||
continue
|
||||
key, value = part.split("=", 1)
|
||||
values.setdefault(key, []).append(value)
|
||||
|
||||
src_values = values.get("src", [])
|
||||
dst_values = values.get("dst", [])
|
||||
if not src_values or not dst_values:
|
||||
return None
|
||||
|
||||
packet_values = [int(value) for value in values.get("packets", []) if value.isdigit()]
|
||||
byte_values = [int(value) for value in values.get("bytes", []) if value.isdigit()]
|
||||
sport = values.get("sport", [None])[0]
|
||||
dport = values.get("dport", [None])[0]
|
||||
return {
|
||||
"source_ip": src_values[0],
|
||||
"destination_ip": dst_values[0],
|
||||
"protocol": protocol,
|
||||
"source_port": int(sport) if sport and sport.isdigit() else None,
|
||||
"destination_port": int(dport) if dport and dport.isdigit() else None,
|
||||
"packets": sum(packet_values),
|
||||
"bytes": sum(byte_values),
|
||||
"state": state,
|
||||
}
|
||||
|
||||
|
||||
def collect_flows(limit: int = 500) -> list[dict[str, Any]]:
|
||||
code, output = run_command(["conntrack", "-L", "-o", "extended"], timeout=10)
|
||||
if code != 0 or not output:
|
||||
return []
|
||||
flows = []
|
||||
seen = set()
|
||||
for line in output.splitlines():
|
||||
flow = parse_conntrack_line(line)
|
||||
if not flow:
|
||||
continue
|
||||
key = (
|
||||
flow["source_ip"],
|
||||
flow["destination_ip"],
|
||||
flow["protocol"],
|
||||
flow.get("source_port"),
|
||||
flow.get("destination_port"),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
flows.append(flow)
|
||||
if len(flows) >= limit:
|
||||
break
|
||||
return flows
|
||||
|
||||
|
||||
def collect_conntrack() -> dict[str, Any]:
|
||||
code, output = run_command(["conntrack", "-C"])
|
||||
if code == 0 and output.isdigit():
|
||||
return {"count": int(output), "source": "conntrack"}
|
||||
for path in ("/proc/net/nf_conntrack", "/proc/net/ip_conntrack"):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", errors="ignore") as handle:
|
||||
return {"count": sum(1 for _ in handle), "source": path}
|
||||
except OSError:
|
||||
continue
|
||||
return {"count": None, "source": "unavailable"}
|
||||
|
||||
|
||||
def collect_firewall() -> dict[str, Any]:
|
||||
status = {}
|
||||
code, output = run_command(["systemctl", "is-active", "pve-firewall"])
|
||||
status["pve_firewall"] = output if code == 0 else "unknown"
|
||||
code, output = run_command(["nft", "-j", "list", "ruleset"], timeout=10)
|
||||
if code == 0 and output:
|
||||
status["nft_ruleset_sha256"] = hashlib.sha256(output.encode("utf-8")).hexdigest()
|
||||
else:
|
||||
status["nft_ruleset_sha256"] = None
|
||||
return status
|
||||
|
||||
|
||||
def collect_payload(config: dict[str, Any]) -> dict[str, Any]:
|
||||
uptime = read_text("/proc/uptime")
|
||||
return {
|
||||
"version": VERSION,
|
||||
"node_name": config.get("node_name"),
|
||||
"collected_at": datetime.now(timezone.utc).isoformat(),
|
||||
"hostname": socket.gethostname(),
|
||||
"kernel": platform.release(),
|
||||
"uptime_seconds": float(uptime.split()[0]) if uptime else None,
|
||||
"loadavg": list(os.getloadavg()) if hasattr(os, "getloadavg") else [],
|
||||
"interfaces": collect_interfaces(),
|
||||
"flows": collect_flows(int(config.get("flow_limit", 500))),
|
||||
"conntrack": collect_conntrack(),
|
||||
"firewall": collect_firewall(),
|
||||
"extra": {"platform": platform.platform()},
|
||||
}
|
||||
|
||||
|
||||
def post_heartbeat(config: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
api_url = str(config["api_url"]).rstrip("/")
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
f"{api_url}/agents/heartbeat",
|
||||
data=data,
|
||||
headers={
|
||||
"Authorization": f"Bearer {config['token']}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": f"nexafabric-agent/{VERSION}",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
context = None
|
||||
if not bool(config.get("verify_tls", True)):
|
||||
context = ssl._create_unverified_context()
|
||||
with urllib.request.urlopen(request, timeout=15, context=context) as response:
|
||||
response.read()
|
||||
|
||||
|
||||
def load_config(path: str) -> dict[str, Any]:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="NexaFabric Proxmox node telemetry agent")
|
||||
parser.add_argument("--config", default="/etc/nexafabric-agent/config.json")
|
||||
parser.add_argument("--once", action="store_true")
|
||||
args = parser.parse_args()
|
||||
config = load_config(args.config)
|
||||
interval = int(config.get("interval_seconds", 30))
|
||||
|
||||
while True:
|
||||
payload = collect_payload(config)
|
||||
try:
|
||||
post_heartbeat(config, payload)
|
||||
print(f"heartbeat ok: {payload['collected_at']}", flush=True)
|
||||
except (OSError, urllib.error.URLError, urllib.error.HTTPError) as exc:
|
||||
print(f"heartbeat failed: {exc}", flush=True)
|
||||
if args.once:
|
||||
return 0
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,10 +1,11 @@
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
import csv
|
||||
import io
|
||||
from ipaddress import ip_address, ip_interface, ip_network
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request
|
||||
from fastapi.responses import FileResponse, PlainTextResponse, StreamingResponse
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -12,6 +13,7 @@ from sqlalchemy.orm import Session
|
||||
from app.api.deps import CurrentUser
|
||||
from app.api.v1 import auth
|
||||
from app.core.security import hash_password
|
||||
from app.core.security import create_token, decode_token
|
||||
from app.db.session import get_db
|
||||
from app.models.domain import (
|
||||
AuditLog,
|
||||
@@ -20,6 +22,7 @@ from app.models.domain import (
|
||||
Job,
|
||||
Network,
|
||||
Node,
|
||||
NodeAgent,
|
||||
Policy,
|
||||
Project,
|
||||
Role,
|
||||
@@ -29,11 +32,13 @@ from app.models.domain import (
|
||||
SystemSetting,
|
||||
Subnet,
|
||||
Tenant,
|
||||
TrafficFlow,
|
||||
User,
|
||||
Workload,
|
||||
)
|
||||
from app.schemas.domain import (
|
||||
AuditLogRead,
|
||||
AgentHeartbeat,
|
||||
ClusterCreate,
|
||||
ClusterRead,
|
||||
ClusterUpdate,
|
||||
@@ -45,6 +50,7 @@ from app.schemas.domain import (
|
||||
NetworkCreate,
|
||||
NetworkRead,
|
||||
NodeRead,
|
||||
NodeWithAgentRead,
|
||||
PolicyCreate,
|
||||
PolicyRead,
|
||||
ProjectCreate,
|
||||
@@ -233,6 +239,13 @@ def endpoint_values(db: Session, cluster: Cluster, ref: str) -> tuple[list[str |
|
||||
return [], [f"Endpoint {ref} is not yet resolvable to a Proxmox firewall matcher."]
|
||||
|
||||
|
||||
def flow_int(value: object, default: int = 0) -> int:
|
||||
try:
|
||||
return int(value) if value not in (None, "") else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def proxmox_action(action: str) -> str:
|
||||
return {"allow": "ACCEPT", "deny": "DROP", "reject": "REJECT"}.get(action, "ACCEPT")
|
||||
|
||||
@@ -489,6 +502,12 @@ def delete_cluster(cluster_id: str, user: CurrentUser, db: Session = Depends(get
|
||||
db.delete(subnet)
|
||||
for network in db.scalars(select(Network).where(Network.id.in_(network_ids))).all():
|
||||
db.delete(network)
|
||||
node_ids = [row[0] for row in db.execute(select(Node.id).where(Node.cluster_id == cluster.id)).all()]
|
||||
if node_ids:
|
||||
for agent in db.scalars(select(NodeAgent).where(NodeAgent.node_id.in_(node_ids))).all():
|
||||
db.delete(agent)
|
||||
for flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id.in_(node_ids))).all():
|
||||
db.delete(flow)
|
||||
for workload in db.scalars(select(Workload).where(Workload.cluster_id == cluster.id)).all():
|
||||
db.delete(workload)
|
||||
for node in db.scalars(select(Node).where(Node.cluster_id == cluster.id)).all():
|
||||
@@ -618,6 +637,207 @@ def nodes(_: CurrentUser, db: Session = Depends(get_db)) -> list[Node]:
|
||||
return db.scalars(select(Node).order_by(Node.name)).all()
|
||||
|
||||
|
||||
def node_agent_payload(node: Node, agent: NodeAgent | None) -> dict:
|
||||
return {
|
||||
"id": node.id,
|
||||
"cluster_id": node.cluster_id,
|
||||
"name": node.name,
|
||||
"status": node.status,
|
||||
"cpu_count": node.cpu_count,
|
||||
"memory_mb": node.memory_mb,
|
||||
"agent": {
|
||||
"node_id": agent.node_id,
|
||||
"status": agent.status,
|
||||
"version": agent.version,
|
||||
"last_seen_at": agent.last_seen_at,
|
||||
"install_count": agent.install_count,
|
||||
"last_payload": agent.last_payload,
|
||||
}
|
||||
if agent
|
||||
else None,
|
||||
}
|
||||
|
||||
|
||||
@api_router.get("/nodes/agents", response_model=list[NodeWithAgentRead])
|
||||
def nodes_with_agents(_: CurrentUser, db: Session = Depends(get_db)) -> list[dict]:
|
||||
agents = {agent.node_id: agent for agent in db.scalars(select(NodeAgent)).all()}
|
||||
return [
|
||||
node_agent_payload(node, agents.get(node.id))
|
||||
for node in db.scalars(select(Node).order_by(Node.name)).all()
|
||||
]
|
||||
|
||||
|
||||
def agent_install_script(base_url: str, token: str, node: Node) -> str:
|
||||
return f"""#!/bin/sh
|
||||
set -eu
|
||||
|
||||
NEXAFABRIC_URL="${{NEXAFABRIC_URL:-{base_url.rstrip("/")}}}"
|
||||
INSTALL_DIR="${{INSTALL_DIR:-/opt/nexafabric-agent}}"
|
||||
CONFIG_DIR="${{CONFIG_DIR:-/etc/nexafabric-agent}}"
|
||||
SERVICE_NAME="${{SERVICE_NAME:-nexafabric-agent}}"
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "python3 is required. Install python3 on this Proxmox node and run this script again." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v conntrack >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y conntrack
|
||||
fi
|
||||
|
||||
mkdir -p "$INSTALL_DIR" "$CONFIG_DIR"
|
||||
curl -fsSL "$NEXAFABRIC_URL/api/v1/agents/download/nexafabric-agent.py" -o "$INSTALL_DIR/nexafabric-agent.py"
|
||||
chmod 0755 "$INSTALL_DIR/nexafabric-agent.py"
|
||||
|
||||
cat > "$CONFIG_DIR/config.json" <<'JSON'
|
||||
{{
|
||||
"api_url": "{base_url.rstrip("/")}/api/v1",
|
||||
"token": "{token}",
|
||||
"node_id": "{node.id}",
|
||||
"node_name": "{node.name}",
|
||||
"interval_seconds": 30,
|
||||
"flow_limit": 500,
|
||||
"verify_tls": true
|
||||
}}
|
||||
JSON
|
||||
chmod 0600 "$CONFIG_DIR/config.json"
|
||||
|
||||
cat > "/etc/systemd/system/$SERVICE_NAME.service" <<EOF
|
||||
[Unit]
|
||||
Description=NexaFabric Node Agent
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/python3 $INSTALL_DIR/nexafabric-agent.py --config $CONFIG_DIR/config.json
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
User=root
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now "$SERVICE_NAME.service"
|
||||
systemctl status "$SERVICE_NAME.service" --no-pager || true
|
||||
else
|
||||
echo "systemctl not found. Run manually: python3 $INSTALL_DIR/nexafabric-agent.py --config $CONFIG_DIR/config.json"
|
||||
fi
|
||||
"""
|
||||
|
||||
|
||||
@api_router.get("/nodes/{node_id}/agent/install", response_class=PlainTextResponse)
|
||||
def node_agent_install(node_id: str, request: Request, user: CurrentUser, db: Session = Depends(get_db)) -> str:
|
||||
node = db.get(Node, node_id)
|
||||
if not node:
|
||||
raise HTTPException(status_code=404, detail="Node not found")
|
||||
agent = db.get(NodeAgent, node.id)
|
||||
if not agent:
|
||||
agent = NodeAgent(node_id=node.id, status="pending_install", install_count=0)
|
||||
db.add(agent)
|
||||
agent.install_count = (agent.install_count or 0) + 1
|
||||
token = create_token(node.id, "agent", timedelta(days=365), {"node_name": node.name})
|
||||
write_audit(db, action="agent.install.generated", object_type="node", object_id=node.id, user_id=user.id)
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
return agent_install_script(base_url, token, node)
|
||||
|
||||
|
||||
@api_router.get("/nodes/{node_id}/agent/install-info")
|
||||
def node_agent_install_info(node_id: str, request: Request, user: CurrentUser, db: Session = Depends(get_db)) -> dict[str, str]:
|
||||
node = db.get(Node, node_id)
|
||||
if not node:
|
||||
raise HTTPException(status_code=404, detail="Node not found")
|
||||
agent = db.get(NodeAgent, node.id)
|
||||
if not agent:
|
||||
agent = NodeAgent(node_id=node.id, status="pending_install", install_count=0)
|
||||
db.add(agent)
|
||||
agent.install_count = (agent.install_count or 0) + 1
|
||||
install_token = create_token(node.id, "agent-install", timedelta(days=7), {"node_name": node.name})
|
||||
write_audit(db, action="agent.install.generated", object_type="node", object_id=node.id, user_id=user.id)
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
install_url = f"{base_url}/api/v1/agents/install/{node.id}?token={install_token}"
|
||||
return {
|
||||
"node_id": node.id,
|
||||
"install_url": install_url,
|
||||
"command": f"curl -fsSL '{install_url}' | sh",
|
||||
}
|
||||
|
||||
|
||||
@api_router.get("/agents/install/{node_id}", response_class=PlainTextResponse)
|
||||
def public_node_agent_install(node_id: str, token: str, request: Request, db: Session = Depends(get_db)) -> str:
|
||||
try:
|
||||
claims = decode_token(token)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=401, detail="Invalid install token") from exc
|
||||
if claims.get("typ") != "agent-install" or claims.get("sub") != node_id:
|
||||
raise HTTPException(status_code=403, detail="Install token does not match this node")
|
||||
node = db.get(Node, node_id)
|
||||
if not node:
|
||||
raise HTTPException(status_code=404, detail="Node not found")
|
||||
agent_token = create_token(node.id, "agent", timedelta(days=365), {"node_name": node.name})
|
||||
base_url = str(request.base_url).rstrip("/")
|
||||
return agent_install_script(base_url, agent_token, node)
|
||||
|
||||
|
||||
@api_router.get("/agents/download/nexafabric-agent.py")
|
||||
def download_node_agent() -> FileResponse:
|
||||
path = Path(__file__).resolve().parents[2] / "agent_assets" / "nexafabric-agent.py"
|
||||
return FileResponse(path, media_type="text/x-python", filename="nexafabric-agent.py")
|
||||
|
||||
|
||||
@api_router.post("/agents/heartbeat")
|
||||
def agent_heartbeat(payload: AgentHeartbeat, authorization: str | None = Header(default=None), db: Session = Depends(get_db)) -> dict:
|
||||
if not authorization or not authorization.lower().startswith("bearer "):
|
||||
raise HTTPException(status_code=401, detail="Missing agent token")
|
||||
token = authorization.split(" ", 1)[1]
|
||||
try:
|
||||
claims = decode_token(token)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=401, detail="Invalid agent token") from exc
|
||||
if claims.get("typ") != "agent":
|
||||
raise HTTPException(status_code=403, detail="Token is not an agent token")
|
||||
node = db.get(Node, claims.get("sub"))
|
||||
if not node:
|
||||
raise HTTPException(status_code=404, detail="Node not found")
|
||||
agent = db.get(NodeAgent, node.id)
|
||||
if not agent:
|
||||
agent = NodeAgent(node_id=node.id, install_count=0)
|
||||
db.add(agent)
|
||||
agent.status = "online"
|
||||
agent.version = payload.version
|
||||
agent.last_seen_at = datetime.utcnow()
|
||||
agent.last_payload = payload.model_dump()
|
||||
for old_flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id == node.id)).all():
|
||||
db.delete(old_flow)
|
||||
for raw_flow in payload.flows[:1000]:
|
||||
source_ip = str(raw_flow.get("source_ip") or "")
|
||||
destination_ip = str(raw_flow.get("destination_ip") or "")
|
||||
if not source_ip or not destination_ip:
|
||||
continue
|
||||
observed_at = payload.collected_at or datetime.utcnow()
|
||||
db.add(
|
||||
TrafficFlow(
|
||||
node_id=node.id,
|
||||
source_ip=source_ip,
|
||||
destination_ip=destination_ip,
|
||||
protocol=str(raw_flow.get("protocol") or "unknown"),
|
||||
source_port=flow_int(raw_flow.get("source_port"), 0) or None,
|
||||
destination_port=flow_int(raw_flow.get("destination_port"), 0) or None,
|
||||
bytes=flow_int(raw_flow.get("bytes")),
|
||||
packets=flow_int(raw_flow.get("packets")),
|
||||
state=str(raw_flow.get("state") or "") or None,
|
||||
observed_at=observed_at.replace(tzinfo=None) if observed_at.tzinfo else observed_at,
|
||||
raw=raw_flow,
|
||||
)
|
||||
)
|
||||
commit_or_400(db)
|
||||
return {"status": "ok", "node_id": node.id}
|
||||
|
||||
|
||||
@api_router.get("/vms", response_model=list[WorkloadRead])
|
||||
def workloads(_: CurrentUser, db: Session = Depends(get_db)) -> list[Workload]:
|
||||
return db.scalars(select(Workload).order_by(Workload.name)).all()
|
||||
@@ -632,7 +852,39 @@ def workload_insights(workload_id: str, _: CurrentUser, db: Session = Depends(ge
|
||||
select(Policy).where((Policy.project_id == workload.project_id) | (Policy.project_id.is_(None))).order_by(Policy.name)
|
||||
).all()
|
||||
assigned_ips = db.scalars(select(IpAddress).where(IpAddress.workload_id == workload.id).order_by(IpAddress.address)).all()
|
||||
workload_ips = [address.address for address in assigned_ips]
|
||||
ip_owners = {
|
||||
address.address: db.get(Workload, address.workload_id)
|
||||
for address in db.scalars(select(IpAddress).where(IpAddress.workload_id.is_not(None))).all()
|
||||
}
|
||||
traffic = []
|
||||
if workload_ips:
|
||||
flows = db.scalars(
|
||||
select(TrafficFlow)
|
||||
.where((TrafficFlow.source_ip.in_(workload_ips)) | (TrafficFlow.destination_ip.in_(workload_ips)))
|
||||
.order_by(TrafficFlow.updated_at.desc())
|
||||
.limit(50)
|
||||
).all()
|
||||
for flow in flows:
|
||||
source_owner = ip_owners.get(flow.source_ip)
|
||||
destination_owner = ip_owners.get(flow.destination_ip)
|
||||
traffic.append(
|
||||
{
|
||||
"source": source_owner.name if source_owner else "external",
|
||||
"destination": destination_owner.name if destination_owner else "external",
|
||||
"source_ip": flow.source_ip,
|
||||
"destination_ip": flow.destination_ip,
|
||||
"protocol": flow.protocol,
|
||||
"port": flow.destination_port,
|
||||
"source_port": flow.source_port,
|
||||
"bytes": flow.bytes,
|
||||
"packets": flow.packets,
|
||||
"state": flow.state,
|
||||
"decision": "observed",
|
||||
"observed_at": flow.observed_at.isoformat() if flow.observed_at else None,
|
||||
"ip_addresses": [flow.source_ip, flow.destination_ip],
|
||||
}
|
||||
)
|
||||
audit_mode_notes = [
|
||||
f"{policy.name} is in audit mode; matching traffic is logged without enforcement."
|
||||
for policy in policies
|
||||
|
||||
@@ -2,7 +2,7 @@ from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import JSON, BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db.session import Base
|
||||
@@ -129,6 +129,18 @@ class Node(Base, TimestampMixin):
|
||||
cluster: Mapped[Cluster] = relationship()
|
||||
|
||||
|
||||
class NodeAgent(Base, TimestampMixin):
|
||||
__tablename__ = "node_agents"
|
||||
|
||||
node_id: Mapped[str] = mapped_column(ForeignKey("nodes.id"), primary_key=True)
|
||||
status: Mapped[str] = mapped_column(String(100), default="not_installed")
|
||||
version: Mapped[str | None] = mapped_column(String(50))
|
||||
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
last_payload: Mapped[dict | None] = mapped_column(JSON)
|
||||
install_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
node: Mapped[Node] = relationship()
|
||||
|
||||
|
||||
class Workload(Base, TimestampMixin):
|
||||
__tablename__ = "workloads"
|
||||
|
||||
@@ -183,6 +195,23 @@ class IpAddress(Base, TimestampMixin):
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class TrafficFlow(Base, TimestampMixin):
|
||||
__tablename__ = "traffic_flows"
|
||||
|
||||
id: Mapped[str] = mapped_column(String, primary_key=True, default=new_id)
|
||||
node_id: Mapped[str] = mapped_column(ForeignKey("nodes.id"), index=True)
|
||||
source_ip: Mapped[str] = mapped_column(String(100), index=True)
|
||||
destination_ip: Mapped[str] = mapped_column(String(100), index=True)
|
||||
protocol: Mapped[str] = mapped_column(String(20), default="unknown")
|
||||
source_port: Mapped[int | None] = mapped_column(Integer)
|
||||
destination_port: Mapped[int | None] = mapped_column(Integer)
|
||||
bytes: Mapped[int] = mapped_column(BigInteger, default=0)
|
||||
packets: Mapped[int] = mapped_column(BigInteger, default=0)
|
||||
state: Mapped[str | None] = mapped_column(String(100))
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
raw: Mapped[dict | None] = mapped_column(JSON)
|
||||
|
||||
|
||||
class SecurityGroup(Base, TimestampMixin):
|
||||
__tablename__ = "security_groups"
|
||||
|
||||
|
||||
@@ -181,6 +181,34 @@ class NodeRead(OrmModel):
|
||||
memory_mb: int
|
||||
|
||||
|
||||
class NodeAgentRead(BaseModel):
|
||||
node_id: str
|
||||
status: str
|
||||
version: str | None = None
|
||||
last_seen_at: datetime | None = None
|
||||
install_count: int = 0
|
||||
last_payload: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class NodeWithAgentRead(NodeRead):
|
||||
agent: NodeAgentRead | None = None
|
||||
|
||||
|
||||
class AgentHeartbeat(BaseModel):
|
||||
version: str
|
||||
node_name: str | None = None
|
||||
collected_at: datetime | None = None
|
||||
hostname: str | None = None
|
||||
kernel: str | None = None
|
||||
uptime_seconds: float | None = None
|
||||
loadavg: list[float] = []
|
||||
interfaces: list[dict[str, Any]] = []
|
||||
flows: list[dict[str, Any]] = []
|
||||
conntrack: dict[str, Any] = {}
|
||||
firewall: dict[str, Any] = {}
|
||||
extra: dict[str, Any] = {}
|
||||
|
||||
|
||||
class WorkloadRead(OrmModel):
|
||||
id: str
|
||||
cluster_id: str
|
||||
|
||||
Reference in New Issue
Block a user