From 0f4734c85cfec80bb8831ac0588b0c55d7df772d Mon Sep 17 00:00:00 2001 From: nessi Date: Fri, 10 Jul 2026 08:42:48 +0200 Subject: [PATCH] feat: reduce agent payload size with flow truncation and increase nginx body size limit to 16MB Reduce agent flow_limit from 2000 to 1500, truncate log_excerpt from 500 to 180 characters in firewall log parsing, increase firewall_log_lines_from_files limit from default to max(limit*2, 1000), add compact_agent_payload to truncate flows array to 50 entries with flow_count/flows_truncated metadata, update agent_heartbeat to store compacted payload instead of full dump, add agentFlowCount helper to --- backend/app/agent_assets/nexafabric-agent.py | 8 ++++---- backend/app/api/v1/router.py | 14 ++++++++++++-- frontend/src/pages/Nodes.tsx | 10 +++++++++- nginx/default.conf | 3 +++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/backend/app/agent_assets/nexafabric-agent.py b/backend/app/agent_assets/nexafabric-agent.py index 203528c..5fb3e51 100644 --- a/backend/app/agent_assets/nexafabric-agent.py +++ b/backend/app/agent_assets/nexafabric-agent.py @@ -18,7 +18,7 @@ from pathlib import Path from typing import Any -VERSION = "0.2.2" +VERSION = "0.2.3" VM_INTERFACE_RE = re.compile(r"(?:tap|fwbr|fwln|fwpr)(\d+)") VM_INTERFACE_DETAIL_RE = re.compile(r"(?:tap|fwbr|fwln|fwpr)(\d+)i(\d+)") LOG_FIELD_RE = re.compile(r"\b([A-Z]+)=([^\s]+)") @@ -358,7 +358,7 @@ def parse_firewall_log_line(line: str) -> dict[str, Any] | None: "state": decision, "decision": decision, "collector": "firewall-log", - "log_excerpt": line[-500:], + "log_excerpt": line[-180:], } @@ -386,7 +386,7 @@ def collect_firewall_log_flows(since_minutes: int = 5, limit: int = 500) -> tupl log_lines.extend(output.splitlines()) else: diagnostics["errors"].append(output or "journalctl returned no firewall log output") - file_lines, file_errors = firewall_log_lines_from_files() + file_lines, file_errors = firewall_log_lines_from_files(max(limit * 2, 1000)) log_lines.extend(file_lines) diagnostics["file_errors"] = file_errors diagnostics["lines_scanned"] = len(log_lines) @@ -500,7 +500,7 @@ def collect_firewall() -> dict[str, Any]: def collect_payload(config: dict[str, Any]) -> dict[str, Any]: uptime = read_text("/proc/uptime") interfaces = collect_interfaces() - flow_limit = int(config.get("flow_limit", 2000)) + flow_limit = int(config.get("flow_limit", 1500)) packet_flows: list[dict[str, Any]] = [] packet_diagnostics: dict[str, Any] | None = None firewall_log_flows: list[dict[str, Any]] = [] diff --git a/backend/app/api/v1/router.py b/backend/app/api/v1/router.py index a1d8b5e..fffd65b 100644 --- a/backend/app/api/v1/router.py +++ b/backend/app/api/v1/router.py @@ -589,6 +589,16 @@ def traffic_flow_key(node_id: str, raw_flow: dict[str, object]) -> tuple[object, ) +def compact_agent_payload(payload: AgentHeartbeat) -> dict: + value = payload.model_dump(mode="json") + flows = value.get("flows") + if isinstance(flows, list): + value["flow_count"] = len(flows) + value["flows"] = flows[:50] + value["flows_truncated"] = len(flows) > 50 + return value + + def policy_read_payload(policy: Policy, deployment_status: dict[str, object] | None = None) -> dict[str, object]: return { "id": policy.id, @@ -1336,7 +1346,7 @@ cat > "$CONFIG_DIR/config.json" <<'JSON' "node_id": "{node.id}", "node_name": "{node.name}", "interval_seconds": 30, - "flow_limit": 2000, + "flow_limit": 1500, "packet_flow_collector": true, "packet_flow_window_seconds": 10, "firewall_log_collector": true, @@ -1454,7 +1464,7 @@ def agent_heartbeat(payload: AgentHeartbeat, authorization: str | None = Header( agent.status = "online" agent.version = payload.version agent.last_seen_at = datetime.utcnow() - agent.last_payload = payload.model_dump(mode="json") + agent.last_payload = compact_agent_payload(payload) retention_hours = runtime_settings_payload(db).flow_retention_hours retention_cutoff = datetime.utcnow() - timedelta(hours=retention_hours) for old_flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id == node.id, TrafficFlow.updated_at < retention_cutoff)).all(): diff --git a/frontend/src/pages/Nodes.tsx b/frontend/src/pages/Nodes.tsx index 6f50332..6db2968 100644 --- a/frontend/src/pages/Nodes.tsx +++ b/frontend/src/pages/Nodes.tsx @@ -47,6 +47,14 @@ export function Nodes() { return Array.isArray(interfaceTraffic) ? interfaceTraffic : []; } + function agentFlowCount(node: Node) { + const count = node.agent?.last_payload?.flow_count; + if (typeof count === "number") { + return count; + } + return agentFlows(node).length; + } + function agentInterfaces(node: Node) { const interfaces = node.agent?.last_payload?.interfaces; return Array.isArray(interfaces) ? interfaces : []; @@ -158,7 +166,7 @@ export function Nodes() {
Flows
-
{agentFlows(detailNode).length}
+
{agentFlowCount(detailNode)}
Conntrack
diff --git a/nginx/default.conf b/nginx/default.conf index d1b32f2..6f53700 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -1,8 +1,10 @@ server { listen 80; server_name _; + client_max_body_size 16m; location /api/ { + client_max_body_size 16m; proxy_pass http://api:8000/api/; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; @@ -12,6 +14,7 @@ server { } location /agents/ { + client_max_body_size 16m; proxy_pass http://api:8000/api/v1/agents/; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr;