feat: reduce agent payload size with flow truncation and increase nginx body size limit to 16MB
Reduce agent flow_limit from 2000 to 1500, truncate log_excerpt from 500 to 180 characters in firewall log parsing, increase firewall_log_lines_from_files limit from default to max(limit*2, 1000), add compact_agent_payload to truncate flows array to 50 entries with flow_count/flows_truncated metadata, update agent_heartbeat to store compacted payload instead of full dump, add agentFlowCount helper to
This commit is contained in:
@@ -589,6 +589,16 @@ def traffic_flow_key(node_id: str, raw_flow: dict[str, object]) -> tuple[object,
|
||||
)
|
||||
|
||||
|
||||
def compact_agent_payload(payload: AgentHeartbeat) -> dict:
|
||||
value = payload.model_dump(mode="json")
|
||||
flows = value.get("flows")
|
||||
if isinstance(flows, list):
|
||||
value["flow_count"] = len(flows)
|
||||
value["flows"] = flows[:50]
|
||||
value["flows_truncated"] = len(flows) > 50
|
||||
return value
|
||||
|
||||
|
||||
def policy_read_payload(policy: Policy, deployment_status: dict[str, object] | None = None) -> dict[str, object]:
|
||||
return {
|
||||
"id": policy.id,
|
||||
@@ -1336,7 +1346,7 @@ cat > "$CONFIG_DIR/config.json" <<'JSON'
|
||||
"node_id": "{node.id}",
|
||||
"node_name": "{node.name}",
|
||||
"interval_seconds": 30,
|
||||
"flow_limit": 2000,
|
||||
"flow_limit": 1500,
|
||||
"packet_flow_collector": true,
|
||||
"packet_flow_window_seconds": 10,
|
||||
"firewall_log_collector": true,
|
||||
@@ -1454,7 +1464,7 @@ def agent_heartbeat(payload: AgentHeartbeat, authorization: str | None = Header(
|
||||
agent.status = "online"
|
||||
agent.version = payload.version
|
||||
agent.last_seen_at = datetime.utcnow()
|
||||
agent.last_payload = payload.model_dump(mode="json")
|
||||
agent.last_payload = compact_agent_payload(payload)
|
||||
retention_hours = runtime_settings_payload(db).flow_retention_hours
|
||||
retention_cutoff = datetime.utcnow() - timedelta(hours=retention_hours)
|
||||
for old_flow in db.scalars(select(TrafficFlow).where(TrafficFlow.node_id == node.id, TrafficFlow.updated_at < retention_cutoff)).all():
|
||||
|
||||
Reference in New Issue
Block a user