From 150a69b60ba2920ba42332161095ea968acf3272 Mon Sep 17 00:00:00 2001 From: nessi Date: Thu, 9 Jul 2026 13:14:01 +0200 Subject: [PATCH] feat: add IPAM discovery from Proxmox with IP enrichment, improve workload insights, and enhance DataTable interactivity Add /ipam/discover endpoint to automatically import IP addresses from Proxmox clusters with error tracking and audit logging, implement ensure_discovered_network helper to create "discovered-ipam" network for auto-discovered IPs, add import_discovered_ips function to parse IP interfaces and create subnet/address records with assignment tracking, enhance ProxmoxProvider.enrich_work --- backend/app/api/v1/router.py | 119 ++++++++++++---- backend/app/services/providers/proxmox.py | 57 +++++++- frontend/src/components/DataTable.tsx | 17 ++- frontend/src/pages/Dashboard.tsx | 5 +- frontend/src/pages/Ipam.tsx | 19 +++ frontend/src/pages/PolicyDesigner.tsx | 159 +++++++++++++++++++--- frontend/src/pages/SetupWizard.tsx | 43 +++++- frontend/src/pages/Workloads.tsx | 19 +-- 8 files changed, 368 insertions(+), 70 deletions(-) diff --git a/backend/app/api/v1/router.py b/backend/app/api/v1/router.py index 9421353..445e2ea 100644 --- a/backend/app/api/v1/router.py +++ b/backend/app/api/v1/router.py @@ -1,6 +1,7 @@ from datetime import datetime import csv import io +from ipaddress import ip_interface from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import StreamingResponse @@ -93,6 +94,48 @@ def setup_setting(db: Session) -> SystemSetting: return setting +def ensure_discovered_network(db: Session, cluster_id: str) -> Network: + network = db.scalar(select(Network).where(Network.cluster_id == cluster_id, Network.name == "discovered-ipam")) + if network: + return network + network = Network( + cluster_id=cluster_id, + name="discovered-ipam", + kind="discovered", + description="Automatically created for IP addresses discovered during Proxmox sync.", + ) + db.add(network) + db.flush() + return network + + +def import_discovered_ips(db: Session, cluster_id: str, workload: Workload, addresses: list[str]) -> int: + imported = 0 + for value in addresses: + try: + interface = ip_interface(value) + except ValueError: + continue + if interface.ip.is_loopback or interface.ip.is_link_local: + continue + network = ensure_discovered_network(db, cluster_id) + cidr = str(interface.network) + subnet = db.scalar(select(Subnet).where(Subnet.network_id == network.id, Subnet.cidr == cidr)) + if not subnet: + subnet = Subnet(network_id=network.id, cidr=cidr) + db.add(subnet) + db.flush() + address_value = str(interface.ip) + existing = db.scalar(select(IpAddress).where(IpAddress.subnet_id == subnet.id, IpAddress.address == address_value)) + if existing: + existing.workload_id = workload.id + existing.status = "assigned" + else: + db.add(IpAddress(subnet_id=subnet.id, address=address_value, status="assigned", workload_id=workload.id)) + imported += 1 + return imported + + @api_router.get("/setup/status", response_model=SetupStatus) def setup_status(db: Session = Depends(get_db)) -> SetupStatus: setting = setup_setting(db) @@ -171,10 +214,7 @@ def dashboard(_: CurrentUser, db: Session = Depends(get_db)) -> dict: {"id": node.id, "name": node.name, "status": node.status, "cluster_id": node.cluster_id} for node in faulty_nodes ], - "top_talkers": [ - {"name": "finance-app-2", "bytes": 942000000}, - {"name": "core-services-1", "bytes": 512000000}, - ], + "top_talkers": [], } @@ -326,6 +366,7 @@ async def sync_cluster(cluster_id: str, user: CurrentUser, db: Session = Depends workload.name = raw_workload.get("name") or workload.name workload.kind = raw_workload.get("type") or raw_workload.get("kind") or workload.kind workload.status = raw_workload.get("status") or workload.status + import_discovered_ips(db, cluster.id, workload, raw_workload.get("ip_addresses", [])) network_by_name = { network.name: network @@ -367,35 +408,29 @@ def workload_insights(workload_id: str, _: CurrentUser, db: Session = Depends(ge policies = db.scalars( select(Policy).where((Policy.project_id == workload.project_id) | (Policy.project_id.is_(None))).order_by(Policy.name) ).all() - traffic = [ - { - "timestamp": datetime.utcnow().isoformat(), - "source": workload.name, - "destination": "finance-db-1" if "web" in workload.tags else "core-services-1", - "protocol": "tcp", - "port": 5432 if "web" in workload.tags else 22, - "bytes": 1489200, - "decision": "allowed", - }, - { - "timestamp": datetime.utcnow().isoformat(), - "source": "unknown-external", - "destination": workload.name, - "protocol": "tcp", - "port": 3389, - "bytes": 22140, - "decision": "would_block", - }, - ] + assigned_ips = db.scalars(select(IpAddress).where(IpAddress.workload_id == workload.id)).all() + traffic = [] audit_mode_notes = [ f"{policy.name} is in audit mode; matching traffic is logged without enforcement." for policy in policies if policy.enforcement_mode == "audit" ] - decision = "audit" if audit_mode_notes else "allowed" + decision = "audit" if audit_mode_notes else "unknown" return WorkloadInsight( workload=workload, - traffic=traffic, + traffic=[ + { + "source": workload.name, + "destination": "unknown", + "protocol": "unknown", + "port": "unknown", + "bytes": 0, + "decision": "no_flow_telemetry", + "ip_addresses": [address.address for address in assigned_ips], + } + ] + if assigned_ips + else traffic, matching_policies=policies, effective_decision=decision, audit_mode_notes=audit_mode_notes, @@ -441,6 +476,38 @@ def ipam_addresses(_: CurrentUser, db: Session = Depends(get_db)) -> list[IpAddr return db.scalars(select(IpAddress).order_by(IpAddress.address)).all() +@api_router.post("/ipam/discover") +async def discover_ipam(user: CurrentUser, db: Session = Depends(get_db)) -> dict: + imported = 0 + errors = [] + clusters = db.scalars(select(Cluster).order_by(Cluster.name)).all() + for cluster in clusters: + try: + inventory = await get_provider(cluster.provider).sync_inventory( + ProviderConnection( + api_url=cluster.api_url, + token=cluster.token_ref or "", + verify_tls=cluster.verify_tls, + read_only=True, + ) + ) + workload_by_external_id = { + workload.external_id: workload + for workload in db.scalars(select(Workload).where(Workload.cluster_id == cluster.id)).all() + } + for raw_workload in inventory.get("workloads", []): + external_id = str(raw_workload.get("vmid") or raw_workload.get("id") or "") + workload = workload_by_external_id.get(external_id) + if workload: + imported += import_discovered_ips(db, cluster.id, workload, raw_workload.get("ip_addresses", [])) + except Exception as exc: + errors.append({"cluster": cluster.name, "error": str(exc)}) + db.add(Job(kind="ipam.discover", status="success" if not errors else "failed", progress=100, logs=[f"Imported {imported} IP addresses"], error=str(errors) if errors else None)) + commit_or_400(db) + write_audit(db, action="ipam.discover", object_type="ipam", user_id=user.id, new_values={"imported": imported, "errors": errors}, result="success" if not errors else "failed") + return {"imported": imported, "errors": errors} + + @api_router.post("/ipam/addresses", response_model=IpAddressRead) def reserve_ip(payload: IpReservationCreate, user: CurrentUser, db: Session = Depends(get_db)) -> IpAddress: if not db.get(Subnet, payload.subnet_id): diff --git a/backend/app/services/providers/proxmox.py b/backend/app/services/providers/proxmox.py index 97f6eb8..db56bae 100644 --- a/backend/app/services/providers/proxmox.py +++ b/backend/app/services/providers/proxmox.py @@ -25,10 +25,11 @@ class ProxmoxProvider(Provider): return response.json().get("data", {}) async def sync_inventory(self, connection: ProviderConnection) -> dict[str, list[dict[str, Any]]]: + base_url = connection.api_url.rstrip("/") headers = {"Authorization": self.auth_header(connection.token)} async with httpx.AsyncClient(verify=connection.verify_tls, timeout=20) as client: resources = await client.get( - f"{connection.api_url.rstrip('/')}/api2/json/cluster/resources", + f"{base_url}/api2/json/cluster/resources", headers=headers, ) resources.raise_for_status() @@ -36,9 +37,63 @@ class ProxmoxProvider(Provider): nodes = [item for item in data if item.get("type") == "node"] workloads = [item for item in data if item.get("type") in {"qemu", "lxc"}] + async with httpx.AsyncClient(verify=connection.verify_tls, timeout=8) as client: + for workload in workloads: + await self.enrich_workload_ips(client, base_url, headers, workload) networks = await self.list_networks(connection) return {"nodes": nodes, "workloads": workloads, "networks": networks} + async def enrich_workload_ips( + self, + client: httpx.AsyncClient, + base_url: str, + headers: dict[str, str], + workload: dict[str, Any], + ) -> None: + node = workload.get("node") + vmid = workload.get("vmid") + kind = workload.get("type") + workload["ip_addresses"] = [] + if not node or not vmid: + return + + if kind == "qemu": + try: + response = await client.get( + f"{base_url}/api2/json/nodes/{node}/qemu/{vmid}/agent/network-get-interfaces", + headers=headers, + ) + if response.status_code >= 400: + return + interfaces = response.json().get("data", {}).get("result", []) + for interface in interfaces: + for address in interface.get("ip-addresses", []): + ip_address = address.get("ip-address") + prefix = address.get("prefix") + if ip_address and ":" not in ip_address and prefix is not None: + workload["ip_addresses"].append(f"{ip_address}/{prefix}") + except httpx.HTTPError: + return + + if kind == "lxc": + try: + response = await client.get( + f"{base_url}/api2/json/nodes/{node}/lxc/{vmid}/config", + headers=headers, + ) + if response.status_code >= 400: + return + config = response.json().get("data", {}) + for key, value in config.items(): + if key.startswith("net") and isinstance(value, str): + for part in value.split(","): + if part.startswith("ip="): + ip_address = part.removeprefix("ip=") + if ip_address != "dhcp" and ":" not in ip_address: + workload["ip_addresses"].append(ip_address) + except httpx.HTTPError: + return + async def list_networks(self, connection: ProviderConnection) -> list[dict[str, Any]]: headers = {"Authorization": self.auth_header(connection.token)} async with httpx.AsyncClient(verify=connection.verify_tls, timeout=20) as client: diff --git a/frontend/src/components/DataTable.tsx b/frontend/src/components/DataTable.tsx index b816192..3b41a25 100644 --- a/frontend/src/components/DataTable.tsx +++ b/frontend/src/components/DataTable.tsx @@ -1,9 +1,13 @@ +import { ReactNode } from "react"; + type DataTableProps> = { - columns: Array<{ key: keyof T; label: string; render?: (row: T) => string }>; + columns: Array<{ key: keyof T; label: string; render?: (row: T) => ReactNode }>; rows: T[]; + onRowClick?: (row: T) => void; + selectedId?: string; }; -export function DataTable>({ columns, rows }: DataTableProps) { +export function DataTable>({ columns, rows, onRowClick, selectedId }: DataTableProps) { return (
@@ -19,7 +23,13 @@ export function DataTable>({ columns, rows }: {rows.map((row, index) => ( - + onRowClick?.(row)} + > {columns.map((column) => ( {column.render ? column.render(row) : String(row[column.key] ?? "")} @@ -34,4 +44,3 @@ export function DataTable>({ columns, rows }:
); } - diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index ad37431..f805597 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -44,15 +44,14 @@ export function Dashboard() {
Top Talkers
- {(data?.top_talkers ?? []).map((item) => ( + {(data?.top_talkers ?? []).length ? (data?.top_talkers ?? []).map((item) => (
{item.name} {Math.round(item.bytes / 1_000_000)} MB
- ))} + )) :
No flow telemetry collected yet.
}
); } - diff --git a/frontend/src/pages/Ipam.tsx b/frontend/src/pages/Ipam.tsx index fda31e6..5f0be90 100644 --- a/frontend/src/pages/Ipam.tsx +++ b/frontend/src/pages/Ipam.tsx @@ -13,6 +13,7 @@ export function Ipam() { const networks = useQuery({ queryKey: ["networks"], queryFn: () => api("/networks") }); const subnets = useQuery({ queryKey: ["subnets"], queryFn: () => api("/ipam/subnets") }); const addresses = useQuery({ queryKey: ["addresses"], queryFn: () => api("/ipam/addresses") }); + const [message, setMessage] = useState(""); const [subnetForm, setSubnetForm] = useState({ network_id: "", cidr: "10.50.0.0/24", gateway: "10.50.0.1", dns: ["10.50.0.10"], dhcp_enabled: false }); const [ipForm, setIpForm] = useState({ subnet_id: "", address: "10.50.0.20", status: "reserved", note: "" }); const [subnetOpen, setSubnetOpen] = useState(false); @@ -21,16 +22,20 @@ export function Ipam() { const createSubnet = useMutation({ mutationFn: () => api("/ipam/subnets", { method: "POST", body: JSON.stringify({ ...subnetForm, network_id: subnetForm.network_id || networks.data?.[0]?.id }) }), onSuccess: () => { + setMessage("Subnet created."); setSubnetOpen(false); queryClient.invalidateQueries({ queryKey: ["subnets"] }); }, + onError: (error) => setMessage(error instanceof Error ? error.message : "Subnet creation failed."), }); const createIp = useMutation({ mutationFn: () => api("/ipam/addresses", { method: "POST", body: JSON.stringify({ ...ipForm, subnet_id: ipForm.subnet_id || subnets.data?.[0]?.id }) }), onSuccess: () => { + setMessage("IP address saved."); setIpOpen(false); queryClient.invalidateQueries({ queryKey: ["addresses"] }); }, + onError: (error) => setMessage(error instanceof Error ? error.message : "IP reservation failed."), }); async function submitSubnet(event: FormEvent) { @@ -56,6 +61,17 @@ export function Ipam() { URL.revokeObjectURL(url); } + async function discoverIpam() { + try { + const result = await api<{ imported: number; errors: Array> }>("/ipam/discover", { method: "POST" }); + setMessage(`Discovery imported ${result.imported} IP addresses${result.errors.length ? " with errors" : ""}.`); + await queryClient.invalidateQueries({ queryKey: ["subnets"] }); + await queryClient.invalidateQueries({ queryKey: ["addresses"] }); + } catch (error) { + setMessage(error instanceof Error ? error.message : "IPAM discovery failed."); + } + } + return ( <> @@ -63,8 +79,10 @@ export function Ipam() {
+
+ {message ?
{message}
: null} setSubnetOpen(false)}>
Add Subnet
@@ -103,6 +121,7 @@ export function Ipam() {
+ []} columns={[{ key: "cidr", label: "Subnet" }, { key: "gateway", label: "Gateway" }, { key: "dhcp_enabled", label: "DHCP" }]} /> []} columns={[{ key: "address", label: "Address" }, { key: "status", label: "Status" }, { key: "note", label: "Note" }]} />
diff --git a/frontend/src/pages/PolicyDesigner.tsx b/frontend/src/pages/PolicyDesigner.tsx index b5d2545..9b2c2b6 100644 --- a/frontend/src/pages/PolicyDesigner.tsx +++ b/frontend/src/pages/PolicyDesigner.tsx @@ -1,48 +1,163 @@ +import { FormEvent, useMemo, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Save, Wand2 } from "lucide-react"; +import { api, Network, Policy, SecurityGroup, ServiceCatalogItem, Workload } from "../api/client"; +import { buttonClass, Field, inputClass, secondaryButtonClass, selectClass } from "../components/FormControls"; import { PageHeader } from "../components/PageHeader"; +type TargetOption = { + label: string; + value: string; +}; + export function PolicyDesigner() { + const queryClient = useQueryClient(); + const workloads = useQuery({ queryKey: ["workloads"], queryFn: () => api("/vms") }); + const securityGroups = useQuery({ queryKey: ["security-groups"], queryFn: () => api("/security-groups") }); + const networks = useQuery({ queryKey: ["networks"], queryFn: () => api("/networks") }); + const services = useQuery({ queryKey: ["service-catalog"], queryFn: () => api("/service-catalog") }); + const [preview, setPreview] = useState | null>(null); + const [form, setForm] = useState({ + name: "Designed Policy", + source: "any", + destination: "any", + service_id: "", + protocol: "tcp", + ports: "443", + action: "allow", + direction: "ingress", + enforcement_mode: "enforced", + logging: true, + description: "Policy intent", + }); + + const targets = useMemo(() => { + return [ + { label: "Any", value: "any" }, + ...(workloads.data ?? []).map((workload) => ({ label: `VM/LXC: ${workload.name}`, value: `workload:${workload.id}` })), + ...(securityGroups.data ?? []).map((group) => ({ label: `Security Group: ${group.name}`, value: `sg:${group.name}` })), + ...(networks.data ?? []).map((network) => ({ label: `Network: ${network.name}`, value: `network:${network.name}` })), + ]; + }, [networks.data, securityGroups.data, workloads.data]); + + function payload() { + const service = services.data?.find((item) => item.id === form.service_id); + return { + project_id: null, + name: form.name, + enabled: true, + definition: { + source: form.source, + destination: form.destination, + service: { protocol: service?.protocol ?? form.protocol, ports: service?.ports ?? form.ports }, + action: form.action, + direction: form.direction, + enforcement_mode: form.enforcement_mode, + logging: form.logging, + description: form.description, + }, + }; + } + + const save = useMutation({ + mutationFn: () => api("/policies", { method: "POST", body: JSON.stringify(payload()) }), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ["policies"] }), + }); + + async function submit(event: FormEvent) { + event.preventDefault(); + await save.mutateAsync(); + } + + function dryRun() { + const current = payload(); + const warnings = []; + if (current.definition.source === "any" && current.definition.destination === "any") { + warnings.push("Policy targets all sources and destinations."); + } + if (current.definition.enforcement_mode === "audit") { + warnings.push("Audit mode logs decisions without enforcing them."); + } + setPreview({ + affected_workloads: (workloads.data ?? []).filter((workload) => + [current.definition.source, current.definition.destination].includes(`workload:${workload.id}`), + ), + generated_rule: current.definition, + conflicts: [], + warnings, + }); + } + return ( <> -
-
+
+
- {["Source", "Destination", "Service", "Action", "Direction", "Logging"].map((label) => ( - - ))} + + + + + + + + + + + + + setForm({ ...form, protocol: event.target.value })} /> + setForm({ ...form, ports: event.target.value })} /> + + + + + +
-
+
); } - diff --git a/frontend/src/pages/SetupWizard.tsx b/frontend/src/pages/SetupWizard.tsx index ffccee6..72f1555 100644 --- a/frontend/src/pages/SetupWizard.tsx +++ b/frontend/src/pages/SetupWizard.tsx @@ -5,9 +5,20 @@ import { publicApi } from "../api/client"; import { buttonClass, Field, inputClass, secondaryButtonClass, selectClass } from "../components/FormControls"; import { useTheme } from "../stores/theme"; +const preparingMessages = [ + "Creating your secure workspace...", + "Preparing the control plane...", + "Registering your Proxmox provider...", + "Setting safe read-only defaults...", + "Warming up inventory services...", + "Almost ready...", +]; + export function SetupWizard() { const [step, setStep] = useState(0); const [error, setError] = useState(""); + const [preparing, setPreparing] = useState(false); + const [prepareProgress, setPrepareProgress] = useState(0); const { dark, toggle } = useTheme(); const [form, setForm] = useState({ admin_email: "admin@nexafabric.local", @@ -26,12 +37,42 @@ export function SetupWizard() { setError(""); try { await publicApi("/setup/complete", { method: "POST", body: JSON.stringify(form) }); + setPreparing(true); + setPrepareProgress(0); + for (let progress = 1; progress <= 100; progress += 1) { + await new Promise((resolve) => window.setTimeout(resolve, 150)); + setPrepareProgress(progress); + } window.location.href = "/login"; } catch (err) { + setPreparing(false); setError(err instanceof Error ? err.message : "Setup failed"); } } + if (preparing) { + const messageIndex = Math.min( + preparingMessages.length - 1, + Math.floor((prepareProgress / 100) * preparingMessages.length), + ); + + return ( +
+
+
+ +
+

Preparing NexaFabric

+

{preparingMessages[messageIndex]}

+
+
+
+
{prepareProgress}%
+
+
+ ); + } + return (
: null}
- {step < 3 ? : } + {step < 3 ? : }
diff --git a/frontend/src/pages/Workloads.tsx b/frontend/src/pages/Workloads.tsx index c3b5b42..d24eb1c 100644 --- a/frontend/src/pages/Workloads.tsx +++ b/frontend/src/pages/Workloads.tsx @@ -1,10 +1,9 @@ import { useState } from "react"; import { useQuery } from "@tanstack/react-query"; -import { Activity, Server } from "lucide-react"; +import { Activity } from "lucide-react"; import { api, Workload, WorkloadInsight } from "../api/client"; import { DataTable } from "../components/DataTable"; -import { secondaryButtonClass } from "../components/FormControls"; import { PageHeader } from "../components/PageHeader"; export function Workloads() { @@ -25,15 +24,9 @@ export function Workloads() { []} columns={[{ key: "name", label: "Name" }, { key: "kind", label: "Kind" }, { key: "status", label: "Status" }, { key: "external_id", label: "VMID" }]} + selectedId={selected} + onRowClick={(row) => setSelectedId(String(row.id))} /> -
- {(workloads.data ?? []).map((workload) => ( - - ))} -