From fa8958934f80fa86210e99dd963d5f500a488e56 Mon Sep 17 00:00:00 2001 From: nessi Date: Thu, 12 Feb 2026 16:54:22 +0100 Subject: [PATCH] Add multi-database discovery and grouping features This update introduces optional automatic discovery and onboarding of all databases on a PostgreSQL instance. It also enhances the frontend UI with grouped target display and navigation, making it easier to view and manage related databases. Additionally, new backend endpoints and logic ensure seamless integration of these features. --- README.md | 2 + backend/app/api/routes/targets.py | 118 ++++++++++++++++++++++-- backend/app/schemas/target.py | 1 + frontend/src/pages/DashboardPage.jsx | 111 ++++++++++++++++++---- frontend/src/pages/TargetDetailPage.jsx | 36 +++++++- frontend/src/pages/TargetsPage.jsx | 32 ++++++- frontend/src/styles.css | 35 +++++++ 7 files changed, 307 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 479c386..92ec134 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ It combines FastAPI, React, and PostgreSQL in a Docker Compose stack with RBAC, ## Highlights - Multi-target monitoring for remote PostgreSQL instances +- Optional one-click target onboarding for "all databases" discovery on an instance - PostgreSQL compatibility support: `14`, `15`, `16`, `17`, `18` - JWT auth (`access` + `refresh`) and RBAC (`admin`, `operator`, `viewer`) - Polling collector for metrics, locks, activity, and optional `pg_stat_statements` @@ -194,6 +195,7 @@ Recommended values for `VITE_API_URL`: - Create, list, edit, delete targets - Test target connection before save +- Optional "discover all databases" mode (creates one monitored target per discovered DB) - Configure SSL mode per target - Toggle `pg_stat_statements` usage per target - Assign responsible users (target owners) diff --git a/backend/app/api/routes/targets.py b/backend/app/api/routes/targets.py index 6e3f94b..6ab0322 100644 --- a/backend/app/api/routes/targets.py +++ b/backend/app/api/routes/targets.py @@ -1,4 +1,5 @@ from datetime import datetime +from uuid import uuid4 import asyncpg from fastapi import APIRouter, Depends, HTTPException, Query, status @@ -60,6 +61,47 @@ async def _set_target_owners(db: AsyncSession, target_id: int, user_ids: list[in db.add(TargetOwner(target_id=target_id, user_id=user_id, assigned_by_user_id=assigned_by_user_id)) +async def _discover_databases(payload: TargetCreate) -> list[str]: + ssl = False if payload.sslmode == "disable" else True + conn = None + try: + conn = await asyncpg.connect( + host=payload.host, + port=payload.port, + database=payload.dbname, + user=payload.username, + password=payload.password, + ssl=ssl, + timeout=8, + ) + rows = await conn.fetch( + """ + SELECT datname + FROM pg_database + WHERE datallowconn + AND NOT datistemplate + ORDER BY datname + """ + ) + return [row["datname"] for row in rows if row["datname"]] + except Exception as exc: + raise HTTPException(status_code=400, detail=f"Database discovery failed: {exc}") + finally: + if conn: + await conn.close() + + +async def _next_unique_target_name(db: AsyncSession, base_name: str) -> str: + candidate = base_name.strip() + suffix = 2 + while True: + exists = await db.scalar(select(Target.id).where(Target.name == candidate)) + if exists is None: + return candidate + candidate = f"{base_name} ({suffix})" + suffix += 1 + + @router.get("", response_model=list[TargetOut]) async def list_targets(user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db)) -> list[TargetOut]: _ = user @@ -101,13 +143,78 @@ async def create_target( user: User = Depends(require_roles("admin", "operator")), db: AsyncSession = Depends(get_db), ) -> TargetOut: + owner_ids = sorted(set(payload.owner_user_ids or [])) + if owner_ids: + owners_exist = (await db.scalars(select(User.id).where(User.id.in_(owner_ids)))).all() + if len(set(owners_exist)) != len(owner_ids): + raise HTTPException(status_code=400, detail="One or more owner users were not found") + + encrypted_password = encrypt_secret(payload.password) + created_targets: list[Target] = [] + + if payload.discover_all_databases: + databases = await _discover_databases(payload) + if not databases: + raise HTTPException(status_code=400, detail="No databases discovered on target") + group_id = str(uuid4()) + base_tags = payload.tags or {} + for dbname in databases: + duplicate = await db.scalar( + select(Target.id).where( + Target.host == payload.host, + Target.port == payload.port, + Target.dbname == dbname, + Target.username == payload.username, + ) + ) + if duplicate is not None: + continue + target_name = await _next_unique_target_name(db, f"{payload.name} / {dbname}") + tags = { + **base_tags, + "monitor_mode": "all_databases", + "monitor_group_id": group_id, + "monitor_group_name": payload.name, + } + target = Target( + name=target_name, + host=payload.host, + port=payload.port, + dbname=dbname, + username=payload.username, + encrypted_password=encrypted_password, + sslmode=payload.sslmode, + use_pg_stat_statements=payload.use_pg_stat_statements, + tags=tags, + ) + db.add(target) + await db.flush() + created_targets.append(target) + if owner_ids: + await _set_target_owners(db, target.id, owner_ids, user.id) + + if not created_targets: + raise HTTPException(status_code=400, detail="All discovered databases already exist as targets") + await db.commit() + for item in created_targets: + await db.refresh(item) + await write_audit_log( + db, + "target.create.all_databases", + user.id, + {"base_name": payload.name, "created_count": len(created_targets), "host": payload.host, "port": payload.port}, + ) + owner_map = await _owners_by_target_ids(db, [created_targets[0].id]) + return _target_out_with_owners(created_targets[0], owner_map.get(created_targets[0].id, [])) + + target_name = await _next_unique_target_name(db, payload.name) target = Target( - name=payload.name, + name=target_name, host=payload.host, port=payload.port, dbname=payload.dbname, username=payload.username, - encrypted_password=encrypt_secret(payload.password), + encrypted_password=encrypted_password, sslmode=payload.sslmode, use_pg_stat_statements=payload.use_pg_stat_statements, tags=payload.tags, @@ -116,11 +223,8 @@ async def create_target( await db.commit() await db.refresh(target) - if payload.owner_user_ids: - owners_exist = (await db.scalars(select(User.id).where(User.id.in_(payload.owner_user_ids)))).all() - if len(set(owners_exist)) != len(set(payload.owner_user_ids)): - raise HTTPException(status_code=400, detail="One or more owner users were not found") - await _set_target_owners(db, target.id, payload.owner_user_ids, user.id) + if owner_ids: + await _set_target_owners(db, target.id, owner_ids, user.id) await db.commit() await write_audit_log(db, "target.create", user.id, {"target_id": target.id, "name": target.name}) diff --git a/backend/app/schemas/target.py b/backend/app/schemas/target.py index f26acfd..da3195c 100644 --- a/backend/app/schemas/target.py +++ b/backend/app/schemas/target.py @@ -16,6 +16,7 @@ class TargetBase(BaseModel): class TargetCreate(TargetBase): password: str + discover_all_databases: bool = False class TargetConnectionTestRequest(BaseModel): diff --git a/frontend/src/pages/DashboardPage.jsx b/frontend/src/pages/DashboardPage.jsx index 553db06..3ba89a3 100644 --- a/frontend/src/pages/DashboardPage.jsx +++ b/frontend/src/pages/DashboardPage.jsx @@ -3,10 +3,20 @@ import { Link } from "react-router-dom"; import { apiFetch } from "../api"; import { useAuth } from "../state"; +function getTargetGroupMeta(target) { + const tags = target?.tags || {}; + if (tags.monitor_mode !== "all_databases" || !tags.monitor_group_id) return null; + return { + id: tags.monitor_group_id, + name: tags.monitor_group_name || target.name || "All databases", + }; +} + export function DashboardPage() { const { tokens, refresh, alertStatus } = useAuth(); const [targets, setTargets] = useState([]); const [search, setSearch] = useState(""); + const [openGroups, setOpenGroups] = useState({}); const [loading, setLoading] = useState(true); const [error, setError] = useState(""); @@ -51,6 +61,21 @@ export function DashboardPage() { (t.dbname || "").toLowerCase().includes(q) ); }); + const groupedRows = []; + const groupedMap = new Map(); + for (const t of filteredTargets) { + const meta = getTargetGroupMeta(t); + if (!meta) { + groupedRows.push({ type: "single", target: t }); + continue; + } + if (!groupedMap.has(meta.id)) { + const groupRow = { type: "group", groupId: meta.id, groupName: meta.name, targets: [] }; + groupedMap.set(meta.id, groupRow); + groupedRows.push(groupRow); + } + groupedMap.get(meta.id).targets.push(t); + } return (
@@ -95,30 +120,84 @@ export function DashboardPage() {
- {filteredTargets.map((t) => { - const severity = targetSeverities.get(t.id) || "ok"; + {groupedRows.map((row) => { + if (row.type === "single") { + const t = row.target; + const severity = targetSeverities.get(t.id) || "ok"; + return ( +
+
+
+

{t.name}

+ + {severity === "alert" ? "Alert" : severity === "warning" ? "Warning" : "OK"} + +
+

Host: {t.host}:{t.port}

+

DB: {t.dbname}

+
+
+ + + Details + +
+
+ ); + } + + const highestSeverity = row.targets.some((t) => targetSeverities.get(t.id) === "alert") + ? "alert" + : row.targets.some((t) => targetSeverities.get(t.id) === "warning") + ? "warning" + : "ok"; + const first = row.targets[0]; + const isOpen = !!openGroups[row.groupId]; return ( -
+
-

{t.name}

- - {severity === "alert" ? "Alert" : severity === "warning" ? "Warning" : "OK"} +

{row.groupName}

+ + {highestSeverity === "alert" ? "Alert" : highestSeverity === "warning" ? "Warning" : "OK"}
-

Host: {t.host}:{t.port}

-

DB: {t.dbname}

+

Host: {first.host}:{first.port}

+

DB: All databases ({row.targets.length})

- - - Details - +
+ {isOpen && ( +
+ {row.targets.map((t) => { + const severity = targetSeverities.get(t.id) || "ok"; + return ( +
+
+ {t.dbname} + + {severity === "alert" ? "Alert" : severity === "warning" ? "Warning" : "OK"} + +
+ + Details + +
+ ); + })} +
+ )}
); })} diff --git a/frontend/src/pages/TargetDetailPage.jsx b/frontend/src/pages/TargetDetailPage.jsx index a222ce2..a24b9ed 100644 --- a/frontend/src/pages/TargetDetailPage.jsx +++ b/frontend/src/pages/TargetDetailPage.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useMemo, useRef, useState } from "react"; -import { useParams } from "react-router-dom"; +import { useNavigate, useParams } from "react-router-dom"; import { Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; import { apiFetch } from "../api"; import { useAuth } from "../state"; @@ -75,6 +75,7 @@ async function loadMetric(targetId, metric, range, tokens, refresh) { export function TargetDetailPage() { const { id } = useParams(); + const navigate = useNavigate(); const { tokens, refresh, uiMode } = useAuth(); const [range, setRange] = useState("1h"); const [liveMode, setLiveMode] = useState(false); @@ -84,6 +85,7 @@ export function TargetDetailPage() { const [overview, setOverview] = useState(null); const [targetMeta, setTargetMeta] = useState(null); const [owners, setOwners] = useState([]); + const [groupTargets, setGroupTargets] = useState([]); const [error, setError] = useState(""); const [loading, setLoading] = useState(true); const refreshRef = useRef(refresh); @@ -99,7 +101,7 @@ export function TargetDetailPage() { setLoading(true); } try { - const [connections, xacts, cache, locksTable, activityTable, overviewData, targetInfo, ownerRows] = await Promise.all([ + const [connections, xacts, cache, locksTable, activityTable, overviewData, targetInfo, ownerRows, allTargets] = await Promise.all([ loadMetric(id, "connections_total", range, tokens, refreshRef.current), loadMetric(id, "xacts_total", range, tokens, refreshRef.current), loadMetric(id, "cache_hit_ratio", range, tokens, refreshRef.current), @@ -108,6 +110,7 @@ export function TargetDetailPage() { apiFetch(`/targets/${id}/overview`, {}, tokens, refreshRef.current), apiFetch(`/targets/${id}`, {}, tokens, refreshRef.current), apiFetch(`/targets/${id}/owners`, {}, tokens, refreshRef.current), + apiFetch("/targets", {}, tokens, refreshRef.current), ]); if (!active) return; setSeries({ connections, xacts, cache }); @@ -116,6 +119,15 @@ export function TargetDetailPage() { setOverview(overviewData); setTargetMeta(targetInfo); setOwners(ownerRows); + const groupId = targetInfo?.tags?.monitor_group_id; + if (groupId) { + const sameGroup = allTargets + .filter((item) => item?.tags?.monitor_group_id === groupId) + .sort((a, b) => (a.dbname || "").localeCompare(b.dbname || "")); + setGroupTargets(sameGroup); + } else { + setGroupTargets([]); + } setError(""); } catch (e) { if (active) setError(String(e.message || e)); @@ -232,6 +244,26 @@ export function TargetDetailPage() { Target Detail {targetMeta?.name || `#${id}`} {targetMeta?.dbname ? ` (${targetMeta.dbname})` : ""} + {groupTargets.length > 1 && ( +
+ + +
+ )}
Responsible users: {owners.length > 0 ? owners.map((item) => {item.email}) : none assigned} diff --git a/frontend/src/pages/TargetsPage.jsx b/frontend/src/pages/TargetsPage.jsx index eef8651..f0785fe 100644 --- a/frontend/src/pages/TargetsPage.jsx +++ b/frontend/src/pages/TargetsPage.jsx @@ -7,11 +7,12 @@ const emptyForm = { name: "", host: "", port: 5432, - dbname: "", + dbname: "postgres", username: "", password: "", sslmode: "prefer", use_pg_stat_statements: true, + discover_all_databases: false, owner_user_ids: [], tags: {}, }; @@ -276,8 +277,18 @@ export function TargetsPage() { setForm({ ...form, port: Number(e.target.value) })} type="number" required />
- - setForm({ ...form, dbname: e.target.value })} required /> + + setForm({ ...form, dbname: e.target.value })} + required + /> + + {form.discover_all_databases + ? "Connection database used to crawl all available databases on this instance." + : "Single database to monitor for this target."} +
@@ -313,6 +324,21 @@ export function TargetsPage() {
+
+ + +
div { + display: inline-flex; + align-items: center; + gap: 8px; +} + .dashboard-target-card:hover { transform: none; border-color: #3f79af; @@ -1813,6 +1843,11 @@ select:-webkit-autofill { flex-wrap: wrap; } +.target-db-switcher { + max-width: 420px; + margin-bottom: 10px; +} + .owner-pill { display: inline-flex; align-items: center;