lomavuokraus/lib/monitoring.ts
Tero Halla-aho 0bb709d9c5
Some checks failed
CI / checks (push) Has been cancelled
chore: fix audit alerts and formatting
2026-02-04 12:43:03 +02:00

358 lines
10 KiB
TypeScript

import fs from "fs";
import https from "https";
import { prisma } from "./prisma";
type HetznerServerSummary = {
id: number;
name: string;
status: string;
type?: string;
datacenter?: string;
publicIp?: string;
privateIp?: string;
created?: string;
};
export type HetznerStatus = {
ok: boolean;
error?: string;
missingToken?: boolean;
servers?: HetznerServerSummary[];
};
type KubernetesClientConfig = {
server: string;
token?: string;
ca?: string;
insecureSkipTlsVerify?: boolean;
};
type KubernetesNodeSummary = {
name: string;
ready: boolean;
status: string;
roles: string[];
internalIp?: string;
kubeletVersion?: string;
osImage?: string;
lastReadyTransition?: string | null;
};
type KubernetesPodContainer = {
name: string;
ready: boolean;
restartCount: number;
state: string;
lastState?: string | null;
};
type KubernetesPodSummary = {
name: string;
namespace: string;
phase: string;
reason?: string | null;
message?: string | null;
readyCount: number;
totalContainers: number;
restarts: number;
nodeName?: string;
hostIP?: string | null;
podIP?: string | null;
startedAt?: string | null;
containers: KubernetesPodContainer[];
};
export type KubernetesStatus = {
ok: boolean;
error?: string;
nodes?: KubernetesNodeSummary[];
pods?: KubernetesPodSummary[];
};
export type DbStatus = {
ok: boolean;
error?: string;
serverTime?: string;
recovery?: boolean;
databaseSizeBytes?: number;
connections?: { state: string; count: number }[];
};
function readFileSafe(path: string): string | null {
try {
return fs.readFileSync(path, "utf8");
} catch {
return null;
}
}
export async function fetchHetznerServers(): Promise<HetznerStatus> {
const token = process.env.HCLOUD_TOKEN ?? process.env.HETZNER_TOKEN;
if (!token) {
return {
ok: false,
missingToken: true,
error: "HCLOUD_TOKEN not configured",
};
}
try {
const res = await fetch("https://api.hetzner.cloud/v1/servers", {
headers: { Authorization: `Bearer ${token}` },
signal: AbortSignal.timeout(8000),
});
if (!res.ok) {
const body = await res.text();
return {
ok: false,
error: `Hetzner API ${res.status}: ${body.slice(0, 200)}`,
};
}
const json = (await res.json()) as { servers?: any[] };
const servers =
json.servers?.map((s) => ({
id: s.id,
name: s.name,
status: s.status,
type: s.server_type?.name,
datacenter:
s.datacenter?.name ??
s.datacenter?.description ??
s.datacenter?.location?.name,
publicIp: s.public_net?.ipv4?.ip,
privateIp: s.private_net?.[0]?.ip,
created: s.created,
})) ?? [];
return { ok: true, servers };
} catch (error: any) {
return { ok: false, error: error?.message ?? "Hetzner API request failed" };
}
}
function loadKubernetesConfig(): KubernetesClientConfig | null {
const token = readFileSafe(
"/var/run/secrets/kubernetes.io/serviceaccount/token",
);
const ca = readFileSafe(
"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
);
const serviceHost =
process.env.KUBERNETES_SERVICE_HOST ?? "kubernetes.default.svc";
const servicePort = process.env.KUBERNETES_SERVICE_PORT ?? "443";
if (token) {
return {
server: `https://${serviceHost}:${servicePort}`,
token: token.trim(),
ca: ca ?? undefined,
};
}
if (process.env.K8S_API_SERVER && process.env.K8S_BEARER_TOKEN) {
const caCert = process.env.K8S_CA_CERT;
return {
server: process.env.K8S_API_SERVER.replace(/\/$/, ""),
token: process.env.K8S_BEARER_TOKEN,
ca: caCert,
insecureSkipTlsVerify: process.env.K8S_INSECURE_SKIP_TLS === "true",
};
}
return null;
}
function k8sRequest(path: string, cfg: KubernetesClientConfig): Promise<any> {
const url = `${cfg.server}${path}`;
const headers: Record<string, string> = { Accept: "application/json" };
if (cfg.token) headers.Authorization = `Bearer ${cfg.token}`;
return new Promise((resolve, reject) => {
const req = https.request(
url,
{
method: "GET",
headers,
ca: cfg.ca,
rejectUnauthorized: cfg.insecureSkipTlsVerify ? false : true,
timeout: 7000,
},
(res) => {
const chunks: Buffer[] = [];
res.on("data", (d) =>
chunks.push(typeof d === "string" ? Buffer.from(d) : d),
);
res.on("end", () => {
const body = Buffer.concat(chunks).toString("utf8");
if (res.statusCode && res.statusCode >= 400) {
reject(
new Error(`Kubernetes ${res.statusCode}: ${body.slice(0, 200)}`),
);
return;
}
try {
resolve(body ? JSON.parse(body) : {});
} catch (err) {
reject(err);
}
});
},
);
req.on("error", reject);
req.on("timeout", () =>
req.destroy(new Error("Kubernetes request timed out")),
);
req.end();
});
}
function parseK8sNodes(data: any): KubernetesNodeSummary[] {
const items: any[] = data?.items ?? [];
return items.map((node) => {
const labels = node?.metadata?.labels ?? {};
const roles = Object.keys(labels)
.filter((key) => key.startsWith("node-role.kubernetes.io/"))
.map(
(key) => key.replace("node-role.kubernetes.io/", "") || "control-plane",
);
const readyCondition = (node?.status?.conditions ?? []).find(
(c: any) => c.type === "Ready",
);
const addresses = node?.status?.addresses ?? [];
const internal = addresses.find(
(a: any) => a.type === "InternalIP",
)?.address;
return {
name: node?.metadata?.name ?? "unknown",
ready: readyCondition?.status === "True",
status:
readyCondition?.status === "True"
? "Ready"
: (readyCondition?.reason ?? "NotReady"),
roles,
internalIp: internal,
kubeletVersion: node?.status?.nodeInfo?.kubeletVersion,
osImage: node?.status?.nodeInfo?.osImage,
lastReadyTransition: readyCondition?.lastTransitionTime ?? null,
};
});
}
function parseContainerState(status: any): string {
if (!status) return "unknown";
if (status.state?.running) return "running";
if (status.state?.waiting?.reason) return status.state.waiting.reason;
if (status.state?.terminated?.reason) return status.state.terminated.reason;
if (status.state?.waiting) return "waiting";
if (status.state?.terminated) return "terminated";
return "unknown";
}
function parseLastState(status: any): string | null {
if (!status?.lastState) return null;
const { lastState } = status;
if (lastState.terminated?.reason)
return `terminated: ${lastState.terminated.reason}`;
if (lastState.terminated?.exitCode !== undefined)
return `terminated: code ${lastState.terminated.exitCode}`;
if (lastState.waiting?.reason) return `waiting: ${lastState.waiting.reason}`;
return "previous state recorded";
}
function parseK8sPods(data: any): KubernetesPodSummary[] {
const items: any[] = data?.items ?? [];
return items
.filter((pod) => {
const ns = pod?.metadata?.namespace ?? "";
return ns.startsWith("lomavuokraus-") || ns === "default";
})
.map((pod) => {
const containers: KubernetesPodContainer[] = (
pod?.status?.containerStatuses ?? []
).map((c: any) => ({
name: c.name,
ready: Boolean(c.ready),
restartCount: Number(c.restartCount ?? 0),
state: parseContainerState(c),
lastState: parseLastState(c),
}));
const readyCount = containers.filter((c) => c.ready).length;
const restarts = containers.reduce((sum, c) => sum + c.restartCount, 0);
return {
name: pod?.metadata?.name ?? "pod",
namespace: pod?.metadata?.namespace ?? "unknown",
phase: pod?.status?.phase ?? "Unknown",
reason: pod?.status?.reason ?? null,
message: pod?.status?.message ?? null,
readyCount,
totalContainers: containers.length,
restarts,
nodeName: pod?.spec?.nodeName,
hostIP: pod?.status?.hostIP ?? null,
podIP: pod?.status?.podIP ?? null,
startedAt: pod?.status?.startTime ?? null,
containers,
};
});
}
export async function fetchKubernetesStatus(): Promise<KubernetesStatus> {
const config = loadKubernetesConfig();
if (!config) {
return {
ok: false,
error:
"Kubernetes config not found (service account or env K8S_API_SERVER/K8S_BEARER_TOKEN required)",
};
}
try {
const [nodes, pods] = await Promise.all([
k8sRequest("/api/v1/nodes", config),
k8sRequest("/api/v1/pods", config),
]);
return { ok: true, nodes: parseK8sNodes(nodes), pods: parseK8sPods(pods) };
} catch (error: any) {
return { ok: false, error: error?.message ?? "Kubernetes query failed" };
}
}
export async function fetchDbStatus(): Promise<DbStatus> {
try {
const [summary, activity] = await Promise.all([
prisma.$queryRaw<
{ server_time: Date; recovery: boolean; size_bytes: bigint | number }[]
>`
SELECT now() as server_time, pg_is_in_recovery() as recovery, pg_database_size(current_database()) as size_bytes
`,
prisma.$queryRaw<{ state: string | null; count: number }[]>`
SELECT COALESCE(state, 'unknown') as state, count(*)::int as count FROM pg_stat_activity GROUP BY state
`,
]);
const row = summary?.[0];
const serverTime =
row?.server_time instanceof Date
? row.server_time.toISOString()
: row?.server_time
? String(row.server_time)
: undefined;
const sizeVal = row?.size_bytes ?? 0;
const sizeNumber =
typeof sizeVal === "bigint" ? Number(sizeVal) : Number(sizeVal);
const connections =
activity?.map((a) => ({
state: a.state ?? "unknown",
count: Number(a.count ?? 0),
})) ?? [];
return {
ok: true,
serverTime,
recovery: Boolean(row?.recovery),
databaseSizeBytes: Number.isFinite(sizeNumber) ? sizeNumber : undefined,
connections,
};
} catch (error: any) {
return { ok: false, error: error?.message ?? "Database query failed" };
}
}