tune shutdown and runner cleanup

scripts/dev-services.ts

@@ -1,3 +1,6 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+
 type ChildSpec = {
   name: string;
   cmd: string[];
@@ -11,6 +14,10 @@ type Child = {
 
 const children: Child[] = [];
 let shuttingDown = false;
+let shutdownPromise: Promise<void> | null = null;
+let forceShutdownPromise: Promise<void> | null = null;
+const stateDir = path.join(process.cwd(), ".tmp");
+const pidFile = path.join(stateDir, "dev-services-runner-pids.json");
 
 const sleep = (delayMs: number): Promise<void> => {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
@@ -24,6 +31,26 @@ const waitForExit = async (proc: Bun.Subprocess, timeoutMs: number): Promise<boo
   return result;
 };
 
+const isPidRunning = (pid: number): boolean => {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const waitForPidExit = async (pid: number, timeoutMs: number): Promise<boolean> => {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (!isPidRunning(pid)) {
+      return true;
+    }
+    await sleep(100);
+  }
+  return !isPidRunning(pid);
+};
+
 const signalProcess = (pid: number, signal: NodeJS.Signals): boolean => {
   try {
     process.kill(-pid, signal);
@@ -43,12 +70,15 @@ const stopChild = async (child: Child, timeoutMs = 5000): Promise<void> => {
   if (!pid) {
     return;
   }
+  await stopPid(pid, timeoutMs);
+};
 
+const stopPid = async (pid: number, timeoutMs = 5000): Promise<void> => {
   if (!signalProcess(pid, "SIGINT")) {
     return;
   }
 
-  const exited = await waitForExit(child.process, timeoutMs);
+  const exited = await waitForPidExit(pid, timeoutMs);
   if (exited) {
     return;
   }
@@ -57,19 +87,62 @@ const stopChild = async (child: Child, timeoutMs = 5000): Promise<void> => {
     return;
   }
 
-  await waitForExit(child.process, 2000);
+  await waitForPidExit(pid, 2000);
+};
+
+const persistChildren = async (): Promise<void> => {
+  await mkdir(stateDir, { recursive: true });
+  const payload = children
+    .map((child) => {
+      const pid = child.process.pid;
+      return pid ? { name: child.name, pid } : null;
+    })
+    .filter((value): value is { name: string; pid: number } => value !== null);
+  await writeFile(pidFile, JSON.stringify(payload, null, 2));
+};
+
+const clearPersistedChildren = async (): Promise<void> => {
+  await rm(pidFile, { force: true });
+};
+
+const cleanupStaleChildren = async (): Promise<void> => {
+  try {
+    const raw = await readFile(pidFile, "utf8");
+    const recorded = JSON.parse(raw) as Array<{ name?: string; pid?: number }>;
+    const stale = recorded.filter(
+      (entry): entry is { name: string; pid: number } =>
+        typeof entry?.name === "string" && typeof entry?.pid === "number" && isPidRunning(entry.pid)
+    );
+
+    if (stale.length > 0) {
+      console.log(
+        `[dev-services] Cleaning up stale processes from previous run: ${stale
+          .map((entry) => `${entry.name}(${entry.pid})`)
+          .join(", ")}`
+      );
+    }
+
+    for (const entry of stale) {
+      await stopPid(entry.pid, 3000);
+    }
+  } catch {
+    // No persisted children from a prior run.
+  } finally {
+    await clearPersistedChildren();
+  }
 };
 
 const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
   const proc = Bun.spawn(cmd, {
     cwd,
+    detached: true,
     stdin: "inherit",
     stdout: "inherit",
-    stderr: "inherit",
-    detached: true
+    stderr: "inherit"
   });
 
   children.push({ name, process: proc });
+  void persistChildren();
 
   proc.exited.then((code) => {
     if (shuttingDown) {
@@ -83,22 +156,68 @@ const spawnChild = ({ name, cmd, cwd }: ChildSpec): void => {
   });
 };
 
-const shutdown = async (code: number): Promise<void> => {
-  if (shuttingDown) {
-    return;
+const forceShutdown = async (code: number): Promise<void> => {
+  if (forceShutdownPromise) {
+    return forceShutdownPromise;
   }
 
   shuttingDown = true;
+  forceShutdownPromise = (async () => {
+    await Promise.all(
+      children.map(async (child) => {
+        const pid = child.process.pid;
+        if (!pid) {
+          return;
+        }
 
-  if (children.length > 0) {
-    await Promise.all(children.map((child) => stopChild(child)));
-  }
+        if (!signalProcess(pid, "SIGKILL")) {
+          return;
+        }
 
-  process.exit(code);
+        await waitForPidExit(pid, 2000);
+      })
+    );
+
+    await clearPersistedChildren();
+    process.exit(code);
+  })();
+
+  return forceShutdownPromise;
 };
 
-process.on("SIGINT", () => void shutdown(0));
-process.on("SIGTERM", () => void shutdown(0));
+const shutdown = async (code: number): Promise<void> => {
+  if (shutdownPromise) {
+    return shutdownPromise;
+  }
+
+  shuttingDown = true;
+  shutdownPromise = (async () => {
+    if (children.length > 0) {
+      await Promise.all(children.map((child) => stopChild(child)));
+    }
+
+    await clearPersistedChildren();
+    process.exit(code);
+  })();
+
+  return shutdownPromise;
+};
+
+const handleSignal = (signal: NodeJS.Signals) => {
+  if (shuttingDown) {
+    if (signal === "SIGINT") {
+      console.error("[dev-services] Force shutdown requested. Terminating remaining processes.");
+      void forceShutdown(130);
+    }
+    return;
+  }
+
+  void shutdown(0);
+};
+
+process.on("SIGINT", () => handleSignal("SIGINT"));
+process.on("SIGTERM", () => handleSignal("SIGTERM"));
+process.on("SIGHUP", () => handleSignal("SIGHUP"));
 
 const tasks: ChildSpec[] = [
   { name: "ingest-options", cmd: ["bun", "run", "dev"], cwd: "services/ingest-options" },
@@ -110,6 +229,7 @@ const tasks: ChildSpec[] = [
   { name: "api", cmd: ["bun", "run", "dev"], cwd: "services/api" }
 ];
 
+await cleanupStaleChildren();
 for (const task of tasks) {
   spawnChild(task);
 }