feat: complete enterprise hardening and storage lock reliability

lib/storage.ts

@@ -109,6 +109,7 @@ export function formatStorageErrorHint(error: unknown, path: string): string {
 }
 
 let storageMutex: Promise<void> = Promise.resolve();
+let accountFileMutex: Promise<void> = Promise.resolve();
 
 function withStorageLock<T>(fn: () => Promise<T>): Promise<T> {
   const previousMutex = storageMutex;
@@ -119,6 +120,24 @@ function withStorageLock<T>(fn: () => Promise<T>): Promise<T> {
   return previousMutex.then(fn).finally(() => releaseLock());
 }
 
+function withAccountFileMutex<T>(fn: () => Promise<T>): Promise<T> {
+	const previousMutex = accountFileMutex;
+	let releaseLock: () => void;
+	accountFileMutex = new Promise<void>((resolve) => {
+		releaseLock = resolve;
+	});
+	return previousMutex.then(fn).finally(() => releaseLock());
+}
+
+async function withStorageSerializedFileLock<T>(path: string, fn: () => Promise<T>): Promise<T> {
+	// Serialize file-lock acquisition to keep save ordering deterministic, then
+	// preserve the historical lock order (file lock -> in-process mutex) so all
+	// account-storage mutation paths share the same acquisition sequence.
+	return withAccountFileMutex(() =>
+		withAccountFileLock(path, () => withStorageLock(fn)),
+	);
+}
+
 type AnyAccountStorage = AccountStorageV1 | AccountStorageV3;
 
 type AccountLike = {
@@ -318,16 +337,70 @@ function getAccountsLockPath(path: string): string {
 	return `${path}.lock`;
 }
 
+async function releaseStorageLockFallback(lockPath: string): Promise<void> {
+	try {
+		await fs.rm(lockPath, { force: true });
+	} catch {
+		// Best-effort lock cleanup fallback.
+	}
+}
+
+async function cleanupDeadProcessStorageLock(lockPath: string): Promise<void> {
+	try {
+		const raw = await fs.readFile(lockPath, "utf-8");
+		const parsed = JSON.parse(raw) as { pid?: number; acquiredAt?: number };
+		const lockPid = Number(parsed?.pid);
+		const lockAcquiredAt = Number(parsed?.acquiredAt);
+
+		if (Number.isFinite(lockPid) && lockPid > 0) {
+			let isDeadProcess = false;
+			try {
+				process.kill(lockPid, 0);
+			} catch (error) {
+				const code = (error as NodeJS.ErrnoException).code;
+				isDeadProcess = code === "ESRCH";
+			}
+
+			if (isDeadProcess) {
+				await releaseStorageLockFallback(lockPath);
+				return;
+			}
+		}
+
+		if (Number.isFinite(lockAcquiredAt) && Date.now() - lockAcquiredAt > ACCOUNT_STORAGE_LOCK_OPTIONS.staleAfterMs) {
+			await releaseStorageLockFallback(lockPath);
+		}
+	} catch (error) {
+		const code = (error as NodeJS.ErrnoException).code;
+		if (code === "ENOENT") {
+			return;
+		}
+		await releaseStorageLockFallback(lockPath);
+	}
+}
+
 async function withAccountFileLock<T>(path: string, fn: () => Promise<T>): Promise<T> {
+	const lockPath = getAccountsLockPath(path);
+	await cleanupDeadProcessStorageLock(lockPath);
 	await fs.mkdir(dirname(path), { recursive: true });
-	const lock = await acquireFileLock(getAccountsLockPath(path), ACCOUNT_STORAGE_LOCK_OPTIONS);
+	const lock = await acquireFileLock(lockPath, ACCOUNT_STORAGE_LOCK_OPTIONS);
 	try {
 		return await fn();
 	} finally {
-		await lock.release();
+		try {
+			await lock.release();
+		} catch (error) {
+			const code = (error as NodeJS.ErrnoException).code;
+			if (code !== "ENOENT") {
+				log.warn("Failed to release account storage lock", {
+					path: lockPath,
+					error: String(error),
+				});
+			}
+		}
+		await releaseStorageLockFallback(lockPath);
 	}
 }
-
 async function copyFileWithRetry(
 	sourcePath: string,
 	destinationPath: string,
@@ -1242,14 +1315,13 @@ export async function withAccountStorageTransaction<T>(
   ) => Promise<T>,
 ): Promise<T> {
 	const path = getStoragePath();
-	return withAccountFileLock(path, () =>
-		withStorageLock(async () => {
-			const current = await loadAccountsInternal(saveAccountsUnlocked);
-			return handler(current, saveAccountsUnlocked);
-		}),
-	);
+	return withStorageSerializedFileLock(path, async () => {
+		const current = await loadAccountsInternal(saveAccountsUnlocked);
+		return handler(current, saveAccountsUnlocked);
+	});
 }
 
+
 /**
  * Persists account storage to disk using atomic write (temp file + rename).
  * Creates the Codex multi-auth storage directory if it doesn't exist.
@@ -1259,23 +1331,21 @@ export async function withAccountStorageTransaction<T>(
  */
 export async function saveAccounts(storage: AccountStorageV3): Promise<void> {
 	const path = getStoragePath();
-	return withAccountFileLock(path, () =>
-		withStorageLock(async () => {
-			await saveAccountsUnlocked(storage);
-		}),
-	);
+	return withStorageSerializedFileLock(path, async () => {
+		await saveAccountsUnlocked(storage);
+	});
 }
 
+
 /**
  * Deletes the account storage file from disk.
  * Silently ignores if file doesn't exist.
  */
 export async function clearAccounts(): Promise<void> {
 	const path = getStoragePath();
-	return withAccountFileLock(path, () =>
-		withStorageLock(async () => {
-			const walPath = getAccountsWalPath(path);
-	const backupPaths = getAccountsBackupRecoveryCandidates(path);
+	return withStorageSerializedFileLock(path, async () => {
+		const walPath = getAccountsWalPath(path);
+		const backupPaths = getAccountsBackupRecoveryCandidates(path);
     const clearPath = async (targetPath: string): Promise<void> => {
       try {
         await fs.unlink(targetPath);
@@ -1295,10 +1365,10 @@ export async function clearAccounts(): Promise<void> {
     } catch {
       // Individual path cleanup is already best-effort with per-artifact logging.
     }
-		}),
-	);
+	});
 }
 
+
 function normalizeFlaggedStorage(data: unknown): FlaggedAccountStorageV1 {
 	if (!isRecord(data) || data.version !== 1 || !Array.isArray(data.accounts)) {
 		return { version: 1, accounts: [] };
@@ -1597,3 +1667,5 @@ export async function rotateStoredSecretEncryption(): Promise<{
 		flaggedAccounts: flaggedCount,
 	};
 }
+
+