|
@@ -17,14 +17,59 @@ interface ServiceProcess {
|
|
|
name: string;
|
|
name: string;
|
|
|
port: number;
|
|
port: number;
|
|
|
ready: boolean;
|
|
ready: boolean;
|
|
|
|
|
+ /** 重启跟踪 */
|
|
|
|
|
+ restartCount: number;
|
|
|
|
|
+ lastRestartTime: number;
|
|
|
|
|
+ /** Python 内存监控 */
|
|
|
|
|
+ lastMemoryMB?: number;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/** 重启管理器:防止频繁重启被系统杀后无限循环 */
|
|
|
|
|
+class RestartManager {
|
|
|
|
|
+ private counts: Map<string, number[]> = new Map();
|
|
|
|
|
+
|
|
|
|
|
+ /**
|
|
|
|
|
+ * 记录一次退出,返回是否允许重启
|
|
|
|
|
+ * @param key 服务标识
|
|
|
|
|
+ * @param maxRestarts 允许的最大重启次数
|
|
|
|
|
+ * @param windowMs 时间窗口(毫秒)
|
|
|
|
|
+ */
|
|
|
|
|
+ canRestart(key: string, maxRestarts = 5, windowMs = 5 * 60 * 1000): boolean {
|
|
|
|
|
+ const now = Date.now();
|
|
|
|
|
+ const times = this.counts.get(key) || [];
|
|
|
|
|
+ // 清理窗口外的记录
|
|
|
|
|
+ const recent = times.filter(t => now - t < windowMs);
|
|
|
|
|
+ recent.push(now);
|
|
|
|
|
+ this.counts.set(key, recent);
|
|
|
|
|
+ const allowed = recent.length <= maxRestarts;
|
|
|
|
|
+ if (!allowed) {
|
|
|
|
|
+ log('WARN', `[RestartManager] ${key} 重启过于频繁(${recent.length}次/${windowMs / 1000}s),暂停重启`);
|
|
|
|
|
+ }
|
|
|
|
|
+ return allowed;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ /** 获取某个服务的重启间隔(指数退避) */
|
|
|
|
|
+ getBackoffMs(key: string, baseMs = 5000): number {
|
|
|
|
|
+ const times = this.counts.get(key) || [];
|
|
|
|
|
+ const recent = times.filter(t => Date.now() - t < 300000); // 5分钟内
|
|
|
|
|
+ // 指数退避:5s, 10s, 20s, 40s, 60s(最多60s)
|
|
|
|
|
+ const backoff = Math.min(baseMs * Math.pow(2, recent.length - 1), 60000);
|
|
|
|
|
+ return backoff;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ reset(key: string): void {
|
|
|
|
|
+ this.counts.delete(key);
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+const restartMgr = new RestartManager();
|
|
|
|
|
+
|
|
|
const NODE_PORT = 14890;
|
|
const NODE_PORT = 14890;
|
|
|
const PYTHON_PORT = 14981;
|
|
const PYTHON_PORT = 14981;
|
|
|
|
|
|
|
|
const services: { node: ServiceProcess; python: ServiceProcess } = {
|
|
const services: { node: ServiceProcess; python: ServiceProcess } = {
|
|
|
- node: { process: null, name: 'Node Server', port: NODE_PORT, ready: false },
|
|
|
|
|
- python: { process: null, name: 'Python Service', port: PYTHON_PORT, ready: false },
|
|
|
|
|
|
|
+ node: { process: null, name: 'Node Server', port: NODE_PORT, ready: false, restartCount: 0, lastRestartTime: 0 },
|
|
|
|
|
+ python: { process: null, name: 'Python Service', port: PYTHON_PORT, ready: false, restartCount: 0, lastRestartTime: 0 },
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
// ==================== 文件日志 ====================
|
|
// ==================== 文件日志 ====================
|
|
@@ -216,6 +261,8 @@ function startNodeServer(): void {
|
|
|
HOST: '127.0.0.1',
|
|
HOST: '127.0.0.1',
|
|
|
PYTHON_PUBLISH_SERVICE_URL: `http://127.0.0.1:${PYTHON_PORT}`,
|
|
PYTHON_PUBLISH_SERVICE_URL: `http://127.0.0.1:${PYTHON_PORT}`,
|
|
|
USE_REDIS_QUEUE: 'false',
|
|
USE_REDIS_QUEUE: 'false',
|
|
|
|
|
+ // 限制 Node 服务端最大堆内存 768MB,防止无限增长被系统杀
|
|
|
|
|
+ NODE_OPTIONS: '--max-old-space-size=768',
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
// 打包模式下,uploads 目录使用 userData(可写),避免 Program Files 权限问题
|
|
// 打包模式下,uploads 目录使用 userData(可写),避免 Program Files 权限问题
|
|
@@ -270,6 +317,22 @@ function startNodeServer(): void {
|
|
|
log('INFO', `Node 服务退出, code=${code}, signal=${signal}`);
|
|
log('INFO', `Node 服务退出, code=${code}, signal=${signal}`);
|
|
|
services.node.process = null;
|
|
services.node.process = null;
|
|
|
services.node.ready = false;
|
|
services.node.ready = false;
|
|
|
|
|
+
|
|
|
|
|
+ // ===== 自动重启:被系统杀(code非0)或异常退出 =====
|
|
|
|
|
+ const key = 'node-server';
|
|
|
|
|
+ const isUnexpected = code !== 0 || signal !== null;
|
|
|
|
|
+ if (isUnexpected && restartMgr.canRestart(key)) {
|
|
|
|
|
+ const backoff = restartMgr.getBackoffMs(key);
|
|
|
|
|
+ log('WARN', `[Node] 异常退出,${backoff / 1000}s 后自动重启...`);
|
|
|
|
|
+ setTimeout(() => {
|
|
|
|
|
+ restartMgr.reset(key);
|
|
|
|
|
+ log('INFO', `[Node] 自动重启第 ${services.node.restartCount + 1} 次`);
|
|
|
|
|
+ startNodeServer();
|
|
|
|
|
+ }, backoff);
|
|
|
|
|
+ services.node.restartCount++;
|
|
|
|
|
+ } else if (!restartMgr.canRestart(key)) {
|
|
|
|
|
+ log('ERROR', '[Node] 重启次数过多,停止自动重启,请检查问题');
|
|
|
|
|
+ }
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
services.node.process = child;
|
|
services.node.process = child;
|
|
@@ -296,6 +359,7 @@ function startPythonService(): void {
|
|
|
...process.env as Record<string, string>,
|
|
...process.env as Record<string, string>,
|
|
|
PYTHONUNBUFFERED: '1',
|
|
PYTHONUNBUFFERED: '1',
|
|
|
PYTHONIOENCODING: 'utf-8',
|
|
PYTHONIOENCODING: 'utf-8',
|
|
|
|
|
+ PYTHONDONTWRITEBYTECODE: '1',
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
const pwPath = getPlaywrightBrowsersPath();
|
|
const pwPath = getPlaywrightBrowsersPath();
|
|
@@ -336,14 +400,96 @@ function startPythonService(): void {
|
|
|
log('INFO', `Python 服务退出, code=${code}, signal=${signal}`);
|
|
log('INFO', `Python 服务退出, code=${code}, signal=${signal}`);
|
|
|
services.python.process = null;
|
|
services.python.process = null;
|
|
|
services.python.ready = false;
|
|
services.python.ready = false;
|
|
|
|
|
+
|
|
|
|
|
+ // ===== 自动重启:被系统杀或异常退出 =====
|
|
|
|
|
+ const key = 'python-service';
|
|
|
|
|
+ const isUnexpected = code !== 0 || signal !== null;
|
|
|
|
|
+ if (isUnexpected && restartMgr.canRestart(key)) {
|
|
|
|
|
+ const backoff = restartMgr.getBackoffMs(key);
|
|
|
|
|
+ log('WARN', `[Python] 异常退出,${backoff / 1000}s 后自动重启...`);
|
|
|
|
|
+ setTimeout(() => {
|
|
|
|
|
+ restartMgr.reset(key);
|
|
|
|
|
+ log('INFO', `[Python] 自动重启第 ${services.python.restartCount + 1} 次`);
|
|
|
|
|
+ startPythonService();
|
|
|
|
|
+ }, backoff);
|
|
|
|
|
+ services.python.restartCount++;
|
|
|
|
|
+ } else if (!restartMgr.canRestart(key)) {
|
|
|
|
|
+ log('ERROR', '[Python] 重启次数过多,停止自动重启,请检查问题');
|
|
|
|
|
+ }
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
services.python.process = child;
|
|
services.python.process = child;
|
|
|
|
|
+
|
|
|
|
|
+ // ===== Python 内存监控(Windows) =====
|
|
|
|
|
+ if (process.platform === 'win32') {
|
|
|
|
|
+ startPythonMemoryMonitor(child.pid);
|
|
|
|
|
+ }
|
|
|
} catch (e: any) {
|
|
} catch (e: any) {
|
|
|
log('ERROR', `Python spawn 异常: ${e.message}`);
|
|
log('ERROR', `Python spawn 异常: ${e.message}`);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+/** 监控 Python 进程内存,超过阈值则重启 */
|
|
|
|
|
+let pythonMemInterval: NodeJS.Timeout | null = null;
|
|
|
|
|
+const PYTHON_MEM_THRESHOLD_MB = 800;
|
|
|
|
|
+
|
|
|
|
|
+function startPythonMemoryMonitor(pid: number): void {
|
|
|
|
|
+ // 清除旧的监控
|
|
|
|
|
+ if (pythonMemInterval) {
|
|
|
|
|
+ clearInterval(pythonMemInterval);
|
|
|
|
|
+ pythonMemInterval = null;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ pythonMemInterval = setInterval(async () => {
|
|
|
|
|
+ const proc = services.python?.process;
|
|
|
|
|
+ if (!proc || proc.pid !== pid || proc.killed) {
|
|
|
|
|
+ clearInterval(pythonMemInterval!);
|
|
|
|
|
+ pythonMemInterval = null;
|
|
|
|
|
+ return;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ try {
|
|
|
|
|
+ const memMB = await getProcessMemoryMB(pid);
|
|
|
|
|
+ services.python.lastMemoryMB = memMB;
|
|
|
|
|
+ if (memMB > PYTHON_MEM_THRESHOLD_MB) {
|
|
|
|
|
+ log('WARN', `[Python] 内存占用过高(${memMB}MB > ${PYTHON_MEM_THRESHOLD_MB}MB),准备重启...`);
|
|
|
|
|
+ proc.kill('SIGTERM');
|
|
|
|
|
+ // 给2秒优雅退出,然后强制杀
|
|
|
|
|
+ setTimeout(() => {
|
|
|
|
|
+ if (!proc.killed) {
|
|
|
|
|
+ log('WARN', '[Python] 优雅退出超时,强制杀进程');
|
|
|
|
|
+ try { proc.kill('SIGKILL'); } catch {}
|
|
|
|
|
+ }
|
|
|
|
|
+ }, 2000);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (e) {
|
|
|
|
|
+ // 进程可能已结束,忽略
|
|
|
|
|
+ }
|
|
|
|
|
+ }, 60000); // 每60秒检查一次
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/** 通过 tasklist 获取进程内存(Windows,MB) */
|
|
|
|
|
+function getProcessMemoryMB(pid: number): Promise<number> {
|
|
|
|
|
+ return new Promise((resolve) => {
|
|
|
|
|
+ const child = spawn('tasklist', ['/FI', `PID eq ${pid}`, '/FO', 'CSV', '/NH'], {
|
|
|
|
|
+ windowsHide: true,
|
|
|
|
|
+ });
|
|
|
|
|
+ let stdout = '';
|
|
|
|
|
+ child.stdout?.on('data', (d: Buffer) => { stdout += d.toString(); });
|
|
|
|
|
+ child.on('close', () => {
|
|
|
|
|
+ // 格式: "python.exe","1234","Console","1,234 KB"
|
|
|
|
|
+ const match = stdout.match(/"(\d+[\d,]*) KB"/);
|
|
|
|
|
+ if (match) {
|
|
|
|
|
+ const kb = parseInt(match[1].replace(/,/g, ''), 10);
|
|
|
|
|
+ resolve(Math.round(kb / 1024));
|
|
|
|
|
+ } else {
|
|
|
|
|
+ resolve(0);
|
|
|
|
|
+ }
|
|
|
|
|
+ });
|
|
|
|
|
+ child.on('error', () => resolve(0));
|
|
|
|
|
+ });
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
// ==================== 导出 API ====================
|
|
// ==================== 导出 API ====================
|
|
|
|
|
|
|
|
export async function startLocalServices(): Promise<{ nodeOk: boolean; pythonOk: boolean }> {
|
|
export async function startLocalServices(): Promise<{ nodeOk: boolean; pythonOk: boolean }> {
|
|
@@ -394,6 +540,12 @@ export async function startLocalServices(): Promise<{ nodeOk: boolean; pythonOk:
|
|
|
export function stopLocalServices(): void {
|
|
export function stopLocalServices(): void {
|
|
|
log('INFO', '正在停止本地服务...');
|
|
log('INFO', '正在停止本地服务...');
|
|
|
|
|
|
|
|
|
|
+ // 清除 Python 内存监控定时器
|
|
|
|
|
+ if (pythonMemInterval) {
|
|
|
|
|
+ clearInterval(pythonMemInterval);
|
|
|
|
|
+ pythonMemInterval = null;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
for (const key of ['node', 'python'] as const) {
|
|
for (const key of ['node', 'python'] as const) {
|
|
|
const svc = services[key];
|
|
const svc = services[key];
|
|
|
if (svc.process) {
|
|
if (svc.process) {
|