From dfa73af1043162f8a89ed9c517796ef75a20c177 Mon Sep 17 00:00:00 2001 From: whyour Date: Tue, 18 Aug 2026 00:03:06 +0800 Subject: [PATCH] fix: preserve container startup diagnostics --- back/schedule/health.ts | 100 +++++++++++++++++++++++++---- ecosystem.config.js | 4 +- shell/check.sh | 76 ++++++++++++++++------ shell/lang/en.sh | 6 ++ shell/lang/zh.sh | 6 ++ test/back/ecosystem.test.cjs | 4 +- test/back/schedule-health.test.cjs | 86 +++++++++++++++++++++++++ 7 files changed, 249 insertions(+), 33 deletions(-) create mode 100644 test/back/schedule-health.test.cjs diff --git a/back/schedule/health.ts b/back/schedule/health.ts index dc90f7e6..04068c67 100644 --- a/back/schedule/health.ts +++ b/back/schedule/health.ts @@ -1,28 +1,104 @@ import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js'; +import fs from 'fs/promises'; +import path from 'path'; +import { request } from 'undici'; import { HealthCheckRequest, HealthCheckResponse } from '../protos/health'; import config from '../config'; -import { promiseExec } from '../config/util'; + +function formatError(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + + const detailedError = error as Error & { + cause?: unknown; + code?: string; + errors?: unknown[]; + }; + const details = [error.message, detailedError.code]; + if (Array.isArray(detailedError.errors)) { + details.push(...detailedError.errors.map(formatError)); + } else if (detailedError.cause) { + details.push(formatError(detailedError.cause)); + } + + return [...new Set(details.filter(Boolean))].join(': ') || error.name; +} + +async function getRecentSystemLog(lineLimit = 300): Promise { + try { + const entries = await fs.readdir(config.systemLogPath); + const latestLog = entries + .filter((entry) => entry.endsWith('.log')) + .sort() + .at(-1); + if (!latestLog) { + return `No system log found in ${config.systemLogPath}`; + } + const content = await fs.readFile( + path.join(config.systemLogPath, latestLog), + 'utf8', + ); + return content.split('\n').slice(-lineLimit).join('\n').trim(); + } catch (error) { + return `Unable to read system log from ${config.systemLogPath}: ${ + error instanceof Error ? error.message : String(error) + }`; + } +} const check = async ( call: ServerUnaryCall, callback: sendUnaryData, ) => { switch (call.request.service) { - case 'cron': - const res = await promiseExec( - `curl -s --noproxy '*' http://localhost:${config.port}/api/system`, - ); - - if (res.includes('200')) { - return callback(null, { status: 1 }); + case 'cron': { + const healthUrl = `http://localhost:${config.port}${ + config.baseUrl || '' + }/api/health`; + let failure = ''; + try { + const response = await request(healthUrl, { + method: 'GET', + headersTimeout: 5000, + bodyTimeout: 5000, + }); + const body = (await response.body.json()) as { + code?: number; + data?: { status?: string }; + }; + if ( + response.statusCode >= 200 && + response.statusCode < 300 && + body.code === 200 && + body.data?.status === 'ok' + ) { + return callback(null, { status: 1 }); + } + failure = `HTTP ${response.statusCode}: ${JSON.stringify(body)}`; + } catch (error) { + failure = formatError(error); } - const qinglongErrLog = await promiseExec( - `tail -n 300 ~/.pm2/logs/qinglong-error.log`, - ); + const systemLog = await getRecentSystemLog(); + const containerHint = + process.env.QL_CONTAINER === 'true' + ? 'PM2 file logging is disabled in containers. Check `docker logs ` for early startup errors.' + : 'Check `pm2 logs qinglong --lines 300` for early startup errors.'; return callback( - new Error(`${qinglongErrLog || ''}\n${res}`.trim()), + new Error( + [ + `HTTP health check failed: ${healthUrl}`, + failure, + containerHint, + `Recent system log (${config.systemLogPath}):`, + systemLog, + ] + .filter(Boolean) + .join('\n'), + ), ); + } default: return callback(null, { status: 1 }); diff --git a/ecosystem.config.js b/ecosystem.config.js index d7c150e2..dc9b1e91 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -11,7 +11,9 @@ module.exports = { source_map_support: true, time: !isContainer, out_file: isContainer ? '/dev/null' : undefined, - error_file: isContainer ? '/dev/null' : undefined, + // Do not persist PM2 logs in containers, but keep early startup errors + // visible through `docker logs` before Winston is initialized. + error_file: isContainer ? '/proc/1/fd/2' : undefined, script: 'static/build/app.js', env: { http_proxy: '', diff --git a/shell/check.sh b/shell/check.sh index ab01fe95..99925104 100755 --- a/shell/check.sh +++ b/shell/check.sh @@ -23,37 +23,75 @@ copy_dep() { } pm2_log() { - t '---> pm2日志' - local panelOut="/root/.pm2/logs/qinglong-out.log" - local panelError="/root/.pm2/logs/qinglong-error.log" - tail -n 300 "$panelOut" - tail -n 300 "$panelError" + t '---> 服务诊断信息' + pm2 status || true + + local systemLogDir="$dir_data/syslog" + local latestSystemLog + latestSystemLog=$(find "$systemLogDir" -maxdepth 1 -type f -name '*.log' 2>/dev/null | sort | tail -n 1) + if [[ -n "$latestSystemLog" ]]; then + t '---> 最近的系统日志: %s' "$latestSystemLog" + tail -n 300 "$latestSystemLog" + else + t '---> 未找到系统日志: %s' "$systemLogDir" + fi + + if [[ "$QL_CONTAINER" == "true" ]]; then + t '---> 容器内 PM2 日志不落盘；早期启动错误请执行 docker logs --tail 300 <容器名>' + return + fi + + local pm2Home="${PM2_HOME:-$HOME/.pm2}" + local panelOut="$pm2Home/logs/qinglong-out.log" + local panelError="$pm2Home/logs/qinglong-error.log" + [[ -f "$panelOut" ]] && tail -n 300 "$panelOut" + [[ -f "$panelError" ]] && tail -n 300 "$panelError" } check_ql() { - local api=$(curl -s --noproxy "*" "http://localhost:${ql_port}") + local basePath="${ql_base_url%/}" + local api + local attempt + for ((attempt = 1; attempt <= 10; attempt++)); do + api=$(curl -s --max-time 2 --noproxy "*" "http://localhost:${ql_port}${basePath}/") + [[ $api =~ "
" ]] && break + sleep 1 + done t '\n=====> 检测面板' echo -e "\n\n$api\n" if [[ $api =~ "
" ]]; then t '=====> 面板服务启动正常\n' + else + t '=====> 面板服务启动异常，请检查上方诊断信息\n' + return 1 fi } check_pm2() { - pm2_log local currentTimeStamp=$(date +%s) - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/api/system?t=$currentTimeStamp" \ - -H 'Accept: */*' \ - -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36' \ - -H "Referer: http://localhost:${ql_port}/crontab" \ - -H 'Accept-Language: en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7' \ - --compressed - ) + local basePath="${ql_base_url%/}" + local api + local attempt + for ((attempt = 1; attempt <= 10; attempt++)); do + api=$( + curl -s --max-time 2 --noproxy "*" "http://localhost:${ql_port}${basePath}/api/health?t=$currentTimeStamp" \ + -H 'Accept: */*' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36' \ + -H "Referer: http://localhost:${ql_port}/crontab" \ + -H 'Accept-Language: en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7' \ + --compressed + ) + [[ $api == *'"code":200'* && $api == *'"status":"ok"'* ]] && break + sleep 1 + done t '\n=====> 检测后台' echo -e "\n\n$api\n" - if [[ $api =~ "{\"code\"" ]]; then + if [[ $api == *'"code":200'* && $api == *'"status":"ok"'* ]]; then t '=====> 后台服务启动正常\n' + else + pm2_log + t '=====> 后台服务启动异常，请检查上方诊断信息\n' + return 1 fi } @@ -63,10 +101,12 @@ main() { reset_env copy_dep - check_ql - check_pm2 reload_pm2 + local checkStatus=0 + check_ql || checkStatus=1 + check_pm2 || checkStatus=1 t '\n=====> 检测结束\n' + return $checkStatus } main diff --git a/shell/lang/en.sh b/shell/lang/en.sh index 7e823d25..38778c9c 100644 --- a/shell/lang/en.sh +++ b/shell/lang/en.sh @@ -82,10 +82,16 @@ declare -gA LANG_MESSAGES=( ['---> 复制一份 %s 为 %s\n']='---> Copying %s to %s\n' ['---> 通知文件复制完成\n']='---> Notification files copied\n' ['---> pm2日志']='---> pm2 log' + ['---> 服务诊断信息']='---> Service diagnostics' + ['---> 最近的系统日志: %s']='---> Recent system log: %s' + ['---> 未找到系统日志: %s']='---> No system log found: %s' + ['---> 容器内 PM2 日志不落盘；早期启动错误请执行 docker logs --tail 300 <容器名>']='---> PM2 logs are not persisted in containers; run docker logs --tail 300 for early startup errors' ['\n=====> 检测面板']='\n=====> Checking panel' ['=====> 面板服务启动正常\n']='=====> Panel service running normally\n' + ['=====> 面板服务启动异常，请检查上方诊断信息\n']='=====> Panel failed to start; check the diagnostics above\n' ['\n=====> 检测后台']='\n=====> Checking backend' ['=====> 后台服务启动正常\n']='=====> Backend service running normally\n' + ['=====> 后台服务启动异常，请检查上方诊断信息\n']='=====> Backend failed to start; check the diagnostics above\n' ['=====> 开始检测']='=====> Starting check' ['\n=====> 检测结束\n']='\n=====> Check complete\n' # rmlog.sh diff --git a/shell/lang/zh.sh b/shell/lang/zh.sh index 0a3c668c..e1935675 100644 --- a/shell/lang/zh.sh +++ b/shell/lang/zh.sh @@ -82,10 +82,16 @@ declare -gA LANG_MESSAGES=( ['---> 复制一份 %s 为 %s\n']='---> 复制一份 %s 为 %s\n' ['---> 通知文件复制完成\n']='---> 通知文件复制完成\n' ['---> pm2日志']='---> pm2日志' + ['---> 服务诊断信息']='---> 服务诊断信息' + ['---> 最近的系统日志: %s']='---> 最近的系统日志: %s' + ['---> 未找到系统日志: %s']='---> 未找到系统日志: %s' + ['---> 容器内 PM2 日志不落盘；早期启动错误请执行 docker logs --tail 300 <容器名>']='---> 容器内 PM2 日志不落盘；早期启动错误请执行 docker logs --tail 300 <容器名>' ['\n=====> 检测面板']='\n=====> 检测面板' ['=====> 面板服务启动正常\n']='=====> 面板服务启动正常\n' + ['=====> 面板服务启动异常，请检查上方诊断信息\n']='=====> 面板服务启动异常，请检查上方诊断信息\n' ['\n=====> 检测后台']='\n=====> 检测后台' ['=====> 后台服务启动正常\n']='=====> 后台服务启动正常\n' + ['=====> 后台服务启动异常，请检查上方诊断信息\n']='=====> 后台服务启动异常，请检查上方诊断信息\n' ['=====> 开始检测']='=====> 开始检测' ['\n=====> 检测结束\n']='\n=====> 检测结束\n' # rmlog.sh diff --git a/test/back/ecosystem.test.cjs b/test/back/ecosystem.test.cjs index 34de98ab..155124c4 100644 --- a/test/back/ecosystem.test.cjs +++ b/test/back/ecosystem.test.cjs @@ -13,10 +13,10 @@ function loadConfig(containerValue) { return require(configPath).apps[0]; } -test('container logging relies on pm2-runtime stdout without persistent copies', () => { +test('container logging discards stdout and forwards startup errors without persistent copies', () => { const app = loadConfig('true'); assert.equal(app.out_file, '/dev/null'); - assert.equal(app.error_file, '/dev/null'); + assert.equal(app.error_file, '/proc/1/fd/2'); assert.equal(app.time, false); }); diff --git a/test/back/schedule-health.test.cjs b/test/back/schedule-health.test.cjs new file mode 100644 index 00000000..7a5eb62b --- /dev/null +++ b/test/back/schedule-health.test.cjs @@ -0,0 +1,86 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const config = require('../../back/config').default; +const { check } = require('../../back/schedule/health'); + +function runCheck(service = 'cron') { + return new Promise((resolve) => { + check({ request: { service } }, (error, response) => { + resolve({ error, response }); + }); + }); +} + +test('schedule health check uses the HTTP health endpoint and system logs', async (t) => { + const originalBaseUrl = config.baseUrl; + const originalPort = config.port; + const originalSystemLogPath = config.systemLogPath; + const originalContainer = process.env.QL_CONTAINER; + + t.after(() => { + config.baseUrl = originalBaseUrl; + config.port = originalPort; + config.systemLogPath = originalSystemLogPath; + if (originalContainer === undefined) { + delete process.env.QL_CONTAINER; + } else { + process.env.QL_CONTAINER = originalContainer; + } + }); + + await t.test( + 'returns serving for a healthy prefixed HTTP service', + async () => { + let requestedUrl = ''; + const server = http.createServer((request, response) => { + requestedUrl = request.url; + response.setHeader('Content-Type', 'application/json'); + response.end(JSON.stringify({ code: 200, data: { status: 'ok' } })); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => server.close(resolve))); + + config.baseUrl = '/ql'; + config.port = server.address().port; + + const result = await runCheck(); + assert.equal(result.error, null); + assert.deepEqual(result.response, { status: 1 }); + assert.equal(requestedUrl, '/ql/api/health'); + }, + ); + + await t.test( + 'reports recent system logs when HTTP startup fails', + async () => { + const logDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-health-')); + t.after(() => fs.rm(logDir, { recursive: true, force: true })); + const lines = Array.from( + { length: 305 }, + (_, index) => `line-${index + 1}`, + ); + await fs.writeFile(path.join(logDir, '2026-08-16.log'), 'older-log'); + await fs.writeFile(path.join(logDir, '2026-08-17.log'), lines.join('\n')); + config.systemLogPath = logDir; + config.port = 1; + process.env.QL_CONTAINER = 'true'; + + const result = await runCheck(); + assert.equal(result.response, undefined); + assert.match(result.error.message, /ECONNREFUSED|connect/); + assert.match( + result.error.message, + /http:\/\/localhost:1\/ql\/api\/health/, + ); + assert.match(result.error.message, /docker logs /); + assert.match(result.error.message, /line-305/); + assert.doesNotMatch(result.error.message, /line-1\n/); + assert.doesNotMatch(result.error.message, /qinglong-error\.log/); + }, + ); +});