fix: verify staged upgrades in the background before restarting

back/app.ts

@@ -41,11 +41,18 @@ class Application {
   }
 
   private startMasterProcess() {
+    const readyTimeout =
+      process.arch === 'arm' &&
+      Number(
+        (process.config.variables as { arm_version?: string | number }).arm_version,
+      ) === 6
+        ? 120000
+        : 30000;
     // Fork gRPC worker first and wait for it to be ready
     const grpcWorker = this.forkWorker('grpc');
     
     // Wait for gRPC worker to signal it's ready before starting HTTP worker
-    this.waitForWorkerReady(grpcWorker, 30000)
+    this.waitForWorkerReady(grpcWorker, readyTimeout)
       .then(() => {
         Logger.info('[boot] gRPC worker is ready, starting HTTP worker');
         this.httpWorker = this.forkWorker('http');
@@ -71,7 +78,7 @@ class Application {
               Logger.warn('Unable to notify HTTP worker of scheduler exit');
             }
             const newGrpcWorker = this.forkWorker('grpc');
-            this.waitForWorkerReady(newGrpcWorker, 30000)
+            this.waitForWorkerReady(newGrpcWorker, readyTimeout)
               .then(() => {
                 Logger.info('gRPC worker restarted and ready');
                 // Re-register cron jobs by notifying the HTTP worker

cli/src/internal/maintenance/panelUpgrade.cjs

@@ -221,12 +221,14 @@ async function startJob({ root, tmp, env = process.env }, action) {
           let selected;
           let metadata;
           if (action === 'apply') {
-            // Fail before launching a process or stopping the working panel.
+            // Keep requests short on slow devices. The detached worker performs
+            // complete file/dependency verification before it can stop the panel.
             try {
               selected = await artifacts.selectedUpgrade(
                 tmp,
                 artifacts.releaseBranch(env),
                 env,
+                { metadataOnly: true },
               );
               metadata = await stagedMetadata(selected);
               if (previous.status === 'ready')
@@ -391,7 +393,7 @@ async function runJob({ root, tmp, env = process.env }, id) {
                       );
                       await artifacts.waitForHealth(
                         env,
-                        60000,
+                        undefined,
                         recover ? undefined : controller.signal,
                       );
                       if (!recover)

cli/src/internal/maintenance/upgradeArtifacts.cjs

@@ -495,7 +495,12 @@ async function stageUpgrade({
   }
 }
 
-async function selectedUpgrade(tmp, branch, env = process.env) {
+async function selectedUpgrade(
+  tmp,
+  branch,
+  env = process.env,
+  { metadataOnly = false } = {},
+) {
   const pointer = JSON.parse(
     await fs.readFile(path.join(tmp, `upgrade-ready-${branch}.json`), 'utf8'),
   );
@@ -519,10 +524,12 @@ async function selectedUpgrade(tmp, branch, env = process.env) {
   );
   if (ready.source !== selected.source || ready.static !== selected.static)
     throw new Error(upgradeMessage('暂存升级就绪记录不匹配。', env));
-  const manifest = await verifyUpgrade(selected.source, selected.static, env);
+  const manifest = metadataOnly
+    ? await manifestAt(selected.static, env)
+    : await verifyUpgrade(selected.source, selected.static, env);
   if (ready.sourceCommit !== manifest.sourceCommit)
     throw new Error(upgradeMessage('暂存升级就绪记录不匹配。', env));
-  if (ready.bundle)
+  if (ready.bundle && !metadataOnly)
     await require('./platformBundle.cjs').verifyStagedBundle(selected);
   if (ready.dependenciesChanged)
     await fs.access(path.join(selected.source, 'node_modules'));
@@ -668,7 +675,16 @@ async function reloadSystem({
     .catch(() => {});
 }
 
-async function waitForHealth(env, timeout = 60000, signal) {
+function startupHealthTimeout(runtime = process) {
+  // ARMv6 starts the database, scheduler and HTTP workers sequentially. Give
+  // slow devices time to become healthy before deciding to restore a backup.
+  return runtime.arch === 'arm' &&
+    Number(runtime.config.variables.arm_version) === 6
+    ? 300000
+    : 60000;
+}
+
+async function waitForHealth(env, timeout = startupHealthTimeout(), signal) {
   const port = Number(env.QlPort || 5700);
   let base = env.QlBaseUrl || '';
   if (base && !base.startsWith('/')) base = `/${base}`;
@@ -856,7 +872,7 @@ async function main() {
             await hook('start', [], recover ? undefined : controller.signal);
             await waitForHealth(
               env,
-              60000,
+              undefined,
               recover ? undefined : controller.signal,
             );
           },
@@ -903,6 +919,7 @@ module.exports = {
   selectedUpgrade,
   replaceAndReload,
   reloadSystem,
+  startupHealthTimeout,
   waitForHealth,
   runProcess,
   withLifecycleHooks,

docs/development/platform-upgrade-bundles.md

@@ -6,7 +6,7 @@ CI 从同次构建的不可变镜像摘要导出完整 tar.gz，包含源码中
 
 `package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移；完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。
 
-面板下载任务与本地离线入口共用 `platformBundle.cjs`：平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 发布就绪指针。重启前重新验证完整载荷。离线升级同步面板任务状态，并与在线任务互斥；中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
+面板下载任务与本地离线入口共用 `platformBundle.cjs`：平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 发布就绪指针。点击重启时，接口只检查就绪记录并提交后台任务；完整文件与依赖复验由独立工作进程执行，复验通过后才停止旧服务，避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态，并与在线任务互斥；中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
 
 渠道索引为 GitHub prerelease `upgrade-develop` 或 `upgrade-master` 下的 `latest.json`。索引记录准确字节数、SHA256 和不可变 `upgrade-<commit>` release 资产 URL。13 个平台均完成归档、解包后复验及依赖试运行才发布渠道；过期工作流不能覆盖更新提交的渠道。
 
@@ -45,7 +45,7 @@ ARMv6 glibc 小设备（例如 Raspberry Pi 1/Zero 的 Raspbian/Raspberry Pi OS
 - glibc、musl 分包，并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。
 - ARMv6 glibc 使用签名校验的 Raspbian bullseye（glibc 2.31）构建环境，以固定 SHA256 的 ARMv6 Node 20.18.1 头文件编译 SQLite，强制 `-march=armv6 -mfpu=vfp -mfloat-abi=hard`。CI 检查 ELF 的 ARMv6 和硬浮点标记，并在 QEMU ARM1176 下完成归档复验、SQLite 读写和空目录离线安装；验证辅助镜像按不可变摘要记录，不进入用户安装包。Debian ARMv7 镜像仅提供同次构建的应用文件，不复用其 Node、libc 或 SQLite 二进制。
 - ARMv6 与 ARMv7 使用不同包名。ARMv6 在目标镜像中从源码编译 SQLite，禁止复用 generic `arm` 的 ARMv7 预编译件；构建需通过加载和内存库读写，失败则不发布渠道。
-- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`，避开 ARM1176 上 V8 的优化编译崩溃；保留现有 Node 二进制，正常面板进程的启动参数不变。依赖加载和 SQLite 读写仍由现有 Node 的正常模式独立进程验证，使 Node 20.18.1 的内置 HTTP/WebAssembly 组件保持可用。
+- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`，避开 ARM1176 上 V8 的优化编译崩溃；保留现有 Node 二进制，正常面板进程的启动参数不变。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟；升级后的整体健康检查最多等待 5 分钟，覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时；其他平台仍为 60 秒，超时或版本校验失败仍执行回退。依赖加载和 SQLite 读写仍由现有 Node 的正常模式独立进程验证，使 Node 20.18.1 的内置 HTTP/WebAssembly 组件保持可用。
 - tar 中允许 pnpm 的包内符号链接；拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖，暂存失败保留上一次可用的就绪包。
 
 自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引；要求 HTTPS，测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源，避免依赖额外的 raw 域名。

shell/install-bundle.cjs

@@ -256,7 +256,7 @@ async function installBundle(argv = process.argv.slice(2), env = process.env) {
                         );
                         await artifacts.waitForHealth(
                           environment,
-                          60000,
+                          undefined,
                           recover ? undefined : controller.signal,
                         );
                         if (!recover)

test/back/armv6-worker-readiness.test.cjs

@@ -0,0 +1,101 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const { EventEmitter } = require('node:events');
+const ts = require('typescript');
+
+const source = ts.createSourceFile(
+  'app.ts',
+  fs.readFileSync('back/app.ts', 'utf8'),
+  ts.ScriptTarget.Latest,
+  true,
+);
+const application = source.statements.find(
+  (node) => ts.isClassDeclaration(node) && node.name.text === 'Application',
+);
+const method = application.members.find(
+  (node) => node.name?.getText(source) === 'startMasterProcess',
+);
+const compiled = ts.transpileModule(
+  `class Application { ${method.getText(
+    source,
+  )} }\nmodule.exports = Application;`,
+  {
+    compilerOptions: {
+      module: ts.ModuleKind.CommonJS,
+      target: ts.ScriptTarget.ES2020,
+    },
+  },
+).outputText;
+
+for (const [arch, armVersion, timeout] of [
+  ['arm', 6, 120000],
+  ['arm', 7, 30000],
+  ['arm64', undefined, 30000],
+]) {
+  test(`${arch} v${
+    armVersion ?? ''
+  } waits for scheduler readiness before HTTP and job recovery`, async () => {
+    const cluster = new EventEmitter();
+    const module = { exports: {} };
+    const logger = { info() {}, warn() {}, error() {} };
+    new Function(
+      'module',
+      'cluster',
+      'Logger',
+      'errStack',
+      'process',
+      compiled,
+    )(module, cluster, logger, String, {
+      arch,
+      config: { variables: { arm_version: armVersion } },
+      exit() {
+        assert.fail('unexpected application exit');
+      },
+    });
+    const app = new module.exports();
+    app.workerMetadataMap = new Map();
+    const forks = [],
+      waits = [],
+      notifications = [];
+    let ready;
+    app.forkWorker = (serviceType) => {
+      const worker = {
+        id: forks.length + 1,
+        process: { pid: 100 + forks.length },
+        send: (message) => notifications.push(message),
+      };
+      forks.push({ serviceType, worker });
+      app.workerMetadataMap.set(worker.id, { serviceType });
+      return worker;
+    };
+    app.waitForWorkerReady = (worker, milliseconds) => {
+      waits.push({ worker, milliseconds });
+      return new Promise((resolve) => {
+        ready = resolve;
+      });
+    };
+    app.setupMasterShutdown = () => {};
+    app.startMasterProcess();
+    assert.deepEqual(
+      forks.map((fork) => fork.serviceType),
+      ['grpc'],
+    );
+    assert.equal(waits[0].milliseconds, timeout);
+    ready();
+    await Promise.resolve();
+    assert.deepEqual(
+      forks.map((fork) => fork.serviceType),
+      ['grpc', 'http'],
+    );
+    cluster.emit('exit', forks[0].worker, 1);
+    assert.equal(waits[1].milliseconds, timeout);
+    assert.deepEqual(notifications, ['scheduler-unavailable']);
+    ready();
+    await Promise.resolve();
+    assert.deepEqual(notifications, [
+      'scheduler-unavailable',
+      'reregister-crons',
+    ]);
+  });
+}

test/back/panel-upgrade.test.cjs

@@ -186,7 +186,7 @@ test('detached download survives its initiating request and is idempotent; apply
   );
 });
 
-test('invalid staged payload is rejected before spawning a restart', async (t) => {
+test('invalid staged payload fails asynchronously before stopping the panel', async (t) => {
   const f = await fixture(t);
   await startJob(f, 'download');
   assert.equal((await finished(f.tmp)).status, 'ready');
@@ -197,9 +197,16 @@ test('invalid staged payload is rejected before spawning a restart', async (t) =
     path.join(f.tmp, pointer.directory, 'qinglong-static-master/build/app.js'),
     'corrupted',
   );
-  await assert.rejects(startJob(f, 'apply'), /checksum|损坏/);
+  const queued = await startJob(f, 'apply');
+  assert.equal(queued.status, 'starting');
+  const failed = await finished(f.tmp);
+  assert.equal(failed.status, 'failed');
+  assert.match(failed.error, /checksum|损坏/);
   assert.equal(syncFs.existsSync(path.join(f.root, 'stops')), false);
-  assert.equal((await getStatus(f.tmp)).status, 'failed');
+  assert.equal(
+    await fs.readFile(path.join(f.root, 'version.yaml'), 'utf8'),
+    'version: 2.23.0',
+  );
 });
 
 test('failed download persists its failure without stopping the panel and can be retried', async (t) => {
@@ -467,3 +474,15 @@ for (const knownSize of [true, false])
     assert.equal(reports.at(-1).downloadedBytes, 100);
     assert.equal(reports.at(-1).percent, knownSize ? 100 : null);
   });
+
+test('asynchronous apply still rejects unsafe stage pointers before dispatch', async (t) => {
+  const f = await fixture(t);
+  await startJob(f, 'download');
+  assert.equal((await finished(f.tmp)).status, 'ready');
+  await fs.writeFile(
+    path.join(f.tmp, 'upgrade-ready-master.json'),
+    JSON.stringify({ directory: '../outside' }),
+  );
+  await assert.rejects(startJob(f, 'apply'), /指针|pointer/);
+  assert.equal(syncFs.existsSync(path.join(f.root, 'stops')), false);
+});

test/back/upgrade-guards.test.cjs

@@ -283,3 +283,54 @@ test('unhealthy, malformed and oversized health responses fail within the deadli
     await assert.rejects(waitForHealth({ QlPort: port }, 20), /health check|后端启动检查失败/);
   }
 });
+
+test('ARMv6 startup can become healthy after the standard one-minute deadline', async (t) => {
+  const { EventEmitter } = require('node:events');
+  const descriptors = ['arch', 'config'].map((key) => [
+    key,
+    Object.getOwnPropertyDescriptor(process, key),
+  ]);
+  t.after(() => {
+    for (const [key, descriptor] of descriptors)
+      Object.defineProperty(process, key, descriptor);
+  });
+  Object.defineProperty(process, 'arch', { value: 'arm', configurable: true });
+  Object.defineProperty(process, 'config', {
+    value: { variables: { arm_version: 6 } },
+    configurable: true,
+  });
+  let now = 0;
+  let probes = 0;
+  t.mock.method(Date, 'now', () => now);
+  t.mock.method(http, 'get', (_options, callback) => {
+    const request = new EventEmitter();
+    request.destroy = (error) => request.emit('error', error);
+    queueMicrotask(() => {
+      const response = new EventEmitter();
+      response.statusCode = ++probes === 1 ? 503 : 200;
+      callback(response);
+      now = probes * 70000;
+      response.emit(
+        'data',
+        JSON.stringify({
+          code: response.statusCode,
+          data: { status: probes > 1 ? 'ok' : 'starting' },
+        }),
+      );
+      response.emit('end');
+    });
+    return request;
+  });
+  await waitForHealth({});
+  assert.equal(probes, 2);
+  assert.equal(now, 140000);
+  const { startupHealthTimeout } = require('../../shell/upgrade.cjs');
+  assert.equal(
+    startupHealthTimeout({
+      arch: 'arm',
+      config: { variables: { arm_version: 7 } },
+    }),
+    60000,
+  );
+  assert.equal(startupHealthTimeout({ arch: 'arm64' }), 60000);
+});