From 035fe8c5d2807c732b060d778fd7465a98b52faf Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 17 Oct 2026 22:42:25 +0800 Subject: [PATCH] fix: verify archived permissions and coordinate offline upgrade state --- cli/scripts/build.cjs | 15 +- cli/src/internal/maintenance/panelUpgrade.cjs | 1 + cli/test/internal/platformBundle.test.cjs | 35 ++ docker/Dockerfile.upgrade-bundle | 1 + docs/development/platform-upgrade-bundles.md | 4 +- scripts/build-upgrade-bundle.cjs | 28 +- shell/install-bundle.cjs | 305 +++++++++++------- test/back/platform-bundle.test.cjs | 173 ++++++++++ 8 files changed, 447 insertions(+), 115 deletions(-) create mode 100644 cli/test/internal/platformBundle.test.cjs diff --git a/cli/scripts/build.cjs b/cli/scripts/build.cjs index 9c7505e4..eff9258e 100644 --- a/cli/scripts/build.cjs +++ b/cli/scripts/build.cjs @@ -17,10 +17,17 @@ fs.copyFileSync( path.join(root, 'src/internal/maintenance/upgradeArtifacts.cjs'), path.join(root, 'dist/internal/maintenance/upgradeArtifacts.cjs'), ); -fs.copyFileSync( - path.join(root, 'src/internal/maintenance/platformBundle.cjs'), - path.join(root, 'dist/internal/maintenance/platformBundle.cjs'), -); +// The operational CLI may live outside the panel and has no panel node_modules. +require('esbuild').buildSync({ + entryPoints: [path.join(root, 'src/internal/maintenance/platformBundle.cjs')], + outfile: path.join(root, 'dist/internal/maintenance/platformBundle.cjs'), + bundle: true, + platform: 'node', + format: 'cjs', + target: 'node20', + minify: true, + legalComments: 'inline', +}); require('esbuild').buildSync({ entryPoints: [path.join(root, 'src/shared/cli/commander.ts')], outfile: path.join(root, 'dist/shared/cli/commander.js'), diff --git a/cli/src/internal/maintenance/panelUpgrade.cjs b/cli/src/internal/maintenance/panelUpgrade.cjs index cd53b012..691f98d0 100644 --- a/cli/src/internal/maintenance/panelUpgrade.cjs +++ b/cli/src/internal/maintenance/panelUpgrade.cjs @@ -431,6 +431,7 @@ async function runJob({ root, tmp, env = process.env }, id) { module.exports = { getStatus, + saveState: save, startJob, runJob, withPanelLifecycle, diff --git a/cli/test/internal/platformBundle.test.cjs b/cli/test/internal/platformBundle.test.cjs new file mode 100644 index 00000000..3ce4a7de --- /dev/null +++ b/cli/test/internal/platformBundle.test.cjs @@ -0,0 +1,35 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +test('the standalone operational CLI verifier carries its archive parser without panel dependencies', (t) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-cli-bundle-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const filename = path.join(directory, 'platformBundle.cjs'); + fs.copyFileSync( + path.resolve( + __dirname, + '../../dist/internal/maintenance/platformBundle.cjs', + ), + filename, + ); + const code = ` + const Module=require('node:module'),load=Module._load; + Module._load=function(name,...args){ + if(name!==process.argv[1]&&!name.startsWith('node:')&&!Module.builtinModules.includes(name))throw new Error('External dependency: '+name); + return load.call(this,name,...args); + }; + const bundle=require(process.argv[1]); + if(typeof bundle.stageBundle!=='function')throw new Error('Missing verifier'); + bundle.runtimeTarget().then(target=>console.log(bundle.targetKey(target))).catch(e=>{console.error(e);process.exitCode=1}); + `; + const result = spawnSync(process.execPath, ['-e', code, filename], { + encoding: 'utf8', + env: { ...process.env, NODE_PATH: '' }, + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, new RegExp(`^${process.platform}-`)); +}); diff --git a/docker/Dockerfile.upgrade-bundle b/docker/Dockerfile.upgrade-bundle index 5fe14d95..54f35da9 100644 --- a/docker/Dockerfile.upgrade-bundle +++ b/docker/Dockerfile.upgrade-bundle @@ -15,6 +15,7 @@ RUN if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \ npm_config_nodedir=/usr npm_config_build_from_source=true QEMU_CPU=arm1176 \ node -e 'require("node:child_process").execFileSync(process.execPath,[require.resolve("@mapbox/node-pre-gyp/bin/node-pre-gyp"),"rebuild","--napi_build_version=6"],{stdio:"inherit"})' && \ readelf -A lib/binding/napi-v6-linux-arm/node_sqlite3.node | grep -E 'Tag_CPU_arch: v6($|K)' && \ + rm -rf build && \ apk del .ql-bundle-build; \ fi RUN if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then export QEMU_CPU=arm1176; fi; \ diff --git a/docs/development/platform-upgrade-bundles.md b/docs/development/platform-upgrade-bundles.md index 452be33d..81438bea 100644 --- a/docs/development/platform-upgrade-bundles.md +++ b/docs/development/platform-upgrade-bundles.md @@ -6,9 +6,9 @@ CI 从同次构建的不可变镜像摘要导出完整 tar.gz，包含源码中 `package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移；完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。 -面板下载任务与本地离线入口共用 `platformBundle.cjs`：平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 发布就绪指针。重启前重新验证完整载荷。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。 +面板下载任务与本地离线入口共用 `platformBundle.cjs`：平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 发布就绪指针。重启前重新验证完整载荷。离线升级同步面板任务状态，并与在线任务互斥；中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。 -渠道索引为 GitHub prerelease `upgrade-develop` 或 `upgrade-master` 下的 `latest.json`。索引记录准确字节数、SHA256 和不可变 `upgrade-` release 资产 URL。12 个平台均构建及试运行成功才发布渠道；过期工作流不能覆盖更新提交的渠道。 +渠道索引为 GitHub prerelease `upgrade-develop` 或 `upgrade-master` 下的 `latest.json`。索引记录准确字节数、SHA256 和不可变 `upgrade-` release 资产 URL。12 个平台均完成归档、解包后复验及依赖试运行才发布渠道；过期工作流不能覆盖更新提交的渠道。 ## 手动离线安装和升级 diff --git a/scripts/build-upgrade-bundle.cjs b/scripts/build-upgrade-bundle.cjs index 8b9adc81..6abd9aed 100644 --- a/scripts/build-upgrade-bundle.cjs +++ b/scripts/build-upgrade-bundle.cjs @@ -6,6 +6,19 @@ const { verifyUpgrade, } = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs'); +async function normalizeModes(directory) { + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + const filename = path.join(directory, entry.name); + if (entry.isDirectory()) { + await fs.chmod(filename, 0o755); + await normalizeModes(filename); + } else if (entry.isFile()) { + const stat = await fs.stat(filename); + await fs.chmod(filename, 0o644 | (stat.mode & 0o111)); + } + } +} + async function buildBundle(root, output) { root = path.resolve(root); output = path.resolve(output); @@ -46,6 +59,8 @@ async function buildBundle(root, output) { recursive: true, verbatimSymlinks: true, }); + await normalizeModes(source); + await normalizeModes(staticRoot); const manifest = { format: 2, sourceCommit: build.sourceCommit, @@ -82,6 +97,17 @@ async function buildBundle(root, output) { }, ['qinglong'], ); + // Verify the bytes users receive, not just the pre-archive build directory. + const roundtrip = path.join(workspace, 'roundtrip'); + await fs.mkdir(roundtrip); + await bundle.extractArchive(path.join(output, filename), roundtrip); + const extracted = { + source: path.join(roundtrip, 'qinglong/source'), + static: path.join(roundtrip, 'qinglong/static'), + }; + await bundle.verifyPayload(extracted, manifest, target); + await verifyUpgrade(extracted.source, extracted.static); + await bundle.smokeDependencies(extracted.source); const bytes = await fs.readFile(path.join(output, filename)); const record = { format: 2, @@ -102,7 +128,7 @@ async function buildBundle(root, output) { await fs.rm(workspace, { recursive: true, force: true }); } } -module.exports = { buildBundle }; +module.exports = { buildBundle, normalizeModes }; if (require.main === module) buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch( (e) => { diff --git a/shell/install-bundle.cjs b/shell/install-bundle.cjs index f66dd03b..086f9969 100644 --- a/shell/install-bundle.cjs +++ b/shell/install-bundle.cjs @@ -3,6 +3,7 @@ const fs = require('node:fs/promises'); const path = require('node:path'); const os = require('node:os'); +const { randomUUID } = require('node:crypto'); const { promisify } = require('node:util'); const execute = promisify(require('node:child_process').execFile); const { parseArgs } = require('node:util'); @@ -68,118 +69,206 @@ async function installBundle(argv = process.argv.slice(2), env = process.env) { ? await fs.mkdtemp(path.join(os.tmpdir(), 'ql-offline-install-')) : undefined; const tmp = freshWorkspace || path.join(root, '.tmp'); + const controller = new AbortController(); + const abort = () => controller.abort(new Error('离线升级已中断')); + process.once('SIGINT', abort); + process.once('SIGTERM', abort); try { - return await artifacts.withUpgradeLock( - tmp, - async () => { - const selected = await artifacts.stageUpgrade({ - root, - tmp, - env: environment, - run: (program, args, capture) => - artifacts.runProcess(environment, program, args, capture), - onProgress: async (p) => console.log(p.phase), - install: async () => { - throw new Error('Offline bundles never install npm dependencies'); - }, - }); - if (values['download-only']) { - console.log('更新已就绪；执行 ql reload system 应用更新。'); - return selected; - } - if (values.install) { - // Build beside the destination, then atomically replace only an empty root. - await fs.mkdir(path.dirname(root), { recursive: true }); - const publication = await fs.mkdtemp( - path.join(path.dirname(root), '.qinglong-install-'), - ); - try { - await fs.cp(selected.source, publication, { - recursive: true, - verbatimSymlinks: true, - }); - await fs.cp(selected.static, path.join(publication, 'static'), { - recursive: true, - }); - await fs.copyFile( - path.join(publication, '.env.example'), - path.join(publication, '.env'), - require('node:fs').constants.COPYFILE_EXCL, - ); - await fs.chmod(path.join(publication, '.env'), 0o600); - await fs.rename(publication, root); - } finally { - await fs.rm(publication, { recursive: true, force: true }); - } - await fs.rm(path.dirname(selected.source), { - recursive: true, - force: true, - }); - await fs.rm( - path.join( - tmp, - `upgrade-ready-${artifacts.releaseBranch(environment)}.json`, - ), - { force: true }, - ); - console.log( - `离线安装完成，保留现有 Node ${ - process.version - }。启动服务：QL_DIR=${JSON.stringify(root)} bash ${JSON.stringify( - path.join(root, 'shell/start-simplify.sh'), - )}`, - ); - return; - } - return jobs.withPanelLifecycle( - root, - tmp, - environment, - async (hook, prepareSelection) => { - const expected = await jobs.stagedMetadata(selected); - const previous = await jobs - .readRunningBuild(environment) - .catch(() => undefined); - const selection = await prepareSelection?.(selected); - try { - return await artifacts.reloadSystem({ - root, - tmp, - env: environment, - data: environment.dir_data || path.join(root, 'data'), - staticRoot: path.join(root, 'static'), - config: - environment.dir_config || path.join(root, 'data/config'), - lifecycle: { - stop: () => hook('stop'), - start: async (recover) => { - await hook('start'); - await artifacts.waitForHealth(environment); - if (!recover) - await jobs.verifyRunningVersion( - environment, - expected, - previous, - ); - }, - }, - additionalReplacements: selection - ? [{ source: selection.source, target: selection.target }] - : [], - }); - } finally { - if (selection) - await fs.rm(selection.directory, { - recursive: true, - force: true, - }); + const operation = () => + artifacts.withUpgradeLock( + tmp, + async () => { + let state; + const report = async (patch) => { + if (patch.phase) console.log(patch.phase); + if (state) { + state = { + ...state, + ...patch, + pid: process.pid, + updatedAt: Date.now(), + }; + await jobs.saveState(tmp, state); } - }, + }; + if (!values.install) { + const previous = await jobs.getStatus(tmp); + if (['starting', 'running', 'restarting'].includes(previous.status)) + throw new Error('已有更新或重载正在执行。'); + state = { + id: randomUUID(), + action: values['download-only'] ? 'download' : 'apply', + status: 'running', + phase: 'verifying', + startedAt: Date.now(), + }; + await report({}); + await fs.writeFile( + path.join(tmp, 'panel-upgrade.log'), + 'Offline bundle verification\n', + { mode: 0o600 }, + ); + } + try { + const selected = await artifacts.stageUpgrade({ + root, + tmp, + env: environment, + run: (program, args, capture) => + artifacts.runProcess( + environment, + program, + args, + capture, + controller.signal, + ), + signal: controller.signal, + onProgress: report, + install: async () => { + throw new Error( + 'Offline bundles never install npm dependencies', + ); + }, + }); + if (state) + await report({ + status: 'ready', + phase: 'ready', + ...(await jobs.stagedMetadata(selected)), + }); + if (values['download-only']) { + console.log('更新已就绪；执行 ql reload system 应用更新。'); + return selected; + } + if (values.install) { + // Build beside the destination, then atomically replace only an empty root. + await fs.mkdir(path.dirname(root), { recursive: true }); + const publication = await fs.mkdtemp( + path.join(path.dirname(root), '.qinglong-install-'), + ); + try { + await fs.chmod(publication, 0o755); + await fs.cp(selected.source, publication, { + recursive: true, + verbatimSymlinks: true, + }); + await fs.cp(selected.static, path.join(publication, 'static'), { + recursive: true, + }); + await fs.copyFile( + path.join(publication, '.env.example'), + path.join(publication, '.env'), + require('node:fs').constants.COPYFILE_EXCL, + ); + await fs.chmod(path.join(publication, '.env'), 0o600); + controller.signal.throwIfAborted(); + await fs.rename(publication, root); + } finally { + await fs.rm(publication, { recursive: true, force: true }); + } + await fs.rm(path.dirname(selected.source), { + recursive: true, + force: true, + }); + await fs.rm( + path.join( + tmp, + `upgrade-ready-${artifacts.releaseBranch(environment)}.json`, + ), + { force: true }, + ); + console.log( + `离线安装完成，保留现有 Node ${ + process.version + }。启动服务：QL_DIR=${JSON.stringify( + root, + )} bash ${JSON.stringify( + path.join(root, 'shell/start-simplify.sh'), + )}`, + ); + return; + } + const result = await jobs.withPanelLifecycle( + root, + tmp, + environment, + async (hook, prepareSelection) => { + const expected = await jobs.stagedMetadata(selected); + const previous = await jobs + .readRunningBuild(environment) + .catch(() => undefined); + const selection = await prepareSelection?.(selected); + try { + return await artifacts.reloadSystem({ + root, + tmp, + env: environment, + data: environment.dir_data || path.join(root, 'data'), + staticRoot: path.join(root, 'static'), + config: + environment.dir_config || path.join(root, 'data/config'), + lifecycle: { + signal: controller.signal, + stop: async () => { + await report({ + status: 'restarting', + phase: 'restarting', + }); + await hook('stop', [], controller.signal); + }, + start: async (recover) => { + await hook( + 'start', + [], + recover ? undefined : controller.signal, + ); + await artifacts.waitForHealth( + environment, + 60000, + recover ? undefined : controller.signal, + ); + if (!recover) + await jobs.verifyRunningVersion( + environment, + expected, + previous, + ); + }, + }, + additionalReplacements: selection + ? [{ source: selection.source, target: selection.target }] + : [], + }); + } finally { + if (selection) + await fs.rm(selection.directory, { + recursive: true, + force: true, + }); + } + }, + controller.signal, + ); + await report({ status: 'completed', phase: 'completed' }); + console.log('升级成功，已保留现有 Node 并运行更新后的版本。'); + return result; + } catch (error) { + if (state) await report({ status: 'failed', error: error.message }); + throw error; + } + }, + environment, + ); + return values.install + ? await operation() + : await artifacts.withUpgradeLock( + path.join(tmp, 'panel-job-launch'), + operation, + environment, ); - }, - environment, - ); } finally { + process.removeListener('SIGINT', abort); + process.removeListener('SIGTERM', abort); if (freshWorkspace) await fs.rm(freshWorkspace, { recursive: true, force: true }); } diff --git a/test/back/platform-bundle.test.cjs b/test/back/platform-bundle.test.cjs index 7e9b1d74..09815915 100644 --- a/test/back/platform-bundle.test.cjs +++ b/test/back/platform-bundle.test.cjs @@ -351,3 +351,176 @@ test('fresh offline installation needs no package manager, keeps Node and publis 'keep', ); }); + +test('offline staging replaces stale panel readiness and refuses an active online job', async (t) => { + const f = await fixture(t); + await fs.writeFile(path.join(f.source, 'version.yaml'), 'version: 2.24.0\n'); + await fs.writeFile( + path.join(f.source, 'node_modules/sqlite3/index.js'), + 'exports.Database=class {serialize(fn){fn()}run(){}get(sql,cb){cb(null,{value:42})}close(){}};', + ); + for (const name of ['sequelize', 'cron-parser', '@grpc/grpc-js', 'undici']) { + const dir = path.join(f.source, 'node_modules', name); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'index.js'), 'module.exports={};'); + } + f.manifest.files = await bundle.inventory({ + source: f.source, + static: f.staticRoot, + }); + await f.pack(); + await fs.mkdir(path.join(f.root, 'shell')); + await fs.writeFile( + path.join(f.root, 'shell/share.sh'), + 'import_config() { :; }\n', + ); + await fs.writeFile( + path.join(f.root, 'shell/env.sh'), + 'export QL_BRANCH=develop\n', + ); + const stateFile = path.join(f.tmp, 'panel-upgrade.json'); + await fs.writeFile( + stateFile, + JSON.stringify({ + id: 'previous-online-job', + status: 'ready', + phase: 'ready', + }), + ); + const install = require('../../shell/install-bundle.cjs').installBundle; + await install(['--root', f.root, '--file', f.archive, '--download-only']); + const state = JSON.parse(await fs.readFile(stateFile, 'utf8')); + assert.equal(state.status, 'ready'); + assert.equal(state.sourceCommit, sourceCommit); + assert.notEqual(state.id, 'previous-online-job'); + assert.equal(state.version, '2.24.0'); + const pointer = await fs.readFile( + path.join(f.tmp, 'upgrade-ready-develop.json'), + 'utf8', + ); + await fs.writeFile( + stateFile, + JSON.stringify({ + id: 'live-online-job', + status: 'running', + pid: process.pid, + startedAt: Date.now(), + }), + ); + await assert.rejects( + install(['--root', f.root, '--file', f.archive, '--download-only']), + /已有更新/, + ); + assert.equal( + JSON.parse(await fs.readFile(stateFile, 'utf8')).id, + 'live-online-job', + ); + assert.equal( + await fs.readFile(path.join(f.tmp, 'upgrade-ready-develop.json'), 'utf8'), + pointer, + ); +}); + +test('interrupting offline staging records failure, preserves prior readiness and releases both locks', async (t) => { + const f = await fixture(t); + const { spawn } = require('node:child_process'); + await fs.writeFile(path.join(f.source, 'version.yaml'), 'version: 2.24.0\n'); + await fs.writeFile( + path.join(f.source, 'node_modules/sqlite3/index.js'), + 'exports.Database=class {serialize(fn){fn()}run(){}get(sql,cb){setTimeout(()=>cb(null,{value:42}),10000)}close(){}};', + ); + for (const name of ['sequelize', 'cron-parser', '@grpc/grpc-js', 'undici']) { + const dir = path.join(f.source, 'node_modules', name); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'index.js'), 'module.exports={};'); + } + f.manifest.files = await bundle.inventory({ + source: f.source, + static: f.staticRoot, + }); + await f.pack(); + await fs.mkdir(path.join(f.root, 'shell')); + await fs.writeFile( + path.join(f.root, 'shell/share.sh'), + 'import_config() { :; }\n', + ); + await fs.writeFile( + path.join(f.root, 'shell/env.sh'), + 'export QL_BRANCH=develop\n', + ); + const pointer = path.join(f.tmp, 'upgrade-ready-develop.json'); + await fs.writeFile(pointer, 'previous-good-readiness'); + const child = spawn( + process.execPath, + [ + path.resolve(__dirname, '../../shell/install-bundle.cjs'), + '--root', + f.root, + '--file', + f.archive, + '--download-only', + ], + { stdio: 'ignore' }, + ); + t.after(() => { + if (child.exitCode === null) child.kill('SIGKILL'); + }); + const exited = new Promise((resolve, reject) => { + child.once('exit', resolve); + child.once('error', reject); + }); + let seen = false; + for (let i = 0; i < 200; i++) { + const state = await fs + .readFile(path.join(f.tmp, 'panel-upgrade.json'), 'utf8') + .then(JSON.parse) + .catch(() => ({})); + if (state.phase === 'dependencies') { + seen = true; + break; + } + await new Promise((r) => setTimeout(r, 20)); + } + assert.ok(seen); + child.kill('SIGTERM'); + assert.equal(await exited, 1); + assert.equal(await fs.readFile(pointer, 'utf8'), 'previous-good-readiness'); + assert.equal( + JSON.parse( + await fs.readFile(path.join(f.tmp, 'panel-upgrade.json'), 'utf8'), + ).status, + 'failed', + ); + await assert.rejects(fs.access(path.join(f.tmp, 'upgrade.lock'))); + await assert.rejects( + fs.access(path.join(f.tmp, 'panel-job-launch/upgrade.lock')), + ); +}); + +test('portable release archives preserve the permission manifest after normalizing image scripts', async (t) => { + const f = await fixture(t); + const script = path.join(f.source, 'sample/config.sample.sh'); + await fs.chmod(script, 0o777); + const { normalizeModes } = require('../../scripts/build-upgrade-bundle.cjs'); + await normalizeModes(f.source); + await normalizeModes(f.staticRoot); + assert.equal((await fs.stat(script)).mode & 0o777, 0o755); + f.manifest.files = await bundle.inventory({ + source: f.source, + static: f.staticRoot, + }); + await f.pack(); + const extracted = path.join(f.directory, 'roundtrip'); + await fs.mkdir(extracted); + await bundle.extractArchive(f.archive, extracted); + const selected = { + source: path.join(extracted, 'qinglong/source'), + static: path.join(extracted, 'qinglong/static'), + }; + await bundle.verifyPayload(selected, f.manifest, f.target); + await fs.chmod(path.join(selected.source, 'sample/config.sample.sh'), 0o777); + await assert.rejects( + bundle.verifyPayload(selected, f.manifest, f.target), + /checksums/, + ); +});