diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index eb05f9c0..d94cceb0 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -8,6 +8,8 @@ on: # CLI is verified separately; its shared Bash upgrade engine ships in images. - "!cli/**" - "cli/src/internal/maintenance/upgradeArtifacts.cjs" + - "cli/src/internal/maintenance/platformBundle.cjs" + - "cli/src/internal/maintenance/panelUpgrade.cjs" - "!.github/workflows/cli-package.yml" branches: - "master" @@ -203,6 +205,8 @@ jobs: build-alpine: if: ${{ !startsWith(github.ref, 'refs/tags/') }} needs: build-static + outputs: + image-digest: ${{ steps.image.outputs.digest }} runs-on: ubuntu-22.04 permissions: packages: write @@ -268,6 +272,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Build and push (Alpine) + id: image uses: docker/build-push-action@v7 with: build-args: | @@ -289,6 +294,8 @@ jobs: build-debian: if: ${{ !startsWith(github.ref, 'refs/tags/') }} needs: build-static + outputs: + image-digest: ${{ steps.image.outputs.digest }} runs-on: ubuntu-22.04 permissions: packages: write @@ -353,6 +360,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Build and push (Debian) + id: image uses: docker/build-push-action@v7 with: build-args: | @@ -563,3 +571,76 @@ jobs: echo '::error::Could not check the published npm version; refusing to publish.' exit 1 fi + + build-upgrade-bundles: + needs: [build-alpine, build-debian] + runs-on: ubuntu-22.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - { platform: linux/amd64, libc: musl, name: x64-musl } + - { platform: linux/arm/v6, libc: musl, name: armv6-musl } + - { platform: linux/arm/v7, libc: musl, name: armv7-musl } + - { platform: linux/arm64, libc: musl, name: arm64-musl } + - { platform: linux/ppc64le, libc: musl, name: ppc64-musl } + - { platform: linux/s390x, libc: musl, name: s390x-musl } + - { platform: linux/386, libc: musl, name: ia32-musl } + - { platform: linux/amd64, libc: glibc, name: x64-glibc } + - { platform: linux/arm/v7, libc: glibc, name: armv7-glibc } + - { platform: linux/arm64, libc: glibc, name: arm64-glibc } + - { platform: linux/ppc64le, libc: glibc, name: ppc64-glibc } + - { platform: linux/s390x, libc: glibc, name: s390x-glibc } + steps: + - uses: actions/checkout@v6 + - uses: docker/setup-qemu-action@v4 + with: + cache-image: false + - uses: docker/setup-buildx-action@v4 + - name: Build and probe the immutable platform package + env: + ALPINE_DIGEST: ${{ needs.build-alpine.outputs.image-digest }} + DEBIAN_DIGEST: ${{ needs.build-debian.outputs.image-digest }} + BUNDLE_LIBC: ${{ matrix.libc }} + BUNDLE_PLATFORM: ${{ matrix.platform }} + run: | + set -euo pipefail + digest="$ALPINE_DIGEST" + if [ "$BUNDLE_LIBC" = glibc ]; then digest="$DEBIAN_DIGEST"; fi + test -n "$digest" + docker buildx build --platform "$BUNDLE_PLATFORM" --build-arg "BASE_IMAGE=ghcr.io/${GITHUB_REPOSITORY}@${digest}" --file docker/Dockerfile.upgrade-bundle --output type=local,dest=output/upgrade . + - uses: actions/upload-artifact@v6 + with: + name: upgrade-${{ matrix.name }}-${{ github.sha }} + path: output/upgrade/* + if-no-files-found: error + + publish-upgrade-bundles: + needs: build-upgrade-bundles + runs-on: ubuntu-latest + permissions: + contents: write + concurrency: + group: upgrade-channel-${{ github.ref_name }} + cancel-in-progress: false + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '20' + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' + - run: pnpm install --prod --frozen-lockfile + - run: npm --prefix cli ci + - uses: actions/download-artifact@v7 + with: + pattern: upgrade-*-${{ github.sha }} + path: output/upgrade + merge-multiple: true + - run: node scripts/build-bundle-installer.cjs output/upgrade/install-bundle.cjs + - name: Publish complete packages before advancing the channel + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/publish-upgrade-bundles.cjs output/upgrade "$GITHUB_SHA" "$GITHUB_REPOSITORY" "$GITHUB_REF_NAME" diff --git a/cli/scripts/build.cjs b/cli/scripts/build.cjs index 0d4d4d92..9c7505e4 100644 --- a/cli/scripts/build.cjs +++ b/cli/scripts/build.cjs @@ -17,6 +17,10 @@ fs.copyFileSync( path.join(root, 'src/internal/maintenance/upgradeArtifacts.cjs'), path.join(root, 'dist/internal/maintenance/upgradeArtifacts.cjs'), ); +fs.copyFileSync( + path.join(root, 'src/internal/maintenance/platformBundle.cjs'), + path.join(root, 'dist/internal/maintenance/platformBundle.cjs'), +); require('esbuild').buildSync({ entryPoints: [path.join(root, 'src/shared/cli/commander.ts')], outfile: path.join(root, 'dist/shared/cli/commander.js'), diff --git a/cli/src/internal/maintenance/panelUpgrade.cjs b/cli/src/internal/maintenance/panelUpgrade.cjs index 72749b72..cd53b012 100644 --- a/cli/src/internal/maintenance/panelUpgrade.cjs +++ b/cli/src/internal/maintenance/panelUpgrade.cjs @@ -429,7 +429,15 @@ async function runJob({ root, tmp, env = process.env }, id) { } } -module.exports = { getStatus, startJob, runJob }; +module.exports = { + getStatus, + startJob, + runJob, + withPanelLifecycle, + readRunningBuild, + verifyRunningVersion, + stagedMetadata, +}; async function launchWorker(id) { // detached creates a session but keeps the original parent. PM2 kills the // backend's entire descendant tree, so an intermediate launcher must exit diff --git a/cli/src/internal/maintenance/platformBundle.cjs b/cli/src/internal/maintenance/platformBundle.cjs new file mode 100644 index 00000000..e486671e --- /dev/null +++ b/cli/src/internal/maintenance/platformBundle.cjs @@ -0,0 +1,471 @@ +// One installed verifier handles release downloads and local offline archives. +const fs = require('node:fs/promises'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const os = require('node:os'); +const { execFile } = require('node:child_process'); +const { promisify } = require('node:util'); +const tar = require('tar'); +const execute = promisify(execFile); +const sha256 = (bytes) => + crypto.createHash('sha256').update(bytes).digest('hex'); +const inside = (root, filename) => { + const relative = path.relative(root, filename); + return ( + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +}; +const versionParts = (v) => String(v).replace(/^v/, '').split('.').map(Number); +function atLeast(actual, minimum) { + const a = versionParts(actual), + b = versionParts(minimum); + return ( + b.every((n) => Number.isFinite(n)) && + a.every((n) => Number.isFinite(n)) && + (a[0] > b[0] || + (a[0] === b[0] && + (a[1] > b[1] || (a[1] === b[1] && (a[2] || 0) >= (b[2] || 0))))) + ); +} + +async function runtimeTarget() { + let libc, libcVersion; + if (process.platform === 'linux') { + const header = process.report.getReport().header; + if (header.glibcVersionRuntime) { + libc = 'glibc'; + libcVersion = header.glibcVersionRuntime; + } else { + let text = ''; + try { + text = (await execute('ldd', ['--version'])).stdout; + } catch (error) { + text = `${error.stdout || ''}\n${error.stderr || ''}`; + } + const match = text.match(/musl[\s\S]*?Version\s+(\d+\.\d+\.\d+)/i); + if (!match) + throw new Error( + 'Cannot identify Linux libc; refusing a native dependency bundle', + ); + libc = 'musl'; + libcVersion = match[1]; + } + } + const armVersion = + process.arch === 'arm' + ? Number(process.config.variables.arm_version) + : undefined; + if (process.arch === 'arm' && ![6, 7, 8].includes(armVersion)) + throw new Error('Cannot identify ARM instruction set'); + const arch = + process.arch === 'arm' ? `armv${armVersion >= 7 ? 7 : 6}` : process.arch; + const armFloatAbi = + process.arch === 'arm' ? process.config.variables.arm_float_abi : undefined; + if ( + process.arch === 'arm' && + !['hard', 'soft', 'softfp'].includes(armFloatAbi) + ) + throw new Error('Cannot identify ARM floating-point ABI'); + return { + platform: process.platform, + arch, + libc, + libcVersion, + endianness: os.endianness(), + armFloatAbi, + }; +} + +function targetKey(target) { + return [target.platform, target.arch, target.libc].filter(Boolean).join('-'); +} + +function assertCompatible(manifest, target, node = process.versions.node) { + if ( + manifest.format !== 2 || + !/^[a-f0-9]{40}$/.test(manifest.sourceCommit) || + !/^[a-f0-9]{64}$/.test(manifest.lockfileSha256) || + !manifest.files || + typeof manifest.files !== 'object' || + Array.isArray(manifest.files) || + !Number.isSafeInteger(manifest.nodeApi) || + manifest.nodeApi < 6 + ) + throw new Error('Invalid platform bundle manifest'); + if (targetKey(manifest.target || {}) !== targetKey(target)) + throw new Error(`Bundle platform mismatch: need ${targetKey(target)}`); + if ( + !atLeast(node, manifest.nodeMinimum) || + Number(process.versions.napi) < manifest.nodeApi + ) + throw new Error( + `Bundle requires Node >= ${manifest.nodeMinimum} and Node-API ${manifest.nodeApi}; the existing Node is retained`, + ); + if ( + manifest.target.endianness !== target.endianness || + manifest.target.armFloatAbi !== target.armFloatAbi + ) + throw new Error( + 'Bundle native byte order or ARM floating-point ABI mismatch', + ); + if (target.libc && !atLeast(target.libcVersion, manifest.target.libcMinimum)) + throw new Error( + `Bundle requires ${target.libc} >= ${manifest.target.libcMinimum}`, + ); +} + +async function inventory(roots) { + const files = {}; + async function visit(directory, prefix) { + for (const name of (await fs.readdir(directory)).sort()) { + const filename = path.join(directory, name), + relative = `${prefix}/${name}`; + const stat = await fs.lstat(filename); + if (stat.isDirectory()) await visit(filename, relative); + else if (stat.isFile()) + files[relative] = { + type: 'file', + mode: stat.mode & 0o777, + sha256: sha256(await fs.readFile(filename)), + }; + else if (stat.isSymbolicLink()) { + const link = await fs.readlink(filename); + const boundary = roots[prefix.split('/')[0]]; + if ( + path.isAbsolute(link) || + !inside(boundary, path.resolve(path.dirname(filename), link)) || + !inside(await fs.realpath(boundary), await fs.realpath(filename)) + ) + throw new Error(`Bundle link escapes its payload: ${relative}`); + files[relative] = { type: 'link', link }; + } else throw new Error(`Unsupported bundle entry: ${relative}`); + } + } + for (const [name, root] of Object.entries(roots)) await visit(root, name); + return files; +} + +async function verifyPayload(selected, manifest, target) { + assertCompatible(manifest, target); + for (const name of ['data', '.tmp', '.git', '.env', 'static']) { + if ( + await fs.lstat(path.join(selected.source, name)).catch((error) => { + if (error.code !== 'ENOENT') throw error; + return null; + }) + ) + throw new Error(`Reserved installation path in bundle: ${name}`); + } + const actual = await inventory({ + source: selected.source, + static: selected.static, + }); + const names = Object.keys(actual).sort(), + expected = Object.keys(manifest.files).sort(); + if ( + JSON.stringify(names) !== JSON.stringify(expected) || + names.some( + (name) => + JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]), + ) + ) + throw new Error( + 'Platform bundle files or dependency checksums do not match', + ); + if ( + sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !== + manifest.lockfileSha256 + ) + throw new Error('Platform bundle lockfile mismatch'); + await fs.access( + path.join(selected.source, 'node_modules/sqlite3/package.json'), + ); +} + +async function smokeDependencies(source, signal) { + // This probe is installed verifier code, never a verifier supplied by an archive. + const probe = ` + const path = require('node:path'),fs = require('node:fs'); + const root = fs.realpathSync(process.cwd()); + const r = require('node:module').createRequire(path.join(root,'package.json')); + for (const name of ['sqlite3','sequelize','cron-parser','@grpc/grpc-js','undici']) { + if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name); + r(name); + } + const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:'); + db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)'); + db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});}); + `; + await execute(process.execPath, ['-e', probe], { + cwd: source, + env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' }, + timeout: 30000, + signal, + }); +} + +async function extractArchive(archive, destination) { + const entries = new Map(), + links = new Set(); + let unpacked = 0, + invalid; + await tar.t({ + file: archive, + strict: true, + onReadEntry(entry) { + try { + const name = entry.path.replace(/\/$/, ''); + if ( + !name || + name.includes('\\') || + /[\x00-\x1f\x7f]/.test(name) || + name.split('/').some((p) => !p || p === '.' || p === '..') || + !(name === 'qinglong' || name.startsWith('qinglong/')) || + entries.has(name) || + !['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type) + ) + throw new Error(`Unsafe bundle archive entry: ${name}`); + unpacked += entry.size; + if (unpacked > 2 * 1024 ** 3 || entries.size > 200000) + throw new Error('Bundle archive exceeds extraction limits'); + entries.set(name, entry.type); + if (entry.type === 'SymbolicLink') { + if ( + path.posix.isAbsolute(entry.linkpath) || + entry.linkpath.includes('\\') || + !inside( + '/qinglong', + path.posix.resolve('/', path.posix.dirname(name), entry.linkpath), + ) + ) + throw new Error('Unsafe bundle symlink'); + links.add(name); + } + } catch (error) { + invalid ||= error; + } + }, + }); + if (invalid) throw invalid; + for (const name of entries.keys()) { + let parent = path.posix.dirname(name); + while (parent !== '.') { + if (links.has(parent)) + throw new Error('Archive writes through a symlink'); + parent = path.posix.dirname(parent); + } + } + await tar.x({ + file: archive, + cwd: destination, + strict: true, + preservePaths: false, + noChown: true, + umask: 0, + }); +} + +async function stageBundle({ + root, + tmp, + env, + run, + signal, + onProgress, + verifyStatic, + download, + probe = smokeDependencies, +}) { + let local = env.QL_UPGRADE_FILE; + if (!local) { + const current = JSON.parse( + await fs + .readFile(path.join(root, 'package.json'), 'utf8') + .catch((error) => { + if (error.code !== 'ENOENT') throw error; + return '{}'; + }), + ); + if (current.qinglongUpgradeFormat !== 2) return null; // One-time compatibility path for older releases. + } + const target = await runtimeTarget(); + const branch = ['develop', 'debian-dev'].includes(env.QL_BRANCH) + ? 'develop' + : 'master'; + const workspace = await fs.mkdtemp(path.join(tmp, 'upgrade-')); + const report = onProgress || (async () => {}); + try { + const archive = path.join(workspace, 'bundle.tar.gz'); + let expectedCommit; + if (local) { + if (!path.isAbsolute(local)) + throw new Error('Offline bundle path must be absolute'); + // Snapshot the local input so replacing it cannot race validation/extraction. + await fs.copyFile(local, archive); + } else { + const indexUrl = + env.QL_UPGRADE_INDEX_URL || + `https://github.com/whyour/qinglong/releases/download/upgrade-${branch}/latest.json`; + const checkUrl = (url) => { + const parsed = new URL(url); + if ( + parsed.protocol !== 'https:' && + !( + parsed.protocol === 'http:' && + ['127.0.0.1', '[::1]'].includes(parsed.hostname) + ) + ) + throw new Error('Upgrade URLs require HTTPS'); + }; + checkUrl(indexUrl); + const indexFile = path.join(workspace, 'index.json'); + await run('curl', [ + '--fail', + '--silent', + '--show-error', + '--location', + '--proto', + new URL(indexUrl).protocol === 'http:' ? '=http,https' : '=https', + '--proto-redir', + '=https', + '--header', + 'Cache-Control: no-cache', + '--connect-timeout', + '10', + '--max-time', + '30', + '--max-filesize', + '1048576', + '--output', + indexFile, + indexUrl, + ]); + const index = JSON.parse(await fs.readFile(indexFile, 'utf8')); + const entry = index.packages?.[targetKey(target)]; + if ( + index.format !== 2 || + !/^[a-f0-9]{40}$/.test(index.sourceCommit) || + !entry || + !Number.isSafeInteger(entry.size) || + entry.size <= 0 || + entry.size > 512 * 1024 ** 2 || + !/^[a-f0-9]{64}$/.test(entry.sha256) + ) + throw new Error(`No verified update bundle for ${targetKey(target)}`); + checkUrl(entry.url); + expectedCommit = index.sourceCommit; + await report({ + phase: 'downloading', + artifact: 'qinglong-bundle', + downloadedBytes: 0, + totalBytes: entry.size, + percent: 0, + }); + await download( + run, + archive, + entry.url, + 'qinglong-bundle', + report, + true, + entry, + ); + } + if ((await fs.stat(archive)).size > 512 * 1024 ** 2) + throw new Error('Bundle is too large'); + await report({ + phase: 'extracting', + artifact: 'qinglong-bundle', + percent: null, + }); + await extractArchive(archive, workspace); + const incoming = path.join(workspace, 'qinglong'); + const manifest = JSON.parse( + await fs.readFile(path.join(incoming, 'bundle.json'), 'utf8'), + ); + if (expectedCommit && manifest.sourceCommit !== expectedCommit) + throw new Error('Update index and bundle commit differ'); + const selected = { + source: path.join(workspace, `qinglong-${branch}`), + static: path.join(workspace, `qinglong-static-${branch}`), + }; + await fs.rename(path.join(incoming, 'source'), selected.source); + await fs.rename(path.join(incoming, 'static'), selected.static); + await report({ phase: 'verifying', percent: null }); + await verifyPayload(selected, manifest, target); + const build = await verifyStatic(selected.source, selected.static, env); + if (build.sourceCommit !== manifest.sourceCommit) + throw new Error('Bundle and build commit differ'); + await report({ phase: 'dependencies', percent: null }); + await probe(selected.source, signal); + await fs.writeFile( + path.join(workspace, 'bundle.json'), + JSON.stringify(manifest), + { flag: 'wx', mode: 0o600 }, + ); + await fs.writeFile( + path.join(workspace, 'ready.json'), + JSON.stringify({ + ...selected, + root: path.resolve(root), + sourceCommit: manifest.sourceCommit, + dependenciesChanged: true, + bundle: true, + }), + { flag: 'wx', mode: 0o600 }, + ); + const pointer = path.join(workspace, 'pointer.json'); + await fs.writeFile( + pointer, + JSON.stringify({ directory: path.basename(workspace) }), + { flag: 'wx', mode: 0o600 }, + ); + signal?.throwIfAborted(); + const readyPointer = path.join(tmp, `upgrade-ready-${branch}.json`); + const previous = await fs + .readFile(readyPointer, 'utf8') + .catch(() => 'null'); + await fs.rename(pointer, readyPointer); + try { + const old = JSON.parse(previous)?.directory; + if ( + /^upgrade-[A-Za-z0-9_-]+$/.test(old) && + old !== path.basename(workspace) + ) + await fs.rm(path.join(tmp, old), { recursive: true, force: true }); + } catch { + /* Committed readiness must survive cleanup failure. */ + } + return selected; + } catch (error) { + await fs.rm(workspace, { recursive: true, force: true }); + throw error; + } +} + +async function verifyStagedBundle(selected) { + const manifest = JSON.parse( + await fs.readFile( + path.join(path.dirname(selected.source), 'bundle.json'), + 'utf8', + ), + ); + await verifyPayload(selected, manifest, await runtimeTarget()); + await smokeDependencies(selected.source); +} + +module.exports = { + atLeast, + runtimeTarget, + targetKey, + assertCompatible, + inventory, + verifyPayload, + smokeDependencies, + extractArchive, + stageBundle, + verifyStagedBundle, + sha256, +}; diff --git a/cli/src/internal/maintenance/upgradeArtifacts.cjs b/cli/src/internal/maintenance/upgradeArtifacts.cjs index af8d0bb7..63a0261c 100644 --- a/cli/src/internal/maintenance/upgradeArtifacts.cjs +++ b/cli/src/internal/maintenance/upgradeArtifacts.cjs @@ -218,7 +218,9 @@ async function downloadWithProgress( '--location', ...(quiet ? ['--silent', '--show-error'] : []), '--proto', - '=https', + /^http:\/\/(?:127\.0\.0\.1|\[::1\])(?::\d+)?\//.test(url) + ? '=http,https' + : '=https', '--proto-redir', '=https', '--connect-timeout', @@ -260,6 +262,26 @@ async function stageUpgrade({ signal, onProgress, }) { + const currentPackage = JSON.parse( + await fs + .readFile(path.join(root, 'package.json'), 'utf8') + .catch((error) => { + if (error.code !== 'ENOENT') throw error; + return '{}'; + }), + ); + if (env.QL_UPGRADE_FILE || currentPackage.qinglongUpgradeFormat === 2) { + return require('./platformBundle.cjs').stageBundle({ + root, + tmp, + env, + run, + signal, + onProgress, + verifyStatic: verifyUpgrade, + download: downloadWithProgress, + }); + } const report = onProgress || (async () => {}); const workspace = await fs.mkdtemp(path.join(tmp, 'upgrade-')); const branch = releaseBranch(env); @@ -496,6 +518,8 @@ async function selectedUpgrade(tmp, branch, env = process.env) { const manifest = await verifyUpgrade(selected.source, selected.static, env); if (ready.sourceCommit !== manifest.sourceCommit) throw new Error(upgradeMessage('暂存升级就绪记录不匹配。', env)); + if (ready.bundle) + await require('./platformBundle.cjs').verifyStagedBundle(selected); if (ready.dependenciesChanged) await fs.access(path.join(selected.source, 'node_modules')); else { diff --git a/docker/Dockerfile.upgrade-bundle b/docker/Dockerfile.upgrade-bundle new file mode 100644 index 00000000..5fe14d95 --- /dev/null +++ b/docker/Dockerfile.upgrade-bundle @@ -0,0 +1,23 @@ +# The image digest is supplied by the same workflow run, never a rolling tag. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS bundle +ARG BASE_IMAGE +ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE} +ARG TARGETARCH +ARG TARGETVARIANT +USER root +# The generic "arm" sqlite release is ARMv7. ARMv6 must compile its own addon. +RUN if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \ + apk add --no-cache --virtual .ql-bundle-build build-base binutils python3 nodejs-dev && \ + cd "$(readlink -f /ql/node_modules/sqlite3)" && \ + CFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \ + CXXFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \ + npm_config_nodedir=/usr npm_config_build_from_source=true QEMU_CPU=arm1176 \ + node -e 'require("node:child_process").execFileSync(process.execPath,[require.resolve("@mapbox/node-pre-gyp/bin/node-pre-gyp"),"rebuild","--napi_build_version=6"],{stdio:"inherit"})' && \ + readelf -A lib/binding/napi-v6-linux-arm/node_sqlite3.node | grep -E 'Tag_CPU_arch: v6($|K)' && \ + apk del .ql-bundle-build; \ + fi +RUN if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then export QEMU_CPU=arm1176; fi; \ + node /ql/scripts/build-upgrade-bundle.cjs /ql /out +FROM scratch +COPY --from=bundle /out/ / diff --git a/docs/development/platform-upgrade-bundles.md b/docs/development/platform-upgrade-bundles.md new file mode 100644 index 00000000..452be33d --- /dev/null +++ b/docs/development/platform-upgrade-bundles.md @@ -0,0 +1,47 @@ +# 平台完整升级包 + +CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。 + +## 在线升级 + +`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。 + +面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 发布就绪指针。重启前重新验证完整载荷。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。 + +渠道索引为 GitHub prerelease `upgrade-develop` 或 `upgrade-master` 下的 `latest.json`。索引记录准确字节数、SHA256 和不可变 `upgrade-` release 资产 URL。12 个平台均构建及试运行成功才发布渠道;过期工作流不能覆盖更新提交的渠道。 + +## 手动离线安装和升级 + +从同一个 `upgrade-` release 下载匹配设备的 `.tar.gz` 和独立 `install-bundle.cjs`。独立安装器已内置解析依赖,在目标设备无需 npm、pnpm 或重新安装 SQLite。可对照同 release 的 `latest.json` 核对归档 SHA256。 + +已有安装(默认应用并重启): + +```sh +node install-bundle.cjs --root /ql --file /absolute/path/qinglong--linux-arm64-musl.tar.gz +``` + +只准备,稍后手动重启: + +```sh +node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only +ql reload system +``` + +安装到空目录: + +```sh +node install-bundle.cjs --root /opt/qinglong --file /absolute/path/package.tar.gz --install +QL_DIR=/opt/qinglong bash /opt/qinglong/shell/start-simplify.sh +``` + +`--install` 拒绝非空目录,不覆盖既有配置。该包安装青龙及其 Node 生产依赖;首次启动仍需已有 Python、Bash、PM2、nginx 等相应运行环境,不会在线安装或替换它们。服务启动和系统自启按原手动安装流程配置。 + +## 平台兼容 + +- 最低 Node 为 20.18.1,Node-API 至少 6;安装器只检查,不替换 Node。 +- 以 Node 实际运行架构、ARM 指令集和 libc 类型/版本选择包;不依据 Docker 架构别名猜测。 +- glibc、musl 分包,并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。 +- ARMv6 与 ARMv7 使用不同包名。ARMv6 在目标镜像中从源码编译 SQLite,禁止复用 generic `arm` 的 ARMv7 预编译件;构建需通过加载和内存库读写,失败则不发布渠道。 +- tar 中允许 pnpm 的包内符号链接;拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖,暂存失败保留上一次可用的就绪包。 + +自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引;要求 HTTPS,测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源,避免依赖额外的 raw 域名。 diff --git a/package.json b/package.json index e36e566e..bb03612f 100644 --- a/package.json +++ b/package.json @@ -127,7 +127,8 @@ "undici": "^7.9.0", "uuid": "^11.0.3", "winston": "^3.17.0", - "winston-daily-rotate-file": "^5.0.0" + "winston-daily-rotate-file": "^5.0.0", + "tar": "7.5.20" }, "devDependencies": { "@ant-design/icons": "^5.0.1", @@ -201,5 +202,6 @@ "typescript": "5.2.2", "vh-check": "^2.0.5", "virtualizedtableforantd4": "1.3.0" - } + }, + "qinglongUpgradeFormat": 2 } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 863bffc4..38aac98d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -127,6 +127,9 @@ dependencies: sqlite3: specifier: npm:@whyour/sqlite3@1.1.2 version: /@whyour/sqlite3@1.1.2 + tar: + specifier: 7.5.20 + version: 7.5.20 toad-scheduler: specifier: ^3.0.1 version: 3.0.1 @@ -3053,7 +3056,6 @@ packages: dependencies: minipass: 7.1.3 dev: false - optional: true /@istanbuljs/load-nyc-config@1.1.0: resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} @@ -6623,7 +6625,6 @@ packages: engines: {node: '>=18'} requiresBuild: true dev: false - optional: true /ci-info@3.9.0: resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} @@ -10701,7 +10702,6 @@ packages: dependencies: minipass: 7.1.3 dev: false - optional: true /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -14706,7 +14706,6 @@ packages: minizlib: 3.1.0 yallist: 5.0.0 dev: false - optional: true /terser@5.36.0: resolution: {integrity: sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==} @@ -15727,7 +15726,6 @@ packages: engines: {node: '>=18'} requiresBuild: true dev: false - optional: true /yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} diff --git a/scripts/build-bundle-installer.cjs b/scripts/build-bundle-installer.cjs new file mode 100644 index 00000000..34784fee --- /dev/null +++ b/scripts/build-bundle-installer.cjs @@ -0,0 +1,22 @@ +const path = require('node:path'); +require('../cli/node_modules/esbuild').buildSync({ + entryPoints: [path.resolve(__dirname, '../shell/install-bundle.cjs')], + outfile: path.resolve(process.argv[2] || 'output/upgrade/install-bundle.cjs'), + bundle: true, + platform: 'node', + format: 'cjs', + target: 'node20', + minify: true, + legalComments: 'inline', +}); +const { spawnSync } = require('node:child_process'); +const result = spawnSync( + process.execPath, + [path.resolve(process.argv[2] || 'output/upgrade/install-bundle.cjs')], + { encoding: 'utf8' }, +); +if ( + result.status !== 1 || + !result.stderr.includes('Usage: node install-bundle.cjs') +) + throw new Error('Standalone offline installer did not start correctly'); diff --git a/scripts/build-upgrade-bundle.cjs b/scripts/build-upgrade-bundle.cjs new file mode 100644 index 00000000..8b9adc81 --- /dev/null +++ b/scripts/build-upgrade-bundle.cjs @@ -0,0 +1,112 @@ +const fs = require('node:fs/promises'); +const path = require('node:path'); +const tar = require('tar'); +const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs'); +const { + verifyUpgrade, +} = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs'); + +async function buildBundle(root, output) { + root = path.resolve(root); + output = path.resolve(output); + const target = await bundle.runtimeTarget(); + if (process.platform !== 'linux') + throw new Error('Release bundles must be built and probed on Linux'); + const build = await verifyUpgrade(root, path.join(root, 'static')); + await bundle.smokeDependencies(root); + await fs.mkdir(output, { recursive: true }); + const workspace = await fs.mkdtemp(path.join(output, '.bundle-')); + try { + const payload = path.join(workspace, 'qinglong'); + const source = path.join(payload, 'source'), + staticRoot = path.join(payload, 'static'); + await fs.mkdir(source, { recursive: true }); + const excluded = new Set([ + '.git', + '.tmp', + '.env', + 'data', + 'static', + 'output', + 'node_modules', + ]); + for (const name of await fs.readdir(root)) { + if (excluded.has(name)) continue; + await fs.cp(path.join(root, name), path.join(source, name), { + recursive: true, + verbatimSymlinks: true, + }); + } + await fs.cp( + path.join(root, 'node_modules'), + path.join(source, 'node_modules'), + { recursive: true, verbatimSymlinks: true }, + ); + await fs.cp(path.join(root, 'static'), staticRoot, { + recursive: true, + verbatimSymlinks: true, + }); + const manifest = { + format: 2, + sourceCommit: build.sourceCommit, + version: JSON.parse( + await fs.readFile(path.join(root, 'package.json'), 'utf8'), + ).version, + lockfileSha256: build.lockfileSha256, + target: { + platform: target.platform, + arch: target.arch, + libc: target.libc, + libcMinimum: target.libcVersion, + endianness: target.endianness, + armFloatAbi: target.armFloatAbi, + }, + nodeMinimum: '20.18.1', + nodeApi: 6, + files: await bundle.inventory({ source, static: staticRoot }), + }; + await fs.writeFile( + path.join(payload, 'bundle.json'), + JSON.stringify(manifest), + ); + const filename = `qinglong-${build.sourceCommit}-${bundle.targetKey( + target, + )}.tar.gz`; + await tar.c( + { + gzip: true, + portable: true, + noMtime: true, + cwd: workspace, + file: path.join(output, filename), + }, + ['qinglong'], + ); + const bytes = await fs.readFile(path.join(output, filename)); + const record = { + format: 2, + sourceCommit: build.sourceCommit, + target: bundle.targetKey(target), + filename, + baseImage: process.env.QL_BUNDLE_BASE_IMAGE, + size: bytes.length, + sha256: bundle.sha256(bytes), + }; + await fs.writeFile( + path.join(output, filename + '.json'), + JSON.stringify(record) + '\n', + ); + console.log(JSON.stringify(record)); + return record; + } finally { + await fs.rm(workspace, { recursive: true, force: true }); + } +} +module.exports = { buildBundle }; +if (require.main === module) + buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch( + (e) => { + console.error(e); + process.exitCode = 1; + }, + ); diff --git a/scripts/publish-upgrade-bundles.cjs b/scripts/publish-upgrade-bundles.cjs new file mode 100644 index 00000000..6155b52d --- /dev/null +++ b/scripts/publish-upgrade-bundles.cjs @@ -0,0 +1,152 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { + sha256, +} = require('../cli/src/internal/maintenance/platformBundle.cjs'); +const platforms = [ + 'linux-x64-musl', + 'linux-armv6-musl', + 'linux-armv7-musl', + 'linux-arm64-musl', + 'linux-ppc64-musl', + 'linux-s390x-musl', + 'linux-ia32-musl', + 'linux-x64-glibc', + 'linux-armv7-glibc', + 'linux-arm64-glibc', + 'linux-ppc64-glibc', + 'linux-s390x-glibc', +]; +function releaseIndex(directory, commit, repository) { + if (!/^[a-f0-9]{40}$/.test(commit) || !/^[\w.-]+\/[\w.-]+$/.test(repository)) + throw new Error('Invalid release source'); + const packages = {}; + for (const target of platforms) { + const filename = `qinglong-${commit}-${target}.tar.gz`; + const record = JSON.parse( + fs.readFileSync(path.join(directory, filename + '.json')), + ); + const bytes = fs.readFileSync(path.join(directory, filename)); + if ( + record.format !== 2 || + record.sourceCommit !== commit || + record.target !== target || + record.filename !== filename || + record.size !== bytes.length || + record.sha256 !== sha256(bytes) + ) + throw new Error(`Unverified platform bundle: ${target}`); + packages[target] = { + size: record.size, + sha256: record.sha256, + url: `https://github.com/${repository}/releases/download/upgrade-${commit}/${filename}`, + }; + } + return { format: 2, sourceCommit: commit, packages }; +} +module.exports = { releaseIndex, platforms }; +if (require.main === module) { + const [directory, commit, repository, branch] = process.argv.slice(2); + const index = releaseIndex(directory, commit, repository); + const installer = fs.readFileSync(path.join(directory, 'install-bundle.cjs')); + index.installer = { + size: installer.length, + sha256: sha256(installer), + url: `https://github.com/${repository}/releases/download/upgrade-${commit}/install-bundle.cjs`, + }; + if (!['develop', 'master'].includes(branch)) + throw new Error('Invalid release channel'); + fs.writeFileSync( + path.join(directory, 'latest.json'), + JSON.stringify(index) + '\n', + ); + const gh = (...args) => + execFileSync('gh', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + const tag = `upgrade-${commit}`; + try { + gh('release', 'view', tag, '--repo', repository); + } catch { + gh( + 'release', + 'create', + tag, + '--repo', + repository, + '--target', + commit, + '--prerelease', + '--title', + `Platform upgrade ${commit.slice(0, 12)}`, + '--notes', + 'Verified platform bundles with production dependencies. Existing Node runtime is retained.', + ); + } + const assets = fs + .readdirSync(directory) + .filter((n) => /\.(?:tar\.gz|cjs|json)$/.test(n) && n !== 'latest.json') + .map((n) => path.join(directory, n)); + const published = JSON.parse( + gh('api', `repos/${repository}/releases/tags/${tag}`), + ); + const missing = []; + for (const file of [...assets, path.join(directory, 'latest.json')]) { + const bytes = fs.readFileSync(file); + const previous = published.assets.find( + (a) => a.name === path.basename(file), + ); + if (!previous) missing.push(file); + else if ( + previous.size !== bytes.length || + previous.digest !== `sha256:${sha256(bytes)}` + ) + throw new Error( + `Immutable release asset differs: ${path.basename(file)}`, + ); + } + if (missing.length) + gh('release', 'upload', tag, ...missing, '--repo', repository); + // An older, slower run may publish immutable assets, but must not regress a channel. + if ( + gh( + 'api', + `repos/${repository}/git/ref/heads/${branch}`, + '--jq', + '.object.sha', + ).trim() !== commit + ) { + console.log('Newer source exists; leaving channel unchanged'); + } else { + const channel = `upgrade-${branch}`; + try { + gh('release', 'view', channel, '--repo', repository); + } catch { + gh( + 'release', + 'create', + channel, + '--repo', + repository, + '--target', + commit, + '--prerelease', + '--title', + `Upgrade channel ${branch}`, + '--notes', + 'The latest.json asset selects immutable, fully verified platform bundles.', + ); + } + gh( + 'release', + 'upload', + channel, + path.join(directory, 'latest.json'), + '--repo', + repository, + '--clobber', + ); + } +} diff --git a/shell/install-bundle.cjs b/shell/install-bundle.cjs new file mode 100644 index 00000000..f66dd03b --- /dev/null +++ b/shell/install-bundle.cjs @@ -0,0 +1,193 @@ +#!/usr/bin/env node +// Also published as a standalone, bundled installer for devices with Node only. +const fs = require('node:fs/promises'); +const path = require('node:path'); +const os = require('node:os'); +const { promisify } = require('node:util'); +const execute = promisify(require('node:child_process').execFile); +const { parseArgs } = require('node:util'); +const artifacts = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs'); +const jobs = require('../cli/src/internal/maintenance/panelUpgrade.cjs'); + +async function installBundle(argv = process.argv.slice(2), env = process.env) { + const { values } = parseArgs({ + args: argv, + options: { + root: { type: 'string' }, + file: { type: 'string' }, + 'download-only': { type: 'boolean' }, + install: { type: 'boolean' }, + }, + }); + if (!values.file || !(values.root || env.QL_DIR)) + throw new Error( + 'Usage: node install-bundle.cjs --root /path/to/qinglong --file /path/to/bundle.tar.gz [--download-only | --install]', + ); + if (values.install && values['download-only']) + throw new Error('Choose --install or --download-only'); + const root = path.resolve(values.root || env.QL_DIR); + const existing = await fs.readdir(root).catch((error) => { + if (error.code !== 'ENOENT') throw error; + return []; + }); + if (values.install && existing.length) + throw new Error('--install requires an empty installation directory'); + if (!values.install) await fs.access(path.join(root, 'shell/share.sh')); + let environment = { + ...env, + QL_DIR: root, + QL_UPGRADE_FILE: path.resolve(values.file), + }; + if (!values.install) { + const captured = await execute( + 'bash', + [ + '--noprofile', + '--norc', + '-c', + '. "$1/shell/share.sh"\n. "$1/shell/env.sh"\nimport_config\nexport QlPort="${ql_port:-${QlPort:-5700}}" QlBaseUrl="${ql_base_url:-${QlBaseUrl:-/}}"\nenv -0', + 'ql-offline', + root, + ], + { env: environment, maxBuffer: 1024 * 1024 }, + ); + const loaded = Object.fromEntries( + captured.stdout.split('\0').flatMap((line) => { + const match = /^([A-Za-z_][A-Za-z_0-9]*)=([\s\S]*)$/.exec(line); + return match ? [[match[1], match[2]]] : []; + }), + ); + environment = { + ...environment, + ...loaded, + QL_DIR: root, + QL_UPGRADE_FILE: path.resolve(values.file), + }; + } + const freshWorkspace = values.install + ? await fs.mkdtemp(path.join(os.tmpdir(), 'ql-offline-install-')) + : undefined; + const tmp = freshWorkspace || path.join(root, '.tmp'); + try { + return await artifacts.withUpgradeLock( + tmp, + async () => { + const selected = await artifacts.stageUpgrade({ + root, + tmp, + env: environment, + run: (program, args, capture) => + artifacts.runProcess(environment, program, args, capture), + onProgress: async (p) => console.log(p.phase), + install: async () => { + throw new Error('Offline bundles never install npm dependencies'); + }, + }); + if (values['download-only']) { + console.log('更新已就绪;执行 ql reload system 应用更新。'); + return selected; + } + if (values.install) { + // Build beside the destination, then atomically replace only an empty root. + await fs.mkdir(path.dirname(root), { recursive: true }); + const publication = await fs.mkdtemp( + path.join(path.dirname(root), '.qinglong-install-'), + ); + try { + await fs.cp(selected.source, publication, { + recursive: true, + verbatimSymlinks: true, + }); + await fs.cp(selected.static, path.join(publication, 'static'), { + recursive: true, + }); + await fs.copyFile( + path.join(publication, '.env.example'), + path.join(publication, '.env'), + require('node:fs').constants.COPYFILE_EXCL, + ); + await fs.chmod(path.join(publication, '.env'), 0o600); + await fs.rename(publication, root); + } finally { + await fs.rm(publication, { recursive: true, force: true }); + } + await fs.rm(path.dirname(selected.source), { + recursive: true, + force: true, + }); + await fs.rm( + path.join( + tmp, + `upgrade-ready-${artifacts.releaseBranch(environment)}.json`, + ), + { force: true }, + ); + console.log( + `离线安装完成,保留现有 Node ${ + process.version + }。启动服务:QL_DIR=${JSON.stringify(root)} bash ${JSON.stringify( + path.join(root, 'shell/start-simplify.sh'), + )}`, + ); + return; + } + return jobs.withPanelLifecycle( + root, + tmp, + environment, + async (hook, prepareSelection) => { + const expected = await jobs.stagedMetadata(selected); + const previous = await jobs + .readRunningBuild(environment) + .catch(() => undefined); + const selection = await prepareSelection?.(selected); + try { + return await artifacts.reloadSystem({ + root, + tmp, + env: environment, + data: environment.dir_data || path.join(root, 'data'), + staticRoot: path.join(root, 'static'), + config: + environment.dir_config || path.join(root, 'data/config'), + lifecycle: { + stop: () => hook('stop'), + start: async (recover) => { + await hook('start'); + await artifacts.waitForHealth(environment); + if (!recover) + await jobs.verifyRunningVersion( + environment, + expected, + previous, + ); + }, + }, + additionalReplacements: selection + ? [{ source: selection.source, target: selection.target }] + : [], + }); + } finally { + if (selection) + await fs.rm(selection.directory, { + recursive: true, + force: true, + }); + } + }, + ); + }, + environment, + ); + } finally { + if (freshWorkspace) + await fs.rm(freshWorkspace, { recursive: true, force: true }); + } +} + +module.exports = { installBundle }; +if (require.main === module) + installBundle().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); diff --git a/test/back/platform-bundle.test.cjs b/test/back/platform-bundle.test.cjs new file mode 100644 index 00000000..7e9b1d74 --- /dev/null +++ b/test/back/platform-bundle.test.cjs @@ -0,0 +1,353 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const path = require('node:path'); +const os = require('node:os'); +const tar = require('tar'); +const bundle = require('../../cli/src/internal/maintenance/platformBundle.cjs'); +const artifacts = require('../../shell/upgrade.cjs'); +const { + writeManifest, + sourceCommit, +} = require('../helpers/upgrade-fixture.cjs'); +const { + releaseIndex, + platforms, +} = require('../../scripts/publish-upgrade-bundles.cjs'); + +async function fixture(t) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-bundle-test-')); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const root = path.join(directory, 'installed'), + tmp = path.join(root, '.tmp'); + const payload = path.join(directory, 'qinglong'); + const source = path.join(payload, 'source'), + staticRoot = path.join(payload, 'static'); + await fs.mkdir(tmp, { recursive: true }); + await fs.mkdir( + path.join(source, 'node_modules/.pnpm/sqlite/node_modules/sqlite3'), + { recursive: true }, + ); + await fs.symlink( + '.pnpm/sqlite/node_modules/sqlite3', + path.join(source, 'node_modules/sqlite3'), + ); + await fs.writeFile( + path.join(source, 'node_modules/sqlite3/package.json'), + '{"name":"sqlite3"}', + ); + await fs.mkdir(path.join(source, 'sample')); + await fs.writeFile(path.join(source, 'sample/config.sample.sh'), '# config'); + await fs.writeFile( + path.join(source, 'package.json'), + '{"version":"2.24.0","qinglongUpgradeFormat":2}', + ); + await fs.writeFile( + path.join(root, 'package.json'), + '{"qinglongUpgradeFormat":2}', + ); + await fs.mkdir(path.join(staticRoot, 'build'), { recursive: true }); + await fs.writeFile(path.join(staticRoot, 'build/app.js'), 'backend'); + writeManifest(source, staticRoot); + const target = await bundle.runtimeTarget(); + const manifest = { + format: 2, + sourceCommit, + lockfileSha256: bundle.sha256( + await fs.readFile(path.join(source, 'pnpm-lock.yaml')), + ), + target: { ...target, libcMinimum: target.libcVersion }, + nodeMinimum: '20.18.1', + nodeApi: 6, + files: await bundle.inventory({ source, static: staticRoot }), + }; + const archive = path.join(directory, 'package.tar.gz'); + const pack = async () => { + await fs.writeFile( + path.join(payload, 'bundle.json'), + JSON.stringify(manifest), + ); + await tar.c( + { + file: archive, + cwd: directory, + gzip: true, + portable: true, + noMtime: true, + }, + ['qinglong'], + ); + }; + await pack(); + return { + directory, + root, + tmp, + source, + staticRoot, + target, + manifest, + archive, + pack, + }; +} + +test('online and offline stage the same complete payload, preserve pnpm links and never install dependencies', async (t) => { + const f = await fixture(t), + bytes = await fs.readFile(f.archive), + progress = []; + const entry = { + size: bytes.length, + sha256: bundle.sha256(bytes), + url: 'https://example.invalid/immutable.tar.gz', + }; + let calls = 0, + probes = 0; + const run = async (program, args) => { + assert.equal(program, 'curl'); + calls++; + const output = args[args.indexOf('--output') + 1]; + if (args.at(-1).endsWith('latest.json')) + await fs.writeFile( + output, + JSON.stringify({ + format: 2, + sourceCommit, + packages: { [bundle.targetKey(f.target)]: entry }, + }), + ); + else { + await fs.writeFile(output, bytes); + await fs.writeFile( + args[args.indexOf('--dump-header') + 1], + 'HTTP/2 200 OK\r\n\r\n', + ); + } + }; + const options = { + root: f.root, + tmp: f.tmp, + env: { QL_BRANCH: 'develop' }, + run, + verifyStatic: artifacts.verifyUpgrade, + download: artifacts.downloadWithProgress, + probe: async () => { + probes++; + }, + onProgress: async (p) => progress.push(p), + }; + const online = await bundle.stageBundle(options); + const first = await bundle.inventory(online); + assert.equal(calls, 2); + assert.equal(probes, 1); + assert.ok(progress.some((p) => p.percent === 100)); + assert.ok(progress.some((p) => p.percent === 0)); + const offline = await bundle.stageBundle({ + ...options, + env: { QL_BRANCH: 'develop', QL_UPGRADE_FILE: f.archive }, + run: () => assert.fail('offline network access'), + }); + assert.equal(probes, 2); + assert.deepEqual(await bundle.inventory(offline), first); + assert.equal( + await fs.readlink(path.join(offline.source, 'node_modules/sqlite3')), + '.pnpm/sqlite/node_modules/sqlite3', + ); + await assert.rejects(fs.access(online.source)); +}); + +test('native target compatibility distinguishes ARMv6/ARMv7 and libc and retains the existing Node requirement', async (t) => { + const f = await fixture(t), + target = { + platform: 'linux', + arch: 'armv6', + libc: 'musl', + libcVersion: '1.2.5', + }; + const manifest = { + ...f.manifest, + target: { ...target, libcMinimum: '1.2.4' }, + }; + bundle.assertCompatible(manifest, target, '20.20.2'); + assert.throws( + () => bundle.assertCompatible(manifest, { ...target, arch: 'armv7' }), + /platform mismatch/, + ); + assert.throws( + () => bundle.assertCompatible(manifest, { ...target, libc: 'glibc' }), + /platform mismatch/, + ); + assert.throws( + () => bundle.assertCompatible(manifest, target, '18.20.0'), + /requires Node/, + ); + assert.throws( + () => + bundle.assertCompatible(manifest, { ...target, libcVersion: '1.1.24' }), + /requires musl/, + ); +}); + +test('dependency tampering and payload link escapes are rejected', async (t) => { + const f = await fixture(t), + selected = { source: f.source, static: f.staticRoot }; + await bundle.verifyPayload(selected, f.manifest, f.target); + await fs.writeFile( + path.join(f.source, 'node_modules/sqlite3/package.json'), + '{"tampered":true}', + ); + await assert.rejects( + bundle.verifyPayload(selected, f.manifest, f.target), + /checksums/, + ); + await fs.symlink('/etc/passwd', path.join(f.source, 'outside')); + await assert.rejects(bundle.inventory(selected), /escapes/); +}); + +test('archive traversal, absolute links and writes through internal symlinks are rejected before extraction', async (t) => { + const f = await fixture(t), + dest = path.join(f.directory, 'extract'); + await fs.mkdir(dest); + await fs.symlink('/etc', path.join(f.directory, 'qinglong/bad')); + await tar.c({ cwd: f.directory, file: f.archive, gzip: true }, [ + 'qinglong/bad', + ]); + await assert.rejects(bundle.extractArchive(f.archive, dest), /symlink/); + await tar.c( + { cwd: f.directory, file: f.archive, gzip: true, prefix: '../outside' }, + ['qinglong/source/package.json'], + ); + await assert.rejects(bundle.extractArchive(f.archive, dest), /Unsafe/); + await fs.mkdir(path.join(f.directory, 'qinglong/real')); + await fs.writeFile(path.join(f.directory, 'qinglong/real/file'), 'value'); + await fs.symlink('real', path.join(f.directory, 'qinglong/link')); + await tar.c({ cwd: f.directory, file: f.archive, gzip: true }, [ + 'qinglong/link', + 'qinglong/link/file', + ]); + await assert.rejects( + bundle.extractArchive(f.archive, dest), + /through a symlink/, + ); + assert.deepEqual(await fs.readdir(dest), []); +}); + +test('an incomplete or corrupt platform download cannot replace an existing ready pointer', async (t) => { + const f = await fixture(t), + pointer = path.join(f.tmp, 'upgrade-ready-develop.json'); + await fs.writeFile(pointer, 'previous-ready'); + await assert.rejects( + bundle.stageBundle({ + root: f.root, + tmp: f.tmp, + env: { QL_BRANCH: 'develop', QL_UPGRADE_FILE: f.archive }, + verifyStatic: artifacts.verifyUpgrade, + probe: async () => { + throw new Error('native addon failed'); + }, + }), + /native addon failed/, + ); + assert.equal(await fs.readFile(pointer, 'utf8'), 'previous-ready'); + assert.equal( + (await fs.readdir(f.tmp)).filter((n) => n.startsWith('upgrade-')).length, + 1, + ); +}); + +test('publishing requires every platform and validates all immutable archive hashes', async (t) => { + const f = await fixture(t), + repository = 'whyour/qinglong'; + assert.throws(() => releaseIndex(f.directory, sourceCommit, repository)); + for (const target of platforms) { + const filename = `qinglong-${sourceCommit}-${target}.tar.gz`, + bytes = Buffer.from(target); + await fs.writeFile(path.join(f.directory, filename), bytes); + await fs.writeFile( + path.join(f.directory, filename + '.json'), + JSON.stringify({ + format: 2, + sourceCommit, + target, + filename, + size: bytes.length, + sha256: bundle.sha256(bytes), + }), + ); + } + const index = releaseIndex(f.directory, sourceCommit, repository); + assert.equal(Object.keys(index.packages).length, 12); + await fs.appendFile( + path.join(f.directory, `qinglong-${sourceCommit}-${platforms[0]}.tar.gz`), + 'corrupt', + ); + assert.throws( + () => releaseIndex(f.directory, sourceCommit, repository), + /Unverified/, + ); +}); + +test('fresh offline installation needs no package manager, keeps Node and publishes a complete directory', async (t) => { + const f = await fixture(t); + await fs.writeFile(path.join(f.source, '.env.example'), 'QlPort=5700\n'); + await fs.writeFile(path.join(f.source, 'version.yaml'), 'version: 2.24.0\n'); + await fs.writeFile( + path.join(f.source, 'node_modules/sqlite3/index.js'), + `exports.Database=class {serialize(fn){fn()}run(){}get(sql,cb){cb(null,{value:42})}close(){}};`, + ); + for (const name of ['sequelize', 'cron-parser', '@grpc/grpc-js', 'undici']) { + const dir = path.join(f.source, 'node_modules', name); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, 'index.js'), 'module.exports={};'); + } + f.manifest.files = await bundle.inventory({ + source: f.source, + static: f.staticRoot, + }); + await f.pack(); + const destination = path.join(f.directory, 'fresh'), + node = process.execPath; + await require('../../shell/install-bundle.cjs').installBundle( + ['--root', destination, '--file', f.archive, '--install'], + { ...process.env, PATH: '', QL_BRANCH: 'develop' }, + ); + assert.equal(process.execPath, node); + assert.equal( + await fs.readFile(path.join(destination, '.env'), 'utf8'), + 'QlPort=5700\n', + ); + assert.equal( + (await fs.stat(path.join(destination, '.env'))).mode & 0o777, + 0o600, + ); + assert.equal( + await fs.readFile(path.join(destination, 'static/build/app.js'), 'utf8'), + 'backend', + ); + assert.ok( + ( + await fs.stat(path.join(destination, 'node_modules/sqlite3/package.json')) + ).isFile(), + ); + assert.equal( + (await fs.readdir(f.directory)).filter((n) => + n.startsWith('.qinglong-install-'), + ).length, + 0, + ); + await fs.writeFile(path.join(destination, 'keep-user-data'), 'keep'); + await assert.rejects( + require('../../shell/install-bundle.cjs').installBundle([ + '--root', + destination, + '--file', + f.archive, + '--install', + ]), + /empty installation directory/, + ); + assert.equal( + await fs.readFile(path.join(destination, 'keep-user-data'), 'utf8'), + 'keep', + ); +});