feat: ship verified platform upgrade bundles with production dependencies

cli/scripts/build.cjs

@@ -17,6 +17,10 @@ fs.copyFileSync(
   path.join(root, 'src/internal/maintenance/upgradeArtifacts.cjs'),
   path.join(root, 'dist/internal/maintenance/upgradeArtifacts.cjs'),
 );
+fs.copyFileSync(
+  path.join(root, 'src/internal/maintenance/platformBundle.cjs'),
+  path.join(root, 'dist/internal/maintenance/platformBundle.cjs'),
+);
 require('esbuild').buildSync({
   entryPoints: [path.join(root, 'src/shared/cli/commander.ts')],
   outfile: path.join(root, 'dist/shared/cli/commander.js'),

cli/src/internal/maintenance/panelUpgrade.cjs

@@ -429,7 +429,15 @@ async function runJob({ root, tmp, env = process.env }, id) {
   }
 }
 
-module.exports = { getStatus, startJob, runJob };
+module.exports = {
+  getStatus,
+  startJob,
+  runJob,
+  withPanelLifecycle,
+  readRunningBuild,
+  verifyRunningVersion,
+  stagedMetadata,
+};
 async function launchWorker(id) {
   // detached creates a session but keeps the original parent. PM2 kills the
   // backend's entire descendant tree, so an intermediate launcher must exit

cli/src/internal/maintenance/platformBundle.cjs

@@ -0,0 +1,471 @@
+// One installed verifier handles release downloads and local offline archives.
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const os = require('node:os');
+const { execFile } = require('node:child_process');
+const { promisify } = require('node:util');
+const tar = require('tar');
+const execute = promisify(execFile);
+const sha256 = (bytes) =>
+  crypto.createHash('sha256').update(bytes).digest('hex');
+const inside = (root, filename) => {
+  const relative = path.relative(root, filename);
+  return (
+    relative !== '..' &&
+    !relative.startsWith(`..${path.sep}`) &&
+    !path.isAbsolute(relative)
+  );
+};
+const versionParts = (v) => String(v).replace(/^v/, '').split('.').map(Number);
+function atLeast(actual, minimum) {
+  const a = versionParts(actual),
+    b = versionParts(minimum);
+  return (
+    b.every((n) => Number.isFinite(n)) &&
+    a.every((n) => Number.isFinite(n)) &&
+    (a[0] > b[0] ||
+      (a[0] === b[0] &&
+        (a[1] > b[1] || (a[1] === b[1] && (a[2] || 0) >= (b[2] || 0)))))
+  );
+}
+
+async function runtimeTarget() {
+  let libc, libcVersion;
+  if (process.platform === 'linux') {
+    const header = process.report.getReport().header;
+    if (header.glibcVersionRuntime) {
+      libc = 'glibc';
+      libcVersion = header.glibcVersionRuntime;
+    } else {
+      let text = '';
+      try {
+        text = (await execute('ldd', ['--version'])).stdout;
+      } catch (error) {
+        text = `${error.stdout || ''}\n${error.stderr || ''}`;
+      }
+      const match = text.match(/musl[\s\S]*?Version\s+(\d+\.\d+\.\d+)/i);
+      if (!match)
+        throw new Error(
+          'Cannot identify Linux libc; refusing a native dependency bundle',
+        );
+      libc = 'musl';
+      libcVersion = match[1];
+    }
+  }
+  const armVersion =
+    process.arch === 'arm'
+      ? Number(process.config.variables.arm_version)
+      : undefined;
+  if (process.arch === 'arm' && ![6, 7, 8].includes(armVersion))
+    throw new Error('Cannot identify ARM instruction set');
+  const arch =
+    process.arch === 'arm' ? `armv${armVersion >= 7 ? 7 : 6}` : process.arch;
+  const armFloatAbi =
+    process.arch === 'arm' ? process.config.variables.arm_float_abi : undefined;
+  if (
+    process.arch === 'arm' &&
+    !['hard', 'soft', 'softfp'].includes(armFloatAbi)
+  )
+    throw new Error('Cannot identify ARM floating-point ABI');
+  return {
+    platform: process.platform,
+    arch,
+    libc,
+    libcVersion,
+    endianness: os.endianness(),
+    armFloatAbi,
+  };
+}
+
+function targetKey(target) {
+  return [target.platform, target.arch, target.libc].filter(Boolean).join('-');
+}
+
+function assertCompatible(manifest, target, node = process.versions.node) {
+  if (
+    manifest.format !== 2 ||
+    !/^[a-f0-9]{40}$/.test(manifest.sourceCommit) ||
+    !/^[a-f0-9]{64}$/.test(manifest.lockfileSha256) ||
+    !manifest.files ||
+    typeof manifest.files !== 'object' ||
+    Array.isArray(manifest.files) ||
+    !Number.isSafeInteger(manifest.nodeApi) ||
+    manifest.nodeApi < 6
+  )
+    throw new Error('Invalid platform bundle manifest');
+  if (targetKey(manifest.target || {}) !== targetKey(target))
+    throw new Error(`Bundle platform mismatch: need ${targetKey(target)}`);
+  if (
+    !atLeast(node, manifest.nodeMinimum) ||
+    Number(process.versions.napi) < manifest.nodeApi
+  )
+    throw new Error(
+      `Bundle requires Node >= ${manifest.nodeMinimum} and Node-API ${manifest.nodeApi}; the existing Node is retained`,
+    );
+  if (
+    manifest.target.endianness !== target.endianness ||
+    manifest.target.armFloatAbi !== target.armFloatAbi
+  )
+    throw new Error(
+      'Bundle native byte order or ARM floating-point ABI mismatch',
+    );
+  if (target.libc && !atLeast(target.libcVersion, manifest.target.libcMinimum))
+    throw new Error(
+      `Bundle requires ${target.libc} >= ${manifest.target.libcMinimum}`,
+    );
+}
+
+async function inventory(roots) {
+  const files = {};
+  async function visit(directory, prefix) {
+    for (const name of (await fs.readdir(directory)).sort()) {
+      const filename = path.join(directory, name),
+        relative = `${prefix}/${name}`;
+      const stat = await fs.lstat(filename);
+      if (stat.isDirectory()) await visit(filename, relative);
+      else if (stat.isFile())
+        files[relative] = {
+          type: 'file',
+          mode: stat.mode & 0o777,
+          sha256: sha256(await fs.readFile(filename)),
+        };
+      else if (stat.isSymbolicLink()) {
+        const link = await fs.readlink(filename);
+        const boundary = roots[prefix.split('/')[0]];
+        if (
+          path.isAbsolute(link) ||
+          !inside(boundary, path.resolve(path.dirname(filename), link)) ||
+          !inside(await fs.realpath(boundary), await fs.realpath(filename))
+        )
+          throw new Error(`Bundle link escapes its payload: ${relative}`);
+        files[relative] = { type: 'link', link };
+      } else throw new Error(`Unsupported bundle entry: ${relative}`);
+    }
+  }
+  for (const [name, root] of Object.entries(roots)) await visit(root, name);
+  return files;
+}
+
+async function verifyPayload(selected, manifest, target) {
+  assertCompatible(manifest, target);
+  for (const name of ['data', '.tmp', '.git', '.env', 'static']) {
+    if (
+      await fs.lstat(path.join(selected.source, name)).catch((error) => {
+        if (error.code !== 'ENOENT') throw error;
+        return null;
+      })
+    )
+      throw new Error(`Reserved installation path in bundle: ${name}`);
+  }
+  const actual = await inventory({
+    source: selected.source,
+    static: selected.static,
+  });
+  const names = Object.keys(actual).sort(),
+    expected = Object.keys(manifest.files).sort();
+  if (
+    JSON.stringify(names) !== JSON.stringify(expected) ||
+    names.some(
+      (name) =>
+        JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
+    )
+  )
+    throw new Error(
+      'Platform bundle files or dependency checksums do not match',
+    );
+  if (
+    sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !==
+    manifest.lockfileSha256
+  )
+    throw new Error('Platform bundle lockfile mismatch');
+  await fs.access(
+    path.join(selected.source, 'node_modules/sqlite3/package.json'),
+  );
+}
+
+async function smokeDependencies(source, signal) {
+  // This probe is installed verifier code, never a verifier supplied by an archive.
+  const probe = `
+    const path = require('node:path'),fs = require('node:fs');
+    const root = fs.realpathSync(process.cwd());
+    const r = require('node:module').createRequire(path.join(root,'package.json'));
+    for (const name of ['sqlite3','sequelize','cron-parser','@grpc/grpc-js','undici']) {
+      if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name);
+      r(name);
+    }
+    const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:');
+    db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)');
+      db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});});
+  `;
+  await execute(process.execPath, ['-e', probe], {
+    cwd: source,
+    env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
+    timeout: 30000,
+    signal,
+  });
+}
+
+async function extractArchive(archive, destination) {
+  const entries = new Map(),
+    links = new Set();
+  let unpacked = 0,
+    invalid;
+  await tar.t({
+    file: archive,
+    strict: true,
+    onReadEntry(entry) {
+      try {
+        const name = entry.path.replace(/\/$/, '');
+        if (
+          !name ||
+          name.includes('\\') ||
+          /[\x00-\x1f\x7f]/.test(name) ||
+          name.split('/').some((p) => !p || p === '.' || p === '..') ||
+          !(name === 'qinglong' || name.startsWith('qinglong/')) ||
+          entries.has(name) ||
+          !['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type)
+        )
+          throw new Error(`Unsafe bundle archive entry: ${name}`);
+        unpacked += entry.size;
+        if (unpacked > 2 * 1024 ** 3 || entries.size > 200000)
+          throw new Error('Bundle archive exceeds extraction limits');
+        entries.set(name, entry.type);
+        if (entry.type === 'SymbolicLink') {
+          if (
+            path.posix.isAbsolute(entry.linkpath) ||
+            entry.linkpath.includes('\\') ||
+            !inside(
+              '/qinglong',
+              path.posix.resolve('/', path.posix.dirname(name), entry.linkpath),
+            )
+          )
+            throw new Error('Unsafe bundle symlink');
+          links.add(name);
+        }
+      } catch (error) {
+        invalid ||= error;
+      }
+    },
+  });
+  if (invalid) throw invalid;
+  for (const name of entries.keys()) {
+    let parent = path.posix.dirname(name);
+    while (parent !== '.') {
+      if (links.has(parent))
+        throw new Error('Archive writes through a symlink');
+      parent = path.posix.dirname(parent);
+    }
+  }
+  await tar.x({
+    file: archive,
+    cwd: destination,
+    strict: true,
+    preservePaths: false,
+    noChown: true,
+    umask: 0,
+  });
+}
+
+async function stageBundle({
+  root,
+  tmp,
+  env,
+  run,
+  signal,
+  onProgress,
+  verifyStatic,
+  download,
+  probe = smokeDependencies,
+}) {
+  let local = env.QL_UPGRADE_FILE;
+  if (!local) {
+    const current = JSON.parse(
+      await fs
+        .readFile(path.join(root, 'package.json'), 'utf8')
+        .catch((error) => {
+          if (error.code !== 'ENOENT') throw error;
+          return '{}';
+        }),
+    );
+    if (current.qinglongUpgradeFormat !== 2) return null; // One-time compatibility path for older releases.
+  }
+  const target = await runtimeTarget();
+  const branch = ['develop', 'debian-dev'].includes(env.QL_BRANCH)
+    ? 'develop'
+    : 'master';
+  const workspace = await fs.mkdtemp(path.join(tmp, 'upgrade-'));
+  const report = onProgress || (async () => {});
+  try {
+    const archive = path.join(workspace, 'bundle.tar.gz');
+    let expectedCommit;
+    if (local) {
+      if (!path.isAbsolute(local))
+        throw new Error('Offline bundle path must be absolute');
+      // Snapshot the local input so replacing it cannot race validation/extraction.
+      await fs.copyFile(local, archive);
+    } else {
+      const indexUrl =
+        env.QL_UPGRADE_INDEX_URL ||
+        `https://github.com/whyour/qinglong/releases/download/upgrade-${branch}/latest.json`;
+      const checkUrl = (url) => {
+        const parsed = new URL(url);
+        if (
+          parsed.protocol !== 'https:' &&
+          !(
+            parsed.protocol === 'http:' &&
+            ['127.0.0.1', '[::1]'].includes(parsed.hostname)
+          )
+        )
+          throw new Error('Upgrade URLs require HTTPS');
+      };
+      checkUrl(indexUrl);
+      const indexFile = path.join(workspace, 'index.json');
+      await run('curl', [
+        '--fail',
+        '--silent',
+        '--show-error',
+        '--location',
+        '--proto',
+        new URL(indexUrl).protocol === 'http:' ? '=http,https' : '=https',
+        '--proto-redir',
+        '=https',
+        '--header',
+        'Cache-Control: no-cache',
+        '--connect-timeout',
+        '10',
+        '--max-time',
+        '30',
+        '--max-filesize',
+        '1048576',
+        '--output',
+        indexFile,
+        indexUrl,
+      ]);
+      const index = JSON.parse(await fs.readFile(indexFile, 'utf8'));
+      const entry = index.packages?.[targetKey(target)];
+      if (
+        index.format !== 2 ||
+        !/^[a-f0-9]{40}$/.test(index.sourceCommit) ||
+        !entry ||
+        !Number.isSafeInteger(entry.size) ||
+        entry.size <= 0 ||
+        entry.size > 512 * 1024 ** 2 ||
+        !/^[a-f0-9]{64}$/.test(entry.sha256)
+      )
+        throw new Error(`No verified update bundle for ${targetKey(target)}`);
+      checkUrl(entry.url);
+      expectedCommit = index.sourceCommit;
+      await report({
+        phase: 'downloading',
+        artifact: 'qinglong-bundle',
+        downloadedBytes: 0,
+        totalBytes: entry.size,
+        percent: 0,
+      });
+      await download(
+        run,
+        archive,
+        entry.url,
+        'qinglong-bundle',
+        report,
+        true,
+        entry,
+      );
+    }
+    if ((await fs.stat(archive)).size > 512 * 1024 ** 2)
+      throw new Error('Bundle is too large');
+    await report({
+      phase: 'extracting',
+      artifact: 'qinglong-bundle',
+      percent: null,
+    });
+    await extractArchive(archive, workspace);
+    const incoming = path.join(workspace, 'qinglong');
+    const manifest = JSON.parse(
+      await fs.readFile(path.join(incoming, 'bundle.json'), 'utf8'),
+    );
+    if (expectedCommit && manifest.sourceCommit !== expectedCommit)
+      throw new Error('Update index and bundle commit differ');
+    const selected = {
+      source: path.join(workspace, `qinglong-${branch}`),
+      static: path.join(workspace, `qinglong-static-${branch}`),
+    };
+    await fs.rename(path.join(incoming, 'source'), selected.source);
+    await fs.rename(path.join(incoming, 'static'), selected.static);
+    await report({ phase: 'verifying', percent: null });
+    await verifyPayload(selected, manifest, target);
+    const build = await verifyStatic(selected.source, selected.static, env);
+    if (build.sourceCommit !== manifest.sourceCommit)
+      throw new Error('Bundle and build commit differ');
+    await report({ phase: 'dependencies', percent: null });
+    await probe(selected.source, signal);
+    await fs.writeFile(
+      path.join(workspace, 'bundle.json'),
+      JSON.stringify(manifest),
+      { flag: 'wx', mode: 0o600 },
+    );
+    await fs.writeFile(
+      path.join(workspace, 'ready.json'),
+      JSON.stringify({
+        ...selected,
+        root: path.resolve(root),
+        sourceCommit: manifest.sourceCommit,
+        dependenciesChanged: true,
+        bundle: true,
+      }),
+      { flag: 'wx', mode: 0o600 },
+    );
+    const pointer = path.join(workspace, 'pointer.json');
+    await fs.writeFile(
+      pointer,
+      JSON.stringify({ directory: path.basename(workspace) }),
+      { flag: 'wx', mode: 0o600 },
+    );
+    signal?.throwIfAborted();
+    const readyPointer = path.join(tmp, `upgrade-ready-${branch}.json`);
+    const previous = await fs
+      .readFile(readyPointer, 'utf8')
+      .catch(() => 'null');
+    await fs.rename(pointer, readyPointer);
+    try {
+      const old = JSON.parse(previous)?.directory;
+      if (
+        /^upgrade-[A-Za-z0-9_-]+$/.test(old) &&
+        old !== path.basename(workspace)
+      )
+        await fs.rm(path.join(tmp, old), { recursive: true, force: true });
+    } catch {
+      /* Committed readiness must survive cleanup failure. */
+    }
+    return selected;
+  } catch (error) {
+    await fs.rm(workspace, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+async function verifyStagedBundle(selected) {
+  const manifest = JSON.parse(
+    await fs.readFile(
+      path.join(path.dirname(selected.source), 'bundle.json'),
+      'utf8',
+    ),
+  );
+  await verifyPayload(selected, manifest, await runtimeTarget());
+  await smokeDependencies(selected.source);
+}
+
+module.exports = {
+  atLeast,
+  runtimeTarget,
+  targetKey,
+  assertCompatible,
+  inventory,
+  verifyPayload,
+  smokeDependencies,
+  extractArchive,
+  stageBundle,
+  verifyStagedBundle,
+  sha256,
+};

cli/src/internal/maintenance/upgradeArtifacts.cjs

@@ -218,7 +218,9 @@ async function downloadWithProgress(
       '--location',
       ...(quiet ? ['--silent', '--show-error'] : []),
       '--proto',
-      '=https',
+      /^http:\/\/(?:127\.0\.0\.1|\[::1\])(?::\d+)?\//.test(url)
+        ? '=http,https'
+        : '=https',
       '--proto-redir',
       '=https',
       '--connect-timeout',
@@ -260,6 +262,26 @@ async function stageUpgrade({
   signal,
   onProgress,
 }) {
+  const currentPackage = JSON.parse(
+    await fs
+      .readFile(path.join(root, 'package.json'), 'utf8')
+      .catch((error) => {
+        if (error.code !== 'ENOENT') throw error;
+        return '{}';
+      }),
+  );
+  if (env.QL_UPGRADE_FILE || currentPackage.qinglongUpgradeFormat === 2) {
+    return require('./platformBundle.cjs').stageBundle({
+      root,
+      tmp,
+      env,
+      run,
+      signal,
+      onProgress,
+      verifyStatic: verifyUpgrade,
+      download: downloadWithProgress,
+    });
+  }
   const report = onProgress || (async () => {});
   const workspace = await fs.mkdtemp(path.join(tmp, 'upgrade-'));
   const branch = releaseBranch(env);
@@ -496,6 +518,8 @@ async function selectedUpgrade(tmp, branch, env = process.env) {
   const manifest = await verifyUpgrade(selected.source, selected.static, env);
   if (ready.sourceCommit !== manifest.sourceCommit)
     throw new Error(upgradeMessage('暂存升级就绪记录不匹配。', env));
+  if (ready.bundle)
+    await require('./platformBundle.cjs').verifyStagedBundle(selected);
   if (ready.dependenciesChanged)
     await fs.access(path.join(selected.source, 'node_modules'));
   else {