Keep develop upgrade bundles in one rolling release

docs/development/platform-upgrade-bundles.md

@@ -14,9 +14,9 @@ CI 从同次构建的不可变镜像摘要导出完整 tar.gz，包含源码中
 gh workflow run build-docker-image.yml --ref develop
 ```
 
-开发包更新共用的 `upgrade-develop/latest.json`。`develop` 和 `debian-dev` 镜像都使用该索引，按实际 Node 架构和 musl/glibc 选择包，无需设置测试专用下载源。开发批次位于 `upgrade-build-develop-<run-id>-<run-attempt>`，tag 不带版本号；索引中仍保留开发版本供客户端判断更新。开发批次不会改写正式版本或 `upgrade-master`。同一开发渠道的镜像构建和包发布串行执行，与 master 独立。
+开发包只使用 `upgrade-develop` 这一个 prerelease。`develop` 和 `debian-dev` 镜像读取其中的 `latest.json`，按实际 Node 架构和 musl/glibc 选择包，无需设置测试专用下载源。新批次先以带构建 ID 的文件名上传并校验，再切换 `latest.json`；索引仍保留开发版本供客户端判断更新。开发渠道不会改写正式版本或 `upgrade-master`。同一开发渠道的镜像构建和包发布串行执行，与 master 独立。
 
-开发包只保留最新一个已验证批次：所有平台资产和渠道索引的大小、SHA256 校验成功后，删除其他名称匹配的开发构建 prerelease；不删除正式包、渠道索引或无法分类的旧 release。构建或上传失败不会删除上一套可用包。旧批次被删除后，已选中旧索引但尚未下载完的客户端可能需要重新发起下载；准备失败仍保留旧服务和数据。
+开发包只保留最新一个已验证批次：所有平台资产和渠道索引的大小、SHA256 校验成功后，清理 `upgrade-develop` 中旧批次资产及旧命名的开发构建 prerelease；不删除正式包或无法分类的 release。构建或上传失败不会删除上一套可用包。旧资产被删除后，已选中旧索引但尚未下载完的客户端可能需要重新发起下载；准备失败仍保留旧服务和数据。
 
 开发和正式包都必须在 11 个平台完成归档、解包后复验及依赖试运行，并且全部上传资产的大小和 SHA256 复验通过后，才更新各自索引。平台范围与现有 Alpine/Debian 镜像一致：ARMv6 和 ARMv7 仅提供 musl 包，Debian 不再发布 ARMv7 镜像或 glibc 平台包。
 

scripts/publish-upgrade-bundles.cjs

@@ -18,11 +18,12 @@ const platforms = [
   'linux-s390x-glibc',
 ];
 function releaseIndex(directory, commit, repository, options = {}) {
-  const { version, releaseTag = `upgrade-${commit}` } = options;
+  const { version, releaseTag = `upgrade-${commit}`, assetBuildId } = options;
   if (
     !/^[a-f0-9]{40}$/.test(commit) ||
     !/^[\w.-]+\/[\w.-]+$/.test(repository) ||
-    !/^[\w.+-]+$/.test(releaseTag)
+    !/^[\w.+-]+$/.test(releaseTag) ||
+    (assetBuildId && !/^\d+-\d+$/.test(assetBuildId))
   )
     throw new Error('Invalid release source');
   const packages = {};
@@ -47,11 +48,45 @@ function releaseIndex(directory, commit, repository, options = {}) {
       sha256: record.sha256,
       url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
         releaseTag,
-      )}/${filename}`,
+      )}/${
+        assetBuildId
+          ? filename.replace(/\.tar\.gz$/, `-${assetBuildId}.tar.gz`)
+          : filename
+      }`,
     };
   }
   return { format: 2, sourceCommit: commit, packages };
 }
+function stageDevelopmentAssets(directory, commit, buildId) {
+  const staged = fs.mkdtempSync(path.join(directory, '.upgrade-develop-'));
+  const assets = [];
+  try {
+    for (const target of platforms) {
+      const filename = `qinglong-${commit}-${target}.tar.gz`;
+      const archiveName = filename.replace(/\.tar\.gz$/, `-${buildId}.tar.gz`);
+      const archive = path.join(staged, archiveName);
+      fs.linkSync(path.join(directory, filename), archive);
+      const record = JSON.parse(
+        fs.readFileSync(path.join(directory, filename + '.json')),
+      );
+      fs.writeFileSync(
+        archive + '.json',
+        JSON.stringify({
+          ...record,
+          filename: archiveName,
+        }) + '\n',
+      );
+      assets.push(archive, archive + '.json');
+    }
+    const installer = path.join(staged, `install-bundle-${buildId}.cjs`);
+    fs.linkSync(path.join(directory, 'install-bundle.cjs'), installer);
+    assets.push(installer);
+    return { staged, assets };
+  } catch (error) {
+    fs.rmSync(staged, { recursive: true, force: true });
+    throw error;
+  }
+}
 function publishBundles({
   directory,
   commit,
@@ -76,12 +111,13 @@ function publishBundles({
   // that an in-flight download has already selected by hash.
   const development = branch === 'develop';
   const tag = development
-    ? `upgrade-build-develop-${buildId}`
+    ? 'upgrade-develop'
     : `upgrade-build-v${version}-${buildId}`;
   const index = {
     ...releaseIndex(directory, commit, repository, {
       version,
       releaseTag: tag,
+      assetBuildId: development ? buildId : undefined,
     }),
     version,
     buildId,
@@ -93,7 +129,7 @@ function publishBundles({
     sha256: sha256(installer),
     url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
       tag,
-    )}/install-bundle.cjs`,
+    )}/${development ? `install-bundle-${buildId}.cjs` : 'install-bundle.cjs'}`,
   };
   fs.writeFileSync(
     path.join(directory, 'latest.json'),
@@ -113,7 +149,9 @@ function publishBundles({
     }
   };
   const notes =
-    `${development ? 'Development' : 'Verified'} platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
+    `${
+      development ? 'Development' : 'Verified'
+    } platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
     Object.entries(index.packages)
       .map(([target, entry]) => `- [${target}](${entry.url})`)
       .join('\n') +
@@ -147,123 +185,153 @@ function publishBundles({
     console.log('Newer source exists; leaving version and channel unchanged');
     return false;
   }
-  ensureRelease(
-    tag,
-    development
-      ? `Development platform upgrade (${buildId})`
-      : `Platform upgrade v${version} (${buildId})`,
-  );
-  const assets = platforms
-    .flatMap((target) => {
-      const archive = path.join(
-        directory,
-        `qinglong-${commit}-${target}.tar.gz`,
-      );
-      return [archive, archive + '.json'];
-    })
-    .concat(
-      path.join(directory, 'install-bundle.cjs'),
-      path.join(directory, 'latest.json'),
-    );
-  const published = lookup(tag);
-  const missing = [];
-  const matches = (asset, file) => {
-    const bytes = fs.readFileSync(file);
-    return (
-      asset?.size === bytes.length && asset.digest === `sha256:${sha256(bytes)}`
-    );
-  };
-  for (const file of assets) {
-    const previous = published.assets.find(
-      (a) => a.name === path.basename(file),
-    );
-    if (!previous) missing.push(file);
-    else if (!matches(previous, file))
-      throw new Error(
-        `Immutable release asset differs: ${path.basename(file)}`,
-      );
-  }
-  if (missing.length)
-    gh('release', 'upload', tag, ...missing, '--repo', repository);
-  const uploaded = lookup(tag);
-  if (
-    !assets.every((file) =>
-      matches(
-        uploaded.assets.find((a) => a.name === path.basename(file)),
-        file,
-      ),
-    )
-  )
-    throw new Error('Published platform bundle verification failed');
-
-  // Each channel serializes its image builds and pointer writes as one release.
-  // A rerun of an earlier workflow is allowed to replace the same
-  // version/commit; an obsolete source commit must never advance either alias.
-  const aliases = development
-    ? ['upgrade-develop']
-    : [`upgrade-v${version}`, 'upgrade-master'];
-  for (const ref of aliases) {
-    if (!currentSource()) {
-      console.log('Newer source exists; leaving version and channel unchanged');
-      return false;
-    }
+  const staged = development
+    ? stageDevelopmentAssets(directory, commit, buildId)
+    : null;
+  try {
     ensureRelease(
-      ref,
-      ref === `upgrade-${branch}`
-        ? `Upgrade channel ${branch}`
-        : `Platform upgrade v${version}`,
+      tag,
+      development
+        ? 'Upgrade channel develop'
+        : `Platform upgrade v${version} (${buildId})`,
     );
-    gh(
-      'release',
-      'upload',
-      ref,
-      path.join(directory, 'latest.json'),
-      '--repo',
-      repository,
-      '--clobber',
-    );
-    gh('release', 'edit', ref, '--repo', repository, '--notes', notes);
+    const assets =
+      staged?.assets ||
+      platforms
+        .flatMap((target) => {
+          const archive = path.join(
+            directory,
+            `qinglong-${commit}-${target}.tar.gz`,
+          );
+          return [archive, archive + '.json'];
+        })
+        .concat(
+          path.join(directory, 'install-bundle.cjs'),
+          path.join(directory, 'latest.json'),
+        );
+    const published = lookup(tag);
+    const missing = [];
+    const matches = (asset, file) => {
+      const bytes = fs.readFileSync(file);
+      return (
+        asset?.size === bytes.length &&
+        asset.digest === `sha256:${sha256(bytes)}`
+      );
+    };
+    for (const file of assets) {
+      const previous = published.assets.find(
+        (a) => a.name === path.basename(file),
+      );
+      if (!previous) missing.push(file);
+      else if (!matches(previous, file))
+        throw new Error(
+          `Immutable release asset differs: ${path.basename(file)}`,
+        );
+    }
+    if (missing.length)
+      gh('release', 'upload', tag, ...missing, '--repo', repository);
+    const uploaded = lookup(tag);
     if (
-      !matches(
-        lookup(ref)?.assets.find((a) => a.name === 'latest.json'),
-        path.join(directory, 'latest.json'),
+      !assets.every((file) =>
+        matches(
+          uploaded.assets.find((a) => a.name === path.basename(file)),
+          file,
+        ),
       )
     )
-      throw new Error('Published channel index verification failed');
-  }
-  // Only prune development releases after the complete new batch and its
-  // channel pointer are verified. Production versions and aliases are retained.
-  if (development && currentSource()) {
-    const pages = JSON.parse(
-      gh(
-        'api',
-        `repos/${repository}/releases?per_page=100`,
-        '--paginate',
-        '--slurp',
-      ),
-    );
-    for (const release of pages.flat()) {
-      if (
-        release.tag_name !== tag &&
-        /^upgrade-build-develop-(?:v\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?-)?\d+-\d+$/.test(
-          release.tag_name,
-        ) &&
-        release.prerelease === true &&
-        release.draft === false
-      ) {
-        if (!currentSource()) break;
-        gh(
-          'release',
-          'delete',
-          release.tag_name,
-          '--repo',
-          repository,
-          '--yes',
+      throw new Error('Published platform bundle verification failed');
+
+    // Each channel serializes its image builds and pointer writes as one release.
+    // A rerun of an earlier workflow is allowed to replace the same
+    // version/commit; an obsolete source commit must never advance either alias.
+    const aliases = development
+      ? ['upgrade-develop']
+      : [`upgrade-v${version}`, 'upgrade-master'];
+    for (const ref of aliases) {
+      if (!currentSource()) {
+        console.log(
+          'Newer source exists; leaving version and channel unchanged',
         );
+        return false;
+      }
+      ensureRelease(
+        ref,
+        ref === `upgrade-${branch}`
+          ? `Upgrade channel ${branch}`
+          : `Platform upgrade v${version}`,
+      );
+      gh(
+        'release',
+        'upload',
+        ref,
+        path.join(directory, 'latest.json'),
+        '--repo',
+        repository,
+        '--clobber',
+      );
+      gh('release', 'edit', ref, '--repo', repository, '--notes', notes);
+      if (
+        !matches(
+          lookup(ref)?.assets.find((a) => a.name === 'latest.json'),
+          path.join(directory, 'latest.json'),
+        )
+      )
+        throw new Error('Published channel index verification failed');
+    }
+    // Only prune development releases after the complete new batch and its
+    // channel pointer are verified. Production versions and aliases are retained.
+    if (development && currentSource()) {
+      const currentAssets = new Set(assets.map((file) => path.basename(file)));
+      for (const asset of lookup(tag).assets) {
+        if (
+          !currentAssets.has(asset.name) &&
+          (/^qinglong-[a-f0-9]{40}-linux-[\w-]+-\d+-\d+\.tar\.gz(?:\.json)?$/.test(
+            asset.name,
+          ) ||
+            /^install-bundle-\d+-\d+\.cjs$/.test(asset.name))
+        ) {
+          if (!currentSource()) break;
+          gh(
+            'api',
+            '-X',
+            'DELETE',
+            `repos/${repository}/releases/assets/${asset.id}`,
+          );
+        }
+      }
+      const pages = JSON.parse(
+        gh(
+          'api',
+          `repos/${repository}/releases?per_page=100`,
+          '--paginate',
+          '--slurp',
+        ),
+      );
+      for (const release of pages.flat()) {
+        if (
+          release.tag_name !== tag &&
+          /^upgrade-build-develop-(?:v\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?-)?\d+-\d+$/.test(
+            release.tag_name,
+          ) &&
+          release.prerelease === true &&
+          release.draft === false
+        ) {
+          if (!currentSource()) break;
+          gh(
+            'release',
+            'delete',
+            release.tag_name,
+            '--repo',
+            repository,
+            '--yes',
+          );
+        }
       }
     }
+    return true;
+  } finally {
+    if (staged) fs.rmSync(staged.staged, { recursive: true, force: true });
   }
-  return true;
 }
 module.exports = { releaseIndex, platforms, publishBundles };
 if (require.main === module) {

test/back/publish-upgrade-bundles.test.cjs

@@ -66,6 +66,17 @@ function fixture(t) {
   const gh = (...args) => {
     state.calls.push(args);
     if (args[0] === 'api') {
+      if (args[1] === '-X' && args[2] === 'DELETE') {
+        if (state.failDelete) throw new Error('Delete unavailable');
+        const id = Number(args[3].split('/').at(-1));
+        for (const release of state.releases.values())
+          for (const asset of release.values())
+            if (asset.id === id) {
+              release.delete(asset.name);
+              return '';
+            }
+        return notFound();
+      }
       const route = args[1];
       if (route.endsWith('/git/ref/heads/master')) return state.head;
       if (route.endsWith('/git/ref/heads/develop')) return state.developHead;
@@ -366,24 +377,29 @@ test('automatic development publication keeps its prerelease version separate fr
     ...Object.values(development.packages),
     development.installer,
   ])
-    assert.ok(entry.url.includes('/upgrade-build-develop-100-1/'));
+    assert.ok(entry.url.includes('/upgrade-develop/'));
+  for (const entry of Object.values(development.packages))
+    assert.ok(entry.url.endsWith('-100-1.tar.gz'));
+  assert.ok(development.installer.url.endsWith('/install-bundle-100-1.cjs'));
   assert.ok(
     f.state.calls.some(
       (args) =>
         args[0] === 'release' &&
         args[1] === 'create' &&
-        args[2] === 'upgrade-build-develop-100-1' &&
-        args[args.indexOf('--title') + 1] ===
-          'Development platform upgrade (100-1)',
+        args[2] === 'upgrade-develop' &&
+        args[args.indexOf('--title') + 1] === 'Upgrade channel develop',
     ),
   );
   assert.deepEqual(f.index(), production);
   assert.deepEqual(f.index('upgrade-v2.24.0'), production);
   assert.equal(f.state.releases.get('upgrade-build-v2.24.0-100-1').size, 24);
-  assert.equal(
-    f.state.releases.get('upgrade-build-develop-100-1').size,
-    24,
+  assert.equal(f.state.releases.get('upgrade-develop').size, 24);
+  assert.ok(!f.state.releases.has('upgrade-build-develop-100-1'));
+  const archiveName = path.basename(development.packages['linux-x64-musl'].url);
+  const sidecar = JSON.parse(
+    f.state.releases.get('upgrade-develop').get(`${archiveName}.json`).bytes,
   );
+  assert.equal(sidecar.filename, archiveName);
 });
 
 test('development builds cannot publish obsolete development source', (t) => {
@@ -435,7 +451,12 @@ for (const fault of [
     f.publish(options);
     const oldTag = 'upgrade-build-develop-100-1';
     const legacyTag = 'upgrade-build-develop-v2.24.0-99-1';
+    f.state.releases.set(oldTag, new Map());
     f.state.releases.set(legacyTag, new Map());
+    const oldAsset = Array.from(
+      f.state.releases.get('upgrade-develop').keys(),
+    ).find((name) => name.endsWith('-100-1.tar.gz'));
+    assert.ok(oldAsset);
     // Never prune production, channel aliases, legacy/unclassified releases,
     // drafts or a non-prerelease even when its name resembles a dev build.
     const protectedTags = [
@@ -460,7 +481,7 @@ for (const fault of [
       assert.equal(f.publish({ ...options, buildId: '101-1' }), true);
       assert.equal(f.index('upgrade-develop').buildId, '101-1');
       const deletion = f.state.calls.findIndex(
-        (a) => a[0] === 'release' && a[1] === 'delete',
+        (a) => a[0] === 'api' && a[1] === '-X' && a[2] === 'DELETE',
       );
       const aliasEdit = f.state.calls.findLastIndex(
         (a) => a[0] === 'release' && a[1] === 'edit',
@@ -474,10 +495,15 @@ for (const fault of [
               a[0] === 'api' && a[1].includes('/releases/tags/upgrade-develop'),
           ),
       );
-      assert.ok(f.state.releases.has('upgrade-build-develop-101-1'));
+      assert.ok(f.state.releases.has('upgrade-develop'));
+      assert.ok(!f.state.releases.has('upgrade-build-develop-101-1'));
     }
     assert.equal(f.state.releases.has(oldTag), Boolean(fault));
     assert.equal(f.state.releases.has(legacyTag), Boolean(fault));
+    assert.equal(
+      f.state.releases.get('upgrade-develop').has(oldAsset),
+      Boolean(fault),
+    );
     for (const tag of protectedTags) assert.ok(f.state.releases.has(tag));
     assert.deepEqual(f.index(), production);
     assert.deepEqual(f.index('upgrade-v2.24.0'), production);
@@ -488,17 +514,21 @@ test('failed development cleanup is retryable without overwriting immutable asse
   const options = { branch: 'develop', commit: f.state.developHead };
   f.write(options.commit, 'development');
   f.publish(options);
+  const oldAsset = Array.from(
+    f.state.releases.get('upgrade-develop').keys(),
+  ).find((name) => name.endsWith('-100-1.tar.gz'));
+  assert.ok(oldAsset);
   f.state.failDelete = true;
   assert.throws(
     () => f.publish({ ...options, buildId: '101-1' }),
     /Delete unavailable/,
   );
   assert.equal(f.index('upgrade-develop').buildId, '101-1');
-  assert.ok(f.state.releases.has('upgrade-build-develop-100-1'));
+  assert.ok(f.state.releases.get('upgrade-develop').has(oldAsset));
   f.state.failDelete = false;
   f.publish({ ...options, buildId: '101-1' });
-  assert.ok(!f.state.releases.has('upgrade-build-develop-100-1'));
-  assert.ok(f.state.releases.has('upgrade-build-develop-101-1'));
+  assert.ok(!f.state.releases.get('upgrade-develop').has(oldAsset));
+  assert.equal(f.state.releases.get('upgrade-develop').size, 24);
 });
 
 test('a newer development commit during upload preserves the published channel and its old package', (t) => {
@@ -507,10 +537,14 @@ test('a newer development commit during upload preserves the published channel a
   f.write(options.commit, 'development');
   f.publish(options);
   const before = f.index('upgrade-develop');
+  const oldAsset = Array.from(
+    f.state.releases.get('upgrade-develop').keys(),
+  ).find((name) => name.endsWith('-100-1.tar.gz'));
+  assert.ok(oldAsset);
   f.state.changeDevelopHeadAfterUpload = true;
   assert.equal(f.publish({ ...options, buildId: '101-1' }), false);
   assert.deepEqual(f.index('upgrade-develop'), before);
-  assert.ok(f.state.releases.has('upgrade-build-develop-100-1'));
+  assert.ok(f.state.releases.get('upgrade-develop').has(oldAsset));
   assert.ok(
     !f.state.calls.some((a) => a[0] === 'release' && a[1] === 'delete'),
   );