mirror of
https://github.com/whyour/qinglong.git
synced 2026-11-02 06:58:52 +08:00
feat: ship verified platform upgrades with channel bundle releases
Package production dependencies per architecture for online and offline upgrades. Validate dependencies and isolated backend startup before readiness, repair once when permitted, and preserve service/data through failed preparation or rollback. Publish release bundles with master images and expose an opt-in develop build sharing the development index across Alpine and Debian images.
This commit is contained in:
whyour
@@ -0,0 +1,22 @@
|
||||
const path = require('node:path');
|
||||
require('../cli/node_modules/esbuild').buildSync({
|
||||
entryPoints: [path.resolve(__dirname, '../shell/install-bundle.cjs')],
|
||||
outfile: path.resolve(process.argv[2] || 'output/upgrade/install-bundle.cjs'),
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'cjs',
|
||||
target: 'node20',
|
||||
minify: true,
|
||||
legalComments: 'inline',
|
||||
});
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[path.resolve(process.argv[2] || 'output/upgrade/install-bundle.cjs')],
|
||||
{ encoding: 'utf8' },
|
||||
);
|
||||
if (
|
||||
result.status !== 1 ||
|
||||
!result.stderr.includes('Usage: node install-bundle.cjs')
|
||||
)
|
||||
throw new Error('Standalone offline installer did not start correctly');
|
||||
@@ -0,0 +1,142 @@
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const tar = require('tar');
|
||||
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
|
||||
const {
|
||||
verifyUpgrade,
|
||||
} = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
|
||||
|
||||
async function normalizeModes(directory) {
|
||||
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
|
||||
const filename = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await fs.chmod(filename, 0o755);
|
||||
await normalizeModes(filename);
|
||||
} else if (entry.isFile()) {
|
||||
const stat = await fs.stat(filename);
|
||||
await fs.chmod(filename, 0o644 | (stat.mode & 0o111));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function buildBundle(root, output) {
|
||||
root = path.resolve(root);
|
||||
output = path.resolve(output);
|
||||
const target = await bundle.runtimeTarget();
|
||||
if (process.platform !== 'linux')
|
||||
throw new Error('Release bundles must be built and probed on Linux');
|
||||
const build = await verifyUpgrade(root, path.join(root, 'static'));
|
||||
await bundle.smokeDependencies(root);
|
||||
await fs.mkdir(output, { recursive: true });
|
||||
const workspace = await fs.mkdtemp(path.join(output, '.bundle-'));
|
||||
try {
|
||||
const payload = path.join(workspace, 'qinglong');
|
||||
const source = path.join(payload, 'source'),
|
||||
staticRoot = path.join(payload, 'static');
|
||||
await fs.mkdir(source, { recursive: true });
|
||||
const excluded = new Set([
|
||||
'.git',
|
||||
'.tmp',
|
||||
'.env',
|
||||
'data',
|
||||
'static',
|
||||
'output',
|
||||
'node_modules',
|
||||
]);
|
||||
for (const name of await fs.readdir(root)) {
|
||||
if (excluded.has(name)) continue;
|
||||
await fs.cp(path.join(root, name), path.join(source, name), {
|
||||
recursive: true,
|
||||
verbatimSymlinks: true,
|
||||
});
|
||||
}
|
||||
await fs.cp(
|
||||
path.join(root, 'node_modules'),
|
||||
path.join(source, 'node_modules'),
|
||||
{ recursive: true, verbatimSymlinks: true },
|
||||
);
|
||||
await fs.cp(path.join(root, 'static'), staticRoot, {
|
||||
recursive: true,
|
||||
verbatimSymlinks: true,
|
||||
});
|
||||
await normalizeModes(source);
|
||||
await normalizeModes(staticRoot);
|
||||
const manifest = {
|
||||
format: 2,
|
||||
sourceCommit: build.sourceCommit,
|
||||
version: JSON.parse(
|
||||
await fs.readFile(path.join(root, 'package.json'), 'utf8'),
|
||||
).version,
|
||||
lockfileSha256: build.lockfileSha256,
|
||||
target: {
|
||||
platform: target.platform,
|
||||
arch: target.arch,
|
||||
libc: target.libc,
|
||||
libcMinimum: target.libcVersion,
|
||||
endianness: target.endianness,
|
||||
armFloatAbi: target.armFloatAbi,
|
||||
},
|
||||
nodeMinimum: '20.18.1',
|
||||
nodeApi: 6,
|
||||
files: await bundle.inventory({ source, static: staticRoot }),
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(payload, 'bundle.json'),
|
||||
JSON.stringify(manifest),
|
||||
);
|
||||
console.log('Bundle inventory verified; writing archive');
|
||||
const filename = `qinglong-${build.sourceCommit}-${bundle.targetKey(
|
||||
target,
|
||||
)}.tar.gz`;
|
||||
await tar.c(
|
||||
{
|
||||
gzip: true,
|
||||
portable: true,
|
||||
noMtime: true,
|
||||
cwd: workspace,
|
||||
file: path.join(output, filename),
|
||||
},
|
||||
['qinglong'],
|
||||
);
|
||||
// Verify the bytes users receive, not just the pre-archive build directory.
|
||||
const roundtrip = path.join(workspace, 'roundtrip');
|
||||
await fs.mkdir(roundtrip);
|
||||
console.log('Verifying archived paths, permissions and checksums');
|
||||
await bundle.extractArchive(path.join(output, filename), roundtrip);
|
||||
const extracted = {
|
||||
source: path.join(roundtrip, 'qinglong/source'),
|
||||
static: path.join(roundtrip, 'qinglong/static'),
|
||||
};
|
||||
await bundle.verifyPayload(extracted, manifest, target);
|
||||
await verifyUpgrade(extracted.source, extracted.static);
|
||||
await bundle.smokeDependencies(extracted.source);
|
||||
console.log('Archived production dependencies and SQLite verified');
|
||||
const bytes = await fs.readFile(path.join(output, filename));
|
||||
const record = {
|
||||
format: 2,
|
||||
sourceCommit: build.sourceCommit,
|
||||
version: manifest.version,
|
||||
target: bundle.targetKey(target),
|
||||
filename,
|
||||
baseImage: process.env.QL_BUNDLE_BASE_IMAGE,
|
||||
size: bytes.length,
|
||||
sha256: bundle.sha256(bytes),
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(output, filename + '.json'),
|
||||
JSON.stringify(record) + '\n',
|
||||
);
|
||||
console.log(JSON.stringify(record));
|
||||
return record;
|
||||
} finally {
|
||||
await fs.rm(workspace, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
module.exports = { buildBundle, normalizeModes };
|
||||
if (require.main === module)
|
||||
buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch(
|
||||
(e) => {
|
||||
console.error(e);
|
||||
process.exitCode = 1;
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,253 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const {
|
||||
sha256,
|
||||
} = require('../cli/src/internal/maintenance/platformBundle.cjs');
|
||||
const platforms = [
|
||||
'linux-x64-musl',
|
||||
'linux-armv6-musl',
|
||||
'linux-armv7-musl',
|
||||
'linux-arm64-musl',
|
||||
'linux-ppc64-musl',
|
||||
'linux-s390x-musl',
|
||||
'linux-ia32-musl',
|
||||
'linux-x64-glibc',
|
||||
'linux-armv6-glibc',
|
||||
'linux-armv7-glibc',
|
||||
'linux-arm64-glibc',
|
||||
'linux-ppc64-glibc',
|
||||
'linux-s390x-glibc',
|
||||
];
|
||||
function releaseIndex(directory, commit, repository, options = {}) {
|
||||
const { version, releaseTag = `upgrade-${commit}` } = options;
|
||||
if (
|
||||
!/^[a-f0-9]{40}$/.test(commit) ||
|
||||
!/^[\w.-]+\/[\w.-]+$/.test(repository) ||
|
||||
!/^[\w.+-]+$/.test(releaseTag)
|
||||
)
|
||||
throw new Error('Invalid release source');
|
||||
const packages = {};
|
||||
for (const target of platforms) {
|
||||
const filename = `qinglong-${commit}-${target}.tar.gz`;
|
||||
const record = JSON.parse(
|
||||
fs.readFileSync(path.join(directory, filename + '.json')),
|
||||
);
|
||||
const bytes = fs.readFileSync(path.join(directory, filename));
|
||||
if (
|
||||
record.format !== 2 ||
|
||||
record.sourceCommit !== commit ||
|
||||
(version !== undefined && record.version !== version) ||
|
||||
record.target !== target ||
|
||||
record.filename !== filename ||
|
||||
record.size !== bytes.length ||
|
||||
record.sha256 !== sha256(bytes)
|
||||
)
|
||||
throw new Error(`Unverified platform bundle: ${target}`);
|
||||
packages[target] = {
|
||||
size: record.size,
|
||||
sha256: record.sha256,
|
||||
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
|
||||
releaseTag,
|
||||
)}/${filename}`,
|
||||
};
|
||||
}
|
||||
return { format: 2, sourceCommit: commit, packages };
|
||||
}
|
||||
function publishBundles({
|
||||
directory,
|
||||
commit,
|
||||
repository,
|
||||
branch,
|
||||
version,
|
||||
buildId,
|
||||
allowDevelopment = false,
|
||||
gh = (...args) =>
|
||||
execFileSync('gh', args, {
|
||||
encoding: 'utf8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}),
|
||||
}) {
|
||||
if (
|
||||
branch !== 'master' &&
|
||||
!(branch === 'develop' && allowDevelopment === true)
|
||||
)
|
||||
throw new Error(
|
||||
'Only master or explicitly requested development images publish bundles',
|
||||
);
|
||||
if (!/^\d+\.\d+\.\d+(?:-[\w.-]+)?(?:\+[\w.-]+)?$/.test(version || ''))
|
||||
throw new Error('Invalid bundle version');
|
||||
if (!/^\d+-\d+$/.test(buildId || ''))
|
||||
throw new Error('Invalid bundle build ID');
|
||||
// Rebuilding a version gets new download URLs, including when its commit is
|
||||
// unchanged. Only the version/channel pointer is overwritten, never bytes
|
||||
// that an in-flight download has already selected by hash.
|
||||
const development = branch === 'develop';
|
||||
const tag = `upgrade-build-${
|
||||
development ? 'develop-' : ''
|
||||
}v${version}-${buildId}`;
|
||||
const index = {
|
||||
...releaseIndex(directory, commit, repository, {
|
||||
version,
|
||||
releaseTag: tag,
|
||||
}),
|
||||
version,
|
||||
buildId,
|
||||
channel: branch,
|
||||
};
|
||||
const installer = fs.readFileSync(path.join(directory, 'install-bundle.cjs'));
|
||||
index.installer = {
|
||||
size: installer.length,
|
||||
sha256: sha256(installer),
|
||||
url: `https://github.com/${repository}/releases/download/${encodeURIComponent(
|
||||
tag,
|
||||
)}/install-bundle.cjs`,
|
||||
};
|
||||
fs.writeFileSync(
|
||||
path.join(directory, 'latest.json'),
|
||||
JSON.stringify(index) + '\n',
|
||||
);
|
||||
const lookup = (ref) => {
|
||||
try {
|
||||
return JSON.parse(
|
||||
gh(
|
||||
'api',
|
||||
`repos/${repository}/releases/tags/${encodeURIComponent(ref)}`,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
if (/\bHTTP 404\b/.test(String(error.stderr))) return null;
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const notes =
|
||||
`Verified platform bundles for v${version}, source ${commit}, build ${buildId}. Existing Node runtime is retained.\n\n` +
|
||||
Object.entries(index.packages)
|
||||
.map(([target, entry]) => `- [${target}](${entry.url})`)
|
||||
.join('\n') +
|
||||
`\n- [Standalone offline installer](${index.installer.url})`;
|
||||
const ensureRelease = (ref, title) => {
|
||||
if (!lookup(ref))
|
||||
gh(
|
||||
'release',
|
||||
'create',
|
||||
ref,
|
||||
'--repo',
|
||||
repository,
|
||||
'--target',
|
||||
commit,
|
||||
'--prerelease',
|
||||
'--latest=false',
|
||||
'--title',
|
||||
title,
|
||||
'--notes',
|
||||
notes,
|
||||
);
|
||||
};
|
||||
const currentSource = () =>
|
||||
gh(
|
||||
'api',
|
||||
`repos/${repository}/git/ref/heads/${branch}`,
|
||||
'--jq',
|
||||
'.object.sha',
|
||||
).trim() === commit;
|
||||
if (!currentSource()) {
|
||||
console.log('Newer source exists; leaving version and channel unchanged');
|
||||
return false;
|
||||
}
|
||||
ensureRelease(tag, `Platform upgrade v${version} (${buildId})`);
|
||||
const assets = platforms
|
||||
.flatMap((target) => {
|
||||
const archive = path.join(
|
||||
directory,
|
||||
`qinglong-${commit}-${target}.tar.gz`,
|
||||
);
|
||||
return [archive, archive + '.json'];
|
||||
})
|
||||
.concat(
|
||||
path.join(directory, 'install-bundle.cjs'),
|
||||
path.join(directory, 'latest.json'),
|
||||
);
|
||||
const published = lookup(tag);
|
||||
const missing = [];
|
||||
const matches = (asset, file) => {
|
||||
const bytes = fs.readFileSync(file);
|
||||
return (
|
||||
asset?.size === bytes.length && asset.digest === `sha256:${sha256(bytes)}`
|
||||
);
|
||||
};
|
||||
for (const file of assets) {
|
||||
const previous = published.assets.find(
|
||||
(a) => a.name === path.basename(file),
|
||||
);
|
||||
if (!previous) missing.push(file);
|
||||
else if (!matches(previous, file))
|
||||
throw new Error(
|
||||
`Immutable release asset differs: ${path.basename(file)}`,
|
||||
);
|
||||
}
|
||||
if (missing.length)
|
||||
gh('release', 'upload', tag, ...missing, '--repo', repository);
|
||||
const uploaded = lookup(tag);
|
||||
if (
|
||||
!assets.every((file) =>
|
||||
matches(
|
||||
uploaded.assets.find((a) => a.name === path.basename(file)),
|
||||
file,
|
||||
),
|
||||
)
|
||||
)
|
||||
throw new Error('Published platform bundle verification failed');
|
||||
|
||||
// Each channel serializes its image builds and pointer writes as one release.
|
||||
// A rerun of an earlier workflow is allowed to replace the same
|
||||
// version/commit; an obsolete source commit must never advance either alias.
|
||||
const aliases = development
|
||||
? ['upgrade-develop']
|
||||
: [`upgrade-v${version}`, 'upgrade-master'];
|
||||
for (const ref of aliases) {
|
||||
if (!currentSource()) {
|
||||
console.log('Newer source exists; leaving version and channel unchanged');
|
||||
return false;
|
||||
}
|
||||
ensureRelease(
|
||||
ref,
|
||||
ref === `upgrade-${branch}`
|
||||
? `Upgrade channel ${branch}`
|
||||
: `Platform upgrade v${version}`,
|
||||
);
|
||||
gh(
|
||||
'release',
|
||||
'upload',
|
||||
ref,
|
||||
path.join(directory, 'latest.json'),
|
||||
'--repo',
|
||||
repository,
|
||||
'--clobber',
|
||||
);
|
||||
gh('release', 'edit', ref, '--repo', repository, '--notes', notes);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
module.exports = { releaseIndex, platforms, publishBundles };
|
||||
if (require.main === module) {
|
||||
const [directory, commit, repository, branch] = process.argv.slice(2);
|
||||
const version = require('../package.json').version;
|
||||
if (
|
||||
require('js-yaml').load(
|
||||
fs.readFileSync(path.join(__dirname, '../version.yaml'), 'utf8'),
|
||||
).version !== version
|
||||
)
|
||||
throw new Error('Package and image release versions differ');
|
||||
publishBundles({
|
||||
directory,
|
||||
commit,
|
||||
repository,
|
||||
branch,
|
||||
version,
|
||||
buildId: `${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT}`,
|
||||
allowDevelopment:
|
||||
process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' &&
|
||||
process.env.QL_BUILD_DEVELOPMENT_BUNDLES === 'true',
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user