ci: verify Gitee Pull mirrors with resilient GitHub notifications

Use native Gitee Pull mirrors for code and static artifacts, and verify branch and tag SHAs alongside image builds. Trigger stale mirrors through small GitHub REST ref updates, retry transient errors with backoff, and keep polling when a notification fails.
This commit is contained in:
whyour
2026-10-08 01:16:43 +08:00
parent ef1c54a620
commit 9d5f58ef99
5 changed files with 960 additions and 51 deletions
+48 -47
View File
@@ -17,7 +17,7 @@ on:
workflow_dispatch:
inputs:
static_sync_only:
description: Only synchronize an existing GitHub static snapshot
description: Synchronize an existing static snapshot to GitLab and verify Gitee
type: boolean
default: false
static_ref:
@@ -56,37 +56,6 @@ jobs:
git push --force --all gitlab 2>&1 || echo "::warning::GitLab push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab tags push failed"
code_gitee:
if: ${{ !inputs.static_sync_only }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Push to Gitee
env:
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git remote add gitee git@gitee.com:whyour/qinglong.git 2>/dev/null
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee push --all succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee push succeeded after repo creation" || echo "::warning::Gitee push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed"
build-static:
if: ${{ !inputs.static_sync_only }}
needs: validate
@@ -146,26 +115,42 @@ jobs:
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh git@gitlab.com:whyour/qinglong-static.git "$STATIC_BRANCH" "$STATIC_REF_TYPE"
static_gitee:
verify-gitee-mirrors:
needs: build-static
runs-on: ubuntu-latest
concurrency:
group: static-gitee-${{ github.ref_name }}
cancel-in-progress: false
timeout-minutes: 14
permissions:
contents: write
strategy:
fail-fast: false
matrix:
repository: [qinglong, qinglong-static]
steps:
- uses: actions/checkout@v6
- name: Push qinglong-static to Gitee
# Gitee Pull mirrors and GitHub push webhooks own synchronization.
# Verification runs alongside Docker builds and needs no Gitee credentials.
- name: Wait for Gitee Pull mirror
if: ${{ matrix.repository == 'qinglong' }}
env:
STATIC_BRANCH: ${{ github.ref_name }}
STATIC_REF_TYPE: ${{ github.ref_type }}
MIRROR_SSH_KEY: ${{ secrets.GITLAB_SSH_PK }}
GH_TOKEN: ${{ github.token }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong
MIRROR_REPOSITORY: ${{ matrix.repository }}
MIRROR_REF: ${{ github.ref_name }}
MIRROR_REF_TYPE: ${{ github.ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: ${{ matrix.repository == 'qinglong-static' }}
QL_MIRROR_NOTIFY_ON_START: 'false'
run: |
set -euo pipefail
mkdir -p ~/.ssh
printf '%s\n' "$MIRROR_SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh git@gitee.com:whyour/qinglong-static.git "$STATIC_BRANCH" "$STATIC_REF_TYPE"
bash scripts/wait-for-mirror.sh "https://github.com/whyour/$MIRROR_REPOSITORY.git" "https://gitee.com/whyour/$MIRROR_REPOSITORY.git" "$MIRROR_REF" "$MIRROR_REF_TYPE"
- name: Notify and wait for Gitee static mirror
if: ${{ matrix.repository == 'qinglong-static' }}
env:
GH_TOKEN: ${{ secrets.API_TOKEN }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
MIRROR_REF: ${{ github.ref_name }}
MIRROR_REF_TYPE: ${{ github.ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
run: |
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
sync-existing-static:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
@@ -173,7 +158,7 @@ jobs:
strategy:
fail-fast: false
matrix:
mirror: [gitlab, gitee]
mirror: [gitlab]
concurrency:
group: static-${{ matrix.mirror }}-${{ inputs.static_ref }}
cancel-in-progress: false
@@ -193,6 +178,22 @@ jobs:
ssh-keyscan -T 10 "$MIRROR_HOST" >> ~/.ssh/known_hosts 2>/dev/null
bash scripts/sync-static.sh "git@$MIRROR_HOST:whyour/qinglong-static.git" "$STATIC_REF" "$STATIC_REF_TYPE"
verify-existing-static-gitee:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.static_sync_only }}
runs-on: ubuntu-latest
timeout-minutes: 14
steps:
- uses: actions/checkout@v6
- name: Wait for existing Gitee static snapshot
env:
GH_TOKEN: ${{ secrets.API_TOKEN }}
QL_MIRROR_NOTIFY_REPO: whyour/qinglong-static
MIRROR_REF: ${{ inputs.static_ref }}
MIRROR_REF_TYPE: ${{ inputs.static_ref_type }}
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true'
run: |
bash scripts/wait-for-mirror.sh https://github.com/whyour/qinglong-static.git https://gitee.com/whyour/qinglong-static.git "$MIRROR_REF" "$MIRROR_REF_TYPE"
build-alpine:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
needs: build-static
+214
View File
@@ -0,0 +1,214 @@
#!/usr/bin/env bash
# Observe Pull mirrors; optionally notify them through a small GitHub push.
set -euo pipefail
if [[ $# -lt 3 || $# -gt 4 ]]; then
echo 'Usage: wait-for-mirror.sh SOURCE MIRROR REF [branch|tag]' >&2
exit 2
fi
source=$1
mirror=$2
ref=$3
ref_type=${4:-branch}
wait_seconds=${QL_MIRROR_WAIT_SECONDS:-600}
poll_interval=${QL_MIRROR_POLL_INTERVAL:-30}
query_timeout=${QL_MIRROR_QUERY_TIMEOUT:-20}
include_version_branch=${QL_MIRROR_INCLUDE_VERSION_BRANCH:-false}
notify_repo=${QL_MIRROR_NOTIFY_REPO:-}
retry_after=${QL_MIRROR_RETRY_AFTER_SECONDS:-310}
notify_on_start=${QL_MIRROR_NOTIFY_ON_START:-true}
timings=("$wait_seconds" "$poll_interval" "$query_timeout")
if [[ -n "$notify_repo" ]]; then timings+=("$retry_after"); fi
for value in "${timings[@]}"; do
if [[ ! "$value" =~ ^[1-9][0-9]{0,4}$ ]] || (( value > 86400 )); then
echo 'Mirror timing values must be integers between 1 and 86400 seconds' >&2
exit 2
fi
done
case "$ref_type" in
branch) refs=("refs/heads/$ref") ;;
tag)
refs=("refs/tags/$ref")
if [[ "$include_version_branch" == true ]]; then
refs+=("refs/heads/$ref")
fi
;;
*) echo 'Mirror ref type must be branch or tag' >&2; exit 2 ;;
esac
if [[ "$include_version_branch" != true && "$include_version_branch" != false ]]; then
echo 'QL_MIRROR_INCLUDE_VERSION_BRANCH must be true or false' >&2
exit 2
fi
if [[ "$notify_on_start" != true && "$notify_on_start" != false ]]; then
echo 'QL_MIRROR_NOTIFY_ON_START must be true or false' >&2
exit 2
fi
for requested_ref in "${refs[@]}"; do
git check-ref-format "$requested_ref"
done
command -v timeout >/dev/null || { echo 'GNU timeout is required' >&2; exit 2; }
if [[ -n "$notify_repo" ]]; then
if [[ ! "$notify_repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
echo 'QL_MIRROR_NOTIFY_REPO must be a GitHub owner/repository' >&2
exit 2
fi
command -v gh >/dev/null || { echo 'GitHub CLI is required for notifications' >&2; exit 2; }
fi
export GIT_TERMINAL_PROMPT=0
polling=false
notification_command() {
local budget=$((notification_deadline - SECONDS))
if (( budget <= 0 )); then return 1; fi
if (( budget > 10 )); then budget=10; fi
timeout --signal=KILL "${budget}s" "$@"
}
notify_static_mirror() {
local notification_deadline=$((SECONDS + 30))
if [[ "$polling" == true ]] && (( notification_deadline > wait_seconds )); then
notification_deadline=$wait_seconds
fi
local trigger_ref=refs/heads/gitee-sync-trigger
local attempt remote_refs source_sha parent tree commit delay remaining
local commit_args
if ! remote_refs=$(notification_command git ls-remote --refs "$source" "${refs[0]}"); then
echo '::warning::Could not read the published source; continuing mirror verification'
return 1
fi
source_sha=${remote_refs%%$'\t'*}
if [[ ! "$source_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::Published source ref ${refs[0]} does not exist"
return 2
fi
for attempt in 1 2 3; do
if (( attempt > 1 )); then
delay=$((attempt * 2 - 2))
remaining=$((notification_deadline - SECONDS))
if (( remaining <= delay )); then break; fi
sleep "$delay"
fi
# Refresh the parent on every attempt. REST avoids the Git push endpoint's
# transient 500s, and non-forced updates preserve concurrent notifications.
if parent=$(notification_command gh api "repos/$notify_repo/git/ref/heads/gitee-sync-trigger" --jq '.object.sha' 2>&1); then
if ! tree=$(notification_command gh api "repos/$notify_repo/git/commits/$parent" --jq '.tree.sha'); then
echo "::warning::Could not read notification history (attempt $attempt)"
continue
fi
elif [[ "$parent" == *'(HTTP 404)'* ]]; then
parent=''
# A new notification branch starts with one tiny marker, never artifacts.
# Later commits reuse this tree and have no changed files in the webhook.
if ! tree=$(printf '%s' '{"tree":[{"path":".gitee-sync-trigger","mode":"100644","type":"blob","content":"Gitee Pull mirror notifications.\n"}]}' |
notification_command gh api --method POST "repos/$notify_repo/git/trees" --input - --jq '.sha'); then
echo "::warning::Could not create notification tree (attempt $attempt)"
continue
fi
else
echo "::warning::Could not read notification ref (attempt $attempt)"
continue
fi
commit_args=(-f "tree=$tree" -f "message=Trigger Gitee mirror for ${refs[0]} at $source_sha")
if [[ -n "$parent" ]]; then commit_args+=(-f "parents[]=$parent"); fi
if ! commit=$(notification_command gh api --method POST "repos/$notify_repo/git/commits" "${commit_args[@]}" --jq '.sha'); then
echo "::warning::Could not create notification commit (attempt $attempt)"
continue
fi
if [[ -n "$parent" ]]; then
if notification_command gh api --method PATCH "repos/$notify_repo/git/refs/heads/gitee-sync-trigger" -f "sha=$commit" -F force=false --silent; then
echo '::notice::Sent a small GitHub push event via REST to trigger the Gitee Pull mirror'
return 0
fi
elif notification_command gh api --method POST "repos/$notify_repo/git/refs" -f "ref=$trigger_ref" -f "sha=$commit" --silent; then
echo '::notice::Sent a small GitHub push event via REST to trigger the Gitee Pull mirror'
return 0
fi
echo "::warning::Mirror notification update failed (attempt $attempt)"
done
echo '::warning::Could not send the mirror notification; continuing mirror verification'
return 1
}
if [[ -n "$notify_repo" && "$notify_on_start" == true ]]; then
if notify_static_mirror; then :; elif [[ $? -eq 2 ]]; then exit 1; fi
fi
SECONDS=0
polling=true
attempt=0
retried=false
last_source='(not read)'
last_mirror='(not read)'
while (( SECONDS < wait_seconds )); do
attempt=$((attempt + 1))
outputs=()
ready=true
# Re-read the source after the mirror so a moving source cannot pass with
# the previous SHA. --refs compares annotated tag objects as well as branches.
urls=("$source" "$mirror" "$source")
labels=('source' 'mirror' 'source confirmation')
for index in 0 1 2; do
remaining=$((wait_seconds - SECONDS))
if (( remaining <= 0 )); then
ready=false
break
fi
budget=$query_timeout
if (( budget > remaining )); then budget=$remaining; fi
if ! output=$(timeout --signal=KILL "${budget}s" git ls-remote --refs \
"${urls[$index]}" "${refs[@]}" | LC_ALL=C sort); then
echo "::warning::Could not read ${labels[$index]} refs on attempt $attempt; retrying"
ready=false
break
fi
if (( index == 0 )); then last_source=$output; fi
if (( index == 1 )); then last_mirror=$output; fi
for requested_ref in "${refs[@]}"; do
found=false
while IFS=$'\t' read -r sha returned_ref; do
if [[ "$returned_ref" == "$requested_ref" && "$sha" =~ ^[0-9a-f]{40}([0-9a-f]{24})?$ ]]; then
found=true
fi
done <<< "$output"
if [[ "$found" == false ]]; then
echo "::notice::${labels[$index]} is missing $requested_ref; waiting"
ready=false
break
fi
done
if [[ "$ready" == false ]]; then break; fi
outputs+=("$output")
# Only confirm a source when the initial source and mirror reads match.
if (( index == 1 )) && [[ "${outputs[0]}" != "${outputs[1]}" ]]; then
ready=false
break
fi
done
if [[ "$ready" == true && "${outputs[0]}" == "${outputs[2]}" ]]; then
echo "::notice::Mirror matches the latest source refs after ${SECONDS}s"
printf '%s\n' "${outputs[2]}"
exit 0
fi
echo "Waiting for mirror (attempt $attempt, ${SECONDS}/${wait_seconds}s)"
remaining=$((wait_seconds - SECONDS))
if (( remaining <= 0 )); then break; fi
# Gitee requires five minutes between sync requests. Retry once after that
# interval without restarting the polling deadline.
if [[ -n "$notify_repo" && "$retried" == false ]] && (( SECONDS >= retry_after )); then
echo '::notice::Retrying the GitHub notification after the Gitee sync interval'
if notify_static_mirror; then :; elif [[ $? -eq 2 ]]; then exit 1; fi
retried=true
continue
fi
delay=$poll_interval
if (( delay > remaining )); then delay=$remaining; fi
if [[ -n "$notify_repo" && "$retried" == false ]] && (( SECONDS < retry_after && delay > retry_after - SECONDS )); then
delay=$((retry_after - SECONDS))
fi
sleep "$delay"
done
echo "::error::Mirror did not match the latest source refs within ${wait_seconds}s"
printf 'Last source refs:\n%s\nLast mirror refs:\n%s\n' "$last_source" "$last_mirror"
exit 1
+258
View File
@@ -0,0 +1,258 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');
const { execFileSync, spawn, spawnSync } = require('node:child_process');
const script = path.resolve('scripts/wait-for-mirror.sh');
const git = (...args) =>
execFileSync('git', args, { encoding: 'utf8', stdio: 'pipe' }).trim();
const options = (env = {}) => ({
encoding: 'utf8',
env: {
...process.env,
QL_MIRROR_WAIT_SECONDS: '3',
QL_MIRROR_POLL_INTERVAL: '1',
QL_MIRROR_QUERY_TIMEOUT: '1',
...env,
},
});
function fixture(t) {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-mirror-status-'));
t.after(() => fs.rmSync(tmp, { recursive: true, force: true }));
const source = path.join(tmp, 'source');
const mirror = path.join(tmp, 'mirror.git');
git('init', source);
git('init', '--bare', mirror);
git('-C', source, 'config', 'user.name', 'Fixture');
git('-C', source, 'config', 'user.email', 'fixture@example.invalid');
git('-C', source, 'checkout', '-b', 'develop');
fs.writeFileSync(path.join(source, 'snapshot'), 'first');
git('-C', source, 'add', '.');
git('-C', source, 'commit', '-m', 'first');
return { tmp, source, mirror };
}
test('mirror verification reads matching refs without changing either repository', (t) => {
const { source, mirror } = fixture(t);
git('-C', source, 'push', mirror, 'develop');
const before = git('ls-remote', '--refs', mirror);
const result = spawnSync(
'bash',
[script, source, mirror, 'develop'],
options(),
);
assert.equal(result.status, 0, result.stderr + result.stdout);
assert.match(result.stdout, /Mirror matches the latest source refs/);
assert.equal(git('ls-remote', '--refs', mirror), before);
assert.equal(git('-C', source, 'status', '--porcelain'), '');
});
test('mirror polling follows a source that advances while waiting', async (t) => {
const { source, mirror } = fixture(t);
git('-C', source, 'push', mirror, 'develop');
fs.writeFileSync(path.join(source, 'snapshot'), 'second');
git('-C', source, 'commit', '-am', 'second');
const child = spawn(
'bash',
[script, source, mirror, 'develop'],
options({
QL_MIRROR_WAIT_SECONDS: '6',
}),
);
let output = '';
let error = '';
let moved = false;
child.stderr.on('data', (data) => {
error += data;
});
child.stdout.on('data', (data) => {
output += data;
if (!moved && output.includes('Waiting for mirror')) {
moved = true;
fs.writeFileSync(path.join(source, 'snapshot'), 'third');
git('-C', source, 'commit', '-am', 'third');
git('-C', source, 'push', mirror, 'develop');
}
});
const status = await new Promise((resolve, reject) => {
child.on('error', reject);
child.on('close', resolve);
});
assert.equal(status, 0, error + output);
assert.ok(moved);
assert.match(output, new RegExp(git('-C', source, 'rev-parse', 'develop')));
});
test('empty or missing refs never count as a synchronized mirror', (t) => {
const { source, mirror } = fixture(t);
for (const ref of ['develop', 'missing']) {
const result = spawnSync(
'bash',
[script, source, mirror, ref],
options({
QL_MIRROR_WAIT_SECONDS: '1',
}),
);
assert.equal(result.status, 1, result.stderr + result.stdout);
assert.match(result.stdout, /is missing refs\/heads\//);
assert.match(result.stdout, /::error::Mirror did not match/);
}
});
test('static releases require both the version branch and annotated tag object', (t) => {
const { source, mirror } = fixture(t);
git('-C', source, 'branch', 'v2.22.0');
git('-C', source, 'tag', '-a', 'v2.22.0', '-m', 'release');
git('-C', source, 'push', mirror, 'refs/tags/v2.22.0');
const args = [script, source, mirror, 'v2.22.0', 'tag'];
const env = {
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true',
QL_MIRROR_WAIT_SECONDS: '1',
};
const partial = spawnSync('bash', args, options(env));
assert.equal(partial.status, 1, partial.stderr + partial.stdout);
assert.match(partial.stdout, /mirror is missing refs\/heads\/v2.22.0/);
git('-C', source, 'push', mirror, 'refs/heads/v2.22.0');
const complete = spawnSync('bash', args, options(env));
assert.equal(complete.status, 0, complete.stderr + complete.stdout);
assert.match(
complete.stdout,
new RegExp(git('-C', source, 'rev-parse', 'refs/tags/v2.22.0')),
);
// A tag pointing to the same commit with different annotation is still stale.
git('-C', source, 'tag', '-fa', 'v2.22.0', '-m', 'updated annotation');
const stale = spawnSync('bash', args, options(env));
assert.equal(stale.status, 1, stale.stderr + stale.stdout);
});
test('network failures and hung ref queries stop within the configured deadline', (t) => {
const { tmp, source, mirror } = fixture(t);
const failure = spawnSync(
'bash',
[script, source, path.join(tmp, 'absent'), 'develop'],
options({
QL_MIRROR_WAIT_SECONDS: '1',
}),
);
assert.equal(failure.status, 1, failure.stderr + failure.stdout);
assert.match(failure.stdout, /Could not read mirror refs/);
const bin = path.join(tmp, 'bin');
fs.mkdirSync(bin);
const realGit = execFileSync('which', ['git'], { encoding: 'utf8' }).trim();
fs.writeFileSync(
path.join(bin, 'git'),
'#!/bin/sh\nif [ "$1" = ls-remote ]; then sleep 30; fi\nexec "$QL_TEST_GIT" "$@"\n',
{ mode: 0o755 },
);
const started = Date.now();
const hung = spawnSync('bash', [script, source, mirror, 'develop'], {
...options({
PATH: `${bin}${path.delimiter}${process.env.PATH}`,
QL_TEST_GIT: realGit,
QL_MIRROR_WAIT_SECONDS: '2',
QL_MIRROR_QUERY_TIMEOUT: '20',
}),
timeout: 5000,
});
assert.equal(hung.status, 1, hung.stderr + hung.stdout);
assert.ok(Date.now() - started < 4500);
assert.match(hung.stdout, /within 2s/);
});
test('invalid mirror parameters fail before polling', (t) => {
const { source, mirror } = fixture(t);
for (const args of [
[],
[source, mirror, 'develop', 'other'],
[source, mirror, 'bad ref'],
]) {
assert.notEqual(spawnSync('bash', [script, ...args], options()).status, 0);
}
for (const value of ['0', '-1', '1.5', '1+2', '9999999999999']) {
const result = spawnSync(
'bash',
[script, source, mirror, 'develop'],
options({
QL_MIRROR_WAIT_SECONDS: value,
}),
);
assert.equal(result.status, 2, result.stderr + result.stdout);
}
});
test('Gitee workflows verify both mirrors and do not gate image publication', () => {
const workflow = require('js-yaml').load(
fs.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'),
);
assert.equal(workflow.jobs.code_gitee, undefined);
assert.equal(workflow.jobs.static_gitee, undefined);
const job = workflow.jobs['verify-gitee-mirrors'];
assert.equal(job.needs, 'build-static');
assert.equal(job['timeout-minutes'], 14);
assert.deepEqual(job.permissions, { contents: 'write' });
assert.deepEqual(job.strategy.matrix.repository, [
'qinglong',
'qinglong-static',
]);
const verify = job.steps.find((step) => step.run);
assert.equal(verify.env.MIRROR_REF, '${{ github.ref_name }}');
assert.equal(verify.env.MIRROR_REF_TYPE, '${{ github.ref_type }}');
assert.equal(
verify.env.QL_MIRROR_INCLUDE_VERSION_BRANCH,
"${{ matrix.repository == 'qinglong-static' }}",
);
assert.match(verify.run, /bash scripts\/wait-for-mirror.sh/);
assert.equal(verify.if, "${{ matrix.repository == 'qinglong' }}");
assert.equal(verify.env.GH_TOKEN, '${{ github.token }}');
assert.equal(verify.env.QL_MIRROR_NOTIFY_ON_START, 'false');
assert.equal(verify.env.QL_MIRROR_NOTIFY_REPO, 'whyour/qinglong');
assert.doesNotMatch(
JSON.stringify(verify),
/secrets\.|git push|sync-static|ssh-keyscan/,
);
const notify = job.steps.find((step) =>
/Notify and wait/.test(step.name || ''),
);
assert.equal(notify.if, "${{ matrix.repository == 'qinglong-static' }}");
assert.equal(notify.env.GH_TOKEN, '${{ secrets.API_TOKEN }}');
assert.match(notify.run, /bash scripts\/wait-for-mirror.sh/);
assert.equal(notify.env.QL_MIRROR_NOTIFY_REPO, 'whyour/qinglong-static');
assert.equal(notify.env.QL_MIRROR_INCLUDE_VERSION_BRANCH, 'true');
assert.doesNotMatch(
JSON.stringify(job),
/GITEE_TOKEN|git@gitee|sync-static|ssh-keyscan/,
);
const backfill = workflow.jobs['verify-existing-static-gitee'];
assert.match(backfill.if, /workflow_dispatch.*inputs.static_sync_only/);
assert.equal(backfill.needs, undefined);
const verifyExisting = backfill.steps.find((step) => step.run);
assert.equal(verifyExisting.env.MIRROR_REF, '${{ inputs.static_ref }}');
assert.equal(
verifyExisting.env.MIRROR_REF_TYPE,
'${{ inputs.static_ref_type }}',
);
assert.equal(verifyExisting.env.GH_TOKEN, '${{ secrets.API_TOKEN }}');
assert.match(verifyExisting.run, /wait-for-mirror.sh/);
assert.equal(
verifyExisting.env.QL_MIRROR_NOTIFY_REPO,
'whyour/qinglong-static',
);
assert.equal(verifyExisting.env.QL_MIRROR_INCLUDE_VERSION_BRANCH, 'true');
assert.deepEqual(workflow.on.push.branches, ['master', 'develop']);
assert.doesNotMatch(
JSON.stringify(backfill),
/GITEE_TOKEN|git@gitee|sync-static|ssh-keyscan/,
);
for (const name of [
'build-alpine',
'build-debian',
'build-alpine310',
'build-debian310',
]) {
assert.equal(workflow.jobs[name].needs, 'build-static');
}
});
+436
View File
@@ -0,0 +1,436 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');
const { execFileSync, spawn, spawnSync } = require('node:child_process');
const notify = path.resolve('scripts/wait-for-mirror.sh');
const verify = notify;
const triggerRef = 'refs/heads/gitee-sync-trigger';
const git = (...args) =>
execFileSync('git', args, { encoding: 'utf8', stdio: 'pipe' }).trim();
function fixture(t) {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-mirror-trigger-'));
t.after(() => fs.rmSync(tmp, { recursive: true, force: true }));
const work = path.join(tmp, 'work');
const source = path.join(tmp, 'source.git');
const mirror = path.join(tmp, 'mirror.git');
for (const repo of [source, mirror]) git('init', '--bare', repo);
git('init', work);
for (const repo of [source, work]) {
git('-C', repo, 'config', 'user.name', 'Fixture');
git('-C', repo, 'config', 'user.email', 'fixture@example.invalid');
}
git('-C', work, 'checkout', '-b', 'develop');
fs.writeFileSync(path.join(work, 'snapshot'), 'static artifact');
git('-C', work, 'add', '.');
git('-C', work, 'commit', '-m', 'snapshot');
git('-C', work, 'push', source, 'develop');
const bin = path.join(tmp, 'bin');
fs.mkdirSync(bin);
// Model GitHub's Git Data API with real Git objects and non-forced ref updates.
// The fixture's post-receive hook simulates Gitee receiving the push event.
fs.writeFileSync(
path.join(bin, 'gh'),
`#!${process.execPath}
const fs = require('node:fs');
const path = require('node:path');
const { execFileSync, spawnSync } = require('node:child_process');
const args = process.argv.slice(2);
const endpoint = args.find((arg) => arg.startsWith('repos/'));
const method = args.includes('--method') ? args[args.indexOf('--method') + 1] : 'GET';
const fields = {};
for (let i = 0; i < args.length; i++) {
if (args[i] === '-f' || args[i] === '-F') {
const field = args[++i];
const separator = field.indexOf('=');
fields[field.slice(0, separator)] = field.slice(separator + 1);
}
}
fs.appendFileSync(process.env.QL_TEST_API_LOG, JSON.stringify({method, endpoint, fields, time: Date.now()}) + '\\n');
const source = process.env.QL_TEST_SOURCE;
const git = (...args) => execFileSync('git', ['-C', source, ...args], {encoding: 'utf8', stdio: 'pipe'}).trim();
const fail = (status) => { console.error('gh: API failure (HTTP ' + status + ')'); process.exit(1); };
const ref = 'refs/heads/gitee-sync-trigger';
const isUpdate = method === 'PATCH' || endpoint.endsWith('/git/refs');
if (process.env.QL_TEST_API_HANG) spawnSync('sleep', ['30'], {stdio: 'ignore'});
if (process.env.QL_TEST_API_REJECT) fail(500);
if (isUpdate && process.env.QL_TEST_COLLISION && !fs.existsSync(process.env.QL_TEST_COLLISION)) {
const current = git('rev-parse', ref);
const tree = git('rev-parse', current + '^{tree}');
const peer = git('commit-tree', tree, '-p', current, '-m', 'Concurrent notifier');
git('update-ref', ref, peer);
fs.writeFileSync(process.env.QL_TEST_COLLISION, peer);
}
if (isUpdate && process.env.QL_TEST_UPDATE_FAILURE && !fs.existsSync(process.env.QL_TEST_UPDATE_FAILURE)) {
fs.writeFileSync(process.env.QL_TEST_UPDATE_FAILURE, 'failed');
fail(500);
}
try {
if (method === 'GET' && endpoint.includes('/git/ref/')) {
console.log(git('rev-parse', '--verify', 'refs/' + endpoint.split('/git/ref/')[1]));
} else if (method === 'GET' && endpoint.includes('/git/commits/')) {
console.log(git('rev-parse', endpoint.split('/git/commits/')[1] + '^{tree}'));
} else if (endpoint.endsWith('/git/trees')) {
const body = JSON.parse(fs.readFileSync(0, 'utf8'));
const entries = body.tree.map((entry) => {
const blob = execFileSync('git', ['-C', source, 'hash-object', '-w', '--stdin'], {encoding: 'utf8', input: entry.content}).trim();
return entry.mode + ' blob ' + blob + '\\t' + entry.path + '\\n';
}).join('');
console.log(execFileSync('git', ['-C', source, 'mktree'], {encoding: 'utf8', input: entries}).trim());
} else if (endpoint.endsWith('/git/commits')) {
const parents = fields['parents[]'] ? ['-p', fields['parents[]']] : [];
console.log(git('commit-tree', fields.tree, ...parents, '-m', fields.message));
} else if (isUpdate) {
const current = spawnSync('git', ['-C', source, 'rev-parse', '--verify', ref], {encoding: 'utf8'});
const old = current.status === 0 ? current.stdout.trim() : '0'.repeat(40);
if (method === 'PATCH') {
if (fields.force !== 'false' || current.status !== 0 || spawnSync('git', ['-C', source, 'merge-base', '--is-ancestor', old, fields.sha]).status !== 0) fail(422);
} else if (current.status === 0 || fields.ref !== ref) fail(422);
git('update-ref', ref, fields.sha, old);
const hook = path.join(source, 'hooks/post-receive');
if (fs.existsSync(hook)) execFileSync(hook, {cwd: source, input: old + ' ' + fields.sha + ' ' + ref + '\\n', env: {...process.env, GIT_DIR: source}, stdio: ['pipe', 'pipe', 'pipe']});
} else fail(400);
} catch (error) { fail(method === 'GET' ? 404 : 422); }
`,
{ mode: 0o755 },
);
const env = {
...process.env,
PATH: `${bin}${path.delimiter}${process.env.PATH}`,
QL_TEST_SOURCE: source,
QL_TEST_API_LOG: path.join(tmp, 'api.log'),
QL_MIRROR_NOTIFY_REPO: 'fixture/source',
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'true',
QL_MIRROR_WAIT_SECONDS: '4',
QL_MIRROR_RETRY_AFTER_SECONDS: '1',
QL_MIRROR_POLL_INTERVAL: '1',
QL_MIRROR_QUERY_TIMEOUT: '1',
};
return { tmp, work, source, mirror, env };
}
test('static notifications use a tiny REST branch with unchanged trees and preserve published refs', (t) => {
const { source, env } = fixture(t);
const snapshot = git('-C', source, 'rev-parse', 'develop');
execFileSync('bash', [notify, source, source, 'develop', 'branch'], {
stdio: 'pipe',
env,
});
const first = git('-C', source, 'rev-parse', triggerRef);
execFileSync('bash', [notify, source, source, 'develop', 'branch'], {
stdio: 'pipe',
env,
});
const second = git('-C', source, 'rev-parse', triggerRef);
assert.notEqual(first, second);
assert.equal(git('-C', source, 'rev-parse', `${second}^`), first);
assert.equal(
git('-C', source, 'ls-tree', '-r', '--name-only', second),
'.gitee-sync-trigger',
);
assert.equal(git('-C', source, 'diff', '--name-only', first, second), '');
assert.equal(git('-C', source, 'rev-parse', 'develop'), snapshot);
assert.match(
git('-C', source, 'log', '-1', '--format=%s', second),
new RegExp(snapshot),
);
});
test('REST notifications retry a newer parent and GitHub 500s with backoff', (t) => {
const { tmp, source, env } = fixture(t);
execFileSync('bash', [notify, source, source, 'develop'], {
stdio: 'pipe',
env,
});
const parent = git('-C', source, 'rev-parse', triggerRef);
const flag = path.join(tmp, 'collision');
const collision = spawnSync('bash', [notify, source, source, 'develop'], {
encoding: 'utf8',
env: { ...env, QL_TEST_COLLISION: flag },
});
assert.equal(collision.status, 0, collision.stderr + collision.stdout);
assert.match(collision.stdout, /notification update failed \(attempt 1\)/);
const peer = fs.readFileSync(flag, 'utf8');
assert.equal(git('-C', source, 'rev-parse', `${peer}^`), parent);
assert.equal(git('-C', source, 'rev-parse', `${triggerRef}^`), peer);
const failed = path.join(tmp, 'failed');
const recovery = spawnSync('bash', [notify, source, source, 'develop'], {
encoding: 'utf8',
env: { ...env, QL_TEST_UPDATE_FAILURE: failed },
});
assert.equal(recovery.status, 0, recovery.stderr + recovery.stdout);
assert.match(recovery.stderr, /HTTP 500/);
const calls = fs
.readFileSync(env.QL_TEST_API_LOG, 'utf8')
.trim()
.split('\n')
.map(JSON.parse);
const updates = calls.filter((call) => call.method === 'PATCH');
assert.ok(updates.every((call) => call.fields.force === 'false'));
const last = updates.slice(-2);
assert.ok(
last[1].time - last[0].time >= 1900,
'failed API updates must back off',
);
});
test('missing snapshots fail before sending any notification', (t) => {
const { source, env } = fixture(t);
const missing = spawnSync('bash', [notify, source, source, 'missing'], {
encoding: 'utf8',
env,
});
assert.equal(missing.status, 1, missing.stderr + missing.stdout);
assert.match(missing.stdout, /Published source ref.*does not exist/);
assert.equal(fs.existsSync(env.QL_TEST_API_LOG), false);
});
test('notification failure does not prevent an already synchronized mirror from passing', (t) => {
const { source, mirror, env } = fixture(t);
git('-C', source, 'push', mirror, 'develop');
const result = spawnSync('bash', [verify, source, mirror, 'develop'], {
encoding: 'utf8',
env: { ...env, QL_TEST_API_REJECT: 'true' },
});
assert.equal(result.status, 0, result.stderr + result.stdout);
assert.match(
result.stdout,
/Could not send the mirror notification; continuing/,
);
assert.match(result.stdout, /Mirror matches the latest source refs/);
});
test('a failed delayed notification keeps polling until the mirror updates', async (t) => {
const { source, mirror, env } = fixture(t);
const child = spawn('bash', [verify, source, mirror, 'develop'], {
env: {
...env,
QL_MIRROR_NOTIFY_ON_START: 'false',
QL_MIRROR_WAIT_SECONDS: '10',
QL_TEST_API_REJECT: 'true',
},
});
let output = '';
let error = '';
let synchronized = false;
child.stderr.on('data', (data) => {
error += data;
});
child.stdout.on('data', (data) => {
output += data;
if (
!synchronized &&
output.includes('Could not read notification ref (attempt 1)')
) {
synchronized = true;
git('-C', source, 'push', mirror, 'develop');
}
});
const status = await new Promise((resolve, reject) => {
child.on('error', reject);
child.on('close', resolve);
});
assert.ok(synchronized);
assert.equal(status, 0, error + output);
assert.match(output, /Mirror matches the latest source refs/);
});
test('failed or hung notifications preserve the mirror polling deadline', (t) => {
const { source, mirror, env } = fixture(t);
for (const failure of ['QL_TEST_API_REJECT', 'QL_TEST_API_HANG']) {
const started = Date.now();
const result = spawnSync('bash', [verify, source, mirror, 'develop'], {
encoding: 'utf8',
timeout: 6000,
env: {
...env,
QL_MIRROR_NOTIFY_ON_START: 'false',
QL_MIRROR_WAIT_SECONDS: '3',
[failure]: 'true',
},
});
assert.equal(result.status, 1, result.stderr + result.stdout);
assert.ok(Date.now() - started < 5000);
assert.match(result.stdout, /::error::Mirror did not match.*within 3s/);
}
});
test('static verification retries a small GitHub notification after a missed sync interval', (t) => {
const { tmp, source, mirror, env } = fixture(t);
const snapshot = git('-C', source, 'rev-parse', 'develop');
git('-C', source, 'update-ref', 'refs/heads/v2.22.0', snapshot);
git('-C', source, 'update-ref', 'refs/tags/v2.22.0', snapshot);
const countFile = path.join(tmp, 'requests');
fs.writeFileSync(
path.join(source, 'hooks/post-receive'),
`#!/bin/sh
while read before after ref; do
if [ "$ref" = refs/heads/gitee-sync-trigger ]; then
count=0
if [ -f "$QL_TEST_COUNT" ]; then count=$(cat "$QL_TEST_COUNT"); fi
count=$((count + 1))
printf '%s' "$count" > "$QL_TEST_COUNT"
if [ "$count" -ge 2 ]; then
git push --quiet "$QL_TEST_MIRROR" refs/heads/v2.22.0:refs/heads/v2.22.0 refs/tags/v2.22.0:refs/tags/v2.22.0
fi
fi
done
`,
{ mode: 0o755 },
);
const result = spawnSync('bash', [verify, source, mirror, 'v2.22.0', 'tag'], {
encoding: 'utf8',
env: {
...env,
QL_TEST_COUNT: countFile,
QL_TEST_MIRROR: mirror,
},
});
assert.equal(result.status, 0, result.stderr + result.stdout);
assert.equal(fs.readFileSync(countFile, 'utf8'), '2');
assert.match(result.stdout, /Retrying the GitHub notification/);
assert.match(result.stdout, /Mirror matches the latest source refs/);
assert.doesNotMatch(result.stdout, /::error::/);
for (const ref of ['refs/heads/v2.22.0', 'refs/tags/v2.22.0']) {
assert.equal(git('-C', mirror, 'rev-parse', ref), snapshot);
}
});
test('static verification still fails when fresh notifications never synchronize the mirror', (t) => {
const { source, mirror, env } = fixture(t);
const result = spawnSync(
'bash',
[verify, source, mirror, 'develop', 'branch'],
{
encoding: 'utf8',
env: {
...env,
QL_MIRROR_WAIT_SECONDS: '2',
},
},
);
assert.equal(result.status, 1, result.stderr + result.stdout);
assert.equal(git('-C', source, 'rev-list', '--count', triggerRef), '2');
assert.match(result.stdout, /::error::Mirror did not match/);
const invalid = spawnSync(
'bash',
[verify, source, mirror, 'develop', 'branch'],
{
encoding: 'utf8',
env: { ...env, QL_MIRROR_WAIT_SECONDS: '0' },
},
);
assert.equal(invalid.status, 2);
});
test('code mirrors that already match do not send another notification', (t) => {
const { source, mirror, env } = fixture(t);
git('-C', source, 'push', mirror, 'develop');
const codeEnv = {
...env,
QL_MIRROR_NOTIFY_REPO: 'fixture/source',
QL_MIRROR_NOTIFY_ON_START: 'false',
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'false',
};
const result = spawnSync(
'bash',
[verify, source, mirror, 'develop', 'branch'],
{
encoding: 'utf8',
env: codeEnv,
},
);
assert.equal(result.status, 0, result.stderr + result.stdout);
assert.match(result.stdout, /Mirror matches the latest source refs/);
assert.doesNotMatch(result.stdout, /Sent a small GitHub push event via REST/);
assert.notEqual(
spawnSync('git', ['-C', source, 'rev-parse', '--verify', triggerRef])
.status,
0,
);
const invalid = spawnSync(
'bash',
[verify, source, mirror, 'develop', 'branch'],
{
encoding: 'utf8',
env: { ...codeEnv, QL_MIRROR_NOTIFY_ON_START: 'later' },
},
);
assert.equal(invalid.status, 2);
});
test('code mirrors notify after the sync interval and verify tags without version branches', (t) => {
const { tmp, work, source, mirror, env } = fixture(t);
const snapshot = git('-C', source, 'rev-parse', 'develop');
git('-C', work, 'tag', '-a', 'v2.22.0', '-m', 'release');
git('-C', work, 'push', source, 'refs/tags/v2.22.0');
const tag = git('-C', source, 'rev-parse', 'refs/tags/v2.22.0');
const countFile = path.join(tmp, 'requests');
fs.writeFileSync(
path.join(source, 'hooks/post-receive'),
`#!/bin/sh
while read before after ref; do
if [ "$ref" = refs/heads/gitee-sync-trigger ]; then
printf 'notification\n' >> "$QL_TEST_COUNT"
git push --quiet "$QL_TEST_MIRROR" refs/tags/v2.22.0:refs/tags/v2.22.0
fi
done
`,
{ mode: 0o755 },
);
const started = Date.now();
const result = spawnSync('bash', [verify, source, mirror, 'v2.22.0', 'tag'], {
encoding: 'utf8',
env: {
...env,
QL_MIRROR_NOTIFY_REPO: 'fixture/source',
QL_MIRROR_NOTIFY_ON_START: 'false',
QL_MIRROR_INCLUDE_VERSION_BRANCH: 'false',
QL_MIRROR_RETRY_AFTER_SECONDS: '1',
QL_TEST_COUNT: countFile,
QL_TEST_MIRROR: mirror,
},
});
assert.equal(result.status, 0, result.stderr + result.stdout);
assert.ok(
Date.now() - started >= 900,
'notification should wait for the sync interval',
);
assert.equal(fs.readFileSync(countFile, 'utf8'), 'notification\n');
assert.equal(git('-C', source, 'rev-list', '--count', triggerRef), '1');
assert.equal(git('-C', source, 'rev-parse', 'develop'), snapshot);
assert.equal(git('-C', mirror, 'rev-parse', 'refs/tags/v2.22.0'), tag);
assert.notEqual(
spawnSync('git', [
'-C',
mirror,
'rev-parse',
'--verify',
'refs/heads/v2.22.0',
]).status,
0,
);
});
test('code mirrors that never synchronize fail after one delayed notification', (t) => {
const { source, mirror, env } = fixture(t);
const result = spawnSync(
'bash',
[verify, source, mirror, 'develop', 'branch'],
{
encoding: 'utf8',
env: {
...env,
QL_MIRROR_NOTIFY_REPO: 'fixture/source',
QL_MIRROR_NOTIFY_ON_START: 'false',
QL_MIRROR_WAIT_SECONDS: '3',
QL_MIRROR_RETRY_AFTER_SECONDS: '1',
},
},
);
assert.equal(result.status, 1, result.stderr + result.stdout);
assert.equal(git('-C', source, 'rev-list', '--count', triggerRef), '1');
assert.match(result.stdout, /::error::Mirror did not match/);
});
+4 -4
View File
@@ -53,7 +53,7 @@ test('mirror jobs select the built branch, retain hidden artifacts and propagate
const workflow = yaml.load(
fs.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'),
);
for (const name of ['static_gitee', 'static_gitlab']) {
for (const name of ['static_gitlab']) {
const job = workflow.jobs[name],
step = job.steps.find((step) => step.run);
assert.equal(job.needs, 'build-static');
@@ -156,18 +156,18 @@ test('static backfill uses existing snapshots and skips all source, build and im
);
const job = workflow.jobs['sync-existing-static'];
assert.match(job.if, /workflow_dispatch.*inputs.static_sync_only/);
assert.deepEqual(job.strategy.matrix.mirror, ['gitlab', 'gitee']);
assert.deepEqual(job.strategy.matrix.mirror, ['gitlab']);
const sync = job.steps.find((step) => step.run);
assert.equal(sync.env.STATIC_REF, '${{ inputs.static_ref }}');
assert.equal(sync.env.STATIC_REF_TYPE, '${{ inputs.static_ref_type }}');
assert.match(sync.run, /bash scripts\/sync-static.sh/);
assert.doesNotMatch(sync.run, /publish-static|docker|pub.sh/);
for (const name of ['validate', 'code_gitlab', 'code_gitee', 'build-static'])
for (const name of ['validate', 'code_gitlab', 'build-static'])
assert.match(workflow.jobs[name].if, /!inputs.static_sync_only/);
// Downstream jobs require the skipped build; none bypass dependency failures.
for (const name of [
'static_gitlab',
'static_gitee',
'verify-gitee-mirrors',
'build-alpine',
'build-debian',
'build-alpine310',