mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 01:32:44 +08:00
372 lines
12 KiB
JavaScript
372 lines
12 KiB
JavaScript
'use strict';
|
|
|
|
const assert = require('node:assert/strict');
|
|
const { mkdtemp, lstat, rm } = require('node:fs/promises');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
const { test } = require('node:test');
|
|
const {
|
|
createClusterTaskExecutionRevision,
|
|
} = require('@qinglong/runtime-core/cluster-execution-revision');
|
|
const {
|
|
createClusterRemoteExecutionOffer,
|
|
} = require('@qinglong/runtime-core/remote-dispatch');
|
|
const {
|
|
createRemoteExecutionOfferPullBody,
|
|
} = require('@qinglong/runtime-core/remote-offer-delivery');
|
|
const {
|
|
digestRunDispatchLeaseToken,
|
|
} = require('@qinglong/runtime-core');
|
|
const {
|
|
WorkerRemoteOfferFileJournal,
|
|
WorkerRemoteOfferPullCoordinator,
|
|
createWorkerRemoteOfferClaimRecord,
|
|
normalizeWorkerRemoteExecutionInboxRecord,
|
|
} = require('../dist/remote-execution/remoteOfferDeliveryEntrypoint');
|
|
|
|
const SESSION = '018f0000-0000-7000-8000-000000000001';
|
|
const SOURCE_DIGEST = 'a'.repeat(64);
|
|
const TASK_REVISION = `qltd:v1:1:${SOURCE_DIGEST}`;
|
|
const STATS = Object.freeze({
|
|
pages: 1,
|
|
candidates: 1,
|
|
plansUnavailable: 0,
|
|
placementMismatches: 0,
|
|
claimAttempts: 1,
|
|
claimRaces: 0,
|
|
});
|
|
|
|
const session = Object.freeze({
|
|
workerId: 'edge-1',
|
|
sessionId: SESSION,
|
|
generation: 2,
|
|
});
|
|
|
|
function executionRevision() {
|
|
return createClusterTaskExecutionRevision({
|
|
projectId: 'project-1',
|
|
taskId: 'task-1',
|
|
taskRevision: TASK_REVISION,
|
|
sourceRevision: 1,
|
|
sourceContentDigest: SOURCE_DIGEST,
|
|
executorType: 'remote_worker',
|
|
planSchema: 'qinglong/command-execution@v1',
|
|
command: { kind: 'argv', file: '/bin/true', args: [] },
|
|
environment: [],
|
|
createdAtMs: 1,
|
|
});
|
|
}
|
|
|
|
function offerFromRequest(request, version = 0) {
|
|
const revision = executionRevision();
|
|
return createClusterRemoteExecutionOffer({
|
|
offerId: request.body.offerId,
|
|
deliveryKind: version === 0 ? 'new_claim' : 'lease_recovery',
|
|
executionDigest: revision.contentDigest,
|
|
candidate: {
|
|
runId: 'run-1',
|
|
attemptId: 'attempt-1',
|
|
projectId: 'project-1',
|
|
taskId: 'task-1',
|
|
taskRevision: TASK_REVISION,
|
|
priority: 1,
|
|
queuedAtMs: 10,
|
|
attemptCreatedAtMs: 11,
|
|
attemptNumber: 1,
|
|
executorType: 'remote_worker',
|
|
},
|
|
worker: {
|
|
workerId: session.workerId,
|
|
sessionId: session.sessionId,
|
|
generation: session.generation,
|
|
},
|
|
lease: {
|
|
attemptId: 'attempt-1',
|
|
runId: 'run-1',
|
|
status: 'leased',
|
|
version,
|
|
leaseGeneration: 1,
|
|
workerId: session.workerId,
|
|
workerSessionId: session.sessionId,
|
|
workerGeneration: session.generation,
|
|
leaseTokenDigest: digestRunDispatchLeaseToken(request.body.leaseToken),
|
|
acquiredAtMs: 20,
|
|
renewedAtMs: 20 + version,
|
|
expiresAtMs: 30_020 + version,
|
|
updatedAtMs: 20 + version,
|
|
},
|
|
leaseToken: request.body.leaseToken,
|
|
executionRevision: revision,
|
|
placementScore: 0,
|
|
});
|
|
}
|
|
|
|
async function journalFixture(t, maximumEntries = 64) {
|
|
const parent = await mkdtemp(path.join(os.tmpdir(), 'ql3-worker-offer-'));
|
|
t.after(() => rm(parent, { recursive: true, force: true }));
|
|
const rootDirectory = path.join(parent, 'inbox');
|
|
const journal = new WorkerRemoteOfferFileJournal({
|
|
rootDirectory,
|
|
maximumEntries,
|
|
ownershipStaleMs: 5_000,
|
|
});
|
|
await journal.acquireOwnership();
|
|
t.after(() => journal.releaseOwnership().catch(() => undefined));
|
|
return { journal, rootDirectory };
|
|
}
|
|
|
|
test('persists the claim intent and accepted capability in private atomic files', async (t) => {
|
|
const { journal, rootDirectory } = await journalFixture(t);
|
|
const claim = createWorkerRemoteOfferClaimRecord({
|
|
workerId: session.workerId,
|
|
workerSessionId: session.sessionId,
|
|
workerGeneration: session.generation,
|
|
offerId: 'offer-1',
|
|
leaseToken: 'worker_generated_lease_capability_0000000000000001',
|
|
}, 1_000);
|
|
await journal.createPendingClaim(claim);
|
|
assert.equal((await journal.readPendingClaim()).offerId, 'offer-1');
|
|
|
|
const request = {
|
|
body: {
|
|
offerId: claim.offerId,
|
|
leaseToken: claim.leaseToken,
|
|
},
|
|
};
|
|
const accepted = await journal.acceptOffer(offerFromRequest(request), 1_001);
|
|
assert.equal(accepted.status, 'accepted');
|
|
const replayed = await journal.acceptOffer(offerFromRequest(request), 1_002);
|
|
assert.equal(replayed.status, 'replayed');
|
|
assert.equal(replayed.record.revision, 0);
|
|
assert.equal(
|
|
(await lstat(path.join(rootDirectory, 'offers', 'offer-1.json'))).mode & 0o777,
|
|
0o600,
|
|
);
|
|
assert.equal((await lstat(rootDirectory)).mode & 0o777, 0o700);
|
|
});
|
|
|
|
test('uses one revision-fenced inbox record through ACK and spawn barriers', async (t) => {
|
|
const { journal } = await journalFixture(t);
|
|
const request = {
|
|
body: {
|
|
offerId: 'offer-state-machine-1',
|
|
leaseToken: 'worker_generated_lease_capability_0000000000000002',
|
|
},
|
|
};
|
|
let record = (await journal.acceptOffer(offerFromRequest(request), 1_000)).record;
|
|
assert.equal(record.state, 'accepted');
|
|
|
|
const advance = async (patch) => {
|
|
const next = normalizeWorkerRemoteExecutionInboxRecord({
|
|
...record,
|
|
...patch,
|
|
revision: record.revision + 1,
|
|
updatedAtMs: record.updatedAtMs + 1,
|
|
});
|
|
await journal.replaceOffer(next, record.revision);
|
|
record = await journal.readOffer(record.offer.offerId);
|
|
};
|
|
|
|
await advance({ state: 'starting_acknowledged' });
|
|
await advance({
|
|
state: 'launching',
|
|
executorStartedAtMs: 1_002,
|
|
logArtifactId: 'log-artifact-1',
|
|
completionReceiptCallbackSequence: 1,
|
|
completionReceiptTokenDigest: 'b'.repeat(64),
|
|
});
|
|
await advance({
|
|
state: 'started',
|
|
executorHandle: 'pid:123:boot:abc',
|
|
executorStartedAtMs: 1_002,
|
|
logArtifactId: 'log-artifact-1',
|
|
});
|
|
await advance({ state: 'running_acknowledged' });
|
|
|
|
assert.equal(record.state, 'running_acknowledged');
|
|
assert.equal(record.revision, 4);
|
|
assert.equal(record.offer.leaseToken, request.body.leaseToken);
|
|
assert.equal(record.executorHandle, 'pid:123:boot:abc');
|
|
assert.equal(record.completionReceiptTokenDigest, 'b'.repeat(64));
|
|
|
|
const regressed = normalizeWorkerRemoteExecutionInboxRecord({
|
|
schemaVersion: 1,
|
|
revision: record.revision + 1,
|
|
state: 'starting_acknowledged',
|
|
offer: record.offer,
|
|
acceptedAtMs: record.acceptedAtMs,
|
|
updatedAtMs: record.updatedAtMs + 1,
|
|
});
|
|
await assert.rejects(
|
|
journal.replaceOffer(regressed, record.revision),
|
|
/invalid_transition/,
|
|
);
|
|
await assert.rejects(
|
|
journal.replaceOffer({ ...record, revision: record.revision + 2 }, record.revision),
|
|
/offer_revision_conflict/,
|
|
);
|
|
assert.equal((await journal.readOffer(record.offer.offerId)).revision, 4);
|
|
});
|
|
|
|
test('lists the single execution inbox authority with a stable bounded cursor', async (t) => {
|
|
const { journal } = await journalFixture(t);
|
|
for (const offerId of ['offer-page-a', 'offer-page-b', 'offer-page-c']) {
|
|
await journal.acceptOffer(offerFromRequest({
|
|
body: {
|
|
offerId,
|
|
leaseToken: `worker_generated_lease_capability_${offerId}`,
|
|
},
|
|
}), 1_000);
|
|
}
|
|
const first = await journal.listOffers({ limit: 2 });
|
|
assert.deepEqual(
|
|
first.records.map((record) => record.offer.offerId),
|
|
['offer-page-a', 'offer-page-b'],
|
|
);
|
|
assert.equal(first.nextAfterOfferId, 'offer-page-b');
|
|
const second = await journal.listOffers({
|
|
afterOfferId: first.nextAfterOfferId,
|
|
limit: 2,
|
|
});
|
|
assert.deepEqual(
|
|
second.records.map((record) => record.offer.offerId),
|
|
['offer-page-c'],
|
|
);
|
|
assert.equal(second.nextAfterOfferId, undefined);
|
|
await assert.rejects(journal.listOffers({ limit: 65 }), /invalid_configuration/);
|
|
});
|
|
|
|
test('keeps one stable claim through transport loss, bounded backoff and restart', async (t) => {
|
|
const { journal, rootDirectory } = await journalFixture(t);
|
|
let now = 1_000;
|
|
const firstRequests = [];
|
|
const first = new WorkerRemoteOfferPullCoordinator({
|
|
journal,
|
|
currentSession: () => session,
|
|
now: () => now,
|
|
random: () => 0.5,
|
|
backoffBaseMs: 1_000,
|
|
transport: {
|
|
async exchange(request) {
|
|
firstRequests.push(request);
|
|
throw new Error('response lost');
|
|
},
|
|
},
|
|
});
|
|
const unavailable = await first.pull(session);
|
|
assert.equal(unavailable.status, 'unavailable');
|
|
assert.equal(unavailable.nextAttemptAtMs, 1_500);
|
|
assert.equal(firstRequests.length, 1);
|
|
|
|
now = 1_400;
|
|
const suppressed = await first.pull(session);
|
|
assert.equal(suppressed.status, 'backoff');
|
|
assert.equal(firstRequests.length, 1);
|
|
|
|
await journal.releaseOwnership();
|
|
const resumedJournal = new WorkerRemoteOfferFileJournal({
|
|
rootDirectory,
|
|
ownershipStaleMs: 5_000,
|
|
});
|
|
await resumedJournal.acquireOwnership();
|
|
t.after(() => resumedJournal.releaseOwnership().catch(() => undefined));
|
|
now = 1_500;
|
|
let resumedRequest;
|
|
const resumed = new WorkerRemoteOfferPullCoordinator({
|
|
journal: resumedJournal,
|
|
currentSession: () => session,
|
|
now: () => now,
|
|
random: () => 0,
|
|
transport: {
|
|
async exchange(request) {
|
|
resumedRequest = request;
|
|
return JSON.stringify(createRemoteExecutionOfferPullBody({
|
|
status: 'offered',
|
|
offer: offerFromRequest(request),
|
|
stats: STATS,
|
|
truncated: false,
|
|
}));
|
|
},
|
|
},
|
|
});
|
|
const result = await resumed.pull(session);
|
|
assert.equal(result.status, 'accepted');
|
|
assert.equal(resumedRequest.body.offerId, firstRequests[0].body.offerId);
|
|
assert.equal(resumedRequest.body.leaseToken, firstRequests[0].body.leaseToken);
|
|
assert.equal(await resumedJournal.readPendingClaim(), undefined);
|
|
assert.equal(
|
|
(await resumedJournal.readOffer(resumedRequest.body.offerId)).offer.leaseToken,
|
|
resumedRequest.body.leaseToken,
|
|
);
|
|
});
|
|
|
|
test('writes the inbox before clearing the pending claim and rejects target drift', async (t) => {
|
|
const events = [];
|
|
let stored;
|
|
let pending;
|
|
const journal = {
|
|
async readPendingClaim() { return pending; },
|
|
async createPendingClaim(record) { pending = record; return record; },
|
|
async replacePendingClaim(record) { pending = record; return record; },
|
|
async clearPendingClaim() { events.push('clear'); pending = undefined; },
|
|
async acceptOffer(offer, acceptedAtMs) {
|
|
events.push('accept');
|
|
stored = { schemaVersion: 1, revision: 0, state: 'accepted', offer, acceptedAtMs, updatedAtMs: acceptedAtMs };
|
|
return { status: 'accepted', record: stored };
|
|
},
|
|
async readOffer() { return stored; },
|
|
};
|
|
const coordinator = new WorkerRemoteOfferPullCoordinator({
|
|
journal,
|
|
currentSession: () => session,
|
|
now: () => 1_000,
|
|
transport: {
|
|
async exchange(request) {
|
|
return JSON.stringify(createRemoteExecutionOfferPullBody({
|
|
status: 'offered',
|
|
offer: offerFromRequest(request),
|
|
stats: STATS,
|
|
truncated: false,
|
|
}));
|
|
},
|
|
},
|
|
});
|
|
assert.equal((await coordinator.pull(session)).status, 'accepted');
|
|
assert.deepEqual(events, ['accept', 'clear']);
|
|
});
|
|
|
|
test('retains the old claim without accepting when the current Session changes', async () => {
|
|
let current = session;
|
|
let pending;
|
|
let accepted = false;
|
|
const journal = {
|
|
async readPendingClaim() { return pending; },
|
|
async createPendingClaim(record) { pending = record; return record; },
|
|
async replacePendingClaim(record) { pending = record; return record; },
|
|
async clearPendingClaim() { pending = undefined; },
|
|
async acceptOffer() { accepted = true; throw new Error('must not accept'); },
|
|
async readOffer() { return undefined; },
|
|
};
|
|
const coordinator = new WorkerRemoteOfferPullCoordinator({
|
|
journal,
|
|
currentSession: () => current,
|
|
now: () => 1_000,
|
|
random: () => 0,
|
|
transport: {
|
|
async exchange(request) {
|
|
current = { ...session, generation: 3 };
|
|
return JSON.stringify(createRemoteExecutionOfferPullBody({
|
|
status: 'offered',
|
|
offer: offerFromRequest(request),
|
|
stats: STATS,
|
|
truncated: false,
|
|
}));
|
|
},
|
|
},
|
|
});
|
|
const result = await coordinator.pull(session);
|
|
assert.equal(result.status, 'unavailable');
|
|
assert.equal(accepted, false);
|
|
assert.equal(pending.workerGeneration, 2);
|
|
});
|