mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,582 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize } = require('sequelize');
|
||||
const {
|
||||
defineSchemaMigrationModel,
|
||||
} = require('../../back/data/schemaMigration');
|
||||
const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
|
||||
const {
|
||||
runCancellationRequestMigration,
|
||||
} = require('../../back/migrations/0004-run-cancellation-request');
|
||||
const {
|
||||
runAttemptDeadlineMigration,
|
||||
} = require('../../back/migrations/0006-run-attempt-deadline');
|
||||
const {
|
||||
workerRegistryMigration,
|
||||
} = require('../../back/migrations/0008-worker-registry');
|
||||
const {
|
||||
runDispatchLeaseMigration,
|
||||
} = require('../../back/migrations/0009-run-dispatch-lease');
|
||||
const { runMigrations } = require('../../back/migrations/runner');
|
||||
const {
|
||||
LegacySequelizeRunDispatchLeaseRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runDispatchLeaseRepository');
|
||||
const {
|
||||
LegacySequelizeRunRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
|
||||
const {
|
||||
LegacySequelizeWorkerRegistryRepository,
|
||||
} = require('../../back/runtime/adapters/legacy-sequelize/workerRegistryRepository');
|
||||
const {
|
||||
PrimaryRunCreator,
|
||||
} = require('../../back/runtime/application/primaryRunCreator');
|
||||
const {
|
||||
RemoteRunActivationService,
|
||||
RemoteRunActivationStateError,
|
||||
} = require('../../back/runtime/application/remoteRunActivationService');
|
||||
const {
|
||||
RunCommandService,
|
||||
} = require('../../back/runtime/application/runCommandService');
|
||||
const {
|
||||
RunDispatchLeaseService,
|
||||
} = require('../../back/runtime/application/runDispatchLeaseService');
|
||||
const {
|
||||
BoundWorkerControlPlaneClient,
|
||||
WorkerControlService,
|
||||
} = require('../../back/runtime/application/workerControlService');
|
||||
const {
|
||||
RunDispatchLeaseFenceRejectedError,
|
||||
} = require('../../back/runtime/domain/runDispatchLease');
|
||||
|
||||
const START = 1_760_200_000_000;
|
||||
const SESSION_A = '019f7c00-0000-7000-8000-000000000001';
|
||||
const SESSION_B = '019f7c00-0000-7000-8000-000000000002';
|
||||
const TOKEN = 'remote_activation_token_abcdefghijklmnopqrstuvwxyz0123456789';
|
||||
const TOKEN_B =
|
||||
'remote_activation_token_b_abcdefghijklmnopqrstuvwxyz0123456789';
|
||||
let idSequence = 3_000;
|
||||
|
||||
function nextId() {
|
||||
idSequence += 1;
|
||||
return `019f7c00-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function capabilities() {
|
||||
return {
|
||||
architecture: 'arm64',
|
||||
operatingSystem: 'linux',
|
||||
executors: ['remote_worker'],
|
||||
runtimes: [{ name: 'node', version: '24.14.0' }],
|
||||
labels: {},
|
||||
capacity: { cpuCores: 1, memoryBytes: 256 * 1024 * 1024 },
|
||||
features: ['direct_file_log'],
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture(t) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => database.close());
|
||||
await runMigrations({
|
||||
database,
|
||||
migrationModel: defineSchemaMigrationModel(database),
|
||||
migrations: [
|
||||
runSchemaMigration,
|
||||
runCancellationRequestMigration,
|
||||
runAttemptDeadlineMigration,
|
||||
workerRegistryMigration,
|
||||
runDispatchLeaseMigration,
|
||||
],
|
||||
logger: { info() {} },
|
||||
});
|
||||
let nowMs = START;
|
||||
const runs = new LegacySequelizeRunRepository(database);
|
||||
const leases = new LegacySequelizeRunDispatchLeaseRepository(database);
|
||||
const workerControl = new WorkerControlService(
|
||||
new LegacySequelizeWorkerRegistryRepository(database),
|
||||
{ leaseDurationMs: 60_000, clock: { now: () => nowMs } },
|
||||
);
|
||||
const leaseControl = new RunDispatchLeaseService(leases, {
|
||||
leaseDurationMs: 5_000,
|
||||
clock: { now: () => nowMs },
|
||||
createEventId: nextId,
|
||||
});
|
||||
return {
|
||||
database,
|
||||
runs,
|
||||
leases,
|
||||
workerControl,
|
||||
leaseControl,
|
||||
activation(repository = leases) {
|
||||
return new RemoteRunActivationService(repository, {
|
||||
clock: { now: () => nowMs },
|
||||
createEventId: nextId,
|
||||
});
|
||||
},
|
||||
setNow(value) {
|
||||
nowMs = value;
|
||||
},
|
||||
getNow() {
|
||||
return nowMs;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function register(context, sessionId = SESSION_A) {
|
||||
return new BoundWorkerControlPlaneClient(context.workerControl, {
|
||||
workerId: 'worker-a',
|
||||
}).register({
|
||||
workerId: 'worker-a',
|
||||
sessionId,
|
||||
capabilities: capabilities(),
|
||||
maxConcurrentRuns: 1,
|
||||
availableSlots: 1,
|
||||
});
|
||||
}
|
||||
|
||||
async function createRun(context) {
|
||||
return new PrimaryRunCreator(context.runs, nextId).create(
|
||||
{
|
||||
projectId: 'default',
|
||||
taskId: `activation:${nextId()}`,
|
||||
taskRevision: 'activation-v1',
|
||||
triggerType: 'manual',
|
||||
executionOrigin: 'manual',
|
||||
acceptedAtMs: context.getNow(),
|
||||
actor: { type: 'user', id: 'user:1' },
|
||||
},
|
||||
'remote_worker',
|
||||
);
|
||||
}
|
||||
|
||||
async function claim(context, reference, worker, leaseToken = TOKEN) {
|
||||
const result = await context.leaseControl.claim(
|
||||
{ workerId: worker.id },
|
||||
{
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
workerId: worker.id,
|
||||
workerSessionId: worker.sessionId,
|
||||
workerGeneration: worker.generation,
|
||||
leaseToken,
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 'claimed');
|
||||
return result;
|
||||
}
|
||||
|
||||
function command(reference, worker, lease, overrides = {}) {
|
||||
return {
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
workerId: worker.id,
|
||||
workerSessionId: worker.sessionId,
|
||||
workerGeneration: worker.generation,
|
||||
leaseGeneration: lease.leaseGeneration,
|
||||
leaseToken: lease.leaseToken,
|
||||
expectedLeaseVersion: lease.version,
|
||||
executorType: 'remote_worker',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('atomically acknowledges starting/running and replays both steps', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
const service = context.activation();
|
||||
|
||||
context.setNow(START + 100);
|
||||
const startCommand = command(reference, worker, claimed.lease);
|
||||
const starting = await service.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
startCommand,
|
||||
);
|
||||
assert.equal(starting.status, 'applied');
|
||||
assert.equal(starting.run.status, 'dispatching');
|
||||
assert.equal(starting.attempt.status, 'starting');
|
||||
assert.equal(starting.attempt.workerId, worker.id);
|
||||
assert.equal(starting.lease.version, 0);
|
||||
assert.equal(starting.events.length, 1);
|
||||
assert.equal(starting.events[0].actorType, 'worker');
|
||||
assert.equal(starting.events[0].actorId, worker.id);
|
||||
assert.equal(
|
||||
(await service.acknowledgeStarting({ workerId: worker.id }, startCommand))
|
||||
.status,
|
||||
'already_starting',
|
||||
);
|
||||
|
||||
context.setNow(START + 200);
|
||||
const runningCommand = command(reference, worker, claimed.lease, {
|
||||
startedAtMs: START + 150,
|
||||
executorHandle: 'remote-worker:durable-handle-1',
|
||||
logArtifactId: '019f7c00-0000-7000-8000-000000000099',
|
||||
});
|
||||
const running = await service.acknowledgeRunning(
|
||||
{ workerId: worker.id },
|
||||
runningCommand,
|
||||
);
|
||||
assert.equal(running.status, 'applied');
|
||||
assert.equal(running.run.status, 'running');
|
||||
assert.equal(running.attempt.status, 'running');
|
||||
assert.equal(running.attempt.executorHandle, runningCommand.executorHandle);
|
||||
assert.equal(running.events.length, 2);
|
||||
assert.deepEqual(
|
||||
running.events.map((event) => event.type),
|
||||
['attempt.running', 'run.running'],
|
||||
);
|
||||
assert.equal(
|
||||
(await service.acknowledgeRunning({ workerId: worker.id }, runningCommand))
|
||||
.status,
|
||||
'already_running',
|
||||
);
|
||||
assert.equal(
|
||||
(await service.acknowledgeStarting({ workerId: worker.id }, startCommand))
|
||||
.status,
|
||||
'already_running',
|
||||
);
|
||||
|
||||
const persistedEvents = await context.runs.listEvents(reference.run.id);
|
||||
assert.deepEqual(
|
||||
persistedEvents.slice(-3).map((event) => event.type),
|
||||
['attempt.starting', 'attempt.running', 'run.running'],
|
||||
);
|
||||
});
|
||||
|
||||
test('requires starting acknowledgement before accepting an executor handle', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
context.setNow(START + 100);
|
||||
await assert.rejects(
|
||||
context.activation().acknowledgeRunning(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease, {
|
||||
startedAtMs: START + 50,
|
||||
executorHandle: 'remote-worker:unexpected',
|
||||
}),
|
||||
),
|
||||
RemoteRunActivationStateError,
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findRunById(reference.run.id)).status,
|
||||
'dispatching',
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findAttemptById(reference.attempt.id)).status,
|
||||
'claimed',
|
||||
);
|
||||
});
|
||||
|
||||
test('fences a stale lease version after renewal and accepts the current version', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
context.setNow(START + 500);
|
||||
const renewed = await context.leaseControl.renew(
|
||||
{ workerId: worker.id },
|
||||
{
|
||||
attemptId: reference.attempt.id,
|
||||
workerId: worker.id,
|
||||
workerSessionId: worker.sessionId,
|
||||
workerGeneration: worker.generation,
|
||||
leaseGeneration: claimed.lease.leaseGeneration,
|
||||
leaseToken: claimed.lease.leaseToken,
|
||||
expectedVersion: claimed.lease.version,
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
context
|
||||
.activation()
|
||||
.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof RunDispatchLeaseFenceRejectedError &&
|
||||
error.reason === 'version_mismatch',
|
||||
);
|
||||
const applied = await context
|
||||
.activation()
|
||||
.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, renewed),
|
||||
);
|
||||
assert.equal(applied.status, 'applied');
|
||||
assert.equal(applied.lease.version, renewed.version);
|
||||
});
|
||||
|
||||
test('rejects expired leases and a replaced Worker session without Run mutation', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const expiredReference = await createRun(context);
|
||||
const expiredClaim = await claim(context, expiredReference, worker);
|
||||
context.setNow(START + 5_000);
|
||||
await assert.rejects(
|
||||
context
|
||||
.activation()
|
||||
.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(expiredReference, worker, expiredClaim.lease),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof RunDispatchLeaseFenceRejectedError &&
|
||||
error.reason === 'lease_expired',
|
||||
);
|
||||
|
||||
context.setNow(START + 6_000);
|
||||
const currentWorker = await register(context, SESSION_B);
|
||||
const replacedReference = await createRun(context);
|
||||
const currentClaim = await claim(
|
||||
context,
|
||||
replacedReference,
|
||||
currentWorker,
|
||||
TOKEN_B,
|
||||
);
|
||||
await assert.rejects(
|
||||
context.activation().acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(replacedReference, worker, {
|
||||
...currentClaim.lease,
|
||||
workerSessionId: worker.sessionId,
|
||||
workerGeneration: worker.generation,
|
||||
}),
|
||||
),
|
||||
RunDispatchLeaseFenceRejectedError,
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findAttemptById(replacedReference.attempt.id)).status,
|
||||
'claimed',
|
||||
);
|
||||
});
|
||||
|
||||
test('atomically records executor start failure and completes the lease', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
const service = context.activation();
|
||||
context.setNow(START + 100);
|
||||
await service.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
context.setNow(START + 200);
|
||||
const failureCommand = command(reference, worker, claimed.lease);
|
||||
const failed = await service.failStart(
|
||||
{ workerId: worker.id },
|
||||
failureCommand,
|
||||
);
|
||||
assert.equal(failed.status, 'applied');
|
||||
assert.equal(failed.run.status, 'failed');
|
||||
assert.equal(failed.attempt.status, 'failed');
|
||||
assert.equal(failed.attempt.callbackSequence, 1);
|
||||
assert.equal(failed.attempt.errorCode, 'EXECUTOR_START_FAILED');
|
||||
assert.equal(failed.lease.status, 'completed');
|
||||
assert.equal(failed.lease.version, claimed.lease.version + 1);
|
||||
assert.equal(
|
||||
(await service.failStart({ workerId: worker.id }, failureCommand)).status,
|
||||
'already_terminal',
|
||||
);
|
||||
});
|
||||
|
||||
test('lets a cancellation request win over executor start failure', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
const service = context.activation();
|
||||
context.setNow(START + 100);
|
||||
const starting = await service.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
await new RunCommandService(context.runs, nextId).requestCancellation({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
expectedVersion: starting.run.version,
|
||||
reason: 'user',
|
||||
atMs: START + 150,
|
||||
actor: { type: 'user', id: 'user:1' },
|
||||
});
|
||||
context.setNow(START + 200);
|
||||
const result = await service.failStart(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
assert.equal(result.run.status, 'cancelled');
|
||||
assert.equal(result.attempt.status, 'cancelled');
|
||||
assert.equal(result.run.errorCode, 'EXECUTION_CANCELLED');
|
||||
assert.equal(result.lease.status, 'completed');
|
||||
});
|
||||
|
||||
test('preserves timeout semantics when timeout wins during executor start', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
const service = context.activation();
|
||||
context.setNow(START + 100);
|
||||
const starting = await service.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
await new RunCommandService(context.runs, nextId).requestCancellation({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
expectedVersion: starting.run.version,
|
||||
reason: 'timeout',
|
||||
atMs: START + 150,
|
||||
actor: { type: 'system', id: 'timeout-supervisor' },
|
||||
});
|
||||
context.setNow(START + 200);
|
||||
const result = await service.failStart(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
assert.equal(result.run.status, 'timed_out');
|
||||
assert.equal(result.attempt.status, 'timed_out');
|
||||
assert.equal(result.run.errorCode, 'EXECUTION_TIMED_OUT');
|
||||
assert.equal(result.lease.status, 'completed');
|
||||
});
|
||||
|
||||
test('rolls back Run, Attempt, events, and lease when start-failure commit crashes', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
context.setNow(START + 100);
|
||||
await context
|
||||
.activation()
|
||||
.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
const eventCount = (await context.runs.listEvents(reference.run.id)).length;
|
||||
const crashingRepository = {
|
||||
completeWithLease(commandValue, work) {
|
||||
return context.leases.completeWithLease(
|
||||
commandValue,
|
||||
async (transaction, leaseValue) => {
|
||||
await work(transaction, leaseValue);
|
||||
throw new Error('crash before commit');
|
||||
},
|
||||
);
|
||||
},
|
||||
};
|
||||
context.setNow(START + 200);
|
||||
await assert.rejects(
|
||||
context
|
||||
.activation(crashingRepository)
|
||||
.failStart(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
),
|
||||
/crash before commit/,
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findRunById(reference.run.id)).status,
|
||||
'dispatching',
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findAttemptById(reference.attempt.id)).status,
|
||||
'starting',
|
||||
);
|
||||
assert.equal(
|
||||
(await context.leases.findByAttemptId(reference.attempt.id)).status,
|
||||
'leased',
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.listEvents(reference.run.id)).length,
|
||||
eventCount,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await context
|
||||
.activation()
|
||||
.failStart(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
)
|
||||
).status,
|
||||
'applied',
|
||||
);
|
||||
});
|
||||
|
||||
test('rolls back both running transitions when the second event write fails', async (t) => {
|
||||
const context = await fixture(t);
|
||||
const worker = await register(context);
|
||||
const reference = await createRun(context);
|
||||
const claimed = await claim(context, reference, worker);
|
||||
context.setNow(START + 100);
|
||||
await context
|
||||
.activation()
|
||||
.acknowledgeStarting(
|
||||
{ workerId: worker.id },
|
||||
command(reference, worker, claimed.lease),
|
||||
);
|
||||
const eventCount = (await context.runs.listEvents(reference.run.id)).length;
|
||||
const crashingRepository = {
|
||||
withLease(commandValue, work) {
|
||||
return context.leases.withLease(
|
||||
commandValue,
|
||||
(transaction, leaseValue) => {
|
||||
let appended = 0;
|
||||
const failingTransaction = new Proxy(transaction, {
|
||||
get(target, property) {
|
||||
if (property === 'appendEvent') {
|
||||
return async (event) => {
|
||||
appended += 1;
|
||||
if (appended === 2) throw new Error('second event failed');
|
||||
return target.appendEvent(event);
|
||||
};
|
||||
}
|
||||
const value = Reflect.get(target, property, target);
|
||||
return typeof value === 'function' ? value.bind(target) : value;
|
||||
},
|
||||
});
|
||||
return work(failingTransaction, leaseValue);
|
||||
},
|
||||
);
|
||||
},
|
||||
};
|
||||
context.setNow(START + 200);
|
||||
const runningCommand = command(reference, worker, claimed.lease, {
|
||||
startedAtMs: START + 150,
|
||||
executorHandle: 'remote-worker:rollback-handle',
|
||||
});
|
||||
await assert.rejects(
|
||||
context
|
||||
.activation(crashingRepository)
|
||||
.acknowledgeRunning({ workerId: worker.id }, runningCommand),
|
||||
/second event failed/,
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findRunById(reference.run.id)).status,
|
||||
'dispatching',
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.findAttemptById(reference.attempt.id)).status,
|
||||
'starting',
|
||||
);
|
||||
assert.equal(
|
||||
(await context.runs.listEvents(reference.run.id)).length,
|
||||
eventCount,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await context
|
||||
.activation()
|
||||
.acknowledgeRunning({ workerId: worker.id }, runningCommand)
|
||||
).status,
|
||||
'applied',
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user