mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): add strong cluster run management
This commit is contained in:
whyour
@@ -163,6 +163,7 @@ function database(serverVersionNum = '160014') {
|
||||
'enforce_plugin_package_stage_provenance',
|
||||
'lock_active_plugin_package_project',
|
||||
'lock_approval_policy_fence',
|
||||
'lock_run_management_policy_fence',
|
||||
'plugin_package_lifecycle_blocking_runs',
|
||||
'plugin_package_automation_start_allowed',
|
||||
'plugin_package_run_start_allowed',
|
||||
|
||||
@@ -12,6 +12,7 @@ const {
|
||||
createClusterAutomationIdentityKeysetFile,
|
||||
createClusterApprovalIdentityKeysetFile,
|
||||
createClusterModelProviderCredentialIdentityKeysetFile,
|
||||
createClusterRunIdentityKeysetFile,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-identity-keyset');
|
||||
|
||||
const NOW_MS = 1_700_000_000_000;
|
||||
@@ -216,6 +217,38 @@ function providerCredentialAssertion(key, overrides = {}) {
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
function runAssertion(key, overrides = {}) {
|
||||
const header = Buffer.from(
|
||||
JSON.stringify({
|
||||
alg: 'EdDSA',
|
||||
kid: key.kid,
|
||||
typ: 'ql3-run-management+jwt',
|
||||
}),
|
||||
).toString('base64url');
|
||||
const now = Math.floor(NOW_MS / 1000);
|
||||
const payload = Buffer.from(
|
||||
JSON.stringify({
|
||||
acr: 'urn:ql3:mfa',
|
||||
amr: ['pwd', 'otp'],
|
||||
aud: 'qinglong3-run-management',
|
||||
auth_time: now - 10,
|
||||
exp: now + 120,
|
||||
iat: now,
|
||||
iss: ISSUER,
|
||||
jti: `run-assertion-${key.kid}`,
|
||||
ql3_purpose: 'run-management',
|
||||
sub: 'run-operator-1',
|
||||
...overrides,
|
||||
}),
|
||||
).toString('base64url');
|
||||
const signed = `${header}.${payload}`;
|
||||
return `${signed}.${sign(
|
||||
null,
|
||||
Buffer.from(signed, 'ascii'),
|
||||
key.privateKey,
|
||||
).toString('base64url')}`;
|
||||
}
|
||||
|
||||
async function atomicWrite(filePath, document) {
|
||||
const nextPath = `${filePath}.next`;
|
||||
await writeFile(nextPath, `${JSON.stringify(document)}\n`, { mode: 0o644 });
|
||||
@@ -374,6 +407,34 @@ test('loads a provider credential keyset isolated by type, purpose and audience'
|
||||
});
|
||||
});
|
||||
|
||||
test('loads a Run keyset isolated from every other management purpose', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const key = reviewedKey('run-identity-key-1');
|
||||
await atomicWrite(filePath, {
|
||||
...keyset(1, [key]),
|
||||
audience: 'qinglong3-run-management',
|
||||
});
|
||||
const provider = createClusterRunIdentityKeysetFile({
|
||||
filePath,
|
||||
now: () => NOW_MS,
|
||||
});
|
||||
const principal = await provider.bind(runAssertion(key)).authenticate();
|
||||
assert.deepEqual(principal.subject, {
|
||||
type: 'user',
|
||||
id: 'run-operator-1',
|
||||
});
|
||||
await assert.rejects(provider.bind(approvalAssertion(key)).authenticate(), {
|
||||
code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID',
|
||||
});
|
||||
await assert.rejects(
|
||||
provider
|
||||
.bind(runAssertion(key, { ql3_purpose: 'approval-management' }))
|
||||
.authenticate(),
|
||||
{ code: 'CLUSTER_PLUGIN_PACKAGE_IDENTITY_ASSERTION_INVALID' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('supports overlap rotation then immediately revokes the previous key', async () => {
|
||||
await fixture(async ({ filePath }) => {
|
||||
const first = reviewedKey('issuer-key-1');
|
||||
|
||||
@@ -197,6 +197,7 @@ function database(serverVersionNum = '160014') {
|
||||
'plugin_package_tool_start_allowed',
|
||||
'plugin_package_workflow_admission_snapshot',
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'lock_run_management_policy_fence',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementAuthorizationError,
|
||||
createClusterRunManagementService,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const EXECUTION_DIGEST = 'b'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
||||
const GENERATED = [
|
||||
'019f9500-0000-4000-8000-000000000010',
|
||||
'019f9500-0000-4000-8000-000000000011',
|
||||
'019f9500-0000-4000-8000-000000000012',
|
||||
'019f9500-0000-4000-8000-000000000013',
|
||||
];
|
||||
|
||||
function request() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: '019f9500-0000-4000-8000-000000000001',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000002',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000003',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function policyRow(role = 'operator') {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
projectName: 'Project 1',
|
||||
projectSlug: 'project-1',
|
||||
projectStatus: 'active',
|
||||
projectVersion: 2,
|
||||
projectCreatedAtMs: '1',
|
||||
projectUpdatedAtMs: '2',
|
||||
bindingProjectId: 'project-1',
|
||||
bindingSubjectType: 'user',
|
||||
bindingSubjectId: 'operator-1',
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: role,
|
||||
bindingMutationId: 'binding-3',
|
||||
bindingChangedByType: 'user',
|
||||
bindingChangedById: 'owner-1',
|
||||
bindingCreatedAtMs: '3',
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(role = 'operator') {
|
||||
const calls = [];
|
||||
const pool = {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'pool', sql: text, params });
|
||||
if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
|
||||
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected pool query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
return {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ scope: 'client', sql: text, params });
|
||||
if (
|
||||
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.startsWith('SELECT set_config')
|
||||
) return { rows: [], rowCount: 0 };
|
||||
if (text.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: NOW }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('lock_run_management_policy_fence')) {
|
||||
return { rows: [{ matches: true }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('idempotency_key = $2')) return { rows: [] };
|
||||
if (text.includes('WHERE run.id = $1')) {
|
||||
return {
|
||||
rows: [{
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
taskName: 'Task 1',
|
||||
taskSnapshotRef: TASK_REVISION,
|
||||
parentRunId: null,
|
||||
triggerType: 'task_start',
|
||||
executionOwner: 'runtime',
|
||||
inputRef: null,
|
||||
priority: 1,
|
||||
runStatus: 'failed',
|
||||
runVersion: 7,
|
||||
attemptExecutorType: 'remote_worker',
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."task_definitions"')) {
|
||||
return { rows: [{ enabled: true }] };
|
||||
}
|
||||
if (text.includes('task_execution_revisions')) {
|
||||
return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
|
||||
}
|
||||
if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
|
||||
throw new Error(`unexpected client query: ${text}`);
|
||||
},
|
||||
release() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
let index = 0;
|
||||
return {
|
||||
calls,
|
||||
service: createClusterRunManagementService({
|
||||
pool,
|
||||
now: () => NOW,
|
||||
randomUuid: () => GENERATED[index++],
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test('authorizes run.retry and keeps all generated aggregate identities server-side', async () => {
|
||||
const { calls, service } = fixture();
|
||||
const result = await service.retry(request());
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.equal(result.runId, GENERATED[0]);
|
||||
assert.equal(result.attemptId, GENERATED[1]);
|
||||
const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
|
||||
assert.equal(runInsert.params[0], GENERATED[0]);
|
||||
assert.equal(runInsert.params.includes(GENERATED[2]), false);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('denied policy writes only the caller-supplied failure audit', async () => {
|
||||
const { calls, service } = fixture('viewer');
|
||||
await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
|
||||
const audits = calls.filter(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
);
|
||||
assert.equal(audits.length, 1);
|
||||
assert.equal(audits[0].params[0], request().failureAuditEventId);
|
||||
assert.equal(calls.some(({ scope }) => scope === 'client'), false);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
validateClusterRunManagementClientResult,
|
||||
} = require('@qinglong/cluster-admin/run-management-client');
|
||||
const {
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
const {
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
} = require('@qinglong/cluster-admin/plugin-package-management-client');
|
||||
|
||||
const command = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9400-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function response(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9400-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9400-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 1_000_000,
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('validates one low-sensitive retry response against the request fence', () => {
|
||||
assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
|
||||
});
|
||||
|
||||
test('rejects response target, execution placement and shape drift', () => {
|
||||
for (const candidate of [
|
||||
response({ projectId: 'project-2' }),
|
||||
response({ executorType: 'local_process' }),
|
||||
response({ sourceRunVersion: 8 }),
|
||||
{ ...response(), principal: { type: 'user', id: 'operator-1' } },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => validateClusterRunManagementClientResult(candidate, command),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { readFileSync } = require('node:fs');
|
||||
const { request: httpsRequest } = require('node:https');
|
||||
const { resolve } = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementRateLimitedError,
|
||||
} = require('@qinglong/cluster-admin/run-management');
|
||||
const {
|
||||
startClusterRunManagementHttp,
|
||||
} = require('@qinglong/cluster-admin/run-management-http');
|
||||
|
||||
const SERVER_KEY = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-key.pem');
|
||||
const SERVER_CERT = resolve(__dirname, '../../ql3-cluster-control/test/fixtures/mtls/server-cert.pem');
|
||||
const PATH = '/api/v3/runs/management';
|
||||
|
||||
function post(port, path = PATH) {
|
||||
const body = Buffer.from(JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9600-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9600-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9600-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
},
|
||||
}));
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const outgoing = httpsRequest({
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path,
|
||||
method: 'POST',
|
||||
rejectUnauthorized: false,
|
||||
agent: false,
|
||||
headers: {
|
||||
authorization: 'Bearer assertion',
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(body.length),
|
||||
},
|
||||
}, (incoming) => {
|
||||
const chunks = [];
|
||||
incoming.on('data', (chunk) => chunks.push(chunk));
|
||||
incoming.once('end', () => {
|
||||
const bytes = Buffer.concat(chunks);
|
||||
resolvePromise({
|
||||
statusCode: incoming.statusCode,
|
||||
headers: incoming.headers,
|
||||
body: bytes.length ? JSON.parse(bytes.toString('utf8')) : null,
|
||||
});
|
||||
});
|
||||
});
|
||||
outgoing.once('error', reject);
|
||||
outgoing.end(body);
|
||||
});
|
||||
}
|
||||
|
||||
test('serves only the Run path and maps durable quota to bounded HTTP facts', async () => {
|
||||
const application = await startClusterRunManagementHttp({
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
tls: {
|
||||
privateKey: Buffer.from(readFileSync(SERVER_KEY)),
|
||||
certificate: Buffer.from(readFileSync(SERVER_CERT)),
|
||||
},
|
||||
identities: {
|
||||
async reload() { throw new Error('not used'); },
|
||||
bind() {
|
||||
return { authenticate: async () => ({
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 900,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'hardware',
|
||||
}) };
|
||||
},
|
||||
},
|
||||
transport: {
|
||||
async execute(_command, authentication) {
|
||||
await authentication.authenticate();
|
||||
throw new ClusterRunManagementRateLimitedError(1_500);
|
||||
},
|
||||
},
|
||||
now: () => 1_000,
|
||||
});
|
||||
try {
|
||||
const limited = await post(application.address.port);
|
||||
assert.equal(limited.statusCode, 429);
|
||||
assert.equal(limited.headers['retry-after'], '2');
|
||||
assert.equal(limited.body.error.code, 'rate_limited');
|
||||
assert.match(limited.body.requestId, /^[0-9a-f-]{36}$/);
|
||||
const absent = await post(application.address.port, '/api/v3/approvals/management');
|
||||
assert.equal(absent.statusCode, 404);
|
||||
assert.equal(absent.body.error.code, 'not_found');
|
||||
} finally {
|
||||
await application.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementProcessConfigError,
|
||||
loadClusterRunManagementProcessConfig,
|
||||
startClusterRunManagementProcess,
|
||||
} = require('@qinglong/cluster-admin/run-management-process');
|
||||
|
||||
function enabled(overrides = {}) {
|
||||
return {
|
||||
QL3_RUN_MANAGEMENT_ENABLED: 'true',
|
||||
QL3_PROFILE: 'cluster-admin',
|
||||
QL3_RUN_MANAGEMENT_TLS_CERT_FILE: '/run/ql3/run/tls.crt',
|
||||
QL3_RUN_MANAGEMENT_TLS_KEY_FILE: '/run/ql3/run/tls.key',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CA_FILE: '/run/ql3/run/client-ca.crt',
|
||||
QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE: '/run/ql3/run/client.crl',
|
||||
QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE: '/run/ql3/run/identity.json',
|
||||
QL3_POSTGRES_RUN_MANAGER_HOST: 'postgres.qinglong3-system.svc',
|
||||
QL3_POSTGRES_RUN_MANAGER_DATABASE: 'qinglong3',
|
||||
QL3_POSTGRES_RUN_MANAGER_USER: 'ql3_run_manager',
|
||||
QL3_POSTGRES_RUN_MANAGER_PASSWORD: 'secret',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: 'postgres.qinglong3-system.svc',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled Run manager acquires no PostgreSQL or file authority', async () => {
|
||||
let opened = false;
|
||||
const runtime = await startClusterRunManagementProcess({
|
||||
environment: {},
|
||||
openDatabase: async () => {
|
||||
opened = true;
|
||||
throw new Error('must not open');
|
||||
},
|
||||
});
|
||||
assert.equal(runtime.status, 'disabled');
|
||||
assert.equal(opened, false);
|
||||
await runtime.close();
|
||||
});
|
||||
|
||||
test('loads a bounded opt-in Run-only process configuration', () => {
|
||||
const config = loadClusterRunManagementProcessConfig(enabled());
|
||||
assert.equal(config.enabled, true);
|
||||
assert.equal(config.port, 8448);
|
||||
assert.equal(config.http.maxConcurrentRequests, 16);
|
||||
assert.equal(config.database.pool.maxConnections, 2);
|
||||
assert.equal(config.database.connection.user, 'ql3_run_manager');
|
||||
assert.deepEqual(config.database.connection.tls, {
|
||||
mode: 'verify-full',
|
||||
servername: 'postgres.qinglong3-system.svc',
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects profile drift and implicit insecure PostgreSQL', () => {
|
||||
assert.throws(
|
||||
() => loadClusterRunManagementProcessConfig(enabled({ QL3_PROFILE: 'cluster-control' })),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
loadClusterRunManagementProcessConfig(
|
||||
enabled({
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_MODE: 'disable',
|
||||
QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME: undefined,
|
||||
}),
|
||||
),
|
||||
ClusterRunManagementProcessConfigError,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
ClusterRunManagementTransportRequestError,
|
||||
createClusterRunManagementTransport,
|
||||
normalizeClusterRunManagementCommand,
|
||||
} = require('@qinglong/cluster-admin/run-management-transport');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
function principal(overrides = {}) {
|
||||
return {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: 999_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000001',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000002',
|
||||
body: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
mutationId: '019f9300-0000-4000-8000-000000000003',
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function retryResult() {
|
||||
return {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: '019f9300-0000-4000-8000-000000000010',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `qltd:v1:1:${'a'.repeat(64)}`,
|
||||
attemptId: '019f9300-0000-4000-8000-000000000011',
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: NOW,
|
||||
};
|
||||
}
|
||||
|
||||
test('routes one exact strong User retry and emits the shared response', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry(request) {
|
||||
calls.push(request);
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(command(), {
|
||||
authenticate: async () => principal(),
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].mutationId, command().request.body.mutationId);
|
||||
assert.equal(calls[0].principal.assurance, 'multi_factor');
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
retry: {
|
||||
schema: 'qinglong/run-manual-retry@v1',
|
||||
...retryResult(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects weak or non-User identity before service authority', async () => {
|
||||
let called = false;
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() {
|
||||
called = true;
|
||||
return retryResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () => principal({ assurance: 'single_factor' }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
await assert.rejects(
|
||||
transport.execute(command(), {
|
||||
authenticate: async () =>
|
||||
principal({ subject: { type: 'agent', id: 'agent-1' } }),
|
||||
}),
|
||||
ClusterRunManagementTransportAuthenticationError,
|
||||
);
|
||||
assert.equal(called, false);
|
||||
});
|
||||
|
||||
test('rejects widened commands and ambiguous audit identity', () => {
|
||||
assert.throws(
|
||||
() => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ failureAuditEventId: command().request.auditEventId }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeClusterRunManagementCommand(
|
||||
command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
|
||||
),
|
||||
ClusterRunManagementTransportRequestError,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user