mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 02:27:44 +08:00
feat(ql3): expose cancellation availability summary
This commit is contained in:
@@ -8,8 +8,10 @@ export {
|
||||
type BlockingCancellationDispatchResult,
|
||||
type PostgresRunCancellationDispatchInspectCommand,
|
||||
type PostgresRunCancellationDispatchRearmCommand,
|
||||
type PostgresRunCancellationDispatchSummaryCommand,
|
||||
type RunCancellationDispatchDiagnostic,
|
||||
type RunCancellationDispatchRearmReceipt,
|
||||
type RunCancellationDispatchSummary,
|
||||
} from '../run-management/runCancellationDispatchManagementRepository';
|
||||
export {
|
||||
PostgresClusterRunCancellationRepository,
|
||||
|
||||
+215
-5
@@ -57,15 +57,47 @@ export type RunCancellationDispatchRearmReceipt = Readonly<{
|
||||
eventSequence: number;
|
||||
}>;
|
||||
|
||||
interface ManagementAuthority {
|
||||
export type RunCancellationDispatchSummary = Readonly<{
|
||||
projectId: string;
|
||||
observedAtMs: number;
|
||||
assessment: 'clear' | 'converging' | 'attention_required';
|
||||
operatorAction: 'none' | 'wait' | 'inspect';
|
||||
dispatches: Readonly<{
|
||||
total: number;
|
||||
pending: number;
|
||||
leased: number;
|
||||
retryWait: number;
|
||||
dispatched: number;
|
||||
blocked: number;
|
||||
}>;
|
||||
signals: Readonly<{
|
||||
due: number;
|
||||
expiredLease: number;
|
||||
}>;
|
||||
blockingResults: Readonly<{
|
||||
identityMismatch: number;
|
||||
pidMismatch: number;
|
||||
unsupported: number;
|
||||
invalid: number;
|
||||
}>;
|
||||
oldestBlockedAtMs?: number;
|
||||
}>;
|
||||
|
||||
interface ProjectManagementAuthority {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
interface ManagementAuthority extends ProjectManagementAuthority {
|
||||
readonly runId: string;
|
||||
}
|
||||
|
||||
export interface PostgresRunCancellationDispatchSummaryCommand
|
||||
extends ProjectManagementAuthority {}
|
||||
|
||||
export interface PostgresRunCancellationDispatchInspectCommand
|
||||
extends ManagementAuthority {}
|
||||
|
||||
@@ -282,6 +314,39 @@ function normalizeInspectCommand(
|
||||
return normalizeAuthority(value, []);
|
||||
}
|
||||
|
||||
function normalizeSummaryCommand(
|
||||
value: Readonly<PostgresRunCancellationDispatchSummaryCommand>,
|
||||
): Readonly<PostgresRunCancellationDispatchSummaryCommand> {
|
||||
const input = exact(value, [
|
||||
'projectId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'principal',
|
||||
'policyFence',
|
||||
]);
|
||||
const principal = exact(input.principal, [
|
||||
'subject',
|
||||
'authenticationId',
|
||||
'authenticatedAtMs',
|
||||
'expiresAtMs',
|
||||
'assurance',
|
||||
]) as unknown as SecurityPrincipal;
|
||||
const fence = exact(input.policyFence, [
|
||||
'projectVersion',
|
||||
'bindingVersion',
|
||||
]);
|
||||
return Object.freeze({
|
||||
projectId: identifier(input.projectId),
|
||||
requestId: identifier(input.requestId),
|
||||
auditEventId: uuid(input.auditEventId),
|
||||
principal,
|
||||
policyFence: Object.freeze({
|
||||
projectVersion: boundedInteger(fence.projectVersion, 1, 2_147_483_647),
|
||||
bindingVersion: boundedInteger(fence.bindingVersion, 1, 2_147_483_647),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeRearmCommand(
|
||||
value: Readonly<PostgresRunCancellationDispatchRearmCommand>,
|
||||
): Readonly<PostgresRunCancellationDispatchRearmCommand> {
|
||||
@@ -383,7 +448,7 @@ function strongPrincipal(
|
||||
|
||||
async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<ManagementAuthority>,
|
||||
command: Readonly<ProjectManagementAuthority>,
|
||||
): Promise<void> {
|
||||
const result = await client.query<Row>(
|
||||
`SELECT "ql3"."lock_run_management_policy_fence"(
|
||||
@@ -406,8 +471,11 @@ async function confirmAuthorization(
|
||||
|
||||
async function recordAllowedAudit(
|
||||
client: PostgresClient,
|
||||
command: Readonly<ManagementAuthority>,
|
||||
operationId: 'run.cancellation.inspect' | 'run.cancellation.rearm',
|
||||
command: Readonly<ProjectManagementAuthority>,
|
||||
operationId:
|
||||
| 'run.cancellation.summary'
|
||||
| 'run.cancellation.inspect'
|
||||
| 'run.cancellation.rearm',
|
||||
observedAtMs: number,
|
||||
): Promise<void> {
|
||||
const inserted = await client.query<Row>(
|
||||
@@ -463,6 +531,78 @@ async function recordAllowedAudit(
|
||||
}
|
||||
}
|
||||
|
||||
function summaryProjection(
|
||||
projectId: string,
|
||||
observedAtMs: number,
|
||||
row: Row,
|
||||
): Readonly<RunCancellationDispatchSummary> {
|
||||
const dispatches = Object.freeze({
|
||||
total: integer(row, 'total'),
|
||||
pending: integer(row, 'pending'),
|
||||
leased: integer(row, 'leased'),
|
||||
retryWait: integer(row, 'retryWait'),
|
||||
dispatched: integer(row, 'dispatched'),
|
||||
blocked: integer(row, 'blocked'),
|
||||
});
|
||||
const signals = Object.freeze({
|
||||
due: integer(row, 'due'),
|
||||
expiredLease: integer(row, 'expiredLease'),
|
||||
});
|
||||
const blockingResults = Object.freeze({
|
||||
identityMismatch: integer(row, 'identityMismatch'),
|
||||
pidMismatch: integer(row, 'pidMismatch'),
|
||||
unsupported: integer(row, 'unsupported'),
|
||||
invalid: integer(row, 'invalid'),
|
||||
});
|
||||
if (
|
||||
dispatches.total !==
|
||||
dispatches.pending +
|
||||
dispatches.leased +
|
||||
dispatches.retryWait +
|
||||
dispatches.dispatched +
|
||||
dispatches.blocked ||
|
||||
dispatches.blocked !==
|
||||
blockingResults.identityMismatch +
|
||||
blockingResults.pidMismatch +
|
||||
blockingResults.unsupported +
|
||||
blockingResults.invalid ||
|
||||
signals.due > dispatches.pending + dispatches.retryWait ||
|
||||
signals.expiredLease > dispatches.leased
|
||||
) {
|
||||
throw new TypeError('PostgreSQL cancellation management summary is invalid');
|
||||
}
|
||||
const oldestBlockedAtMs = optionalInteger(row, 'oldestBlockedAtMs');
|
||||
if (
|
||||
(dispatches.blocked === 0) !== (oldestBlockedAtMs === undefined) ||
|
||||
(oldestBlockedAtMs !== undefined && oldestBlockedAtMs > observedAtMs)
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL cancellation management blocked summary is invalid',
|
||||
);
|
||||
}
|
||||
const active =
|
||||
dispatches.pending +
|
||||
dispatches.leased +
|
||||
dispatches.retryWait +
|
||||
dispatches.blocked;
|
||||
return Object.freeze({
|
||||
projectId,
|
||||
observedAtMs,
|
||||
assessment:
|
||||
dispatches.blocked > 0
|
||||
? 'attention_required'
|
||||
: active > 0
|
||||
? 'converging'
|
||||
: 'clear',
|
||||
operatorAction:
|
||||
dispatches.blocked > 0 ? 'inspect' : active > 0 ? 'wait' : 'none',
|
||||
dispatches,
|
||||
signals,
|
||||
blockingResults,
|
||||
...(oldestBlockedAtMs === undefined ? {} : { oldestBlockedAtMs }),
|
||||
});
|
||||
}
|
||||
|
||||
function runStatus(row: Row): RunStatus {
|
||||
const value = text(row, 'runStatus') as RunStatus;
|
||||
if (!RUN_STATUSES.includes(value)) {
|
||||
@@ -608,6 +748,76 @@ export class PostgresRunCancellationDispatchManagementRepository {
|
||||
}
|
||||
}
|
||||
|
||||
summary(
|
||||
value: Readonly<PostgresRunCancellationDispatchSummaryCommand>,
|
||||
): Promise<Readonly<RunCancellationDispatchSummary>> {
|
||||
const command = normalizeSummaryCommand(value);
|
||||
return this.transaction(async (client) => {
|
||||
const observedAtMs = await databaseNow(client);
|
||||
const authorized = Object.freeze({
|
||||
...command,
|
||||
principal: strongPrincipal(command.principal, observedAtMs),
|
||||
});
|
||||
await confirmAuthorization(client, authorized);
|
||||
const result = await client.query<Row>(
|
||||
`SELECT count(*)::bigint AS total,
|
||||
count(*) FILTER (WHERE dispatch.status = 'pending')::bigint
|
||||
AS pending,
|
||||
count(*) FILTER (WHERE dispatch.status = 'leased')::bigint
|
||||
AS leased,
|
||||
count(*) FILTER (WHERE dispatch.status = 'retry_wait')::bigint
|
||||
AS "retryWait",
|
||||
count(*) FILTER (WHERE dispatch.status = 'dispatched')::bigint
|
||||
AS dispatched,
|
||||
count(*) FILTER (WHERE dispatch.status = 'blocked')::bigint
|
||||
AS blocked,
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status IN ('pending', 'retry_wait')
|
||||
AND dispatch.next_attempt_at_ms <= $2
|
||||
)::bigint AS due,
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'leased'
|
||||
AND dispatch.lease_expires_at_ms <= $2
|
||||
)::bigint AS "expiredLease",
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'identity_mismatch'
|
||||
)::bigint AS "identityMismatch",
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'pid_mismatch'
|
||||
)::bigint AS "pidMismatch",
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'unsupported'
|
||||
)::bigint AS unsupported,
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'invalid'
|
||||
)::bigint AS invalid,
|
||||
min(dispatch.updated_at_ms) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
) AS "oldestBlockedAtMs"
|
||||
FROM "ql3"."run_cancellation_dispatches" AS dispatch
|
||||
JOIN "ql3"."runs" AS run ON run.id = dispatch.run_id
|
||||
WHERE run.project_id = $1`,
|
||||
[command.projectId, observedAtMs],
|
||||
);
|
||||
if (result.rows.length !== 1) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL cancellation management summary row is invalid',
|
||||
);
|
||||
}
|
||||
await recordAllowedAudit(
|
||||
client,
|
||||
authorized,
|
||||
'run.cancellation.summary',
|
||||
observedAtMs,
|
||||
);
|
||||
return summaryProjection(command.projectId, observedAtMs, result.rows[0]!);
|
||||
});
|
||||
}
|
||||
|
||||
inspect(
|
||||
value: Readonly<PostgresRunCancellationDispatchInspectCommand>,
|
||||
): Promise<Readonly<RunCancellationDispatchDiagnostic>> {
|
||||
|
||||
+117
@@ -42,6 +42,11 @@ function rearmCommand(overrides = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
function summaryCommand(overrides = {}) {
|
||||
const { runId: _runId, ...authority } = command();
|
||||
return { ...authority, ...overrides };
|
||||
}
|
||||
|
||||
function runRow() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
@@ -104,6 +109,30 @@ function fixture(options = {}) {
|
||||
) {
|
||||
return { rows: options.replay ? [options.replay] : [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches" AS dispatch')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
options.summary ?? {
|
||||
total: '5',
|
||||
pending: '1',
|
||||
leased: '1',
|
||||
retryWait: '1',
|
||||
dispatched: '1',
|
||||
blocked: '1',
|
||||
due: '1',
|
||||
expiredLease: '1',
|
||||
identityMismatch: '1',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: String(NOW - 1_500),
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT attempt_id AS "attemptId"') &&
|
||||
!text.includes('dispatchStatus') &&
|
||||
@@ -179,6 +208,94 @@ test('inspects one low-sensitive blocked dispatch under run.read authority', asy
|
||||
);
|
||||
});
|
||||
|
||||
test('summarizes one Project without exposing Run, Attempt or lease identity', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
const result = await repository.summary(summaryCommand());
|
||||
assert.deepEqual(result, {
|
||||
projectId: 'project-1',
|
||||
observedAtMs: NOW,
|
||||
assessment: 'attention_required',
|
||||
operatorAction: 'inspect',
|
||||
dispatches: {
|
||||
total: 5,
|
||||
pending: 1,
|
||||
leased: 1,
|
||||
retryWait: 1,
|
||||
dispatched: 1,
|
||||
blocked: 1,
|
||||
},
|
||||
signals: { due: 1, expiredLease: 1 },
|
||||
blockingResults: {
|
||||
identityMismatch: 1,
|
||||
pidMismatch: 0,
|
||||
unsupported: 0,
|
||||
invalid: 0,
|
||||
},
|
||||
oldestBlockedAtMs: NOW - 1_500,
|
||||
});
|
||||
const read = calls.find(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."run_cancellation_dispatches" AS dispatch'),
|
||||
);
|
||||
assert.deepEqual(read.params, ['project-1', NOW]);
|
||||
for (const forbidden of [
|
||||
'attempt_id AS',
|
||||
'run.id AS',
|
||||
'lease_owner',
|
||||
'lease_token',
|
||||
'lease_token_digest',
|
||||
]) {
|
||||
assert.equal(read.sql.includes(forbidden), false);
|
||||
}
|
||||
const audit = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
params[2] === 'run.cancellation.summary',
|
||||
);
|
||||
assert.equal(audit.params[0], summaryCommand().auditEventId);
|
||||
});
|
||||
|
||||
test('derives clear and converging assessments from fixed status counts', async () => {
|
||||
const clear = fixture({
|
||||
summary: {
|
||||
total: '1',
|
||||
pending: '0',
|
||||
leased: '0',
|
||||
retryWait: '0',
|
||||
dispatched: '1',
|
||||
blocked: '0',
|
||||
due: '0',
|
||||
expiredLease: '0',
|
||||
identityMismatch: '0',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: null,
|
||||
},
|
||||
});
|
||||
assert.equal((await clear.repository.summary(summaryCommand())).assessment, 'clear');
|
||||
|
||||
const converging = fixture({
|
||||
summary: {
|
||||
total: '1',
|
||||
pending: '0',
|
||||
leased: '0',
|
||||
retryWait: '1',
|
||||
dispatched: '0',
|
||||
blocked: '0',
|
||||
due: '1',
|
||||
expiredLease: '0',
|
||||
identityMismatch: '0',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: null,
|
||||
},
|
||||
});
|
||||
const result = await converging.repository.summary(summaryCommand());
|
||||
assert.equal(result.assessment, 'converging');
|
||||
assert.equal(result.operatorAction, 'wait');
|
||||
});
|
||||
|
||||
test('rearms an exact blocked dispatch with one event and allowed audit', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
const result = await repository.rearm(rearmCommand());
|
||||
|
||||
Reference in New Issue
Block a user