open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 20:15:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(local): bind service completion restart

Browse Source
This commit is contained in:
whyour
2026-08-23 00:34:46 +08:00
parent 4d71dfb937
commit 5f62f2f724
11 changed files with 659 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts
+1
View File
@@ -474,6 +474,7 @@ export function advanceLocalCutoverInstanceHead(
(state === 'manual_required' &&
(current.state === 'legacy_stopped' ||
current.state === 'target_active' ||
current.state === 'reconciliation_completed' ||
current.state === 'rollback_prepared' ||
current.state === 'legacy_restart_requested'));
if (!allowed)
packages/ql3-local-owner-cli/src/deployment/service-manager/serviceBridgeContract.ts
+57 -13
View File
@@ -27,10 +27,14 @@ export type LocalServiceManagerIntentLineage =
generation: number;
expectedActivationDigest: string;
previousRecordDigest: string;
completionFence?: Readonly<{
expectedInstanceHeadDigest: string;
expectedCompletionDigest: string;
}>;
}>;
export interface LocalServiceManagerIntent {
readonly schemaVersion: 1;
readonly schemaVersion: 1 | 2;
readonly kind: 'qinglong3-local-service-manager-intent';
readonly actionId: string;
readonly action: LocalServiceManagerAction;
@@ -169,7 +173,10 @@ function safeInteger(value: unknown, label: string, maximum: number): number {
return value as number;
}
function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage {
function normalizedLineage(
value: unknown,
schemaVersion: 1 | 2,
): LocalServiceManagerIntentLineage {
const lineage = object(value, 'lineage');
if (lineage.mode === 'fresh') {
exact(lineage, ['mode'], 'lineage');
@@ -179,6 +186,7 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage {
lineage,
[
'cutoverId',
...(schemaVersion === 2 ? ['completionFence'] : []),
'expectedActivationDigest',
'generation',
'mode',
@@ -186,6 +194,17 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage {
],
'lineage',
);
const completionFence =
schemaVersion === 2
? object(lineage.completionFence, 'completion fence')
: undefined;
if (completionFence !== undefined) {
exact(
completionFence,
['expectedCompletionDigest', 'expectedInstanceHeadDigest'],
'completion fence',
);
}
if (
lineage.mode !== 'adopted' ||
typeof lineage.cutoverId !== 'string' ||
@@ -196,7 +215,12 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage {
typeof lineage.expectedActivationDigest !== 'string' ||
!DIGEST_PATTERN.test(lineage.expectedActivationDigest) ||
typeof lineage.previousRecordDigest !== 'string' ||
!DIGEST_PATTERN.test(lineage.previousRecordDigest)
!DIGEST_PATTERN.test(lineage.previousRecordDigest) ||
(completionFence !== undefined &&
(typeof completionFence.expectedInstanceHeadDigest !== 'string' ||
!DIGEST_PATTERN.test(completionFence.expectedInstanceHeadDigest) ||
typeof completionFence.expectedCompletionDigest !== 'string' ||
!DIGEST_PATTERN.test(completionFence.expectedCompletionDigest)))
) {
configurationError('adopted lineage is invalid');
}
@@ -206,6 +230,16 @@ function normalizedLineage(value: unknown): LocalServiceManagerIntentLineage {
generation: lineage.generation as number,
expectedActivationDigest: lineage.expectedActivationDigest,
previousRecordDigest: lineage.previousRecordDigest,
...(completionFence === undefined
? {}
: {
completionFence: Object.freeze({
expectedInstanceHeadDigest:
completionFence.expectedInstanceHeadDigest as string,
expectedCompletionDigest:
completionFence.expectedCompletionDigest as string,
}),
}),
});
}
@@ -297,9 +331,12 @@ export function normalizeLocalServiceManagerIntent(
: '/etc/init.d/qinglong3';
const expectedSourceMode = service.kind === 'systemd' ? 0o600 : 0o700;
const expectedDestinationMode = service.kind === 'systemd' ? 0o644 : 0o755;
const lineage = normalizedLineage(intent.lineage);
if (intent.schemaVersion !== 1 && intent.schemaVersion !== 2) {
configurationError('service manager intent schema version is invalid');
}
const schemaVersion = intent.schemaVersion;
const lineage = normalizedLineage(intent.lineage, schemaVersion);
if (
intent.schemaVersion !== 1 ||
intent.kind !== 'qinglong3-local-service-manager-intent' ||
typeof intent.actionId !== 'string' ||
!UUID_V4_PATTERN.test(intent.actionId) ||
@@ -336,20 +373,27 @@ export function normalizeLocalServiceManagerIntent(
}
if (
lineage.mode === 'adopted' &&
((lineage.generation === 1 &&
intent.action !== 'install-enable-start' &&
intent.action !== 'start' &&
intent.action !== 'stop') ||
(lineage.generation >= 2 &&
intent.action !== 'restart' &&
intent.action !== 'stop'))
(schemaVersion === 2
? lineage.generation < 2 || intent.action !== 'restart'
: (lineage.generation === 1 &&
intent.action !== 'install-enable-start' &&
intent.action !== 'start' &&
intent.action !== 'stop') ||
(lineage.generation >= 2 &&
intent.action !== 'restart' &&
intent.action !== 'stop'))
) {
configurationError(
'service manager action does not match the adopted generation',
);
}
if (schemaVersion === 2 && lineage.mode !== 'adopted') {
configurationError(
'service manager v2 intent requires adopted completion lineage',
);
}
const payload = Object.freeze({
schemaVersion: 1 as const,
schemaVersion,
kind: 'qinglong3-local-service-manager-intent' as const,
actionId: intent.actionId,
action: intent.action,
packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverConsumer.ts
+68 -7
View File
@@ -84,6 +84,7 @@ export interface LocalServiceManagerCutoverDependencies {
readonly procRoot?: string;
readonly now?: () => number;
readonly wait?: (milliseconds: number) => Promise<void>;
readonly afterRecordPublished?: () => void;
}
interface AdoptedBinding {
@@ -736,16 +737,28 @@ function replayResult(
): Readonly<LocalServiceManagerCutoverConsumeResult> | undefined {
if (intent.lineage.mode !== 'adopted') return undefined;
const state = desiredState(intent, outcome);
const recordPath = localServiceManagerCutoverRecordPath(intent, state);
if (!fs.existsSync(recordPath)) return undefined;
const candidatePaths = [
localServiceManagerCutoverRecordPath(intent, state),
...(state === 'manual_required'
? []
: [localServiceManagerCutoverRecordPath(intent, 'manual_required')]),
].filter((candidate) => fs.existsSync(candidate));
if (candidatePaths.length === 0) return undefined;
if (candidatePaths.length !== 1) {
configurationError('service manager cutover replay is ambiguous');
}
const record = normalizeLocalServiceManagerCutoverRecord(
readPrivateLocalCommandFile(recordPath),
readPrivateLocalCommandFile(candidatePaths[0]!),
);
const head = readLocalCutoverInstanceHead(
let head = readLocalCutoverInstanceHead(
intent.deployment.root,
intent.instanceId,
currentIdentity().uid,
);
const completionFence =
intent.schemaVersion === 2 && intent.lineage.mode === 'adopted'
? intent.lineage.completionFence
: undefined;
const stoppedCaptureProgress =
record.state === 'target_stopped' &&
(head.state === 'reconciliation_capture_prepared' ||
@@ -759,17 +772,58 @@ function replayResult(
head.state === 'reconciliation_automation_planned' ||
head.state === 'reconciliation_automation_decision_prepared' ||
head.state === 'reconciliation_automation_reviewed');
const completionRestartPendingHead =
(record.state === 'target_active' || record.state === 'manual_required') &&
completionFence !== undefined &&
record.schemaVersion === 3 &&
record.evidence.completionFence?.expectedInstanceHeadDigest ===
completionFence.expectedInstanceHeadDigest &&
record.evidence.completionFence.expectedCompletionDigest ===
completionFence.expectedCompletionDigest &&
head.state === 'reconciliation_completed' &&
head.headDigest === completionFence.expectedInstanceHeadDigest &&
head.sourceRecordDigest === completionFence.expectedCompletionDigest;
if (
record.actionId !== intent.actionId ||
record.intentDigest !== intent.intentDigest ||
record.evidence.managerOutcomeDigest !== outcome.outcomeDigest ||
record.profile !== intent.profile ||
record.instanceId !== intent.instanceId ||
record.activationDigest !== intent.lineage.expectedActivationDigest ||
record.generation !== intent.lineage.generation ||
record.previousRecordDigest !== intent.lineage.previousRecordDigest ||
head.cutoverId !== intent.lineage.cutoverId ||
head.generation !== intent.lineage.generation ||
(!stoppedCaptureProgress && head.state !== record.state) ||
(!stoppedCaptureProgress && head.sourceRecordDigest !== record.recordDigest)
head.generation !==
(completionRestartPendingHead
? intent.lineage.generation - 1
: intent.lineage.generation) ||
(!stoppedCaptureProgress &&
!completionRestartPendingHead &&
head.state !== record.state) ||
(!stoppedCaptureProgress &&
!completionRestartPendingHead &&
head.sourceRecordDigest !== record.recordDigest)
) {
configurationError('service manager cutover replay drifted');
}
if (completionRestartPendingHead) {
head = advanceLocalCutoverInstanceHead(
{
options: { deploymentRoot: intent.deployment.root },
request: {
cutoverId: intent.lineage.cutoverId,
profile: intent.profile,
instanceId: intent.instanceId,
expectedActivationDigest: intent.lineage.expectedActivationDigest,
requestedAtMs: record.completedAtMs,
},
},
currentIdentity().uid,
record.state,
intent.lineage.generation,
record.recordDigest,
);
}
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
@@ -895,6 +949,12 @@ export async function consumeLocalServiceManagerCutoverOutcome(
shutdownReceiptDigest,
processIdentityDigest,
manualReason,
...(intent.schemaVersion === 2 &&
intent.lineage.completionFence !== undefined
? {
completionFence: intent.lineage.completionFence,
}
: {}),
});
const record = localServiceManagerCutoverRecord(
intent,
@@ -907,6 +967,7 @@ export async function consumeLocalServiceManagerCutoverOutcome(
record,
identity.uid,
);
dependencies.afterRecordPublished?.();
const head = advanceLocalCutoverInstanceHead(
{
options: { deploymentRoot: intent.deployment.root },
packages/ql3-local-owner-cli/src/deployment/service-manager/serviceCutoverJournal.ts
+51 -4
View File
@@ -33,11 +33,15 @@ export interface LocalServiceManagerCutoverEvidence {
readonly shutdownReceiptDigest: string | null;
readonly processIdentityDigest: string | null;
readonly manualReason: string | null;
readonly completionFence?: Readonly<{
expectedInstanceHeadDigest: string;
expectedCompletionDigest: string;
}>;
}
export interface LocalServiceManagerCutoverRecord {
readonly schema: typeof SCHEMA;
readonly schemaVersion: 1 | 2;
readonly schemaVersion: 1 | 2 | 3;
readonly actionId: string;
readonly action: LocalServiceManagerAction;
readonly state: LocalServiceManagerCutoverState;
@@ -145,6 +149,7 @@ export function localServiceManagerCutoverRecord(
evidence,
'legacyDataApplicationReceiptDigest',
);
const hasCompletionFence = Object.hasOwn(evidence, 'completionFence');
if (
hasCommitDigest !== hasReceiptDigest ||
(hasCommitDigest &&
@@ -153,9 +158,25 @@ export function localServiceManagerCutoverRecord(
) {
configurationError('legacy data application evidence is incomplete');
}
if (
hasCompletionFence !== (intent.schemaVersion === 2) ||
(hasCompletionFence &&
(intent.lineage.mode !== 'adopted' ||
intent.lineage.completionFence === undefined ||
evidence.completionFence?.expectedInstanceHeadDigest !==
intent.lineage.completionFence.expectedInstanceHeadDigest ||
evidence.completionFence?.expectedCompletionDigest !==
intent.lineage.completionFence.expectedCompletionDigest))
) {
configurationError('service manager completion evidence is incomplete');
}
const payload = Object.freeze({
schema: SCHEMA,
schemaVersion: hasCommitDigest ? (2 as const) : (1 as const),
schemaVersion: hasCompletionFence
? (3 as const)
: hasCommitDigest
? (2 as const)
: (1 as const),
actionId: intent.actionId,
action: intent.action,
state,
@@ -213,6 +234,12 @@ export function normalizeLocalServiceManagerCutoverRecord(
'legacyDataApplicationCommitDigest',
'legacyDataApplicationReceiptDigest',
]
: record.schemaVersion === 3
? [
'completionFence',
'legacyDataApplicationCommitDigest',
'legacyDataApplicationReceiptDigest',
]
: []),
'manualReason',
'processIdentityDigest',
@@ -226,9 +253,22 @@ export function normalizeLocalServiceManagerCutoverRecord(
candidate === null ||
(typeof candidate === 'string' && DIGEST_PATTERN.test(candidate));
const { recordDigest, ...payload } = record;
const completionFence =
record.schemaVersion === 3
? object(evidence.completionFence, 'service manager completion evidence')
: undefined;
if (completionFence !== undefined) {
exact(
completionFence,
['expectedCompletionDigest', 'expectedInstanceHeadDigest'],
'service manager completion evidence',
);
}
if (
record.schema !== SCHEMA ||
(record.schemaVersion !== 1 && record.schemaVersion !== 2) ||
(record.schemaVersion !== 1 &&
record.schemaVersion !== 2 &&
record.schemaVersion !== 3) ||
typeof record.actionId !== 'string' ||
(record.action !== 'install-enable-start' &&
record.action !== 'start' &&
@@ -244,6 +284,8 @@ export function normalizeLocalServiceManagerCutoverRecord(
!DIGEST_PATTERN.test(record.activationDigest) ||
!Number.isSafeInteger(record.generation) ||
(record.generation as number) < 1 ||
(record.schemaVersion === 3 &&
(record.action !== 'restart' || (record.generation as number) < 2)) ||
typeof record.previousRecordDigest !== 'string' ||
!DIGEST_PATTERN.test(record.previousRecordDigest) ||
typeof record.intentDigest !== 'string' ||
@@ -264,11 +306,16 @@ export function normalizeLocalServiceManagerCutoverRecord(
!DIGEST_PATTERN.test(evidence.commitmentDigest) ||
typeof evidence.targetDataIdentityDigest !== 'string' ||
!DIGEST_PATTERN.test(evidence.targetDataIdentityDigest) ||
(record.schemaVersion === 2 &&
((record.schemaVersion === 2 || record.schemaVersion === 3) &&
(!nullableDigest(evidence.legacyDataApplicationCommitDigest) ||
!nullableDigest(evidence.legacyDataApplicationReceiptDigest) ||
(evidence.legacyDataApplicationCommitDigest === null) !==
(evidence.legacyDataApplicationReceiptDigest === null))) ||
(completionFence !== undefined &&
(typeof completionFence.expectedInstanceHeadDigest !== 'string' ||
!DIGEST_PATTERN.test(completionFence.expectedInstanceHeadDigest) ||
typeof completionFence.expectedCompletionDigest !== 'string' ||
!DIGEST_PATTERN.test(completionFence.expectedCompletionDigest))) ||
!nullableDigest(evidence.startupReceiptDigest) ||
!nullableDigest(evidence.shutdownReceiptDigest) ||
!nullableDigest(evidence.processIdentityDigest) ||
packages/ql3-local-owner-cli/src/deployment/service-manager/serviceManagerIntent.ts
+38 -4
View File
@@ -34,6 +34,7 @@ import {
normalizeLocalServiceManagerOutcome,
type LocalServiceManagerOutcome,
} from './serviceOutcomeContract';
import { readLocalServiceManagerActiveRecord } from './serviceCutoverJournal';
const MAX_PATH_BYTES = 4_096;
const MAX_DESCRIPTOR_BYTES = 64 * 1024;
@@ -43,7 +44,7 @@ const UUID_V4_PATTERN =
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export interface LocalServiceManagerIntentPrepareCommand {
readonly schemaVersion: 1;
readonly schemaVersion: 1 | 2;
readonly operation: 'local.deployment.service-manager.intent.prepare';
readonly options: Readonly<{
deploymentRoot: string;
@@ -454,6 +455,39 @@ function assertIntentLineageHead(
}),
uid,
);
if (intent.schemaVersion === 2) {
if (
intent.action !== 'restart' ||
intent.lineage.completionFence === undefined ||
head.state !== 'reconciliation_completed' ||
head.generation !== intent.lineage.generation - 1 ||
head.headDigest !==
intent.lineage.completionFence.expectedInstanceHeadDigest ||
head.sourceRecordDigest !==
intent.lineage.completionFence.expectedCompletionDigest ||
intent.requestedAtMs < head.updatedAtMs
) {
configurationError(
'service manager completion restart lost the instance head compare-and-swap',
);
}
const previous = readLocalServiceManagerActiveRecord(
intent.deployment.root,
intent.lineage.cutoverId,
intent.lineage.generation - 1,
);
if (
previous.recordDigest !== intent.lineage.previousRecordDigest ||
previous.profile !== intent.profile ||
previous.instanceId !== intent.instanceId ||
previous.activationDigest !== intent.lineage.expectedActivationDigest
) {
configurationError(
'service manager completion restart lost the previous active record',
);
}
return;
}
const expected =
intent.action === 'restart'
? Object.freeze({
@@ -495,7 +529,7 @@ function normalizePrepareCommand(
'request',
);
if (
command.schemaVersion !== 1 ||
(command.schemaVersion !== 1 && command.schemaVersion !== 2) ||
command.operation !== 'local.deployment.service-manager.intent.prepare' ||
typeof request.actionId !== 'string' ||
!UUID_V4_PATTERN.test(request.actionId) ||
@@ -510,7 +544,7 @@ function normalizePrepareCommand(
configurationError('service manager intent command is invalid');
}
return Object.freeze({
schemaVersion: 1 as const,
schemaVersion: command.schemaVersion,
operation: 'local.deployment.service-manager.intent.prepare' as const,
options: Object.freeze({
deploymentRoot: safeAbsolutePath(
@@ -576,7 +610,7 @@ export function prepareLocalServiceManagerIntent(
const application = parseApplicationIdentity(applicationBytes);
const payload: Omit<LocalServiceManagerIntent, 'intentDigest'> =
Object.freeze({
schemaVersion: 1 as const,
schemaVersion: command.schemaVersion,
kind: 'qinglong3-local-service-manager-intent' as const,
actionId: command.request.actionId,
action: command.request.action,