mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
test(ql3): gate failed plugin upgrades before activation
This commit is contained in:
whyour
@@ -35,7 +35,13 @@ const POSTGRES_REPOSITORY_DIGEST = `postgres@${POSTGRES_INDEX_DIGEST}`;
|
||||
const DEFAULT_ADMIN_IMAGE = 'qinglong3-cluster-admin:ql3-plugin-recovery-e2e';
|
||||
const DEFAULT_CONTROL_IMAGE =
|
||||
'qinglong3-cluster-control:ql3-plugin-recovery-e2e';
|
||||
const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v1';
|
||||
const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v2';
|
||||
const INITIAL_SEED_JOB = 'ql3-plugin-package-e2e-seed-initial';
|
||||
const INITIAL_RECOVERY_JOB = 'ql3-plugin-package-recovery-initial';
|
||||
const UPGRADE_SEED_JOB = 'ql3-plugin-package-e2e-seed-upgrade';
|
||||
const UPGRADE_STAGE_JOB = 'ql3-plugin-package-recovery-stage-upgrade';
|
||||
const TRANSITION_JOB = 'ql3-plugin-package-e2e-transition';
|
||||
const UPGRADE_REJECTION_JOB = 'ql3-plugin-package-recovery-reject-upgrade';
|
||||
const SAFE_CLUSTER =
|
||||
/^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/;
|
||||
|
||||
@@ -727,11 +733,11 @@ function migrationJob() {
|
||||
return job;
|
||||
}
|
||||
|
||||
function seedJob() {
|
||||
function seedJob(name, mode) {
|
||||
return {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name: 'ql3-plugin-package-e2e-seed', namespace: NAMESPACE },
|
||||
metadata: { name, namespace: NAMESPACE },
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: 300,
|
||||
@@ -754,7 +760,7 @@ function seedJob() {
|
||||
imagePullPolicy: 'Never',
|
||||
command: ['node', '/opt/ql3-e2e/fixture.cjs'],
|
||||
env: [
|
||||
{ name: 'QL3_E2E_MODE', value: 'seed' },
|
||||
{ name: 'QL3_E2E_MODE', value: mode },
|
||||
{
|
||||
name: 'QL3_E2E_FIXTURE_FILE',
|
||||
value: '/opt/ql3-e2e/fixture.json',
|
||||
@@ -811,7 +817,7 @@ function seedJob() {
|
||||
};
|
||||
}
|
||||
|
||||
function recoveryResources(fixture) {
|
||||
function recoveryResources(fixture, jobName) {
|
||||
const rbac = readYamlDocuments(
|
||||
path.join(
|
||||
ROOT,
|
||||
@@ -830,6 +836,7 @@ function recoveryResources(fixture) {
|
||||
'deploy/kubernetes/ql3-cluster/operations/plugin-package-recovery/base/recover-job.yaml',
|
||||
),
|
||||
);
|
||||
job.metadata.name = jobName;
|
||||
job.metadata.namespace = NAMESPACE;
|
||||
job.spec.activeDeadlineSeconds = 300;
|
||||
const container = job.spec.template.spec.containers[0];
|
||||
@@ -956,7 +963,14 @@ function recoveryResources(fixture) {
|
||||
];
|
||||
}
|
||||
|
||||
function waitForJob(name, timeoutMs = 5 * 60 * 1000) {
|
||||
function waitForJob(
|
||||
name,
|
||||
expectedStatus = 'complete',
|
||||
timeoutMs = 5 * 60 * 1000,
|
||||
) {
|
||||
if (!['complete', 'failed'].includes(expectedStatus)) {
|
||||
fail('expected Job status is invalid');
|
||||
}
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const job = kubectlJson(['-n', NAMESPACE, 'get', 'job', name]);
|
||||
@@ -964,11 +978,17 @@ function waitForJob(name, timeoutMs = 5 * 60 * 1000) {
|
||||
(condition) =>
|
||||
condition.type === 'Complete' && condition.status === 'True',
|
||||
);
|
||||
if (complete) return job;
|
||||
if (complete) {
|
||||
if (expectedStatus !== 'complete') {
|
||||
fail(`${name} completed but failure was required`);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
const failed = job.status?.conditions?.find(
|
||||
(condition) => condition.type === 'Failed' && condition.status === 'True',
|
||||
);
|
||||
if (failed) {
|
||||
if (expectedStatus === 'failed') return job;
|
||||
const logs = kubectl(
|
||||
['-n', NAMESPACE, 'logs', `job/${name}`, '--all-containers=true'],
|
||||
{ capture: true, quiet: true, allowFailure: true },
|
||||
@@ -1039,7 +1059,72 @@ function canI(verb, resource) {
|
||||
return result.stdout === 'yes';
|
||||
}
|
||||
|
||||
function databaseEvidence(lockDigest) {
|
||||
function upgradeStageEvidence(fixture, transitionReceiptCount) {
|
||||
const generationDigest = fixture.upgrade.generation.generationDigest;
|
||||
assert.match(generationDigest, /^[0-9a-f]{64}$/);
|
||||
const sql = `
|
||||
SELECT json_build_object(
|
||||
'state', (
|
||||
SELECT state FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'previousActiveLockDigest', (
|
||||
SELECT previous_active_lock_digest FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'activeLockDigest', (
|
||||
SELECT active_lock_digest FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'mutationCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_install_mutations
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'transitionReceiptCount', (
|
||||
SELECT count(*)
|
||||
FROM ql3.plugin_package_secret_binding_transition_receipts
|
||||
WHERE generation_digest = '${generationDigest}'
|
||||
),
|
||||
'candidateRevisionCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_materialized_revisions
|
||||
WHERE generation_digest = '${generationDigest}'
|
||||
)
|
||||
)::text;
|
||||
`.trim();
|
||||
const output = kubectl(
|
||||
[
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'exec',
|
||||
POSTGRES_NAME,
|
||||
'--',
|
||||
'psql',
|
||||
'--username',
|
||||
'postgres',
|
||||
'--dbname',
|
||||
'qinglong',
|
||||
'--tuples-only',
|
||||
'--no-align',
|
||||
'--command',
|
||||
sql,
|
||||
],
|
||||
{ capture: true, quiet: true },
|
||||
).stdout;
|
||||
const value = JSON.parse(output);
|
||||
assert.equal(value.state, 'staged');
|
||||
assert.equal(value.previousActiveLockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(value.activeLockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(value.mutationCount, 2);
|
||||
assert.equal(value.transitionReceiptCount, transitionReceiptCount);
|
||||
assert.equal(value.candidateRevisionCount, 0);
|
||||
return value;
|
||||
}
|
||||
|
||||
function databaseEvidence(fixture) {
|
||||
const initialGenerationDigest = fixture.initial.generation.generationDigest;
|
||||
const upgradeGenerationDigest = fixture.upgrade.generation.generationDigest;
|
||||
assert.match(initialGenerationDigest, /^[0-9a-f]{64}$/);
|
||||
assert.match(upgradeGenerationDigest, /^[0-9a-f]{64}$/);
|
||||
const sql = `
|
||||
SELECT json_build_object(
|
||||
'migrationCount', (SELECT count(*) FROM ql3.schema_migrations),
|
||||
@@ -1048,25 +1133,63 @@ SELECT json_build_object(
|
||||
FROM ql3.schema_capabilities
|
||||
WHERE contract_name = 'control-core'
|
||||
),
|
||||
'state', (
|
||||
'initialState', (
|
||||
SELECT state
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = 'install-plugin-recovery-e2e'
|
||||
WHERE installation_id = '${fixture.initial.installationId}'
|
||||
),
|
||||
'lockDigest', (
|
||||
SELECT lock_digest
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = 'install-plugin-recovery-e2e'
|
||||
),
|
||||
'activeLockDigest', (
|
||||
'initialActiveLockDigest', (
|
||||
SELECT active_lock_digest
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = 'install-plugin-recovery-e2e'
|
||||
WHERE installation_id = '${fixture.initial.installationId}'
|
||||
),
|
||||
'mutationCount', (
|
||||
'upgradeState', (
|
||||
SELECT state
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'upgradePreviousActiveLockDigest', (
|
||||
SELECT previous_active_lock_digest
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'upgradeActiveLockDigest', (
|
||||
SELECT active_lock_digest
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'upgradeFailureReason', (
|
||||
SELECT record_json #>> '{failure,reason}'
|
||||
FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'initialMutationCount', (
|
||||
SELECT count(*)
|
||||
FROM ql3.plugin_package_install_mutations
|
||||
WHERE installation_id = 'install-plugin-recovery-e2e'
|
||||
WHERE installation_id = '${fixture.initial.installationId}'
|
||||
),
|
||||
'upgradeMutationCount', (
|
||||
SELECT count(*)
|
||||
FROM ql3.plugin_package_install_mutations
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'headInstallationId', (
|
||||
SELECT installation_id
|
||||
FROM ql3.plugin_package_install_heads
|
||||
WHERE project_id = 'default' AND package_name = 'e2e-monitor'
|
||||
),
|
||||
'transitionReceiptCount', (
|
||||
SELECT count(*)
|
||||
FROM ql3.plugin_package_secret_binding_transition_receipts
|
||||
WHERE generation_digest = '${upgradeGenerationDigest}'
|
||||
),
|
||||
'initialRevisionCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_materialized_revisions
|
||||
WHERE generation_digest = '${initialGenerationDigest}'
|
||||
),
|
||||
'upgradeRevisionCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_materialized_revisions
|
||||
WHERE generation_digest = '${upgradeGenerationDigest}'
|
||||
),
|
||||
'recoverableCount', (
|
||||
SELECT count(*)
|
||||
@@ -1095,17 +1218,28 @@ SELECT json_build_object(
|
||||
{ capture: true, quiet: true },
|
||||
).stdout;
|
||||
const value = JSON.parse(output);
|
||||
assert.equal(value.migrationCount, 22);
|
||||
assert.equal(value.capabilityVersion, 21);
|
||||
assert.equal(value.state, 'active');
|
||||
assert.equal(value.lockDigest, lockDigest);
|
||||
assert.equal(value.activeLockDigest, lockDigest);
|
||||
assert.equal(value.mutationCount, 4);
|
||||
assert.equal(value.migrationCount, 65);
|
||||
assert.equal(value.capabilityVersion, 64);
|
||||
assert.equal(value.initialState, 'active');
|
||||
assert.equal(value.initialActiveLockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(value.upgradeState, 'failed');
|
||||
assert.equal(
|
||||
value.upgradePreviousActiveLockDigest,
|
||||
fixture.initial.lock.lockDigest,
|
||||
);
|
||||
assert.equal(value.upgradeActiveLockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(value.upgradeFailureReason, 'activation_fact_conflict');
|
||||
assert.equal(value.initialMutationCount, 4);
|
||||
assert.equal(value.upgradeMutationCount, 3);
|
||||
assert.equal(value.headInstallationId, fixture.upgrade.installationId);
|
||||
assert.equal(value.transitionReceiptCount, 1);
|
||||
assert.equal(value.initialRevisionCount, 1);
|
||||
assert.equal(value.upgradeRevisionCount, 0);
|
||||
assert.equal(value.recoverableCount, 0);
|
||||
return value;
|
||||
}
|
||||
|
||||
function activePointerEvidence(lockDigest) {
|
||||
function activePointerEvidence(fixture) {
|
||||
const values = kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
@@ -1117,13 +1251,14 @@ function activePointerEvidence(lockDigest) {
|
||||
assert.equal(values.items.length, 1);
|
||||
const configMap = values.items[0];
|
||||
const pointer = JSON.parse(configMap.data['active.json']);
|
||||
assert.equal(pointer.intent.installationId, 'install-plugin-recovery-e2e');
|
||||
assert.equal(pointer.intent.lockDigest, lockDigest);
|
||||
assert.equal(pointer.intent.installationId, fixture.initial.installationId);
|
||||
assert.equal(pointer.intent.lockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(pointer.receipt.generation, 1);
|
||||
return Object.freeze({
|
||||
name: configMap.metadata.name,
|
||||
uid: configMap.metadata.uid,
|
||||
resourceVersion: configMap.metadata.resourceVersion,
|
||||
activeJson: configMap.data['active.json'],
|
||||
intentDigest: pointer.intent.intentDigest,
|
||||
activationRef: pointer.receipt.activationRef,
|
||||
});
|
||||
@@ -1141,10 +1276,17 @@ function registryEvidence(fixture) {
|
||||
.map((line) => JSON.parse(line))
|
||||
.filter((value) => value.schema === REGISTRY_EVENT_SCHEMA);
|
||||
const packageRequests = events.filter((event) => event.path !== '/v2/');
|
||||
assert.equal(packageRequests.length, fixture.routes.length);
|
||||
const expectedPaths = [
|
||||
...fixture.initial.routes.map((routeValue) => routeValue.path),
|
||||
...fixture.upgrade.routes.flatMap((routeValue) => [
|
||||
routeValue.path,
|
||||
routeValue.path,
|
||||
]),
|
||||
].sort();
|
||||
assert.equal(packageRequests.length, expectedPaths.length);
|
||||
assert.deepEqual(
|
||||
packageRequests.map((event) => event.path).sort(),
|
||||
fixture.routes.map((routeValue) => routeValue.path).sort(),
|
||||
expectedPaths,
|
||||
);
|
||||
assert.ok(packageRequests.every((event) => event.status === 200));
|
||||
assert.ok(packageRequests.every((event) => event.authenticated === true));
|
||||
@@ -1154,6 +1296,8 @@ function registryEvidence(fixture) {
|
||||
authenticatedRequestCount: packageRequests.length,
|
||||
requestCount: packageRequests.length,
|
||||
uniquePaths: new Set(packageRequests.map((event) => event.path)).size,
|
||||
initialRequestCount: fixture.initial.routes.length,
|
||||
upgradeRequestCount: fixture.upgrade.routes.length * 2,
|
||||
redirects: 0,
|
||||
});
|
||||
}
|
||||
@@ -1572,36 +1716,109 @@ async function main() {
|
||||
(value) => value.event === 'migration_completed',
|
||||
);
|
||||
|
||||
apply(seedJob(), 'persist one durable queued Plugin Package installation');
|
||||
waitForJob('ql3-plugin-package-e2e-seed');
|
||||
const seed = lastJsonLine(
|
||||
jobLog('ql3-plugin-package-e2e-seed'),
|
||||
apply(
|
||||
seedJob(INITIAL_SEED_JOB, 'seed-initial'),
|
||||
'persist initial durable queued Plugin Package installation',
|
||||
);
|
||||
waitForJob(INITIAL_SEED_JOB);
|
||||
const initialSeed = lastJsonLine(
|
||||
jobLog(INITIAL_SEED_JOB),
|
||||
(value) => value.event === 'seed_completed',
|
||||
);
|
||||
assert.equal(seed.status, 'created');
|
||||
assert.equal(seed.state, 'queued');
|
||||
assert.equal(seed.lockDigest, fixture.lock.lockDigest);
|
||||
assert.equal(initialSeed.phase, 'initial');
|
||||
assert.equal(initialSeed.status, 'created');
|
||||
assert.equal(initialSeed.state, 'queued');
|
||||
assert.equal(initialSeed.lockDigest, fixture.initial.lock.lockDigest);
|
||||
|
||||
for (const resource of recoveryResources(fixture)) {
|
||||
apply(resource, `apply recovery ${resource.kind}`);
|
||||
for (const resource of recoveryResources(fixture, INITIAL_RECOVERY_JOB)) {
|
||||
apply(resource, `apply initial recovery ${resource.kind}`);
|
||||
}
|
||||
const recovered = waitForJob('ql3-plugin-package-recovery');
|
||||
const recoveryLog = jobLog('ql3-plugin-package-recovery');
|
||||
const completed = lastJsonLine(
|
||||
recoveryLog,
|
||||
const initialRecovered = waitForJob(INITIAL_RECOVERY_JOB);
|
||||
const initialCompleted = lastJsonLine(
|
||||
jobLog(INITIAL_RECOVERY_JOB),
|
||||
(value) => value.event === 'recovery_completed',
|
||||
);
|
||||
assert.equal(completed.recovery.safeToAdmit, true);
|
||||
assert.equal(completed.recovery.remaining, false);
|
||||
assert.equal(completed.recovery.manualRequired, 0);
|
||||
assert.equal(initialCompleted.recovery.safeToAdmit, true);
|
||||
assert.equal(initialCompleted.recovery.remaining, false);
|
||||
assert.equal(initialCompleted.recovery.manualRequired, 0);
|
||||
const pointerBeforeUpgrade = activePointerEvidence(fixture);
|
||||
|
||||
const database = databaseEvidence(fixture.lock.lockDigest);
|
||||
const pointer = activePointerEvidence(fixture.lock.lockDigest);
|
||||
apply(
|
||||
seedJob(UPGRADE_SEED_JOB, 'seed-upgrade'),
|
||||
'persist invalid durable queued Plugin Package upgrade',
|
||||
);
|
||||
waitForJob(UPGRADE_SEED_JOB);
|
||||
const upgradeSeed = lastJsonLine(
|
||||
jobLog(UPGRADE_SEED_JOB),
|
||||
(value) => value.event === 'seed_completed',
|
||||
);
|
||||
assert.equal(upgradeSeed.phase, 'upgrade');
|
||||
assert.equal(upgradeSeed.status, 'created');
|
||||
assert.equal(upgradeSeed.state, 'queued');
|
||||
assert.equal(upgradeSeed.lockDigest, fixture.upgrade.lock.lockDigest);
|
||||
|
||||
for (const resource of recoveryResources(fixture, UPGRADE_STAGE_JOB)) {
|
||||
apply(resource, `apply upgrade staging recovery ${resource.kind}`);
|
||||
}
|
||||
const stagedUpgrade = waitForJob(UPGRADE_STAGE_JOB, 'failed');
|
||||
const stagedFailureCondition = stagedUpgrade.status.conditions.find(
|
||||
(condition) => condition.type === 'Failed' && condition.status === 'True',
|
||||
);
|
||||
assert.ok(stagedFailureCondition?.lastTransitionTime);
|
||||
const stageFailure = lastJsonLine(
|
||||
jobLog(UPGRADE_STAGE_JOB),
|
||||
(value) => value.event === 'recovery_failed',
|
||||
);
|
||||
assert.equal(
|
||||
stageFailure.name,
|
||||
'ClusterPluginPackageRecoveryRequiredError',
|
||||
);
|
||||
const stagedDatabase = upgradeStageEvidence(fixture, 0);
|
||||
assert.deepEqual(activePointerEvidence(fixture), pointerBeforeUpgrade);
|
||||
|
||||
apply(
|
||||
seedJob(TRANSITION_JOB, 'commit-transition'),
|
||||
'commit durable no-secret binding transition receipt',
|
||||
);
|
||||
const committedTransition = waitForJob(TRANSITION_JOB);
|
||||
const transition = lastJsonLine(
|
||||
jobLog(TRANSITION_JOB),
|
||||
(value) => value.event === 'transition_completed',
|
||||
);
|
||||
assert.equal(transition.status, 'created');
|
||||
assert.equal(
|
||||
transition.generationDigest,
|
||||
fixture.upgrade.generation.generationDigest,
|
||||
);
|
||||
assert.equal(transition.bindingDigest, null);
|
||||
upgradeStageEvidence(fixture, 1);
|
||||
|
||||
for (const resource of recoveryResources(fixture, UPGRADE_REJECTION_JOB)) {
|
||||
apply(resource, `apply upgrade rejection recovery ${resource.kind}`);
|
||||
}
|
||||
const rejectedUpgrade = waitForJob(UPGRADE_REJECTION_JOB);
|
||||
const rejectionCompleted = lastJsonLine(
|
||||
jobLog(UPGRADE_REJECTION_JOB),
|
||||
(value) => value.event === 'recovery_completed',
|
||||
);
|
||||
assert.equal(rejectionCompleted.recovery.safeToAdmit, true);
|
||||
assert.equal(rejectionCompleted.recovery.remaining, false);
|
||||
assert.equal(rejectionCompleted.recovery.manualRequired, 0);
|
||||
|
||||
const database = databaseEvidence(fixture);
|
||||
const pointerAfterRejection = activePointerEvidence(fixture);
|
||||
assert.deepEqual(pointerAfterRejection, pointerBeforeUpgrade);
|
||||
const oci = registryEvidence(fixture);
|
||||
const rbac = recoveryRbacEvidence();
|
||||
assertRuntimeCannotReadPluginAuthority();
|
||||
const runtime = applyRuntimeAfterRecovery(recovered, migrated, secrets);
|
||||
const recoveryImageId = jobImageId('ql3-plugin-package-recovery');
|
||||
const runtime = applyRuntimeAfterRecovery(
|
||||
rejectedUpgrade,
|
||||
migrated,
|
||||
secrets,
|
||||
);
|
||||
const initialRecoveryImageId = jobImageId(INITIAL_RECOVERY_JOB);
|
||||
const stageRecoveryImageId = jobImageId(UPGRADE_STAGE_JOB);
|
||||
const rejectionRecoveryImageId = jobImageId(UPGRADE_REJECTION_JOB);
|
||||
const migrationImageId = jobImageId('ql3-cluster-migration');
|
||||
const postgresPod = kubectlJson([
|
||||
'-n',
|
||||
@@ -1621,21 +1838,40 @@ async function main() {
|
||||
controlBuildId: imageId(CONTROL_IMAGE),
|
||||
postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST,
|
||||
migrationImageId,
|
||||
recoveryImageId,
|
||||
initialRecoveryImageId,
|
||||
stageRecoveryImageId,
|
||||
rejectionRecoveryImageId,
|
||||
postgresImageId: postgresPod.status.containerStatuses[0].imageID,
|
||||
}),
|
||||
ordering: Object.freeze({
|
||||
migrationJobUid: migrated.metadata.uid,
|
||||
migrationCompletedAt: migrated.status.completionTime,
|
||||
recoveryJobUid: recovered.metadata.uid,
|
||||
recoveryCompletedAt: recovered.status.completionTime,
|
||||
initialRecoveryJobUid: initialRecovered.metadata.uid,
|
||||
initialRecoveryCompletedAt: initialRecovered.status.completionTime,
|
||||
upgradeStageJobUid: stagedUpgrade.metadata.uid,
|
||||
upgradeStageFailedAt: stagedFailureCondition.lastTransitionTime,
|
||||
transitionJobUid: committedTransition.metadata.uid,
|
||||
transitionCompletedAt: committedTransition.status.completionTime,
|
||||
rejectionRecoveryJobUid: rejectedUpgrade.metadata.uid,
|
||||
rejectionRecoveryCompletedAt: rejectedUpgrade.status.completionTime,
|
||||
runtimeCreatedAt: runtime.creationTimestamp,
|
||||
runtimeBoundRecoveryJobUid: runtime.recoveryJobUid,
|
||||
}),
|
||||
failedUpgrade: Object.freeze({
|
||||
stageFailure: Object.freeze({
|
||||
jobUid: stagedUpgrade.metadata.uid,
|
||||
reason: stageFailure.name,
|
||||
durableState: stagedDatabase.state,
|
||||
}),
|
||||
transitionReceiptDigest: transition.receiptDigest,
|
||||
rejectionReason: database.upgradeFailureReason,
|
||||
candidateRevisionCount: database.upgradeRevisionCount,
|
||||
activePointerUnchanged: true,
|
||||
}),
|
||||
database,
|
||||
oci,
|
||||
kubernetes: Object.freeze({
|
||||
activePointer: pointer,
|
||||
activePointer: pointerAfterRejection,
|
||||
rbac,
|
||||
}),
|
||||
runtime,
|
||||
|
||||
Reference in New Issue
Block a user