mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
fix(ci): align recovery gate with durable admission
This commit is contained in:
whyour
@@ -340,6 +340,172 @@ function signedPackageRoutes(material, lock, privateKey) {
|
||||
]);
|
||||
}
|
||||
|
||||
function createInstallAuthority(actionInput, phase, proposedAtMs, eventBase) {
|
||||
const {
|
||||
consumeApprovalRequest,
|
||||
createApprovalRequest,
|
||||
decideApprovalRequest,
|
||||
} = ql3Require('@qinglong/runtime-core/approved-action');
|
||||
const {
|
||||
createPluginPackageInstallProposal,
|
||||
resolvePluginPackageInstallProposal,
|
||||
} = ql3Require('@qinglong/runtime-core/plugin-package-proposal');
|
||||
const requester = { type: 'user', id: 'plugin-recovery-e2e-owner' };
|
||||
const reviewer = { type: 'user', id: 'plugin-recovery-e2e-reviewer' };
|
||||
const consumer = { type: 'system', id: 'plugin-recovery-e2e-dispatcher' };
|
||||
const fence = { projectVersion: 1, bindingVersion: 1 };
|
||||
const requestedAtMs = proposedAtMs + 1;
|
||||
const decidedAtMs = proposedAtMs + 2;
|
||||
const consumedAtMs = proposedAtMs + 3;
|
||||
const lockCreatedAtMs = proposedAtMs + 4;
|
||||
const expiresAtMs = proposedAtMs + 60 * 60 * 1000;
|
||||
const actionRef = `plugin-package:e2e-monitor:${phase}`;
|
||||
const proposal = createPluginPackageInstallProposal({
|
||||
actionRef,
|
||||
actionInput,
|
||||
proposedBy: requester,
|
||||
proposalFence: fence,
|
||||
createdAtMs: proposedAtMs,
|
||||
});
|
||||
const action = {
|
||||
actionRef: proposal.actionRef,
|
||||
actionType: proposal.actionType,
|
||||
permission: proposal.permission,
|
||||
actionDigest: proposal.actionDigest,
|
||||
previewDigest: proposal.previewDigest,
|
||||
};
|
||||
const request = createApprovalRequest({
|
||||
id: `approval-plugin-recovery-e2e-${phase}`,
|
||||
projectId: proposal.projectId,
|
||||
action,
|
||||
risk: 'high',
|
||||
decisionMode: 'separation_of_duty',
|
||||
requestedBy: requester,
|
||||
requestedAtMs,
|
||||
expiresAtMs,
|
||||
requestFence: fence,
|
||||
});
|
||||
const decision = {
|
||||
expectedVersion: 1,
|
||||
decisionId: `decision-plugin-recovery-e2e-${phase}`,
|
||||
decision: 'approved',
|
||||
reasonCode: 'reviewed',
|
||||
principal: {
|
||||
subject: reviewer,
|
||||
authenticationId: `auth-plugin-recovery-e2e-reviewer-${phase}`,
|
||||
authenticatedAtMs: proposedAtMs,
|
||||
expiresAtMs,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
decidedAtMs,
|
||||
authorizationFence: fence,
|
||||
};
|
||||
const decided = decideApprovalRequest(request, decision);
|
||||
const consumption = {
|
||||
expectedVersion: 2,
|
||||
consumptionId: `consume-plugin-recovery-e2e-${phase}`,
|
||||
dispatchId: `dispatch-plugin-recovery-e2e-${phase}`,
|
||||
action,
|
||||
requestedBy: requester,
|
||||
consumedBy: consumer,
|
||||
consumedAtMs,
|
||||
authorizationFence: fence,
|
||||
};
|
||||
const consumed = consumeApprovalRequest(decided, consumption);
|
||||
const audit = (
|
||||
offset,
|
||||
requestId,
|
||||
operationId,
|
||||
subject,
|
||||
authenticationId,
|
||||
outcome,
|
||||
reasons,
|
||||
occurredAtMs,
|
||||
) => ({
|
||||
eventId: `10000000-0000-4000-8000-${String(
|
||||
eventBase + offset,
|
||||
).padStart(12, '0')}`,
|
||||
requestId,
|
||||
operationId,
|
||||
projectId: proposal.projectId,
|
||||
subject,
|
||||
authenticationId,
|
||||
outcome,
|
||||
reasons,
|
||||
fence,
|
||||
occurredAtMs,
|
||||
});
|
||||
return Object.freeze({
|
||||
requester,
|
||||
reviewer,
|
||||
consumer,
|
||||
fence,
|
||||
proposalCommand: {
|
||||
proposal,
|
||||
audit: audit(
|
||||
0,
|
||||
proposal.actionRef,
|
||||
'plugin_package.propose',
|
||||
requester,
|
||||
`auth-plugin-recovery-e2e-owner-${phase}`,
|
||||
'allowed',
|
||||
['package_proposal'],
|
||||
proposedAtMs,
|
||||
),
|
||||
},
|
||||
requestCommand: {
|
||||
request,
|
||||
audit: audit(
|
||||
1,
|
||||
`request-plugin-recovery-e2e-${phase}`,
|
||||
'approval.request',
|
||||
requester,
|
||||
`auth-plugin-recovery-e2e-owner-${phase}`,
|
||||
'approval_required',
|
||||
['package_review'],
|
||||
requestedAtMs,
|
||||
),
|
||||
},
|
||||
decisionCommand: {
|
||||
requestId: request.id,
|
||||
...decision,
|
||||
audit: audit(
|
||||
2,
|
||||
`decision-plugin-recovery-e2e-${phase}`,
|
||||
'approval.decide',
|
||||
reviewer,
|
||||
decision.principal.authenticationId,
|
||||
'allowed',
|
||||
['role_grant'],
|
||||
decidedAtMs,
|
||||
),
|
||||
},
|
||||
consumptionCommand: {
|
||||
requestId: request.id,
|
||||
...consumption,
|
||||
audit: audit(
|
||||
3,
|
||||
`consume-plugin-recovery-e2e-${phase}`,
|
||||
'approval.consume',
|
||||
consumer,
|
||||
`auth-plugin-recovery-e2e-dispatcher-${phase}`,
|
||||
'allowed',
|
||||
['role_grant'],
|
||||
consumedAtMs,
|
||||
),
|
||||
},
|
||||
dispatch: consumed.dispatch,
|
||||
lock: resolvePluginPackageInstallProposal(
|
||||
proposal,
|
||||
consumed.dispatch,
|
||||
lockCreatedAtMs,
|
||||
),
|
||||
admissionAuditEventId: `10000000-0000-4000-8000-${String(
|
||||
eventBase + 4,
|
||||
).padStart(12, '0')}`,
|
||||
});
|
||||
}
|
||||
|
||||
function createFixture({ registry, architecture, createdAtMs = Date.now() }) {
|
||||
if (
|
||||
typeof registry !== 'string' ||
|
||||
@@ -356,11 +522,6 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) {
|
||||
const { planPluginPackageInstall } = ql3Require(
|
||||
'@qinglong/runtime-core/plugin-package',
|
||||
);
|
||||
const {
|
||||
createPluginPackageLock,
|
||||
pluginPackageInstallActionDigest,
|
||||
pluginPackageInstallPlanDigest,
|
||||
} = ql3Require('@qinglong/runtime-core/plugin-package-install');
|
||||
const { createPluginPackageResourceGenerationFromReferences } = ql3Require(
|
||||
'@qinglong/runtime-core/plugin-package-resource-generation',
|
||||
);
|
||||
@@ -386,22 +547,14 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) {
|
||||
deploymentProfile: 'cluster-control',
|
||||
targetGeneration: 1,
|
||||
};
|
||||
const initialLock = createPluginPackageLock({
|
||||
...initialAction,
|
||||
approval: {
|
||||
requestId: 'approval-plugin-recovery-e2e-initial',
|
||||
requestVersion: 1,
|
||||
dispatchId: 'dispatch-plugin-recovery-e2e-initial',
|
||||
actionDigest: pluginPackageInstallActionDigest(initialAction),
|
||||
previewDigest: pluginPackageInstallPlanDigest(initialPlan),
|
||||
approvedBy: { type: 'user', id: 'e2e-owner' },
|
||||
approvedAtMs: createdAtMs - 1,
|
||||
expiresAtMs: createdAtMs + 60 * 60 * 1000,
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
},
|
||||
const initialAuthority = createInstallAuthority(
|
||||
initialAction,
|
||||
'initial',
|
||||
createdAtMs,
|
||||
});
|
||||
const upgradeCreatedAtMs = createdAtMs + 10;
|
||||
1,
|
||||
);
|
||||
const initialLock = initialAuthority.lock;
|
||||
const upgradeCreatedAtMs = initialLock.createdAtMs + 10;
|
||||
const upgradeManifest = pluginManifest(architecture, '2.0.0', true);
|
||||
const upgradeMaterial = packageMaterial(
|
||||
registry,
|
||||
@@ -426,21 +579,13 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) {
|
||||
targetGeneration: 2,
|
||||
previousLockDigest: initialLock.lockDigest,
|
||||
};
|
||||
const upgradeLock = createPluginPackageLock({
|
||||
...upgradeAction,
|
||||
approval: {
|
||||
requestId: 'approval-plugin-recovery-e2e-upgrade',
|
||||
requestVersion: 1,
|
||||
dispatchId: 'dispatch-plugin-recovery-e2e-upgrade',
|
||||
actionDigest: pluginPackageInstallActionDigest(upgradeAction),
|
||||
previewDigest: pluginPackageInstallPlanDigest(upgradePlan),
|
||||
approvedBy: { type: 'user', id: 'e2e-owner' },
|
||||
approvedAtMs: upgradeCreatedAtMs - 1,
|
||||
expiresAtMs: upgradeCreatedAtMs + 60 * 60 * 1000,
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
},
|
||||
createdAtMs: upgradeCreatedAtMs,
|
||||
});
|
||||
const upgradeAuthority = createInstallAuthority(
|
||||
upgradeAction,
|
||||
'upgrade',
|
||||
upgradeCreatedAtMs,
|
||||
11,
|
||||
);
|
||||
const upgradeLock = upgradeAuthority.lock;
|
||||
const { publicKey, privateKey } = generateKeyPairSync('ed25519');
|
||||
const publicKeyPem = publicKey.export({ format: 'pem', type: 'spki' });
|
||||
const trust = {
|
||||
@@ -467,6 +612,7 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) {
|
||||
privateKey,
|
||||
);
|
||||
const initial = Object.freeze({
|
||||
authority: initialAuthority,
|
||||
installationId: 'install-plugin-recovery-e2e-initial',
|
||||
manifest: initialManifest,
|
||||
lock: initialLock,
|
||||
@@ -483,6 +629,7 @@ function createFixture({ registry, architecture, createdAtMs = Date.now() }) {
|
||||
routes: initialRoutes,
|
||||
});
|
||||
const upgrade = Object.freeze({
|
||||
authority: upgradeAuthority,
|
||||
installationId: 'install-plugin-recovery-e2e-upgrade',
|
||||
manifest: upgradeManifest,
|
||||
lock: upgradeLock,
|
||||
@@ -518,8 +665,12 @@ function readFixture(filePath) {
|
||||
value.schema !== FIXTURE_SCHEMA ||
|
||||
!Array.isArray(value.routes) ||
|
||||
!value.initial?.lock ||
|
||||
!value.initial?.authority?.proposalCommand?.proposal ||
|
||||
!value.initial?.authority?.dispatch ||
|
||||
!value.initial?.generation ||
|
||||
!value.upgrade?.lock ||
|
||||
!value.upgrade?.authority?.proposalCommand?.proposal ||
|
||||
!value.upgrade?.authority?.dispatch ||
|
||||
!value.upgrade?.generation ||
|
||||
!value.trust
|
||||
) {
|
||||
@@ -624,32 +775,31 @@ async function runSeed() {
|
||||
assertPostgresPackageExecutorSchemaReady,
|
||||
createPostgresDatabaseOpener,
|
||||
loadPostgresConnectionEnvironment,
|
||||
PostgresPluginPackageSecretBindingTransitionRepository,
|
||||
PostgresApprovedActionExecutionRepository,
|
||||
} = ql3Require('@qinglong/cluster-postgres/package-executor');
|
||||
const {
|
||||
assertPostgresPackageManagerSchemaReady,
|
||||
PostgresApprovalRequestRepository,
|
||||
PostgresPluginPackagePublisherTrustAuthorityRepository,
|
||||
} = ql3Require('@qinglong/cluster-postgres/package-manager');
|
||||
const { PostgresProjectPolicyRepository } = ql3Require(
|
||||
'@qinglong/cluster-postgres/project-policy',
|
||||
);
|
||||
const { PostgresPluginPackageInstallRepository } = ql3Require(
|
||||
'@qinglong/cluster-postgres/plugin-package-install',
|
||||
);
|
||||
const {
|
||||
createPluginPackageInstall,
|
||||
normalizePluginPackageLock,
|
||||
pluginPackageInstallCreate,
|
||||
} = ql3Require('@qinglong/runtime-core/plugin-package-install');
|
||||
const { createPluginPackageSecretBindingTarget } = ql3Require(
|
||||
'@qinglong/runtime-core/plugin-package-secret-binding',
|
||||
const { PostgresPluginPackageInstallProposalRepository } = ql3Require(
|
||||
'@qinglong/cluster-postgres/plugin-package-proposal',
|
||||
);
|
||||
const { createPluginPackageSecretBindingTransitionPlan } = ql3Require(
|
||||
'@qinglong/runtime-core/plugin-package-secret-binding-transition-plan',
|
||||
const { normalizePluginPackageLock } = ql3Require(
|
||||
'@qinglong/runtime-core/plugin-package-install',
|
||||
);
|
||||
const { createPluginPackagePublisherTrustSnapshot } = ql3Require(
|
||||
'@qinglong/runtime-core/plugin-package-publisher-trust',
|
||||
);
|
||||
const fixture = readFixture(process.env.QL3_E2E_FIXTURE_FILE);
|
||||
const mode = process.env.QL3_E2E_MODE;
|
||||
if (!['seed-initial', 'seed-upgrade', 'commit-transition'].includes(mode)) {
|
||||
if (!['seed-initial', 'seed-upgrade'].includes(mode)) {
|
||||
throw new Error('Plugin Package E2E seed mode is invalid');
|
||||
}
|
||||
const connection = loadPostgresConnectionEnvironment(process.env, {
|
||||
@@ -671,72 +821,82 @@ async function runSeed() {
|
||||
})();
|
||||
try {
|
||||
await assertPostgresPackageExecutorSchemaReady(database.pool);
|
||||
if (mode === 'commit-transition') {
|
||||
const plannedAtMs = Date.now();
|
||||
const transitionPlan = createPluginPackageSecretBindingTransitionPlan({
|
||||
previousTarget: createPluginPackageSecretBindingTarget(
|
||||
fixture.initial.generation,
|
||||
fixture.initial.manifest,
|
||||
),
|
||||
previousBinding: null,
|
||||
previousAttemptGeneration: 1,
|
||||
nextGeneration: fixture.upgrade.generation,
|
||||
nextManifest: fixture.upgrade.manifest,
|
||||
assignments: [],
|
||||
plannedAtMs,
|
||||
});
|
||||
const result =
|
||||
await new PostgresPluginPackageSecretBindingTransitionRepository(
|
||||
database.pool,
|
||||
).apply({
|
||||
transitionPlan,
|
||||
evidenceDigest: transitionPlan.transitionDigest,
|
||||
committedAtMs: plannedAtMs + 1,
|
||||
});
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schema: 'qinglong/plugin-package-recovery-e2e-transition-result@v1',
|
||||
event: 'transition_completed',
|
||||
status: result.status,
|
||||
generationDigest: transitionPlan.nextTarget.generationDigest,
|
||||
transitionDigest: transitionPlan.transitionDigest,
|
||||
bindingDigest: result.receipt.bindingDigest,
|
||||
receiptDigest: result.receipt.receiptDigest,
|
||||
})}\n`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const selected =
|
||||
mode === 'seed-initial' ? fixture.initial : fixture.upgrade;
|
||||
let publisherTrustStatus = null;
|
||||
if (mode === 'seed-initial') {
|
||||
const packageManagerConnection = loadPostgresConnectionEnvironment(
|
||||
process.env,
|
||||
{
|
||||
host: 'QL3_E2E_POSTGRES_HOST',
|
||||
port: 'QL3_E2E_POSTGRES_PORT',
|
||||
database: 'QL3_E2E_POSTGRES_DATABASE',
|
||||
user: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER',
|
||||
password: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_PASSWORD',
|
||||
},
|
||||
const runtimeConnection = loadPostgresConnectionEnvironment(process.env, {
|
||||
host: 'QL3_E2E_POSTGRES_HOST',
|
||||
port: 'QL3_E2E_POSTGRES_PORT',
|
||||
database: 'QL3_E2E_POSTGRES_DATABASE',
|
||||
user: 'QL3_E2E_POSTGRES_RUNTIME_USER',
|
||||
password: 'QL3_E2E_POSTGRES_RUNTIME_PASSWORD',
|
||||
});
|
||||
const runtimeDatabase = await createPostgresDatabaseOpener({
|
||||
role: 'runtime',
|
||||
connection: { ...runtimeConnection, tls: { mode: 'disable' } },
|
||||
pool: {
|
||||
applicationName: 'qinglong3-plugin-package-e2e-policy-seed',
|
||||
maxConnections: 1,
|
||||
connectionTimeoutMs: 15_000,
|
||||
},
|
||||
onPoolError() {},
|
||||
})();
|
||||
try {
|
||||
const policies = new PostgresProjectPolicyRepository(
|
||||
runtimeDatabase.pool,
|
||||
);
|
||||
const packageManagerDatabase = await createPostgresDatabaseOpener({
|
||||
role: 'package-manager',
|
||||
connection: {
|
||||
...packageManagerConnection,
|
||||
tls: { mode: 'disable' },
|
||||
},
|
||||
pool: {
|
||||
applicationName: 'qinglong3-plugin-package-e2e-trust-seed',
|
||||
maxConnections: 1,
|
||||
connectionTimeoutMs: 15_000,
|
||||
},
|
||||
onPoolError() {},
|
||||
})();
|
||||
try {
|
||||
await assertPostgresPackageManagerSchemaReady(
|
||||
packageManagerDatabase.pool,
|
||||
);
|
||||
for (const [subject, role] of [
|
||||
[selected.authority.requester, 'owner'],
|
||||
[selected.authority.reviewer, 'admin'],
|
||||
]) {
|
||||
const policy = await policies.append({
|
||||
expectedCurrentVersion: 0,
|
||||
binding: {
|
||||
projectId: selected.lock.projectId,
|
||||
subject,
|
||||
version: 1,
|
||||
state: 'active',
|
||||
role,
|
||||
mutationId: `plugin-recovery-e2e-grant-${role}`,
|
||||
changedBy: { type: 'system', id: 'plugin-recovery-e2e-bootstrap' },
|
||||
createdAtMs: 1,
|
||||
},
|
||||
});
|
||||
if (!['inserted', 'existing'].includes(policy.status)) {
|
||||
throw new Error('Plugin Package E2E policy seed was not durable');
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await runtimeDatabase.close();
|
||||
}
|
||||
let publisherTrustStatus = null;
|
||||
const packageManagerConnection = loadPostgresConnectionEnvironment(
|
||||
process.env,
|
||||
{
|
||||
host: 'QL3_E2E_POSTGRES_HOST',
|
||||
port: 'QL3_E2E_POSTGRES_PORT',
|
||||
database: 'QL3_E2E_POSTGRES_DATABASE',
|
||||
user: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER',
|
||||
password: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_PASSWORD',
|
||||
},
|
||||
);
|
||||
const packageManagerDatabase = await createPostgresDatabaseOpener({
|
||||
role: 'package-manager',
|
||||
connection: {
|
||||
...packageManagerConnection,
|
||||
tls: { mode: 'disable' },
|
||||
},
|
||||
pool: {
|
||||
applicationName: 'qinglong3-plugin-package-e2e-management-seed',
|
||||
maxConnections: 1,
|
||||
connectionTimeoutMs: 15_000,
|
||||
},
|
||||
onPoolError() {},
|
||||
})();
|
||||
try {
|
||||
await assertPostgresPackageManagerSchemaReady(
|
||||
packageManagerDatabase.pool,
|
||||
);
|
||||
if (mode === 'seed-initial') {
|
||||
const observed =
|
||||
await new PostgresPluginPackagePublisherTrustAuthorityRepository(
|
||||
packageManagerDatabase.pool,
|
||||
@@ -754,9 +914,17 @@ async function runSeed() {
|
||||
);
|
||||
}
|
||||
publisherTrustStatus = observed.status;
|
||||
} finally {
|
||||
await packageManagerDatabase.close();
|
||||
}
|
||||
await new PostgresPluginPackageInstallProposalRepository(
|
||||
packageManagerDatabase.pool,
|
||||
).createProposal(selected.authority.proposalCommand);
|
||||
const approvals = new PostgresApprovalRequestRepository(
|
||||
packageManagerDatabase.pool,
|
||||
);
|
||||
await approvals.create(selected.authority.requestCommand);
|
||||
await approvals.decide(selected.authority.decisionCommand);
|
||||
} finally {
|
||||
await packageManagerDatabase.close();
|
||||
}
|
||||
const lock = normalizePluginPackageLock(selected.lock);
|
||||
const repository = new PostgresPluginPackageInstallRepository(
|
||||
@@ -772,16 +940,68 @@ async function runSeed() {
|
||||
) {
|
||||
throw new Error('Plugin Package E2E previous install head is invalid');
|
||||
}
|
||||
const record = createPluginPackageInstall(lock, {
|
||||
const consumed = await new PostgresApprovalRequestRepository(
|
||||
database.pool,
|
||||
).consume(selected.authority.consumptionCommand);
|
||||
if (
|
||||
JSON.stringify(consumed.dispatch) !==
|
||||
JSON.stringify(selected.authority.dispatch)
|
||||
) {
|
||||
throw new Error('Plugin Package E2E durable dispatch drifted');
|
||||
}
|
||||
const executions = new PostgresApprovedActionExecutionRepository(
|
||||
database.pool,
|
||||
);
|
||||
const executionAtMs = Date.now();
|
||||
const owner = 'plugin_recovery_e2e_dispatcher';
|
||||
const leaseToken = `lease-plugin-recovery-e2e-${
|
||||
mode === 'seed-initial' ? 'initial' : 'upgrade'
|
||||
}`;
|
||||
const claimed = await executions.claimExecution({
|
||||
dispatchId: consumed.dispatch.id,
|
||||
owner,
|
||||
leaseToken,
|
||||
nowMs: executionAtMs,
|
||||
leaseDurationMs: 60_000,
|
||||
});
|
||||
if (claimed.status !== 'claimed') {
|
||||
throw new Error('Plugin Package E2E execution was not claimable');
|
||||
}
|
||||
const started = await executions.startExecution({
|
||||
dispatchId: consumed.dispatch.id,
|
||||
approvalRequestId: consumed.dispatch.approvalRequestId,
|
||||
actionDigest: consumed.dispatch.action.actionDigest,
|
||||
owner,
|
||||
leaseToken,
|
||||
expectedVersion: claimed.snapshot.execution.version,
|
||||
startedAtMs: executionAtMs + 1,
|
||||
});
|
||||
const admittedAtMs = executionAtMs + 2;
|
||||
const result = await repository.admit({
|
||||
lock,
|
||||
proposalDigest:
|
||||
selected.authority.proposalCommand.proposal.proposalDigest,
|
||||
execution: started.execution,
|
||||
installationId: selected.installationId,
|
||||
mutationId: `mutation-plugin-recovery-e2e-${
|
||||
mode === 'seed-initial' ? 'initial' : 'upgrade'
|
||||
}-create`,
|
||||
occurredAtMs: lock.createdAtMs + 1,
|
||||
}-admit`,
|
||||
admittedAtMs,
|
||||
audit: {
|
||||
eventId: selected.authority.admissionAuditEventId,
|
||||
requestId: consumed.dispatch.id,
|
||||
operationId: 'plugin_package.admit',
|
||||
projectId: lock.projectId,
|
||||
subject: selected.authority.consumer,
|
||||
authenticationId: `auth-plugin-recovery-e2e-dispatcher-${
|
||||
mode === 'seed-initial' ? 'initial' : 'upgrade'
|
||||
}`,
|
||||
outcome: 'allowed',
|
||||
reasons: ['approved_action'],
|
||||
fence: selected.authority.fence,
|
||||
occurredAtMs: admittedAtMs,
|
||||
},
|
||||
});
|
||||
const result = await repository.create(
|
||||
pluginPackageInstallCreate(lock, record, previous),
|
||||
);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schema: 'qinglong/plugin-package-recovery-e2e-seed-result@v1',
|
||||
@@ -806,15 +1026,13 @@ async function main() {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
['seed-initial', 'seed-upgrade', 'commit-transition'].includes(
|
||||
process.env.QL3_E2E_MODE,
|
||||
)
|
||||
['seed-initial', 'seed-upgrade'].includes(process.env.QL3_E2E_MODE)
|
||||
) {
|
||||
await runSeed();
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
'QL3_E2E_MODE must be registry, seed-initial, seed-upgrade or commit-transition',
|
||||
'QL3_E2E_MODE must be registry, seed-initial or seed-upgrade',
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v2';
|
||||
const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v3';
|
||||
const LIMITATIONS = Object.freeze([
|
||||
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
|
||||
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
|
||||
@@ -13,7 +13,6 @@ const LIMITATIONS = Object.freeze([
|
||||
]);
|
||||
const GATE_KEYS = Object.freeze([
|
||||
'healthyInitialActivation',
|
||||
'missingTransitionFailedClosed',
|
||||
'invalidUpgradeRejectedBeforeActivation',
|
||||
'activePointerUidUnchanged',
|
||||
'activePointerResourceVersionUnchanged',
|
||||
@@ -119,8 +118,8 @@ function validRuntime(report) {
|
||||
]) &&
|
||||
runtime.replicas === 2 &&
|
||||
runtime.creationTimestamp === ordering?.runtimeCreatedAt &&
|
||||
runtime.recoveryJobUid === ordering?.rejectionRecoveryJobUid &&
|
||||
runtime.recoveryCompletedAt === ordering?.rejectionRecoveryCompletedAt &&
|
||||
runtime.recoveryJobUid === ordering?.upgradeRecoveryJobUid &&
|
||||
runtime.recoveryCompletedAt === ordering?.upgradeRecoveryCompletedAt &&
|
||||
Array.isArray(runtime.nodes) &&
|
||||
runtime.nodes.length === 2 &&
|
||||
new Set(runtime.nodes).size === 2 &&
|
||||
@@ -179,8 +178,7 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
'postgresRepositoryDigest',
|
||||
'migrationImageId',
|
||||
'initialRecoveryImageId',
|
||||
'stageRecoveryImageId',
|
||||
'rejectionRecoveryImageId',
|
||||
'upgradeRecoveryImageId',
|
||||
'postgresImageId',
|
||||
]) ||
|
||||
!SHA256_ID.test(images?.adminBuildId ?? '') ||
|
||||
@@ -192,9 +190,7 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
!imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.initialRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.stageRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.rejectionRecoveryImageId) !==
|
||||
imageDigest(images?.upgradeRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
!imageDigest(images?.postgresImageId)
|
||||
) {
|
||||
@@ -207,17 +203,13 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
const timeKeys = [
|
||||
'migrationCompletedAt',
|
||||
'initialRecoveryCompletedAt',
|
||||
'upgradeStageFailedAt',
|
||||
'transitionCompletedAt',
|
||||
'rejectionRecoveryCompletedAt',
|
||||
'upgradeRecoveryCompletedAt',
|
||||
'runtimeCreatedAt',
|
||||
];
|
||||
const uidKeys = [
|
||||
'migrationJobUid',
|
||||
'initialRecoveryJobUid',
|
||||
'upgradeStageJobUid',
|
||||
'transitionJobUid',
|
||||
'rejectionRecoveryJobUid',
|
||||
'upgradeRecoveryJobUid',
|
||||
'runtimeBoundRecoveryJobUid',
|
||||
];
|
||||
const times = timeKeys.map((key) => Date.parse(ordering?.[key]));
|
||||
@@ -225,7 +217,7 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
!exactKeys(ordering, [...uidKeys, ...timeKeys]) ||
|
||||
uidKeys.some((key) => !UUID.test(ordering?.[key] ?? '')) ||
|
||||
ordering?.runtimeBoundRecoveryJobUid !==
|
||||
ordering?.rejectionRecoveryJobUid ||
|
||||
ordering?.upgradeRecoveryJobUid ||
|
||||
timeKeys.some((key) => !validIso(ordering?.[key])) ||
|
||||
times.some((value, index) => index > 0 && value < times[index - 1])
|
||||
) {
|
||||
@@ -240,18 +232,12 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
const failed = report?.failedUpgrade;
|
||||
if (
|
||||
!exactKeys(failed, [
|
||||
'stageFailure',
|
||||
'transitionReceiptDigest',
|
||||
'recoveryJobUid',
|
||||
'rejectionReason',
|
||||
'candidateRevisionCount',
|
||||
'activePointerUnchanged',
|
||||
]) ||
|
||||
!exactKeys(failed?.stageFailure, ['jobUid', 'reason', 'durableState']) ||
|
||||
failed?.stageFailure?.jobUid !== ordering?.upgradeStageJobUid ||
|
||||
failed?.stageFailure?.reason !==
|
||||
'ClusterPluginPackageRecoveryRequiredError' ||
|
||||
failed?.stageFailure?.durableState !== 'staged' ||
|
||||
!SHA256.test(failed?.transitionReceiptDigest ?? '') ||
|
||||
failed?.recoveryJobUid !== ordering?.upgradeRecoveryJobUid ||
|
||||
failed?.rejectionReason !== 'activation_fact_conflict' ||
|
||||
failed?.candidateRevisionCount !== 0 ||
|
||||
failed?.activePointerUnchanged !== true
|
||||
@@ -277,7 +263,6 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
'initialMutationCount',
|
||||
'upgradeMutationCount',
|
||||
'headInstallationId',
|
||||
'transitionReceiptCount',
|
||||
'initialRevisionCount',
|
||||
'upgradeRevisionCount',
|
||||
'recoverableCount',
|
||||
@@ -296,7 +281,6 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
database?.initialMutationCount !== 4 ||
|
||||
database?.upgradeMutationCount !== 3 ||
|
||||
database?.headInstallationId !== 'install-plugin-recovery-e2e-upgrade' ||
|
||||
database?.transitionReceiptCount !== 1 ||
|
||||
database?.initialRevisionCount !== 1 ||
|
||||
database?.upgradeRevisionCount !== 0 ||
|
||||
database?.recoverableCount !== 0
|
||||
@@ -323,11 +307,11 @@ function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
]) ||
|
||||
oci?.https !== true ||
|
||||
oci?.authentication !== 'exact-registry-basic' ||
|
||||
oci?.authenticatedRequestCount !== 18 ||
|
||||
oci?.requestCount !== 18 ||
|
||||
oci?.authenticatedRequestCount !== 12 ||
|
||||
oci?.requestCount !== 12 ||
|
||||
oci?.uniquePaths !== 12 ||
|
||||
oci?.initialRequestCount !== 6 ||
|
||||
oci?.upgradeRequestCount !== 12 ||
|
||||
oci?.upgradeRequestCount !== 6 ||
|
||||
oci?.redirects !== 0
|
||||
) {
|
||||
findings.push(
|
||||
|
||||
@@ -38,13 +38,11 @@ const POSTGRES_RUNTIME_IMAGE =
|
||||
const DEFAULT_ADMIN_IMAGE = 'qinglong3-cluster-admin:ql3-plugin-recovery-e2e';
|
||||
const DEFAULT_CONTROL_IMAGE =
|
||||
'qinglong3-cluster-control:ql3-plugin-recovery-e2e';
|
||||
const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v2';
|
||||
const REPORT_SCHEMA = 'qinglong/plugin-package-recovery-e2e-live-contract@v3';
|
||||
const INITIAL_SEED_JOB = 'ql3-plugin-package-e2e-seed-initial';
|
||||
const INITIAL_RECOVERY_JOB = 'ql3-plugin-package-recovery-initial';
|
||||
const UPGRADE_SEED_JOB = 'ql3-plugin-package-e2e-seed-upgrade';
|
||||
const UPGRADE_STAGE_JOB = 'ql3-plugin-package-recovery-stage-upgrade';
|
||||
const TRANSITION_JOB = 'ql3-plugin-package-e2e-transition';
|
||||
const UPGRADE_REJECTION_JOB = 'ql3-plugin-package-recovery-reject-upgrade';
|
||||
const UPGRADE_RECOVERY_JOB = 'ql3-plugin-package-recovery-reject-upgrade';
|
||||
const SAFE_CLUSTER =
|
||||
/^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/;
|
||||
|
||||
@@ -879,6 +877,19 @@ function seedJob(name, mode) {
|
||||
{ name: 'QL3_E2E_POSTGRES_HOST', value: POSTGRES_NAME },
|
||||
{ name: 'QL3_E2E_POSTGRES_PORT', value: '5432' },
|
||||
{ name: 'QL3_E2E_POSTGRES_DATABASE', value: 'qinglong' },
|
||||
{
|
||||
name: 'QL3_E2E_POSTGRES_RUNTIME_USER',
|
||||
value: 'ql3_runtime',
|
||||
},
|
||||
{
|
||||
name: 'QL3_E2E_POSTGRES_RUNTIME_PASSWORD',
|
||||
valueFrom: {
|
||||
secretKeyRef: {
|
||||
name: 'ql3-e2e-postgres-auth',
|
||||
key: 'runtime-password',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'QL3_E2E_POSTGRES_PACKAGE_MANAGER_USER',
|
||||
value: 'ql3_package_manager',
|
||||
@@ -1179,67 +1190,6 @@ function canI(verb, resource) {
|
||||
return result.stdout === 'yes';
|
||||
}
|
||||
|
||||
function upgradeStageEvidence(fixture, transitionReceiptCount) {
|
||||
const generationDigest = fixture.upgrade.generation.generationDigest;
|
||||
assert.match(generationDigest, /^[0-9a-f]{64}$/);
|
||||
const sql = `
|
||||
SELECT json_build_object(
|
||||
'state', (
|
||||
SELECT state FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'previousActiveLockDigest', (
|
||||
SELECT previous_active_lock_digest FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'activeLockDigest', (
|
||||
SELECT active_lock_digest FROM ql3.plugin_package_installs
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'mutationCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_install_mutations
|
||||
WHERE installation_id = '${fixture.upgrade.installationId}'
|
||||
),
|
||||
'transitionReceiptCount', (
|
||||
SELECT count(*)
|
||||
FROM ql3.plugin_package_secret_binding_transition_receipts
|
||||
WHERE generation_digest = '${generationDigest}'
|
||||
),
|
||||
'candidateRevisionCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_materialized_revisions
|
||||
WHERE generation_digest = '${generationDigest}'
|
||||
)
|
||||
)::text;
|
||||
`.trim();
|
||||
const output = kubectl(
|
||||
[
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'exec',
|
||||
POSTGRES_NAME,
|
||||
'--',
|
||||
'psql',
|
||||
'--username',
|
||||
'postgres',
|
||||
'--dbname',
|
||||
'qinglong',
|
||||
'--tuples-only',
|
||||
'--no-align',
|
||||
'--command',
|
||||
sql,
|
||||
],
|
||||
{ capture: true, quiet: true },
|
||||
).stdout;
|
||||
const value = JSON.parse(output);
|
||||
assert.equal(value.state, 'staged');
|
||||
assert.equal(value.previousActiveLockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(value.activeLockDigest, fixture.initial.lock.lockDigest);
|
||||
assert.equal(value.mutationCount, 2);
|
||||
assert.equal(value.transitionReceiptCount, transitionReceiptCount);
|
||||
assert.equal(value.candidateRevisionCount, 0);
|
||||
return value;
|
||||
}
|
||||
|
||||
function databaseEvidence(fixture) {
|
||||
const initialGenerationDigest = fixture.initial.generation.generationDigest;
|
||||
const upgradeGenerationDigest = fixture.upgrade.generation.generationDigest;
|
||||
@@ -1298,11 +1248,6 @@ SELECT json_build_object(
|
||||
FROM ql3.plugin_package_install_heads
|
||||
WHERE project_id = 'default' AND package_name = 'e2e-monitor'
|
||||
),
|
||||
'transitionReceiptCount', (
|
||||
SELECT count(*)
|
||||
FROM ql3.plugin_package_secret_binding_transition_receipts
|
||||
WHERE generation_digest = '${upgradeGenerationDigest}'
|
||||
),
|
||||
'initialRevisionCount', (
|
||||
SELECT count(*) FROM ql3.plugin_package_materialized_revisions
|
||||
WHERE generation_digest = '${initialGenerationDigest}'
|
||||
@@ -1352,7 +1297,6 @@ SELECT json_build_object(
|
||||
assert.equal(value.initialMutationCount, 4);
|
||||
assert.equal(value.upgradeMutationCount, 3);
|
||||
assert.equal(value.headInstallationId, fixture.upgrade.installationId);
|
||||
assert.equal(value.transitionReceiptCount, 1);
|
||||
assert.equal(value.initialRevisionCount, 1);
|
||||
assert.equal(value.upgradeRevisionCount, 0);
|
||||
assert.equal(value.recoverableCount, 0);
|
||||
@@ -1411,10 +1355,7 @@ function registryEvidence(fixture) {
|
||||
const packageRequests = events.filter((event) => event.path !== '/v2/');
|
||||
const expectedPaths = [
|
||||
...fixture.initial.routes.map((routeValue) => routeValue.path),
|
||||
...fixture.upgrade.routes.flatMap((routeValue) => [
|
||||
routeValue.path,
|
||||
routeValue.path,
|
||||
]),
|
||||
...fixture.upgrade.routes.map((routeValue) => routeValue.path),
|
||||
].sort();
|
||||
assert.equal(packageRequests.length, expectedPaths.length);
|
||||
assert.deepEqual(
|
||||
@@ -1430,7 +1371,7 @@ function registryEvidence(fixture) {
|
||||
requestCount: packageRequests.length,
|
||||
uniquePaths: new Set(packageRequests.map((event) => event.path)).size,
|
||||
initialRequestCount: fixture.initial.routes.length,
|
||||
upgradeRequestCount: fixture.upgrade.routes.length * 2,
|
||||
upgradeRequestCount: fixture.upgrade.routes.length,
|
||||
redirects: 0,
|
||||
});
|
||||
}
|
||||
@@ -1874,7 +1815,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
(value) => value.event === 'seed_completed',
|
||||
);
|
||||
assert.equal(initialSeed.phase, 'initial');
|
||||
assert.equal(initialSeed.status, 'created');
|
||||
assert.equal(initialSeed.status, 'admitted');
|
||||
assert.equal(initialSeed.state, 'queued');
|
||||
assert.equal(initialSeed.lockDigest, fixture.initial.lock.lockDigest);
|
||||
|
||||
@@ -1901,52 +1842,16 @@ async function main(argv = process.argv.slice(2)) {
|
||||
(value) => value.event === 'seed_completed',
|
||||
);
|
||||
assert.equal(upgradeSeed.phase, 'upgrade');
|
||||
assert.equal(upgradeSeed.status, 'created');
|
||||
assert.equal(upgradeSeed.status, 'admitted');
|
||||
assert.equal(upgradeSeed.state, 'queued');
|
||||
assert.equal(upgradeSeed.lockDigest, fixture.upgrade.lock.lockDigest);
|
||||
|
||||
for (const resource of recoveryResources(fixture, UPGRADE_STAGE_JOB)) {
|
||||
apply(resource, `apply upgrade staging recovery ${resource.kind}`);
|
||||
}
|
||||
const stagedUpgrade = waitForJob(UPGRADE_STAGE_JOB, 'failed');
|
||||
const stagedFailureCondition = stagedUpgrade.status.conditions.find(
|
||||
(condition) => condition.type === 'Failed' && condition.status === 'True',
|
||||
);
|
||||
assert.ok(stagedFailureCondition?.lastTransitionTime);
|
||||
const stageFailure = lastJsonLine(
|
||||
jobLog(UPGRADE_STAGE_JOB),
|
||||
(value) => value.event === 'recovery_failed',
|
||||
);
|
||||
assert.equal(
|
||||
stageFailure.name,
|
||||
'ClusterPluginPackageRecoveryRequiredError',
|
||||
);
|
||||
const stagedDatabase = upgradeStageEvidence(fixture, 0);
|
||||
assert.deepEqual(activePointerEvidence(fixture), pointerBeforeUpgrade);
|
||||
|
||||
apply(
|
||||
seedJob(TRANSITION_JOB, 'commit-transition'),
|
||||
'commit durable no-secret binding transition receipt',
|
||||
);
|
||||
const committedTransition = waitForJob(TRANSITION_JOB);
|
||||
const transition = lastJsonLine(
|
||||
jobLog(TRANSITION_JOB),
|
||||
(value) => value.event === 'transition_completed',
|
||||
);
|
||||
assert.equal(transition.status, 'created');
|
||||
assert.equal(
|
||||
transition.generationDigest,
|
||||
fixture.upgrade.generation.generationDigest,
|
||||
);
|
||||
assert.equal(transition.bindingDigest, null);
|
||||
upgradeStageEvidence(fixture, 1);
|
||||
|
||||
for (const resource of recoveryResources(fixture, UPGRADE_REJECTION_JOB)) {
|
||||
for (const resource of recoveryResources(fixture, UPGRADE_RECOVERY_JOB)) {
|
||||
apply(resource, `apply upgrade rejection recovery ${resource.kind}`);
|
||||
}
|
||||
const rejectedUpgrade = waitForJob(UPGRADE_REJECTION_JOB);
|
||||
const rejectedUpgrade = waitForJob(UPGRADE_RECOVERY_JOB);
|
||||
const rejectionCompleted = lastJsonLine(
|
||||
jobLog(UPGRADE_REJECTION_JOB),
|
||||
jobLog(UPGRADE_RECOVERY_JOB),
|
||||
(value) => value.event === 'recovery_completed',
|
||||
);
|
||||
assert.equal(rejectionCompleted.recovery.safeToAdmit, true);
|
||||
@@ -1965,8 +1870,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
secrets,
|
||||
);
|
||||
const initialRecoveryImageId = jobImageId(INITIAL_RECOVERY_JOB);
|
||||
const stageRecoveryImageId = jobImageId(UPGRADE_STAGE_JOB);
|
||||
const rejectionRecoveryImageId = jobImageId(UPGRADE_REJECTION_JOB);
|
||||
const upgradeRecoveryImageId = jobImageId(UPGRADE_RECOVERY_JOB);
|
||||
const migrationImageId = jobImageId('ql3-cluster-migration');
|
||||
const postgresPod = kubectlJson([
|
||||
'-n',
|
||||
@@ -1991,8 +1895,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST,
|
||||
migrationImageId,
|
||||
initialRecoveryImageId,
|
||||
stageRecoveryImageId,
|
||||
rejectionRecoveryImageId,
|
||||
upgradeRecoveryImageId,
|
||||
postgresImageId: postgresPod.status.containerStatuses[0].imageID,
|
||||
}),
|
||||
ordering: Object.freeze({
|
||||
@@ -2000,22 +1903,13 @@ async function main(argv = process.argv.slice(2)) {
|
||||
migrationCompletedAt: migrated.status.completionTime,
|
||||
initialRecoveryJobUid: initialRecovered.metadata.uid,
|
||||
initialRecoveryCompletedAt: initialRecovered.status.completionTime,
|
||||
upgradeStageJobUid: stagedUpgrade.metadata.uid,
|
||||
upgradeStageFailedAt: stagedFailureCondition.lastTransitionTime,
|
||||
transitionJobUid: committedTransition.metadata.uid,
|
||||
transitionCompletedAt: committedTransition.status.completionTime,
|
||||
rejectionRecoveryJobUid: rejectedUpgrade.metadata.uid,
|
||||
rejectionRecoveryCompletedAt: rejectedUpgrade.status.completionTime,
|
||||
upgradeRecoveryJobUid: rejectedUpgrade.metadata.uid,
|
||||
upgradeRecoveryCompletedAt: rejectedUpgrade.status.completionTime,
|
||||
runtimeCreatedAt: runtime.creationTimestamp,
|
||||
runtimeBoundRecoveryJobUid: runtime.recoveryJobUid,
|
||||
}),
|
||||
failedUpgrade: Object.freeze({
|
||||
stageFailure: Object.freeze({
|
||||
jobUid: stagedUpgrade.metadata.uid,
|
||||
reason: stageFailure.name,
|
||||
durableState: stagedDatabase.state,
|
||||
}),
|
||||
transitionReceiptDigest: transition.receiptDigest,
|
||||
recoveryJobUid: rejectedUpgrade.metadata.uid,
|
||||
rejectionReason: database.upgradeFailureReason,
|
||||
candidateRevisionCount: database.upgradeRevisionCount,
|
||||
activePointerUnchanged: true,
|
||||
@@ -2029,7 +1923,6 @@ async function main(argv = process.argv.slice(2)) {
|
||||
runtime,
|
||||
gates: Object.freeze({
|
||||
healthyInitialActivation: true,
|
||||
missingTransitionFailedClosed: true,
|
||||
invalidUpgradeRejectedBeforeActivation: true,
|
||||
activePointerUidUnchanged: true,
|
||||
activePointerResourceVersionUnchanged: true,
|
||||
|
||||
Reference in New Issue
Block a user