open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 19:29:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): materialize package secret bindings

Browse Source
This commit is contained in:
whyour
2026-08-13 10:31:06 +08:00
parent 55ef085a6e
commit d72631a2b0
34 changed files with 839 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-local-sqlite/src/migration/migration.ts
+4
View File
@@ -102,6 +102,8 @@ import { local0089PluginPackageAutomationDispositionEventsMigration } from '../m
import { local0090CapabilityV45Migration } from '../migrations/0090-capability-v45';
import { local0091PluginPackageSecretBindingsMigration } from '../migrations/0091-plugin-package-secret-bindings';
import { local0092CapabilityV46Migration } from '../migrations/0092-capability-v46';
import { local0093PluginPackageSecretMaterializationGuardMigration } from '../migrations/0093-plugin-package-secret-materialization-guard';
import { local0094CapabilityV47Migration } from '../migrations/0094-capability-v47';
import type { LocalSqliteMigrationContext } from '../migrations/sqlMigration';
import {
LOCAL_SQLITE_MIGRATION_STREAM_ID,
@@ -216,6 +218,8 @@ export const localSqliteMigrationDefinition: MigrationStreamDefinition<LocalSqli
local0090CapabilityV45Migration,
local0091PluginPackageSecretBindingsMigration,
local0092CapabilityV46Migration,
local0093PluginPackageSecretMaterializationGuardMigration,
local0094CapabilityV47Migration,
]),
});
packages/ql3-local-sqlite/src/migration/migrationManifest.ts
+10
View File
@@ -472,5 +472,15 @@ export const localSqliteMigrationManifest: MigrationStreamManifest =
checksum:
'a1b058bb7b0259069202632d27d4a55466828cde831c49821eb862feeba6ce35',
}),
Object.freeze({
id: '0093-plugin-package-secret-materialization-guard',
checksum:
'e549ea0932c5b3e48f954ff72f205d84795ae03d473312070c3a9c4939169c42',
}),
Object.freeze({
id: '0094-capability-v47',
checksum:
'6ecbbef0b9d9b3c738cc47a80868e0871c47916fb67e7c37f8620f9099de735e',
}),
]),
});
packages/ql3-local-sqlite/src/migrations/0093-plugin-package-secret-materialization-guard.ts
+8
View File
@@ -0,0 +1,8 @@
import { defineLocalSqliteMigration } from './sqlMigration';
import { LOCAL_PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGER_SQL } from '../plugin-package/pluginPackageSecretMaterializationSchemaContract';
export const local0093PluginPackageSecretMaterializationGuardMigration =
defineLocalSqliteMigration({
id: '0093-plugin-package-secret-materialization-guard',
statements: [LOCAL_PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGER_SQL],
});
packages/ql3-local-sqlite/src/migrations/0094-capability-v47.ts
+14
View File
@@ -0,0 +1,14 @@
import { CAPABILITIES_V46 } from './0092-capability-v46';
import { defineLocalSqliteMigration } from './sqlMigration';
export const CAPABILITIES_V47 = CAPABILITIES_V46.replace(
'"plugin_package_secret_binding":1,',
'"plugin_package_secret_binding":1,"plugin_package_secret_materialization":1,',
);
export const local0094CapabilityV47Migration = defineLocalSqliteMigration({
id: '0094-capability-v47',
statements: [
`UPDATE "QingLong3SchemaCapabilities" SET contract_version = 47, migration_id = '0093-plugin-package-secret-materialization-guard', capabilities = '${CAPABILITIES_V47}', updated_at_ms = CAST(unixepoch('subsec') * 1000 AS INTEGER) WHERE contract_name = 'local-control-core' AND contract_version = 46 AND migration_id = '0091-plugin-package-secret-bindings' AND capabilities = '${CAPABILITIES_V46}'`,
],
});
packages/ql3-local-sqlite/src/plugin-package/pluginPackageSecretMaterializationSchemaContract.ts
+67
View File
@@ -0,0 +1,67 @@
/**
* Immutable DDL contract shared by the migration writer and read-only
* readiness auditor. Keeping it outside /migrations preserves lazy DDL
* loading on constrained Local Profiles.
*/
export const LOCAL_PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGER_SQL = `
CREATE TRIGGER ql3_plugin_package_secret_materialization_guard
BEFORE INSERT ON "QingLong3PluginPackageMaterializedRevisions"
BEGIN
SELECT CASE
WHEN json_type(
NEW.revision_json,
'$.manifest.spec.permissions.secrets'
) IS NOT 'array'
THEN RAISE(ABORT, 'Package Secret permission declarations are malformed')
WHEN json_array_length(json_extract(
NEW.revision_json,
'$.manifest.spec.permissions.secrets'
)) = 0 AND json_type(NEW.revision_json, '$.secretBinding') IS NOT NULL
THEN RAISE(ABORT, 'unexpected Package Secret binding')
WHEN json_array_length(json_extract(
NEW.revision_json,
'$.manifest.spec.permissions.secrets'
)) > 0 AND NOT EXISTS (
SELECT 1
FROM "QingLong3PluginPackageSecretBindings" AS binding
WHERE binding.generation_digest = NEW.generation_digest
AND json(binding.binding_json) = json_extract(
NEW.revision_json,
'$.secretBinding'
)
)
THEN RAISE(ABORT, 'Package Secret binding is absent or mismatched')
WHEN EXISTS (
SELECT 1
FROM json_each(NEW.revision_json, '$.resources') AS resource
JOIN json_each(
resource.value,
'$.value.spec.config.environment'
) AS environment
WHERE json_extract(resource.value, '$.kind') = 'task'
AND json_extract(environment.value, '$.kind') = 'package-secret'
)
THEN RAISE(ABORT, 'unresolved Package Secret placeholder')
WHEN EXISTS (
SELECT 1
FROM json_each(NEW.revision_json, '$.resources') AS resource
JOIN json_each(
resource.value,
'$.value.spec.config.environment'
) AS environment
WHERE json_extract(resource.value, '$.kind') = 'task'
AND json_extract(environment.value, '$.kind') = 'secret'
AND NOT EXISTS (
SELECT 1
FROM json_each(
NEW.revision_json,
'$.secretBinding.entries'
) AS binding_entry
WHERE json_extract(binding_entry.value, '$.secretRef') =
json_extract(environment.value, '$.secretRef')
)
)
THEN RAISE(ABORT, 'Task SecretRef is outside Package binding')
END;
END
`.trim();
packages/ql3-local-sqlite/src/profile/localProfile.ts
+2
View File
@@ -56,6 +56,7 @@ export type LocalProfileStorageBootstrapResult =
readonly ownerPepper: LocalSqliteRuntimeDatabase['ownerPepper'];
readonly pluginPackageInstalls: LocalSqliteRuntimeDatabase['pluginPackageInstalls'];
readonly pluginPackageMaterializedRevisions: LocalSqliteRuntimeDatabase['pluginPackageMaterializedRevisions'];
readonly pluginPackageSecretBindings: LocalSqliteRuntimeDatabase['pluginPackageSecretBindings'];
readonly pluginPackageTaskReconciliations: LocalSqliteRuntimeDatabase['pluginPackageTaskReconciliations'];
readonly pluginPackageAutomationPublications: LocalSqliteRuntimeDatabase['pluginPackageAutomationPublications'];
readonly projectToolDefinitionSnapshots: LocalSqliteRuntimeDatabase['projectToolDefinitionSnapshots'];
@@ -151,6 +152,7 @@ export async function bootstrapLocalProfileStorage(
pluginPackageInstalls: database.pluginPackageInstalls,
pluginPackageMaterializedRevisions:
database.pluginPackageMaterializedRevisions,
pluginPackageSecretBindings: database.pluginPackageSecretBindings,
pluginPackageTaskReconciliations:
database.pluginPackageTaskReconciliations,
pluginPackageAutomationPublications:
packages/ql3-local-sqlite/src/readiness/readiness.ts
+13 -3
View File
@@ -2,13 +2,22 @@ import { auditMigrationStreamHistory } from '@qinglong/runtime-core/migration-st
import type { DatabaseSync } from 'node:sqlite';
import { localSqliteMigrationManifest } from '../migration/migrationManifest';
import { LocalSqliteMigrationStreamStore } from '../migration/migrationStreamStore';
import { LOCAL_PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGER_SQL } from '../plugin-package/pluginPackageSecretMaterializationSchemaContract';
import {
LOCAL_STEP_RUN_REFERENCE_TRIGGERS,
normalizeLocalSqliteSchemaSql,
} from '../run/stepRunSchemaContract';
export const LOCAL_SQLITE_CONTRACT_NAME = 'local-control-core';
export const LOCAL_SQLITE_CONTRACT_VERSION = 46;
export const LOCAL_SQLITE_CONTRACT_VERSION = 47;
const PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGERS = Object.freeze([
Object.freeze({
name: 'ql3_plugin_package_secret_materialization_guard',
tableName: 'QingLong3PluginPackageMaterializedRevisions',
sql: LOCAL_PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGER_SQL,
}),
]);
const PLUGIN_PACKAGE_AUTOMATION_DISPOSITION_TRIGGERS = Object.freeze([
Object.freeze({
@@ -1743,6 +1752,7 @@ function assertRequiredSchema(client: DatabaseSync): number {
const expectedTriggers = [
...LOCAL_STEP_RUN_REFERENCE_TRIGGERS,
...PLUGIN_PACKAGE_AUTOMATION_DISPOSITION_TRIGGERS,
...PLUGIN_PACKAGE_SECRET_MATERIALIZATION_TRIGGERS,
].sort((left, right) => left.name.localeCompare(right.name));
if (
triggerRows.length !== expectedTriggers.length ||
@@ -2554,10 +2564,10 @@ export async function auditLocalSqliteReadiness(
capability.contract_name !== LOCAL_SQLITE_CONTRACT_NAME ||
capability.contract_version !== LOCAL_SQLITE_CONTRACT_VERSION ||
capability.migration_id !==
'0091-plugin-package-secret-bindings' ||
'0093-plugin-package-secret-materialization-guard' ||
typeof capability.capabilities !== 'string' ||
capability.capabilities !==
'{"run_core":1,"run_retry_policy":1,"completion_receipt_journal":1,"local_dispatch_plan":1,"local_secret_envelope":1,"local_project_policy":1,"local_project_administration":1,"local_security_audit":1,"local_security_audit_compaction":1,"local_secret_authorized_mutation":1,"local_identity":1,"local_api_credential":1,"local_identity_provisioning":1,"local_identity_credential_administration":1,"local_owner_bootstrap":1,"local_owner_delivery_acknowledgement":1,"api_credential_pepper_binding":1,"local_owner_pepper_catalog":1,"local_owner_credential_recovery":1,"local_owner_pepper_reference_inspection":1,"local_owner_pepper_material_gc":1,"local_owner_delivery_acknowledgement_gc":1,"task_definition":1,"local_execution_revision_digest":1,"trigger_definition":1,"legacy_adoption_ledger":1,"local_scheduler_admission":1,"plugin_package_install":1,"approved_action":1,"plugin_package_admission":1,"approved_action_execution":1,"plugin_package_proposal":1,"plugin_package_materialized_revision":1,"plugin_package_secret_binding":1,"plugin_package_task_reconciliation":1,"project_tool_definition_snapshot":1,"step_run":1,"tool_execution_evidence":1,"tool_execution_start_barrier":1,"tool_invocation_artifact":1,"tool_execution_artifact_binding":1,"tool_execution_completion":1,"tool_execution_failure_completion":1,"tool_result_key_catalog":1,"tool_result_rekey":1,"plugin_package_quarantine":1,"plugin_package_lifecycle":1,"plugin_package_automation_publication":1,"plugin_package_automation_security_withdrawal":1,"plugin_package_workflow_admission":1,"plugin_package_workflow_run_list":1,"run_attempt_log_retention":1,"plugin_package_workflow_task_attempt_admission":1}' ||
'{"run_core":1,"run_retry_policy":1,"completion_receipt_journal":1,"local_dispatch_plan":1,"local_secret_envelope":1,"local_project_policy":1,"local_project_administration":1,"local_security_audit":1,"local_security_audit_compaction":1,"local_secret_authorized_mutation":1,"local_identity":1,"local_api_credential":1,"local_identity_provisioning":1,"local_identity_credential_administration":1,"local_owner_bootstrap":1,"local_owner_delivery_acknowledgement":1,"api_credential_pepper_binding":1,"local_owner_pepper_catalog":1,"local_owner_credential_recovery":1,"local_owner_pepper_reference_inspection":1,"local_owner_pepper_material_gc":1,"local_owner_delivery_acknowledgement_gc":1,"task_definition":1,"local_execution_revision_digest":1,"trigger_definition":1,"legacy_adoption_ledger":1,"local_scheduler_admission":1,"plugin_package_install":1,"approved_action":1,"plugin_package_admission":1,"approved_action_execution":1,"plugin_package_proposal":1,"plugin_package_materialized_revision":1,"plugin_package_secret_binding":1,"plugin_package_secret_materialization":1,"plugin_package_task_reconciliation":1,"project_tool_definition_snapshot":1,"step_run":1,"tool_execution_evidence":1,"tool_execution_start_barrier":1,"tool_invocation_artifact":1,"tool_execution_artifact_binding":1,"tool_execution_completion":1,"tool_execution_failure_completion":1,"tool_result_key_catalog":1,"tool_result_rekey":1,"plugin_package_quarantine":1,"plugin_package_lifecycle":1,"plugin_package_automation_publication":1,"plugin_package_automation_security_withdrawal":1,"plugin_package_workflow_admission":1,"plugin_package_workflow_run_list":1,"run_attempt_log_retention":1,"plugin_package_workflow_task_attempt_admission":1}' ||
typeof capability.updated_at_ms !== 'number' ||
!Number.isSafeInteger(capability.updated_at_ms) ||
capability.updated_at_ms < 0
packages/ql3-local-sqlite/src/runtime/runtimeDatabase.ts
+14
View File
@@ -29,6 +29,7 @@ import type {
PluginPackageAutomationPublicationRecoverySource,
} from '@qinglong/runtime-core/plugin-package-automation-publication';
import type { PluginPackageMaterializedRevisionRepository } from '@qinglong/runtime-core/plugin-package-resource-materialization';
import type { PluginPackageSecretBindingRepository } from '@qinglong/runtime-core/plugin-package-secret-binding';
import type { PluginPackageTaskReconciliationRepository } from '@qinglong/runtime-core/plugin-package-task-reconciliation';
import type { PluginPackageTaskPublicationRecoverySource } from '@qinglong/runtime-core/plugin-package-task-publication';
import type { StepRunRepository } from '@qinglong/runtime-core/step-run';
@@ -109,6 +110,7 @@ export interface LocalSqliteRuntimeDatabase {
readonly ownerPepper: LocalOwnerPepperRepository;
pluginPackageInstalls(): Promise<PluginPackageInstallRepository>;
pluginPackageMaterializedRevisions(): Promise<PluginPackageMaterializedRevisionRepository>;
pluginPackageSecretBindings(): Promise<PluginPackageSecretBindingRepository>;
pluginPackageTaskReconciliations(): Promise<
PluginPackageTaskReconciliationRepository &
PluginPackageTaskPublicationRecoverySource
@@ -195,6 +197,9 @@ export async function openLocalSqliteRuntimeDatabase(
let pluginPackageMaterializedRevisionsPromise:
| Promise<PluginPackageMaterializedRevisionRepository>
| undefined;
let pluginPackageSecretBindingsPromise:
| Promise<PluginPackageSecretBindingRepository>
| undefined;
let pluginPackageTaskReconciliationsPromise:
| Promise<
PluginPackageTaskReconciliationRepository &
@@ -271,6 +276,15 @@ export async function openLocalSqliteRuntimeDatabase(
);
return pluginPackageMaterializedRevisionsPromise;
},
pluginPackageSecretBindings() {
pluginPackageSecretBindingsPromise ??= import(
'../plugin-package/pluginPackageSecretBindingRepository.js'
).then(
({ LocalSqlitePluginPackageSecretBindingRepository }) =>
new LocalSqlitePluginPackageSecretBindingRepository(authority),
);
return pluginPackageSecretBindingsPromise;
},
pluginPackageTaskReconciliations() {
pluginPackageTaskReconciliationsPromise ??= import(
'../plugin-package/pluginPackageTaskReconciliationRepository.js'
packages/ql3-local-sqlite/test/database.test.cjs
+88 -3
View File
@@ -142,9 +142,11 @@ test('creates a reviewed edge database and opens runtime only after readiness',
'0090-capability-v45',
'0091-plugin-package-secret-bindings',
'0092-capability-v46',
'0093-plugin-package-secret-materialization-guard',
'0094-capability-v47',
]);
assert.equal(migrated.readiness.contractName, 'local-control-core');
assert.equal(migrated.readiness.contractVersion, 46);
assert.equal(migrated.readiness.contractVersion, 47);
assert.equal(migrated.readiness.journalMode, 'delete');
assert.equal(fs.statSync(databasePath).mode & 0o777, 0o600);
@@ -228,6 +230,89 @@ test('creates a reviewed edge database and opens runtime only after readiness',
}
});
test('rejects malformed and unbound Package Secret materialized revisions at the database boundary', async () => {
const client = new DatabaseSync(':memory:');
try {
client.exec('PRAGMA foreign_keys = ON');
await runMigrationStream({
stream: localSqliteMigrationDefinition,
store: new LocalSqliteMigrationStreamStore(client),
});
const insert = client.prepare(
`INSERT INTO "QingLong3PluginPackageMaterializedRevisions" (
generation_digest, project_id, package_name, generation,
lock_digest, manifest_digest, revision_digest, revision_json,
created_at_ms
) VALUES (?, 'default', 'secret-guard', 1, ?, ?, ?, ?, 1)`,
);
const digest = 'a'.repeat(64);
const revision = ({ manifest, resources = [] }) => ({
schema: 'qinglong/plugin-package-materialized-revision@v1',
generation: {
installationId: 'install-secret-guard',
projectId: 'default',
packageName: 'secret-guard',
lockDigest: digest,
generation: 1,
generationDigest: digest,
},
manifestDigest: digest,
manifest,
resources,
revisionDigest: digest,
createdAtMs: 1,
});
assert.throws(
() =>
insert.run(
digest,
digest,
digest,
digest,
JSON.stringify(revision({ manifest: {} })),
),
/permission declarations are malformed/,
);
assert.throws(
() =>
insert.run(
digest,
digest,
digest,
digest,
JSON.stringify(
revision({
manifest: { spec: { permissions: { secrets: [] } } },
resources: [
{
kind: 'task',
value: {
spec: {
config: {
environment: [
{
name: 'TOKEN',
kind: 'secret',
secretRef:
'qlsecret://default/runtime-token?version=1',
},
],
},
},
},
},
],
}),
),
),
/outside Package binding/,
);
} finally {
client.close();
}
});
test('standalone opts into bounded WAL while edge keeps rollback journal', async (t) => {
const { databasePath } = fixture(t);
const options = { databasePath, profile: 'standalone' };
@@ -507,8 +592,8 @@ test('backfills v14 execution revisions with a verified independent digest', asy
.get(),
},
{
contract_version: 46,
migration_id: '0091-plugin-package-secret-bindings',
contract_version: 47,
migration_id: '0093-plugin-package-secret-materialization-guard',
},
);
} finally {
packages/ql3-local-sqlite/test/pluginPackageWorkflowAdmissionRepository.test.cjs
+2 -2
View File
@@ -156,7 +156,7 @@ test('atomically admits one generation-bound Workflow Run and exactly replays it
},
{ runs: 1, steps: 2, events: 3, mutations: 2, admissions: 1 },
);
assert.equal((await auditLocalSqliteReadiness(client)).contractVersion, 46);
assert.equal((await auditLocalSqliteReadiness(client)).contractVersion, 47);
});
test('runs an optional authorization guard inside new and replay transactions', async (t) => {
@@ -288,7 +288,7 @@ test('exactly replays immutable admission after the Workflow StepRun advances',
},
{ status: 'running', version: 5, eventSequence: 5 },
);
assert.equal((await auditLocalSqliteReadiness(client)).contractVersion, 46);
assert.equal((await auditLocalSqliteReadiness(client)).contractVersion, 47);
});
test('fails closed before writing when the exact installation is not active', async (t) => {
packages/ql3-local-sqlite/test/pluginPackageWorkflowTaskAttemptAdmissionRepository.test.cjs
+1 -1
View File
@@ -231,7 +231,7 @@ test('atomically admits the exact reconciled local Task revision and replays it'
stepAttemptCount: 0,
},
);
assert.equal((await auditLocalSqliteReadiness(client)).contractVersion, 46);
assert.equal((await auditLocalSqliteReadiness(client)).contractVersion, 47);
});
test('bounds candidate paging before SQL and fences cancellation', async (t) => {
packages/ql3-local-sqlite/test/rolloutSafety.test.cjs
+3 -3
View File
@@ -40,9 +40,9 @@ test('creates and exactly replays a reviewed rollout backup', async (t) => {
await migrateLocalSqlitePath(state);
const prepared = await createLocalSqliteRolloutBackup(state);
assert.equal(prepared.status, 'prepared');
assert.equal(prepared.contractVersion, 46);
assert.equal(prepared.writeContractVersion, 46);
assert.equal(LOCAL_SQLITE_WRITE_CONTRACT_VERSION, 46);
assert.equal(prepared.contractVersion, 47);
assert.equal(prepared.writeContractVersion, 47);
assert.equal(LOCAL_SQLITE_WRITE_CONTRACT_VERSION, 47);
assert.match(prepared.sha256, /^[0-9a-f]{64}$/);
assert.equal(prepared.bytes > 0, true);
assert.equal(prepared.pageCount > 0, true);