mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 01:32:44 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,635 @@
|
||||
import { LocalExecutionCoordinator } from '@qinglong/local-execution/execution';
|
||||
import {
|
||||
LocalCompletionReceiptProcessor,
|
||||
LocalExecutionControlCoordinator,
|
||||
LocalExecutionControlLifecycle,
|
||||
LocalExecutionControlScanner,
|
||||
type LocalExecutionControlCycleSummary,
|
||||
} from '@qinglong/local-execution/control';
|
||||
import {
|
||||
LocalDispatchPlanMaterializer,
|
||||
LocalFileArtifactAllocator,
|
||||
LocalRunDispatcher,
|
||||
localArtifactCapacityPolicyForProfile,
|
||||
} from '@qinglong/local-execution/dispatch';
|
||||
import {
|
||||
LocalRunStartupRecoveryCoordinator,
|
||||
LocalWorkflowTaskStartupRecoveryCoordinator,
|
||||
type LocalRunStartupRecoverySummary,
|
||||
type LocalWorkflowTaskStartupRecoverySummary,
|
||||
} from '@qinglong/local-execution/recovery';
|
||||
import {
|
||||
LocalSchedulerCoordinator,
|
||||
LocalSchedulerLifecycle,
|
||||
LocalWorkflowSchedulerCoordinator,
|
||||
} from '@qinglong/local-execution/scheduler';
|
||||
import {
|
||||
CompletionReceiptFileStore,
|
||||
LocalCompletionReceiptCleanupScanner,
|
||||
LocalProcessController,
|
||||
LocalProcessPersistedExecutionInspector,
|
||||
LocalProcessLauncher,
|
||||
type LocalCompletionReceiptCleanupSummary,
|
||||
} from '@qinglong/local-process';
|
||||
import {
|
||||
EncryptedLocalSecretService,
|
||||
LocalSecretKeyringFileProvider,
|
||||
} from '@qinglong/local-secret';
|
||||
import { MAX_PLUGIN_PACKAGE_INSTALL_RECOVERY_PAGE_SIZE } from '@qinglong/runtime-core/plugin-package-install';
|
||||
import { MAX_PLUGIN_PACKAGE_RECOVERY_PAGES } from '@qinglong/runtime-core/plugin-package-recovery';
|
||||
import {
|
||||
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGES,
|
||||
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGE_SIZE,
|
||||
} from '@qinglong/runtime-core/plugin-package-task-publication';
|
||||
import {
|
||||
type LocalApplicationActivationAudit,
|
||||
type LocalApplicationBootstrapOptions,
|
||||
type LocalApplicationBootstrapResult,
|
||||
type LocalApplicationEnabledBootstrapOptions,
|
||||
type LocalApplicationProfile,
|
||||
type LocalApplicationProductSurfaceLifecycle,
|
||||
type LocalApplicationStopResult,
|
||||
} from './contract';
|
||||
import {
|
||||
recoverLocalApplicationPluginPackages,
|
||||
type LocalApplicationPluginPackageStartup,
|
||||
} from './pluginPackageStartup';
|
||||
import {
|
||||
openLocalApplicationStorage,
|
||||
type LocalApplicationReadyStorage,
|
||||
} from './storageActivation';
|
||||
import { LocalApplicationStartupRecoveryRequiredError } from './startupErrors';
|
||||
export {
|
||||
LocalApplicationPluginPackageAutomationPublicationRequiredError,
|
||||
LocalApplicationPluginPackageRecoveryRequiredError,
|
||||
LocalApplicationPluginPackageTaskPublicationRequiredError,
|
||||
LocalApplicationPluginPackageToolSnapshotRequiredError,
|
||||
LocalApplicationStartupRecoveryRequiredError,
|
||||
} from './startupErrors';
|
||||
|
||||
const EXECUTION_CONTROL_POLICIES = Object.freeze({
|
||||
edge: Object.freeze({
|
||||
cleanupIntervalMs: 5 * 60_000,
|
||||
cleanupPageSize: 8,
|
||||
controlIntervalMs: 5_000,
|
||||
controlPageSize: 4,
|
||||
maxDrainPages: 2,
|
||||
retentionMs: 24 * 60 * 60_000,
|
||||
stopTimeoutMs: 5_000,
|
||||
}),
|
||||
standalone: Object.freeze({
|
||||
cleanupIntervalMs: 60_000,
|
||||
cleanupPageSize: 32,
|
||||
controlIntervalMs: 1_000,
|
||||
controlPageSize: 32,
|
||||
maxDrainPages: 8,
|
||||
retentionMs: 60 * 60_000,
|
||||
stopTimeoutMs: 10_000,
|
||||
}),
|
||||
});
|
||||
|
||||
function assertProfile(
|
||||
profile: unknown,
|
||||
): asserts profile is LocalApplicationProfile {
|
||||
if (profile !== 'edge' && profile !== 'standalone') {
|
||||
throw new TypeError('Local application Profile is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertEnabledBoundary(
|
||||
options: LocalApplicationBootstrapOptions,
|
||||
): void {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new TypeError('Local application bootstrap options are invalid');
|
||||
}
|
||||
if (options.enabled !== undefined && typeof options.enabled !== 'boolean') {
|
||||
throw new TypeError('Local application enabled flag is invalid');
|
||||
}
|
||||
assertProfile(options.profile);
|
||||
if (typeof options.applicationAudit !== 'function') {
|
||||
throw new TypeError('Local application audit sink is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertActiveBoundary(
|
||||
options: LocalApplicationBootstrapOptions,
|
||||
): asserts options is LocalApplicationEnabledBootstrapOptions {
|
||||
if (options.enabled !== true) {
|
||||
throw new TypeError('Local application enabled configuration is invalid');
|
||||
}
|
||||
if (typeof options.audit !== 'function') {
|
||||
throw new TypeError('Local application storage audit sink is invalid');
|
||||
}
|
||||
if (
|
||||
options.storageMode !== 'fresh' &&
|
||||
typeof options.adoptionAudit !== 'function'
|
||||
) {
|
||||
throw new TypeError('Local application adoption audit sink is invalid');
|
||||
}
|
||||
if (
|
||||
options.storageMode === 'fresh' &&
|
||||
typeof options.databasePath !== 'string'
|
||||
) {
|
||||
throw new TypeError('Local application fresh database path is invalid');
|
||||
}
|
||||
if (typeof options.receiptRoot !== 'string') {
|
||||
throw new TypeError('Local application receipt root is invalid');
|
||||
}
|
||||
if (typeof options.artifactRoot !== 'string') {
|
||||
throw new TypeError('Local application Artifact root is invalid');
|
||||
}
|
||||
if (typeof options.secretKeyringPath !== 'string') {
|
||||
throw new TypeError('Local application Secret keyring path is invalid');
|
||||
}
|
||||
if (
|
||||
options.productSurface !== undefined &&
|
||||
typeof options.productSurface?.start !== 'function'
|
||||
) {
|
||||
throw new TypeError('Local application product surface is invalid');
|
||||
}
|
||||
const pluginPackages = options.pluginPackages;
|
||||
if (
|
||||
!pluginPackages ||
|
||||
typeof pluginPackages !== 'object' ||
|
||||
Array.isArray(pluginPackages) ||
|
||||
Object.keys(pluginPackages).some(
|
||||
(key) =>
|
||||
![
|
||||
'stageProvider',
|
||||
'stagingRoot',
|
||||
'activationRoot',
|
||||
'now',
|
||||
'pageSize',
|
||||
'maxPages',
|
||||
'taskPublicationPageSize',
|
||||
'taskPublicationMaxPages',
|
||||
].includes(key),
|
||||
) ||
|
||||
!pluginPackages.stageProvider ||
|
||||
typeof pluginPackages.stageProvider.stage !== 'function' ||
|
||||
typeof pluginPackages.stagingRoot !== 'string' ||
|
||||
typeof pluginPackages.activationRoot !== 'string' ||
|
||||
typeof pluginPackages.now !== 'function' ||
|
||||
(pluginPackages.pageSize !== undefined &&
|
||||
(!Number.isSafeInteger(pluginPackages.pageSize) ||
|
||||
pluginPackages.pageSize < 1 ||
|
||||
pluginPackages.pageSize >
|
||||
MAX_PLUGIN_PACKAGE_INSTALL_RECOVERY_PAGE_SIZE)) ||
|
||||
(pluginPackages.maxPages !== undefined &&
|
||||
(!Number.isSafeInteger(pluginPackages.maxPages) ||
|
||||
pluginPackages.maxPages < 1 ||
|
||||
pluginPackages.maxPages > MAX_PLUGIN_PACKAGE_RECOVERY_PAGES)) ||
|
||||
(pluginPackages.taskPublicationPageSize !== undefined &&
|
||||
(!Number.isSafeInteger(pluginPackages.taskPublicationPageSize) ||
|
||||
pluginPackages.taskPublicationPageSize < 1 ||
|
||||
pluginPackages.taskPublicationPageSize >
|
||||
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGE_SIZE)) ||
|
||||
(pluginPackages.taskPublicationMaxPages !== undefined &&
|
||||
(!Number.isSafeInteger(pluginPackages.taskPublicationMaxPages) ||
|
||||
pluginPackages.taskPublicationMaxPages < 1 ||
|
||||
pluginPackages.taskPublicationMaxPages >
|
||||
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGES))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local application Plugin Package recovery configuration is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function oneOrAggregate(errors: unknown[], message: string): unknown {
|
||||
return errors.length === 1 ? errors[0] : new AggregateError(errors, message);
|
||||
}
|
||||
|
||||
async function bestEffortAudit(
|
||||
options: LocalApplicationBootstrapOptions,
|
||||
record: LocalApplicationActivationAudit,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await options.applicationAudit(record);
|
||||
} catch {
|
||||
// Diagnostics cannot replace the activation or shutdown result.
|
||||
}
|
||||
}
|
||||
|
||||
export async function bootstrapLocalApplication(
|
||||
options: LocalApplicationBootstrapOptions,
|
||||
): Promise<LocalApplicationBootstrapResult> {
|
||||
assertEnabledBoundary(options);
|
||||
if (options.enabled !== true) {
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'disabled',
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'disabled' as const,
|
||||
profile: options.profile,
|
||||
stop: async () => 'stopped' as const,
|
||||
});
|
||||
}
|
||||
assertActiveBoundary(options);
|
||||
|
||||
let storage: LocalApplicationReadyStorage | undefined;
|
||||
let schedulerLifecycle: LocalSchedulerLifecycle | undefined;
|
||||
let executionControlLifecycle: LocalExecutionControlLifecycle | undefined;
|
||||
let runRecovery: LocalRunStartupRecoverySummary | undefined;
|
||||
let workflowTaskRecovery: LocalWorkflowTaskStartupRecoverySummary | undefined;
|
||||
let receiptCleanup: LocalCompletionReceiptCleanupSummary | undefined;
|
||||
let executionControl: LocalExecutionControlCycleSummary | undefined;
|
||||
let pluginPackageStartup: LocalApplicationPluginPackageStartup | undefined;
|
||||
let productSurfaceLifecycle:
|
||||
| Readonly<LocalApplicationProductSurfaceLifecycle>
|
||||
| undefined;
|
||||
|
||||
try {
|
||||
storage = await openLocalApplicationStorage(options);
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'storage_ready',
|
||||
});
|
||||
|
||||
pluginPackageStartup = await recoverLocalApplicationPluginPackages(
|
||||
options,
|
||||
storage,
|
||||
);
|
||||
|
||||
const secretKeys = new LocalSecretKeyringFileProvider(
|
||||
options.secretKeyringPath,
|
||||
);
|
||||
const activeSecretKey = await secretKeys.active();
|
||||
activeSecretKey.key.fill(0);
|
||||
const localSecrets = new EncryptedLocalSecretService(
|
||||
storage.localSecrets,
|
||||
secretKeys,
|
||||
);
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'secrets_ready',
|
||||
});
|
||||
|
||||
const executionPolicy = EXECUTION_CONTROL_POLICIES[options.profile];
|
||||
const receipts = new CompletionReceiptFileStore(options.receiptRoot);
|
||||
const localProcessLauncher = new LocalProcessLauncher(
|
||||
storage.completionReceipts,
|
||||
{
|
||||
receiptRoot: options.receiptRoot,
|
||||
},
|
||||
);
|
||||
const localProcessController = new LocalProcessController();
|
||||
const workflowRuntime = await storage.pluginPackageWorkflowRuntime();
|
||||
const localProcess = new LocalExecutionCoordinator(
|
||||
storage.runs,
|
||||
localProcessLauncher,
|
||||
localProcessController,
|
||||
{ workflowTasks: workflowRuntime.executions },
|
||||
);
|
||||
const completionProcessor = new LocalCompletionReceiptProcessor(
|
||||
storage.runs,
|
||||
receipts,
|
||||
{
|
||||
journal: storage.completionReceipts,
|
||||
quarantineRetentionMs: executionPolicy.retentionMs,
|
||||
workflowTasks: workflowRuntime.executions,
|
||||
},
|
||||
);
|
||||
executionControlLifecycle = new LocalExecutionControlLifecycle(
|
||||
completionProcessor,
|
||||
new LocalExecutionControlScanner(
|
||||
storage.executionControl,
|
||||
new LocalExecutionControlCoordinator(
|
||||
storage.runs,
|
||||
completionProcessor,
|
||||
localProcessController,
|
||||
{ workflowTasks: workflowRuntime.executions },
|
||||
),
|
||||
),
|
||||
new LocalCompletionReceiptCleanupScanner(
|
||||
storage.completionReceipts,
|
||||
receipts,
|
||||
{
|
||||
terminalMissingRetentionMs: executionPolicy.retentionMs,
|
||||
},
|
||||
),
|
||||
{
|
||||
intervalMs: executionPolicy.controlIntervalMs,
|
||||
pageSize: executionPolicy.controlPageSize,
|
||||
cleanupIntervalMs: executionPolicy.cleanupIntervalMs,
|
||||
cleanupPageSize: executionPolicy.cleanupPageSize,
|
||||
stopTimeoutMs: executionPolicy.stopTimeoutMs,
|
||||
maxDrainPages: executionPolicy.maxDrainPages,
|
||||
onDiagnostic: async (error) => {
|
||||
if (error === undefined) return;
|
||||
await bestEffortAudit(options, {
|
||||
profile: options.profile,
|
||||
state: 'execution_control_degraded',
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
const localProcessDispatcher = new LocalRunDispatcher(
|
||||
storage.dispatch,
|
||||
new LocalDispatchPlanMaterializer(
|
||||
storage.dispatch,
|
||||
new LocalFileArtifactAllocator(
|
||||
options.artifactRoot,
|
||||
localArtifactCapacityPolicyForProfile(options.profile),
|
||||
),
|
||||
localSecrets,
|
||||
),
|
||||
localProcess,
|
||||
{
|
||||
pageSize: options.profile === 'edge' ? 4 : 16,
|
||||
maxPages: 1,
|
||||
onCompletion: (attemptId) => {
|
||||
executionControlLifecycle?.notifyCompletion(attemptId);
|
||||
},
|
||||
},
|
||||
);
|
||||
const scheduler = new LocalSchedulerCoordinator(storage.schedules, {
|
||||
pageSize: options.profile === 'edge' ? 4 : 16,
|
||||
misfireGraceMs: options.profile === 'edge' ? 30_000 : 5_000,
|
||||
});
|
||||
const workflowScheduler = new LocalWorkflowSchedulerCoordinator(
|
||||
scheduler,
|
||||
workflowRuntime.cancellation,
|
||||
workflowRuntime.frontier,
|
||||
workflowRuntime.taskAttempts,
|
||||
localProcessDispatcher,
|
||||
{
|
||||
cancellationPageSize: options.profile === 'edge' ? 4 : 32,
|
||||
cancellationMaxPages: options.profile === 'edge' ? 1 : 4,
|
||||
frontierPageSize: options.profile === 'edge' ? 1 : 16,
|
||||
frontierMaxPages: options.profile === 'edge' ? 1 : 4,
|
||||
taskAttemptPageSize: options.profile === 'edge' ? 1 : 16,
|
||||
taskAttemptMaxPages: options.profile === 'edge' ? 1 : 4,
|
||||
maxDispatches: options.profile === 'edge' ? 1 : 4,
|
||||
},
|
||||
);
|
||||
schedulerLifecycle = new LocalSchedulerLifecycle(workflowScheduler, {
|
||||
intervalMs: options.profile === 'edge' ? 5_000 : 1_000,
|
||||
stopTimeoutMs: executionPolicy.stopTimeoutMs,
|
||||
onDiagnostic: async (error) => {
|
||||
if (error === undefined) return;
|
||||
await bestEffortAudit(options, {
|
||||
profile: options.profile,
|
||||
state: 'scheduler_degraded',
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
runRecovery = await new LocalRunStartupRecoveryCoordinator(
|
||||
storage.runs,
|
||||
storage.startupRecovery,
|
||||
receipts,
|
||||
new LocalProcessPersistedExecutionInspector(),
|
||||
{
|
||||
receiptPublishGraceMs: options.profile === 'edge' ? 50 : 100,
|
||||
journal: storage.completionReceipts,
|
||||
quarantineRetentionMs: executionPolicy.retentionMs,
|
||||
completionProcessor,
|
||||
},
|
||||
).recover();
|
||||
if (!runRecovery.safe) {
|
||||
throw new LocalApplicationStartupRecoveryRequiredError(
|
||||
Math.max(
|
||||
runRecovery.remaining + runRecovery.failed,
|
||||
runRecovery.scanned - runRecovery.recovered,
|
||||
),
|
||||
runRecovery.truncated,
|
||||
);
|
||||
}
|
||||
workflowTaskRecovery =
|
||||
await new LocalWorkflowTaskStartupRecoveryCoordinator(
|
||||
storage.runs,
|
||||
workflowRuntime.executions,
|
||||
workflowRuntime.executions,
|
||||
completionProcessor,
|
||||
new LocalProcessPersistedExecutionInspector(),
|
||||
{
|
||||
receiptPublishGraceMs: options.profile === 'edge' ? 50 : 100,
|
||||
},
|
||||
).recover();
|
||||
if (!workflowTaskRecovery.safe) {
|
||||
throw new LocalApplicationStartupRecoveryRequiredError(
|
||||
workflowTaskRecovery.remaining + workflowTaskRecovery.failed,
|
||||
workflowTaskRecovery.truncated,
|
||||
);
|
||||
}
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'runs_recovered',
|
||||
runRecovery,
|
||||
workflowTaskRecovery,
|
||||
});
|
||||
executionControl = await executionControlLifecycle.runOnce(true);
|
||||
receiptCleanup = executionControl.cleanup;
|
||||
if (!receiptCleanup) {
|
||||
throw new Error('Local completion receipt cleanup did not run');
|
||||
}
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'receipts_reconciled',
|
||||
runRecovery,
|
||||
workflowTaskRecovery,
|
||||
receiptCleanup,
|
||||
executionControl,
|
||||
});
|
||||
await schedulerLifecycle.runOnce();
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'recovered',
|
||||
runRecovery,
|
||||
workflowTaskRecovery,
|
||||
receiptCleanup,
|
||||
executionControl,
|
||||
});
|
||||
|
||||
executionControlLifecycle.start();
|
||||
schedulerLifecycle.start();
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'lifecycles_started',
|
||||
runRecovery,
|
||||
workflowTaskRecovery,
|
||||
receiptCleanup,
|
||||
executionControl,
|
||||
});
|
||||
|
||||
if (options.productSurface) {
|
||||
const [stepRuns, runCancellation, taskStart] = await Promise.all([
|
||||
storage.stepRunReader(),
|
||||
storage.runCancellationRepository(),
|
||||
storage.taskStartRepository(),
|
||||
]);
|
||||
productSurfaceLifecycle = await options.productSurface.start(
|
||||
Object.freeze({
|
||||
profile: options.profile,
|
||||
runs: storage.runs,
|
||||
stepRuns,
|
||||
runCancellation,
|
||||
taskStart,
|
||||
taskDefinitions: storage.taskDefinitions,
|
||||
apiCredentials: storage.apiCredentials,
|
||||
ownerPepper: storage.ownerPepper,
|
||||
projectPolicy: storage.projectPolicy,
|
||||
securityAudit: storage.securityAudit,
|
||||
}),
|
||||
);
|
||||
if (
|
||||
!productSurfaceLifecycle ||
|
||||
typeof productSurfaceLifecycle.stopAndDrain !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local application product surface lifecycle is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'active',
|
||||
runRecovery,
|
||||
workflowTaskRecovery,
|
||||
receiptCleanup,
|
||||
executionControl,
|
||||
});
|
||||
|
||||
let stopPromise: Promise<LocalApplicationStopResult> | undefined;
|
||||
const activeStorage = storage;
|
||||
const activeSchedulerLifecycle = schedulerLifecycle;
|
||||
const activeExecutionControlLifecycle = executionControlLifecycle;
|
||||
const activeRunRecovery = runRecovery;
|
||||
const activeWorkflowTaskRecovery = workflowTaskRecovery;
|
||||
const activeReceiptCleanup = receiptCleanup;
|
||||
const activeExecutionControl = executionControl;
|
||||
const activePluginPackageStartup = pluginPackageStartup;
|
||||
const activeProductSurfaceLifecycle = productSurfaceLifecycle;
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
profile: options.profile,
|
||||
evidence: activeStorage.evidence,
|
||||
runs: activeStorage.runs,
|
||||
runRecovery: activeRunRecovery,
|
||||
workflowTaskRecovery: activeWorkflowTaskRecovery,
|
||||
receiptCleanup: activeReceiptCleanup,
|
||||
executionControl: activeExecutionControl,
|
||||
pluginPackageRecovery: activePluginPackageStartup.pluginPackageRecovery,
|
||||
pluginPackageTaskPublicationRecovery:
|
||||
activePluginPackageStartup.pluginPackageTaskPublicationRecovery,
|
||||
pluginPackageAutomationPublicationRecovery:
|
||||
activePluginPackageStartup.pluginPackageAutomationPublicationRecovery,
|
||||
pluginPackageToolSnapshotRecovery:
|
||||
activePluginPackageStartup.pluginPackageToolSnapshotRecovery,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
const errors: unknown[] = [];
|
||||
let timedOut = false;
|
||||
await bestEffortAudit(options, {
|
||||
profile: options.profile,
|
||||
state: 'draining',
|
||||
runRecovery: activeRunRecovery,
|
||||
workflowTaskRecovery: activeWorkflowTaskRecovery,
|
||||
receiptCleanup: activeReceiptCleanup,
|
||||
executionControl: activeExecutionControl,
|
||||
});
|
||||
if (activeProductSurfaceLifecycle) {
|
||||
try {
|
||||
const surfaceStop =
|
||||
await activeProductSurfaceLifecycle.stopAndDrain();
|
||||
timedOut = surfaceStop === 'timed_out' || timedOut;
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const schedulerStop = await activeSchedulerLifecycle.stopAndDrain();
|
||||
timedOut = schedulerStop.status === 'timed_out' || timedOut;
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
try {
|
||||
const executionStop =
|
||||
await activeExecutionControlLifecycle.stopAndDrain();
|
||||
timedOut = executionStop.status === 'timed_out' || timedOut;
|
||||
await bestEffortAudit(options, {
|
||||
profile: options.profile,
|
||||
state: 'draining',
|
||||
runRecovery: activeRunRecovery,
|
||||
workflowTaskRecovery: activeWorkflowTaskRecovery,
|
||||
receiptCleanup: executionStop.cleanup ?? activeReceiptCleanup,
|
||||
executionControl: activeExecutionControl,
|
||||
...(executionStop.drain === undefined
|
||||
? {}
|
||||
: { executionDrain: executionStop.drain }),
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
try {
|
||||
await activeStorage.stop();
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
await bestEffortAudit(options, {
|
||||
profile: options.profile,
|
||||
state: 'stopped',
|
||||
runRecovery: activeRunRecovery,
|
||||
workflowTaskRecovery: activeWorkflowTaskRecovery,
|
||||
receiptCleanup: activeReceiptCleanup,
|
||||
executionControl: activeExecutionControl,
|
||||
});
|
||||
if (errors.length > 0) {
|
||||
throw oneOrAggregate(errors, 'Local application stop failed');
|
||||
}
|
||||
return timedOut ? 'timed_out' : 'stopped';
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const cleanupErrors: unknown[] = [];
|
||||
if (productSurfaceLifecycle) {
|
||||
try {
|
||||
await productSurfaceLifecycle.stopAndDrain();
|
||||
} catch (cleanupError) {
|
||||
cleanupErrors.push(cleanupError);
|
||||
}
|
||||
}
|
||||
if (schedulerLifecycle) {
|
||||
try {
|
||||
await schedulerLifecycle.stopAndDrain();
|
||||
} catch (cleanupError) {
|
||||
cleanupErrors.push(cleanupError);
|
||||
}
|
||||
}
|
||||
if (executionControlLifecycle) {
|
||||
try {
|
||||
await executionControlLifecycle.stopAndDrain();
|
||||
} catch (cleanupError) {
|
||||
cleanupErrors.push(cleanupError);
|
||||
}
|
||||
}
|
||||
if (storage) {
|
||||
try {
|
||||
await storage.stop();
|
||||
} catch (cleanupError) {
|
||||
cleanupErrors.push(cleanupError);
|
||||
}
|
||||
}
|
||||
await bestEffortAudit(options, {
|
||||
profile: options.profile,
|
||||
state: 'failed',
|
||||
...(runRecovery ? { runRecovery } : {}),
|
||||
...(workflowTaskRecovery ? { workflowTaskRecovery } : {}),
|
||||
...(receiptCleanup ? { receiptCleanup } : {}),
|
||||
...(executionControl ? { executionControl } : {}),
|
||||
});
|
||||
if (cleanupErrors.length > 0) {
|
||||
throw new AggregateError(
|
||||
[error, ...cleanupErrors],
|
||||
'Local application activation failed and cleanup was incomplete',
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,727 @@
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
import type {
|
||||
ActiveModelGatewayCapability,
|
||||
ModelGatewayProfileAudit,
|
||||
ModelGatewayProviderAuthority,
|
||||
} from '@qinglong/ai/profile';
|
||||
import type { LocalModelInvocationFeatureTransition } from '@qinglong/ai/local-feature-activation';
|
||||
import type { PluginPackagePromptExecutor } from '@qinglong/ai/plugin-package-prompt-executor';
|
||||
import type {
|
||||
PluginPackagePromptOutputArtifactKeyProvider,
|
||||
PluginPackagePromptOutputArtifactReadAuthorizer,
|
||||
} from '@qinglong/ai/plugin-package-prompt-output-artifact';
|
||||
import type { PluginPackagePromptOutputCompletionCapability } from '@qinglong/ai/plugin-package-prompt-output-completion';
|
||||
import type {
|
||||
PluginPackagePromptOutputArtifactRetentionStateReader,
|
||||
PluginPackagePromptOutputReadService,
|
||||
} from '@qinglong/ai/plugin-package-prompt-output-read';
|
||||
import type { PluginPackagePromptExecutionOutputReadService } from '@qinglong/ai/plugin-package-prompt-execution-output-read';
|
||||
|
||||
import { bootstrapLocalApplication } from './activation';
|
||||
import type {
|
||||
LocalApplicationBootstrapOptions,
|
||||
LocalApplicationBootstrapResult,
|
||||
LocalApplicationProfile,
|
||||
LocalApplicationStopResult,
|
||||
} from './contract';
|
||||
|
||||
const MIN_DRAIN_TIMEOUT_MS = 100;
|
||||
const MAX_DRAIN_TIMEOUT_MS = 60_000;
|
||||
const MIN_DRAIN_POLL_MS = 10;
|
||||
const MAX_DRAIN_POLL_MS = 1_000;
|
||||
|
||||
export const LOCAL_AI_FEATURE_APPLICATION_STATES = [
|
||||
'application_disabled',
|
||||
'deployment_excluded',
|
||||
'schema_absent',
|
||||
'feature_inactive',
|
||||
'feature_active',
|
||||
'storage_ready',
|
||||
'recovery_ready',
|
||||
'active',
|
||||
'draining',
|
||||
'drain_timed_out',
|
||||
'stopped',
|
||||
'failed',
|
||||
] as const;
|
||||
|
||||
export type LocalAiFeatureApplicationState =
|
||||
(typeof LOCAL_AI_FEATURE_APPLICATION_STATES)[number];
|
||||
|
||||
export interface LocalAiFeatureApplicationAudit {
|
||||
readonly profile: LocalApplicationProfile;
|
||||
readonly state: LocalAiFeatureApplicationState;
|
||||
readonly generation?: number;
|
||||
readonly recovered?: number;
|
||||
readonly alreadyCompleted?: number;
|
||||
}
|
||||
|
||||
export type LocalAiFeatureDeploymentOptions =
|
||||
| Readonly<{
|
||||
deployment: 'excluded';
|
||||
audit: (
|
||||
record: Readonly<LocalAiFeatureApplicationAudit>,
|
||||
) => void | Promise<void>;
|
||||
}>
|
||||
| Readonly<{
|
||||
deployment: 'installed';
|
||||
loadProviders: () => Promise<ModelGatewayProviderAuthority>;
|
||||
audit: (
|
||||
record: Readonly<LocalAiFeatureApplicationAudit>,
|
||||
) => void | Promise<void>;
|
||||
maxConcurrent?: number;
|
||||
recoveryLimit?: number;
|
||||
drainTimeoutMs?: number;
|
||||
drainPollMs?: number;
|
||||
now?: () => number;
|
||||
promptOutputKeys?: PluginPackagePromptOutputArtifactKeyProvider;
|
||||
promptOutputRead?: Readonly<{
|
||||
authorizer: PluginPackagePromptOutputArtifactReadAuthorizer;
|
||||
retention: PluginPackagePromptOutputArtifactRetentionStateReader;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export interface BootstrapLocalAiFeatureApplicationOptions {
|
||||
readonly application: LocalApplicationBootstrapOptions;
|
||||
readonly ai: LocalAiFeatureDeploymentOptions;
|
||||
}
|
||||
|
||||
type ActiveLocalApplication = Extract<
|
||||
LocalApplicationBootstrapResult,
|
||||
{ status: 'active' }
|
||||
>;
|
||||
|
||||
export type LocalAiFeatureStartupResult =
|
||||
| Readonly<{
|
||||
status: 'deployment_excluded' | 'schema_absent';
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'inactive';
|
||||
generation: number;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'active';
|
||||
generation: number;
|
||||
capability: ActiveModelGatewayCapability;
|
||||
prompts: PluginPackagePromptExecutor;
|
||||
promptOutputs?: PluginPackagePromptOutputReadService;
|
||||
promptExecutionOutputs?: PluginPackagePromptExecutionOutputReadService;
|
||||
}>;
|
||||
|
||||
export type BootstrapLocalAiFeatureApplicationResult =
|
||||
| Readonly<{
|
||||
status: 'disabled';
|
||||
profile: LocalApplicationProfile;
|
||||
ai: Readonly<{ status: 'application_disabled' }>;
|
||||
stop(): Promise<'stopped'>;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'active';
|
||||
profile: LocalApplicationProfile;
|
||||
application: Readonly<Omit<ActiveLocalApplication, 'stop'>>;
|
||||
ai: LocalAiFeatureStartupResult;
|
||||
stop(): Promise<LocalApplicationStopResult>;
|
||||
}>;
|
||||
|
||||
export class LocalAiFeatureApplicationUnavailableError extends Error {
|
||||
readonly code = 'LOCAL_AI_FEATURE_APPLICATION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('The local AI feature application is unavailable', options);
|
||||
this.name = 'LocalAiFeatureApplicationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactObject(
|
||||
value: unknown,
|
||||
expectedKeys: readonly string[],
|
||||
label: string,
|
||||
): asserts value is Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !== [...expectedKeys].sort().join('\0')
|
||||
) {
|
||||
throw new TypeError(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function boundedInteger(
|
||||
value: unknown,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
label: string,
|
||||
): number {
|
||||
if (value === undefined) return fallback;
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
(value as number) < minimum ||
|
||||
(value as number) > maximum
|
||||
) {
|
||||
throw new TypeError(`${label} is invalid`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function assertOptions(value: BootstrapLocalAiFeatureApplicationOptions): void {
|
||||
exactObject(value, ['ai', 'application'], 'Local AI feature application');
|
||||
const ai = value.ai;
|
||||
if (!ai || typeof ai !== 'object' || Array.isArray(ai)) {
|
||||
throw new TypeError('Local AI feature deployment is invalid');
|
||||
}
|
||||
if (ai.deployment === 'excluded') {
|
||||
exactObject(ai, ['audit', 'deployment'], 'Excluded local AI feature');
|
||||
} else if (ai.deployment === 'installed') {
|
||||
const optionalKeys = [
|
||||
'drainPollMs',
|
||||
'drainTimeoutMs',
|
||||
'maxConcurrent',
|
||||
'now',
|
||||
'promptOutputKeys',
|
||||
'promptOutputRead',
|
||||
'recoveryLimit',
|
||||
].filter((key) => Object.hasOwn(ai, key));
|
||||
exactObject(
|
||||
ai,
|
||||
['audit', 'deployment', 'loadProviders', ...optionalKeys],
|
||||
'Installed local AI feature',
|
||||
);
|
||||
if (typeof ai.loadProviders !== 'function') {
|
||||
throw new TypeError('Local AI provider loader is invalid');
|
||||
}
|
||||
boundedInteger(
|
||||
ai.drainTimeoutMs,
|
||||
5_000,
|
||||
MIN_DRAIN_TIMEOUT_MS,
|
||||
MAX_DRAIN_TIMEOUT_MS,
|
||||
'Local AI drain timeout',
|
||||
);
|
||||
boundedInteger(
|
||||
ai.drainPollMs,
|
||||
25,
|
||||
MIN_DRAIN_POLL_MS,
|
||||
MAX_DRAIN_POLL_MS,
|
||||
'Local AI drain poll interval',
|
||||
);
|
||||
if (
|
||||
ai.maxConcurrent !== undefined &&
|
||||
(!Number.isSafeInteger(ai.maxConcurrent) ||
|
||||
ai.maxConcurrent < 1 ||
|
||||
ai.maxConcurrent > 64)
|
||||
) {
|
||||
throw new TypeError('Local AI concurrency is invalid');
|
||||
}
|
||||
if (
|
||||
ai.recoveryLimit !== undefined &&
|
||||
(!Number.isSafeInteger(ai.recoveryLimit) ||
|
||||
ai.recoveryLimit < 1 ||
|
||||
ai.recoveryLimit > 128)
|
||||
) {
|
||||
throw new TypeError('Local AI recovery limit is invalid');
|
||||
}
|
||||
if (ai.now !== undefined && typeof ai.now !== 'function') {
|
||||
throw new TypeError('Local AI clock is invalid');
|
||||
}
|
||||
if (
|
||||
ai.promptOutputKeys !== undefined &&
|
||||
(!ai.promptOutputKeys ||
|
||||
typeof ai.promptOutputKeys !== 'object' ||
|
||||
typeof ai.promptOutputKeys.active !== 'function' ||
|
||||
typeof ai.promptOutputKeys.resolve !== 'function')
|
||||
) {
|
||||
throw new TypeError('Local Prompt output key provider is invalid');
|
||||
}
|
||||
if (ai.promptOutputRead !== undefined) {
|
||||
exactObject(
|
||||
ai.promptOutputRead,
|
||||
['authorizer', 'retention'],
|
||||
'Local Prompt output read capability',
|
||||
);
|
||||
if (
|
||||
ai.promptOutputKeys === undefined ||
|
||||
!ai.promptOutputRead.authorizer ||
|
||||
typeof ai.promptOutputRead.authorizer.authorize !== 'function' ||
|
||||
!ai.promptOutputRead.retention ||
|
||||
typeof ai.promptOutputRead.retention.inspect !== 'function'
|
||||
) {
|
||||
throw new TypeError('Local Prompt output read capability is invalid');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new TypeError('Local AI deployment state is invalid');
|
||||
}
|
||||
if (typeof ai.audit !== 'function') {
|
||||
throw new TypeError('Local AI application audit sink is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async function bestEffortAudit(
|
||||
audit: LocalAiFeatureDeploymentOptions['audit'],
|
||||
record: Readonly<LocalAiFeatureApplicationAudit>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Diagnostics cannot replace activation or shutdown results.
|
||||
}
|
||||
}
|
||||
|
||||
interface RawFeatureHead {
|
||||
readonly state: 'schema_absent' | 'inactive' | 'active';
|
||||
readonly generation: number;
|
||||
readonly transitionDigest: string | null;
|
||||
}
|
||||
|
||||
function rawFeatureHead(client: {
|
||||
prepare(sql: string): {
|
||||
all(...values: unknown[]): Record<string, unknown>[];
|
||||
get(...values: unknown[]): Record<string, unknown> | undefined;
|
||||
};
|
||||
}): Readonly<RawFeatureHead> {
|
||||
const schema = client
|
||||
.prepare(
|
||||
`SELECT name
|
||||
FROM sqlite_schema
|
||||
WHERE type = 'table'
|
||||
AND name IN (
|
||||
'ModelInvocationFeatureHead',
|
||||
'ModelInvocationFeatureTransitions'
|
||||
)
|
||||
ORDER BY name`,
|
||||
)
|
||||
.all();
|
||||
if (schema.length === 0) {
|
||||
return Object.freeze({
|
||||
state: 'schema_absent',
|
||||
generation: 0,
|
||||
transitionDigest: null,
|
||||
});
|
||||
}
|
||||
if (
|
||||
schema.length !== 2 ||
|
||||
schema[0]?.name !== 'ModelInvocationFeatureHead' ||
|
||||
schema[1]?.name !== 'ModelInvocationFeatureTransitions'
|
||||
) {
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
const rows = client
|
||||
.prepare(
|
||||
`SELECT generation, state, transition_digest AS "transitionDigest"
|
||||
FROM "ModelInvocationFeatureHead"
|
||||
WHERE feature_id = 'model-invocation'
|
||||
LIMIT 2`,
|
||||
)
|
||||
.all();
|
||||
if (rows.length === 0) {
|
||||
return Object.freeze({
|
||||
state: 'schema_absent',
|
||||
generation: 0,
|
||||
transitionDigest: null,
|
||||
});
|
||||
}
|
||||
const row = rows[0];
|
||||
if (
|
||||
rows.length !== 1 ||
|
||||
!row ||
|
||||
!Number.isSafeInteger(row.generation) ||
|
||||
(row.generation as number) < 1 ||
|
||||
(row.state !== 'active' && row.state !== 'inactive') ||
|
||||
typeof row.transitionDigest !== 'string' ||
|
||||
!/^[0-9a-f]{64}$/.test(row.transitionDigest)
|
||||
) {
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
return Object.freeze({
|
||||
state: row.state,
|
||||
generation: row.generation as number,
|
||||
transitionDigest: row.transitionDigest,
|
||||
});
|
||||
}
|
||||
|
||||
function sameActivation(
|
||||
expected: Readonly<LocalModelInvocationFeatureTransition>,
|
||||
observed: Readonly<LocalModelInvocationFeatureTransition> | null,
|
||||
): boolean {
|
||||
return (
|
||||
observed?.state === 'active' &&
|
||||
observed.generation === expected.generation &&
|
||||
observed.transitionDigest === expected.transitionDigest
|
||||
);
|
||||
}
|
||||
|
||||
function applicationView(
|
||||
application: ActiveLocalApplication,
|
||||
): Readonly<Omit<ActiveLocalApplication, 'stop'>> {
|
||||
const { stop: _stop, ...view } = application;
|
||||
void _stop;
|
||||
return Object.freeze(view);
|
||||
}
|
||||
|
||||
function oneOrAggregate(errors: unknown[], message: string): unknown {
|
||||
return errors.length === 1 ? errors[0] : new AggregateError(errors, message);
|
||||
}
|
||||
|
||||
function delay(milliseconds: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, milliseconds);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Product composition for an optional local AI deployment. The base
|
||||
* application remains AI-free. Only an installed deployment with a durable
|
||||
* active head imports the AI runtime, recovers it, and reaches providers.
|
||||
*/
|
||||
export async function bootstrapLocalAiFeatureApplication(
|
||||
options: BootstrapLocalAiFeatureApplicationOptions,
|
||||
): Promise<BootstrapLocalAiFeatureApplicationResult> {
|
||||
assertOptions(options);
|
||||
const application = await bootstrapLocalApplication(options.application);
|
||||
if (application.status === 'disabled') {
|
||||
await options.ai.audit({
|
||||
profile: application.profile,
|
||||
state: 'application_disabled',
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'disabled',
|
||||
profile: application.profile,
|
||||
ai: Object.freeze({ status: 'application_disabled' as const }),
|
||||
stop: application.stop,
|
||||
});
|
||||
}
|
||||
if (options.application.enabled !== true) {
|
||||
await application.stop();
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
const applicationOptions = options.application;
|
||||
if (options.ai.deployment === 'excluded') {
|
||||
await options.ai.audit({
|
||||
profile: application.profile,
|
||||
state: 'deployment_excluded',
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'active',
|
||||
profile: application.profile,
|
||||
application: applicationView(application),
|
||||
ai: Object.freeze({ status: 'deployment_excluded' as const }),
|
||||
stop: application.stop,
|
||||
});
|
||||
}
|
||||
|
||||
const aiOptions = options.ai;
|
||||
let featureDatabase:
|
||||
| Awaited<
|
||||
ReturnType<
|
||||
typeof import('@qinglong/local-sqlite/optional-feature-runtime')['openLocalSqliteOptionalFeatureRuntimeDatabase']
|
||||
>
|
||||
>
|
||||
| undefined;
|
||||
let featureOwnedByProfile = false;
|
||||
try {
|
||||
const { openLocalSqliteOptionalFeatureRuntimeDatabase } = await import(
|
||||
'@qinglong/local-sqlite/optional-feature-runtime'
|
||||
);
|
||||
featureDatabase = await openLocalSqliteOptionalFeatureRuntimeDatabase({
|
||||
databasePath:
|
||||
applicationOptions.storageMode === 'fresh'
|
||||
? applicationOptions.databasePath
|
||||
: applicationOptions.targetPath,
|
||||
profile: applicationOptions.profile,
|
||||
...(applicationOptions.busyTimeoutMs === undefined
|
||||
? {}
|
||||
: { busyTimeoutMs: applicationOptions.busyTimeoutMs }),
|
||||
});
|
||||
const head = rawFeatureHead(featureDatabase.authority.client);
|
||||
if (head.state !== 'active') {
|
||||
await featureDatabase.close();
|
||||
featureDatabase = undefined;
|
||||
await aiOptions.audit({
|
||||
profile: application.profile,
|
||||
state:
|
||||
head.state === 'schema_absent' ? 'schema_absent' : 'feature_inactive',
|
||||
...(head.generation === 0 ? {} : { generation: head.generation }),
|
||||
});
|
||||
const ai: LocalAiFeatureStartupResult =
|
||||
head.state === 'schema_absent'
|
||||
? Object.freeze({ status: 'schema_absent' as const })
|
||||
: Object.freeze({
|
||||
status: 'inactive' as const,
|
||||
generation: head.generation,
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'active',
|
||||
profile: application.profile,
|
||||
application: applicationView(application),
|
||||
ai,
|
||||
stop: application.stop,
|
||||
});
|
||||
}
|
||||
|
||||
const [
|
||||
{ LocalModelInvocationFeatureActivationRepository },
|
||||
{ LocalModelInvocationRepository },
|
||||
{ LocalModelPriceCatalogRepository },
|
||||
{ bootstrapModelGatewayProfile },
|
||||
{ LocalPluginPackagePromptAdmissionRepository },
|
||||
{ PluginPackagePromptExecutor },
|
||||
{ PluginPackagePromptOutputCompletionCoordinator },
|
||||
] = await Promise.all([
|
||||
import('@qinglong/ai/local-feature-activation'),
|
||||
import('@qinglong/ai/local-model-invocation-storage'),
|
||||
import('@qinglong/ai/local-price-catalog-storage'),
|
||||
import('@qinglong/ai/profile'),
|
||||
import('@qinglong/ai/local-plugin-package-prompt-admission-storage'),
|
||||
import('@qinglong/ai/plugin-package-prompt-executor'),
|
||||
import('@qinglong/ai/plugin-package-prompt-output-completion'),
|
||||
]);
|
||||
const activationRepository =
|
||||
new LocalModelInvocationFeatureActivationRepository(
|
||||
featureDatabase.authority.client,
|
||||
);
|
||||
const activation = activationRepository.findCurrent();
|
||||
if (
|
||||
!activation ||
|
||||
activation.state !== 'active' ||
|
||||
activation.generation !== head.generation ||
|
||||
activation.transitionDigest !== head.transitionDigest
|
||||
) {
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
await aiOptions.audit({
|
||||
profile: application.profile,
|
||||
state: 'feature_active',
|
||||
generation: activation.generation,
|
||||
});
|
||||
const repository = new LocalModelInvocationRepository(
|
||||
featureDatabase.authority,
|
||||
);
|
||||
const pricing = new LocalModelPriceCatalogRepository(
|
||||
featureDatabase.authority,
|
||||
);
|
||||
const activeDatabase = featureDatabase;
|
||||
let durableOutput:
|
||||
| PluginPackagePromptOutputCompletionCapability
|
||||
| undefined;
|
||||
const gateway = await bootstrapModelGatewayProfile({
|
||||
enabled: true,
|
||||
profile: application.profile,
|
||||
loadStorage: async () => {
|
||||
featureOwnedByProfile = true;
|
||||
return Object.freeze({
|
||||
repository,
|
||||
pricing,
|
||||
close: () => activeDatabase.close(),
|
||||
});
|
||||
},
|
||||
loadProviders: aiOptions.loadProviders,
|
||||
...(aiOptions.promptOutputKeys === undefined
|
||||
? {}
|
||||
: {
|
||||
createSuccessfulCompletion: (coordinator) => {
|
||||
durableOutput =
|
||||
new PluginPackagePromptOutputCompletionCoordinator({
|
||||
coordinator,
|
||||
keys: aiOptions.promptOutputKeys!,
|
||||
...(aiOptions.now === undefined ? {} : { now: aiOptions.now }),
|
||||
});
|
||||
return durableOutput;
|
||||
},
|
||||
}),
|
||||
confirmActive: async () => {
|
||||
await activeDatabase.authority.enqueue(
|
||||
async () => {
|
||||
if (
|
||||
!sameActivation(activation, activationRepository.findCurrent())
|
||||
) {
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
},
|
||||
() => new LocalAiFeatureApplicationUnavailableError(),
|
||||
);
|
||||
},
|
||||
audit: async (record: Readonly<ModelGatewayProfileAudit>) => {
|
||||
if (record.state === 'disabled') {
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
await aiOptions.audit({
|
||||
profile: application.profile,
|
||||
state: record.state,
|
||||
generation: activation.generation,
|
||||
...(record.recovered === undefined
|
||||
? {}
|
||||
: { recovered: record.recovered }),
|
||||
...(record.alreadyCompleted === undefined
|
||||
? {}
|
||||
: { alreadyCompleted: record.alreadyCompleted }),
|
||||
});
|
||||
},
|
||||
...(aiOptions.maxConcurrent === undefined
|
||||
? {}
|
||||
: { maxConcurrent: aiOptions.maxConcurrent }),
|
||||
...(aiOptions.recoveryLimit === undefined
|
||||
? {}
|
||||
: { recoveryLimit: aiOptions.recoveryLimit }),
|
||||
...(aiOptions.now === undefined ? {} : { now: aiOptions.now }),
|
||||
});
|
||||
if (gateway.status !== 'active') {
|
||||
throw new LocalAiFeatureApplicationUnavailableError();
|
||||
}
|
||||
const capability = gateway.capability;
|
||||
const prompts = new PluginPackagePromptExecutor({
|
||||
admissions: new LocalPluginPackagePromptAdmissionRepository(
|
||||
activeDatabase.authority,
|
||||
),
|
||||
invocations: repository,
|
||||
gateway: capability,
|
||||
...(durableOutput === undefined ? {} : { durableOutput }),
|
||||
});
|
||||
let promptOutputs: PluginPackagePromptOutputReadService | undefined;
|
||||
let promptExecutionOutputs:
|
||||
| PluginPackagePromptExecutionOutputReadService
|
||||
| undefined;
|
||||
if (aiOptions.promptOutputRead !== undefined) {
|
||||
const [
|
||||
{ LocalPluginPackagePromptOutputArtifactRepository },
|
||||
{ PluginPackagePromptOutputReadService },
|
||||
{ PluginPackagePromptExecutionOutputReadService },
|
||||
{ LocalPluginPackagePromptExecutionOutputReferenceRepository },
|
||||
] = await Promise.all([
|
||||
import(
|
||||
'@qinglong/ai/local-plugin-package-prompt-output-artifact-storage'
|
||||
),
|
||||
import('@qinglong/ai/plugin-package-prompt-output-read'),
|
||||
import('@qinglong/ai/plugin-package-prompt-execution-output-read'),
|
||||
import(
|
||||
'@qinglong/ai/local-plugin-package-prompt-execution-output-reference-storage'
|
||||
),
|
||||
]);
|
||||
promptOutputs = new PluginPackagePromptOutputReadService({
|
||||
artifacts: new LocalPluginPackagePromptOutputArtifactRepository(
|
||||
activeDatabase.authority,
|
||||
),
|
||||
authorizer: aiOptions.promptOutputRead.authorizer,
|
||||
retention: aiOptions.promptOutputRead.retention,
|
||||
keys: aiOptions.promptOutputKeys!,
|
||||
...(aiOptions.now === undefined ? {} : { now: aiOptions.now }),
|
||||
});
|
||||
promptExecutionOutputs =
|
||||
new PluginPackagePromptExecutionOutputReadService({
|
||||
references:
|
||||
new LocalPluginPackagePromptExecutionOutputReferenceRepository(
|
||||
activeDatabase.authority,
|
||||
),
|
||||
outputs: promptOutputs,
|
||||
});
|
||||
}
|
||||
const drainTimeoutMs = boundedInteger(
|
||||
aiOptions.drainTimeoutMs,
|
||||
5_000,
|
||||
MIN_DRAIN_TIMEOUT_MS,
|
||||
MAX_DRAIN_TIMEOUT_MS,
|
||||
'Local AI drain timeout',
|
||||
);
|
||||
const drainPollMs = boundedInteger(
|
||||
aiOptions.drainPollMs,
|
||||
25,
|
||||
MIN_DRAIN_POLL_MS,
|
||||
MAX_DRAIN_POLL_MS,
|
||||
'Local AI drain poll interval',
|
||||
);
|
||||
let stopPromise: Promise<LocalApplicationStopResult> | undefined;
|
||||
return Object.freeze({
|
||||
status: 'active',
|
||||
profile: application.profile,
|
||||
application: applicationView(application),
|
||||
ai: Object.freeze({
|
||||
status: 'active' as const,
|
||||
generation: activation.generation,
|
||||
capability,
|
||||
prompts,
|
||||
...(promptOutputs === undefined ? {} : { promptOutputs }),
|
||||
...(promptExecutionOutputs === undefined
|
||||
? {}
|
||||
: { promptExecutionOutputs }),
|
||||
}),
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
const errors: unknown[] = [];
|
||||
let timedOut = false;
|
||||
await bestEffortAudit(aiOptions.audit, {
|
||||
profile: application.profile,
|
||||
state: 'draining',
|
||||
generation: activation.generation,
|
||||
});
|
||||
try {
|
||||
const deadline = performance.now() + drainTimeoutMs;
|
||||
let result = await capability.stop();
|
||||
while (result === 'draining' && performance.now() < deadline) {
|
||||
await delay(
|
||||
Math.min(
|
||||
drainPollMs,
|
||||
Math.max(1, deadline - performance.now()),
|
||||
),
|
||||
);
|
||||
result = await capability.stop();
|
||||
}
|
||||
timedOut = result === 'draining';
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
try {
|
||||
timedOut = (await application.stop()) === 'timed_out' || timedOut;
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
await bestEffortAudit(aiOptions.audit, {
|
||||
profile: application.profile,
|
||||
state: timedOut ? 'drain_timed_out' : 'stopped',
|
||||
generation: activation.generation,
|
||||
});
|
||||
if (errors.length > 0) {
|
||||
throw oneOrAggregate(
|
||||
errors,
|
||||
'Local AI feature application stop failed',
|
||||
);
|
||||
}
|
||||
return timedOut ? 'timed_out' : 'stopped';
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const cleanupErrors: unknown[] = [];
|
||||
if (featureDatabase && !featureOwnedByProfile) {
|
||||
try {
|
||||
await featureDatabase.close();
|
||||
} catch (cleanupError) {
|
||||
cleanupErrors.push(cleanupError);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await application.stop();
|
||||
} catch (cleanupError) {
|
||||
cleanupErrors.push(cleanupError);
|
||||
}
|
||||
await bestEffortAudit(aiOptions.audit, {
|
||||
profile: application.profile,
|
||||
state: 'failed',
|
||||
});
|
||||
const cause =
|
||||
error instanceof LocalAiFeatureApplicationUnavailableError
|
||||
? error
|
||||
: new LocalAiFeatureApplicationUnavailableError({
|
||||
cause: error instanceof Error ? error : undefined,
|
||||
});
|
||||
if (cleanupErrors.length > 0) {
|
||||
throw new AggregateError(
|
||||
[cause, ...cleanupErrors],
|
||||
'Local AI feature activation failed and cleanup was incomplete',
|
||||
);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
import type {
|
||||
LocalAdoptedProfileBootstrapOptions,
|
||||
LocalAdoptedProfileBootstrapResult,
|
||||
} from '@qinglong/local-admin/adopted-profile';
|
||||
import type {
|
||||
LocalProfileStorageAudit,
|
||||
LocalProfileStorageBootstrapResult,
|
||||
} from '@qinglong/local-sqlite/profile';
|
||||
import type { LocalRunStartupRecoverySummary } from '@qinglong/local-execution/recovery';
|
||||
import type { LocalWorkflowTaskStartupRecoverySummary } from '@qinglong/local-execution/recovery';
|
||||
import type { LocalCompletionReceiptCleanupSummary } from '@qinglong/local-process';
|
||||
import type {
|
||||
LocalExecutionControlCycleSummary,
|
||||
LocalExecutionDrainSummary,
|
||||
} from '@qinglong/local-execution/control';
|
||||
import type { PluginPackageStageProvider } from '@qinglong/runtime-core/plugin-package-installation';
|
||||
import type { PluginPackageRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-recovery';
|
||||
import type { PluginPackageAutomationPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-automation-publication';
|
||||
import type { PluginPackageTaskPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-task-publication';
|
||||
import type { ProjectToolDefinitionSnapshotRecoveryCycleResult } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
|
||||
export type LocalApplicationProfile = 'edge' | 'standalone';
|
||||
|
||||
export type LocalApplicationActivationState =
|
||||
| 'disabled'
|
||||
| 'storage_ready'
|
||||
| 'plugin_packages_recovered'
|
||||
| 'plugin_package_tasks_published'
|
||||
| 'plugin_package_automations_published'
|
||||
| 'plugin_package_tools_snapshotted'
|
||||
| 'secrets_ready'
|
||||
| 'runs_recovered'
|
||||
| 'receipts_reconciled'
|
||||
| 'execution_control_degraded'
|
||||
| 'scheduler_degraded'
|
||||
| 'recovered'
|
||||
| 'lifecycles_started'
|
||||
| 'active'
|
||||
| 'draining'
|
||||
| 'failed'
|
||||
| 'stopped';
|
||||
|
||||
export type LocalApplicationStopResult = 'stopped' | 'timed_out';
|
||||
|
||||
export interface LocalApplicationProductSurfaceAuthority {
|
||||
readonly profile: LocalApplicationProfile;
|
||||
readonly runs: Pick<
|
||||
ReadyFreshStorage['runs'],
|
||||
'findRunById' | 'listEvents' | 'listRunsByProject'
|
||||
>;
|
||||
readonly stepRuns: Awaited<ReturnType<ReadyFreshStorage['stepRunReader']>>;
|
||||
readonly runCancellation: Awaited<
|
||||
ReturnType<ReadyFreshStorage['runCancellationRepository']>
|
||||
>;
|
||||
readonly taskStart: Awaited<
|
||||
ReturnType<ReadyFreshStorage['taskStartRepository']>
|
||||
>;
|
||||
readonly taskDefinitions: Pick<
|
||||
ReadyFreshStorage['taskDefinitions'],
|
||||
'findCurrentTaskDefinition' | 'listTaskDefinitions'
|
||||
>;
|
||||
readonly apiCredentials: ReadyFreshStorage['apiCredentials'];
|
||||
readonly ownerPepper: ReadyFreshStorage['ownerPepper'];
|
||||
readonly projectPolicy: ReadyFreshStorage['projectPolicy'];
|
||||
readonly securityAudit: ReadyFreshStorage['securityAudit'];
|
||||
}
|
||||
|
||||
export interface LocalApplicationProductSurfaceLifecycle {
|
||||
stopAndDrain(): Promise<LocalApplicationStopResult>;
|
||||
}
|
||||
|
||||
export interface LocalApplicationProductSurface {
|
||||
start(
|
||||
authority: Readonly<LocalApplicationProductSurfaceAuthority>,
|
||||
): Promise<Readonly<LocalApplicationProductSurfaceLifecycle>>;
|
||||
}
|
||||
|
||||
export interface LocalApplicationPluginPackageRecoveryOptions {
|
||||
readonly stageProvider: PluginPackageStageProvider;
|
||||
readonly stagingRoot: string;
|
||||
readonly activationRoot: string;
|
||||
readonly now: () => number;
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly taskPublicationPageSize?: number;
|
||||
readonly taskPublicationMaxPages?: number;
|
||||
}
|
||||
|
||||
type ReadyAdoptedStorage = Extract<
|
||||
LocalAdoptedProfileBootstrapResult,
|
||||
{ status: 'adopted_storage_ready' }
|
||||
>;
|
||||
type ReadyFreshStorage = Extract<
|
||||
LocalProfileStorageBootstrapResult,
|
||||
{ status: 'storage_ready' }
|
||||
>;
|
||||
|
||||
export interface LocalApplicationActivationAudit {
|
||||
readonly profile: LocalApplicationProfile;
|
||||
readonly state: LocalApplicationActivationState;
|
||||
readonly runRecovery?: LocalRunStartupRecoverySummary;
|
||||
readonly workflowTaskRecovery?: LocalWorkflowTaskStartupRecoverySummary;
|
||||
readonly receiptCleanup?: LocalCompletionReceiptCleanupSummary;
|
||||
readonly executionControl?: LocalExecutionControlCycleSummary;
|
||||
readonly executionDrain?: LocalExecutionDrainSummary;
|
||||
readonly pluginPackageRecovery?: PluginPackageRecoveryCycleResult;
|
||||
readonly pluginPackageTaskPublicationRecovery?: PluginPackageTaskPublicationRecoveryCycleResult;
|
||||
readonly pluginPackageAutomationPublicationRecovery?: PluginPackageAutomationPublicationRecoveryCycleResult;
|
||||
readonly pluginPackageToolSnapshotRecovery?: ProjectToolDefinitionSnapshotRecoveryCycleResult;
|
||||
}
|
||||
|
||||
export interface LocalApplicationDisabledBootstrapOptions {
|
||||
readonly enabled?: false;
|
||||
readonly profile: LocalApplicationProfile;
|
||||
readonly applicationAudit: (
|
||||
record: LocalApplicationActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
interface LocalApplicationEnabledBootstrapCommon {
|
||||
readonly enabled: true;
|
||||
readonly profile: LocalApplicationProfile;
|
||||
readonly receiptRoot: string;
|
||||
readonly artifactRoot: string;
|
||||
readonly secretKeyringPath: string;
|
||||
readonly pluginPackages: LocalApplicationPluginPackageRecoveryOptions;
|
||||
readonly productSurface?: LocalApplicationProductSurface;
|
||||
readonly applicationAudit: (
|
||||
record: LocalApplicationActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalApplicationAdoptedBootstrapOptions
|
||||
extends LocalApplicationEnabledBootstrapCommon,
|
||||
Omit<LocalAdoptedProfileBootstrapOptions, 'enabled' | 'profile'> {
|
||||
readonly storageMode?: 'adopted';
|
||||
}
|
||||
|
||||
export interface LocalApplicationFreshBootstrapOptions
|
||||
extends LocalApplicationEnabledBootstrapCommon {
|
||||
readonly storageMode: 'fresh';
|
||||
readonly databasePath: string;
|
||||
readonly busyTimeoutMs?: number;
|
||||
readonly audit: (record: LocalProfileStorageAudit) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export type LocalApplicationEnabledBootstrapOptions =
|
||||
| LocalApplicationAdoptedBootstrapOptions
|
||||
| LocalApplicationFreshBootstrapOptions;
|
||||
|
||||
export type LocalApplicationBootstrapOptions =
|
||||
| LocalApplicationDisabledBootstrapOptions
|
||||
| LocalApplicationEnabledBootstrapOptions;
|
||||
|
||||
export type LocalApplicationBootstrapResult =
|
||||
| {
|
||||
readonly status: 'disabled';
|
||||
readonly profile: LocalApplicationProfile;
|
||||
stop(): Promise<'stopped'>;
|
||||
}
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly profile: LocalApplicationProfile;
|
||||
readonly evidence:
|
||||
| ReadyAdoptedStorage['evidence']
|
||||
| ReadyFreshStorage['evidence'];
|
||||
readonly runs: ReadyAdoptedStorage['runs'] | ReadyFreshStorage['runs'];
|
||||
readonly runRecovery: LocalRunStartupRecoverySummary;
|
||||
readonly workflowTaskRecovery: LocalWorkflowTaskStartupRecoverySummary;
|
||||
readonly receiptCleanup: LocalCompletionReceiptCleanupSummary;
|
||||
readonly executionControl: LocalExecutionControlCycleSummary;
|
||||
readonly pluginPackageRecovery: PluginPackageRecoveryCycleResult;
|
||||
readonly pluginPackageTaskPublicationRecovery: PluginPackageTaskPublicationRecoveryCycleResult;
|
||||
readonly pluginPackageAutomationPublicationRecovery: PluginPackageAutomationPublicationRecoveryCycleResult;
|
||||
readonly pluginPackageToolSnapshotRecovery: ProjectToolDefinitionSnapshotRecoveryCycleResult;
|
||||
stop(): Promise<LocalApplicationStopResult>;
|
||||
};
|
||||
@@ -0,0 +1,192 @@
|
||||
import { LocalPluginPackageActivationPublisher } from '@qinglong/local-admin/package-activation';
|
||||
import { LocalPluginPackageResourceByteSource } from '@qinglong/local-admin/package-resource-materialization';
|
||||
import {
|
||||
PluginPackageRecoveryCoordinator,
|
||||
type PluginPackageRecoveryCycleResult,
|
||||
} from '@qinglong/runtime-core/plugin-package-recovery';
|
||||
import {
|
||||
PluginPackageAutomationPublicationCoordinator,
|
||||
PluginPackageAutomationPublicationRecoveryCoordinator,
|
||||
type PluginPackageAutomationPublicationRecoveryCycleResult,
|
||||
} from '@qinglong/runtime-core/plugin-package-automation-publication';
|
||||
import {
|
||||
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGES,
|
||||
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGE_SIZE,
|
||||
PluginPackageTaskPublicationCoordinator,
|
||||
PluginPackageTaskPublicationRecoveryCoordinator,
|
||||
type PluginPackageTaskPublicationRecoveryCycleResult,
|
||||
} from '@qinglong/runtime-core/plugin-package-task-publication';
|
||||
import {
|
||||
MAX_PROJECT_TOOL_SNAPSHOT_RECOVERY_PAGES,
|
||||
MAX_PROJECT_TOOL_SNAPSHOT_SOURCE_PAGE_SIZE,
|
||||
ProjectToolDefinitionSnapshotPublicationCoordinator,
|
||||
ProjectToolDefinitionSnapshotRecoveryCoordinator,
|
||||
type ProjectToolDefinitionSnapshotRecoveryCycleResult,
|
||||
} from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
import { createBuiltInTaskSpecSemanticRegistry } from '@qinglong/runtime-core/task-spec-semantic';
|
||||
import type { LocalApplicationEnabledBootstrapOptions } from './contract';
|
||||
import type { LocalApplicationReadyStorage } from './storageActivation';
|
||||
import {
|
||||
LocalApplicationPluginPackageAutomationPublicationRequiredError,
|
||||
LocalApplicationPluginPackageRecoveryRequiredError,
|
||||
LocalApplicationPluginPackageTaskPublicationRequiredError,
|
||||
LocalApplicationPluginPackageToolSnapshotRequiredError,
|
||||
} from './startupErrors';
|
||||
|
||||
export interface LocalApplicationPluginPackageStartup {
|
||||
readonly pluginPackageRecovery: Readonly<PluginPackageRecoveryCycleResult>;
|
||||
readonly pluginPackageTaskPublicationRecovery: Readonly<PluginPackageTaskPublicationRecoveryCycleResult>;
|
||||
readonly pluginPackageAutomationPublicationRecovery: Readonly<PluginPackageAutomationPublicationRecoveryCycleResult>;
|
||||
readonly pluginPackageToolSnapshotRecovery: Readonly<ProjectToolDefinitionSnapshotRecoveryCycleResult>;
|
||||
}
|
||||
|
||||
export async function recoverLocalApplicationPluginPackages(
|
||||
options: LocalApplicationEnabledBootstrapOptions,
|
||||
storage: LocalApplicationReadyStorage,
|
||||
): Promise<LocalApplicationPluginPackageStartup> {
|
||||
const pluginPackageInstalls = await storage.pluginPackageInstalls();
|
||||
const pluginPackageActivation = new LocalPluginPackageActivationPublisher({
|
||||
stagingRoot: options.pluginPackages.stagingRoot,
|
||||
activationRoot: options.pluginPackages.activationRoot,
|
||||
now: options.pluginPackages.now,
|
||||
});
|
||||
const pluginPackageRecovery = await new PluginPackageRecoveryCoordinator({
|
||||
repository: pluginPackageInstalls,
|
||||
stageProvider: options.pluginPackages.stageProvider,
|
||||
publisher: pluginPackageActivation,
|
||||
now: options.pluginPackages.now,
|
||||
}).recover({
|
||||
...(options.pluginPackages.pageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.pluginPackages.pageSize }),
|
||||
...(options.pluginPackages.maxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.pluginPackages.maxPages }),
|
||||
});
|
||||
if (!pluginPackageRecovery.safeToAdmit) {
|
||||
throw new LocalApplicationPluginPackageRecoveryRequiredError(
|
||||
pluginPackageRecovery,
|
||||
);
|
||||
}
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'plugin_packages_recovered',
|
||||
pluginPackageRecovery,
|
||||
});
|
||||
|
||||
const taskSpecSemanticRegistry = createBuiltInTaskSpecSemanticRegistry();
|
||||
const pluginPackageTaskReconciliations =
|
||||
await storage.pluginPackageTaskReconciliations();
|
||||
const pluginPackageMaterializedRevisions =
|
||||
await storage.pluginPackageMaterializedRevisions();
|
||||
const pluginPackageTaskPublicationRecovery =
|
||||
await new PluginPackageTaskPublicationRecoveryCoordinator({
|
||||
source: pluginPackageTaskReconciliations,
|
||||
publisher: new PluginPackageTaskPublicationCoordinator({
|
||||
generationSource: pluginPackageActivation,
|
||||
lockSource: pluginPackageInstalls,
|
||||
byteSource: new LocalPluginPackageResourceByteSource({
|
||||
stagingRoot: options.pluginPackages.stagingRoot,
|
||||
}),
|
||||
materializedRepository: pluginPackageMaterializedRevisions,
|
||||
reconciliationRepository: pluginPackageTaskReconciliations,
|
||||
taskSpecSemanticRegistry,
|
||||
}),
|
||||
}).recover({
|
||||
...(options.pluginPackages.taskPublicationPageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.pluginPackages.taskPublicationPageSize }),
|
||||
...(options.pluginPackages.taskPublicationMaxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.pluginPackages.taskPublicationMaxPages }),
|
||||
});
|
||||
if (!pluginPackageTaskPublicationRecovery.safeToAdmit) {
|
||||
throw new LocalApplicationPluginPackageTaskPublicationRequiredError(
|
||||
pluginPackageTaskPublicationRecovery,
|
||||
);
|
||||
}
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'plugin_package_tasks_published',
|
||||
pluginPackageRecovery,
|
||||
pluginPackageTaskPublicationRecovery,
|
||||
});
|
||||
|
||||
const pluginPackageAutomationPublications =
|
||||
await storage.pluginPackageAutomationPublications();
|
||||
const pluginPackageAutomationPublicationRecovery =
|
||||
await new PluginPackageAutomationPublicationRecoveryCoordinator({
|
||||
source: pluginPackageAutomationPublications,
|
||||
publisher: new PluginPackageAutomationPublicationCoordinator({
|
||||
generationSource: pluginPackageActivation,
|
||||
materializedRepository: pluginPackageMaterializedRevisions,
|
||||
repository: pluginPackageAutomationPublications,
|
||||
taskSpecSemanticRegistry,
|
||||
now: options.pluginPackages.now,
|
||||
}),
|
||||
}).recover({
|
||||
...(options.pluginPackages.taskPublicationPageSize === undefined
|
||||
? {}
|
||||
: { pageSize: options.pluginPackages.taskPublicationPageSize }),
|
||||
...(options.pluginPackages.taskPublicationMaxPages === undefined
|
||||
? {}
|
||||
: { maxPages: options.pluginPackages.taskPublicationMaxPages }),
|
||||
});
|
||||
if (!pluginPackageAutomationPublicationRecovery.safeToAdmit) {
|
||||
throw new LocalApplicationPluginPackageAutomationPublicationRequiredError(
|
||||
pluginPackageAutomationPublicationRecovery,
|
||||
);
|
||||
}
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'plugin_package_automations_published',
|
||||
pluginPackageRecovery,
|
||||
pluginPackageTaskPublicationRecovery,
|
||||
pluginPackageAutomationPublicationRecovery,
|
||||
});
|
||||
|
||||
const projectToolDefinitionSnapshots =
|
||||
await storage.projectToolDefinitionSnapshots();
|
||||
const pluginPackageToolSnapshotRecovery =
|
||||
await new ProjectToolDefinitionSnapshotRecoveryCoordinator({
|
||||
source: projectToolDefinitionSnapshots,
|
||||
publisher: new ProjectToolDefinitionSnapshotPublicationCoordinator({
|
||||
source: projectToolDefinitionSnapshots,
|
||||
materializedRepository: pluginPackageMaterializedRevisions,
|
||||
repository: projectToolDefinitionSnapshots,
|
||||
taskSpecSemanticRegistry,
|
||||
pageSize: Math.min(
|
||||
options.pluginPackages.taskPublicationPageSize ??
|
||||
(options.profile === 'edge' ? 4 : 16),
|
||||
MAX_PROJECT_TOOL_SNAPSHOT_SOURCE_PAGE_SIZE,
|
||||
),
|
||||
}),
|
||||
}).recover({
|
||||
pageSize:
|
||||
options.pluginPackages.taskPublicationPageSize ??
|
||||
(options.profile === 'edge' ? 1 : 8),
|
||||
maxPages:
|
||||
options.pluginPackages.taskPublicationMaxPages ??
|
||||
MAX_PROJECT_TOOL_SNAPSHOT_RECOVERY_PAGES,
|
||||
});
|
||||
if (!pluginPackageToolSnapshotRecovery.safeToAdmit) {
|
||||
throw new LocalApplicationPluginPackageToolSnapshotRequiredError(
|
||||
pluginPackageToolSnapshotRecovery,
|
||||
);
|
||||
}
|
||||
await options.applicationAudit({
|
||||
profile: options.profile,
|
||||
state: 'plugin_package_tools_snapshotted',
|
||||
pluginPackageRecovery,
|
||||
pluginPackageTaskPublicationRecovery,
|
||||
pluginPackageAutomationPublicationRecovery,
|
||||
pluginPackageToolSnapshotRecovery,
|
||||
});
|
||||
|
||||
return Object.freeze({
|
||||
pluginPackageRecovery,
|
||||
pluginPackageTaskPublicationRecovery,
|
||||
pluginPackageAutomationPublicationRecovery,
|
||||
pluginPackageToolSnapshotRecovery,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import type { PluginPackageRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-recovery';
|
||||
import type { PluginPackageAutomationPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-automation-publication';
|
||||
import type { PluginPackageTaskPublicationRecoveryCycleResult } from '@qinglong/runtime-core/plugin-package-task-publication';
|
||||
import type { ProjectToolDefinitionSnapshotRecoveryCycleResult } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
|
||||
export class LocalApplicationStartupRecoveryRequiredError extends Error {
|
||||
constructor(
|
||||
readonly observedCandidates: number,
|
||||
readonly truncated: boolean,
|
||||
) {
|
||||
super('Local application has unresolved startup recovery candidates');
|
||||
this.name = 'LocalApplicationStartupRecoveryRequiredError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalApplicationPluginPackageRecoveryRequiredError extends Error {
|
||||
constructor(readonly recovery: Readonly<PluginPackageRecoveryCycleResult>) {
|
||||
super('Local application has unresolved Plugin Package recovery work');
|
||||
this.name = 'LocalApplicationPluginPackageRecoveryRequiredError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalApplicationPluginPackageTaskPublicationRequiredError extends Error {
|
||||
constructor(
|
||||
readonly recovery: Readonly<PluginPackageTaskPublicationRecoveryCycleResult>,
|
||||
) {
|
||||
super(
|
||||
'Local application has unresolved Plugin Package Task publication work',
|
||||
);
|
||||
this.name = 'LocalApplicationPluginPackageTaskPublicationRequiredError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalApplicationPluginPackageAutomationPublicationRequiredError extends Error {
|
||||
constructor(
|
||||
readonly recovery: Readonly<PluginPackageAutomationPublicationRecoveryCycleResult>,
|
||||
) {
|
||||
super(
|
||||
'Local application has unresolved Plugin Package Workflow/Prompt publication work',
|
||||
);
|
||||
this.name =
|
||||
'LocalApplicationPluginPackageAutomationPublicationRequiredError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalApplicationPluginPackageToolSnapshotRequiredError extends Error {
|
||||
constructor(
|
||||
readonly recovery: Readonly<ProjectToolDefinitionSnapshotRecoveryCycleResult>,
|
||||
) {
|
||||
super('Local application has unresolved Plugin Package Tool snapshot work');
|
||||
this.name = 'LocalApplicationPluginPackageToolSnapshotRequiredError';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import {
|
||||
bootstrapLocalAdoptedProfileStorage,
|
||||
type LocalAdoptedProfileBootstrapResult,
|
||||
} from '@qinglong/local-admin/adopted-profile';
|
||||
import {
|
||||
bootstrapLocalProfileStorage,
|
||||
type LocalProfileStorageBootstrapResult,
|
||||
} from '@qinglong/local-sqlite/profile';
|
||||
import type { LocalApplicationEnabledBootstrapOptions } from './contract';
|
||||
|
||||
export type LocalApplicationReadyStorage =
|
||||
| Extract<
|
||||
LocalAdoptedProfileBootstrapResult,
|
||||
{ status: 'adopted_storage_ready' }
|
||||
>
|
||||
| Extract<LocalProfileStorageBootstrapResult, { status: 'storage_ready' }>;
|
||||
|
||||
export async function openLocalApplicationStorage(
|
||||
options: LocalApplicationEnabledBootstrapOptions,
|
||||
): Promise<LocalApplicationReadyStorage> {
|
||||
const opened =
|
||||
options.storageMode === 'fresh'
|
||||
? await bootstrapLocalProfileStorage({
|
||||
enabled: true,
|
||||
profile: options.profile,
|
||||
databasePath: options.databasePath,
|
||||
...(options.busyTimeoutMs === undefined
|
||||
? {}
|
||||
: { busyTimeoutMs: options.busyTimeoutMs }),
|
||||
audit: options.audit,
|
||||
})
|
||||
: await bootstrapLocalAdoptedProfileStorage({
|
||||
enabled: true,
|
||||
profile: options.profile,
|
||||
sourcePath: options.sourcePath,
|
||||
targetPath: options.targetPath,
|
||||
recoveryPath: options.recoveryPath,
|
||||
manifestPath: options.manifestPath,
|
||||
activationPath: options.activationPath,
|
||||
expectedActivationDigest: options.expectedActivationDigest,
|
||||
...(options.busyTimeoutMs === undefined
|
||||
? {}
|
||||
: { busyTimeoutMs: options.busyTimeoutMs }),
|
||||
audit: options.audit,
|
||||
adoptionAudit: options.adoptionAudit,
|
||||
});
|
||||
if (
|
||||
opened.status !== 'adopted_storage_ready' &&
|
||||
opened.status !== 'storage_ready'
|
||||
) {
|
||||
throw new Error('Enabled local application storage did not become ready');
|
||||
}
|
||||
return opened;
|
||||
}
|
||||
Reference in New Issue
Block a user