mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,265 @@
|
||||
import type {
|
||||
GenerateRequest,
|
||||
GenerateResult,
|
||||
ModelChunk,
|
||||
ModelInvocationContext,
|
||||
ModelInvocationPolicyProvider,
|
||||
ModelProvider,
|
||||
} from '../../model-gateway/model';
|
||||
import type {
|
||||
DurableModelInvocationCoordinator,
|
||||
ModelInvocationRecoverySummary,
|
||||
} from '../../model-invocation/durableModelInvocationCoordinator';
|
||||
import type { ModelInvocationSuccessfulCompletionSink } from '../../model-gateway/gateway';
|
||||
import type {
|
||||
ModelInvocationResolutionDecision,
|
||||
ModelInvocationResolutionRecord,
|
||||
ModelInvocationResolutionRepository,
|
||||
ResolveModelInvocationOptions,
|
||||
} from '../../model-invocation/modelInvocationResolution';
|
||||
import type {
|
||||
ModelInvocationUsageLedgerPage,
|
||||
ModelInvocationUsageLedgerQuery,
|
||||
ModelInvocationUsageLedgerRepository,
|
||||
ModelInvocationUsageLedgerSummary,
|
||||
ModelInvocationUsageLedgerSummaryQuery,
|
||||
} from '../../usage/usageLedger';
|
||||
import type {
|
||||
ModelInvocationQuotaRepository,
|
||||
ModelInvocationQuotaWindowUsage,
|
||||
} from '../../usage/usageQuota';
|
||||
import type {
|
||||
ModelInvocationPriceQuote,
|
||||
ModelInvocationPriceSettlement,
|
||||
ModelInvocationPricingRepository,
|
||||
ModelPriceCatalogResolver,
|
||||
} from '../../pricing/pricing';
|
||||
import type {
|
||||
CommitAuthorizedModelPriceCatalogHeadResult,
|
||||
CommitAuthorizedModelPriceCatalogPublicationResult,
|
||||
ModelPriceCatalogAuthorizedAdministrationRepository,
|
||||
ModelPriceCatalogManagementAuthorizer,
|
||||
ModelPriceCatalogManagementDecisionMode,
|
||||
ModelPriceCatalogManagementQuota,
|
||||
PublishModelPriceCatalogRequest,
|
||||
TransitionModelPriceCatalogRequest,
|
||||
} from '../../pricing/modelPriceCatalogManagement';
|
||||
|
||||
export const MODEL_GATEWAY_PROFILES = [
|
||||
'edge',
|
||||
'standalone',
|
||||
'cluster',
|
||||
] as const;
|
||||
export const MODEL_GATEWAY_PROFILE_STATES = [
|
||||
'disabled',
|
||||
'storage_ready',
|
||||
'recovery_ready',
|
||||
'active',
|
||||
'draining',
|
||||
'stopped',
|
||||
'failed',
|
||||
] as const;
|
||||
|
||||
export type ModelGatewayProfile = (typeof MODEL_GATEWAY_PROFILES)[number];
|
||||
export type ModelGatewayProfileState =
|
||||
(typeof MODEL_GATEWAY_PROFILE_STATES)[number];
|
||||
|
||||
export interface ModelGatewayProfileAudit {
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly state: ModelGatewayProfileState;
|
||||
readonly maxConcurrent?: number;
|
||||
readonly recoveryLimit?: number;
|
||||
readonly recovered?: number;
|
||||
readonly alreadyCompleted?: number;
|
||||
}
|
||||
|
||||
export interface ModelGatewayStorageAuthority {
|
||||
readonly repository: ModelInvocationResolutionRepository &
|
||||
ModelInvocationUsageLedgerRepository &
|
||||
ModelInvocationQuotaRepository &
|
||||
ModelInvocationPricingRepository;
|
||||
readonly pricing: ModelPriceCatalogResolver;
|
||||
close?(): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ModelGatewayProviderAuthority {
|
||||
readonly providers: readonly ModelProvider[];
|
||||
readonly policies: ModelInvocationPolicyProvider;
|
||||
dispose?(): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface BootstrapModelGatewayProfileOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly loadStorage: () => Promise<ModelGatewayStorageAuthority>;
|
||||
readonly loadProviders: () => Promise<ModelGatewayProviderAuthority>;
|
||||
readonly confirmActive?: () => void | Promise<void>;
|
||||
readonly createSuccessfulCompletion?: (
|
||||
coordinator: DurableModelInvocationCoordinator,
|
||||
) => ModelInvocationSuccessfulCompletionSink;
|
||||
readonly audit: (
|
||||
record: Readonly<ModelGatewayProfileAudit>,
|
||||
) => void | Promise<void>;
|
||||
readonly maxConcurrent?: number;
|
||||
readonly recoveryLimit?: number;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
export interface ActiveModelGatewayCapability {
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly recovery: Readonly<ModelInvocationRecoverySummary>;
|
||||
readonly maxConcurrent: number;
|
||||
readonly recoveryLimit: number;
|
||||
readonly accepting: boolean;
|
||||
readonly activeOperations: number;
|
||||
supportsSuccessfulCompletionSink(
|
||||
sink: ModelInvocationSuccessfulCompletionSink,
|
||||
): boolean;
|
||||
generate(
|
||||
request: GenerateRequest,
|
||||
context: ModelInvocationContext,
|
||||
): Promise<Readonly<GenerateResult>>;
|
||||
stream(
|
||||
request: GenerateRequest,
|
||||
context: ModelInvocationContext,
|
||||
): AsyncIterable<Readonly<ModelChunk>>;
|
||||
resolveUnknown(options: {
|
||||
readonly invocationId: string;
|
||||
readonly decision: ModelInvocationResolutionDecision;
|
||||
readonly resolvedByUserId: string;
|
||||
readonly resolvedAtMs: number;
|
||||
}): Promise<
|
||||
Readonly<{
|
||||
status: 'created' | 'existing';
|
||||
record: Readonly<ModelInvocationResolutionRecord>;
|
||||
}>
|
||||
>;
|
||||
listProjectUsage(
|
||||
query: ModelInvocationUsageLedgerQuery,
|
||||
): Promise<Readonly<ModelInvocationUsageLedgerPage>>;
|
||||
summarizeProjectUsage(
|
||||
query: ModelInvocationUsageLedgerSummaryQuery,
|
||||
): Promise<Readonly<ModelInvocationUsageLedgerSummary>>;
|
||||
readQuotaWindowUsage(
|
||||
projectId: string,
|
||||
atMs?: number,
|
||||
): Promise<Readonly<ModelInvocationQuotaWindowUsage> | null>;
|
||||
findPriceQuote(
|
||||
invocationId: string,
|
||||
): Promise<Readonly<ModelInvocationPriceQuote> | null>;
|
||||
findPriceSettlement(
|
||||
invocationId: string,
|
||||
): Promise<Readonly<ModelInvocationPriceSettlement> | null>;
|
||||
stop(): Promise<'draining' | 'stopped'>;
|
||||
}
|
||||
|
||||
export type BootstrapModelGatewayProfileResult =
|
||||
| {
|
||||
readonly status: 'disabled';
|
||||
readonly profile: ModelGatewayProfile;
|
||||
stop(): Promise<'stopped'>;
|
||||
}
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly capability: ActiveModelGatewayCapability;
|
||||
};
|
||||
|
||||
export class ModelGatewayProfileUnavailableError extends Error {
|
||||
readonly code = 'MODEL_GATEWAY_PROFILE_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('The model gateway Profile is unavailable', options);
|
||||
this.name = 'ModelGatewayProfileUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ModelGatewayProfileDrainingError extends Error {
|
||||
readonly code = 'MODEL_GATEWAY_PROFILE_DRAINING';
|
||||
|
||||
constructor() {
|
||||
super('The model gateway Profile is draining');
|
||||
this.name = 'ModelGatewayProfileDrainingError';
|
||||
}
|
||||
}
|
||||
|
||||
export const MODEL_PRICE_CATALOG_MANAGEMENT_PROFILE_STATES = [
|
||||
'disabled',
|
||||
'authority_ready',
|
||||
'active',
|
||||
'draining',
|
||||
'stopped',
|
||||
'failed',
|
||||
] as const;
|
||||
|
||||
export type ModelPriceCatalogManagementProfileState =
|
||||
(typeof MODEL_PRICE_CATALOG_MANAGEMENT_PROFILE_STATES)[number];
|
||||
|
||||
export interface ModelPriceCatalogManagementProfileAudit {
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly state: ModelPriceCatalogManagementProfileState;
|
||||
readonly decisionMode: ModelPriceCatalogManagementDecisionMode;
|
||||
}
|
||||
|
||||
export interface ModelPriceCatalogManagementAuthority {
|
||||
readonly repository: ModelPriceCatalogAuthorizedAdministrationRepository;
|
||||
readonly authorizer: ModelPriceCatalogManagementAuthorizer;
|
||||
readonly quota?: ModelPriceCatalogManagementQuota;
|
||||
close?(): void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface BootstrapModelPriceCatalogManagementProfileOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly loadAuthority: () => Promise<ModelPriceCatalogManagementAuthority>;
|
||||
readonly audit: (
|
||||
record: Readonly<ModelPriceCatalogManagementProfileAudit>,
|
||||
) => void | Promise<void>;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
export interface ActiveModelPriceCatalogManagementCapability {
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly decisionMode: ModelPriceCatalogManagementDecisionMode;
|
||||
readonly accepting: boolean;
|
||||
readonly activeOperations: number;
|
||||
publish(
|
||||
request: Readonly<PublishModelPriceCatalogRequest>,
|
||||
): Promise<Readonly<CommitAuthorizedModelPriceCatalogPublicationResult>>;
|
||||
transition(
|
||||
request: Readonly<TransitionModelPriceCatalogRequest>,
|
||||
): Promise<Readonly<CommitAuthorizedModelPriceCatalogHeadResult>>;
|
||||
stop(): Promise<'draining' | 'stopped'>;
|
||||
}
|
||||
|
||||
export type BootstrapModelPriceCatalogManagementProfileResult =
|
||||
| {
|
||||
readonly status: 'disabled';
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly decisionMode: ModelPriceCatalogManagementDecisionMode;
|
||||
stop(): Promise<'stopped'>;
|
||||
}
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly profile: ModelGatewayProfile;
|
||||
readonly decisionMode: ModelPriceCatalogManagementDecisionMode;
|
||||
readonly capability: ActiveModelPriceCatalogManagementCapability;
|
||||
};
|
||||
|
||||
export class ModelPriceCatalogManagementProfileUnavailableError extends Error {
|
||||
readonly code = 'MODEL_PRICE_CATALOG_MANAGEMENT_PROFILE_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('The model price catalog management Profile is unavailable', options);
|
||||
this.name = 'ModelPriceCatalogManagementProfileUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ModelPriceCatalogManagementProfileDrainingError extends Error {
|
||||
readonly code = 'MODEL_PRICE_CATALOG_MANAGEMENT_PROFILE_DRAINING';
|
||||
|
||||
constructor() {
|
||||
super('The model price catalog management Profile is draining');
|
||||
this.name = 'ModelPriceCatalogManagementProfileDrainingError';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type {
|
||||
ModelGatewayProviderAuthority,
|
||||
ModelGatewayStorageAuthority,
|
||||
ModelPriceCatalogManagementAuthority,
|
||||
} from './contracts';
|
||||
|
||||
export async function bestEffortAudit<T>(
|
||||
audit: (record: Readonly<T>) => void | Promise<void>,
|
||||
record: Readonly<T>,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Diagnostic failure cannot replace the activation failure.
|
||||
}
|
||||
}
|
||||
|
||||
export async function dispose(
|
||||
authority: ModelGatewayProviderAuthority | ModelGatewayStorageAuthority,
|
||||
method: 'dispose' | 'close',
|
||||
): Promise<void> {
|
||||
const operation =
|
||||
method === 'dispose'
|
||||
? (authority as ModelGatewayProviderAuthority).dispose
|
||||
: (authority as ModelGatewayStorageAuthority).close;
|
||||
if (operation) await operation.call(authority);
|
||||
}
|
||||
|
||||
export async function closeModelPriceCatalogManagementAuthority(
|
||||
authority: ModelPriceCatalogManagementAuthority,
|
||||
): Promise<void> {
|
||||
if (authority.close) await authority.close.call(authority);
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
import type {
|
||||
GenerateRequest,
|
||||
ModelInvocationContext,
|
||||
} from '../../model-gateway/model';
|
||||
import type { ModelInvocationSuccessfulCompletionSink } from '../../model-gateway/gateway';
|
||||
import type { ResolveModelInvocationOptions } from '../../model-invocation/modelInvocationResolution';
|
||||
import type {
|
||||
ModelInvocationUsageLedgerQuery,
|
||||
ModelInvocationUsageLedgerSummaryQuery,
|
||||
} from '../../usage/usageLedger';
|
||||
import {
|
||||
MODEL_GATEWAY_PROFILES,
|
||||
ModelGatewayProfileDrainingError,
|
||||
ModelGatewayProfileUnavailableError,
|
||||
type ActiveModelGatewayCapability,
|
||||
type BootstrapModelGatewayProfileOptions,
|
||||
type BootstrapModelGatewayProfileResult,
|
||||
type ModelGatewayProviderAuthority,
|
||||
type ModelGatewayStorageAuthority,
|
||||
} from './contracts';
|
||||
import { bestEffortAudit, dispose } from './lifecycle';
|
||||
|
||||
const DEFAULT_PROFILE_BUDGETS = Object.freeze({
|
||||
edge: Object.freeze({ maxConcurrent: 1, recoveryLimit: 4 }),
|
||||
standalone: Object.freeze({ maxConcurrent: 4, recoveryLimit: 32 }),
|
||||
cluster: Object.freeze({ maxConcurrent: 32, recoveryLimit: 128 }),
|
||||
});
|
||||
|
||||
function integer(
|
||||
value: unknown,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
label: string,
|
||||
): number {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
(value as number) < minimum ||
|
||||
(value as number) > maximum
|
||||
) {
|
||||
throw new TypeError(`${label} is invalid`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function assertOptions(options: BootstrapModelGatewayProfileOptions): void {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!MODEL_GATEWAY_PROFILES.includes(options.profile) ||
|
||||
(options.enabled !== undefined && typeof options.enabled !== 'boolean') ||
|
||||
typeof options.loadStorage !== 'function' ||
|
||||
typeof options.loadProviders !== 'function' ||
|
||||
(options.confirmActive !== undefined &&
|
||||
typeof options.confirmActive !== 'function') ||
|
||||
(options.createSuccessfulCompletion !== undefined &&
|
||||
typeof options.createSuccessfulCompletion !== 'function') ||
|
||||
typeof options.audit !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new TypeError('Model gateway Profile options are invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function assertStorage(
|
||||
value: ModelGatewayStorageAuthority,
|
||||
): ModelGatewayStorageAuthority {
|
||||
const repository = value?.repository;
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
!repository ||
|
||||
typeof repository.findStart !== 'function' ||
|
||||
typeof repository.findCompletion !== 'function' ||
|
||||
typeof repository.findResolution !== 'function' ||
|
||||
typeof repository.readAuthority !== 'function' ||
|
||||
typeof repository.listIncomplete !== 'function' ||
|
||||
typeof repository.admit !== 'function' ||
|
||||
typeof repository.complete !== 'function' ||
|
||||
typeof repository.resolve !== 'function' ||
|
||||
typeof repository.findUsage !== 'function' ||
|
||||
typeof repository.listProjectUsage !== 'function' ||
|
||||
typeof repository.summarizeProjectUsage !== 'function' ||
|
||||
typeof repository.findQuotaReservation !== 'function' ||
|
||||
typeof repository.findQuotaSettlement !== 'function' ||
|
||||
typeof repository.readQuotaWindowUsage !== 'function' ||
|
||||
typeof repository.findPriceQuote !== 'function' ||
|
||||
typeof repository.findPriceSettlement !== 'function' ||
|
||||
!value.pricing ||
|
||||
typeof value.pricing.resolve !== 'function' ||
|
||||
(value.close !== undefined && typeof value.close !== 'function')
|
||||
) {
|
||||
throw new TypeError('Model gateway storage authority is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertProviders(
|
||||
value: ModelGatewayProviderAuthority,
|
||||
): ModelGatewayProviderAuthority {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
!Array.isArray(value.providers) ||
|
||||
!value.policies ||
|
||||
typeof value.policies.resolve !== 'function' ||
|
||||
(value.dispose !== undefined && typeof value.dispose !== 'function')
|
||||
) {
|
||||
throw new TypeError('Model gateway provider authority is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Profile-gated optional AI composition root. Disabled mode never invokes the
|
||||
* storage/provider loaders. Enabled mode proves durable storage and bounded
|
||||
* recovery before provider credentials become reachable.
|
||||
*/
|
||||
export async function bootstrapModelGatewayProfile(
|
||||
options: BootstrapModelGatewayProfileOptions,
|
||||
): Promise<BootstrapModelGatewayProfileResult> {
|
||||
assertOptions(options);
|
||||
if (!(options.enabled ?? false)) {
|
||||
await options.audit({ profile: options.profile, state: 'disabled' });
|
||||
return Object.freeze({
|
||||
status: 'disabled',
|
||||
profile: options.profile,
|
||||
stop: async () => 'stopped' as const,
|
||||
});
|
||||
}
|
||||
|
||||
const defaults = DEFAULT_PROFILE_BUDGETS[options.profile];
|
||||
const maxConcurrent = integer(
|
||||
options.maxConcurrent ?? defaults.maxConcurrent,
|
||||
1,
|
||||
64,
|
||||
'Model gateway Profile concurrency',
|
||||
);
|
||||
const recoveryLimit = integer(
|
||||
options.recoveryLimit ?? defaults.recoveryLimit,
|
||||
1,
|
||||
128,
|
||||
'Model gateway Profile recovery limit',
|
||||
);
|
||||
let storage: ModelGatewayStorageAuthority | undefined;
|
||||
let providers: ModelGatewayProviderAuthority | undefined;
|
||||
try {
|
||||
storage = await options.loadStorage();
|
||||
storage = assertStorage(storage);
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'storage_ready',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
});
|
||||
|
||||
const [coordinatorModule, { BoundedModelGateway }] = await Promise.all([
|
||||
import('../../model-invocation/durableModelInvocationCoordinator.js'),
|
||||
import('../../model-gateway/gateway.js'),
|
||||
]);
|
||||
const recovery = await new coordinatorModule.DurableModelInvocationRecovery(
|
||||
storage.repository,
|
||||
).recover(recoveryLimit);
|
||||
if (recovery.failed !== 0 || recovery.hasMore) {
|
||||
throw new ModelGatewayProfileUnavailableError();
|
||||
}
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'recovery_ready',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
recovered: recovery.recovered,
|
||||
alreadyCompleted: recovery.alreadyCompleted,
|
||||
});
|
||||
|
||||
providers = await options.loadProviders();
|
||||
providers = assertProviders(providers);
|
||||
const { DurableModelInvocationResolutionCoordinator } = await import(
|
||||
'../../model-invocation/modelInvocationResolution.js'
|
||||
);
|
||||
const coordinator = new coordinatorModule.DurableModelInvocationCoordinator(
|
||||
storage.repository,
|
||||
);
|
||||
const successfulCompletion =
|
||||
options.createSuccessfulCompletion?.(coordinator);
|
||||
if (
|
||||
successfulCompletion !== undefined &&
|
||||
(!successfulCompletion ||
|
||||
typeof successfulCompletion.record !== 'function')
|
||||
) {
|
||||
throw new ModelGatewayProfileUnavailableError();
|
||||
}
|
||||
const gateway = new BoundedModelGateway({
|
||||
providers: providers.providers,
|
||||
policies: providers.policies,
|
||||
pricing: storage.pricing,
|
||||
audit: coordinator,
|
||||
...(successfulCompletion === undefined ? {} : { successfulCompletion }),
|
||||
maxConcurrent,
|
||||
...(options.now === undefined ? {} : { now: options.now }),
|
||||
});
|
||||
const resolver = new DurableModelInvocationResolutionCoordinator(
|
||||
storage.repository,
|
||||
);
|
||||
let accepting = true;
|
||||
let activeOperations = 0;
|
||||
let stopPromise: Promise<'stopped'> | undefined;
|
||||
let drainingAudited = false;
|
||||
|
||||
const auditDraining = async (): Promise<void> => {
|
||||
if (drainingAudited) return;
|
||||
drainingAudited = true;
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'draining',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
});
|
||||
};
|
||||
const finalizeStop = (): Promise<'stopped'> => {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
await dispose(providers!, 'dispose');
|
||||
await dispose(storage!, 'close');
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'stopped',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
});
|
||||
return 'stopped' as const;
|
||||
})();
|
||||
return stopPromise;
|
||||
};
|
||||
const beginOperation = async (): Promise<void> => {
|
||||
if (!accepting) throw new ModelGatewayProfileDrainingError();
|
||||
if (options.confirmActive) {
|
||||
try {
|
||||
await options.confirmActive();
|
||||
} catch {
|
||||
accepting = false;
|
||||
await auditDraining();
|
||||
if (activeOperations === 0) await finalizeStop();
|
||||
throw new ModelGatewayProfileDrainingError();
|
||||
}
|
||||
}
|
||||
if (!accepting) throw new ModelGatewayProfileDrainingError();
|
||||
activeOperations += 1;
|
||||
};
|
||||
const finishOperation = (): void => {
|
||||
activeOperations -= 1;
|
||||
if (!accepting && activeOperations === 0) {
|
||||
void finalizeStop().catch(async () => {
|
||||
await bestEffortAudit(options.audit, {
|
||||
profile: options.profile,
|
||||
state: 'failed',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const capability: ActiveModelGatewayCapability = Object.freeze({
|
||||
profile: options.profile,
|
||||
recovery,
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
get accepting() {
|
||||
return accepting;
|
||||
},
|
||||
get activeOperations() {
|
||||
return activeOperations;
|
||||
},
|
||||
supportsSuccessfulCompletionSink(
|
||||
sink: ModelInvocationSuccessfulCompletionSink,
|
||||
) {
|
||||
return gateway.supportsSuccessfulCompletionSink(sink);
|
||||
},
|
||||
async generate(
|
||||
request: GenerateRequest,
|
||||
context: ModelInvocationContext,
|
||||
) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await gateway.generate(request, context);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async *stream(request: GenerateRequest, context: ModelInvocationContext) {
|
||||
await beginOperation();
|
||||
try {
|
||||
yield* gateway.stream(request, context);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async resolveUnknown(resolutionOptions: ResolveModelInvocationOptions) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await resolver.resolve(resolutionOptions);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async listProjectUsage(query: ModelInvocationUsageLedgerQuery) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await storage!.repository.listProjectUsage(query);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async summarizeProjectUsage(
|
||||
query: ModelInvocationUsageLedgerSummaryQuery,
|
||||
) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await storage!.repository.summarizeProjectUsage(query);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async readQuotaWindowUsage(projectId: string, atMs?: number) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await storage!.repository.readQuotaWindowUsage(
|
||||
projectId,
|
||||
atMs,
|
||||
);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async findPriceQuote(invocationId: string) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await storage!.repository.findPriceQuote(invocationId);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async findPriceSettlement(invocationId: string) {
|
||||
await beginOperation();
|
||||
try {
|
||||
return await storage!.repository.findPriceSettlement(invocationId);
|
||||
} finally {
|
||||
finishOperation();
|
||||
}
|
||||
},
|
||||
async stop() {
|
||||
accepting = false;
|
||||
if (activeOperations !== 0) {
|
||||
await auditDraining();
|
||||
return 'draining';
|
||||
}
|
||||
return finalizeStop();
|
||||
},
|
||||
});
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'active',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
recovered: recovery.recovered,
|
||||
alreadyCompleted: recovery.alreadyCompleted,
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'active',
|
||||
profile: options.profile,
|
||||
capability,
|
||||
});
|
||||
} catch (cause) {
|
||||
if (providers) {
|
||||
try {
|
||||
await dispose(providers, 'dispose');
|
||||
} catch {
|
||||
// Preserve the activation failure.
|
||||
}
|
||||
}
|
||||
if (storage) {
|
||||
try {
|
||||
await dispose(storage, 'close');
|
||||
} catch {
|
||||
// Preserve the activation failure.
|
||||
}
|
||||
}
|
||||
await bestEffortAudit(options.audit, {
|
||||
profile: options.profile,
|
||||
state: 'failed',
|
||||
maxConcurrent,
|
||||
recoveryLimit,
|
||||
});
|
||||
throw cause instanceof ModelGatewayProfileUnavailableError
|
||||
? cause
|
||||
: new ModelGatewayProfileUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
import type {
|
||||
ModelPriceCatalogManagementDecisionMode,
|
||||
PublishModelPriceCatalogRequest,
|
||||
TransitionModelPriceCatalogRequest,
|
||||
} from '../../pricing/modelPriceCatalogManagement';
|
||||
import {
|
||||
MODEL_GATEWAY_PROFILES,
|
||||
ModelPriceCatalogManagementProfileDrainingError,
|
||||
ModelPriceCatalogManagementProfileUnavailableError,
|
||||
type ActiveModelPriceCatalogManagementCapability,
|
||||
type BootstrapModelPriceCatalogManagementProfileOptions,
|
||||
type BootstrapModelPriceCatalogManagementProfileResult,
|
||||
type ModelGatewayProfile,
|
||||
type ModelPriceCatalogManagementAuthority,
|
||||
} from './contracts';
|
||||
import {
|
||||
bestEffortAudit,
|
||||
closeModelPriceCatalogManagementAuthority,
|
||||
} from './lifecycle';
|
||||
|
||||
function modelPriceCatalogManagementDecisionMode(
|
||||
profile: ModelGatewayProfile,
|
||||
): ModelPriceCatalogManagementDecisionMode {
|
||||
return profile === 'cluster' ? 'separation_of_duty' : 'human_confirmation';
|
||||
}
|
||||
|
||||
function assertModelPriceCatalogManagementProfileOptions(
|
||||
options: BootstrapModelPriceCatalogManagementProfileOptions,
|
||||
): void {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!MODEL_GATEWAY_PROFILES.includes(options.profile) ||
|
||||
(options.enabled !== undefined && typeof options.enabled !== 'boolean') ||
|
||||
typeof options.loadAuthority !== 'function' ||
|
||||
typeof options.audit !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Model price catalog management Profile options are invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertModelPriceCatalogManagementAuthority(
|
||||
value: ModelPriceCatalogManagementAuthority,
|
||||
profile: ModelGatewayProfile,
|
||||
): ModelPriceCatalogManagementAuthority {
|
||||
const repository = value?.repository;
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
!repository ||
|
||||
typeof repository.findPublication !== 'function' ||
|
||||
typeof repository.findCurrent !== 'function' ||
|
||||
typeof repository.findAuthorization !== 'function' ||
|
||||
typeof repository.publishAuthorized !== 'function' ||
|
||||
typeof repository.transitionAuthorized !== 'function' ||
|
||||
!value.authorizer ||
|
||||
typeof value.authorizer.authorize !== 'function' ||
|
||||
(value.quota !== undefined &&
|
||||
(!value.quota || typeof value.quota.consume !== 'function')) ||
|
||||
(profile === 'cluster' && !value.quota) ||
|
||||
(value.close !== undefined && typeof value.close !== 'function')
|
||||
) {
|
||||
throw new TypeError('Model price catalog management authority is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Optional management composition root shared by constrained and clustered
|
||||
* Profiles. Disabled mode is loader-free. Cluster mode is fail-closed unless
|
||||
* quota and separation-of-duty authorities are both available.
|
||||
*/
|
||||
export async function bootstrapModelPriceCatalogManagementProfile(
|
||||
options: BootstrapModelPriceCatalogManagementProfileOptions,
|
||||
): Promise<BootstrapModelPriceCatalogManagementProfileResult> {
|
||||
assertModelPriceCatalogManagementProfileOptions(options);
|
||||
const decisionMode = modelPriceCatalogManagementDecisionMode(options.profile);
|
||||
if (!(options.enabled ?? false)) {
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'disabled',
|
||||
decisionMode,
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'disabled',
|
||||
profile: options.profile,
|
||||
decisionMode,
|
||||
stop: async () => 'stopped' as const,
|
||||
});
|
||||
}
|
||||
|
||||
let authority: ModelPriceCatalogManagementAuthority | undefined;
|
||||
try {
|
||||
authority = await options.loadAuthority();
|
||||
authority = assertModelPriceCatalogManagementAuthority(
|
||||
authority,
|
||||
options.profile,
|
||||
);
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'authority_ready',
|
||||
decisionMode,
|
||||
});
|
||||
const { createModelPriceCatalogManagementService } = await import(
|
||||
'../../pricing/modelPriceCatalogManagement.js'
|
||||
);
|
||||
const service = createModelPriceCatalogManagementService(
|
||||
authority.repository,
|
||||
{
|
||||
decisionMode,
|
||||
authorizer: authority.authorizer,
|
||||
...(authority.quota === undefined ? {} : { quota: authority.quota }),
|
||||
...(options.now === undefined ? {} : { now: options.now }),
|
||||
},
|
||||
);
|
||||
let accepting = true;
|
||||
let activeOperations = 0;
|
||||
let stopPromise: Promise<'stopped'> | undefined;
|
||||
let drainingAudited = false;
|
||||
|
||||
const capability: ActiveModelPriceCatalogManagementCapability =
|
||||
Object.freeze({
|
||||
profile: options.profile,
|
||||
decisionMode,
|
||||
get accepting() {
|
||||
return accepting;
|
||||
},
|
||||
get activeOperations() {
|
||||
return activeOperations;
|
||||
},
|
||||
async publish(request: Readonly<PublishModelPriceCatalogRequest>) {
|
||||
if (!accepting) {
|
||||
throw new ModelPriceCatalogManagementProfileDrainingError();
|
||||
}
|
||||
activeOperations += 1;
|
||||
try {
|
||||
return await service.publish(request);
|
||||
} finally {
|
||||
activeOperations -= 1;
|
||||
}
|
||||
},
|
||||
async transition(
|
||||
request: Readonly<TransitionModelPriceCatalogRequest>,
|
||||
) {
|
||||
if (!accepting) {
|
||||
throw new ModelPriceCatalogManagementProfileDrainingError();
|
||||
}
|
||||
activeOperations += 1;
|
||||
try {
|
||||
return await service.transition(request);
|
||||
} finally {
|
||||
activeOperations -= 1;
|
||||
}
|
||||
},
|
||||
async stop() {
|
||||
accepting = false;
|
||||
if (activeOperations !== 0) {
|
||||
if (!drainingAudited) {
|
||||
drainingAudited = true;
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'draining',
|
||||
decisionMode,
|
||||
});
|
||||
}
|
||||
return 'draining';
|
||||
}
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
await closeModelPriceCatalogManagementAuthority(authority!);
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'stopped',
|
||||
decisionMode,
|
||||
});
|
||||
return 'stopped' as const;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
});
|
||||
await options.audit({
|
||||
profile: options.profile,
|
||||
state: 'active',
|
||||
decisionMode,
|
||||
});
|
||||
return Object.freeze({
|
||||
status: 'active',
|
||||
profile: options.profile,
|
||||
decisionMode,
|
||||
capability,
|
||||
});
|
||||
} catch (cause) {
|
||||
if (authority) {
|
||||
try {
|
||||
await closeModelPriceCatalogManagementAuthority(authority);
|
||||
} catch {
|
||||
// Preserve the activation failure.
|
||||
}
|
||||
}
|
||||
await bestEffortAudit(options.audit, {
|
||||
profile: options.profile,
|
||||
state: 'failed',
|
||||
decisionMode,
|
||||
});
|
||||
throw cause instanceof ModelPriceCatalogManagementProfileUnavailableError
|
||||
? cause
|
||||
: new ModelPriceCatalogManagementProfileUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
export {
|
||||
MODEL_GATEWAY_PROFILES,
|
||||
MODEL_GATEWAY_PROFILE_STATES,
|
||||
MODEL_PRICE_CATALOG_MANAGEMENT_PROFILE_STATES,
|
||||
ModelGatewayProfileDrainingError,
|
||||
ModelGatewayProfileUnavailableError,
|
||||
ModelPriceCatalogManagementProfileDrainingError,
|
||||
ModelPriceCatalogManagementProfileUnavailableError,
|
||||
type ActiveModelGatewayCapability,
|
||||
type ActiveModelPriceCatalogManagementCapability,
|
||||
type BootstrapModelGatewayProfileOptions,
|
||||
type BootstrapModelGatewayProfileResult,
|
||||
type BootstrapModelPriceCatalogManagementProfileOptions,
|
||||
type BootstrapModelPriceCatalogManagementProfileResult,
|
||||
type ModelGatewayProfile,
|
||||
type ModelGatewayProfileAudit,
|
||||
type ModelGatewayProfileState,
|
||||
type ModelGatewayProviderAuthority,
|
||||
type ModelGatewayStorageAuthority,
|
||||
type ModelPriceCatalogManagementAuthority,
|
||||
type ModelPriceCatalogManagementProfileAudit,
|
||||
type ModelPriceCatalogManagementProfileState,
|
||||
} from './profile-composition/contracts';
|
||||
export { bootstrapModelGatewayProfile } from './profile-composition/modelGatewayProfile';
|
||||
export { bootstrapModelPriceCatalogManagementProfile } from './profile-composition/modelPriceCatalogManagementProfile';
|
||||
Reference in New Issue
Block a user