From a94e66505495c7a36a6e17e2fe85d9283990af85 Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 13 Sep 2026 00:32:41 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=94=9F=E5=91=BD=E5=91=A8=E6=9C=9F=E4=B8=8E=E8=B0=83=E5=BA=A6?= =?UTF-8?q?=E5=B0=B1=E7=BB=AA=EF=BC=8C=E4=BC=98=E5=8C=96=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E5=92=8C=E6=9E=84=E5=BB=BA=E5=BC=80=E9=94=80=20(#3069)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: harden task lifecycle and scheduler readiness * fix: confine log writes to the configured log directory * fix: verify complete build artifacts and untracked inputs * fix: reconcile scheduler state and make stop win startup races * fix: isolate cron generations and serialize scheduler recovery --- .dockerignore | 13 + .github/workflows/build-docker-image.yml | 42 +- .github/workflows/validate.yml | 31 + back/api/cron.ts | 14 + back/api/health.ts | 4 +- back/api/system.ts | 7 +- back/app.ts | 29 +- back/config/util.ts | 56 +- back/data/cron.ts | 2 + back/loaders/db.ts | 38 +- back/loaders/initData.ts | 4 +- back/protos/cron.proto | 5 +- back/protos/cron.ts | 25 +- back/schedule/addCron.ts | 9 + back/schedule/client.ts | 72 ++- back/schedule/health.ts | 3 + back/services/cron.ts | 644 ++++++++++++-------- back/services/health.ts | 6 +- back/services/http.ts | 8 +- back/services/schedule.ts | 147 +++-- back/services/subscription.ts | 84 +-- back/shared/childProcess.ts | 78 +++ back/shared/logStreamManager.ts | 160 ++--- back/shared/runCron.ts | 69 ++- back/shared/schedulerMutationLock.ts | 38 ++ back/shared/schedulerReadiness.ts | 77 +++ back/shared/schemaMigrations.ts | 70 +++ docker/Dockerfile | 16 +- docker/Dockerfile.310 | 16 +- docker/Dockerfile.debian | 24 +- docker/Dockerfile.debian310 | 24 +- docker/build-manifest.cjs | 28 + docker/verify-build.cjs | 49 ++ ecosystem.config.js | 3 + package.json | 3 + scripts/benchmark-execution.cjs | 96 +++ scripts/write-build-info.cjs | 27 + shell/api.sh | 47 +- shell/node_path_cache.sh | 118 ++++ shell/otask.sh | 8 +- shell/share.sh | 24 +- shell/task.sh | 16 +- test/back/build-provenance.test.cjs | 150 +++++ test/back/env-name-parsing.test.cjs | 96 +++ test/back/execution-lifecycle.test.cjs | 236 +++++++ test/back/http-exclusive-listen.test.cjs | 31 + test/back/log-path-security.test.cjs | 158 +++++ test/back/manual-execution.test.cjs | 85 +++ test/back/manual-stop-claim.test.cjs | 215 +++++++ test/back/node-path-cache.test.cjs | 173 ++++++ test/back/node-path-lock.test.cjs | 178 ++++++ test/back/primary-apm.test.cjs | 24 + test/back/scheduler-mutation.test.cjs | 274 +++++++++ test/back/scheduler-readiness.test.cjs | 85 +++ test/back/scheduler-reconciliation.test.cjs | 152 +++++ test/back/schema-migrations.test.cjs | 127 ++++ test/back/shell-api-parsing.test.cjs | 184 ++++++ test/back/stop-race.test.cjs | 167 +++++ test/back/subscription-cleanup.test.cjs | 53 ++ test/back/task-time.test.cjs | 116 ++++ test/back/worker-apm.test.cjs | 84 +++ 61 files changed, 4214 insertions(+), 608 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/validate.yml create mode 100644 back/shared/childProcess.ts create mode 100644 back/shared/schedulerMutationLock.ts create mode 100644 back/shared/schedulerReadiness.ts create mode 100644 back/shared/schemaMigrations.ts create mode 100644 docker/build-manifest.cjs create mode 100644 docker/verify-build.cjs create mode 100644 scripts/benchmark-execution.cjs create mode 100644 scripts/write-build-info.cjs create mode 100644 shell/node_path_cache.sh create mode 100644 test/back/build-provenance.test.cjs create mode 100644 test/back/env-name-parsing.test.cjs create mode 100644 test/back/execution-lifecycle.test.cjs create mode 100644 test/back/http-exclusive-listen.test.cjs create mode 100644 test/back/log-path-security.test.cjs create mode 100644 test/back/manual-execution.test.cjs create mode 100644 test/back/manual-stop-claim.test.cjs create mode 100644 test/back/node-path-cache.test.cjs create mode 100644 test/back/node-path-lock.test.cjs create mode 100644 test/back/primary-apm.test.cjs create mode 100644 test/back/scheduler-mutation.test.cjs create mode 100644 test/back/scheduler-readiness.test.cjs create mode 100644 test/back/scheduler-reconciliation.test.cjs create mode 100644 test/back/schema-migrations.test.cjs create mode 100644 test/back/shell-api-parsing.test.cjs create mode 100644 test/back/stop-race.test.cjs create mode 100644 test/back/subscription-cleanup.test.cjs create mode 100644 test/back/task-time.test.cjs create mode 100644 test/back/worker-apm.test.cjs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1f28245d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.gitnexus +.claude +.codex +.env* +node_modules +data +audit +docs +test +src/.umi* +**/__pycache__ +**/*.log diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index e00a3c54..7fbe1dcb 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -15,6 +15,9 @@ permissions: contents: read jobs: + validate: + uses: ./.github/workflows/validate.yml + code_gitlab: runs-on: ubuntu-latest steps: @@ -63,6 +66,7 @@ jobs: git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed" build-static: + needs: validate runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -72,6 +76,7 @@ jobs: - uses: actions/setup-node@v6 with: + node-version: "20" cache: "pnpm" cache-dependency-path: pnpm-lock.yaml @@ -80,6 +85,14 @@ jobs: pnpm install --frozen-lockfile pnpm build:front pnpm build:back + pnpm build:info + + - uses: actions/upload-artifact@v6 + with: + name: qinglong-static-${{ github.sha }} + path: static/ + if-no-files-found: error + include-hidden-files: true - name: copy to static repo env: @@ -151,11 +164,16 @@ jobs: contents: read steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: qinglong-static-${{ github.sha }} + path: static/ - uses: pnpm/action-setup@v6 with: version: "8.3.1" - uses: actions/setup-node@v6 with: + node-version: "20" cache: "pnpm" cache-dependency-path: pnpm-lock.yaml @@ -208,7 +226,7 @@ jobs: uses: docker/build-push-action@v7 with: build-args: | - MAINTAINER=${{ github.repository_owner }} + QL_MAINTAINER=${{ github.repository_owner }} QL_BRANCH=${{ github.ref_name }} SOURCE_COMMIT=${{ github.sha }} network: host @@ -232,11 +250,16 @@ jobs: contents: read steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: qinglong-static-${{ github.sha }} + path: static/ - uses: pnpm/action-setup@v6 with: version: "8.3.1" - uses: actions/setup-node@v6 with: + node-version: "20" cache: "pnpm" cache-dependency-path: pnpm-lock.yaml @@ -288,7 +311,7 @@ jobs: uses: docker/build-push-action@v7 with: build-args: | - MAINTAINER=${{ github.repository_owner }} + QL_MAINTAINER=${{ github.repository_owner }} QL_BRANCH=${{ github.ref_name }} SOURCE_COMMIT=${{ github.sha }} network: host @@ -313,11 +336,16 @@ jobs: contents: read steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: qinglong-static-${{ github.sha }} + path: static/ - uses: pnpm/action-setup@v6 with: version: "8.3.1" - uses: actions/setup-node@v6 with: + node-version: "20" cache: "pnpm" cache-dependency-path: pnpm-lock.yaml @@ -355,7 +383,7 @@ jobs: uses: docker/build-push-action@v7 with: build-args: | - MAINTAINER=${{ github.repository_owner }} + QL_MAINTAINER=${{ github.repository_owner }} QL_BRANCH=${{ github.ref_name }} SOURCE_COMMIT=${{ github.sha }} network: host @@ -380,11 +408,16 @@ jobs: contents: read steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: qinglong-static-${{ github.sha }} + path: static/ - uses: pnpm/action-setup@v6 with: version: "8.3.1" - uses: actions/setup-node@v6 with: + node-version: "20" cache: "pnpm" cache-dependency-path: pnpm-lock.yaml @@ -422,7 +455,7 @@ jobs: uses: docker/build-push-action@v7 with: build-args: | - MAINTAINER=${{ github.repository_owner }} + QL_MAINTAINER=${{ github.repository_owner }} QL_BRANCH=${{ github.ref_name }} SOURCE_COMMIT=${{ github.sha }} network: host @@ -451,6 +484,7 @@ jobs: - uses: actions/setup-node@v3 with: + node-version: "20" cache: "pnpm" - name: build front and back diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..e6cfeb9f --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,31 @@ +name: Validate + +on: + pull_request: + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + regression: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' + - uses: actions/setup-node@v6 + with: + node-version: '20' + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: cp .env.example .env + - run: pnpm test + - run: pnpm build:back + - run: pnpm benchmark:execution -- --runs=10 --output=execution-baseline.json + - uses: actions/upload-artifact@v6 + with: + name: execution-baseline + path: execution-baseline.json diff --git a/back/api/cron.ts b/back/api/cron.ts index abd7f236..a13114ee 100644 --- a/back/api/cron.ts +++ b/back/api/cron.ts @@ -10,12 +10,26 @@ import { InstanceStatus, } from '../data/runningInstance'; import { t } from '../shared/i18n'; +import cronClient from '../schedule/client'; const route = Router(); export default (app: Router) => { app.use('/crons', route); + route.use(async (req, res, next) => { + // Keep stop/status callbacks available even when the scheduler is down. + if (['POST', 'PUT', 'DELETE'].includes(req.method) && + ['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) { + try { + await cronClient.readiness.ensureReady(); + } catch (error) { + return next(error); + } + } + return next(); + }); + route.get( '/views', async (req: Request, res: Response, next: NextFunction) => { diff --git a/back/api/health.ts b/back/api/health.ts index bb961fa1..738e8ec1 100644 --- a/back/api/health.ts +++ b/back/api/health.ts @@ -11,8 +11,8 @@ export default (app: Router) => { try { const healthService = Container.get(HealthService); const health = await healthService.check(); - res.status(200).send({ - code: 200, + res.status(health.status === 'ok' ? 200 : 503).send({ + code: health.status === 'ok' ? 200 : 503, data: health, }); } catch (err: any) { diff --git a/back/api/system.ts b/back/api/system.ts index 8b3829ce..831725a6 100644 --- a/back/api/system.ts +++ b/back/api/system.ts @@ -273,8 +273,11 @@ export default (app: Router) => { }, onEnd: async (cp, endTime, diff) => { // Close the stream after task completion - await logStreamManager.closeStream(await handleLogPath(logPath)); - res.end(); + try { + await logStreamManager.closeStream(await handleLogPath(logPath)); + } finally { + res.end(); + } }, onError: async (message: string) => { res.write(message); diff --git a/back/app.ts b/back/app.ts index 96df1a80..693e34e1 100644 --- a/back/app.ts +++ b/back/app.ts @@ -11,6 +11,7 @@ import { monitoringMiddleware } from './middlewares/monitoring'; import { errStack } from './config/util'; import { type GrpcServerService } from './services/grpc'; import { type HttpServerService } from './services/http'; +import cronClient from './schedule/client'; interface WorkerMetadata { id: number; @@ -79,6 +80,11 @@ class Application { ); // If gRPC worker died, restart it and wait for it to be ready if (metadata.serviceType === 'grpc') { + try { + this.httpWorker?.send('scheduler-unavailable'); + } catch (error) { + Logger.warn('Unable to notify HTTP worker of scheduler exit'); + } const newGrpcWorker = this.forkWorker('grpc'); this.waitForWorkerReady(newGrpcWorker, 30000) .then(() => { @@ -132,7 +138,14 @@ class Application { } private forkWorker(serviceType: string): Worker { - const worker = cluster.fork({ SERVICE_TYPE: serviceType }); + const workerEnv: NodeJS.ProcessEnv = { SERVICE_TYPE: serviceType }; + // PM2's fork launcher is inherited by our own cluster workers. Their APM + // messages go to this primary, not PM2, and duplicate its sampling work. + // Keep primary monitoring and allow restoring the inherited worker APM. + if (process.env.pm_id !== undefined && process.env.QL_WORKER_APM !== 'true') { + workerEnv.pmx = 'false'; + } + const worker = cluster.fork(workerEnv); this.workerMetadataMap.set(worker.id, { id: worker.id, @@ -264,17 +277,9 @@ class Application { process.on('message', async (msg) => { if (msg === 'shutdown') { this.gracefulShutdown(serviceType); - } else if (msg === 'reregister-crons' && serviceType === 'http') { - // Re-register cron jobs when gRPC worker restarts - try { - Logger.info('[boot] Received reregister-crons message, re-registering cron jobs...'); - const CronService = (await import('./services/cron')).default; - const cronService = Container.get(CronService); - await cronService.autosave_crontab(); - Logger.info('[boot] Cron jobs re-registered successfully'); - } catch (error) { - Logger.error(`[boot] Failed to re-register cron jobs:\n${errStack(error)}`); - } + } else if (serviceType === 'http' && + (msg === 'reregister-crons' || msg === 'scheduler-unavailable')) { + cronClient.readiness.invalidate(); } }); diff --git a/back/config/util.ts b/back/config/util.ts index 649f97a1..e6390ea9 100644 --- a/back/config/util.ts +++ b/back/config/util.ts @@ -12,6 +12,7 @@ import { DependenceTypes } from '../data/dependence'; import { FormData } from 'undici'; import os from 'os'; import { maybeSudo, isInContainer } from './container'; +import { resolveFileAccess } from '../shared/fileAccess'; export * from './share'; @@ -144,7 +145,8 @@ export async function handleLogPath( logPath: string, data: string = '', ): Promise { - const absolutePath = path.resolve(config.logPath, logPath); + const absolutePath = resolveFileAccess(config.logPath, [logPath]); + if (!absolutePath) throw new Error('Log path is outside the log directory'); const logFileExist = await fileExist(absolutePath); if (!logFileExist) { await createFile(absolutePath, data); @@ -487,18 +489,48 @@ export function psTree(pid: number): Promise { }); } -export async function killTask(pid: number) { - const pids = await psTree(pid); - - if (pids.length) { - try { - [pid, ...pids].reverse().forEach((x) => { - process.kill(x, 15); - }); - } catch (error) { } - } else { - process.kill(pid, 2); +export async function killTask(pid: number, waitForExit = false) { + const descendants = await psTree(pid); + if (!waitForExit) { + if (descendants.length) { + try { + [pid, ...descendants] + .reverse() + .forEach((target) => process.kill(target, 15)); + } catch {} + } else process.kill(pid, 2); + return; } + const pids = [...descendants.reverse(), pid]; + const signal = (target: number, sig: NodeJS.Signals) => { + try { + process.kill(target, sig); + } catch (error: any) { + if (error.code !== 'ESRCH') throw error; + } + }; + for (const target of pids) signal(target, 'SIGTERM'); + const alive = (target: number) => { + try { + process.kill(target, 0); + return true; + } catch (error: any) { + if (error.code === 'ESRCH') return false; + throw error; + } + }; + const wait = async () => { + const deadline = Date.now() + 1000; + while (pids.some(alive) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return pids.filter(alive); + }; + let remaining = await wait(); + for (const target of remaining) signal(target, 'SIGKILL'); + remaining = await wait(); + if (remaining.length) + throw new Error(`Task processes did not exit: ${remaining.join(', ')}`); } export async function getPid(cmd: string) { diff --git a/back/data/cron.ts b/back/data/cron.ts index 32b0b554..c0c0ad0e 100644 --- a/back/data/cron.ts +++ b/back/data/cron.ts @@ -13,6 +13,7 @@ export class Crontab { pid?: number; isDisabled?: 1 | 0; log_path?: string; + queued_token?: string | null; isPinned?: 1 | 0; labels?: string[]; last_running_time?: number; @@ -83,6 +84,7 @@ export const CrontabModel = sequelize.define('Crontab', { isDisabled: DataTypes.NUMBER, isPinned: DataTypes.NUMBER, log_path: DataTypes.STRING, + queued_token: DataTypes.STRING, labels: DataTypes.JSON, last_running_time: DataTypes.NUMBER, last_execution_time: DataTypes.NUMBER, diff --git a/back/loaders/db.ts b/back/loaders/db.ts index 9a70d91a..44a9c076 100644 --- a/back/loaders/db.ts +++ b/back/loaders/db.ts @@ -9,6 +9,7 @@ import { CrontabViewModel } from '../data/cronView'; import { CrontabStatModel } from '../data/cronStats'; import { RunningInstanceModel } from '../data/runningInstance'; import { sequelize } from '../data'; +import { migrateSchema } from '../shared/schemaMigrations'; export default async () => { try { @@ -22,44 +23,11 @@ export default async () => { await CrontabStatModel.sync(); await RunningInstanceModel.sync(); - // 初始化新增字段 - const migrations = [ - { - table: 'CrontabViews', - column: 'filterRelation', - type: 'VARCHAR(255)', - }, - { table: 'Subscriptions', column: 'proxy', type: 'VARCHAR(255)' }, - { table: 'CrontabViews', column: 'type', type: 'NUMBER' }, - { table: 'Subscriptions', column: 'autoAddCron', type: 'NUMBER' }, - { table: 'Subscriptions', column: 'autoDelCron', type: 'NUMBER' }, - { table: 'Crontabs', column: 'sub_id', type: 'NUMBER' }, - { table: 'Crontabs', column: 'extra_schedules', type: 'JSON' }, - { table: 'Crontabs', column: 'task_before', type: 'TEXT' }, - { table: 'Crontabs', column: 'task_after', type: 'TEXT' }, - { table: 'Crontabs', column: 'log_name', type: 'VARCHAR(255)' }, - { - table: 'Crontabs', - column: 'allow_multiple_instances', - type: 'NUMBER', - }, - { table: 'Crontabs', column: 'work_dir', type: 'VARCHAR(255)' }, - { table: 'Envs', column: 'isPinned', type: 'NUMBER' }, - { table: 'Envs', column: 'labels', type: 'JSON' }, - ]; - - for (const migration of migrations) { - try { - await sequelize.query( - `alter table ${migration.table} add column ${migration.column} ${migration.type}`, - ); - } catch (error) { - // Column already exists or other error, continue - } - } + await migrateSchema(sequelize); Logger.info('[boot] DB loaded'); } catch (error) { Logger.error('[boot] DB load failed', error); + throw error; } }; diff --git a/back/loaders/initData.ts b/back/loaders/initData.ts index 7662ae87..8854841a 100644 --- a/back/loaders/initData.ts +++ b/back/loaders/initData.ts @@ -17,6 +17,7 @@ import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../conf import OpenService from '../services/open'; import { shareStore } from '../shared/store'; import Logger from './logger'; +import cronClient from '../schedule/client'; import { AppModel } from '../data/open'; import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance'; import { setLang, systemLang } from '../shared/i18n'; @@ -236,7 +237,8 @@ export default async () => { } catch { } // 初始化保存一次ck和定时任务数据 - await cronService.autosave_crontab(); + cronClient.readiness.configure(() => cronService.autosave_crontab(true)); + await cronClient.readiness.recover(); await envService.set_envs(); diff --git a/back/protos/cron.proto b/back/protos/cron.proto index e88d6b1d..505eabe6 100644 --- a/back/protos/cron.proto +++ b/back/protos/cron.proto @@ -17,7 +17,10 @@ message ICron { string name = 5; } -message AddCronRequest { repeated ICron crons = 1; } +message AddCronRequest { + repeated ICron crons = 1; + bool replace = 2; +} message AddCronResponse {} diff --git a/back/protos/cron.ts b/back/protos/cron.ts index 82d1c44c..75832810 100644 --- a/back/protos/cron.ts +++ b/back/protos/cron.ts @@ -1,7 +1,7 @@ // Code generated by protoc-gen-ts_proto. DO NOT EDIT. // versions: // protoc-gen-ts_proto v2.6.1 -// protoc v3.21.12 +// protoc v3.17.3 // source: back/protos/cron.proto /* eslint-disable */ @@ -35,6 +35,7 @@ export interface ICron { export interface AddCronRequest { crons: ICron[]; + replace: boolean; } export interface AddCronResponse { @@ -232,7 +233,7 @@ export const ICron: MessageFns = { }; function createBaseAddCronRequest(): AddCronRequest { - return { crons: [] }; + return { crons: [], replace: false }; } export const AddCronRequest: MessageFns = { @@ -240,6 +241,9 @@ export const AddCronRequest: MessageFns = { for (const v of message.crons) { ICron.encode(v!, writer.uint32(10).fork()).join(); } + if (message.replace !== false) { + writer.uint32(16).bool(message.replace); + } return writer; }, @@ -258,6 +262,14 @@ export const AddCronRequest: MessageFns = { message.crons.push(ICron.decode(reader, reader.uint32())); continue; } + case 2: { + if (tag !== 16) { + break; + } + + message.replace = reader.bool(); + continue; + } } if ((tag & 7) === 4 || tag === 0) { break; @@ -268,7 +280,10 @@ export const AddCronRequest: MessageFns = { }, fromJSON(object: any): AddCronRequest { - return { crons: globalThis.Array.isArray(object?.crons) ? object.crons.map((e: any) => ICron.fromJSON(e)) : [] }; + return { + crons: globalThis.Array.isArray(object?.crons) ? object.crons.map((e: any) => ICron.fromJSON(e)) : [], + replace: isSet(object.replace) ? globalThis.Boolean(object.replace) : false, + }; }, toJSON(message: AddCronRequest): unknown { @@ -276,6 +291,9 @@ export const AddCronRequest: MessageFns = { if (message.crons?.length) { obj.crons = message.crons.map((e) => ICron.toJSON(e)); } + if (message.replace !== false) { + obj.replace = message.replace; + } return obj; }, @@ -285,6 +303,7 @@ export const AddCronRequest: MessageFns = { fromPartial, I>>(object: I): AddCronRequest { const message = createBaseAddCronRequest(); message.crons = object.crons?.map((e) => ICron.fromPartial(e)) || []; + message.replace = object.replace ?? false; return message; }, }; diff --git a/back/schedule/addCron.ts b/back/schedule/addCron.ts index d651de10..d3c2fdcf 100644 --- a/back/schedule/addCron.ts +++ b/back/schedule/addCron.ts @@ -69,6 +69,15 @@ const addCron = ( return; } + // Recovery replaces the whole snapshot, including deletions and disabled jobs. + // Validation above must finish before touching the previous schedule. + if (call.request.replace) { + for (const jobs of scheduleStacks.values()) { + for (const job of jobs) job?.cancel(); + } + scheduleStacks.clear(); + } + // ===== 第二遍：注册所有任务 ===== for (const item of call.request.crons) { const { id, schedule, command, extra_schedules, name } = item; diff --git a/back/schedule/client.ts b/back/schedule/client.ts index 30daacf9..3802288e 100644 --- a/back/schedule/client.ts +++ b/back/schedule/client.ts @@ -1,4 +1,4 @@ -import { credentials } from '@grpc/grpc-js'; +import { credentials, status, Metadata } from '@grpc/grpc-js'; import { AddCronRequest, AddCronResponse, @@ -9,7 +9,41 @@ import { import config from '../config'; import { getGrpcCerts } from '../config/grpcCerts'; +import { HealthService } from '../protos/health'; +import { SchedulerReadiness } from '../shared/schedulerReadiness'; + class Client { + readonly readiness = new SchedulerReadiness(() => this.probe()); + + private async waitForReady(timeoutMs: number) { + try { + await new Promise((resolve, reject) => { + this.client.waitForReady(Date.now() + timeoutMs, (err) => + err ? reject(err) : resolve() + ); + }); + } catch (error) { + this.readiness.invalidate(); + throw Object.assign( + error instanceof Error ? error : new Error(String(error)), + { status: 503 } + ); + } + } + + private async probe(): Promise { + await this.waitForReady(1000); + await new Promise((resolve, reject) => { + this.client.makeUnaryRequest( + HealthService.check.path, + HealthService.check.requestSerialize, + HealthService.check.responseDeserialize, + { service: 'scheduler' }, + { deadline: Date.now() + 1000 }, + (err, res) => err ? reject(err) : res?.status === 1 ? resolve() : reject(new Error('Scheduler unavailable')), + ); + }); + } private _client: CronClient | null = null; private get client(): CronClient { @@ -28,22 +62,40 @@ class Client { return this._client; } - addCron(request: AddCronRequest['crons']): Promise { + async addCron( + request: AddCronRequest['crons'], + replace = false + ): Promise { + await this.waitForReady(2000); return new Promise((resolve, reject) => { - this.client.addCron({ crons: request }, (err, res) => { - if (err) { - reject(err); + this.client.addCron( + { crons: request, replace }, + new Metadata(), + { deadline: Date.now() + 5000 }, + (err, res) => { + if (err) { + if (err.code === status.UNAVAILABLE) { + this.readiness.invalidate(); + Object.assign(err, { status: 503 }); + } + return reject(err); + } + resolve(res); } - resolve(res); - }); + ); }); } - delCron(request: DeleteCronRequest['ids']): Promise { + async delCron(request: DeleteCronRequest['ids']): Promise { + await this.waitForReady(2000); return new Promise((resolve, reject) => { - this.client.delCron({ ids: request }, (err, res) => { + this.client.delCron({ ids: request }, new Metadata(), { deadline: Date.now() + 5000 }, (err, res) => { if (err) { - reject(err); + if (err.code === status.UNAVAILABLE) { + this.readiness.invalidate(); + Object.assign(err, { status: 503 }); + } + return reject(err); } resolve(res); }); diff --git a/back/schedule/health.ts b/back/schedule/health.ts index 04068c67..2184f775 100644 --- a/back/schedule/health.ts +++ b/back/schedule/health.ts @@ -52,6 +52,9 @@ const check = async ( callback: sendUnaryData, ) => { switch (call.request.service) { + // Local scheduler liveness only: never call HTTP from this probe. + case 'scheduler': + return callback(null, { status: 1 }); case 'cron': { const healthUrl = `http://localhost:${config.port}${ config.baseUrl || '' diff --git a/back/services/cron.ts b/back/services/cron.ts index e28e710c..a20f0966 100644 --- a/back/services/cron.ts +++ b/back/services/cron.ts @@ -1,3 +1,8 @@ +import { randomUUID } from 'crypto'; +import { + withSchedulerMutation, + schedulerRegistrationError, +} from '../shared/schedulerMutationLock'; import { Service, Inject } from 'typedi'; import winston from 'winston'; import config from '../config'; @@ -13,7 +18,6 @@ import { getFileContentByName, fileExist, killTask, - killAllTasks, getUniqPath, safeJSONParse, isDemoEnv, @@ -31,8 +35,10 @@ import { writeFileWithLock } from '../shared/utils'; import { t } from '../shared/i18n'; import { ScheduleType } from '../interface/schedule'; import { logStreamManager } from '../shared/logStreamManager'; +import { observeChildProcess, asError } from '../shared/childProcess'; import { isEmpty } from 'lodash'; import { LogReadOptions, readLogChunk } from '../shared/logReader'; +import { resolveFileAccess } from '../shared/fileAccess'; @Service() export default class CronService { @@ -97,42 +103,45 @@ export default class CronService { } public async create(payload: Crontab): Promise { - const tab = new Crontab(payload); - tab.saved = false; - tab.log_name = await this.getLogName(tab); - const doc = await this.insert(tab); + return withSchedulerMutation(async () => { + const tab = new Crontab(payload); + tab.saved = false; + tab.log_name = await this.getLogName(tab); + const doc = await this.insert(tab); - if (isDemoEnv()) { - return doc; - } - - if (this.shouldUseCronClient(doc)) { - try { - await cronClient.addCron([ - { - name: doc.name || '', - id: String(doc.id), - schedule: doc.schedule!, - command: this.makeCommand(doc), - extra_schedules: doc.extra_schedules || [], - }, - ]); - } catch (error: any) { - // gRPC 注册失败时回滚 DB 记录，避免产生"僵尸任务" - // （DB 和 crontab.list 有记录但调度器永远不会执行） - await CrontabModel.destroy({ where: { id: doc.id } }); - this.logger.error( - '[crontab] Failed to register cron job in scheduler, task creation rolled back:', - error?.message || error, - ); - throw new Error( - `${t('调度器注册失败，任务创建已回滚')}: ${(error as any)?.details || error?.message}`, - ); + if (isDemoEnv()) { + return doc; } - } - await this.setCrontab(); - return doc; + if (this.shouldUseCronClient(doc)) { + try { + await cronClient.addCron([ + { + name: doc.name || '', + id: String(doc.id), + schedule: doc.schedule!, + command: this.makeCommand(doc), + extra_schedules: doc.extra_schedules || [], + }, + ]); + } catch (error: any) { + // gRPC 注册失败时回滚 DB 记录，避免产生"僵尸任务" + // （DB 和 crontab.list 有记录但调度器永远不会执行） + await CrontabModel.destroy({ where: { id: doc.id } }); + this.logger.error( + '[crontab] Failed to register cron job in scheduler, task creation rolled back:', + error?.message || error, + ); + throw schedulerRegistrationError( + `${t('调度器注册失败，任务创建已回滚')}: ${(error as any)?.details || error?.message}`, + error, + ); + } + } + + await this.setCrontab(); + return doc; + }); } public async insert(payload: Crontab): Promise { @@ -140,69 +149,74 @@ export default class CronService { } public async update(payload: Partial): Promise { - const doc = await this.getDb({ id: payload.id }); - const tab = new Crontab({ ...doc, ...payload }); - tab.saved = false; - tab.log_name = await this.getLogName(tab); - const newDoc = await this.updateDb(tab); + return withSchedulerMutation(async () => { + const doc = await this.getDb({ id: payload.id }); + const tab = new Crontab({ ...doc, ...payload }); + tab.saved = false; + tab.log_name = await this.getLogName(tab); + const newDoc = await this.updateDb(tab); - if (doc.isDisabled === 1 || isDemoEnv()) { - return newDoc; - } + if (doc.isDisabled === 1 || isDemoEnv()) { + return newDoc; + } - try { - await cronClient.delCron([String(newDoc.id)]); - } catch (error: any) { - this.logger.warn( - '[crontab] Failed to unregister cron job in scheduler:', - error?.message || error, - ); - } - - if (this.shouldUseCronClient(newDoc)) { try { - await cronClient.addCron([ - { - name: doc.name || '', - id: String(newDoc.id), - schedule: newDoc.schedule!, - command: this.makeCommand(newDoc), - extra_schedules: newDoc.extra_schedules || [], - }, - ]); + await cronClient.delCron([String(newDoc.id)]); } catch (error: any) { - // gRPC 注册新任务失败 → 回滚 DB 到旧数据，并尝试恢复旧调度注册 - await CrontabModel.update(doc, { where: { id: doc.id } }); - if (this.shouldUseCronClient(doc)) { - try { - await cronClient.addCron([ - { - name: doc.name || '', - id: String(doc.id), - schedule: doc.schedule!, - command: this.makeCommand(doc), - extra_schedules: doc.extra_schedules || [], - }, - ]); - } catch (_recoveryError: any) { - this.logger.warn( - '[crontab] Failed to restore old cron job in scheduler after rollback:', - _recoveryError?.message || _recoveryError, - ); - } - } - this.logger.error( - '[crontab] Failed to register updated cron job in scheduler, update rolled back:', + this.logger.warn( + '[crontab] Failed to unregister cron job in scheduler:', error?.message || error, ); - throw new Error( - `${t('调度器注册失败，任务更新已回滚')}: ${(error as any)?.details || error?.message}`, - ); } - } - await this.setCrontab(); - return newDoc; + if (this.shouldUseCronClient(newDoc)) { + try { + await cronClient.addCron([ + { + name: doc.name || '', + id: String(newDoc.id), + schedule: newDoc.schedule!, + command: this.makeCommand(newDoc), + extra_schedules: newDoc.extra_schedules || [], + }, + ]); + } catch (error: any) { + // gRPC 注册新任务失败 → 回滚 DB 到旧数据，并尝试恢复旧调度注册 + await CrontabModel.update(omit(doc, ['queued_token']), { + where: { id: doc.id }, + }); + if (this.shouldUseCronClient(doc)) { + try { + await cronClient.addCron([ + { + name: doc.name || '', + id: String(doc.id), + schedule: doc.schedule!, + command: this.makeCommand(doc), + extra_schedules: doc.extra_schedules || [], + }, + ]); + } catch (_recoveryError: any) { + this.logger.warn( + '[crontab] Failed to restore old cron job in scheduler after rollback:', + _recoveryError?.message || _recoveryError, + ); + } + } + this.logger.error( + '[crontab] Failed to register updated cron job in scheduler, update rolled back:', + error?.message || error, + ); + throw schedulerRegistrationError( + `${t('调度器注册失败，任务更新已回滚')}: ${(error as any)?.details || error?.message}`, + error, + ); + } + } + + await this.setCrontab(); + return newDoc; + }); } public async updateDb(payload: Crontab): Promise { @@ -290,16 +304,18 @@ export default class CronService { } public async remove(ids: number[]) { - await CrontabModel.destroy({ where: { id: ids } }); - try { - await cronClient.delCron(ids.map(String)); - } catch (error: any) { - this.logger.warn( - '[crontab] Failed to unregister cron job in scheduler:', - error?.message || error, - ); - } - await this.setCrontab(); + return withSchedulerMutation(async () => { + await CrontabModel.destroy({ where: { id: ids } }); + try { + await cronClient.delCron(ids.map(String)); + } catch (error: any) { + this.logger.warn( + '[crontab] Failed to unregister cron job in scheduler:', + error?.message || error, + ); + } + await this.setCrontab(); + }); } public async pin(ids: number[]) { @@ -570,46 +586,99 @@ export default class CronService { } public async run(ids: number[]) { + const queuedToken = randomUUID(); await CrontabModel.update( - { status: CrontabStatus.queued }, + { status: CrontabStatus.queued, queued_token: queuedToken }, { where: { id: ids } }, ); ids.forEach((id) => { - this.runSingle(id); + this.runSingle(id, queuedToken); }); } public async stop(ids: number[]) { const docs = await CrontabModel.findAll({ where: { id: ids } }); + // Cancel the queued snapshot first, so a late spawn cannot claim it. for (const doc of docs) { - // Kill all running instances of this task - try { - if (doc.pid) { - await killTask(doc.pid); - } - const command = doc.command.replace(/\s+/g, ' ').trim(); - await killAllTasks(command); - this.logger.info( - `[panel][停止所有运行中的任务实例] 任务ID: ${doc.id}, 命令: ${command}`, + if (doc.status === CrontabStatus.queued) { + const [cancelled] = await CrontabModel.update( + { status: CrontabStatus.idle, pid: null, queued_token: null } as any, + { + where: { + id: doc.id, + status: CrontabStatus.queued, + [Op.and]: [ + where(colFn('log_path'), { [Op.eq]: doc.log_path ?? null }), + where(colFn('queued_token'), { + [Op.eq]: doc.queued_token ?? null, + }), + ], + }, + } ); + // A concurrent claim may have won; capture its PID before signalling. + if (!cancelled) await doc.reload(); + } + } + const stoppingInstances = await RunningInstanceModel.findAll({ + attributes: ['id', 'pid', 'cron_id'], + where: { cron_id: ids, status: InstanceStatus.running }, + }); + const targets = new Set( + [ + ...stoppingInstances.map((instance) => instance.pid), + ...docs + .filter((doc) => doc.status === CrontabStatus.running) + .map((doc) => doc.pid), + ].filter((pid): pid is number => typeof pid === 'number' && pid > 0) + ); + const stopped = new Set(); + for (const pid of targets) { + try { + await killTask(pid, true); + stopped.add(pid); } catch (error) { this.logger.error( - `[panel][停止任务失败] 任务ID: ${doc.id}, 错误: ${error}`, + '[panel][停止任务失败] PID: %s, 错误: %s', + pid, + asError(error).message ); } } - - // Mark all running instances as stopped - const finishedAt = dayjs().unix(); - await RunningInstanceModel.update( - { status: InstanceStatus.stopped, finished_at: finishedAt }, - { where: { cron_id: ids, status: InstanceStatus.running } }, - ); - - await CrontabModel.update( - { status: CrontabStatus.idle, pid: undefined }, - { where: { id: ids } }, - ); + const stoppedIds = stoppingInstances + .filter((instance) => instance.pid && stopped.has(instance.pid)) + .map((instance) => instance.id!); + if (stoppedIds.length) { + await RunningInstanceModel.update( + { status: InstanceStatus.stopped, finished_at: dayjs().unix() }, + { where: { id: stoppedIds } } + ); + } + for (const doc of docs) { + if ( + doc.status !== CrontabStatus.running || + (doc.pid && !stopped.has(doc.pid)) + ) + continue; + const remaining = await RunningInstanceModel.count({ + where: { cron_id: doc.id, status: InstanceStatus.running }, + }); + if (remaining) continue; + await CrontabModel.update( + { status: CrontabStatus.idle, pid: null, queued_token: null } as any, + { + where: { + id: doc.id, + status: CrontabStatus.running, + [Op.and]: [ + where(colFn('queued_token'), { [Op.eq]: doc.queued_token ?? null }), + where(colFn('pid'), { [Op.eq]: doc.pid ?? null }), + where(colFn('log_path'), { [Op.eq]: doc.log_path ?? null }), + ], + }, + } + ); + } } public async stopInstance(instanceId: number) { @@ -646,125 +715,191 @@ export default class CronService { return { code: 200, message: t('实例已停止') }; } - private async runSingle(cronId: number): Promise { - return taskLimit.manualRunWithCronLimit(() => { - return new Promise(async (resolve: any) => { + private async runSingle( + cronId: number, + expectedToken?: string, + ): Promise { + return taskLimit.manualRunWithCronLimit(async () => { + let absolutePath: string | undefined; + let logPath: string | undefined; + let queuedLogPath: string | null | undefined; + let queuedToken: string | null = null; + let claimed = false; + try { const cron = await this.getDb({ id: cronId }); - const params = { - name: cron.name, - command: cron.command, - schedule: cron.schedule, - extra_schedules: cron.extra_schedules, - }; - if (cron.status !== CrontabStatus.queued) { - resolve(params); - return; - } - - this.logger.info( - `[panel][开始执行任务] 参数: ${JSON.stringify(params)}`, - ); - - let { id, command, log_name } = cron; - + if ( + cron.status !== CrontabStatus.queued || + (expectedToken !== undefined && cron.queued_token !== expectedToken) + ) return; + queuedToken = cron.queued_token ?? null; + queuedLogPath = cron.log_path ?? null; + const { id, command, log_name } = cron; const uniqPath = log_name === '/dev/null' || !log_name ? await getUniqPath(command, `${id}`) : log_name; const logTime = dayjs().format('YYYY-MM-DD-HH-mm-ss-SSS'); - const logDirPath = path.resolve(config.logPath, `${uniqPath}`); - await fs.mkdir(logDirPath, { recursive: true }); - const logPath = `${uniqPath}/${logTime}.log`; - const absolutePath = path.resolve(config.logPath, `${logPath}`); + logPath = `${uniqPath}/${logTime}.log`; + absolutePath = resolveFileAccess(config.logPath, [logPath]); + if (!absolutePath) + throw new Error('Log path is outside the log directory'); + await fs.mkdir(path.dirname(absolutePath), { recursive: true }); + const outputPath = absolutePath; const cp = spawn( `real_log_path=${logPath} no_delay=true ${this.makeCommand( cron, - true, + true )}`, - { shell: '/bin/bash' }, + { shell: '/bin/bash' } ); - - await CrontabModel.update( - { status: CrontabStatus.running, pid: cp.pid, log_path: logPath }, - { where: { id } }, - ); - cp.stdout.on('data', async (data) => { - await logStreamManager.write(absolutePath, data.toString()); + // Install observers before the first await: very short children may already exit. + const { completed } = observeChildProcess(cp, { + onStart: async () => { + try { + const [count] = await CrontabModel.update( + { + status: CrontabStatus.running, + pid: cp.pid, + log_path: logPath, + }, + { + where: { + id, + status: CrontabStatus.queued, + [Op.and]: [ + where(colFn('queued_token'), { [Op.eq]: queuedToken }), + where(colFn('log_path'), { [Op.eq]: queuedLogPath }), + ], + }, + } + ); + if (count !== 1) + throw new Error( + 'Task was stopped or superseded before startup' + ); + claimed = true; + } catch (error) { + if (cp.pid) await killTask(cp.pid, true); + throw error; + } + }, + onStdout: (message) => logStreamManager.write(outputPath, message), + onStderr: (message) => logStreamManager.write(outputPath, message), }); - cp.stderr.on('data', async (data) => { - this.logger.info( - '[panel][执行任务失败] 命令: %s, 错误信息: %j', - command, - data.toString(), - ); - await logStreamManager.write(absolutePath, data.toString()); - }); - cp.on('error', async (err) => { + const result = await completed; + if (result.error) { this.logger.error( - '[panel][创建任务失败] 命令: %s, 错误信息: %j', - command, - err, + '[panel][执行任务失败] 任务ID: %s, 错误: %s', + id, + result.error.message ); - await logStreamManager.write(absolutePath, JSON.stringify(err)); - }); - - cp.on('exit', async (code) => { - this.logger.info( - '[panel][执行任务结束] 参数: %s, 退出码: %j', - JSON.stringify(params), - code, + } + this.logger.info( + '[panel][执行任务结束] 任务ID: %s, 退出码: %j', + id, + result.code + ); + return { ...cron, pid: cp.pid, ...result } as any; + } catch (error) { + this.logger.error( + '[panel][创建任务失败] 任务ID: %s, 错误: %s', + cronId, + asError(error).message + ); + } finally { + try { + if (absolutePath) await logStreamManager.closeStream(absolutePath); + } catch (error) { + this.logger.error( + '[panel][关闭任务日志失败] %s', + asError(error).message ); - await logStreamManager.closeStream(absolutePath); - resolve({ ...params, pid: cp.pid, code }); - }); - }); + } + try { + // Do not overwrite a newer run's state or its script-reported exit code. + await CrontabModel.update( + { status: CrontabStatus.idle, pid: null, queued_token: null } as any, + { + where: { + id: cronId, + [Op.and]: where(colFn('queued_token'), { [Op.eq]: queuedToken }), + [Op.or]: [ + ...(queuedLogPath !== undefined && !claimed + ? [ + { + status: CrontabStatus.queued, + [Op.and]: where(colFn('log_path'), { + [Op.eq]: queuedLogPath, + }), + }, + ] + : []), + ...(claimed && logPath + ? [{ log_path: logPath, status: CrontabStatus.running }] + : []), + ], + }, + } + ); + } catch (error) { + this.logger.error( + '[panel][清理任务状态失败] %s', + asError(error).message + ); + } + } }); } public async disabled(ids: number[]) { - await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } }); - try { - await cronClient.delCron(ids.map(String)); - } catch (error: any) { - this.logger.warn( - '[crontab] Failed to unregister cron job in scheduler:', - error?.message || error, - ); - } - await this.setCrontab(); + return withSchedulerMutation(async () => { + await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } }); + try { + await cronClient.delCron(ids.map(String)); + } catch (error: any) { + this.logger.warn( + '[crontab] Failed to unregister cron job in scheduler:', + error?.message || error, + ); + } + await this.setCrontab(); + }); } public async enabled(ids: number[]) { - await CrontabModel.update({ isDisabled: 0 }, { where: { id: ids } }); - const docs = await CrontabModel.findAll({ where: { id: ids } }); - const crons = docs - .filter((x) => this.shouldUseCronClient(x)) - .map((doc) => ({ - name: doc.name || '', - id: String(doc.id), - schedule: doc.schedule!, - command: this.makeCommand(doc), - extra_schedules: doc.extra_schedules || [], - })); + return withSchedulerMutation(async () => { + await CrontabModel.update({ isDisabled: 0 }, { where: { id: ids } }); + const docs = await CrontabModel.findAll({ where: { id: ids } }); + const crons = docs + .filter((x) => this.shouldUseCronClient(x)) + .map((doc) => ({ + name: doc.name || '', + id: String(doc.id), + schedule: doc.schedule!, + command: this.makeCommand(doc), + extra_schedules: doc.extra_schedules || [], + })); - if (isDemoEnv()) { - return; - } + if (isDemoEnv()) { + return; + } - try { - await cronClient.addCron(crons); - } catch (error: any) { - // gRPC 注册失败 → 回滚启用状态，避免 DB 显示已启用但调度器未注册 - await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } }); - this.logger.error( - '[crontab] Failed to register cron job in scheduler, enable rolled back:', - error?.message || error, - ); - throw new Error( - `${t('调度器注册失败，任务启用已回滚')}: ${(error as any)?.details || error?.message}`, - ); - } - await this.setCrontab(); + try { + await cronClient.addCron(crons); + } catch (error: any) { + // gRPC 注册失败 → 回滚启用状态，避免 DB 显示已启用但调度器未注册 + await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } }); + this.logger.error( + '[crontab] Failed to register cron job in scheduler, enable rolled back:', + error?.message || error, + ); + throw schedulerRegistrationError( + `${t('调度器注册失败，任务启用已回滚')}: ${(error as any)?.details || error?.message}`, + error, + ); + } + await this.setCrontab(); + }); } public async log( @@ -957,39 +1092,42 @@ export default class CronService { }); } - public async autosave_crontab() { - const tabs = await this.crontabs(); - const regularCrons = tabs.data - .filter( - (x) => - x.isDisabled !== 1 && - this.shouldUseCronClient(x), - ) - .map((doc) => ({ - name: doc.name || '', - id: String(doc.id), - schedule: doc.schedule!, - command: this.makeCommand(doc), - extra_schedules: doc.extra_schedules || [], - })); + public async autosave_crontab(requireScheduler = false) { + return withSchedulerMutation(async () => { + const tabs = await this.crontabs(); + const regularCrons = tabs.data + .filter( + (x) => + x.isDisabled !== 1 && + this.shouldUseCronClient(x), + ) + .map((doc) => ({ + name: doc.name || '', + id: String(doc.id), + schedule: doc.schedule!, + command: this.makeCommand(doc), + extra_schedules: doc.extra_schedules || [], + })); - if (isDemoEnv()) { - await writeFileWithLock(config.crontabFile, ''); - return; - } + if (isDemoEnv()) { + await writeFileWithLock(config.crontabFile, ''); + return; + } - // 先同步 crontab.list 与系统 crontab，确保其始终反映数据库真实状态。 - // gRPC 调度注册为尽力而为：失败时不阻断文件同步，调度器重启后会重新注册。 - // 这避免了因调度器短暂不可用导致 crontab.list 与数据库脱节（订阅更新误判任务已存在）。 - await this.setCrontab(tabs); - try { - await cronClient.addCron(regularCrons); - } catch (error: any) { - this.logger.warn( - '[crontab] Failed to register cron job in scheduler:', - error?.message || error, - ); - } + // 先同步 crontab.list 与系统 crontab，确保其始终反映数据库真实状态。 + // gRPC 调度注册为尽力而为：失败时不阻断文件同步，调度器重启后会重新注册。 + // 这避免了因调度器短暂不可用导致 crontab.list 与数据库脱节（订阅更新误判任务已存在）。 + await this.setCrontab(tabs); + try { + await cronClient.addCron(regularCrons, requireScheduler); + } catch (error: any) { + this.logger.warn( + '[crontab] Failed to register cron job in scheduler:', + error?.message || error, + ); + if (requireScheduler) throw error; + } + }); } public async bootTask() { @@ -998,13 +1136,7 @@ export default class CronService { (x) => !x.isDisabled && this.isBootSchedule(x.schedule), ); if (bootTasks.length > 0) { - await CrontabModel.update( - { status: CrontabStatus.queued }, - { where: { id: bootTasks.map((t) => t.id!) } }, - ); - for (const task of bootTasks) { - this.runSingle(task.id!); - } + await this.run(bootTasks.map((task) => task.id!)); } } } diff --git a/back/services/health.ts b/back/services/health.ts index d971768d..3c86f5fa 100644 --- a/back/services/health.ts +++ b/back/services/health.ts @@ -1,6 +1,6 @@ import { Service } from 'typedi'; import Logger from '../loaders/logger'; -import { GrpcServerService } from './grpc'; +import cronClient from '../schedule/client'; import { HttpServerService } from './http'; interface HealthStatus { @@ -23,7 +23,6 @@ export class HealthService { private startTime = Date.now(); constructor( - private grpcServerService: GrpcServerService, private httpServerService: HttpServerService, ) {} @@ -56,8 +55,7 @@ export class HealthService { } try { - const grpcServer = this.grpcServerService.getServer(); - if (!grpcServer) { + if (!(await cronClient.readiness.check())) { status.services.grpc = false; status.status = 'error'; } diff --git a/back/services/http.ts b/back/services/http.ts index f6d6dba3..44ee30ee 100644 --- a/back/services/http.ts +++ b/back/services/http.ts @@ -39,7 +39,13 @@ export class HttpServerService { private async tryListen(expressApp: express.Application, port: number, host: string): Promise { return new Promise((resolve, reject) => { - const server = expressApp.listen(port, host, () => { + // There is one HTTP worker; accepting here avoids primary IPC handoff + // for every connection. Restore shared listening for custom clusters. + const server = expressApp.listen({ + port, + host, + exclusive: process.env.QL_HTTP_SHARED_LISTEN !== 'true', + }, () => { resolve(server); }); diff --git a/back/services/schedule.ts b/back/services/schedule.ts index df938674..884894d9 100644 --- a/back/services/schedule.ts +++ b/back/services/schedule.ts @@ -11,6 +11,7 @@ import { import dayjs from 'dayjs'; import taskLimit from '../shared/pLimit'; import { spawn } from 'cross-spawn'; +import { observeChildProcess, asError, ProcessResult } from '../shared/childProcess'; export interface ScheduleTaskType { id?: number; @@ -27,7 +28,7 @@ export interface TaskCallbacks { startTime: dayjs.Dayjs, ) => Promise; onEnd?: ( - cp: ChildProcessWithoutNullStreams, + cp: ChildProcessWithoutNullStreams | undefined, endTime: dayjs.Dayjs, diff: number, ) => Promise; @@ -63,73 +64,85 @@ export default class ScheduleService { ) { const { runOrigin, ...others } = params; - return taskLimit[this.taskLimitMap[runOrigin]](others, () => { - return new Promise(async (resolve, reject) => { - this.logger.info( - `[panel][开始执行任务] 参数: ${JSON.stringify({ - ...others, - command, - })}`, - ); - - try { - const startTime = dayjs(); - await callbacks.onBefore?.(startTime); - - const cp = spawn(command, { shell: '/bin/bash' }); - - callbacks.onStart?.(cp, startTime); - completionTime === 'start' && resolve(cp.pid); - - cp.stdout.on('data', async (data) => { - await callbacks.onLog?.(data.toString()); - }); - - cp.stderr.on('data', async (data) => { - this.logger.info( - '[panel][执行任务失败] 命令: %s, 错误信息: %j', - command, - data.toString(), - ); - await callbacks.onError?.(data.toString()); - }); - - cp.on('error', async (err) => { - this.logger.error( - '[panel][创建任务失败] 命令: %s, 错误信息: %j', - command, - err, - ); - await callbacks.onError?.(JSON.stringify(err)); - }); - - cp.on('exit', async (code) => { - this.logger.info( - '[panel][执行任务结束] 参数: %s, 退出码: %j', - JSON.stringify({ - ...others, - command, - }), - code, - ); - const endTime = dayjs(); - await callbacks.onEnd?.( - cp, - endTime, - endTime.diff(startTime, 'seconds'), - ); - resolve({ ...others, pid: cp.pid, code }); - }); - } catch (error) { - this.logger.error( - '[panel][执行任务失败] 命令: %s, 错误信息: %j', - command, - error, - ); - await callbacks.onError?.(JSON.stringify(error)); - } - }); + let resolveStart!: (pid: number | undefined) => void; + let rejectStart!: (error: Error) => void; + const startResult = new Promise((resolve, reject) => { + resolveStart = resolve; + rejectStart = reject; }); + // Most scheduled callers only observe completion (or intentionally detach). + void startResult.catch(() => {}); + const completion = taskLimit[this.taskLimitMap[runOrigin]]( + others, + async () => { + const startTime = dayjs(); + let cp: ChildProcessWithoutNullStreams | undefined; + let result: ProcessResult = { code: null, signal: null }; + try { + this.logger.info('[panel][开始执行任务] 任务ID: %s', others.id); + await callbacks.onBefore?.(startTime); + cp = spawn(command, { shell: '/bin/bash' }); + const child = cp; + const observed = observeChildProcess(child, { + onStart: async () => { + await callbacks.onStart?.(child, startTime); + }, + onStdout: callbacks.onLog, + onStderr: callbacks.onError, + }); + observed.started.then(resolveStart, rejectStart); + result = await observed.completed; + } catch (error) { + result.error = asError(error); + rejectStart(result.error); + } + + if (result.error) { + this.logger.error( + '[panel][执行任务失败] 任务ID: %s, 错误: %s', + others.id, + result.error.message, + ); + try { + await callbacks.onError?.(result.error.message); + } catch (error) { + this.logger.error( + '[panel][任务错误回调失败] %s', + asError(error).message, + ); + } + } + // Cleanup also runs after setup/spawn failure, and only after both pipes drain. + const endTime = dayjs(); + try { + await callbacks.onEnd?.( + cp, + endTime, + endTime.diff(startTime, 'seconds'), + ); + } catch (error) { + result.error ??= asError(error); + this.logger.error( + '[panel][任务结束回调失败] %s', + asError(error).message, + ); + } + this.logger.info( + '[panel][执行任务结束] 任务ID: %s, 退出码: %j', + others.id, + result.code, + ); + return { ...others, pid: cp?.pid, ...result }; + }, + ).catch((error) => { + // Queue/setup failures must not become unhandled rejections in detached callers. + rejectStart(asError(error)); + this.logger.error('[panel][任务队列失败] %s', asError(error).message); + return { ...others, code: null, signal: null, error: asError(error) }; + }); + + // Returning a PID must not release the execution slot while the process runs. + return completionTime === 'start' ? startResult : completion; } async createCronTask( diff --git a/back/services/subscription.ts b/back/services/subscription.ts index 6d9e35fa..513d239f 100644 --- a/back/services/subscription.ts +++ b/back/services/subscription.ts @@ -159,48 +159,54 @@ export default class SubscriptionService { ); }, onEnd: async (cp, endTime, diff) => { - const sub = await this.getDb({ id: doc.id }); - const absolutePath = await handleLogPath(sub.log_path as string); - - // 执行 sub_after - let afterStr = ''; + let absolutePath: string | undefined; try { - if (sub.sub_after) { - await logStreamManager.write(absolutePath, `\n\n## ${t('执行after命令...')}\n\n`); - afterStr = await promiseExec(sub.sub_after); + const sub = await this.getDb({ id: doc.id }); + absolutePath = await handleLogPath(sub.log_path as string); + + // 执行 sub_after + let afterStr = ''; + try { + if (sub.sub_after) { + await logStreamManager.write( + absolutePath, + `\n\n## ${t('执行after命令...')}\n\n`, + ); + afterStr = await promiseExec(sub.sub_after); + } + } catch (error: any) { + afterStr = + (error.stderr && error.stderr.toString()) || JSON.stringify(error); + } + if (afterStr) { + await logStreamManager.write(absolutePath, `${afterStr}\n`); + } + + await logStreamManager.write( + absolutePath, + '\n' + + tf( + '## 执行结束... %s 耗时 %s 秒', + endTime.format('YYYY-MM-DD HH:mm:ss'), + String(diff), + ) + + LOG_END_SYMBOL, + ); + } finally { + try { + if (absolutePath) await logStreamManager.closeStream(absolutePath); + } finally { + await SubscriptionModel.update( + { status: SubscriptionStatus.idle, pid: null } as any, + { where: { id: doc.id } }, + ); + this.sockService.sendMessage({ + type: 'runSubscriptionEnd', + message: t('订阅执行完成'), + references: [doc.id as number], + }); } - } catch (error: any) { - afterStr = - (error.stderr && error.stderr.toString()) || JSON.stringify(error); } - if (afterStr) { - await logStreamManager.write(absolutePath, `${afterStr}\n`); - } - - await logStreamManager.write( - absolutePath, - '\n' + - tf( - '## 执行结束... %s 耗时 %s 秒', - endTime.format('YYYY-MM-DD HH:mm:ss'), - String(diff), - ) + - LOG_END_SYMBOL, - ); - - // Close the stream after task completion - await logStreamManager.closeStream(absolutePath); - - await SubscriptionModel.update( - { status: SubscriptionStatus.idle, pid: undefined }, - { where: { id: sub.id } }, - ); - - this.sockService.sendMessage({ - type: 'runSubscriptionEnd', - message: t('订阅执行完成'), - references: [doc.id as number], - }); }, onError: async (message: string) => { const sub = await this.getDb({ id: doc.id }); diff --git a/back/shared/childProcess.ts b/back/shared/childProcess.ts new file mode 100644 index 00000000..a5cacb5f --- /dev/null +++ b/back/shared/childProcess.ts @@ -0,0 +1,78 @@ +import { ChildProcessWithoutNullStreams } from 'child_process'; +import { Readable } from 'stream'; + +export interface ProcessResult { + code: number | null; + signal: NodeJS.Signals | null; + error?: Error; +} + +export function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +/** Attach immediately after spawn, before awaiting database or user callbacks. */ +export function observeChildProcess( + child: ChildProcessWithoutNullStreams, + callbacks: { + onStart?: () => Promise; + onStdout?: (message: string) => Promise; + onStderr?: (message: string) => Promise; + } = {}, +) { + let failure: Error | undefined; + const recordError = (error: unknown) => { + failure ??= asError(error); + }; + const spawned = new Promise((resolve, reject) => { + child.once('spawn', resolve); + // Keep the listener through close: errors can occur after a successful spawn. + child.on('error', (error) => { + recordError(error); + reject(error); + }); + }); + const closed = new Promise((resolve) => { + child.once('close', (code, signal) => resolve({ code, signal })); + }); + const started = spawned.then(async () => { + await callbacks.onStart?.(); + return child.pid; + }); + // The caller can ask only for completion, without an unhandled start rejection. + const ready = started.catch(recordError); + + const consume = async ( + stream: Readable, + callback?: (message: string) => Promise, + ) => { + // StringDecoder in Readable preserves UTF-8 characters split across chunks. + stream.setEncoding('utf8'); + let callbackFailed = false; + try { + for await (const chunk of stream) { + await ready; + if (!callbackFailed && callback) { + try { + await callback(String(chunk)); + } catch (error) { + recordError(error); + callbackFailed = true; + } + } + // Even if a log sink fails, drain the pipe so the child can finish. + } + } catch (error) { + recordError(error); + } + }; + const output = Promise.all([ + consume(child.stdout, callbacks.onStdout), + consume(child.stderr, callbacks.onStderr), + ]); + const completed = Promise.all([closed, ready, output]).then(([result]) => ({ + ...result, + error: failure, + })); + return { started, completed }; +} diff --git a/back/shared/logStreamManager.ts b/back/shared/logStreamManager.ts index 815ce409..ff2ff1ec 100644 --- a/back/shared/logStreamManager.ts +++ b/back/shared/logStreamManager.ts @@ -1,5 +1,8 @@ import { createWriteStream, WriteStream } from 'fs'; import { EventEmitter } from 'events'; +import path from 'path'; +import config from '../config'; +import { resolveFileAccess } from './fileAccess'; /** * Manages write streams for log files to improve performance by avoiding repeated file opens @@ -8,83 +11,90 @@ export class LogStreamManager extends EventEmitter { private streams: Map = new Map(); private pendingWrites: Map> = new Map(); - /** - * Write data to a log file using a managed stream - * @param filePath - Absolute path to the log file - * @param data - Data to write to the log file - */ - async write(filePath: string, data: string): Promise { - // Wait for any pending writes to this file to complete - const pending = this.pendingWrites.get(filePath); - if (pending) { - await pending; - } + private closingStreams = new Map>(); + private closedStreams = new WeakSet(); + private streamErrors = new Map(); - // Create a new promise for this write operation - const writePromise = new Promise((resolve, reject) => { - let stream = this.streams.get(filePath); - - if (!stream) { - // Create a new write stream if one doesn't exist - stream = createWriteStream(filePath, { flags: 'a' }); - this.streams.set(filePath, stream); - - // Handle stream errors - stream.on('error', (error) => { - this.emit('error', { filePath, error }); - // Remove the stream from the map on error - this.streams.delete(filePath); - reject(error); - }); - } - - // Write the data - const canContinue = stream.write(data, 'utf8', (error) => { - if (error) { - reject(error); - } else { - resolve(); - } - }); - - // Handle backpressure - if (!canContinue) { - stream.once('drain', () => { - // Stream is ready for more data - }); - } - }); - - this.pendingWrites.set(filePath, writePromise); - - try { - await writePromise; - } finally { - this.pendingWrites.delete(filePath); - } + constructor(private readonly logRoot = config.logPath) { + super(); } - /** - * Close the stream for a specific file path - * @param filePath - Absolute path to the log file - */ - async closeStream(filePath: string): Promise { - // Wait for any pending writes to complete - const pending = this.pendingWrites.get(filePath); - if (pending) { - await pending.catch(() => { - // Ignore errors on pending writes during close - }); + /** Register each write synchronously, so concurrent callers cannot lose the tail. */ + async write(filePath: string, data: string): Promise { + if (this.closingStreams.has(filePath)) { + throw new Error(`Log stream is closing: ${filePath}`); } + const previous = this.pendingWrites.get(filePath) || Promise.resolve(); + const pending = previous.then( + () => + new Promise((resolve, reject) => { + const failure = this.streamErrors.get(filePath); + if (failure) return reject(failure); + let stream = this.streams.get(filePath); + if (!stream) { + // Validate only when opening: subsequent chunks reuse the same descriptor. + const root = path.resolve(this.logRoot); + const target = path.resolve(filePath); + if ( + !target.startsWith(root + path.sep) || + !resolveFileAccess(root, [target]) + ) { + return reject(new Error('Log path is outside the log directory')); + } + stream = createWriteStream(target, { flags: 'a' }); + this.streams.set(filePath, stream); + const current = stream; + stream.once('close', () => this.closedStreams.add(current)); + stream.on('error', (error) => { + this.streamErrors.set(filePath, error); + // EventEmitter's unobserved "error" event would crash the caller. + if (this.listenerCount('error') > 0) + this.emit('error', { filePath, error }); + }); + } + stream.write(data, 'utf8', (error) => + error ? reject(error) : resolve(), + ); + }), + ); + this.pendingWrites.set(filePath, pending); + // Keep the tail until close, including failures; never reopen a failed log mid-run. + return pending; + } - const stream = this.streams.get(filePath); - if (stream) { - return new Promise((resolve) => { - stream.end(() => { - this.streams.delete(filePath); - resolve(); - }); - }); + async closeStream(filePath: string): Promise { + const closing = this.closingStreams.get(filePath); + if (closing) return closing; + const pending = this.pendingWrites.get(filePath); + const result = (async () => { + let failure: unknown; + try { + await pending; + } catch (error) { + failure = error; + } + const stream = this.streams.get(filePath); + try { + if (stream && !this.closedStreams.has(stream)) { + await new Promise((resolve) => { + stream.once('close', resolve); + if (failure || stream.destroyed) stream.destroy(); + else stream.end(); + }); + } + failure ||= this.streamErrors.get(filePath); + if (failure) throw failure; + } finally { + this.streams.delete(filePath); + this.pendingWrites.delete(filePath); + this.streamErrors.delete(filePath); + } + })(); + this.closingStreams.set(filePath, result); + try { + await result; + } finally { + this.closingStreams.delete(filePath); } } @@ -92,7 +102,11 @@ export class LogStreamManager extends EventEmitter { * Close all open streams */ async closeAll(): Promise { - const closePromises = Array.from(this.streams.keys()).map((filePath) => + const paths = new Set([ + ...this.streams.keys(), + ...this.pendingWrites.keys(), + ]); + const closePromises = Array.from(paths).map((filePath) => this.closeStream(filePath), ); await Promise.all(closePromises); diff --git a/back/shared/runCron.ts b/back/shared/runCron.ts index c451bf4f..29609e7a 100644 --- a/back/shared/runCron.ts +++ b/back/shared/runCron.ts @@ -4,15 +4,13 @@ import Logger from '../loaders/logger'; import { ICron } from '../protos/cron'; import { CrontabModel, CrontabStatus } from '../data/cron'; import { killTask } from '../config/util'; -import { - RunningInstanceModel, - InstanceStatus, -} from '../data/runningInstance'; +import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance'; import dayjs from 'dayjs'; +import { observeChildProcess, asError } from './childProcess'; export function runCron(cmd: string, cron: ICron): Promise { - return taskLimit.runWithCronLimit(cron, () => { - return new Promise(async (resolve: any) => { + return taskLimit.runWithCronLimit(cron, async () => { + try { // Check if the cron is already running and stop it (only if multiple instances are not allowed) try { const existingCron = await CrontabModel.findOne({ @@ -38,7 +36,12 @@ export function runCron(cmd: string, cron: ICron): Promise { const stoppedAt = dayjs().unix(); await RunningInstanceModel.update( { status: InstanceStatus.stopped, finished_at: stoppedAt }, - { where: { cron_id: Number(cron.id), status: InstanceStatus.running } }, + { + where: { + cron_id: Number(cron.id), + status: InstanceStatus.running, + }, + }, ); // Update the status to idle after killing await CrontabModel.update( @@ -60,33 +63,37 @@ export function runCron(cmd: string, cron: ICron): Promise { ); const cp = spawn(cmd, { shell: '/bin/bash' }); - cp.stderr.on('data', (data) => { - Logger.info( - '[schedule][执行任务失败] 命令: %s, 错误信息: %j', - cmd, - data.toString(), - ); + const { completed } = observeChildProcess(cp, { + onStderr: async (message) => { + Logger.info( + '[schedule][任务标准错误] 命令: %s, 信息: %s', + cmd, + message, + ); + }, }); - cp.on('error', (err) => { + const result = await completed; + if (result.error) { Logger.error( - '[schedule][创建任务失败] 命令: %s, 错误信息: %j', + '[schedule][执行任务失败] 命令: %s, 错误: %s', cmd, - err, + result.error.message, ); - }); - - cp.on('exit', async (code) => { - taskLimit.removeQueuedCron(cron.id); - Logger.info( - '[schedule][执行任务结束] 参数: %s, 退出码: %j', - JSON.stringify({ - ...cron, - command: cmd, - }), - code, - ); - resolve({ ...cron, command: cmd, pid: cp.pid, code }); - }); - }); + } + Logger.info( + '[schedule][执行任务结束] 任务ID: %s, 退出码: %j', + cron.id, + result.code, + ); + return { ...cron, command: cmd, pid: cp.pid, ...result } as any; + } catch (error) { + Logger.error( + '[schedule][创建任务失败] 命令: %s, 错误: %s', + cmd, + asError(error).message, + ); + } finally { + taskLimit.removeQueuedCron(cron.id); + } }); } diff --git a/back/shared/schedulerMutationLock.ts b/back/shared/schedulerMutationLock.ts new file mode 100644 index 00000000..afa2d4e0 --- /dev/null +++ b/back/shared/schedulerMutationLock.ts @@ -0,0 +1,38 @@ +import lockfile from 'proper-lockfile'; +import config from '../config'; + +// HTTP and gRPC both mutate cron definitions. Hold one shared lock from the +// initial DB read/write through scheduler registration (including rollback). +// Recovery takes the same lock before reading its replacement snapshot. +export async function withSchedulerMutation( + operation: () => Promise, +): Promise { + let release: () => Promise; + try { + release = await lockfile.lock(config.crontabFile, { + realpath: false, + lockfilePath: `${config.crontabFile}.scheduler.lock`, + stale: 30000, + update: 10000, + retries: { retries: 50, factor: 1, minTimeout: 100, maxTimeout: 100 }, + }); + } catch (cause) { + throw Object.assign( + new Error('Scheduler configuration is busy', { cause }), + { + status: 503, + }, + ); + } + try { + return await operation(); + } finally { + await release(); + } +} + +export function schedulerRegistrationError(message: string, cause: any): Error { + return Object.assign(new Error(message, { cause }), { + status: cause?.status === 503 ? 503 : 500, + }); +} diff --git a/back/shared/schedulerReadiness.ts b/back/shared/schedulerReadiness.ts new file mode 100644 index 00000000..f2100695 --- /dev/null +++ b/back/shared/schedulerReadiness.ts @@ -0,0 +1,77 @@ +// Recovery is single-flight and retried only while unavailable, never at idle. +export class SchedulerReadiness { + private ready = false; + private generation = 0; + private restore?: () => Promise; + private pending?: Promise; + private retry?: NodeJS.Timeout; + + constructor(private probe: () => Promise, private retryMs = 1000) {} + + configure(restore: () => Promise) { + this.restore = restore; + } + + invalidate() { + this.ready = false; + this.generation++; + void this.recover(); + } + + recover(): Promise { + if (this.pending) return this.pending; + if (!this.restore) return Promise.resolve(false); + clearTimeout(this.retry); + const generation = this.generation; + this.ready = false; + this.pending = (async () => { + try { + await this.probe(); + await this.restore!(); + await this.probe(); + this.ready = generation === this.generation; + } catch { + this.ready = false; + } + return this.ready; + })().finally(() => { + this.pending = undefined; + if (!this.ready) { + this.retry = setTimeout(() => void this.recover(), this.retryMs); + this.retry.unref(); + } + }); + return this.pending; + } + + async check(): Promise { + if (!this.ready) return false; + const generation = this.generation; + try { + await this.probe(); + return this.ready && generation === this.generation; + } catch { + this.invalidate(); + return false; + } + } + + async ensureReady(timeoutMs = 2000): Promise { + let timer: NodeJS.Timeout | undefined; + try { + const available = await Promise.race([ + (async () => (await this.check()) || (await this.recover()))(), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + if (!available) { + throw Object.assign(new Error('Scheduler is recovering; try again later'), { + status: 503, + }); + } + } finally { + clearTimeout(timer); + } + } +} diff --git a/back/shared/schemaMigrations.ts b/back/shared/schemaMigrations.ts new file mode 100644 index 00000000..ed7a6658 --- /dev/null +++ b/back/shared/schemaMigrations.ts @@ -0,0 +1,70 @@ +import { QueryTypes, Sequelize } from 'sequelize'; + +// Append new entries; IDs are persisted and must not be renumbered or reused. +const columns = [ + { + table: 'CrontabViews', + column: 'filterRelation', + type: 'VARCHAR(255)', + }, + { table: 'Subscriptions', column: 'proxy', type: 'VARCHAR(255)' }, + { table: 'CrontabViews', column: 'type', type: 'NUMBER' }, + { table: 'Subscriptions', column: 'autoAddCron', type: 'NUMBER' }, + { table: 'Subscriptions', column: 'autoDelCron', type: 'NUMBER' }, + { table: 'Crontabs', column: 'sub_id', type: 'NUMBER' }, + { table: 'Crontabs', column: 'extra_schedules', type: 'JSON' }, + { table: 'Crontabs', column: 'task_before', type: 'TEXT' }, + { table: 'Crontabs', column: 'task_after', type: 'TEXT' }, + { table: 'Crontabs', column: 'log_name', type: 'VARCHAR(255)' }, + { + table: 'Crontabs', + column: 'allow_multiple_instances', + type: 'NUMBER', + }, + { table: 'Crontabs', column: 'work_dir', type: 'VARCHAR(255)' }, + { table: 'Envs', column: 'isPinned', type: 'NUMBER' }, + { table: 'Envs', column: 'labels', type: 'JSON' }, + { table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' }, +]; + +export async function migrateSchema(database: Sequelize): Promise { + await database.transaction(async (transaction) => { + await database.query( + 'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)', + { transaction }, + ); + const applied = await database.query<{ id: string }>( + 'SELECT "id" FROM "SchemaMigrations"', + { type: QueryTypes.SELECT, transaction }, + ); + const appliedIds = new Set(applied.map(({ id }) => id)); + for (const { table, column, type } of columns) { + const id = `add-${table}-${column}`; + const fields = await database.query<{ name: string }>( + `PRAGMA table_info("${table}")`, + { + type: QueryTypes.SELECT, + transaction, + }, + ); + if (fields.length === 0) + throw new Error(`Migration table is missing: ${table}`); + if (!fields.some((field) => field.name === column)) { + // table/column/type come only from the static migration manifest above. + await database.query( + `ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`, + { transaction }, + ); + } + if (!appliedIds.has(id)) { + await database.query( + 'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)', + { + replacements: { id, appliedAt: new Date().toISOString() }, + transaction, + }, + ); + } + } + }); +} diff --git a/docker/Dockerfile b/docker/Dockerfile index f61ef404..7b75859e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ RUN set -x \ && npm_config_target_platform=linux \ npm_config_target_arch="${NODE_ARCH}" \ npm_config_target_libc=musl \ - pnpm install --prod + pnpm install --prod --frozen-lockfile FROM python:3.11-alpine @@ -73,15 +73,19 @@ RUN set -x \ && ulimit -c 0 ARG SOURCE_COMMIT +LABEL org.opencontainers.image.revision=${SOURCE_COMMIT} RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \ && cd ${QL_DIR} \ + && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \ && cp -f .env.example .env \ && chmod 777 ${QL_DIR}/shell/*.sh \ - && chmod 777 ${QL_DIR}/docker/*.sh \ - && git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /static \ - && mkdir -p ${QL_DIR}/static \ - && cp -rf /static/* ${QL_DIR}/static \ - && rm -rf /static + && chmod 777 ${QL_DIR}/docker/*.sh + +# Downloaded by CI from the build-static job in this workflow run. +COPY static/ /ql/static/ +COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/ +COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml +RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \ PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \ diff --git a/docker/Dockerfile.310 b/docker/Dockerfile.310 index 520b1e0e..7b414513 100644 --- a/docker/Dockerfile.310 +++ b/docker/Dockerfile.310 @@ -20,7 +20,7 @@ RUN set -x \ && npm_config_target_platform=linux \ npm_config_target_arch="${NODE_ARCH}" \ npm_config_target_libc=musl \ - pnpm install --prod + pnpm install --prod --frozen-lockfile FROM python:3.10-alpine @@ -73,15 +73,19 @@ RUN set -x \ && ulimit -c 0 ARG SOURCE_COMMIT +LABEL org.opencontainers.image.revision=${SOURCE_COMMIT} RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \ && cd ${QL_DIR} \ + && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \ && cp -f .env.example .env \ && chmod 777 ${QL_DIR}/shell/*.sh \ - && chmod 777 ${QL_DIR}/docker/*.sh \ - && git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /static \ - && mkdir -p ${QL_DIR}/static \ - && cp -rf /static/* ${QL_DIR}/static \ - && rm -rf /static + && chmod 777 ${QL_DIR}/docker/*.sh + +# Downloaded by CI from the build-static job in this workflow run. +COPY static/ /ql/static/ +COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/ +COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml +RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \ PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \ diff --git a/docker/Dockerfile.debian b/docker/Dockerfile.debian index 09bbbeae..fed71e8b 100644 --- a/docker/Dockerfile.debian +++ b/docker/Dockerfile.debian @@ -12,7 +12,7 @@ RUN set -x && \ apt-get install --no-install-recommends -y libatomic1 && \ npm i -g pnpm@8.3.1 && \ cd /tmp/build && \ - pnpm install --prod + pnpm install --prod --frozen-lockfile FROM python:3.11.14-slim-bookworm @@ -82,15 +82,19 @@ RUN mkdir -p ${QL_DIR} && \ USER qinglong ARG SOURCE_COMMIT -RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} && \ - cd ${QL_DIR} && \ - cp -f .env.example .env && \ - chmod 777 ${QL_DIR}/shell/*.sh && \ - chmod 777 ${QL_DIR}/docker/*.sh && \ - git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /tmp/static && \ - mkdir -p ${QL_DIR}/static && \ - cp -rf /tmp/static/* ${QL_DIR}/static && \ - rm -rf /tmp/static +LABEL org.opencontainers.image.revision=${SOURCE_COMMIT} +RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \ + && cd ${QL_DIR} \ + && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \ + && cp -f .env.example .env \ + && chmod 777 ${QL_DIR}/shell/*.sh \ + && chmod 777 ${QL_DIR}/docker/*.sh + +# Downloaded by CI from the build-static job in this workflow run. +COPY --chown=qinglong:qinglong static/ /ql/static/ +COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/ +COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml +RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \ PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \ diff --git a/docker/Dockerfile.debian310 b/docker/Dockerfile.debian310 index 36bc0d41..907da2a1 100644 --- a/docker/Dockerfile.debian310 +++ b/docker/Dockerfile.debian310 @@ -12,7 +12,7 @@ RUN set -x && \ apt-get install --no-install-recommends -y libatomic1 && \ npm i -g pnpm@8.3.1 && \ cd /tmp/build && \ - pnpm install --prod + pnpm install --prod --frozen-lockfile FROM python:3.10-slim-bookworm @@ -82,15 +82,19 @@ RUN mkdir -p ${QL_DIR} && \ USER qinglong ARG SOURCE_COMMIT -RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} && \ - cd ${QL_DIR} && \ - cp -f .env.example .env && \ - chmod 777 ${QL_DIR}/shell/*.sh && \ - chmod 777 ${QL_DIR}/docker/*.sh && \ - git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /tmp/static && \ - mkdir -p ${QL_DIR}/static && \ - cp -rf /tmp/static/* ${QL_DIR}/static && \ - rm -rf /tmp/static +LABEL org.opencontainers.image.revision=${SOURCE_COMMIT} +RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \ + && cd ${QL_DIR} \ + && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \ + && cp -f .env.example .env \ + && chmod 777 ${QL_DIR}/shell/*.sh \ + && chmod 777 ${QL_DIR}/docker/*.sh + +# Downloaded by CI from the build-static job in this workflow run. +COPY --chown=qinglong:qinglong static/ /ql/static/ +COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/ +COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml +RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \ PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \ diff --git a/docker/build-manifest.cjs b/docker/build-manifest.cjs new file mode 100644 index 00000000..d2ef9620 --- /dev/null +++ b/docker/build-manifest.cjs @@ -0,0 +1,28 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); + +// Include every artifact, including imported backend modules and frontend chunks. +function collectBuildFiles(root = 'static') { + const files = {}; + function visit(relative) { + for (const name of fs.readdirSync(path.join(root, relative)).sort()) { + const entry = relative ? `${relative}/${name}` : name; + if (entry === 'build-info.json') continue; + const file = path.join(root, entry); + const stat = fs.lstatSync(file); + if (stat.isSymbolicLink()) + throw new Error(`Build output symlink: ${entry}`); + if (stat.isDirectory()) visit(entry); + else if (stat.isFile()) { + files[entry] = crypto + .createHash('sha256') + .update(fs.readFileSync(file)) + .digest('hex'); + } else throw new Error(`Unsupported build output: ${entry}`); + } + } + visit(''); + return files; +} +module.exports = { collectBuildFiles }; diff --git a/docker/verify-build.cjs b/docker/verify-build.cjs new file mode 100644 index 00000000..8e117da4 --- /dev/null +++ b/docker/verify-build.cjs @@ -0,0 +1,49 @@ +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const { execFileSync } = require('node:child_process'); +const { collectBuildFiles } = require('./build-manifest.cjs'); + +const manifest = JSON.parse(fs.readFileSync('static/build-info.json', 'utf8')); +const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +const lockfileSha256 = crypto + .createHash('sha256') + .update(fs.readFileSync('pnpm-lock.yaml')) + .digest('hex'); +if ( + manifest.dirty !== false || + manifest.sourceCommit !== sourceCommit || + manifest.lockfileSha256 !== lockfileSha256 +) { + throw new Error( + 'Build artifacts do not match the checked-out source and lockfile. Rebuild from the same clean commit.', + ); +} + +if (process.argv[2]) { + const dependencyLock = crypto + .createHash('sha256') + .update(fs.readFileSync(process.argv[2])) + .digest('hex'); + if (dependencyLock !== manifest.lockfileSha256) + throw new Error( + 'Production dependencies were built from a different lockfile.', + ); +} + +const actual = collectBuildFiles(); +if ( + manifest.version !== 1 || + !manifest.files || + JSON.stringify(Object.keys(actual).sort()) !== + JSON.stringify(Object.keys(manifest.files).sort()) +) + throw new Error( + 'Build output file set mismatch. Rebuild the complete artifacts.', + ); +for (const [file, hash] of Object.entries(actual)) { + if (hash !== manifest.files[file]) + throw new Error(`Build output checksum mismatch: ${file}`); +} +console.log(`Verified build artifacts for ${sourceCommit}`); diff --git a/ecosystem.config.js b/ecosystem.config.js index dc9b1e91..6546ed4f 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -4,6 +4,9 @@ module.exports = { apps: [ { name: 'qinglong', + // Keep process supervision; enable injected diagnostics only on demand + // in containers. Standalone installs retain PM2's monitoring default. + pmx: !isContainer || process.env.QL_PRIMARY_APM === 'true', max_restarts: 5, kill_timeout: 1000, wait_ready: true, diff --git a/package.json b/package.json index 56ff1e32..6018bbab 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,9 @@ "start:front": "max dev", "build:front": "max build", "build:back": "tsc -p back/tsconfig.json", + "test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs", + "benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs", + "build:info": "node scripts/write-build-info.cjs", "panel": "npm run build:back && node static/build/app.js", "gen:proto": "protoc --experimental_allow_proto3_optional --plugin=./node_modules/.bin/protoc-gen-ts_proto ./back/protos/*.proto --ts_proto_out=./ --ts_proto_opt=outputServices=grpc-js,env=node,esModuleInterop=true,snakeToCamel=false", "prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'", diff --git a/scripts/benchmark-execution.cjs b/scripts/benchmark-execution.cjs new file mode 100644 index 00000000..553e5694 --- /dev/null +++ b/scripts/benchmark-execution.cjs @@ -0,0 +1,96 @@ +// Microbenchmark of the process observer, not a container or scheduler benchmark. +const fs = require('node:fs'); +const os = require('node:os'); +const crypto = require('node:crypto'); +const { spawn, execFileSync } = require('node:child_process'); +const { performance } = require('node:perf_hooks'); +const { observeChildProcess } = require('../back/shared/childProcess'); + +const args = process.argv.slice(2).filter((arg) => arg !== '--'); +const runsArg = args.find((arg) => arg.startsWith('--runs=')); +const runs = runsArg ? Number(runsArg.slice(7)) : 30; +if (!Number.isInteger(runs) || runs < 3 || runs > 10000) + throw new Error('runs must be between 3 and 10000'); +const output = args.find((arg) => arg.startsWith('--output='))?.slice(9); +const git = (...args) => execFileSync('git', args, { encoding: 'utf8' }).trim(); +const scenarios = [ + { name: 'empty-node', command: '' }, + { + name: 'log-1MiB', + command: 'process.stdout.write("x".repeat(1024 * 1024))', + bytes: 1024 * 1024, + }, + { name: 'spawn-failure', shell: '/nonexistent-ql-benchmark-shell' }, +]; + +(async () => { + const report = { + schema: 1, + observerSha256: crypto + .createHash('sha256') + .update(fs.readFileSync('back/shared/childProcess.ts')) + .digest('hex'), + benchmarkSha256: crypto + .createHash('sha256') + .update(fs.readFileSync(__filename)) + .digest('hex'), + scope: + 'local process observer only; parent CPU excludes child CPU; no panel/container load measured', + sourceCommit: git('rev-parse', 'HEAD'), + dirty: git('status', '--porcelain') !== '', + lockfileSha256: crypto + .createHash('sha256') + .update(fs.readFileSync('pnpm-lock.yaml')) + .digest('hex'), + node: process.version, + platform: process.platform, + arch: process.arch, + cpu: os.cpus()[0]?.model, + cores: os.cpus().length, + timestamp: new Date().toISOString(), + runs, + scenarios: [], + }; + for (const scenario of scenarios) { + const samples = []; + const cpuStart = process.cpuUsage(); + for (let index = 0; index < runs; index++) { + const start = performance.now(); + let bytes = 0; + const child = scenario.shell + ? spawn('true', { shell: scenario.shell }) + : spawn(process.execPath, ['-e', scenario.command]); + const result = await observeChildProcess(child, { + onStdout: async (chunk) => { + bytes += Buffer.byteLength(chunk); + }, + }).completed; + if (scenario.shell ? !result.error : result.error || result.code !== 0) + throw new Error(`Unexpected result: ${scenario.name}`); + if (bytes !== (scenario.bytes || 0)) + throw new Error(`Output loss: ${scenario.name}`); + samples.push({ + wallMs: performance.now() - start, + bytes, + code: result.code, + error: result.error?.message, + }); + } + const sorted = samples.map((sample) => sample.wallMs).sort((a, b) => a - b); + const percentile = (p) => sorted[Math.ceil(sorted.length * p) - 1]; + report.scenarios.push({ + name: scenario.name, + parentCpuMicros: process.cpuUsage(cpuStart), + p50Ms: percentile(0.5), + p95Ms: percentile(0.95), + ...(runs >= 100 ? { p99Ms: percentile(0.99) } : {}), + samples, + }); + } + const json = JSON.stringify(report, null, 2) + '\n'; + if (output) fs.writeFileSync(output, json); + else process.stdout.write(json); +})().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/write-build-info.cjs b/scripts/write-build-info.cjs new file mode 100644 index 00000000..44358b7b --- /dev/null +++ b/scripts/write-build-info.cjs @@ -0,0 +1,27 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { execFileSync } = require('node:child_process'); +const { collectBuildFiles } = require('../docker/build-manifest.cjs'); + +const root = process.cwd(); +for (const output of ['static/build/app.js', 'static/dist/index.html']) { + if (!fs.existsSync(path.join(root, output))) + throw new Error(`Missing build output: ${output}`); +} +const git = (...args) => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim(); +const manifest = { + version: 1, + files: collectBuildFiles(), + sourceCommit: git('rev-parse', 'HEAD'), + dirty: git('status', '--porcelain', '--untracked-files=all') !== '', + lockfileSha256: crypto + .createHash('sha256') + .update(fs.readFileSync('pnpm-lock.yaml')) + .digest('hex'), +}; +fs.writeFileSync( + 'static/build-info.json', + JSON.stringify(manifest, null, 2) + '\n', +); diff --git a/shell/api.sh b/shell/api.sh index 59f3b5f9..2cf9a9ca 100755 --- a/shell/api.sh +++ b/shell/api.sh @@ -10,16 +10,39 @@ create_token() { } get_token() { - if [[ -f $file_auth_token ]]; then - __ql_token__=$(cat $file_auth_token | jq -r .value) - local expiration=$(cat $file_auth_token | jq -r .expiration) - local currentTimeStamp=$(date +%s) - if [[ $currentTimeStamp -ge $expiration ]]; then - create_token + local token_data expiration token_value + local currentTimeStamp=${EPOCHSECONDS:-$(date +%s)} + if [[ -f "$file_auth_token" ]] && token_data=$(jq -er \ + 'select((.expiration | type) == "number" and (.value | type) == "string" and (.value | length) > 0 and (.value | test("[\r\n]") | not)) | .expiration, .value' \ + "$file_auth_token" 2>/dev/null); then + expiration=${token_data%%$'\n'*} + token_value=${token_data#*$'\n'} + if [[ "$expiration" =~ ^[1-9][0-9]{0,10}$ && -n "$token_value" && "$token_value" != *$'\n'* && "$token_value" != *$'\r'* ]] && \ + (( currentTimeStamp < expiration )); then + __ql_token__=$token_value + return 0 fi - else - create_token fi + create_token +} + +# Read the status response once; preserve multiline error messages and the +# legacy code/message variables used by the shell API callers. +ql_parse_status_response() { + # Both task status and statistics endpoints normally return this exact body. + # Match the whole document; all other JSON and malformed responses still + # use jq. Keep its missing-message result for existing callers. + if [[ "$1" == '{"code":200}' ]]; then + code=200 + message=null + return 0 + fi + local parsed + code="" + message="" + parsed=$(jq -r '.code, .message' <<< "$1") || return $? + code=${parsed%%$'\n'*} + message=${parsed#*$'\n'} } add_cron_api() { @@ -142,7 +165,7 @@ update_cron() { local lastExecutingTime="${5:-0}" local runningTime="${6:-0}" local exitCode="${7:-}" - local currentTimeStamp=$(date +%s) + local currentTimeStamp=${EPOCHSECONDS:-$(date +%s)} local dataRaw="{\"ids\":[$ids],\"status\":\"$status\",\"pid\":\"$pid\",\"log_path\":\"$logPath\",\"last_execution_time\":$lastExecutingTime,\"last_running_time\":$runningTime" if [[ -n $exitCode ]]; then dataRaw="${dataRaw},\"exit_code\":$exitCode" @@ -156,8 +179,7 @@ update_cron() { --data-raw "$dataRaw" \ --compressed ) - code=$(echo "$api" | jq -r .code) - message=$(echo "$api" | jq -r .message) + ql_parse_status_response "$api" || true if [[ $code != 200 ]]; then if [[ ! $message ]]; then message="$api" @@ -240,8 +262,7 @@ record_cron_stat() { --data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}" \ --compressed ) - code=$(echo "$api" | jq -r .code) - message=$(echo "$api" | jq -r .message) + ql_parse_status_response "$api" || true if [[ $code != 200 ]]; then if [[ ! $message ]]; then message="$api" diff --git a/shell/node_path_cache.sh b/shell/node_path_cache.sh new file mode 100644 index 00000000..69a0691f --- /dev/null +++ b/shell/node_path_cache.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# Cache discovery only, never the installed packages. Ordinary installs at the +# same root are visible immediately. TTL bounds changes outside known inputs. +ql_node_path_cache_key() { + local pnpm_bin node_bin name directory file + pnpm_bin=$(type -P pnpm) || return 1 + node_bin=$(type -P node) || return 1 + { + printf '%s\n' 'v1' "$EUID" "$PWD" "$PATH" "${HOME:-}" \ + "${PNPM_HOME:-}" "${XDG_CONFIG_HOME:-}" "${XDG_DATA_HOME:-}" \ + "$pnpm_bin" "$node_bin" + for name in ${!npm_config_@} ${!NPM_CONFIG_@}; do + printf '%s=%s\n' "$name" "${!name}" + done + # Follow executable symlinks, detecting upgrades at the same command path. + stat -Lc '%n:%i:%s:%Y:%Z' "$pnpm_bin" "$node_bin" 2>/dev/null || \ + stat -Lf '%N:%i:%z:%m:%c' "$pnpm_bin" "$node_bin" 2>/dev/null + directory=$PWD + while :; do + for file in "$directory/.npmrc" "$directory/pnpm-workspace.yaml"; do + if [[ -f "$file" ]]; then + printf '%s\n' "$file" + cat -- "$file" + printf '\n' + fi + done + [[ "$directory" == / ]] && break + directory=${directory%/*} + [[ -n "$directory" ]] || directory=/ + done + for file in "${HOME:-}/.npmrc" \ + "${XDG_CONFIG_HOME:-${HOME:-}/.config}/pnpm/rc" \ + "${npm_config_userconfig:-${NPM_CONFIG_USERCONFIG:-/dev/null}}" \ + "${npm_config_globalconfig:-${NPM_CONFIG_GLOBALCONFIG:-/dev/null}}"; do + if [[ -f "$file" ]]; then + printf '%s\n' "$file" + cat -- "$file" + printf '\n' + fi + done + } | cksum +} + +ql_read_node_path_cache() { + local cache="$1" key="$2" now + local stored_key="" stored_at="" stored_path="" + now=${EPOCHSECONDS:-$(date +%s)} + if [[ -f "$cache" && ! -L "$cache" && -O "$cache" ]]; then + if ! { + IFS= read -r stored_key && IFS= read -r stored_at && IFS= read -r stored_path + } < "$cache"; then + return 1 + fi + if [[ "$stored_key" == "$key" && "$stored_at" =~ ^(0|[1-9][0-9]{0,10})$ && "$stored_path" == /* ]] && \ + (( now >= stored_at && now - stored_at < 60 )); then + printf '%s\n' "$stored_path" + return 0 + fi + fi + return 1 +} + +# Keep the lock descriptor in a subshell so sourcing this helper never changes +# the caller's descriptors. A crashed refresher releases the kernel lock. +ql_refresh_node_global_path() ( + local cache="$1" key="$2" now result previous_umask + local lock="${cache}.lock" + previous_umask=$(umask) + umask 077 + if type -P flock &>/dev/null && mkdir -p -- "$dir_tmp" 2>/dev/null; then + if [[ ! -L "$lock" && ( ! -e "$lock" || ( -f "$lock" && -O "$lock" ) ) ]] && \ + { exec 9>> "$lock"; } 2>/dev/null; then + # Bound the wait; absent/unsupported flock or contention falls back to + # independent discovery. Never remove the lock file while waiters exist. + if flock -w 2 9 2>/dev/null; then + if ql_read_node_path_cache "$cache" "$key"; then + return 0 + fi + fi + fi + fi + + # Private lock creation must not change pnpm's inherited creation mask. + umask "$previous_umask" + now=${EPOCHSECONDS:-$(date +%s)} + result=$(pnpm root -g 9>&- 2>/dev/null) || return $? + # Never cache failed, empty, multiline or non-absolute answers. + if [[ "$result" == /* && "$result" != *$'\n'* && "$result" != *$'\r'* ]]; then + ( + umask 077 + mkdir -p -- "$dir_tmp" || exit 0 + local temporary + temporary=$(mktemp "${cache}.XXXXXX") || exit 0 + if printf '%s\n%s\n%s\n' "$key" "$now" "$result" > "$temporary"; then + mv -f -- "$temporary" "$cache" || rm -f -- "$temporary" + else + rm -f -- "$temporary" + fi + ) 2>/dev/null + fi + printf '%s\n' "$result" +) + +ql_get_node_global_path() { + if [[ "${QL_NODE_PATH_CACHE:-1}" == 0 ]]; then + pnpm root -g 2>/dev/null + return $? + fi + + local key cache + cache="${dir_tmp}/pnpm-root-${EUID}.cache" + key=$(ql_node_path_cache_key) || { pnpm root -g 2>/dev/null; return $?; } + if ql_read_node_path_cache "$cache" "$key"; then + return 0 + fi + ql_refresh_node_global_path "$cache" "$key" +} diff --git a/shell/otask.sh b/shell/otask.sh index d311cce7..5dcb579d 100755 --- a/shell/otask.sh +++ b/shell/otask.sh @@ -95,7 +95,13 @@ append_node_dependency_path() { # 用户依赖目录加入 NODE_PATH，替代 symlink 到 node_modules 的方式 export NODE_PATH="${NODE_PATH:+${NODE_PATH}:}${dir_dep}" - local pnpm_global_path=$(pnpm root -g 2>/dev/null) + local pnpm_global_path + if [[ -f "$dir_shell/node_path_cache.sh" ]]; then + . "$dir_shell/node_path_cache.sh" + pnpm_global_path=$(ql_get_node_global_path) || true + else + pnpm_global_path=$(pnpm root -g 2>/dev/null) || true + fi if [[ -n "$pnpm_global_path" ]]; then export QL_NODE_GLOBAL_PATH="$pnpm_global_path" export NODE_PATH="${NODE_PATH:+${NODE_PATH}:}${pnpm_global_path}" diff --git a/shell/share.sh b/shell/share.sh index 2f02840e..8a38a746 100755 --- a/shell/share.sh +++ b/shell/share.sh @@ -368,9 +368,16 @@ format_timestamp() { get_env_array() { exported_variables=() - while IFS= read -r line; do - exported_variables+=("$line") - done < <(grep '^export ' $file_env | awk '{print $2}' | cut -d= -f1) + # Preserve the legacy export-line/second-field rules without evaluating values. + local export_name_program='/^export / { name = $2; sub(/=.*/, "", name); print name }' + if [[ ${BASH_VERSINFO[0]} -ge 4 ]]; then + builtin mapfile -t exported_variables < <(awk "$export_name_program" "$file_env") + else + # macOS still ships Bash 3, which does not provide mapfile. + while IFS= read -r line; do + exported_variables+=("$line") + done < <(awk "$export_name_program" "$file_env") + fi } clear_env() { @@ -411,9 +418,14 @@ run_task_after() { } handle_task_end() { - local etime=$(date "+$time_format") - local end_time=$(format_time "$time_format" "$etime") - local end_timestamp=$(format_timestamp "$time_format" "$etime") + local etime end_time end_timestamp + if [[ $is_macos -ne 1 && $time_format == '%Y-%m-%d %H:%M:%S' ]]; then + IFS='|' read -r end_time end_timestamp < <(date "+$time_format|%s") + else + etime=$(date "+$time_format") + end_time=$(format_time "$time_format" "$etime") + end_timestamp=$(format_timestamp "$time_format" "$etime") + fi local diff_time=$(($end_timestamp - $begin_timestamp)) local exit_code="${_task_exit_code:-0}" [[ "$diff_time" == 0 ]] && diff_time=1 diff --git a/shell/task.sh b/shell/task.sh index e18e18cb..a98bd827 100755 --- a/shell/task.sh +++ b/shell/task.sh @@ -52,8 +52,14 @@ handle_log_path() { fi fi - time=$(date "+$mtime_format") - log_time=$(format_log_time "$mtime_format" "$time") + if [[ $is_macos -ne 1 && $mtime_format == '%Y-%m-%d %H:%M:%S.%3N' ]]; then + # Render both representations from one clock snapshot, without parsing it + # again in a second date process. + IFS='|' read -r time log_time < <(date "+$mtime_format|%Y-%m-%d-%H-%M-%S-%3N") + else + time=$(date "+$mtime_format") + log_time=$(format_log_time "$mtime_format" "$time") + fi if [[ -z $log_name ]]; then log_dir_tmp="${file_param##*/}" if [[ $file_param =~ "/" ]]; then @@ -124,7 +130,11 @@ format_params() { } init_begin_time() { - begin_time=$(format_time "$time_format" "$time") + if [[ $is_macos -ne 1 && $mtime_format == '%Y-%m-%d %H:%M:%S.%3N' && $time_format == '%Y-%m-%d %H:%M:%S' ]]; then + begin_time=${time%.*} + else + begin_time=$(format_time "$time_format" "$time") + fi begin_timestamp=$(format_timestamp "$time_format" "$time") } diff --git a/test/back/build-provenance.test.cjs b/test/back/build-provenance.test.cjs new file mode 100644 index 00000000..d473f3b1 --- /dev/null +++ b/test/back/build-provenance.test.cjs @@ -0,0 +1,150 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync, spawnSync } = require('node:child_process'); + +test('build verification accepts matching source and rejects stale or dirty artifacts', (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-build-source-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const git = (...args) => + execFileSync('git', args, { cwd: dir, stdio: 'pipe' }); + git('init'); + git('config', 'user.name', 'Build test'); + git('config', 'user.email', 'build-test@example.invalid'); + fs.writeFileSync(path.join(dir, 'pnpm-lock.yaml'), 'lockfileVersion: 6.0\n'); + fs.writeFileSync(path.join(dir, '.gitignore'), 'static/\n'); + git('add', '.'); + git('commit', '-m', 'fixture'); + fs.mkdirSync(path.join(dir, 'static/build'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'static/dist'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'static/build/app.js'), ''); + fs.writeFileSync(path.join(dir, 'static/dist/index.html'), ''); + const write = path.resolve('scripts/write-build-info.cjs'); + const verify = path.resolve('docker/verify-build.cjs'); + execFileSync(process.execPath, [write], { cwd: dir }); + execFileSync(process.execPath, [verify], { cwd: dir }); + const manifestPath = path.join(dir, 'static/build-info.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath)); + for (const invalid of [ + { ...manifest, sourceCommit: '0'.repeat(40) }, + { ...manifest, dirty: true }, + { ...manifest, lockfileSha256: '0'.repeat(64) }, + ]) { + fs.writeFileSync(manifestPath, JSON.stringify(invalid)); + assert.notEqual( + spawnSync(process.execPath, [verify], { cwd: dir }).status, + 0, + ); + } +}); + +test('release images receive the same-run artifact and verify it before use', () => { + const yaml = require('js-yaml'); + const workflow = yaml.load( + fs.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'), + ); + assert.equal(workflow.jobs['build-static'].needs, 'validate'); + const upload = workflow.jobs['build-static'].steps.find(step => step.uses?.startsWith('actions/upload-artifact@')); + assert.equal(upload.with['include-hidden-files'], true); + for (const name of [ + 'build-alpine', + 'build-debian', + 'build-alpine310', + 'build-debian310', + ]) { + const job = workflow.jobs[name]; + assert.equal(job.needs, 'build-static'); + const download = job.steps.find((step) => + step.uses?.startsWith('actions/download-artifact@'), + ); + assert.equal(download.with.name, 'qinglong-static-${{ github.sha }}'); + assert.equal(download.with.path, 'static/'); + const build = job.steps.find((step) => + step.uses?.startsWith('docker/build-push-action@'), + ); + assert.match( + build.with['build-args'], + /SOURCE_COMMIT=\$\{\{ github.sha \}\}/, + ); + const dockerfile = fs.readFileSync(build.with.file, 'utf8'); + assert.doesNotMatch(dockerfile, /git clone.*qinglong-static/); + assert.match( + dockerfile, + /git fetch --depth=1 origin "\$\{SOURCE_COMMIT\}"/, + ); + assert.match(dockerfile, /node \/tmp\/verify-build.cjs/); + } +}); + +test('complete artifact manifests reject changed, missing, extra files and untracked build inputs', (t) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-build-manifest-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const git = (...args) => + execFileSync('git', args, { cwd: dir, stdio: 'pipe' }); + git('init'); + git('config', 'user.name', 'Build test'); + git('config', 'user.email', 'build-test@example.invalid'); + fs.writeFileSync(path.join(dir, '.gitignore'), 'static/\n'); + fs.writeFileSync(path.join(dir, 'pnpm-lock.yaml'), 'lockfileVersion: 6.0\n'); + git('add', '.'); + git('commit', '-m', 'fixture'); + const artifacts = [ + 'build/app.js', + 'build/services/http.js', + 'dist/index.html', + 'dist/chunks/main.js', + 'dist/assets/main.css', + 'dist/.well-known/config', + ]; + for (const file of artifacts) { + fs.mkdirSync(path.dirname(path.join(dir, 'static', file)), { + recursive: true, + }); + fs.writeFileSync(path.join(dir, 'static', file), file); + } + const write = path.resolve('scripts/write-build-info.cjs'); + const verify = path.resolve('docker/verify-build.cjs'); + const run = (script) => + spawnSync(process.execPath, [script], { cwd: dir, encoding: 'utf8' }); + assert.equal(run(write).status, 0); + const manifestFile = path.join(dir, 'static/build-info.json'); + const original = fs.readFileSync(manifestFile, 'utf8'); + assert.deepEqual( + Object.keys(JSON.parse(original).files).sort(), + artifacts.sort(), + ); + assert.equal(run(write).status, 0); + assert.equal(fs.readFileSync(manifestFile, 'utf8'), original); + assert.equal(run(verify).status, 0); + for (const name of [ + 'build/services/http.js', + 'dist/chunks/main.js', + 'dist/assets/main.css', + 'dist/.well-known/config', + ]) { + const file = path.join(dir, 'static', name); + fs.writeFileSync(file, 'stale'); + assert.notEqual(run(verify).status, 0); + fs.unlinkSync(file); + assert.notEqual(run(verify).status, 0); + fs.writeFileSync(file, name); + assert.equal(run(verify).status, 0); + } + const extra = path.join(dir, 'static/build/stale.js'); + fs.writeFileSync(extra, 'extra'); + assert.notEqual(run(verify).status, 0); + fs.unlinkSync(extra); + const source = path.join(dir, 'custom.config.js'); + fs.writeFileSync(source, 'untracked build input'); + assert.equal(run(write).status, 0); + assert.equal(JSON.parse(fs.readFileSync(manifestFile)).dirty, true); + assert.notEqual(run(verify).status, 0); + fs.unlinkSync(source); + assert.equal(run(write).status, 0); + assert.equal(run(verify).status, 0); + fs.symlinkSync(path.join(dir, 'pnpm-lock.yaml'), extra); + assert.notEqual(run(write).status, 0); + assert.notEqual(run(verify).status, 0); +}); diff --git a/test/back/env-name-parsing.test.cjs b/test/back/env-name-parsing.test.cjs new file mode 100644 index 00000000..05f99bf1 --- /dev/null +++ b/test/back/env-name-parsing.test.cjs @@ -0,0 +1,96 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const source = fs.readFileSync('shell/share.sh', 'utf8'); +function extract(name) { + const start = source.indexOf(name + '() {'); + assert.ok(start >= 0); + return source.slice(start, source.indexOf('\n}', start) + 2); +} +function fixture(t, content) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-env-names-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const file = path.join(root, 'env file.sh'); + fs.writeFileSync(file, content); + return { + root, + file, + run(script) { + const r = spawnSync( + '/bin/bash', + [ + '-ec', + extract('get_env_array') + + '\n' + + extract('clear_env') + + '\n' + + script, + ], + { + encoding: 'utf8', + env: { ...process.env, file_env: file, FIXTURE_ROOT: root }, + }, + ); + assert.equal(r.status, 0, r.stderr); + return r.stdout; + }, + }; +} +test('matches legacy pipeline for whitespace, malformed lines, CRLF and final unterminated line', (t) => { + const f = fixture( + t, + '# comment\nexport A=one\nexport B="two words"\n export IGNORED=1\nexport\tIGNORED2=1\nexport C = spaced\nexport \nexport -x X=1\nexport D=a=b\nexport E=crlf\r\nexport F\r\nexport G=last', + ); + assert.equal( + f.run('get_env_array; printf "%s\\0" "${exported_variables[@]}"'), + f.run( + 'while IFS= read -r line; do printf "%s\\0" "$line"; done < <(grep "^export " "$file_env" | awk \'{print $2}\' | cut -d= -f1)', + ), + ); +}); +test('resets prior names for an empty file', (t) => { + const f = fixture(t, ''); + assert.equal( + f.run( + 'exported_variables=(OLD); get_env_array; echo "${#exported_variables[@]}"', + ), + '0\n', + ); +}); +test('extracting names never evaluates command substitutions or shell syntax in values', (t) => { + const f = fixture( + t, + 'export A=$(touch "$FIXTURE_ROOT/unsafe")\nexport B=`touch "$FIXTURE_ROOT/unsafe2"`\n', + ); + assert.equal( + f.run('get_env_array; printf "%s\\n" "${exported_variables[@]}"'), + 'A\nB\n', + ); + assert.equal(fs.existsSync(path.join(f.root, 'unsafe')), false); + assert.equal(fs.existsSync(path.join(f.root, 'unsafe2')), false); +}); +test('clear_env removes listed exports and preserves unrelated values', (t) => { + const f = fixture(t, 'export USER_ONE=1\nexport USER_TWO=2\n'); + f.run( + 'USER_ONE=old; USER_TWO=old; UNRELATED=kept; get_env_array; clear_env; [[ ! ${USER_ONE+x} && ! ${USER_TWO+x} && "$UNRELATED" == kept ]]', + ); +}); +test('large configuration preserves order and duplicate names', (t) => { + const text = Array.from( + { length: 5000 }, + (_, i) => 'export V' + (i % 1000) + '=value with spaces', + ).join('\n'); + const f = fixture(t, text); + const lines = f + .run('get_env_array; printf "%s\\n" "${exported_variables[@]}"') + .trim() + .split('\n'); + assert.equal(lines.length, 5000); + assert.deepEqual( + lines, + Array.from({ length: 5000 }, (_, i) => 'V' + (i % 1000)), + ); +}); diff --git a/test/back/execution-lifecycle.test.cjs b/test/back/execution-lifecycle.test.cjs new file mode 100644 index 00000000..3e0b9e91 --- /dev/null +++ b/test/back/execution-lifecycle.test.cjs @@ -0,0 +1,236 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { spawn } = require('node:child_process'); +const { EventEmitter } = require('node:events'); +const { PassThrough } = require('node:stream'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const load = require('../helpers/load-security-module.cjs'); +const { observeChildProcess } = require('../../back/shared/childProcess'); +const { LogStreamManager } = require('../../back/shared/logStreamManager'); + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const logger = { info() {}, error() {} }; + +test( + 'spawn failure settles without exit, and the next scheduled run can proceed', + { timeout: 3000 }, + async () => { + let releases = 0; + let active = 0; + const { runCron } = load(path.resolve('back/shared/runCron.ts'), { + 'cross-spawn': { + spawn: () => spawn('true', { shell: '/nonexistent-ql-test-shell' }), + }, + './pLimit': { + runWithCronLimit: async (_cron, fn) => { + active++; + try { + return await fn(); + } finally { + active--; + } + }, + removeQueuedCron: () => releases++, + }, + '../loaders/logger': logger, + '../data/cron': { + CrontabModel: { findOne: async () => null }, + CrontabStatus: {}, + }, + '../data/runningInstance': { + RunningInstanceModel: {}, + InstanceStatus: {}, + }, + '../config/util': { killTask: async () => {} }, + }); + await runCron('true', { id: '1' }); + await runCron('true', { id: '2' }); + assert.equal(active, 0); + assert.equal(releases, 2); + }, +); + +test( + 'completion waits for slow log consumers and preserves the final output', + { timeout: 5000 }, + async () => { + const chunks = []; + const child = spawn(process.execPath, [ + '-e', + 'process.stdout.write("尾行\\n"); process.stderr.write("错误\\n")', + ]); + const observed = observeChildProcess(child, { + onStart: () => delay(25), + onStdout: async (message) => { + await delay(40); + chunks.push(message); + }, + onStderr: async (message) => { + await delay(30); + chunks.push(message); + }, + }); + const result = await observed.completed; + assert.equal(result.code, 0); + assert.equal(result.error, undefined); + assert.match(chunks.join(''), /尾行/); + assert.match(chunks.join(''), /错误/); + }, +); + +test('exit is not completion, and UTF-8 split across writes remains intact', async () => { + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.pid = 123; + const chunks = []; + const observed = observeChildProcess(child, { + onStdout: async (data) => chunks.push(data), + }); + child.emit('spawn'); + let done = false; + observed.completed.then(() => { + done = true; + }); + child.emit('exit', 0, null); + const data = Buffer.from('末尾中文'); + child.stdout.write(data.subarray(0, 2)); + await delay(10); + assert.equal(done, false); + child.stdout.end(data.subarray(2)); + child.stderr.end(); + child.emit('close', 0, null); + await observed.completed; + assert.equal(chunks.join(''), '末尾中文'); +}); + +test( + 'failed log sink drains large output instead of blocking the child', + { timeout: 5000 }, + async () => { + const child = spawn(process.execPath, [ + '-e', + 'process.stdout.write("x".repeat(2 * 1024 * 1024))', + ]); + const { completed } = observeChildProcess(child, { + onStdout: async () => { + throw new Error('ENOSPC'); + }, + }); + const result = await completed; + assert.equal(result.code, 0); + assert.equal(result.error.message, 'ENOSPC'); + }, +); + +function scheduleFixture() { + let active = 0; + const limit = async (_params, fn) => { + active++; + try { + return await fn(); + } finally { + active--; + } + }; + const Schedule = load(path.resolve('back/services/schedule.ts'), { + '../shared/pLimit': { + runWithScriptLimit: limit, + runWithSystemLimit: limit, + }, + 'cross-spawn': { + spawn: () => + spawn(process.execPath, [ + '-e', + 'setTimeout(() => process.stdout.write("done"), 150)', + ]), + }, + }).default; + return { service: new Schedule(logger), active: () => active }; +} + +test( + 'PID response keeps queue capacity occupied until cleanup completes', + { timeout: 4000 }, + async () => { + const fixture = scheduleFixture(); + let finish; + const finished = new Promise((resolve) => { + finish = resolve; + }); + const pid = await fixture.service.runTask( + 'ignored', + { + onEnd: async () => { + await delay(20); + finish(); + }, + }, + { id: 'script', runOrigin: 'script' }, + 'start', + ); + assert.ok(pid > 0); + assert.equal(fixture.active(), 1); + await finished; + await delay(0); + assert.equal(fixture.active(), 0); + }, +); + +test( + 'before/error/end callback failures settle and release capacity', + { timeout: 3000 }, + async () => { + const fixture = scheduleFixture(); + let ended = 0; + const result = await fixture.service.runTask( + 'ignored', + { + onBefore: async () => { + throw new Error('setup failed'); + }, + onError: async () => { + throw new Error('sink failed'); + }, + onEnd: async (child) => { + assert.equal(child, undefined); + ended++; + throw new Error('cleanup failed'); + }, + }, + { id: 'system', runOrigin: 'system' }, + ); + assert.equal(result.error.message, 'setup failed'); + assert.equal(ended, 1); + assert.equal(fixture.active(), 0); + }, +); + +test( + 'concurrent writes and closes preserve all bytes, and failed files can be closed', + { timeout: 5000 }, + async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-lifecycle-')); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + const manager = new LogStreamManager(dir); + const file = path.join(dir, 'out.log'); + const lines = Array.from({ length: 500 }, (_, i) => `${i}:末尾\n`); + const writes = lines.map((line) => manager.write(file, line)); + await Promise.all([ + ...writes, + manager.closeStream(file), + manager.closeStream(file), + ]); + assert.equal(await fs.readFile(file, 'utf8'), lines.join('')); + assert.equal(manager.getOpenStreamCount(), 0); + const invalid = path.join(dir, 'missing', 'out.log'); + await assert.rejects(manager.write(invalid, 'fail'), /ENOENT/); + await assert.rejects(manager.closeStream(invalid), /ENOENT/); + assert.equal(manager.getOpenStreamCount(), 0); + await manager.write(file, 'reopened'); + await manager.closeAll(); + assert.match(await fs.readFile(file, 'utf8'), /reopened$/); + }, +); diff --git a/test/back/http-exclusive-listen.test.cjs b/test/back/http-exclusive-listen.test.cjs new file mode 100644 index 00000000..da2157d8 --- /dev/null +++ b/test/back/http-exclusive-listen.test.cjs @@ -0,0 +1,31 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const express = require('express'); +const load = require('../helpers/load-security-module.cjs'); +const Http = load('back/services/http.ts', { + '../config': {bindHost: '127.0.0.1'}, + '../loaders/logger': {debug(){},warn(){},error(){}}, + './metrics': {metricsService: {record(){}}}, + typedi: {Service:()=>x=>x}, +}).HttpServerService; +test('HTTP listener preserves private binding and releases its port after shutdown',async(t)=>{ + const app=express();app.get('/',(_q,r)=>r.send('ok')); + const first=new Http();const server=await first.initialize(app,0); + t.after(()=>first.shutdown()); + assert.equal(server.address().address,'127.0.0.1'); + const port=server.address().port; + const response=await fetch(`http://127.0.0.1:${port}/`,{headers:{connection:'close'}});assert.equal(await response.text(),'ok'); + await assert.rejects(new Http().initialize(app,port),e=>e.code==='EADDRINUSE'); + await first.shutdown();const replacement=new Http();t.after(()=>replacement.shutdown()); + const next=await replacement.initialize(app,port);assert.equal(next.address().port,port); +}); +test('custom cluster deployments can restore shared listening',async(t)=>{ + const previous=process.env.QL_HTTP_SHARED_LISTEN; + t.after(()=>{if(previous===undefined)delete process.env.QL_HTTP_SHARED_LISTEN;else process.env.QL_HTTP_SHARED_LISTEN=previous;}); + for(const value of [undefined,'true']){ + if(value===undefined)delete process.env.QL_HTTP_SHARED_LISTEN;else process.env.QL_HTTP_SHARED_LISTEN=value; + let options;const service=new Http();const app=express();const listen=app.listen.bind(app); + app.listen=(opts,cb)=>{options=opts;return listen(opts,cb);}; + await service.initialize(app,0);assert.equal(options.exclusive,value!=='true');await service.shutdown(); + } +}); diff --git a/test/back/log-path-security.test.cjs b/test/back/log-path-security.test.cjs new file mode 100644 index 00000000..b925b977 --- /dev/null +++ b/test/back/log-path-security.test.cjs @@ -0,0 +1,158 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const load = require('../helpers/load-security-module.cjs'); +const { LogStreamManager } = require('../../back/shared/logStreamManager'); + +async function fixture(t) { + const base = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-boundary-')); + t.after(() => fs.rm(base, { recursive: true, force: true })); + const root = path.join(base, 'log'); + const outside = path.join(base, 'log-other'); + await fs.mkdir(root); + await fs.mkdir(outside); + const victim = path.join(outside, 'victim.log'); + await fs.writeFile(victim, 'unchanged'); + await fs.symlink(outside, path.join(root, 'escape-dir')); + await fs.symlink(victim, path.join(root, 'escape-file')); + await fs.symlink( + path.join(outside, 'missing.log'), + path.join(root, 'dangling'), + ); + const invalid = [ + path.join(root, '..', 'log-other', 'new.log'), + path.join(root, 'escape-dir', 'new.log'), + path.join(root, 'escape-file'), + path.join(root, 'dangling'), + root, + path.join(root, 'bad\0name'), + ]; + return { root, outside, victim, invalid }; +} + +test('log streams reject traversal, sibling prefixes and escaping symlinks before writing', async (t) => { + const { root, outside, victim, invalid } = await fixture(t); + const manager = new LogStreamManager(root); + for (const target of invalid) { + await assert.rejects( + manager.write(target, 'overwrite'), + /outside the log directory/, + ); + await assert.rejects( + manager.closeStream(target), + /outside the log directory/, + ); + assert.equal(manager.getOpenStreamCount(), 0); + } + assert.equal(await fs.readFile(victim, 'utf8'), 'unchanged'); + assert.deepEqual(await fs.readdir(outside), ['victim.log']); + const folder = path.join(root, '中文 日志'); + await fs.mkdir(folder); + const log = path.join(folder, 'task.log'); + await Promise.all([ + manager.write(log, '开始\n'), + manager.write(log, '结束\n'), + ]); + await manager.closeAll(); + assert.equal(await fs.readFile(log, 'utf8'), '开始\n结束\n'); +}); + +test('log initialization rejects unsafe paths before mkdir or file writes', async (t) => { + const { root, outside, victim, invalid } = await fixture(t); + const { handleLogPath } = load(path.resolve('back/config/util.ts'), { + './index': { logPath: root }, + './share': {}, + '../loaders/logger': {}, + '../shared/utils': { + writeFileWithLock: (file, data) => fs.writeFile(file, data), + }, + '../data/dependence': { DependenceTypes: {} }, + }); + for (const target of invalid) { + await assert.rejects( + handleLogPath(target, 'overwrite'), + /outside the log directory/, + ); + } + await assert.rejects( + handleLogPath('../log-other/new/sub/task.log', 'overwrite'), + /outside the log directory/, + ); + assert.equal(await fs.readFile(victim, 'utf8'), 'unchanged'); + assert.deepEqual(await fs.readdir(outside), ['victim.log']); + const log = await handleLogPath('中文 日志/nested/task.log', 'initial'); + assert.equal(await fs.readFile(log, 'utf8'), 'initial'); + assert.equal(await handleLogPath(log, 'ignored'), log); + assert.equal(await fs.readFile(log, 'utf8'), 'initial'); +}); + +test('manual execution rejects escaping log names before creating directories or spawning', async (t) => { + const { root, outside, victim } = await fixture(t); + let spawned = 0; + let releases = 0; + const errors = []; + const CronService = load(path.resolve('back/services/cron.ts'), { + '../config': { logPath: root }, + '../data/cron': { + CrontabStatus: { queued: 3, idle: 1 }, + CrontabModel: { update: async () => {} }, + }, + '../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} }, + '../config/util': {}, + '../config/const': {}, + '../schedule/client': {}, + '../shared/pLimit': { + manualRunWithCronLimit: async (fn) => { + try { + return await fn(); + } finally { + releases++; + } + }, + }, + '../shared/utils': {}, + '../shared/i18n': { t: (s) => s }, + '../shared/logReader': {}, + '../shared/logStreamManager': { + logStreamManager: { closeStream: async () => {} }, + }, + 'cross-spawn': { + spawn: () => { + spawned++; + throw new Error('must not spawn'); + }, + }, + }).default; + const service = new CronService({ + info() {}, + error: (...args) => errors.push(args), + }); + for (const log_name of [ + '../log-other/new', + outside, + 'escape-dir/new', + 'dangling', + 'bad\0name', + ]) { + service.getDb = async () => ({ + id: 1, + status: 3, + command: 'ignored', + log_path: '', + log_name, + }); + await service.runSingle(1); + } + assert.equal(spawned, 0); + assert.equal(releases, 5); + assert.equal( + errors.filter((args) => + args.includes('Log path is outside the log directory'), + ).length, + 5, + ); + assert.equal(await fs.readFile(victim, 'utf8'), 'unchanged'); + assert.deepEqual(await fs.readdir(outside), ['victim.log']); +}); diff --git a/test/back/manual-execution.test.cjs b/test/back/manual-execution.test.cjs new file mode 100644 index 00000000..d60897e2 --- /dev/null +++ b/test/back/manual-execution.test.cjs @@ -0,0 +1,85 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs/promises'); +const path = require('node:path'); +const os = require('node:os'); +const { spawn } = require('node:child_process'); +const load = require('../helpers/load-security-module.cjs'); +const { LogStreamManager } = require('../../back/shared/logStreamManager'); + +test( + 'manual execution captures a short child before slow status storage and flushes before release', + { timeout: 5000 }, + async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-')); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const logs = new LogStreamManager(root); + const updates = []; + let active = 0; + let failSpawn = false; + const CronService = load(path.resolve('back/services/cron.ts'), { + '../config': { logPath: root }, + '../data/cron': { + CrontabStatus: { queued: 3, running: 0, idle: 1 }, + CrontabModel: { + update: async (data, options) => { + await new Promise((resolve) => setTimeout(resolve, 40)); + updates.push({ data, options }); + return [1]; + }, + }, + }, + '../data/runningInstance': { + RunningInstanceModel: {}, + InstanceStatus: {}, + }, + '../config/util': { getUniqPath: async () => 'task' }, + '../config/const': { TASK_PREFIX: 'task ', QL_PREFIX: 'ql ' }, + '../schedule/client': {}, + '../shared/pLimit': { + manualRunWithCronLimit: async (fn) => { + active++; + try { + return await fn(); + } finally { + active--; + } + }, + }, + '../shared/utils': {}, + '../shared/i18n': { t: (s) => s }, + '../shared/logReader': {}, + '../shared/logStreamManager': { logStreamManager: logs }, + 'cross-spawn': { + spawn: () => + failSpawn + ? spawn('true', { shell: '/nonexistent-ql-manual-shell' }) + : spawn(process.execPath, [ + '-e', + 'process.stdout.write("末尾\\n")', + ]), + }, + }).default; + const service = new CronService({ info() {}, error() {} }); + service.getDb = async () => ({ + id: 1, + status: 3, + command: 'ignored', + log_path: '', + }); + service.makeCommand = () => 'ignored'; + await service.runSingle(1); + const [file] = await fs.readdir(path.join(root, 'task')); + assert.equal( + await fs.readFile(path.join(root, 'task', file), 'utf8'), + '末尾\n', + ); + assert.equal(logs.getOpenStreamCount(), 0); + assert.equal(active, 0); + assert.equal(updates.at(-1).data.status, 1); + failSpawn = true; + await service.runSingle(1); + assert.equal(active, 0); + assert.equal(updates.at(-1).data.status, 1); + }, +); diff --git a/test/back/manual-stop-claim.test.cjs b/test/back/manual-stop-claim.test.cjs new file mode 100644 index 00000000..91866276 --- /dev/null +++ b/test/back/manual-stop-claim.test.cjs @@ -0,0 +1,215 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { Sequelize, DataTypes } = require('sequelize'); +const load = require('../helpers/load-security-module.cjs'); +const { killTask } = require('../../back/config/util'); +const { LogStreamManager } = require('../../back/shared/logStreamManager'); + +for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) { + test( + `manual startup loses its conditional claim after ${conflict} and terminates the late child`, + { timeout: 10000 }, + async (t) => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-claim-')); + const db = new Sequelize({ + dialect: 'sqlite', + storage: ':memory:', + logging: false, + }); + t.after(async () => { + await db.close(); + await fs.rm(root, { recursive: true, force: true }); + }); + const crons = db.define('Cron', { + status: DataTypes.INTEGER, + pid: DataTypes.INTEGER, + log_path: DataTypes.STRING, + queued_token: DataTypes.STRING, + command: DataTypes.STRING, + }); + const instances = db.define('Instance', { + cron_id: DataTypes.INTEGER, + status: DataTypes.INTEGER, + pid: DataTypes.INTEGER, + }); + await db.sync(); + await crons.create({ + id: 1, + status: 3, + pid: null, + log_path: 'previous.log', + command: 'ignored', + }); + let release, + entered, + child, + released = 0; + const pending = new Promise((resolve) => (entered = resolve)); + const gate = new Promise((resolve) => (release = resolve)); + const logs = new LogStreamManager(root); + t.after(() => logs.closeAll()); + const CronService = load('back/services/cron.ts', { + '../config': { logPath: root }, + '../data/cron': { + CrontabModel: crons, + CrontabStatus: { queued: 3, running: 0, idle: 1 }, + }, + '../data/runningInstance': { + RunningInstanceModel: instances, + InstanceStatus: { running: 0, stopped: 2 }, + }, + '../config/util': { + getUniqPath: async () => { + entered(); + await gate; + return 'task'; + }, + killTask, + }, + '../config/const': {}, + '../schedule/client': {}, + '../shared/pLimit': { + manualRunWithCronLimit: async (fn) => { + try { + return await fn(); + } finally { + released++; + } + }, + }, + '../shared/utils': {}, + '../shared/i18n': { t: (s) => s }, + '../shared/logReader': {}, + '../shared/logStreamManager': { logStreamManager: logs }, + 'cross-spawn': { + spawn: () => { + child = spawn(process.execPath, [ + '-e', + 'setInterval(() => {}, 1000)', + ]); + t.after(() => { + if (child.exitCode === null && child.signalCode === null) + child.kill('SIGKILL'); + }); + return child; + }, + }, + }).default; + const service = new CronService({ info() {}, error() {} }); + service.getDb = async () => + (await crons.findByPk(1)).get({ plain: true }); + service.makeCommand = () => 'ignored'; + const running = service.runSingle(1); + await pending; + if (conflict === 'stop-requeue') { + await service.stop([1]); + // Queue again through the real API; leave the new runner pending. + const pendingRuns = []; + service.runSingle = (id, token) => pendingRuns.push({ id, token }); + await service.run([1]); + assert.equal(pendingRuns.length, 1); + assert.equal( + pendingRuns[0].token, + (await crons.findByPk(1)).queued_token, + ); + } else if (conflict === 'stop') await service.stop([1]); + else await crons.update({ log_path: 'newer.log' }, { where: { id: 1 } }); + release(); + const result = await running; + assert.match(result.error.message, /stopped or superseded/); + assert.ok(child.signalCode || child.exitCode !== null); + assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' }); + const row = await crons.findByPk(1); + assert.equal(row.status, conflict === 'stop' ? 1 : 3); + assert.equal( + row.log_path, + conflict === 'newer-queue' ? 'newer.log' : 'previous.log', + ); + assert.equal(row.pid, null); + assert.equal(released, 1); + }, + ); +} + +test( + 'verified termination waits for a SIGTERM-resistant target to exit', + { timeout: 5000 }, + async (t) => { + const child = spawn(process.execPath, [ + '-e', + 'process.on("SIGTERM",()=>{}); process.stdout.write("ready"); setInterval(()=>{},1000)', + ]); + t.after(() => { + if (child.exitCode === null && child.signalCode === null) + child.kill('SIGKILL'); + }); + await new Promise((resolve) => child.stdout.once('data', resolve)); + await killTask(child.pid, true); + assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' }); + assert.equal(child.signalCode, 'SIGKILL'); + }, +); + +test('a runner waiting for a concurrency slot cannot adopt a newer queued generation', async (t) => { + const db = new Sequelize({ + dialect: 'sqlite', + storage: ':memory:', + logging: false, + }); + t.after(() => db.close()); + const crons = db.define('Cron', { + status: DataTypes.INTEGER, + pid: DataTypes.INTEGER, + log_path: DataTypes.STRING, + queued_token: DataTypes.STRING, + }); + await db.sync(); + await crons.create({ id: 1, status: 1, log_path: 'same.log' }); + const waiting = []; + const CronService = load('back/services/cron.ts', { + '../config': {}, + '../data/cron': { + CrontabModel: crons, + CrontabStatus: { queued: 3, idle: 1, running: 0 }, + }, + '../data/runningInstance': { + RunningInstanceModel: { findAll: async () => [] }, + InstanceStatus: { running: 0 }, + }, + '../config/util': { + getUniqPath: () => assert.fail('stale runner must not prepare a child'), + }, + '../config/const': {}, + '../schedule/client': {}, + '../shared/pLimit': { + manualRunWithCronLimit: (fn) => { + waiting.push(fn); + return Promise.resolve(); + }, + }, + '../shared/utils': {}, + '../shared/i18n': {}, + '../shared/logReader': {}, + '../shared/logStreamManager': {}, + 'cross-spawn': { spawn: () => assert.fail('stale runner must not spawn') }, + }).default; + const errors = []; + const service = new CronService({ error: (...args) => errors.push(args) }); + await service.run([1]); + const first = (await crons.findByPk(1)).queued_token; + await service.stop([1]); + await service.run([1]); + const second = (await crons.findByPk(1)).queued_token; + assert.notEqual(first, second); + assert.equal(waiting.length, 2); + await waiting[0](); + const row = await crons.findByPk(1); + assert.equal(row.status, 3); + assert.equal(row.queued_token, second); + assert.equal(row.log_path, 'same.log'); + assert.deepEqual(errors, []); +}); diff --git a/test/back/node-path-cache.test.cjs b/test/back/node-path-cache.test.cjs new file mode 100644 index 00000000..08dab052 --- /dev/null +++ b/test/back/node-path-cache.test.cjs @@ -0,0 +1,173 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync, spawn } = require('node:child_process'); +const helper = path.resolve('shell/node_path_cache.sh'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-node-path-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + fs.symlinkSync(process.execPath, path.join(bin, 'node')); + const calls = path.join(root, 'calls'); + fs.writeFileSync( + path.join(bin, 'pnpm'), + '#!/bin/bash\necho call >> "$CALLS"\n[[ "${FAIL:-0}" == 1 ]] && exit 1\nprintf "%s\\n" "${ANSWER:-/test/global/node_modules}"\n', + { mode: 0o755 }, + ); + const env = { + ...process.env, + HOME: root, + PATH: `${bin}:${process.env.PATH}`, + dir_tmp: path.join(root, 'cache'), + CALLS: calls, + HELPER: helper, + }; + const command = '. "$HELPER"; ql_get_node_global_path'; + const run = (extra = {}, code = command) => + spawnSync('/bin/bash', ['-uc', code], { + cwd: root, + env: { ...env, ...extra }, + encoding: 'utf8', + }); + const count = () => + fs.existsSync(calls) + ? fs.readFileSync(calls, 'utf8').trim().split('\n').length + : 0; + return { root, bin, env, command, run, count }; +} +test('warm cache avoids pnpm, preserves spaces, and never evaluates cached text', (t) => { + const f = fixture(t); + const answer = '/test/path with spaces/$(touch injected)'; + assert.equal(f.run({ ANSWER: answer }).stdout.trim(), answer); + assert.equal(f.run({ ANSWER: '/different' }).stdout.trim(), answer); + assert.equal(f.count(), 1); + assert.equal(fs.existsSync(path.join(f.root, 'injected')), false); +}); +test('config, environment, cwd, and executable changes invalidate discovery', (t) => { + const f = fixture(t); + f.run(); + fs.writeFileSync(path.join(f.root, '.npmrc'), 'global-dir=/other\n'); + f.run(); + assert.equal(f.count(), 2); + f.run({ npm_config_global_dir: '/third' }); + assert.equal(f.count(), 3); + f.run({ PNPM_HOME: '/fourth' }); + assert.equal(f.count(), 4); + fs.mkdirSync(path.join(f.root, 'child')); + f.run({}, 'cd child; ' + f.command); + assert.equal(f.count(), 5); + f.run(); + const before = f.count(); + fs.appendFileSync(path.join(f.bin, 'pnpm'), '# upgrade\n'); + f.run(); + assert.equal(f.count(), before + 1); +}); +test('expired or malformed records refresh and failed lookups are not cached', (t) => { + const f = fixture(t); + f.run(); + const file = path.join( + f.env.dir_tmp, + fs.readdirSync(f.env.dir_tmp).find((name) => name.endsWith('.cache')), + ); + let lines = fs.readFileSync(file, 'utf8').split('\n'); + lines[1] = '0'; + fs.writeFileSync(file, lines.join('\n')); + f.run(); + assert.equal(f.count(), 2); + fs.writeFileSync(file, lines[0] + '\n08\n/incorrect\n'); + assert.equal(f.run().status, 0); + assert.equal(f.count(), 3); + fs.writeFileSync(file, lines[0]); + assert.equal(f.run().status, 0); + assert.equal(f.count(), 4); + fs.rmSync(file); + assert.equal(f.run({ FAIL: '1' }).status, 1); + assert.equal(fs.existsSync(file), false); + assert.equal(f.run().status, 0); + assert.equal(f.count(), 6); +}); +test('disabled or unavailable cache falls back without changing discovery output', (t) => { + const f = fixture(t); + for (let i = 0; i < 2; i++) + assert.equal( + f.run({ QL_NODE_PATH_CACHE: '0' }).stdout.trim(), + '/test/global/node_modules', + ); + assert.equal(f.count(), 2); + assert.equal(fs.existsSync(f.env.dir_tmp), false); + const blocked = path.join(f.root, 'file'); + fs.writeFileSync(blocked, ''); + assert.equal( + f.run({ dir_tmp: blocked }).stdout.trim(), + '/test/global/node_modules', + ); + assert.equal(f.count(), 3); +}); +test('concurrent refreshes publish complete records and do not overwrite symlink targets', async (t) => { + const f = fixture(t); + await Promise.all( + Array.from( + { length: 8 }, + () => + new Promise((resolve, reject) => { + const p = spawn('/bin/bash', ['-uc', f.command], { + cwd: f.root, + env: f.env, + }); + let out = ''; + p.stdout.on('data', (c) => (out += c)); + p.once('error', reject); + p.once('close', (code) => { + try { + assert.equal(code, 0); + assert.equal(out.trim(), '/test/global/node_modules'); + resolve(); + } catch (e) { + reject(e); + } + }); + }), + ), + ); + const before = f.count(); + f.run(); + assert.equal(f.count(), before); + const files = fs + .readdirSync(f.env.dir_tmp) + .filter((file) => file.endsWith('.cache')); + assert.equal(files.length, 1); + const file = path.join(f.env.dir_tmp, files[0]); + fs.rmSync(file); + const victim = path.join(f.root, 'victim'); + fs.writeFileSync(victim, 'unchanged'); + fs.symlinkSync(victim, file); + assert.equal(f.run().status, 0); + assert.equal(fs.readFileSync(victim, 'utf8'), 'unchanged'); + assert.equal(fs.lstatSync(file).isSymbolicLink(), false); +}); + +test('task entry tolerates discovery failure with errexit enabled', (t) => { + const f = fixture(t); + const source = fs.readFileSync(path.resolve('shell/otask.sh'), 'utf8'); + const start = source.indexOf('append_node_dependency_path() {'); + const end = source.indexOf('\nenter_script_workdir()', start); + const script = + source.slice(start, end) + + '\nappend_node_dependency_path; printf "continued:%s" "$NODE_PATH"'; + for (const dir of [path.dirname(helper), f.root]) { + const r = f.run( + { + FAIL: '1', + dir_shell: dir, + dir_dep: '/legacy/deps', + NODE_PATH: '/existing', + }, + 'set -e; ' + script, + ); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, 'continued:/existing:/legacy/deps'); + } +}); diff --git a/test/back/node-path-lock.test.cjs b/test/back/node-path-lock.test.cjs new file mode 100644 index 00000000..94d03e32 --- /dev/null +++ b/test/back/node-path-lock.test.cjs @@ -0,0 +1,178 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn, spawnSync } = require('node:child_process'); +const helper = path.resolve('shell/node_path_cache.sh'); +const hasFlock = spawnSync('/bin/bash', ['-c', 'type -P flock']).status === 0; +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-path-lock-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + fs.symlinkSync(process.execPath, path.join(bin, 'node')); + fs.writeFileSync( + path.join(bin, 'pnpm'), + '#!/bin/bash\nprintf "call\\n" >> "$CALLS"\n[[ -n "${UMASK_FILE:-}" ]] && umask > "$UMASK_FILE"\nsleep "${LOOKUP_DELAY:-0.2}"\n[[ "${FAIL:-0}" == 1 ]] && exit 17\nprintf "%s\\n" "${npm_config_global_dir:-/test/global/node_modules}"\n', + { mode: 0o755 }, + ); + const env = { + ...process.env, + HOME: root, + PATH: bin + ':' + process.env.PATH, + dir_tmp: path.join(root, 'cache'), + CALLS: path.join(root, 'calls'), + HELPER: helper, + }; + const command = '. "$HELPER"; ql_get_node_global_path'; + const run = (extra = {}, code = command) => + spawnSync('/bin/bash', ['-euc', code], { + cwd: root, + env: { ...env, ...extra }, + encoding: 'utf8', + }); + const asyncRun = (extra = {}) => + new Promise((resolve, reject) => { + const child = spawn('/bin/bash', ['-euc', command], { + cwd: root, + env: { ...env, ...extra }, + }); + let out = '', + err = ''; + child.stdout.on('data', (c) => (out += c)); + child.stderr.on('data', (c) => (err += c)); + child.on('error', reject); + child.on('close', (status) => + resolve({ status, stdout: out, stderr: err }), + ); + }); + const count = () => + fs.existsSync(env.CALLS) + ? fs.readFileSync(env.CALLS, 'utf8').trim().split('\n').length + : 0; + const cache = path.join(env.dir_tmp, `pnpm-root-${process.getuid()}.cache`); + return { + root, + bin, + env, + command, + run, + asyncRun, + count, + cache, + lock: cache + '.lock', + }; +} +function check(r, output = '/test/global/node_modules') { + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout.trim(), output); +} +test( + 'simultaneous cold and expired lookups share one successful refresh', + { skip: !hasFlock }, + async (t) => { + const f = fixture(t); + for (let batch = 0; batch < 2; batch++) { + if (batch) { + const lines = fs.readFileSync(f.cache, 'utf8').split('\n'); + lines[1] = '0'; + fs.writeFileSync(f.cache, lines.join('\n')); + } + const rows = await Promise.all( + Array.from({ length: 8 }, () => f.asyncRun({ LOOKUP_DELAY: '0.5' })), + ); + rows.forEach((r) => check(r)); + assert.equal(f.count(), batch + 1); + } + assert.equal(fs.statSync(f.lock).mode & 0o777, 0o600); + check(f.run()); + assert.equal(f.count(), 2); + }, +); +test( + 'waiters with different configuration do not reuse another key', + { skip: !hasFlock }, + async (t) => { + const f = fixture(t); + const rows = await Promise.all( + ['/first', '/second'].map((v) => + f.asyncRun({ npm_config_global_dir: v }), + ), + ); + rows.forEach((r, i) => check(r, ['/first', '/second'][i])); + assert.equal(f.count(), 2); + }, +); +test( + 'lock timeout falls back while a holder is still alive', + { skip: !hasFlock }, + async (t) => { + const f = fixture(t); + fs.mkdirSync(f.env.dir_tmp); + const holder = spawn( + '/bin/bash', + ['-c', 'exec 9>> "$LOCK"; flock 9; echo ready; sleep 30'], + { env: { ...f.env, LOCK: f.lock }, detached: true }, + ); + t.after(() => { + try { + process.kill(-holder.pid, 'SIGKILL'); + } catch {} + }); + await new Promise((resolve, reject) => { + holder.stdout.once('data', resolve); + holder.once('error', reject); + }); + const start = Date.now(); + check(f.run()); + assert.ok(Date.now() - start >= 1800); + assert.equal(holder.exitCode, null); + assert.equal(f.count(), 1); + }, +); +test('absent or unsupported flock and unsafe lock paths preserve discovery', (t) => { + const f = fixture(t); + check( + f.run( + {}, + 'type(){ if [[ "$*" == "-P flock" ]]; then return 1; fi; builtin type "$@"; }; ' + + f.command, + ), + ); + fs.rmSync(f.cache); + fs.writeFileSync(path.join(f.bin, 'flock'), '#!/bin/bash\nexit 64\n', { + mode: 0o755, + }); + check(f.run()); + fs.rmSync(f.cache); + fs.rmSync(f.lock, { force: true }); + const victim = path.join(f.root, 'victim'); + fs.writeFileSync(victim, 'untouched'); + fs.symlinkSync(victim, f.lock); + check(f.run()); + assert.equal(fs.readFileSync(victim, 'utf8'), 'untouched'); + assert.equal(f.count(), 3); +}); +test('failed refresh releases lock and does not publish a success', (t) => { + const f = fixture(t); + assert.equal(f.run({ FAIL: '1' }).status, 17); + assert.equal(fs.existsSync(f.cache), false); + check(f.run()); + assert.equal(f.count(), 2); +}); +test('refresh leaves the caller file descriptor and umask unchanged', (t) => { + const f = fixture(t); + const sentinel = path.join(f.root, 'fd'); + const mask = path.join(f.root, 'mask'); + const r = f.run( + { SENTINEL: sentinel, UMASK_FILE: mask }, + 'umask 022; exec 9> "$SENTINEL"; ' + + f.command + + ' >/dev/null; printf preserved >&9; umask', + ); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout.trim(), '0022'); + assert.equal(fs.readFileSync(sentinel, 'utf8'), 'preserved'); + assert.equal(fs.readFileSync(mask, 'utf8').trim(), '0022'); +}); diff --git a/test/back/primary-apm.test.cjs b/test/back/primary-apm.test.cjs new file mode 100644 index 00000000..ab0e09e1 --- /dev/null +++ b/test/back/primary-apm.test.cjs @@ -0,0 +1,24 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const vm = require('node:vm'); +const fs = require('node:fs'); +const source = fs.readFileSync('ecosystem.config.js', 'utf8'); +const appSource = fs.readFileSync('back/app.ts', 'utf8'); +const ts = require('typescript'); +const compiled = ts.transpileModule(appSource.slice(0, appSource.indexOf('\nconst app = new Application();')) + '\nmodule.exports = Application;', {compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText; +function config(env) {const module={exports:{}};vm.runInNewContext(source,{module,process:{env}});return module.exports.apps[0];} +test('container APM policy keeps worker isolation and can restore all inherited probes', () => { + for (const primary of [undefined, 'true']) for (const workers of [undefined, 'true']) { + const env={QL_CONTAINER:'true',QL_PRIMARY_APM:primary,QL_WORKER_APM:workers}; + const conf=config(env);env.pm_id='0';env.pmx=String(conf.pmx); + const calls=[],module={exports:{}}; + vm.runInNewContext(compiled,{module,exports:module.exports,process:{env},require(name){if(name==='cluster')return {fork(options){calls.push({...env,...options});return {id:1,process:{pid:123}}}};if(name==='express')return ()=>({use(){}});return {};}}); + new module.exports().forkWorker('http'); + assert.equal(conf.pmx,primary==='true'); + assert.equal(calls[0].pmx,primary==='true'&&workers==='true'?'true':'false'); + assert.equal(conf.max_restarts,5);assert.equal(conf.script,'static/build/app.js'); + } +}); +test('standalone PM2 monitoring stays enabled', () => { + for(const QL_PRIMARY_APM of [undefined,'true','false']) assert.equal(config({QL_PRIMARY_APM}).pmx,true); +}); diff --git a/test/back/scheduler-mutation.test.cjs b/test/back/scheduler-mutation.test.cjs new file mode 100644 index 00000000..e975e913 --- /dev/null +++ b/test/back/scheduler-mutation.test.cjs @@ -0,0 +1,274 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); +const { setTimeout: delay } = require('node:timers/promises'); +const { Sequelize, DataTypes } = require('sequelize'); +const load = require('../helpers/load-security-module.cjs'); + +function gate() { + let release; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { promise, release }; +} + +async function fixture(t) { + const root = await fs.mkdtemp( + path.join(os.tmpdir(), 'ql-scheduler-mutation-'), + ); + const db = new Sequelize({ + dialect: 'sqlite', + storage: ':memory:', + logging: false, + }); + const crons = db.define('Cron', { + name: DataTypes.STRING, + command: DataTypes.STRING, + schedule: DataTypes.STRING, + isDisabled: DataTypes.INTEGER, + saved: DataTypes.BOOLEAN, + queued_token: DataTypes.STRING, + }); + await db.sync(); + t.after(async () => { + await db.close(); + await fs.rm(root, { recursive: true, force: true }); + }); + const jobs = new Map(); + const client = { + addCron: async (items, replace) => { + if (replace) jobs.clear(); + items.forEach((item) => jobs.set(item.id, item.command)); + }, + delCron: async (ids) => ids.forEach((id) => jobs.delete(id)), + }; + const CronService = load('back/services/cron.ts', { + '../config': { + crontabFile: path.join(root, 'crontab.list'), + logPath: root, + }, + '../data/cron': { + CrontabModel: crons, + Crontab: class { + constructor(options) { + Object.assign(this, { isDisabled: 0 }, options); + } + }, + }, + '../data/runningInstance': {}, + '../config/util': { isDemoEnv: () => false }, + '../config/const': {}, + '../schedule/client': client, + '../shared/pLimit': {}, + '../shared/utils': {}, + '../shared/i18n': { t: (s) => s }, + '../shared/logReader': {}, + '../shared/logStreamManager': {}, + }).default; + const service = new CronService({ error() {}, warn() {}, info() {} }); + service.getLogName = async () => 'task'; + service.shouldUseCronClient = () => true; + service.makeCommand = (doc) => doc.command; + service.crontabs = async () => ({ data: await crons.findAll({ raw: true }) }); + service.setCrontab = async () => {}; + return { service, crons, jobs, client, root }; +} + +for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) { + test(`recovery snapshot cannot overwrite concurrent ${operation}`, async (t) => { + const { service, crons, jobs } = await fixture(t); + await crons.create({ + id: 1, + command: 'old', + schedule: '* * * * *', + isDisabled: operation === 'enabled' ? 1 : 0, + }); + const entered = gate(), + release = gate(); + let first = true; + service.setCrontab = async () => { + if (first) { + first = false; + entered.release(); + await release.promise; + } + }; + const recovery = service.autosave_crontab(true); + await entered.promise; + const before = await crons.findAll({ raw: true }); + const mutation = + operation === 'create' + ? service.create({ command: 'new', schedule: '* * * * *' }) + : operation === 'update' + ? service.update({ id: 1, command: 'new' }) + : service[operation]([1]); + await delay(50); + assert.deepEqual( + await crons.findAll({ raw: true }), + before, + 'DB writes must also wait for recovery', + ); + release.release(); + await Promise.all([recovery, mutation]); + const rows = await crons.findAll({ raw: true }); + assert.deepEqual( + [...jobs.entries()].sort(), + rows + .filter((row) => !row.isDisabled) + .map((row) => [String(row.id), row.command]) + .sort(), + ); + }); +} + +test('recovery waits for an admitted mutation to finish registration before reading the DB', async (t) => { + const { service, client, jobs } = await fixture(t); + const entered = gate(), + release = gate(); + const add = client.addCron; + client.addCron = async (items, replace) => { + if (!replace) { + entered.release(); + await release.promise; + } + return add(items, replace); + }; + const mutation = service.create({ command: 'new', schedule: '* * * * *' }); + await entered.promise; + let reads = 0; + const read = service.crontabs; + service.crontabs = async () => { + reads++; + return read(); + }; + const recovery = service.autosave_crontab(true); + await delay(50); + assert.equal(reads, 0); + release.release(); + await Promise.all([mutation, recovery]); + assert.deepEqual([...jobs.values()], ['new']); +}); + +for (const operation of ['create', 'update', 'enabled']) { + test(`${operation} preserves scheduler 503 after rollback and releases its lock`, async (t) => { + const { service, crons, client } = await fixture(t); + if (operation !== 'create') + await crons.create({ + id: 1, + command: 'old', + schedule: '* * * * *', + isDisabled: operation === 'enabled' ? 1 : 0, + }); + const cause = Object.assign(Error('unavailable'), { status: 503 }); + client.addCron = async () => { + throw cause; + }; + const mutation = + operation === 'create' + ? service.create({ command: 'new', schedule: '* * * * *' }) + : operation === 'update' + ? service.update({ id: 1, command: 'new' }) + : service.enabled([1]); + await assert.rejects( + mutation, + (err) => + err.status === 503 && err.cause === cause && /回滚/.test(err.message), + ); + if (operation === 'create') assert.equal(await crons.count(), 0); + else { + const row = await crons.findByPk(1); + assert.equal(row.command, 'old'); + assert.equal(row.isDisabled, operation === 'enabled' ? 1 : 0); + } + client.addCron = async () => {}; + await service.autosave_crontab(true); + }); +} + +test( + 'scheduler lock excludes a second OS process and leaves no lock after completion', + { timeout: 10000 }, + async (t) => { + const root = await fs.mkdtemp( + path.join(os.tmpdir(), 'ql-scheduler-process-'), + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const config = { crontabFile: path.join(root, 'crontab.list') }; + const { withSchedulerMutation } = load( + 'back/shared/schedulerMutationLock.ts', + { '../config': config }, + ); + const entered = gate(), + release = gate(); + const owner = withSchedulerMutation(async () => { + entered.release(); + await release.promise; + }); + await entered.promise; + const child = spawn( + process.execPath, + [ + '-e', + ` + const load = require('./test/helpers/load-security-module.cjs'); + const { withSchedulerMutation } = load('back/shared/schedulerMutationLock.ts', {'../config': JSON.parse(process.argv[1])}); + process.send('attempting'); + withSchedulerMutation(async () => process.send('entered')).then(() => process.disconnect()).catch(err => { console.error(err); process.exit(1); }); + `, + JSON.stringify(config), + ], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }, + ); + t.after(() => { + if (child.exitCode === null) child.kill('SIGKILL'); + release.release(); + }); + const closed = once(child, 'close'); + assert.equal((await once(child, 'message'))[0], 'attempting'); + let acquired = false; + const acquisition = once(child, 'message').then(([message]) => { + acquired = true; + assert.equal(message, 'entered'); + }); + await delay(150); + assert.equal(acquired, false); + release.release(); + await owner; + await acquisition; + assert.equal((await closed)[0], 0); + await assert.rejects(fs.stat(`${config.crontabFile}.scheduler.lock`), { + code: 'ENOENT', + }); + }, +); + +test('configuration rollback cannot restore an obsolete manual queue token', async (t) => { + const { service, crons, client } = await fixture(t); + await crons.create({ + id: 1, + command: 'old', + schedule: '* * * * *', + isDisabled: 0, + queued_token: 'old-token', + }); + let first = true; + client.addCron = async () => { + if (first) { + first = false; + await crons.update({ queued_token: 'new-token' }, { where: { id: 1 } }); + throw Object.assign(Error('unavailable'), { status: 503 }); + } + }; + await assert.rejects( + service.update({ id: 1, command: 'new' }), + (err) => err.status === 503, + ); + const row = await crons.findByPk(1); + assert.equal(row.command, 'old'); + assert.equal(row.queued_token, 'new-token'); +}); diff --git a/test/back/scheduler-readiness.test.cjs b/test/back/scheduler-readiness.test.cjs new file mode 100644 index 00000000..1fe68834 --- /dev/null +++ b/test/back/scheduler-readiness.test.cjs @@ -0,0 +1,85 @@ +const test=require('node:test'); +const assert=require('node:assert/strict'); +const {setTimeout:sleep}=require('node:timers/promises'); +const {SchedulerReadiness}=require('../../back/shared/schedulerReadiness'); +const load=require('../helpers/load-security-module.cjs'); +const fs=require('node:fs'); +const ts=require('typescript'); + +test('readiness stays false through failed restoration and retries without idle polling',async()=>{ + let probes=0,restores=0,fail=true; + const state=new SchedulerReadiness(async()=>{probes++;},10); + state.configure(async()=>{restores++;if(fail)throw Error('registration failed');}); + assert.equal(await state.recover(),false);assert.equal(await state.check(),false); + fail=false;await sleep(40);assert.equal(await state.check(),true);assert.ok(restores>=2); + const before=probes;await sleep(40);assert.equal(probes,before); +}); +test('single-flight recovery cannot mark an invalidated generation ready',async()=>{ + let release,restores=0; + const state=new SchedulerReadiness(async()=>{},10); + state.configure(async()=>{if(++restores===1)await new Promise(r=>release=r);}); + const a=state.recover();assert.equal(state.recover(),a);await sleep(1); + state.invalidate();release();assert.equal(await a,false);assert.equal(await state.check(),false); + await sleep(30);assert.equal(await state.check(),true);assert.equal(restores,2); +}); +test('mutation waiting is bounded, a later recovery can succeed',async()=>{ + let release; + const state=new SchedulerReadiness(async()=>{},10);state.configure(()=>new Promise(r=>release=r)); + await assert.rejects(state.ensureReady(15),e=>e.status===503); + release();await sleep(1);assert.equal(await state.check(),true); +}); +test('probe failure immediately invalidates a previously ready scheduler',async()=>{ + let fail=false; + const state=new SchedulerReadiness(async()=>{if(fail)throw Error('unavailable');},10); + state.configure(async()=>{});assert.equal(await state.recover(),true); + fail=true;assert.equal(await state.check(),false); + fail=false;await sleep(40);assert.equal(await state.check(),true); +}); +test('health uses actual readiness and returns HTTP 503 until recovery',async(t)=>{ + let ready=false;const express=require('express'); + const {HealthService}=load('back/services/health.ts',{ + typedi:{Service:()=>x=>x},'../loaders/logger':{error(){}},'./http':{}, + '../schedule/client':{readiness:{check:async()=>ready}}, + }); + const service=new HealthService({getServer:()=>({})}); + const app=express();load('back/api/health.ts',{ + typedi:{get:()=>service},'../services/health':{HealthService},'../loaders/logger':{error(){}}, + }).default(app); + const server=app.listen(0,'127.0.0.1');await new Promise(r=>server.once('listening',r)); + t.after(()=>new Promise(r=>{server.close(r);server.closeAllConnections();})); + for(const expected of [503,200,503]){ + ready=expected===200;const r=await fetch(`http://127.0.0.1:${server.address().port}/health`);const b=await r.json(); + assert.equal(r.status,expected);assert.equal(b.code,expected);assert.equal(b.data.services.grpc,ready); + } +}); +test('recovery registration errors propagate while ordinary autosave retains file synchronization',async()=>{ + const source=fs.readFileSync('back/services/cron.ts','utf8');const a=source.indexOf(' public async autosave_crontab('),z=source.indexOf(' public async bootTask',a); + const js=ts.transpileModule('class Fixture {\n'+source.slice(a,z)+'}\nmodule.exports=Fixture;', {compilerOptions:{target:ts.ScriptTarget.ES2020}}).outputText; + const module={exports:{}};let files=0; + new Function('module','isDemoEnv','cronClient','withSchedulerMutation',js)(module,()=>false,{addCron:async()=>{throw Error('registration unavailable');}},fn=>fn()); + const fixture=new module.exports();fixture.crontabs=async()=>({data:[]});fixture.setCrontab=async()=>{files++;};fixture.logger={warn(){}}; + await fixture.autosave_crontab();assert.equal(files,1); + await assert.rejects(fixture.autosave_crontab(true),/registration unavailable/);assert.equal(files,2); +}); +test('scheduler probe uses the cron channel and failed writes are not replayed',async()=>{ + const calls=[];const fake={ + waitForReady(deadline,cb){calls.push(['wait',deadline]);cb();}, + makeUnaryRequest(path,serialize,deserialize,request,options,cb){calls.push(['probe',path,request,options]);cb(null,{status:1});}, + addCron(request,metadata,options,cb){calls.push(['add',request,options]);cb(Object.assign(Error('invalid'),{code:3}));}, + }; + const client=load('back/schedule/client.ts',{ + '../protos/cron':{CronClient:class{constructor(){return fake;}}}, + '../config':{grpcPort:5500},'../config/grpcCerts':{getGrpcCerts:()=>({caCert:'ca',clientKey:'key',clientCert:'cert'})}, + '@grpc/grpc-js':{...require('@grpc/grpc-js'),credentials:{createSsl:()=>({})},status:{UNAVAILABLE:14},Metadata:class{}}, + }).default; + client.readiness.configure(async()=>{});assert.equal(await client.readiness.recover(),true); + assert.ok(calls.filter(x=>x[0]==='probe').every(x=>x[1]==='/com.ql.health.Health/Check'&&x[2].service==='scheduler'&&x[3].deadline>Date.now())); + await assert.rejects(client.addCron([]),/invalid/);assert.equal(calls.filter(x=>x[0]==='add').length,1); +}); +test('scheduler health probe never calls back into HTTP health',async()=>{ + const {check}=load('back/schedule/health.ts',{ + '../config':{},undici:{request:()=>{throw Error('recursive HTTP health call');}}, + }); + const result=await new Promise(resolve=>check({request:{service:'scheduler'}},(error,response)=>resolve({error,response}))); + assert.equal(result.error,null);assert.deepEqual(result.response,{status:1}); +}); diff --git a/test/back/scheduler-reconciliation.test.cjs b/test/back/scheduler-reconciliation.test.cjs new file mode 100644 index 00000000..1380c623 --- /dev/null +++ b/test/back/scheduler-reconciliation.test.cjs @@ -0,0 +1,152 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const ts = require('typescript'); +const load = require('../helpers/load-security-module.cjs'); +const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness'); +const { AddCronRequest } = require('../../back/protos/cron'); + +function fixture(client) { + const source = fs.readFileSync('back/services/cron.ts', 'utf8'); + const a = source.indexOf(' public async autosave_crontab('); + const z = source.indexOf(' public async bootTask', a); + const js = ts.transpileModule( + 'class Fixture {\n' + source.slice(a, z) + '}\nmodule.exports=Fixture;', + { compilerOptions: { target: ts.ScriptTarget.ES2020 } }, + ).outputText; + const module = { exports: {} }; + new Function('module', 'isDemoEnv', 'cronClient', 'withSchedulerMutation', js)( + module, + () => false, + client, + (fn) => fn(), + ); + const service = new module.exports(); + service.setCrontab = async () => {}; + service.logger = { warn() {} }; + service.shouldUseCronClient = () => true; + service.makeCommand = () => 'true'; + return service; +} + +test('recovery reconciles a surviving scheduler after missed deletes and disables, including an empty DB', async () => { + const cancelled = []; + const stacks = new Map( + ['removed', 'disabled', 'kept'].map((id) => [ + id, + [{ cancel: () => cancelled.push(id) }], + ]), + ); + const { addCron } = load('back/schedule/addCron.ts', { + './data': { scheduleStacks: stacks }, + 'node-schedule': { + scheduleJob: (id) => ({ cancel: () => cancelled.push(id) }), + }, + '../shared/runCron': {}, + '../loaders/logger': { info() {}, warn() {} }, + '../shared/i18n': { tf: (s) => s }, + }); + const client = { + addCron: async (crons, replace) => { + // Exercise the real protobuf field, not just a JavaScript-only flag. + const request = AddCronRequest.decode( + AddCronRequest.encode({ crons, replace }).finish(), + ); + await new Promise((resolve, reject) => + addCron({ request }, (err) => (err ? reject(err) : resolve())), + ); + }, + }; + const service = fixture(client); + let rows = [ + { id: 'kept', schedule: '* * * * *', isDisabled: 0 }, + { id: 'disabled', schedule: '* * * * *', isDisabled: 1 }, + ]; + service.crontabs = async () => ({ data: rows }); + const state = new SchedulerReadiness(async () => {}); + state.configure(() => service.autosave_crontab(true)); + assert.equal(await state.recover(), true); + assert.equal(await state.check(), true); + assert.deepEqual([...stacks.keys()], ['kept']); + assert.deepEqual(cancelled.sort(), ['disabled', 'kept', 'removed']); + rows = []; + state.invalidate(); + assert.equal(await state.recover(), true); + assert.equal(stacks.size, 0); + assert.equal(await state.check(), true); +}); + +test('invalid replacement leaves the previous schedule intact', async () => { + const stacks = new Map([ + [ + 'old', + [ + { + cancel: () => { + throw Error('must not cancel'); + }, + }, + ], + ], + ]); + const { addCron } = load('back/schedule/addCron.ts', { + './data': { scheduleStacks: stacks }, + '../shared/runCron': {}, + '../loaders/logger': {}, + '../shared/i18n': { tf: (s) => s }, + }); + await assert.rejects( + new Promise((resolve, reject) => + addCron( + { + request: { + replace: true, + crons: [{ id: 'bad', schedule: '?', extra_schedules: [] }], + }, + }, + (err) => (err ? reject(err) : resolve()), + ), + ), + ); + assert.deepEqual([...stacks.keys()], ['old']); +}); + +test('pre-RPC channel failures invalidate readiness and return 503 without executing or replaying writes', async () => { + for (const method of ['addCron', 'delCron']) { + let invalidations = 0, + writes = 0; + const fake = { + waitForReady: (_deadline, cb) => cb(Error('channel unavailable')), + addCron: () => writes++, + delCron: () => writes++, + }; + const client = load('back/schedule/client.ts', { + '../protos/cron': { + CronClient: class { + constructor() { + return fake; + } + }, + }, + '../config': { grpcPort: 5500 }, + '../config/grpcCerts': { + getGrpcCerts: () => ({ + caCert: 'ca', + clientKey: 'key', + clientCert: 'cert', + }), + }, + '@grpc/grpc-js': { + ...require('@grpc/grpc-js'), + credentials: { createSsl: () => ({}) }, + }, + }).default; + client.readiness.invalidate = () => invalidations++; + await assert.rejects( + client[method]([]), + (err) => err.status === 503 && /channel unavailable/.test(err.message), + ); + assert.equal(invalidations, 1); + assert.equal(writes, 0); + } +}); diff --git a/test/back/schema-migrations.test.cjs b/test/back/schema-migrations.test.cjs new file mode 100644 index 00000000..e1249aff --- /dev/null +++ b/test/back/schema-migrations.test.cjs @@ -0,0 +1,127 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs/promises'); +const path = require('node:path'); +const os = require('node:os'); +const { Sequelize, QueryTypes } = require('sequelize'); +const { migrateSchema } = require('../../back/shared/schemaMigrations'); +const load = require('../helpers/load-security-module.cjs'); + +async function createLegacy(storage = ':memory:', omitEnvs = false) { + const database = new Sequelize({ + dialect: 'sqlite', + storage, + logging: false, + }); + for (const table of [ + 'CrontabViews', + 'Subscriptions', + 'Crontabs', + ...(omitEnvs ? [] : ['Envs']), + ]) { + await database.query( + `CREATE TABLE "${table}" (id INTEGER PRIMARY KEY, name TEXT)`, + ); + } + await database.query( + 'INSERT INTO "Crontabs" (id, name) VALUES (1, \'keep-me\')', + ); + return database; +} + +test('legacy database upgrades without losing rows and repeated migration is idempotent', async (t) => { + const database = await createLegacy(); + t.after(() => database.close()); + await migrateSchema(database); + await migrateSchema(database); + const rows = await database.query('SELECT * FROM "Crontabs"', { + type: QueryTypes.SELECT, + }); + assert.equal(rows[0].name, 'keep-me'); + assert.equal(rows[0].queued_token, null); + assert.ok(Object.hasOwn(rows[0], 'allow_multiple_instances')); + const applied = await database.query('SELECT id FROM "SchemaMigrations"', { + type: QueryTypes.SELECT, + }); + assert.equal(applied.length, 15); +}); + +test('migration failure rolls back added columns and can be retried after repair', async (t) => { + const database = await createLegacy(':memory:', true); + t.after(() => database.close()); + await assert.rejects(migrateSchema(database)); + const columns = await database.getQueryInterface().describeTable('Crontabs'); + assert.equal(Object.hasOwn(columns, 'work_dir'), false); + await database.query( + 'CREATE TABLE "Envs" (id INTEGER PRIMARY KEY, name TEXT)', + ); + await migrateSchema(database); + assert.ok( + Object.hasOwn( + await database.getQueryInterface().describeTable('Crontabs'), + 'work_dir', + ), + ); +}); + +test('offline pre-upgrade backup restores the legacy schema and data', async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-migration-')); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + const file = path.join(dir, 'database.sqlite'); + const backup = path.join(dir, 'before.sqlite'); + let database = await createLegacy(file); + await database.close(); + await fs.copyFile(file, backup); + database = new Sequelize({ + dialect: 'sqlite', + storage: file, + logging: false, + }); + await migrateSchema(database); + await database.close(); + await fs.copyFile(backup, file); + database = new Sequelize({ + dialect: 'sqlite', + storage: file, + logging: false, + }); + try { + assert.equal( + Object.hasOwn( + await database.getQueryInterface().describeTable('Crontabs'), + 'work_dir', + ), + false, + ); + const rows = await database.query('SELECT name FROM "Crontabs"', { + type: QueryTypes.SELECT, + }); + assert.equal(rows[0].name, 'keep-me'); + } finally { + await database.close(); + } +}); + +test('database loader rejects initialization failure rather than allowing workers to start', async () => { + const mocks = { './logger': { error() {}, info() {} } }; + for (const [file, model] of [ + ['env', 'EnvModel'], + ['cron', 'CrontabModel'], + ['dependence', 'DependenceModel'], + ['open', 'AppModel'], + ['system', 'SystemModel'], + ['subscription', 'SubscriptionModel'], + ['cronView', 'CrontabViewModel'], + ['cronStats', 'CrontabStatModel'], + ['runningInstance', 'RunningInstanceModel'], + ]) + mocks[`../data/${file}`] = { [model]: { sync: async () => {} } }; + mocks['../data'] = { sequelize: {} }; + mocks['../shared/schemaMigrations'] = { + migrateSchema: async () => { + throw new Error('SQLITE_FULL'); + }, + }; + const initialize = load(path.resolve('back/loaders/db.ts'), mocks).default; + await assert.rejects(initialize(), /SQLITE_FULL/); +}); diff --git a/test/back/shell-api-parsing.test.cjs b/test/back/shell-api-parsing.test.cjs new file mode 100644 index 00000000..8c7960b4 --- /dev/null +++ b/test/back/shell-api-parsing.test.cjs @@ -0,0 +1,184 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const source = fs + .readFileSync(path.resolve('shell/api.sh'), 'utf8') + .replace(/\nget_token\s*$/, '\n'); +const realJq = spawnSync('which', ['jq'], { encoding: 'utf8' }).stdout.trim(); +function fixture(t) { + assert.ok(realJq, 'jq is required'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-api-parse-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync( + path.join(bin, 'jq'), + '#!/bin/bash\nprintf "call\\n" >> "$JQ_CALLS"\nexec "$REAL_JQ" "$@"\n', + { mode: 0o755 }, + ); + const token = path.join(root, 'token.json'), + response = path.join(root, 'response.json'), + calls = path.join(root, 'calls'); + const run = (code) => + spawnSync( + '/bin/bash', + [ + '-euc', + source + + '\ncreate_token(){ __ql_token__=generated; generated=1; }; curl(){ cat "$RESPONSE_FILE"; };\n' + + code, + ], + { + encoding: 'utf8', + env: { + ...process.env, + PATH: bin + ':' + process.env.PATH, + REAL_JQ: realJq, + JQ_CALLS: calls, + file_auth_token: token, + RESPONSE_FILE: response, + ql_port: '5700', + __ql_token__: 'test-token', + }, + }, + ); + return { + token, + response, + run, + count: () => + fs.existsSync(calls) + ? fs.readFileSync(calls, 'utf8').trim().split('\n').length + : 0, + }; +} +test('valid token cache is read with one jq and reused without generation', (t) => { + const f = fixture(t); + fs.writeFileSync( + f.token, + JSON.stringify({ + value: 'header.payload.signature', + expiration: 4102444800, + }), + ); + const r = f.run( + 'get_token; printf "%s:%s" "$__ql_token__" "${generated:-0}"', + ); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, 'header.payload.signature:0'); + assert.equal(f.count(), 1); +}); +test('expired, corrupt, missing or header-unsafe token cache refreshes under errexit', (t) => { + const f = fixture(t); + for (const content of [ + undefined, + 'broken', + '{}', + JSON.stringify({ value: 'old', expiration: 1 }), + JSON.stringify({ value: 'old', expiration: '4102444800' }), + JSON.stringify({ value: '', expiration: 4102444800 }), + JSON.stringify({ value: 'bad\ninjected', expiration: 4102444800 }), + JSON.stringify({ value: 'bad\n', expiration: 4102444800 }), + JSON.stringify({ value: 'bad\rinjected', expiration: 4102444800 }), + ]) { + if (content === undefined) fs.rmSync(f.token, { force: true }); + else fs.writeFileSync(f.token, content); + const r = f.run('get_token; printf "%s:%s" "$__ql_token__" "$generated"'); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, 'generated:1'); + } +}); +test('success with a message still parses once and stays silent', (t) => { + const f = fixture(t); + fs.writeFileSync(f.response, JSON.stringify({ code: 200, message: 'ok' })); + const r = f.run('update_cron 1 0 123 log 1'); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, ''); + assert.equal(f.count(), 1); +}); +test('status and statistics errors preserve multiline Unicode messages', (t) => { + const f = fixture(t); + fs.writeFileSync( + f.response, + JSON.stringify({ code: 500, message: '写入失败\n请重试 "原任务"' }), + ); + for (const command of [ + 'update_cron 1 1 123 log 1 2 7', + 'record_cron_stat 1 7 2', + ]) { + const r = f.run(command); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, '写入失败\n请重试 "原任务"\n'); + } + assert.equal(f.count(), 2); +}); +test('non-JSON errors fall back to raw text and absent messages retain legacy null', (t) => { + const f = fixture(t); + fs.writeFileSync(f.response, 'upstream unavailable'); + let r = f.run('update_cron 1 0 123 log 1'); + assert.equal(r.status, 0); + assert.equal(r.stdout, 'upstream unavailable\n'); + fs.writeFileSync(f.response, '{"code":500}'); + r = f.run('record_cron_stat 1 7 2'); + assert.equal(r.status, 0); + assert.equal(r.stdout, 'null\n'); +}); +test('statistics without a task id perform no JSON parsing', (t) => { + const f = fixture(t); + const r = f.run('record_cron_stat "" 0 1 || true'); + assert.equal(r.status, 0, r.stderr); + assert.equal(f.count(), 0); +}); + +test('canonical success resets prior errors and matches jq missing-message semantics', (t) => { + const f = fixture(t); + fs.writeFileSync(f.response, '{"code":200}'); + const r = f.run( + 'code=500; message=old; ql_parse_status_response "$(cat "$RESPONSE_FILE")"; printf "%s|%s" "$code" "$message"; record_cron_stat 1 0 1', + ); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, '200|null'); + assert.equal(f.count(), 0); +}); + +test('noncanonical JSON always uses jq, including whitespace, duplicate keys and embedded success text', (t) => { + const f = fixture(t); + const cases = [ + [' {"code":200} ', '200|null'], + ['{"code": 200}', '200|null'], + ['{"code":200,"message":"完成\\n第二行"}', '200|完成\n第二行'], + ['{"code":200,"data":{"code":500}}', '200|null'], + ['{"code":"200"}', '200|null'], + ['{"code":200,"code":500,"message":"error"}', '500|error'], + ['{"code":500,"message":"{\\"code\\":200}"}', '500|{"code":200}'], + ['{"code":500}', '500|null'], + ]; + for (const [body, expected] of cases) { + fs.writeFileSync(f.response, body); + const r = f.run( + 'ql_parse_status_response "$(cat "$RESPONSE_FILE")"; printf "%s|%s" "$code" "$message"', + ); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, expected); + } + assert.equal(f.count(), cases.length); +}); + +test('success-shaped invalid documents retain the raw-response error fallback', (t) => { + const f = fixture(t); + const cases = [ + '{"code":200}trailing', + '{"code":200', + '{"code":200}', + ]; + for (const body of cases) { + fs.writeFileSync(f.response, body); + const r = f.run('update_cron 1 0 123 log 1'); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, body + '\n'); + } + assert.equal(f.count(), cases.length); +}); diff --git a/test/back/stop-race.test.cjs b/test/back/stop-race.test.cjs new file mode 100644 index 00000000..5662b46d --- /dev/null +++ b/test/back/stop-race.test.cjs @@ -0,0 +1,167 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { Sequelize, DataTypes } = require('sequelize'); +const load = require('../helpers/load-security-module.cjs'); +async function fixture(t, onKill = async () => {}) { + const db = new Sequelize({ + dialect: 'sqlite', + storage: ':memory:', + logging: false, + }); + t.after(() => db.close()); + const crons = db.define('Cron', { + command: DataTypes.STRING, + status: DataTypes.INTEGER, + pid: DataTypes.INTEGER, + log_path: DataTypes.STRING, + queued_token: DataTypes.STRING, + last_execution_time: DataTypes.INTEGER, + last_running_time: DataTypes.INTEGER, + }); + const instances = db.define('Instance', { + cron_id: DataTypes.INTEGER, + pid: DataTypes.INTEGER, + status: DataTypes.INTEGER, + exit_code: DataTypes.INTEGER, + finished_at: DataTypes.INTEGER, + }); + await db.sync(); + await crons.bulkCreate([ + { id: 1, command: 'task one.js', status: 0, pid: 101, log_path: 'one.log' }, + { id: 2, command: 'task two.js', status: 0, pid: 201, log_path: 'two.log' }, + ]); + await instances.bulkCreate([ + { id: 1, cron_id: 1, pid: 101, status: 0 }, + { id: 2, cron_id: 1, pid: 102, status: 0 }, + { id: 3, cron_id: 1, pid: 99, status: 1, exit_code: 0 }, + { id: 4, cron_id: 1, pid: 98, status: 3, exit_code: 7 }, + { id: 5, cron_id: 2, pid: 201, status: 0 }, + ]); + let service; + const killed = []; + const CronService = load(path.resolve('back/services/cron.ts'), { + '../config': {}, + '../data/cron': { + CrontabModel: crons, + CrontabStatus: { queued: 3, running: 0, idle: 1 }, + }, + '../data/runningInstance': { + RunningInstanceModel: instances, + InstanceStatus: { running: 0, finished: 1, stopped: 2, error: 3 }, + }, + '../config/util': { + killTask: async (pid, wait) => { + assert.equal(wait, true); + killed.push(pid); + await onKill({ pid, service, instances, crons }); + }, + killAllTasks: async () => { + throw Error('command-wide scans must not run'); + }, + }, + '../config/const': {}, + '../schedule/client': {}, + '../shared/pLimit': {}, + '../shared/utils': {}, + '../shared/i18n': { t: (s) => s }, + '../shared/logReader': {}, + '../shared/logStreamManager': {}, + '../shared/childProcess': require('../../back/shared/childProcess'), + }).default; + service = new CronService({ info() {}, error() {} }); + service.getDb = ({ id }) => crons.findByPk(id); + return { service, instances, crons, killed }; +} +async function report(service, pid, code) { + await service.status({ + ids: [1], + status: 1, + pid, + log_path: 'one.log', + last_running_time: 1, + last_execution_time: 100, + exit_code: code, + }); +} +test('stop wins when shell exit writes error or success during termination, preserving exit codes', async (t) => { + const f = await fixture(t, async ({ service }) => { + await report(service, 101, 143); + await report(service, 102, 0); + }); + await f.service.stop([1]); + const rows = await f.instances.findAll({ order: [['id', 'ASC']], raw: true }); + assert.deepEqual( + rows.map((r) => [r.id, r.status, r.exit_code]), + [ + [1, 2, 143], + [2, 2, 0], + [3, 1, 0], + [4, 3, 7], + [5, 0, null], + ], + ); +}); +test('late shell completion cannot overwrite stopped instances', async (t) => { + const f = await fixture(t); + await f.service.stop([1]); + await report(f.service, 101, 143); + assert.equal((await f.instances.findByPk(1)).status, 2); + assert.equal((await f.instances.findByPk(2)).status, 2); +}); +test('rows created after the stop snapshot are not relabelled', async (t) => { + const f = await fixture(t, async ({ instances }) => { + if (!(await instances.findByPk(6))) + await instances.create({ id: 6, cron_id: 1, pid: 103, status: 0 }); + }); + await f.service.stop([1]); + assert.equal((await f.instances.findByPk(6)).status, 0); + assert.equal((await f.instances.findByPk(1)).status, 2); +}); +test('repeated stop does not rewrite historical rows or their finished timestamps', async (t) => { + const f = await fixture(t); + await f.service.stop([1]); + const before = await f.instances.findAll({ + raw: true, + order: [['id', 'ASC']], + }); + await f.service.stop([1]); + assert.deepEqual( + await f.instances.findAll({ raw: true, order: [['id', 'ASC']] }), + before, + ); +}); +test('batch stop captures running instances from every requested cron', async (t) => { + const f = await fixture(t); + await f.service.stop([1, 2]); + assert.equal((await f.instances.findByPk(5)).status, 2); + assert.equal((await f.instances.findByPk(3)).status, 1); +}); + +test('stop signals every snapshotted PID exactly once and preserves a later running instance', async (t) => { + const f = await fixture(t, async ({ instances, crons }) => { + if (!(await instances.findByPk(6))) { + await instances.create({ id: 6, cron_id: 1, pid: 103, status: 0 }); + await crons.update( + { pid: 103, log_path: 'later.log', status: 0 }, + { where: { id: 1 } }, + ); + } + }); + await f.service.stop([1]); + assert.deepEqual(f.killed, [101, 102]); + assert.equal((await f.instances.findByPk(6)).status, 0); + assert.equal((await f.crons.findByPk(1)).pid, 103); + assert.equal((await f.crons.findByPk(1)).status, 0); +}); + +test('failed termination is not finalized as stopped', async (t) => { + const f = await fixture(t, async ({ pid }) => { + if (pid === 102) throw Error('still alive'); + }); + await f.service.stop([1]); + assert.deepEqual(f.killed, [101, 102]); + assert.equal((await f.instances.findByPk(1)).status, 2); + assert.equal((await f.instances.findByPk(2)).status, 0); + assert.equal((await f.crons.findByPk(1)).status, 0); +}); diff --git a/test/back/subscription-cleanup.test.cjs b/test/back/subscription-cleanup.test.cjs new file mode 100644 index 00000000..fb9acf90 --- /dev/null +++ b/test/back/subscription-cleanup.test.cjs @@ -0,0 +1,53 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const path = require('node:path'); +const dayjs = require('dayjs'); +const load = require('../helpers/load-security-module.cjs'); + +test('subscription becomes idle and closes its log even when completion logging fails', async () => { + const updates = []; + let closed = 0; + let notified = 0; + const Subscription = load(path.resolve('back/services/subscription.ts'), { + '../config': {}, + '../data/subscription': { + SubscriptionModel: { update: async (values) => updates.push(values) }, + SubscriptionStatus: { idle: 1 }, + }, + '../data/cron': {}, + '../config/util': { handleLogPath: async () => '/tmp/unused-log' }, + '../config/const': { LOG_END_SYMBOL: 'end' }, + '../config/subscription': {}, + '../shared/i18n': { t: (s) => s, tf: (s) => s }, + '../shared/pLimit': {}, + '../shared/logReader': {}, + '../shared/logStreamManager': { + logStreamManager: { + write: async () => { + throw new Error('ENOSPC'); + }, + closeStream: async () => { + closed++; + throw new Error('ENOSPC'); + }, + }, + }, + './schedule': {}, + './sock': {}, + './sshKey': {}, + './cron': {}, + }).default; + const service = new Subscription( + {}, + {}, + { sendMessage: () => notified++ }, + {}, + {}, + ); + service.getDb = async () => ({ id: 1, log_path: 'log' }); + const callbacks = service.taskCallbacks({ id: 1 }); + await assert.rejects(callbacks.onEnd(undefined, dayjs(), 1), /ENOSPC/); + assert.equal(closed, 1); + assert.equal(updates.at(-1).status, 1); + assert.equal(notified, 1); +}); diff --git a/test/back/task-time.test.cjs b/test/back/task-time.test.cjs new file mode 100644 index 00000000..449252f5 --- /dev/null +++ b/test/back/task-time.test.cjs @@ -0,0 +1,116 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +function extract(file, name) { + const text = fs.readFileSync(file, 'utf8'); + const start = text.indexOf(name + '() {'); + assert.ok(start >= 0); + return text.slice(start, text.indexOf('\n}', start) + 2); +} +const source = + ['handle_log_path', 'init_begin_time'] + .map((n) => extract('shell/task.sh', n)) + .join('\n') + + '\n' + + ['format_time', 'format_log_time', 'format_timestamp', 'handle_task_end'] + .map((n) => extract('shell/share.sh', n)) + .join('\n'); +function run(t, body) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-task-time-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const result = spawnSync( + '/bin/bash', + [ + '-ec', + source + + ` +is_macos=0 +mtime_format='%Y-%m-%d %H:%M:%S.%3N' +time_format='%Y-%m-%d %H:%M:%S' +ID=42; dir_log=/tmp; log_name=''; real_log_path=''; no_tee=''; real_time='' +make_dir() { :; } +t() { printf '%s\\n' "$*" >> "$TEST_DIR/messages"; } +update_cron() { printf '%s\\n' "$*" >> "$TEST_DIR/status"; } +record_cron_stat() { printf '%s\\n' "$*" >> "$TEST_DIR/stats"; } +date() { + printf '%s\\n' "$*" >> "$TEST_DIR/dates" + case "$1" in + '+%Y-%m-%d %H:%M:%S.%3N|%Y-%m-%d-%H-%M-%S-%3N') printf '%s\\n' '2026-09-12 23:59:59.999|2026-09-12-23-59-59-999';; + '+%Y-%m-%d %H:%M:%S|%s') printf '%s\\n' '2026-09-13 00:00:00|101';; + '-d') [[ "$3" == '+%s' ]] && echo 100;; + *) echo raw-time;; + esac +} +` + + body, + ], + { encoding: 'utf8', env: { ...process.env, TEST_DIR: dir } }, + ); + assert.equal(result.status, 0, result.stderr + result.stdout); + return { + dir, + text: (name) => + fs.existsSync(path.join(dir, name)) + ? fs.readFileSync(path.join(dir, name), 'utf8') + : '', + }; +} +test('Linux task timestamps share a snapshot and retain millisecond log names across midnight', (t) => { + const r = run( + t, + `handle_log_path 'folder/example.js' +init_begin_time +[[ "$log_path" == 'folder_example_42/2026-09-12-23-59-59-999.log' ]] +[[ "$time" == '2026-09-12 23:59:59.999' && "$begin_time" == '2026-09-12 23:59:59' && "$begin_timestamp" == 100 ]] +handle_task_end`, + ); + assert.equal(r.text('dates').trim().split('\n').length, 3); + assert.match(r.text('stats'), /^42 0 1\n$/); + assert.match(r.text('messages'), /2026-09-13 00:00:00/); +}); +test('explicit log paths and output mode remain intact', (t) => { + run( + t, + `real_log_path='custom/output.log'; real_time=true +handle_log_path 'folder/example.js' +[[ "$log_path" == 'custom/output.log' && "$cmd" == '' ]]`, + ); +}); +test('task failure keeps exit code and minimum one-second runtime', (t) => { + const r = run( + t, + `begin_timestamp=101; _task_exit_code=7; log_path=test.log +handle_task_end`, + ); + assert.equal(r.text('stats'), '42 7 1\n'); + assert.match(r.text('messages'), /失败/); + assert.match(r.text('status'), /101 1 7/); +}); +test('manual stop retains stopped message and exit status reporting', (t) => { + const r = run( + t, + `begin_timestamp=99; _task_exit_code=1; log_path=test.log; MANUAL=true +handle_task_end`, + ); + assert.equal(r.text('stats'), '42 1 2\n'); + assert.match(r.text('messages'), /已停止/); +}); +for (const mode of ['macos', 'custom']) + test(`${mode} keeps legacy time conversion helpers`, (t) => { + run( + t, + ` +${mode === 'macos' ? 'is_macos=1' : 'mtime_format=custom; time_format=custom'} +format_log_time() { echo legacy-log; } +format_time() { echo legacy-time; } +format_timestamp() { echo 100; } +handle_log_path example.js +init_begin_time +[[ "$log_time" == legacy-log && "$begin_time" == legacy-time && "$begin_timestamp" == 100 ]] +handle_task_end +`, + ); + }); diff --git a/test/back/worker-apm.test.cjs b/test/back/worker-apm.test.cjs new file mode 100644 index 00000000..595188d3 --- /dev/null +++ b/test/back/worker-apm.test.cjs @@ -0,0 +1,84 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const ts = require('typescript'); +const source = fs.readFileSync('back/app.ts', 'utf8'); +const entry = source.indexOf('\nconst app = new Application();'); +assert.ok(entry > 0, 'application entry point exists'); +const compiled = ts.transpileModule( + source.slice(0, entry) + '\nmodule.exports = Application;', + { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + esModuleInterop: true, + }, + }, +).outputText; +function create(env) { + const calls = []; + const cluster = { + fork(options) { + const worker = { + id: calls.length + 1, + process: { pid: 100 + calls.length }, + }; + calls.push({ options, inherited: { ...env, ...options }, worker }); + return worker; + }, + }; + const module = { exports: {} }; + const sandbox = { + module, + exports: module.exports, + process: { env }, + require(name) { + if (name === 'cluster') return cluster; + if (name === 'express') return () => ({ use() {} }); + return {}; + }, + }; + vm.runInNewContext(compiled, sandbox); + return { app: new module.exports(), calls }; +} +test('PM2-managed primary disables only inherited worker APM and keeps worker metadata', () => { + const env = { pm_id: '0', pmx: 'true', CUSTOM_VALUE: 'kept' }; + const { app, calls } = create(env); + for (const role of ['grpc', 'http']) app.forkWorker(role); + assert.deepEqual( + calls.map((x) => ({ ...x.options })), + [ + { SERVICE_TYPE: 'grpc', pmx: 'false' }, + { SERVICE_TYPE: 'http', pmx: 'false' }, + ], + ); + assert.equal(env.pmx, 'true'); + assert.equal(calls[0].inherited.CUSTOM_VALUE, 'kept'); + for (const call of calls) { + const entry = app.workerMetadataMap.get(call.worker.id); + assert.equal(entry.pid, call.worker.process.pid); + assert.equal(entry.serviceType, call.options.SERVICE_TYPE); + } +}); +test('worker APM opt-in restores inherited PM2 settings, including explicit disable', () => { + for (const pmx of ['true', 'false']) { + const { app, calls } = create({ pm_id: '0', pmx, QL_WORKER_APM: 'true' }); + app.forkWorker('http'); + assert.equal(Object.hasOwn(calls[0].options, 'pmx'), false); + assert.equal(calls[0].inherited.pmx, pmx); + } +}); +test('standalone startup does not introduce a PM2-specific override', () => { + const { app, calls } = create({ CUSTOM_VALUE: 'kept' }); + app.forkWorker('grpc'); + assert.deepEqual({ ...calls[0].options }, { SERVICE_TYPE: 'grpc' }); + assert.equal(calls[0].inherited.CUSTOM_VALUE, 'kept'); +}); +test('replacement workers receive the same monitoring policy', () => { + const { app, calls } = create({ pm_id: '0', pmx: 'true' }); + app.forkWorker('grpc'); + app.forkWorker('grpc'); + assert.equal(calls[1].inherited.pmx, 'false'); + assert.notEqual(calls[0].worker.id, calls[1].worker.id); +});