From 89c6355635112574a7e1c368a527ac49e6582cd4 Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 11 Oct 2026 01:51:04 +0800 Subject: [PATCH] fix: renew system tokens and serialize authentication cache updates --- back/loaders/initData.ts | 27 +- back/loaders/initTask.ts | 1 + back/services/open.ts | 150 ++++-- back/token.ts | 21 +- shell/api.sh | 126 +++-- test/back/restart-instance-recovery.test.cjs | 3 +- test/back/shell-api-parsing.test.cjs | 8 +- test/back/shellApiToken.test.cjs | 151 ++++++ test/back/systemTokenRenewal.test.cjs | 497 +++++++++++++++++++ 9 files changed, 835 insertions(+), 149 deletions(-) create mode 100644 test/back/shellApiToken.test.cjs create mode 100644 test/back/systemTokenRenewal.test.cjs diff --git a/back/loaders/initData.ts b/back/loaders/initData.ts index bfe3e61b..7e36171b 100644 --- a/back/loaders/initData.ts +++ b/back/loaders/initData.ts @@ -13,12 +13,11 @@ import { AuthDataType, SystemModel } from '../data/system'; import SystemService from '../services/system'; import UserService from '../services/user'; import { writeFile, readFile } from 'fs/promises'; -import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../config/util'; +import { fileExist, isDemoEnv, safeJSONParse } from '../config/util'; import OpenService from '../services/open'; import { shareStore } from '../shared/store'; import Logger from './logger'; import cronClient from '../schedule/client'; -import { AppModel } from '../data/open'; import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance'; import { setLang, systemLang } from '../shared/i18n'; @@ -31,24 +30,7 @@ export default async () => { const openService = Container.get(OpenService); // 初始化增加系统配置 - let systemApp = ( - await AppModel.findOne({ - where: { name: 'system' }, - }) - )?.get({ plain: true }); - if (!systemApp) { - systemApp = await AppModel.create({ - name: 'system', - scopes: ['crons', 'system', 'dashboard'], - client_id: createRandomString(12, 12), - client_secret: createRandomString(24, 24), - }); - } else if (!systemApp.scopes.includes('dashboard')) { - await AppModel.update( - { scopes: [...systemApp.scopes, 'dashboard'] }, - { where: { name: 'system' } }, - ); - } + await openService.initializeSystemApp(); const [systemConfig] = await SystemModel.findOrCreate({ where: { type: AuthDataType.systemConfig }, }); @@ -250,9 +232,6 @@ export default async () => { await envService.set_envs(); const authInfo = await userService.getAuthInfo(); - const apps = await openService.findApps(); await shareStore.updateAuthInfo(authInfo); - if (apps?.length) { - await shareStore.updateApps(apps); - } + await openService.refreshApps(); }; diff --git a/back/loaders/initTask.ts b/back/loaders/initTask.ts index 608f00e9..7424a6b5 100644 --- a/back/loaders/initTask.ts +++ b/back/loaders/initTask.ts @@ -26,6 +26,7 @@ export default async () => { if (await fileExist(tokenFile)) { tokenCommand = `node ${tokenFile}`; } + tokenCommand += ' --renew'; const cron = { id: NaN, name: t('生成token'), diff --git a/back/services/open.ts b/back/services/open.ts index 19d846c0..e39486a6 100644 --- a/back/services/open.ts +++ b/back/services/open.ts @@ -6,27 +6,77 @@ import { v4 as uuidV4 } from 'uuid'; import sequelize, { Op } from 'sequelize'; import { shareStore } from '../shared/store'; import { t } from '../shared/i18n'; +import { lock } from 'proper-lockfile'; +import path from 'path'; +import os from 'os'; +import { writeFile, rename, rm, stat, chmod } from 'fs/promises'; +import config from '../config'; @Service() export default class OpenService { constructor(@Inject('logger') private logger: winston.Logger) {} + // Every database mutation and full authentication-cache publication shares + // this cross-process lock. Lock a separate marker, not token.json itself. + private async withAppsLock(operation: () => Promise): Promise { + const target = path.join(config.configPath, '.apps-state'); + const release = await lock(target, { + realpath: false, + stale: 10_000, + retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 }, + }); + try { + return await operation(); + } finally { + await release(); + } + } + + public async initializeSystemApp(): Promise { + await this.withAppsLock(async () => { + const doc = await AppModel.findOne({ where: { name: 'system' } }); + const app = doc?.get({ plain: true }); + if (!app) { + await this.insert({ + name: 'system', + scopes: ['crons', 'system', 'dashboard'], + client_id: createRandomString(12, 12), + client_secret: createRandomString(24, 24), + }); + } else if (!app.scopes.includes('dashboard')) { + await AppModel.update( + { scopes: [...app.scopes, 'dashboard'] }, + { where: { id: app.id } }, + ); + } + await shareStore.updateApps(await this.find({})); + }); + } + + public async refreshApps(): Promise { + await this.withAppsLock(async () => { + await shareStore.updateApps(await this.find({})); + }); + } + public async findApps(): Promise { const docs = await this.find({}); return docs; } public async create(payload: App): Promise { - const tab = { ...payload }; - tab.client_id = createRandomString(12, 12); - tab.client_secret = createRandomString(24, 24); - const doc = await this.insert(tab); - const apps = await this.find({}); - await shareStore.updateApps(apps); - return { ...doc, tokens: [] }; + return this.withAppsLock(async () => { + const tab = { ...payload }; + tab.client_id = createRandomString(12, 12); + tab.client_secret = createRandomString(24, 24); + const doc = await this.insert(tab); + const apps = await this.find({}); + await shareStore.updateApps(apps); + return { ...doc, tokens: [] }; + }); } - public async insert(payload: App): Promise { + private async insert(payload: App): Promise { const doc = await AppModel.create(payload, { returning: true }); return doc.get({ plain: true }); } @@ -41,10 +91,12 @@ export default class OpenService { } private async updateDb(payload: Partial): Promise { - await AppModel.update(payload, { where: { id: payload.id } }); - const apps = await this.find({}); - await shareStore.updateApps(apps); - return apps?.find((x) => x.id === payload.id) as App; + return this.withAppsLock(async () => { + await AppModel.update(payload, { where: { id: payload.id } }); + const apps = await this.find({}); + await shareStore.updateApps(apps); + return apps?.find((x) => x.id === payload.id) as App; + }); } public async getDb(query: Record): Promise { @@ -56,9 +108,11 @@ export default class OpenService { } public async remove(ids: number[]) { - await AppModel.destroy({ where: { id: ids } }); - const apps = await this.find({}); - await shareStore.updateApps(apps); + return this.withAppsLock(async () => { + await AppModel.destroy({ where: { id: ids } }); + const apps = await this.find({}); + await shareStore.updateApps(apps); + }); } public async resetSecret(id: number): Promise { @@ -112,7 +166,15 @@ export default class OpenService { return docs.map((x) => x.get({ plain: true })); } - public async authToken({ + public async authToken(credentials: { + client_id: string; + client_secret: string; + }): Promise { + return this.withAppsLock(() => this.issueToken(credentials)); + } + + // Caller holds withAppsLock through database update and cache publication. + private async issueToken({ client_id, client_secret, }: { @@ -156,27 +218,47 @@ export default class OpenService { } } - public async generateSystemToken(): Promise<{ + public async generateSystemToken(renew = false): Promise<{ value: string; expiration: number; }> { - const apps = await shareStore.getApps(); - const systemApp = apps?.find((x) => x.name === 'system'); - if (!systemApp) { - throw new Error('system app not found'); - } - const now = Math.round(Date.now() / 1000); - const currentToken = systemApp.tokens?.find((x) => x.expiration > now); - if (currentToken) { - return currentToken; - } - const { data } = await this.authToken({ - client_id: systemApp.client_id, - client_secret: systemApp.client_secret, + return this.withAppsLock(async () => { + // Read after acquiring the lock; a concurrent issuer may have renewed. + const systemApp = await this.getDb({ name: 'system' }); + const now = Math.round(Date.now() / 1000); + let appToken = systemApp.tokens + ?.filter((token) => token.expiration > now) + .sort((left, right) => right.expiration - left.expiration)[0]; + if (renew || !appToken) { + const { data } = await this.issueToken({ + client_id: systemApp.client_id, + client_secret: systemApp.client_secret, + }); + appToken = { value: data.token, expiration: data.expiration }; + } else { + // Recover a prior database commit whose cache publication failed. + await shareStore.updateApps(await this.find({})); + } + const tokenFile = path.join(config.configPath, 'token.json'); + const temporaryFile = `${tokenFile}.${process.pid}.tmp`; + const mode = await stat(tokenFile).then( + (file) => file.mode & 0o777, + (error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return 0o600; + throw error; + }, + ); + try { + await writeFile(temporaryFile, `${JSON.stringify(appToken)}${os.EOL}`, { + mode, + }); + await chmod(temporaryFile, mode); + // Shell readers see a complete old or new document, never a partial one. + await rename(temporaryFile, tokenFile); + } finally { + await rm(temporaryFile, { force: true }); + } + return appToken; }); - return { - ...data, - value: data.token, - }; } } diff --git a/back/token.ts b/back/token.ts index d307c623..e4400ca3 100755 --- a/back/token.ts +++ b/back/token.ts @@ -2,30 +2,19 @@ import 'reflect-metadata'; import OpenService from './services/open'; import { Container } from 'typedi'; import LoggerInstance from './loaders/logger'; -import config from './config'; -import path from 'path'; -import os from 'os'; -import { writeFileWithLock } from './shared/utils'; - -const tokenFile = path.join(config.configPath, 'token.json'); async function getToken() { try { Container.set('logger', LoggerInstance); const openService = Container.get(OpenService); - const appToken = await openService.generateSystemToken(); + const appToken = await openService.generateSystemToken( + process.argv.includes('--renew'), + ); console.log(appToken.value); - await writeFile({ - value: appToken.value, - expiration: appToken.expiration, - }); } catch (error) { - console.log(error); + console.error(error); + process.exitCode = 1; } } -async function writeFile(data: any) { - await writeFileWithLock(tokenFile, `${JSON.stringify(data)}${os.EOL}`); -} - getToken(); diff --git a/shell/api.sh b/shell/api.sh index 2cf9a9ca..e4b1afe8 100755 --- a/shell/api.sh +++ b/shell/api.sh @@ -26,6 +26,28 @@ get_token() { create_token } +# Check the cached token before every authenticated API request. A rejected +# token is refreshed and retried once; other failures are never retried. +# Arguments are curl options, including the URL, method and optional body. +ql_api_request() { + local api="" + local attempt + get_token || return $? + for attempt in 1 2; do + api=$(curl -s --noproxy "*" \ + -H "Authorization: Bearer ${__ql_token__}" \ + -H "Content-Type: application/json;charset=UTF-8" \ + "$@" \ + --compressed) + if [[ "$attempt" == 2 || "$api" == '{"code":200}' ]] || + [[ "$(jq -r '.code' <<< "$api" 2>/dev/null)" != 401 ]]; then + break + fi + create_token || return $? + done + printf '%s' "$api" +} + # Read the status response once; preserve multiline error messages and the # legacy code/message variables used by the shell API callers. ql_parse_status_response() { @@ -63,13 +85,10 @@ add_cron_api() { sub_id="null" fi - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"sub_id\":$sub_id}" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ + --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"sub_id\":$sub_id}" \ + -X POST) code=$(echo "$api" | jq -r .code) message=$(echo "$api" | jq -r .message) if [[ $code == 200 ]]; then @@ -93,14 +112,10 @@ update_cron_api() { local id="$4" fi - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ - -X 'PUT' \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"id\":\"$id\"}" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ + -X PUT \ + --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"id\":\"$id\"}") code=$(echo "$api" | jq -r .code) message=$(echo "$api" | jq -r .message) if [[ $code == 200 ]]; then @@ -120,14 +135,10 @@ update_cron_command_api() { local id="$2" fi - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ - -X 'PUT' \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "{\"command\":\"${command//\"/\\\"}\",\"id\":\"$id\"}" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ + -X PUT \ + --data-raw "{\"command\":\"${command//\"/\\\"}\",\"id\":\"$id\"}") code=$(echo "$api" | jq -r .code) message=$(echo "$api" | jq -r .message) if [[ $code == 200 ]]; then @@ -140,14 +151,10 @@ update_cron_command_api() { del_cron_api() { local ids="$1" local currentTimeStamp=$(date +%s) - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ - -X 'DELETE' \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "[$ids]" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \ + -X DELETE \ + --data-raw "[$ids]") code=$(echo "$api" | jq -r .code) message=$(echo "$api" | jq -r .message) if [[ $code == 200 ]]; then @@ -171,14 +178,10 @@ update_cron() { dataRaw="${dataRaw},\"exit_code\":$exitCode" fi dataRaw="${dataRaw}}" - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \ - -X 'PUT' \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "$dataRaw" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \ + -X PUT \ + --data-raw "$dataRaw") ql_parse_status_response "$api" || true if [[ $code != 200 ]]; then if [[ ! $message ]]; then @@ -192,14 +195,10 @@ notify_api() { local title="$1" local content="$2" local currentTimeStamp=$(date +%s) - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/system/notify?t=$currentTimeStamp" \ - -X 'PUT' \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "{\"title\":\"${title//\"/\\\"}\",\"content\":\"${content//\"/\\\"}\"}" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/system/notify?t=$currentTimeStamp" \ + -X PUT \ + --data-raw "{\"title\":\"${title//\"/\\\"}\",\"content\":\"${content//\"/\\\"}\"}") code=$(echo "$api" | jq -r .code) message=$(echo "$api" | jq -r .message) if [[ $code == 200 ]]; then @@ -212,12 +211,9 @@ notify_api() { find_cron_api() { local params="$1" local currentTimeStamp=$(date +%s) - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/detail?$params&t=$currentTimeStamp" \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/crons/detail?$params&t=$currentTimeStamp" \ + -X GET) data=$(echo "$api" | jq -r .data) if [[ $data == 'null' ]]; then echo -e "" @@ -231,14 +227,10 @@ update_auth_config() { local body="$1" local tip="$2" local currentTimeStamp=$(date +%s) - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port}/open/system/auth/reset?t=$currentTimeStamp" \ - -X 'PUT' \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "{$body}" \ - --compressed - ) + local api=$(ql_api_request \ + "http://localhost:${ql_port}/open/system/auth/reset?t=$currentTimeStamp" \ + -X PUT \ + --data-raw "{$body}") code=$(echo "$api" | jq -r .code) message=$(echo "$api" | jq -r .message) if [[ $code == 200 ]]; then @@ -254,14 +246,10 @@ record_cron_stat() { local elapsed="${3:-0}" [[ $ref_id ]] && [[ $ref_id -gt 0 ]] 2>/dev/null || return - local api=$( - curl -s --noproxy "*" "http://localhost:${ql_port:-5700}/open/dashboard/record" \ + local api=$(ql_api_request \ + "http://localhost:${ql_port:-5700}/open/dashboard/record" \ -X POST \ - -H "Authorization: Bearer ${__ql_token__}" \ - -H "Content-Type: application/json;charset=UTF-8" \ - --data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}" \ - --compressed - ) + --data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}") ql_parse_status_response "$api" || true if [[ $code != 200 ]]; then if [[ ! $message ]]; then @@ -270,5 +258,3 @@ record_cron_stat() { echo -e "${message}" fi } - -get_token diff --git a/test/back/restart-instance-recovery.test.cjs b/test/back/restart-instance-recovery.test.cjs index a0e3c03d..ac0f78bd 100644 --- a/test/back/restart-instance-recovery.test.cjs +++ b/test/back/restart-instance-recovery.test.cjs @@ -27,7 +27,8 @@ test('startup closes interrupted instances without inventing exit codes or rewri const service = { set_envs: async () => {}, getAuthInfo: async () => ({}), - findApps: async () => [], + initializeSystemApp: async () => {}, + refreshApps: async () => {}, }; const initData = load('back/loaders/initData.ts', { typedi: { Container: { get: () => service } }, diff --git a/test/back/shell-api-parsing.test.cjs b/test/back/shell-api-parsing.test.cjs index 8c7960b4..c0e60061 100644 --- a/test/back/shell-api-parsing.test.cjs +++ b/test/back/shell-api-parsing.test.cjs @@ -91,13 +91,13 @@ test('expired, corrupt, missing or header-unsafe token cache refreshes under err assert.equal(r.stdout, 'generated:1'); } }); -test('success with a message still parses once and stays silent', (t) => { +test('success with a message checks authentication and stays silent', (t) => { const f = fixture(t); fs.writeFileSync(f.response, JSON.stringify({ code: 200, message: 'ok' })); const r = f.run('update_cron 1 0 123 log 1'); assert.equal(r.status, 0, r.stderr); assert.equal(r.stdout, ''); - assert.equal(f.count(), 1); + assert.equal(f.count(), 2); }); test('status and statistics errors preserve multiline Unicode messages', (t) => { const f = fixture(t); @@ -113,7 +113,7 @@ test('status and statistics errors preserve multiline Unicode messages', (t) => assert.equal(r.status, 0, r.stderr); assert.equal(r.stdout, '写入失败\n请重试 "原任务"\n'); } - assert.equal(f.count(), 2); + assert.equal(f.count(), 4); }); test('non-JSON errors fall back to raw text and absent messages retain legacy null', (t) => { const f = fixture(t); @@ -180,5 +180,5 @@ test('success-shaped invalid documents retain the raw-response error fallback', assert.equal(r.status, 0, r.stderr); assert.equal(r.stdout, body + '\n'); } - assert.equal(f.count(), cases.length); + assert.equal(f.count(), cases.length * 2); }); diff --git a/test/back/shellApiToken.test.cjs b/test/back/shellApiToken.test.cjs new file mode 100644 index 00000000..92b02f42 --- /dev/null +++ b/test/back/shellApiToken.test.cjs @@ -0,0 +1,151 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); + +const apiFile = path.resolve(__dirname, '../../shell/api.sh'); + +const endpoints = { + status: { + method: 'PUT', url: /\/open\/crons\/status\?t=\d+$/, + body: { ids: ['37'], status: '1', pid: '4321', log_path: 'task/run.log', + last_execution_time: 1787004000, last_running_time: 90, exit_code: 0 }, + success: '', failure: (message) => message, + }, + statistics: { + method: 'POST', url: /\/open\/dashboard\/record$/, + body: { ref_id: 37, code: 0, elapsed: 90 }, + success: '', failure: (message) => message, + }, + add: { + method: 'POST', url: /\/open\/crons\?t=\d+$/, + body: { name: 'sample job', command: 'task job.py', schedule: '0 * * * *', sub_id: null }, + success: 'sample job -> 添加成功', failure: (message) => `sample job -> 添加失败(${message})`, + }, + update: { + method: 'PUT', url: /\/open\/crons\?t=\d+$/, + body: { name: 'sample job', command: 'task job.py', schedule: '0 * * * *', id: '37' }, + success: 'sample job -> 更新成功', failure: (message) => `sample job -> 更新失败(${message})`, + }, + command: { + method: 'PUT', url: /\/open\/crons\?t=\d+$/, + body: { command: 'task job.py', id: '37' }, + success: 'task job.py -> 更新成功', failure: (message) => `task job.py -> 更新失败(${message})`, + }, + delete: { + method: 'DELETE', url: /\/open\/crons\?t=\d+$/, + body: [37, 38], success: '成功', failure: (message) => `失败(${message})`, + }, + notify: { + method: 'PUT', url: /\/open\/system\/notify\?t=\d+$/, + body: { title: 'title', content: 'body' }, + success: '通知发送成功🎉', failure: (message) => `通知失败(${message})`, + }, + find: { + method: 'GET', url: /\/open\/crons\/detail\?log_path=task\/run.log&t=\d+$/, + body: null, success: 'sample job', failure: () => '', + }, + auth: { + method: 'PUT', url: /\/open\/system\/auth\/reset\?t=\d+$/, + body: { retries: 0 }, success: '重置登录错误次数成功🎉', + failure: (message) => `重置登录错误次数失败(${message})`, + }, +}; + +function runCallback(endpoint, scenario) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-api-token-')); + try { + const tokenFile = path.join(directory, 'token.json'); + fs.writeFileSync(tokenFile, JSON.stringify({ + value: 'initial-token', expiration: Math.floor(Date.now() / 1000) + 3600, + })); + const child = spawnSync('bash', ['-c', ` + . "$API_FILE" + create_token() { + printf 'refresh\\n' >> "$CAPTURE" + __ql_token__=fresh-token + printf '{"value":"fresh-token","expiration":4102444800}' > "$file_auth_token" + } + case "$SCENARIO" in + expired) printf '{"value":"initial-token","expiration":1}' > "$file_auth_token" ;; + replaced) printf '{"value":"fresh-token","expiration":4102444800}' > "$file_auth_token" ;; + esac + curl() { + local auth body url method arg + while [[ $# -gt 0 ]]; do + arg="$1"; shift + case "$arg" in + -H) [[ "$1" == Authorization:* ]] && auth="$1"; shift ;; + --data-raw) body="$1"; shift ;; + -X) method="$1"; shift ;; + http:*) url="$arg" ;; + esac + done + printf '%s\\n%s\\n%s\\n%s\\n' "$auth" "$method" "$url" "\${body:-null}" >> "$CAPTURE" + if [[ "$SCENARIO" == forbidden ]]; then + printf '{"code":403,"message":"forbidden"}' + elif [[ "$SCENARIO" == unavailable ]]; then + printf '{"code":500,"message":"unavailable"}' + elif [[ "$SCENARIO" == persistent || ( "$SCENARIO" != valid && "$auth" != 'Authorization: Bearer fresh-token' ) ]]; then + printf '{"code":401,"message":"Token 已失效"}' + else + printf '{"code":200,"data":{"name":"sample job"}}' + fi + } + t() { printf "$@"; } + case "$ENDPOINT" in + status) update_cron '"37"' 1 4321 'task/run.log' 1787004000 90 0 execution-37 ;; + statistics) record_cron_stat 37 0 90 ;; + add) add_cron_api '0 * * * *' 'task job.py' 'sample job' ;; + update) update_cron_api '0 * * * *' 'task job.py' 'sample job' 37 ;; + command) update_cron_command_api 'task job.py' 37 ;; + delete) del_cron_api '37,38' ;; + notify) notify_api title body ;; + find) find_cron_api 'log_path=task/run.log' ;; + auth) update_auth_config '"retries":0' '重置登录错误次数' ;; + esac + `], { + env: { ...process.env, PATH: `/usr/bin:/bin:${process.env.PATH}`, API_FILE: apiFile, file_auth_token: tokenFile, + CAPTURE: path.join(directory, 'requests'), SCENARIO: scenario, + ENDPOINT: endpoint, ql_port: '5700' }, + encoding: 'utf8', timeout: 10_000, + }); + assert.equal(child.status, 0, child.stderr); + assert.equal(child.stderr, ''); + const lines = fs.readFileSync(path.join(directory, 'requests'), 'utf8').trim().split('\n'); + const refreshes = lines.filter((line) => line === 'refresh').length; + const requests = lines.filter((line) => line !== 'refresh'); + const captures = []; + for (let i = 0; i < requests.length; i += 4) { + captures.push({ auth: requests[i], method: requests[i + 1], + url: requests[i + 2], body: JSON.parse(requests[i + 3]) }); + } + return { output: child.stdout, refreshes, captures }; + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +for (const [endpoint, expected] of Object.entries(endpoints)) { + for (const scenario of ['valid', 'expired', 'replaced', 'rejected', 'persistent', 'forbidden', 'unavailable']) { + test(`${endpoint} API handles ${scenario} token without unbounded retries`, () => { + const { output, refreshes, captures } = runCallback(endpoint, scenario); + const retries = ['rejected', 'persistent'].includes(scenario); + assert.equal(captures.length, retries ? 2 : 1); + assert.equal(refreshes, ['expired', 'rejected', 'persistent'].includes(scenario) ? 1 : 0); + for (const capture of captures) { + assert.deepEqual(capture.body, expected.body); + assert.equal(capture.method, expected.method); + assert.match(capture.url, expected.url); + } + if (['expired', 'replaced', 'rejected'].includes(scenario)) { + assert.equal(captures.at(-1).auth, 'Authorization: Bearer fresh-token'); + } + const error = scenario === 'persistent' ? 'Token 已失效' + : scenario === 'forbidden' ? 'forbidden' : scenario === 'unavailable' ? 'unavailable' : ''; + assert.equal(output.trim(), error ? expected.failure(error) : expected.success); + }); + } +} diff --git a/test/back/systemTokenRenewal.test.cjs b/test/back/systemTokenRenewal.test.cjs new file mode 100644 index 00000000..ad185674 --- /dev/null +++ b/test/back/systemTokenRenewal.test.cjs @@ -0,0 +1,497 @@ +require('reflect-metadata'); +require('ts-node/register/transpile-only'); + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFile } = require('node:child_process'); +const { promisify } = require('node:util'); +const environment = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-token-env-')); +fs.writeFileSync(path.join(environment, '.env'), ''); +fs.mkdirSync(path.join(environment, 'data/db'), { recursive: true }); +process.env.QL_DIR = environment; +process.env.QL_DATA_DIR = path.join(environment, 'data'); +require('node:test').after(async () => { + await require('../../back/shared/store').keyvStore.disconnect(); + await require('../../back/data').sequelize.close(); + fs.rmSync(environment, { recursive: true, force: true }); +}); +const OpenService = require('../../back/services/open').default; +const { AppModel } = require('../../back/data/open'); +const { shareStore } = require('../../back/shared/store'); +const config = require('../../back/config').default; + +const DAY = 86400; +const START = 1787004000; +const execute = promisify(execFile); + +function fixture(t, initialTokens = []) { + const previousConfigPath = config.configPath; + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-token-renewal-')); + config.configPath = directory; + t.after(() => { + config.configPath = previousConfigPath; + fs.rmSync(directory, { recursive: true, force: true }); + }); + let now = START; + let app = { + name: 'system', client_id: 'system-client', client_secret: 'system-secret', + scopes: ['crons', 'system', 'dashboard'], tokens: initialTokens, + }; + t.mock.method(Date, 'now', () => now * 1000); + t.mock.method(shareStore, 'getApps', async () => [app]); + t.mock.method(shareStore, 'updateApps', async (apps) => { [app] = apps; }); + t.mock.method(AppModel, 'findOne', async ({ where }) => { + if (where.name) assert.equal(where.name, app.name); + else { + assert.equal(where.client_id, app.client_id); + assert.equal(where.client_secret, app.client_secret); + } + return { tokens: app.tokens, get: () => app }; + }); + t.mock.method(AppModel, 'update', async ({ tokens }) => { + app = { ...app, tokens }; + return [1]; + }); + t.mock.method(AppModel, 'findAll', async () => [{ get: () => app }]); + return { + service: new OpenService({}), + atDay(day) { now = START + day * DAY; }, + tokens() { return app.tokens; }, + }; +} + +test('28-day token generation renews ahead of the 30-day expiry', async (t) => { + const state = fixture(t); + const initial = await state.service.generateSystemToken(); + assert.equal(initial.expiration, START + 30 * DAY); + state.atDay(28); + const renewed = await state.service.generateSystemToken(true); + assert.equal(renewed.expiration, START + 58 * DAY); + assert.notEqual(renewed.value, initial.value); + assert.ok(state.tokens().some((token) => token.value === initial.value)); + state.atDay(56); + const next = await state.service.generateSystemToken(true); + assert.equal(next.expiration, START + 86 * DAY); + assert.ok(state.tokens().some((token) => token.value === renewed.value)); + assert.ok(!state.tokens().some((token) => token.value === initial.value)); +}); + +test('startup generation renews before a restarted 28-day interval', async (t) => { + const state = fixture(t); + await state.service.generateSystemToken(); + state.atDay(20); + const restarted = await state.service.generateSystemToken(true); + assert.equal(restarted.expiration, START + 50 * DAY); + state.atDay(48); + const renewed = await state.service.generateSystemToken(true); + assert.equal(renewed.expiration, START + 78 * DAY); + assert.ok(state.tokens().some((token) => token.value === restarted.value)); +}); + +test('generation at the five-token limit extends the reused token without revoking active tokens', async (t) => { + const initial = Array.from({ length: 5 }, (_, index) => ({ + value: `token-${index}`, expiration: START + 2 * DAY, + })); + const state = fixture(t, initial); + const renewed = await state.service.generateSystemToken(true); + assert.equal(renewed.value, 'token-4'); + assert.equal(renewed.expiration, START + 30 * DAY); + assert.equal(state.tokens().length, 5); + for (const token of initial) { + assert.ok(state.tokens().some((current) => current.value === token.value)); + } +}); + +test('ordinary acquisition reuses the newest valid database token and republishes its cache', async (t) => { + const initial = [ + { value: 'older-token', expiration: START + DAY }, + { value: 'newer-token', expiration: START + 20 * DAY }, + ]; + const state = fixture(t, initial); + // A stale process cache must not override the token chosen from the database. + shareStore.getApps = async () => [{ name: 'system', tokens: [] }]; + const token = await state.service.generateSystemToken(); + assert.equal(token.value, 'newer-token'); + assert.deepEqual(state.tokens(), initial); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(config.configPath, 'token.json'), 'utf8')), + token, + ); +}); + +test('a failed generation releases its lock and does not publish a token', async (t) => { + const state = fixture(t); + const update = AppModel.update; + AppModel.update = async () => { throw new Error('generation failed'); }; + await assert.rejects(state.service.generateSystemToken(), /generation failed/); + assert.equal(fs.existsSync(path.join(config.configPath, 'token.json')), false); + assert.equal(fs.existsSync(path.join(config.configPath, 'token.json.refresh.lock')), false); + AppModel.update = update; + const token = await state.service.generateSystemToken(); + assert.ok(state.tokens().some((current) => current.value === token.value)); +}); + +test('acquisition recovers an authentication cache failure after the database commit', async (t) => { + const state = fixture(t); + const updateCache = shareStore.updateApps; + let attempts = 0; + shareStore.updateApps = async (apps) => { + if (++attempts === 1) throw new Error('authentication cache failed'); + return updateCache(apps); + }; + await assert.rejects(state.service.generateSystemToken(), /authentication cache failed/); + assert.equal(fs.existsSync(path.join(config.configPath, 'token.json')), false); + const committed = state.tokens()[0]; + const token = await state.service.generateSystemToken(); + assert.equal(token.value, committed.value); + assert.equal(state.tokens().length, 1); + assert.equal(attempts, 2); +}); + +async function processFixture(t, initialTokens = []) { + const root = path.resolve(__dirname, '../..'); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-token-process-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const databaseFile = path.join(directory, 'database.json'); + fs.writeFileSync(databaseFile, JSON.stringify({ + name: 'system', client_id: 'system-client', client_secret: 'system-secret', + scopes: ['crons', 'system', 'dashboard'], tokens: initialTokens, + })); + const worker = path.join(directory, 'worker.cjs'); + fs.writeFileSync(worker, ` + const fs = require('node:fs/promises'); + const path = require('node:path'); + const root = process.cwd(); + const directory = __dirname; + const config = require(path.join(root, 'back/config')).default; + const { AppModel } = require(path.join(root, 'back/data/open')); + const { shareStore } = require(path.join(root, 'back/shared/store')); + config.configPath = directory; + const databaseFile = path.join(directory, 'database.json'); + const read = async () => JSON.parse(await fs.readFile(databaseFile, 'utf8')); + AppModel.findOne = async () => { + if (process.env.QL_TOKEN_TEST_HOLD === '1') { + await fs.writeFile(path.join(directory, 'locked'), 'ready'); + await new Promise(() => setInterval(() => {}, 1000)); + } + const app = await read(); + // Widen the read/write overlap so the original race is reproducible. + await new Promise(resolve => setTimeout(resolve, 75)); + return { ...app, get: () => app }; + }; + AppModel.update = async ({ tokens }) => { + if (process.env.QL_TOKEN_TEST_FAIL === '1') throw new Error('generation failed'); + const app = await read(); + await fs.writeFile(databaseFile, JSON.stringify({ ...app, tokens })); + return [1]; + }; + AppModel.findAll = async () => { + const app = await read(); + return [{ get: () => app }]; + }; + shareStore.getApps = async () => { throw new Error('stale cache must not be read'); }; + shareStore.updateApps = async () => {}; + require(path.join(root, 'back/token')); + `); + return { + directory, + run(renew = false, fail = false, hold = false) { + return execute(process.execPath, [ + '-r', require.resolve('ts-node/register/transpile-only'), worker, + ...(renew ? ['--renew'] : []), + ], { + cwd: root, timeout: 20_000, + env: { ...process.env, QL_TOKEN_TEST_FAIL: fail ? '1' : '0', QL_TOKEN_TEST_HOLD: hold ? '1' : '0' }, + }); + }, + tokens() { return JSON.parse(fs.readFileSync(databaseFile, 'utf8')).tokens; }, + cached() { return JSON.parse(fs.readFileSync(path.join(directory, 'token.json'), 'utf8')); }, + }; +} + +test('four independent token processes converge on one usable token when the cache is missing', async (t) => { + const state = await processFixture(t); + const results = await Promise.all(Array.from({ length: 4 }, () => state.run())); + const issued = results.map(({ stdout, stderr }) => { + assert.equal(stderr, ''); + return stdout.trim(); + }); + assert.equal(new Set(issued).size, 1); + assert.equal(state.tokens().length, 1); + assert.equal(state.tokens()[0].value, issued[0]); + assert.equal(state.cached().value, issued[0]); + assert.equal(fs.existsSync(path.join(state.directory, 'token.json.refresh.lock')), false); +}); + +test('concurrent scheduled renewal and ordinary acquisition preserve all returned tokens', async (t) => { + const original = { value: 'original-token', expiration: Math.round(Date.now() / 1000) + DAY }; + const state = await processFixture(t, [original]); + const results = await Promise.all([state.run(true), state.run(), state.run(true), state.run()]); + const tokens = state.tokens(); + assert.equal(tokens.length, 3); + assert.ok(tokens.some((token) => token.value === original.value)); + for (const { stdout, stderr } of results) { + assert.equal(stderr, ''); + assert.ok(tokens.some((token) => token.value === stdout.trim())); + } + assert.ok(tokens.some((token) => token.value === state.cached().value)); +}); + +test('token CLI reports generation failure on stderr with a nonzero exit code', async (t) => { + const state = await processFixture(t); + await assert.rejects(state.run(false, true), (error) => { + assert.equal(error.code, 1); + assert.equal(error.stdout, ''); + assert.match(error.stderr, /generation failed/); + return true; + }); + assert.equal(fs.existsSync(path.join(state.directory, 'token.json')), false); + const recovered = await state.run(); + assert.equal(recovered.stdout.trim(), state.cached().value); +}); + +test('startup configures the 28-day token task with explicit renewal', async (t) => { + const { Container } = require('typedi'); + const initialize = require('../../back/loaders/initTask').default; + const SystemService = require('../../back/services/system').default; + const ScheduleService = require('../../back/services/schedule').default; + const SubscriptionService = require('../../back/services/subscription').default; + const SshKeyService = require('../../back/services/sshKey').default; + const scheduled = []; + const services = new Map([ + [SystemService, { getSystemConfig: async () => undefined }], + [ScheduleService, { + cancelIntervalTask: async () => {}, + createIntervalTask: (...args) => scheduled.push(args), + }], + [SubscriptionService, { setSshConfig: async () => {}, list: async () => [] }], + [SshKeyService, {}], + ]); + t.mock.method(Container, 'get', (service) => { + assert.ok(services.has(service)); + return services.get(service); + }); + await initialize(); + assert.equal(scheduled.length, 1); + const [task, interval, immediate] = scheduled[0]; + assert.match(task.command, /(?:token\.ts|token\.js) --renew$/); + assert.equal(task.runOrigin, 'system'); + assert.deepEqual(interval, { days: 28 }); + assert.equal(immediate, true); +}); + +function appsFixture(t) { + const previousConfigPath = config.configPath; + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-apps-concurrency-')); + config.configPath = directory; + t.after(() => { + config.configPath = previousConfigPath; + fs.rmSync(directory, { recursive: true, force: true }); + }); + t.mock.method(Date, 'now', () => START * 1000); + const clone = (value) => JSON.parse(JSON.stringify(value)); + let apps = [ + { id: 1, name: 'system', client_id: 'system-client', client_secret: 'system-secret', + scopes: ['crons', 'system', 'dashboard'], tokens: [{ value: 'system-token', expiration: START + DAY }] }, + { id: 2, name: 'external', client_id: 'external-client', client_secret: 'external-secret', + scopes: ['crons'], tokens: [] }, + ]; + let cached = clone(apps); + const matches = (app, where) => Object.entries(where).every(([key, value]) => + Array.isArray(value) ? value.includes(app[key]) : app[key] === value); + t.mock.method(AppModel, 'findOne', async ({ where }) => { + const app = apps.find((app) => matches(app, where)); + if (!app) return null; + const snapshot = clone(app); + await new Promise((resolve) => setTimeout(resolve, 10)); + return { ...snapshot, get: () => snapshot }; + }); + t.mock.method(AppModel, 'findAll', async () => clone(apps).map((app) => ({ get: () => app }))); + t.mock.method(AppModel, 'update', async (values, { where }) => { + apps = apps.map((app) => matches(app, where) ? { ...app, ...clone(values) } : app); + return [1]; + }); + t.mock.method(AppModel, 'destroy', async ({ where }) => { + apps = apps.filter((app) => !matches(app, where)); + }); + t.mock.method(AppModel, 'create', async (values) => { + const app = { id: Math.max(0, ...apps.map((app) => app.id)) + 1, tokens: [], ...values }; + apps.push(app); + return { get: () => clone(app) }; + }); + t.mock.method(shareStore, 'updateApps', async (values) => { cached = clone(values); }); + t.mock.method(shareStore, 'getApps', async () => clone(cached)); + return { service: new OpenService({}), apps: () => clone(apps), cached: () => clone(cached) }; +} + +for (const operation of ['acquisition', 'startup refresh']) { + test(`${operation} cannot overwrite another application's newly issued token`, async (t) => { + const state = appsFixture(t); + let resume; + let entered; + const paused = new Promise((resolve) => { entered = resolve; }); + const barrier = new Promise((resolve) => { resume = resolve; }); + const findAll = AppModel.findAll; + let first = true; + AppModel.findAll = async (...args) => { + const snapshot = await findAll(...args); + if (first) { + first = false; + entered(); + await barrier; + } + return snapshot; + }; + const publication = operation === 'acquisition' + ? state.service.generateSystemToken() : state.service.refreshApps(); + await paused; + const issuance = state.service.authToken({ client_id: 'external-client', client_secret: 'external-secret' }); + await new Promise((resolve) => setTimeout(resolve, 150)); + resume(); + const [, response] = await Promise.all([publication, issuance]); + assert.equal(response.code, 200); + assert.ok(state.apps()[1].tokens.some((token) => token.value === response.data.token)); + assert.deepEqual(state.cached(), state.apps()); + }); +} + +test('concurrent token API calls and scheduled generation preserve every issued token', async (t) => { + const state = appsFixture(t); + const credentials = { client_id: 'system-client', client_secret: 'system-secret' }; + const results = await Promise.all([ + state.service.authToken(credentials), state.service.generateSystemToken(true), + state.service.authToken(credentials), state.service.generateSystemToken(true), + ]); + const tokens = state.apps()[0].tokens; + assert.equal(tokens.length, 5); + for (const response of results) { + assert.ok(tokens.some((token) => token.value === (response.value || response.data.token))); + } + assert.deepEqual(state.cached(), state.apps()); +}); + +for (const mutation of ['resetSecret', 'remove', 'update', 'create']) { + test(`concurrent cache refresh and ${mutation} retain the latest application state`, async (t) => { + const state = appsFixture(t); + await state.service.authToken({ client_id: 'external-client', client_secret: 'external-secret' }); + const change = () => mutation === 'resetSecret' ? state.service.resetSecret(2) + : mutation === 'remove' ? state.service.remove([2]) + : mutation === 'update' ? state.service.update({ id: 2, name: 'external', scopes: ['dashboard'] }) + : state.service.create({ name: 'created', scopes: ['crons'] }); + await Promise.all([state.service.refreshApps(), change(), state.service.generateSystemToken()]); + assert.deepEqual(state.cached(), state.apps()); + if (mutation === 'resetSecret') { + assert.equal(state.cached()[1].tokens.length, 0); + assert.notEqual(state.cached()[1].client_secret, 'external-secret'); + assert.equal((await state.service.authToken({ client_id: 'external-client', client_secret: 'external-secret' })).code, 400); + } else if (mutation === 'remove') { + assert.equal(state.cached().length, 1); + } else if (mutation === 'update') { + assert.deepEqual(state.cached()[1].scopes, ['dashboard']); + } else { + assert.equal(state.cached().length, 3); + } + }); +} + +test('system initialization preserves active tokens while migrating dashboard scope', async (t) => { + const state = appsFixture(t); + await AppModel.update({ scopes: ['crons', 'system'] }, { where: { id: 1 } }); + await Promise.all([state.service.initializeSystemApp(), state.service.generateSystemToken(true)]); + assert.deepEqual(state.cached(), state.apps()); + assert.deepEqual(state.apps()[0].scopes, ['crons', 'system', 'dashboard']); + assert.equal(state.apps()[0].tokens.length, 2); +}); + +test('failed atomic token-file publication releases the application lock for recovery', async (t) => { + const state = appsFixture(t); + const temporary = path.join(config.configPath, `token.json.${process.pid}.tmp`); + fs.mkdirSync(temporary); + await assert.rejects(state.service.generateSystemToken(), /EISDIR/); + fs.rmSync(temporary, { recursive: true }); + const token = await state.service.generateSystemToken(); + assert.equal(token.value, 'system-token'); + assert.equal(fs.existsSync(path.join(config.configPath, '.apps-state.lock')), false); +}); + +test('real SQLite and authentication cache retain concurrent process updates', async (t) => { + fs.mkdirSync(config.configPath, { recursive: true }); + await AppModel.sync({ force: true }); + const service = new OpenService({}); + await service.initializeSystemApp(); + const external = await service.create({ name: 'sqlite-external', scopes: ['crons'] }); + const worker = path.join(environment, 'sqlite-worker.cjs'); + const root = path.resolve(__dirname, '../..'); + fs.writeFileSync(worker, ` + require(${JSON.stringify(require.resolve('reflect-metadata'))}); + const path = require('node:path'); + const root = process.cwd(); + const OpenService = require(path.join(root, 'back/services/open')).default; + const { shareStore, keyvStore } = require(path.join(root, 'back/shared/store')); + const { sequelize } = require(path.join(root, 'back/data')); + const service = new OpenService({}); + (async () => { + const action = process.argv[2]; + const result = action === 'auth' + ? await service.authToken({ client_id: process.argv[3], client_secret: process.argv[4] }) + : action === 'refresh' ? await service.refreshApps() + : await service.generateSystemToken(action === 'renew'); + console.log(JSON.stringify(result || {})); + })().catch(error => { console.error(error); process.exitCode = 1; }) + .finally(async () => { await keyvStore.disconnect(); await sequelize.close(); }); + `); + const actions = ['renew', 'acquire', 'auth', 'refresh', 'renew', 'auth', 'auth']; + const outcomes = await Promise.allSettled(actions.map((action) => execute(process.execPath, [ + '-r', require.resolve('ts-node/register/transpile-only'), worker, action, + external.client_id, external.client_secret, + ], { cwd: root, timeout: 20_000, env: process.env }))); + const results = outcomes.map((outcome) => { + assert.equal(outcome.status, 'fulfilled', outcome.reason?.stderr); + return outcome.value; + }); + const database = await service.findApps(); + const cached = await shareStore.getApps(); + assert.deepEqual(cached, JSON.parse(JSON.stringify(database))); + for (let index = 0; index < results.length; index++) { + assert.equal(results[index].stderr, ''); + const result = JSON.parse(results[index].stdout.trim()); + const value = result.value || result.data?.token; + if (value) assert.ok(cached.some((app) => app.tokens?.some((token) => token.value === value))); + } + assert.equal(database.find((app) => app.name === 'sqlite-external').tokens.length, 3); + const file = JSON.parse(fs.readFileSync(path.join(config.configPath, 'token.json'), 'utf8')); + assert.ok(cached.find((app) => app.name === 'system').tokens.some((token) => token.value === file.value)); +}); + + +test('atomic publication preserves restricted token-file permissions', async (t) => { + const state = fixture(t, [{ value: 'current', expiration: START + DAY }]); + const file = path.join(config.configPath, 'token.json'); + await state.service.generateSystemToken(); + assert.equal(fs.statSync(file).mode & 0o777, 0o600); + fs.chmodSync(file, 0o640); + await state.service.generateSystemToken(true); + assert.equal(fs.statSync(file).mode & 0o777, 0o640); +}); + + +test('a SIGKILLed generator leaves a lock that the next acquisition recovers', async (t) => { + const state = await processFixture(t); + const running = state.run(false, false, true); + const stopped = running.catch((error) => error); + const deadline = Date.now() + 10_000; + while (!fs.existsSync(path.join(state.directory, 'locked'))) { + assert.ok(Date.now() < deadline, 'generator did not acquire the lock'); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + running.child.kill('SIGKILL'); + assert.equal((await stopped).signal, 'SIGKILL'); + assert.equal(fs.existsSync(path.join(state.directory, '.apps-state.lock')), true); + const recovered = await state.run(); + assert.equal(recovered.stderr, ''); + assert.equal(recovered.stdout.trim(), state.cached().value); + assert.equal(fs.existsSync(path.join(state.directory, '.apps-state.lock')), false); +});