fix: renew system tokens and serialize authentication cache updates

back/loaders/initData.ts

@@ -13,12 +13,11 @@ import { AuthDataType, SystemModel } from '../data/system';
 import SystemService from '../services/system';
 import UserService from '../services/user';
 import { writeFile, readFile } from 'fs/promises';
-import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../config/util';
+import { fileExist, isDemoEnv, safeJSONParse } from '../config/util';
 import OpenService from '../services/open';
 import { shareStore } from '../shared/store';
 import Logger from './logger';
 import cronClient from '../schedule/client';
-import { AppModel } from '../data/open';
 import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
 import { setLang, systemLang } from '../shared/i18n';
 
@@ -31,24 +30,7 @@ export default async () => {
   const openService = Container.get(OpenService);
 
   // 初始化增加系统配置
-  let systemApp = (
-    await AppModel.findOne({
-      where: { name: 'system' },
-    })
-  )?.get({ plain: true });
-  if (!systemApp) {
-    systemApp = await AppModel.create({
-      name: 'system',
-      scopes: ['crons', 'system', 'dashboard'],
-      client_id: createRandomString(12, 12),
-      client_secret: createRandomString(24, 24),
-    });
-  } else if (!systemApp.scopes.includes('dashboard')) {
-    await AppModel.update(
-      { scopes: [...systemApp.scopes, 'dashboard'] },
-      { where: { name: 'system' } },
-    );
-  }
+  await openService.initializeSystemApp();
   const [systemConfig] = await SystemModel.findOrCreate({
     where: { type: AuthDataType.systemConfig },
   });
@@ -250,9 +232,6 @@ export default async () => {
   await envService.set_envs();
 
   const authInfo = await userService.getAuthInfo();
-  const apps = await openService.findApps();
   await shareStore.updateAuthInfo(authInfo);
-  if (apps?.length) {
-    await shareStore.updateApps(apps);
-  }
+  await openService.refreshApps();
 };

back/loaders/initTask.ts

@@ -26,6 +26,7 @@ export default async () => {
   if (await fileExist(tokenFile)) {
     tokenCommand = `node ${tokenFile}`;
   }
+  tokenCommand += ' --renew';
   const cron = {
     id: NaN,
     name: t('生成token'),

back/services/open.ts

@@ -6,27 +6,77 @@ import { v4 as uuidV4 } from 'uuid';
 import sequelize, { Op } from 'sequelize';
 import { shareStore } from '../shared/store';
 import { t } from '../shared/i18n';
+import { lock } from 'proper-lockfile';
+import path from 'path';
+import os from 'os';
+import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
+import config from '../config';
 
 @Service()
 export default class OpenService {
   constructor(@Inject('logger') private logger: winston.Logger) {}
 
+  // Every database mutation and full authentication-cache publication shares
+  // this cross-process lock. Lock a separate marker, not token.json itself.
+  private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
+    const target = path.join(config.configPath, '.apps-state');
+    const release = await lock(target, {
+      realpath: false,
+      stale: 10_000,
+      retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
+    });
+    try {
+      return await operation();
+    } finally {
+      await release();
+    }
+  }
+
+  public async initializeSystemApp(): Promise<void> {
+    await this.withAppsLock(async () => {
+      const doc = await AppModel.findOne({ where: { name: 'system' } });
+      const app = doc?.get({ plain: true });
+      if (!app) {
+        await this.insert({
+          name: 'system',
+          scopes: ['crons', 'system', 'dashboard'],
+          client_id: createRandomString(12, 12),
+          client_secret: createRandomString(24, 24),
+        });
+      } else if (!app.scopes.includes('dashboard')) {
+        await AppModel.update(
+          { scopes: [...app.scopes, 'dashboard'] },
+          { where: { id: app.id } },
+        );
+      }
+      await shareStore.updateApps(await this.find({}));
+    });
+  }
+
+  public async refreshApps(): Promise<void> {
+    await this.withAppsLock(async () => {
+      await shareStore.updateApps(await this.find({}));
+    });
+  }
+
   public async findApps(): Promise<App[] | null> {
     const docs = await this.find({});
     return docs;
   }
 
   public async create(payload: App): Promise<App> {
-    const tab = { ...payload };
-    tab.client_id = createRandomString(12, 12);
-    tab.client_secret = createRandomString(24, 24);
-    const doc = await this.insert(tab);
-    const apps = await this.find({});
-    await shareStore.updateApps(apps);
-    return { ...doc, tokens: [] };
+    return this.withAppsLock(async () => {
+      const tab = { ...payload };
+      tab.client_id = createRandomString(12, 12);
+      tab.client_secret = createRandomString(24, 24);
+      const doc = await this.insert(tab);
+      const apps = await this.find({});
+      await shareStore.updateApps(apps);
+      return { ...doc, tokens: [] };
+    });
   }
 
-  public async insert(payload: App): Promise<App> {
+  private async insert(payload: App): Promise<App> {
     const doc = await AppModel.create(payload, { returning: true });
     return doc.get({ plain: true });
   }
@@ -41,10 +91,12 @@ export default class OpenService {
   }
 
   private async updateDb(payload: Partial<App>): Promise<App> {
-    await AppModel.update(payload, { where: { id: payload.id } });
-    const apps = await this.find({});
-    await shareStore.updateApps(apps);
-    return apps?.find((x) => x.id === payload.id) as App;
+    return this.withAppsLock(async () => {
+      await AppModel.update(payload, { where: { id: payload.id } });
+      const apps = await this.find({});
+      await shareStore.updateApps(apps);
+      return apps?.find((x) => x.id === payload.id) as App;
+    });
   }
 
   public async getDb(query: Record<string, any>): Promise<App> {
@@ -56,9 +108,11 @@ export default class OpenService {
   }
 
   public async remove(ids: number[]) {
-    await AppModel.destroy({ where: { id: ids } });
-    const apps = await this.find({});
-    await shareStore.updateApps(apps);
+    return this.withAppsLock(async () => {
+      await AppModel.destroy({ where: { id: ids } });
+      const apps = await this.find({});
+      await shareStore.updateApps(apps);
+    });
   }
 
   public async resetSecret(id: number): Promise<App> {
@@ -112,7 +166,15 @@ export default class OpenService {
     return docs.map((x) => x.get({ plain: true }));
   }
 
-  public async authToken({
+  public async authToken(credentials: {
+    client_id: string;
+    client_secret: string;
+  }): Promise<any> {
+    return this.withAppsLock(() => this.issueToken(credentials));
+  }
+
+  // Caller holds withAppsLock through database update and cache publication.
+  private async issueToken({
     client_id,
     client_secret,
   }: {
@@ -156,27 +218,47 @@ export default class OpenService {
     }
   }
 
-  public async generateSystemToken(): Promise<{
+  public async generateSystemToken(renew = false): Promise<{
     value: string;
     expiration: number;
   }> {
-    const apps = await shareStore.getApps();
-    const systemApp = apps?.find((x) => x.name === 'system');
-    if (!systemApp) {
-      throw new Error('system app not found');
-    }
-    const now = Math.round(Date.now() / 1000);
-    const currentToken = systemApp.tokens?.find((x) => x.expiration > now);
-    if (currentToken) {
-      return currentToken;
-    }
-    const { data } = await this.authToken({
-      client_id: systemApp.client_id,
-      client_secret: systemApp.client_secret,
+    return this.withAppsLock(async () => {
+      // Read after acquiring the lock; a concurrent issuer may have renewed.
+      const systemApp = await this.getDb({ name: 'system' });
+      const now = Math.round(Date.now() / 1000);
+      let appToken = systemApp.tokens
+        ?.filter((token) => token.expiration > now)
+        .sort((left, right) => right.expiration - left.expiration)[0];
+      if (renew || !appToken) {
+        const { data } = await this.issueToken({
+          client_id: systemApp.client_id,
+          client_secret: systemApp.client_secret,
+        });
+        appToken = { value: data.token, expiration: data.expiration };
+      } else {
+        // Recover a prior database commit whose cache publication failed.
+        await shareStore.updateApps(await this.find({}));
+      }
+      const tokenFile = path.join(config.configPath, 'token.json');
+      const temporaryFile = `${tokenFile}.${process.pid}.tmp`;
+      const mode = await stat(tokenFile).then(
+        (file) => file.mode & 0o777,
+        (error: NodeJS.ErrnoException) => {
+          if (error.code === 'ENOENT') return 0o600;
+          throw error;
+        },
+      );
+      try {
+        await writeFile(temporaryFile, `${JSON.stringify(appToken)}${os.EOL}`, {
+          mode,
+        });
+        await chmod(temporaryFile, mode);
+        // Shell readers see a complete old or new document, never a partial one.
+        await rename(temporaryFile, tokenFile);
+      } finally {
+        await rm(temporaryFile, { force: true });
+      }
+      return appToken;
     });
-    return {
-      ...data,
-      value: data.token,
-    };
   }
 }

back/token.ts

@@ -2,30 +2,19 @@ import 'reflect-metadata';
 import OpenService from './services/open';
 import { Container } from 'typedi';
 import LoggerInstance from './loaders/logger';
-import config from './config';
-import path from 'path';
-import os from 'os';
-import { writeFileWithLock } from './shared/utils';
-
-const tokenFile = path.join(config.configPath, 'token.json');
 
 async function getToken() {
   try {
     Container.set('logger', LoggerInstance);
     const openService = Container.get(OpenService);
-    const appToken = await openService.generateSystemToken();
+    const appToken = await openService.generateSystemToken(
+      process.argv.includes('--renew'),
+    );
     console.log(appToken.value);
-    await writeFile({
-      value: appToken.value,
-      expiration: appToken.expiration,
-    });
   } catch (error) {
-    console.log(error);
+    console.error(error);
+    process.exitCode = 1;
   }
 }
 
-async function writeFile(data: any) {
-  await writeFileWithLock(tokenFile, `${JSON.stringify(data)}${os.EOL}`);
-}
-
 getToken();

shell/api.sh

@@ -26,6 +26,28 @@ get_token() {
   create_token
 }
 
+# Check the cached token before every authenticated API request. A rejected
+# token is refreshed and retried once; other failures are never retried.
+# Arguments are curl options, including the URL, method and optional body.
+ql_api_request() {
+  local api=""
+  local attempt
+  get_token || return $?
+  for attempt in 1 2; do
+    api=$(curl -s --noproxy "*" \
+      -H "Authorization: Bearer ${__ql_token__}" \
+      -H "Content-Type: application/json;charset=UTF-8" \
+      "$@" \
+      --compressed)
+    if [[ "$attempt" == 2 || "$api" == '{"code":200}' ]] ||
+      [[ "$(jq -r '.code' <<< "$api" 2>/dev/null)" != 401 ]]; then
+      break
+    fi
+    create_token || return $?
+  done
+  printf '%s' "$api"
+}
+
 # Read the status response once; preserve multiline error messages and the
 # legacy code/message variables used by the shell API callers.
 ql_parse_status_response() {
@@ -63,13 +85,10 @@ add_cron_api() {
     sub_id="null"
   fi
 
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"sub_id\":$sub_id}" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
+    --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"sub_id\":$sub_id}" \
+    -X POST)
   code=$(echo "$api" | jq -r .code)
   message=$(echo "$api" | jq -r .message)
   if [[ $code == 200 ]]; then
@@ -93,14 +112,10 @@ update_cron_api() {
     local id="$4"
   fi
 
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-      -X 'PUT' \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"id\":\"$id\"}" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
+    -X PUT \
+    --data-raw "{\"name\":\"${name//\"/\\\"}\",\"command\":\"${command//\"/\\\"}\",\"schedule\":\"$schedule\",\"id\":\"$id\"}")
   code=$(echo "$api" | jq -r .code)
   message=$(echo "$api" | jq -r .message)
   if [[ $code == 200 ]]; then
@@ -120,14 +135,10 @@ update_cron_command_api() {
     local id="$2"
   fi
 
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-      -X 'PUT' \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "{\"command\":\"${command//\"/\\\"}\",\"id\":\"$id\"}" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
+    -X PUT \
+    --data-raw "{\"command\":\"${command//\"/\\\"}\",\"id\":\"$id\"}")
   code=$(echo "$api" | jq -r .code)
   message=$(echo "$api" | jq -r .message)
   if [[ $code == 200 ]]; then
@@ -140,14 +151,10 @@ update_cron_command_api() {
 del_cron_api() {
   local ids="$1"
   local currentTimeStamp=$(date +%s)
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
-      -X 'DELETE' \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "[$ids]" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/crons?t=$currentTimeStamp" \
+    -X DELETE \
+    --data-raw "[$ids]")
   code=$(echo "$api" | jq -r .code)
   message=$(echo "$api" | jq -r .message)
   if [[ $code == 200 ]]; then
@@ -171,14 +178,10 @@ update_cron() {
     dataRaw="${dataRaw},\"exit_code\":$exitCode"
   fi
   dataRaw="${dataRaw}}"
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \
-      -X 'PUT' \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "$dataRaw" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/crons/status?t=$currentTimeStamp" \
+    -X PUT \
+    --data-raw "$dataRaw")
   ql_parse_status_response "$api" || true
   if [[ $code != 200 ]]; then
     if [[ ! $message ]]; then
@@ -192,14 +195,10 @@ notify_api() {
   local title="$1"
   local content="$2"
   local currentTimeStamp=$(date +%s)
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/system/notify?t=$currentTimeStamp" \
-      -X 'PUT' \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "{\"title\":\"${title//\"/\\\"}\",\"content\":\"${content//\"/\\\"}\"}" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/system/notify?t=$currentTimeStamp" \
+    -X PUT \
+    --data-raw "{\"title\":\"${title//\"/\\\"}\",\"content\":\"${content//\"/\\\"}\"}")
   code=$(echo "$api" | jq -r .code)
   message=$(echo "$api" | jq -r .message)
   if [[ $code == 200 ]]; then
@@ -212,12 +211,9 @@ notify_api() {
 find_cron_api() {
   local params="$1"
   local currentTimeStamp=$(date +%s)
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/crons/detail?$params&t=$currentTimeStamp" \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/crons/detail?$params&t=$currentTimeStamp" \
+    -X GET)
   data=$(echo "$api" | jq -r .data)
   if [[ $data == 'null' ]]; then
     echo -e ""
@@ -231,14 +227,10 @@ update_auth_config() {
   local body="$1"
   local tip="$2"
   local currentTimeStamp=$(date +%s)
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port}/open/system/auth/reset?t=$currentTimeStamp" \
-      -X 'PUT' \
-      -H "Authorization: Bearer ${__ql_token__}" \
-      -H "Content-Type: application/json;charset=UTF-8" \
-      --data-raw "{$body}" \
-      --compressed
-  )
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port}/open/system/auth/reset?t=$currentTimeStamp" \
+    -X PUT \
+    --data-raw "{$body}")
   code=$(echo "$api" | jq -r .code)
   message=$(echo "$api" | jq -r .message)
   if [[ $code == 200 ]]; then
@@ -254,14 +246,10 @@ record_cron_stat() {
   local elapsed="${3:-0}"
   [[ $ref_id ]] && [[ $ref_id -gt 0 ]] 2>/dev/null || return
 
-  local api=$(
-    curl -s --noproxy "*" "http://localhost:${ql_port:-5700}/open/dashboard/record" \
+  local api=$(ql_api_request \
+    "http://localhost:${ql_port:-5700}/open/dashboard/record" \
     -X POST \
-    -H "Authorization: Bearer ${__ql_token__}" \
-    -H "Content-Type: application/json;charset=UTF-8" \
-    --data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}" \
-    --compressed
-  )
+    --data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}")
   ql_parse_status_response "$api" || true
   if [[ $code != 200 ]]; then
     if [[ ! $message ]]; then
@@ -270,5 +258,3 @@ record_cron_stat() {
     echo -e "${message}"
   fi
 }
-
-get_token

test/back/restart-instance-recovery.test.cjs

@@ -27,7 +27,8 @@ test('startup closes interrupted instances without inventing exit codes or rewri
   const service = {
     set_envs: async () => {},
     getAuthInfo: async () => ({}),
-    findApps: async () => [],
+    initializeSystemApp: async () => {},
+    refreshApps: async () => {},
   };
   const initData = load('back/loaders/initData.ts', {
     typedi: { Container: { get: () => service } },

test/back/shell-api-parsing.test.cjs

@@ -91,13 +91,13 @@ test('expired, corrupt, missing or header-unsafe token cache refreshes under err
     assert.equal(r.stdout, 'generated:1');
   }
 });
-test('success with a message still parses once and stays silent', (t) => {
+test('success with a message checks authentication and stays silent', (t) => {
   const f = fixture(t);
   fs.writeFileSync(f.response, JSON.stringify({ code: 200, message: 'ok' }));
   const r = f.run('update_cron 1 0 123 log 1');
   assert.equal(r.status, 0, r.stderr);
   assert.equal(r.stdout, '');
-  assert.equal(f.count(), 1);
+  assert.equal(f.count(), 2);
 });
 test('status and statistics errors preserve multiline Unicode messages', (t) => {
   const f = fixture(t);
@@ -113,7 +113,7 @@ test('status and statistics errors preserve multiline Unicode messages', (t) =>
     assert.equal(r.status, 0, r.stderr);
     assert.equal(r.stdout, '写入失败\n请重试 "原任务"\n');
   }
-  assert.equal(f.count(), 2);
+  assert.equal(f.count(), 4);
 });
 test('non-JSON errors fall back to raw text and absent messages retain legacy null', (t) => {
   const f = fixture(t);
@@ -180,5 +180,5 @@ test('success-shaped invalid documents retain the raw-response error fallback',
     assert.equal(r.status, 0, r.stderr);
     assert.equal(r.stdout, body + '\n');
   }
-  assert.equal(f.count(), cases.length);
+  assert.equal(f.count(), cases.length * 2);
 });

test/back/shellApiToken.test.cjs

@@ -0,0 +1,151 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const { test } = require('node:test');
+
+const apiFile = path.resolve(__dirname, '../../shell/api.sh');
+
+const endpoints = {
+  status: {
+    method: 'PUT', url: /\/open\/crons\/status\?t=\d+$/,
+    body: { ids: ['37'], status: '1', pid: '4321', log_path: 'task/run.log',
+      last_execution_time: 1787004000, last_running_time: 90, exit_code: 0 },
+    success: '', failure: (message) => message,
+  },
+  statistics: {
+    method: 'POST', url: /\/open\/dashboard\/record$/,
+    body: { ref_id: 37, code: 0, elapsed: 90 },
+    success: '', failure: (message) => message,
+  },
+  add: {
+    method: 'POST', url: /\/open\/crons\?t=\d+$/,
+    body: { name: 'sample job', command: 'task job.py', schedule: '0 * * * *', sub_id: null },
+    success: 'sample job -> 添加成功', failure: (message) => `sample job -> 添加失败(${message})`,
+  },
+  update: {
+    method: 'PUT', url: /\/open\/crons\?t=\d+$/,
+    body: { name: 'sample job', command: 'task job.py', schedule: '0 * * * *', id: '37' },
+    success: 'sample job -> 更新成功', failure: (message) => `sample job -> 更新失败(${message})`,
+  },
+  command: {
+    method: 'PUT', url: /\/open\/crons\?t=\d+$/,
+    body: { command: 'task job.py', id: '37' },
+    success: 'task job.py -> 更新成功', failure: (message) => `task job.py -> 更新失败(${message})`,
+  },
+  delete: {
+    method: 'DELETE', url: /\/open\/crons\?t=\d+$/,
+    body: [37, 38], success: '成功', failure: (message) => `失败(${message})`,
+  },
+  notify: {
+    method: 'PUT', url: /\/open\/system\/notify\?t=\d+$/,
+    body: { title: 'title', content: 'body' },
+    success: '通知发送成功🎉', failure: (message) => `通知失败(${message})`,
+  },
+  find: {
+    method: 'GET', url: /\/open\/crons\/detail\?log_path=task\/run.log&t=\d+$/,
+    body: null, success: 'sample job', failure: () => '',
+  },
+  auth: {
+    method: 'PUT', url: /\/open\/system\/auth\/reset\?t=\d+$/,
+    body: { retries: 0 }, success: '重置登录错误次数成功🎉',
+    failure: (message) => `重置登录错误次数失败(${message})`,
+  },
+};
+
+function runCallback(endpoint, scenario) {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-api-token-'));
+  try {
+    const tokenFile = path.join(directory, 'token.json');
+    fs.writeFileSync(tokenFile, JSON.stringify({
+      value: 'initial-token', expiration: Math.floor(Date.now() / 1000) + 3600,
+    }));
+    const child = spawnSync('bash', ['-c', `
+      . "$API_FILE"
+      create_token() {
+        printf 'refresh\\n' >> "$CAPTURE"
+        __ql_token__=fresh-token
+        printf '{"value":"fresh-token","expiration":4102444800}' > "$file_auth_token"
+      }
+      case "$SCENARIO" in
+        expired) printf '{"value":"initial-token","expiration":1}' > "$file_auth_token" ;;
+        replaced) printf '{"value":"fresh-token","expiration":4102444800}' > "$file_auth_token" ;;
+      esac
+      curl() {
+        local auth body url method arg
+        while [[ $# -gt 0 ]]; do
+          arg="$1"; shift
+          case "$arg" in
+            -H) [[ "$1" == Authorization:* ]] && auth="$1"; shift ;;
+            --data-raw) body="$1"; shift ;;
+            -X) method="$1"; shift ;;
+            http:*) url="$arg" ;;
+          esac
+        done
+        printf '%s\\n%s\\n%s\\n%s\\n' "$auth" "$method" "$url" "\${body:-null}" >> "$CAPTURE"
+        if [[ "$SCENARIO" == forbidden ]]; then
+          printf '{"code":403,"message":"forbidden"}'
+        elif [[ "$SCENARIO" == unavailable ]]; then
+          printf '{"code":500,"message":"unavailable"}'
+        elif [[ "$SCENARIO" == persistent || ( "$SCENARIO" != valid && "$auth" != 'Authorization: Bearer fresh-token' ) ]]; then
+          printf '{"code":401,"message":"Token 已失效"}'
+        else
+          printf '{"code":200,"data":{"name":"sample job"}}'
+        fi
+      }
+      t() { printf "$@"; }
+      case "$ENDPOINT" in
+        status) update_cron '"37"' 1 4321 'task/run.log' 1787004000 90 0 execution-37 ;;
+        statistics) record_cron_stat 37 0 90 ;;
+        add) add_cron_api '0 * * * *' 'task job.py' 'sample job' ;;
+        update) update_cron_api '0 * * * *' 'task job.py' 'sample job' 37 ;;
+        command) update_cron_command_api 'task job.py' 37 ;;
+        delete) del_cron_api '37,38' ;;
+        notify) notify_api title body ;;
+        find) find_cron_api 'log_path=task/run.log' ;;
+        auth) update_auth_config '"retries":0' '重置登录错误次数' ;;
+      esac
+    `], {
+      env: { ...process.env, PATH: `/usr/bin:/bin:${process.env.PATH}`, API_FILE: apiFile, file_auth_token: tokenFile,
+        CAPTURE: path.join(directory, 'requests'), SCENARIO: scenario,
+        ENDPOINT: endpoint, ql_port: '5700' },
+      encoding: 'utf8', timeout: 10_000,
+    });
+    assert.equal(child.status, 0, child.stderr);
+    assert.equal(child.stderr, '');
+    const lines = fs.readFileSync(path.join(directory, 'requests'), 'utf8').trim().split('\n');
+    const refreshes = lines.filter((line) => line === 'refresh').length;
+    const requests = lines.filter((line) => line !== 'refresh');
+    const captures = [];
+    for (let i = 0; i < requests.length; i += 4) {
+      captures.push({ auth: requests[i], method: requests[i + 1],
+        url: requests[i + 2], body: JSON.parse(requests[i + 3]) });
+    }
+    return { output: child.stdout, refreshes, captures };
+  } finally {
+    fs.rmSync(directory, { recursive: true, force: true });
+  }
+}
+
+for (const [endpoint, expected] of Object.entries(endpoints)) {
+  for (const scenario of ['valid', 'expired', 'replaced', 'rejected', 'persistent', 'forbidden', 'unavailable']) {
+    test(`${endpoint} API handles ${scenario} token without unbounded retries`, () => {
+      const { output, refreshes, captures } = runCallback(endpoint, scenario);
+      const retries = ['rejected', 'persistent'].includes(scenario);
+      assert.equal(captures.length, retries ? 2 : 1);
+      assert.equal(refreshes, ['expired', 'rejected', 'persistent'].includes(scenario) ? 1 : 0);
+      for (const capture of captures) {
+        assert.deepEqual(capture.body, expected.body);
+        assert.equal(capture.method, expected.method);
+        assert.match(capture.url, expected.url);
+      }
+      if (['expired', 'replaced', 'rejected'].includes(scenario)) {
+        assert.equal(captures.at(-1).auth, 'Authorization: Bearer fresh-token');
+      }
+      const error = scenario === 'persistent' ? 'Token 已失效'
+        : scenario === 'forbidden' ? 'forbidden' : scenario === 'unavailable' ? 'unavailable' : '';
+      assert.equal(output.trim(), error ? expected.failure(error) : expected.success);
+    });
+  }
+}

test/back/systemTokenRenewal.test.cjs

@@ -0,0 +1,497 @@
+require('reflect-metadata');
+require('ts-node/register/transpile-only');
+
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { execFile } = require('node:child_process');
+const { promisify } = require('node:util');
+const environment = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-token-env-'));
+fs.writeFileSync(path.join(environment, '.env'), '');
+fs.mkdirSync(path.join(environment, 'data/db'), { recursive: true });
+process.env.QL_DIR = environment;
+process.env.QL_DATA_DIR = path.join(environment, 'data');
+require('node:test').after(async () => {
+  await require('../../back/shared/store').keyvStore.disconnect();
+  await require('../../back/data').sequelize.close();
+  fs.rmSync(environment, { recursive: true, force: true });
+});
+const OpenService = require('../../back/services/open').default;
+const { AppModel } = require('../../back/data/open');
+const { shareStore } = require('../../back/shared/store');
+const config = require('../../back/config').default;
+
+const DAY = 86400;
+const START = 1787004000;
+const execute = promisify(execFile);
+
+function fixture(t, initialTokens = []) {
+  const previousConfigPath = config.configPath;
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-token-renewal-'));
+  config.configPath = directory;
+  t.after(() => {
+    config.configPath = previousConfigPath;
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+  let now = START;
+  let app = {
+    name: 'system', client_id: 'system-client', client_secret: 'system-secret',
+    scopes: ['crons', 'system', 'dashboard'], tokens: initialTokens,
+  };
+  t.mock.method(Date, 'now', () => now * 1000);
+  t.mock.method(shareStore, 'getApps', async () => [app]);
+  t.mock.method(shareStore, 'updateApps', async (apps) => { [app] = apps; });
+  t.mock.method(AppModel, 'findOne', async ({ where }) => {
+    if (where.name) assert.equal(where.name, app.name);
+    else {
+      assert.equal(where.client_id, app.client_id);
+      assert.equal(where.client_secret, app.client_secret);
+    }
+    return { tokens: app.tokens, get: () => app };
+  });
+  t.mock.method(AppModel, 'update', async ({ tokens }) => {
+    app = { ...app, tokens };
+    return [1];
+  });
+  t.mock.method(AppModel, 'findAll', async () => [{ get: () => app }]);
+  return {
+    service: new OpenService({}),
+    atDay(day) { now = START + day * DAY; },
+    tokens() { return app.tokens; },
+  };
+}
+
+test('28-day token generation renews ahead of the 30-day expiry', async (t) => {
+  const state = fixture(t);
+  const initial = await state.service.generateSystemToken();
+  assert.equal(initial.expiration, START + 30 * DAY);
+  state.atDay(28);
+  const renewed = await state.service.generateSystemToken(true);
+  assert.equal(renewed.expiration, START + 58 * DAY);
+  assert.notEqual(renewed.value, initial.value);
+  assert.ok(state.tokens().some((token) => token.value === initial.value));
+  state.atDay(56);
+  const next = await state.service.generateSystemToken(true);
+  assert.equal(next.expiration, START + 86 * DAY);
+  assert.ok(state.tokens().some((token) => token.value === renewed.value));
+  assert.ok(!state.tokens().some((token) => token.value === initial.value));
+});
+
+test('startup generation renews before a restarted 28-day interval', async (t) => {
+  const state = fixture(t);
+  await state.service.generateSystemToken();
+  state.atDay(20);
+  const restarted = await state.service.generateSystemToken(true);
+  assert.equal(restarted.expiration, START + 50 * DAY);
+  state.atDay(48);
+  const renewed = await state.service.generateSystemToken(true);
+  assert.equal(renewed.expiration, START + 78 * DAY);
+  assert.ok(state.tokens().some((token) => token.value === restarted.value));
+});
+
+test('generation at the five-token limit extends the reused token without revoking active tokens', async (t) => {
+  const initial = Array.from({ length: 5 }, (_, index) => ({
+    value: `token-${index}`, expiration: START + 2 * DAY,
+  }));
+  const state = fixture(t, initial);
+  const renewed = await state.service.generateSystemToken(true);
+  assert.equal(renewed.value, 'token-4');
+  assert.equal(renewed.expiration, START + 30 * DAY);
+  assert.equal(state.tokens().length, 5);
+  for (const token of initial) {
+    assert.ok(state.tokens().some((current) => current.value === token.value));
+  }
+});
+
+test('ordinary acquisition reuses the newest valid database token and republishes its cache', async (t) => {
+  const initial = [
+    { value: 'older-token', expiration: START + DAY },
+    { value: 'newer-token', expiration: START + 20 * DAY },
+  ];
+  const state = fixture(t, initial);
+  // A stale process cache must not override the token chosen from the database.
+  shareStore.getApps = async () => [{ name: 'system', tokens: [] }];
+  const token = await state.service.generateSystemToken();
+  assert.equal(token.value, 'newer-token');
+  assert.deepEqual(state.tokens(), initial);
+  assert.deepEqual(
+    JSON.parse(fs.readFileSync(path.join(config.configPath, 'token.json'), 'utf8')),
+    token,
+  );
+});
+
+test('a failed generation releases its lock and does not publish a token', async (t) => {
+  const state = fixture(t);
+  const update = AppModel.update;
+  AppModel.update = async () => { throw new Error('generation failed'); };
+  await assert.rejects(state.service.generateSystemToken(), /generation failed/);
+  assert.equal(fs.existsSync(path.join(config.configPath, 'token.json')), false);
+  assert.equal(fs.existsSync(path.join(config.configPath, 'token.json.refresh.lock')), false);
+  AppModel.update = update;
+  const token = await state.service.generateSystemToken();
+  assert.ok(state.tokens().some((current) => current.value === token.value));
+});
+
+test('acquisition recovers an authentication cache failure after the database commit', async (t) => {
+  const state = fixture(t);
+  const updateCache = shareStore.updateApps;
+  let attempts = 0;
+  shareStore.updateApps = async (apps) => {
+    if (++attempts === 1) throw new Error('authentication cache failed');
+    return updateCache(apps);
+  };
+  await assert.rejects(state.service.generateSystemToken(), /authentication cache failed/);
+  assert.equal(fs.existsSync(path.join(config.configPath, 'token.json')), false);
+  const committed = state.tokens()[0];
+  const token = await state.service.generateSystemToken();
+  assert.equal(token.value, committed.value);
+  assert.equal(state.tokens().length, 1);
+  assert.equal(attempts, 2);
+});
+
+async function processFixture(t, initialTokens = []) {
+  const root = path.resolve(__dirname, '../..');
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-token-process-'));
+  t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+  const databaseFile = path.join(directory, 'database.json');
+  fs.writeFileSync(databaseFile, JSON.stringify({
+    name: 'system', client_id: 'system-client', client_secret: 'system-secret',
+    scopes: ['crons', 'system', 'dashboard'], tokens: initialTokens,
+  }));
+  const worker = path.join(directory, 'worker.cjs');
+  fs.writeFileSync(worker, `
+    const fs = require('node:fs/promises');
+    const path = require('node:path');
+    const root = process.cwd();
+    const directory = __dirname;
+    const config = require(path.join(root, 'back/config')).default;
+    const { AppModel } = require(path.join(root, 'back/data/open'));
+    const { shareStore } = require(path.join(root, 'back/shared/store'));
+    config.configPath = directory;
+    const databaseFile = path.join(directory, 'database.json');
+    const read = async () => JSON.parse(await fs.readFile(databaseFile, 'utf8'));
+    AppModel.findOne = async () => {
+      if (process.env.QL_TOKEN_TEST_HOLD === '1') {
+        await fs.writeFile(path.join(directory, 'locked'), 'ready');
+        await new Promise(() => setInterval(() => {}, 1000));
+      }
+      const app = await read();
+      // Widen the read/write overlap so the original race is reproducible.
+      await new Promise(resolve => setTimeout(resolve, 75));
+      return { ...app, get: () => app };
+    };
+    AppModel.update = async ({ tokens }) => {
+      if (process.env.QL_TOKEN_TEST_FAIL === '1') throw new Error('generation failed');
+      const app = await read();
+      await fs.writeFile(databaseFile, JSON.stringify({ ...app, tokens }));
+      return [1];
+    };
+    AppModel.findAll = async () => {
+      const app = await read();
+      return [{ get: () => app }];
+    };
+    shareStore.getApps = async () => { throw new Error('stale cache must not be read'); };
+    shareStore.updateApps = async () => {};
+    require(path.join(root, 'back/token'));
+  `);
+  return {
+    directory,
+    run(renew = false, fail = false, hold = false) {
+      return execute(process.execPath, [
+        '-r', require.resolve('ts-node/register/transpile-only'), worker,
+        ...(renew ? ['--renew'] : []),
+      ], {
+        cwd: root, timeout: 20_000,
+        env: { ...process.env, QL_TOKEN_TEST_FAIL: fail ? '1' : '0', QL_TOKEN_TEST_HOLD: hold ? '1' : '0' },
+      });
+    },
+    tokens() { return JSON.parse(fs.readFileSync(databaseFile, 'utf8')).tokens; },
+    cached() { return JSON.parse(fs.readFileSync(path.join(directory, 'token.json'), 'utf8')); },
+  };
+}
+
+test('four independent token processes converge on one usable token when the cache is missing', async (t) => {
+  const state = await processFixture(t);
+  const results = await Promise.all(Array.from({ length: 4 }, () => state.run()));
+  const issued = results.map(({ stdout, stderr }) => {
+    assert.equal(stderr, '');
+    return stdout.trim();
+  });
+  assert.equal(new Set(issued).size, 1);
+  assert.equal(state.tokens().length, 1);
+  assert.equal(state.tokens()[0].value, issued[0]);
+  assert.equal(state.cached().value, issued[0]);
+  assert.equal(fs.existsSync(path.join(state.directory, 'token.json.refresh.lock')), false);
+});
+
+test('concurrent scheduled renewal and ordinary acquisition preserve all returned tokens', async (t) => {
+  const original = { value: 'original-token', expiration: Math.round(Date.now() / 1000) + DAY };
+  const state = await processFixture(t, [original]);
+  const results = await Promise.all([state.run(true), state.run(), state.run(true), state.run()]);
+  const tokens = state.tokens();
+  assert.equal(tokens.length, 3);
+  assert.ok(tokens.some((token) => token.value === original.value));
+  for (const { stdout, stderr } of results) {
+    assert.equal(stderr, '');
+    assert.ok(tokens.some((token) => token.value === stdout.trim()));
+  }
+  assert.ok(tokens.some((token) => token.value === state.cached().value));
+});
+
+test('token CLI reports generation failure on stderr with a nonzero exit code', async (t) => {
+  const state = await processFixture(t);
+  await assert.rejects(state.run(false, true), (error) => {
+    assert.equal(error.code, 1);
+    assert.equal(error.stdout, '');
+    assert.match(error.stderr, /generation failed/);
+    return true;
+  });
+  assert.equal(fs.existsSync(path.join(state.directory, 'token.json')), false);
+  const recovered = await state.run();
+  assert.equal(recovered.stdout.trim(), state.cached().value);
+});
+
+test('startup configures the 28-day token task with explicit renewal', async (t) => {
+  const { Container } = require('typedi');
+  const initialize = require('../../back/loaders/initTask').default;
+  const SystemService = require('../../back/services/system').default;
+  const ScheduleService = require('../../back/services/schedule').default;
+  const SubscriptionService = require('../../back/services/subscription').default;
+  const SshKeyService = require('../../back/services/sshKey').default;
+  const scheduled = [];
+  const services = new Map([
+    [SystemService, { getSystemConfig: async () => undefined }],
+    [ScheduleService, {
+      cancelIntervalTask: async () => {},
+      createIntervalTask: (...args) => scheduled.push(args),
+    }],
+    [SubscriptionService, { setSshConfig: async () => {}, list: async () => [] }],
+    [SshKeyService, {}],
+  ]);
+  t.mock.method(Container, 'get', (service) => {
+    assert.ok(services.has(service));
+    return services.get(service);
+  });
+  await initialize();
+  assert.equal(scheduled.length, 1);
+  const [task, interval, immediate] = scheduled[0];
+  assert.match(task.command, /(?:token\.ts|token\.js) --renew$/);
+  assert.equal(task.runOrigin, 'system');
+  assert.deepEqual(interval, { days: 28 });
+  assert.equal(immediate, true);
+});
+
+function appsFixture(t) {
+  const previousConfigPath = config.configPath;
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-apps-concurrency-'));
+  config.configPath = directory;
+  t.after(() => {
+    config.configPath = previousConfigPath;
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+  t.mock.method(Date, 'now', () => START * 1000);
+  const clone = (value) => JSON.parse(JSON.stringify(value));
+  let apps = [
+    { id: 1, name: 'system', client_id: 'system-client', client_secret: 'system-secret',
+      scopes: ['crons', 'system', 'dashboard'], tokens: [{ value: 'system-token', expiration: START + DAY }] },
+    { id: 2, name: 'external', client_id: 'external-client', client_secret: 'external-secret',
+      scopes: ['crons'], tokens: [] },
+  ];
+  let cached = clone(apps);
+  const matches = (app, where) => Object.entries(where).every(([key, value]) =>
+    Array.isArray(value) ? value.includes(app[key]) : app[key] === value);
+  t.mock.method(AppModel, 'findOne', async ({ where }) => {
+    const app = apps.find((app) => matches(app, where));
+    if (!app) return null;
+    const snapshot = clone(app);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    return { ...snapshot, get: () => snapshot };
+  });
+  t.mock.method(AppModel, 'findAll', async () => clone(apps).map((app) => ({ get: () => app })));
+  t.mock.method(AppModel, 'update', async (values, { where }) => {
+    apps = apps.map((app) => matches(app, where) ? { ...app, ...clone(values) } : app);
+    return [1];
+  });
+  t.mock.method(AppModel, 'destroy', async ({ where }) => {
+    apps = apps.filter((app) => !matches(app, where));
+  });
+  t.mock.method(AppModel, 'create', async (values) => {
+    const app = { id: Math.max(0, ...apps.map((app) => app.id)) + 1, tokens: [], ...values };
+    apps.push(app);
+    return { get: () => clone(app) };
+  });
+  t.mock.method(shareStore, 'updateApps', async (values) => { cached = clone(values); });
+  t.mock.method(shareStore, 'getApps', async () => clone(cached));
+  return { service: new OpenService({}), apps: () => clone(apps), cached: () => clone(cached) };
+}
+
+for (const operation of ['acquisition', 'startup refresh']) {
+  test(`${operation} cannot overwrite another application's newly issued token`, async (t) => {
+    const state = appsFixture(t);
+    let resume;
+    let entered;
+    const paused = new Promise((resolve) => { entered = resolve; });
+    const barrier = new Promise((resolve) => { resume = resolve; });
+    const findAll = AppModel.findAll;
+    let first = true;
+    AppModel.findAll = async (...args) => {
+      const snapshot = await findAll(...args);
+      if (first) {
+        first = false;
+        entered();
+        await barrier;
+      }
+      return snapshot;
+    };
+    const publication = operation === 'acquisition'
+      ? state.service.generateSystemToken() : state.service.refreshApps();
+    await paused;
+    const issuance = state.service.authToken({ client_id: 'external-client', client_secret: 'external-secret' });
+    await new Promise((resolve) => setTimeout(resolve, 150));
+    resume();
+    const [, response] = await Promise.all([publication, issuance]);
+    assert.equal(response.code, 200);
+    assert.ok(state.apps()[1].tokens.some((token) => token.value === response.data.token));
+    assert.deepEqual(state.cached(), state.apps());
+  });
+}
+
+test('concurrent token API calls and scheduled generation preserve every issued token', async (t) => {
+  const state = appsFixture(t);
+  const credentials = { client_id: 'system-client', client_secret: 'system-secret' };
+  const results = await Promise.all([
+    state.service.authToken(credentials), state.service.generateSystemToken(true),
+    state.service.authToken(credentials), state.service.generateSystemToken(true),
+  ]);
+  const tokens = state.apps()[0].tokens;
+  assert.equal(tokens.length, 5);
+  for (const response of results) {
+    assert.ok(tokens.some((token) => token.value === (response.value || response.data.token)));
+  }
+  assert.deepEqual(state.cached(), state.apps());
+});
+
+for (const mutation of ['resetSecret', 'remove', 'update', 'create']) {
+  test(`concurrent cache refresh and ${mutation} retain the latest application state`, async (t) => {
+    const state = appsFixture(t);
+    await state.service.authToken({ client_id: 'external-client', client_secret: 'external-secret' });
+    const change = () => mutation === 'resetSecret' ? state.service.resetSecret(2)
+      : mutation === 'remove' ? state.service.remove([2])
+      : mutation === 'update' ? state.service.update({ id: 2, name: 'external', scopes: ['dashboard'] })
+      : state.service.create({ name: 'created', scopes: ['crons'] });
+    await Promise.all([state.service.refreshApps(), change(), state.service.generateSystemToken()]);
+    assert.deepEqual(state.cached(), state.apps());
+    if (mutation === 'resetSecret') {
+      assert.equal(state.cached()[1].tokens.length, 0);
+      assert.notEqual(state.cached()[1].client_secret, 'external-secret');
+      assert.equal((await state.service.authToken({ client_id: 'external-client', client_secret: 'external-secret' })).code, 400);
+    } else if (mutation === 'remove') {
+      assert.equal(state.cached().length, 1);
+    } else if (mutation === 'update') {
+      assert.deepEqual(state.cached()[1].scopes, ['dashboard']);
+    } else {
+      assert.equal(state.cached().length, 3);
+    }
+  });
+}
+
+test('system initialization preserves active tokens while migrating dashboard scope', async (t) => {
+  const state = appsFixture(t);
+  await AppModel.update({ scopes: ['crons', 'system'] }, { where: { id: 1 } });
+  await Promise.all([state.service.initializeSystemApp(), state.service.generateSystemToken(true)]);
+  assert.deepEqual(state.cached(), state.apps());
+  assert.deepEqual(state.apps()[0].scopes, ['crons', 'system', 'dashboard']);
+  assert.equal(state.apps()[0].tokens.length, 2);
+});
+
+test('failed atomic token-file publication releases the application lock for recovery', async (t) => {
+  const state = appsFixture(t);
+  const temporary = path.join(config.configPath, `token.json.${process.pid}.tmp`);
+  fs.mkdirSync(temporary);
+  await assert.rejects(state.service.generateSystemToken(), /EISDIR/);
+  fs.rmSync(temporary, { recursive: true });
+  const token = await state.service.generateSystemToken();
+  assert.equal(token.value, 'system-token');
+  assert.equal(fs.existsSync(path.join(config.configPath, '.apps-state.lock')), false);
+});
+
+test('real SQLite and authentication cache retain concurrent process updates', async (t) => {
+  fs.mkdirSync(config.configPath, { recursive: true });
+  await AppModel.sync({ force: true });
+  const service = new OpenService({});
+  await service.initializeSystemApp();
+  const external = await service.create({ name: 'sqlite-external', scopes: ['crons'] });
+  const worker = path.join(environment, 'sqlite-worker.cjs');
+  const root = path.resolve(__dirname, '../..');
+  fs.writeFileSync(worker, `
+    require(${JSON.stringify(require.resolve('reflect-metadata'))});
+    const path = require('node:path');
+    const root = process.cwd();
+    const OpenService = require(path.join(root, 'back/services/open')).default;
+    const { shareStore, keyvStore } = require(path.join(root, 'back/shared/store'));
+    const { sequelize } = require(path.join(root, 'back/data'));
+    const service = new OpenService({});
+    (async () => {
+      const action = process.argv[2];
+      const result = action === 'auth'
+        ? await service.authToken({ client_id: process.argv[3], client_secret: process.argv[4] })
+        : action === 'refresh' ? await service.refreshApps()
+        : await service.generateSystemToken(action === 'renew');
+      console.log(JSON.stringify(result || {}));
+    })().catch(error => { console.error(error); process.exitCode = 1; })
+      .finally(async () => { await keyvStore.disconnect(); await sequelize.close(); });
+  `);
+  const actions = ['renew', 'acquire', 'auth', 'refresh', 'renew', 'auth', 'auth'];
+  const outcomes = await Promise.allSettled(actions.map((action) => execute(process.execPath, [
+    '-r', require.resolve('ts-node/register/transpile-only'), worker, action,
+    external.client_id, external.client_secret,
+  ], { cwd: root, timeout: 20_000, env: process.env })));
+  const results = outcomes.map((outcome) => {
+    assert.equal(outcome.status, 'fulfilled', outcome.reason?.stderr);
+    return outcome.value;
+  });
+  const database = await service.findApps();
+  const cached = await shareStore.getApps();
+  assert.deepEqual(cached, JSON.parse(JSON.stringify(database)));
+  for (let index = 0; index < results.length; index++) {
+    assert.equal(results[index].stderr, '');
+    const result = JSON.parse(results[index].stdout.trim());
+    const value = result.value || result.data?.token;
+    if (value) assert.ok(cached.some((app) => app.tokens?.some((token) => token.value === value)));
+  }
+  assert.equal(database.find((app) => app.name === 'sqlite-external').tokens.length, 3);
+  const file = JSON.parse(fs.readFileSync(path.join(config.configPath, 'token.json'), 'utf8'));
+  assert.ok(cached.find((app) => app.name === 'system').tokens.some((token) => token.value === file.value));
+});
+
+
+test('atomic publication preserves restricted token-file permissions', async (t) => {
+  const state = fixture(t, [{ value: 'current', expiration: START + DAY }]);
+  const file = path.join(config.configPath, 'token.json');
+  await state.service.generateSystemToken();
+  assert.equal(fs.statSync(file).mode & 0o777, 0o600);
+  fs.chmodSync(file, 0o640);
+  await state.service.generateSystemToken(true);
+  assert.equal(fs.statSync(file).mode & 0o777, 0o640);
+});
+
+
+test('a SIGKILLed generator leaves a lock that the next acquisition recovers', async (t) => {
+  const state = await processFixture(t);
+  const running = state.run(false, false, true);
+  const stopped = running.catch((error) => error);
+  const deadline = Date.now() + 10_000;
+  while (!fs.existsSync(path.join(state.directory, 'locked'))) {
+    assert.ok(Date.now() < deadline, 'generator did not acquire the lock');
+    await new Promise((resolve) => setTimeout(resolve, 20));
+  }
+  running.child.kill('SIGKILL');
+  assert.equal((await stopped).signal, 'SIGKILL');
+  assert.equal(fs.existsSync(path.join(state.directory, '.apps-state.lock')), true);
+  const recovered = await state.run();
+  assert.equal(recovered.stderr, '');
+  assert.equal(recovered.stdout.trim(), state.cached().value);
+  assert.equal(fs.existsSync(path.join(state.directory, '.apps-state.lock')), false);
+});