diff --git a/back/api/config.ts b/back/api/config.ts index f91feb77..99649fb3 100644 --- a/back/api/config.ts +++ b/back/api/config.ts @@ -7,6 +7,7 @@ import { celebrate, Joi } from 'celebrate'; import { join } from 'path'; import { SAMPLE_FILES } from '../config/const'; import ConfigService from '../services/config'; +import { writeFileWithLock } from '../shared/utils'; const route = Router(); export default (app: Router) => { @@ -77,7 +78,7 @@ export default (app: Router) => { if (name.startsWith('data/scripts/')) { path = join(config.rootPath, name); } - await fs.writeFile(path, content); + await writeFileWithLock(path, content); res.send({ code: 200, message: '保存成功' }); } catch (e) { return next(e); diff --git a/back/api/script.ts b/back/api/script.ts index b9a22330..721db871 100644 --- a/back/api/script.ts +++ b/back/api/script.ts @@ -8,6 +8,7 @@ import { celebrate, Joi } from 'celebrate'; import path, { join, parse } from 'path'; import ScriptService from '../services/script'; import multer from 'multer'; +import { writeFileWithLock } from '../shared/utils'; const route = Router(); const storage = multer.diskStorage({ @@ -156,7 +157,7 @@ export default (app: Router) => { await rmPath(originFilePath); } } - await fs.writeFile(filePath, content); + await writeFileWithLock(filePath, content); return res.send({ code: 200 }); } catch (e) { return next(e); @@ -182,7 +183,7 @@ export default (app: Router) => { path: string; }; const filePath = join(config.scriptPath, path, filename); - await fs.writeFile(filePath, content); + await writeFileWithLock(filePath, content); return res.send({ code: 200 }); } catch (e) { return next(e); @@ -261,7 +262,7 @@ export default (app: Router) => { let { filename, content, path } = req.body; const { name, ext } = parse(filename); const filePath = join(config.scriptPath, path, `${name}.swap${ext}`); - await fs.writeFile(filePath, content || '', { encoding: 'utf8' }); + await writeFileWithLock(filePath, content || ''); const scriptService = Container.get(ScriptService); const result = await scriptService.runScript(filePath); diff --git a/back/config/util.ts b/back/config/util.ts index b5b25d5b..a6cc5367 100644 --- a/back/config/util.ts +++ b/back/config/util.ts @@ -10,6 +10,7 @@ import { load } from 'js-yaml'; import config from './index'; import { TASK_COMMAND } from './const'; import Logger from '../loaders/logger'; +import { writeFileWithLock } from '../shared/utils'; export * from './share'; @@ -170,7 +171,7 @@ export async function fileExist(file: any) { export async function createFile(file: string, data: string = '') { await fs.mkdir(path.dirname(file), { recursive: true }); - await fs.writeFile(file, data); + await writeFileWithLock(file, data); } export async function handleLogPath( diff --git a/back/loaders/initFile.ts b/back/loaders/initFile.ts index f4a321a6..4fb20a03 100644 --- a/back/loaders/initFile.ts +++ b/back/loaders/initFile.ts @@ -3,6 +3,7 @@ import path from 'path'; import os from 'os'; import Logger from './logger'; import { fileExist } from '../config/util'; +import { writeFileWithLock } from '../shared/utils'; const rootPath = process.env.QL_DIR as string; let dataPath = path.join(rootPath, 'data/'); @@ -99,46 +100,46 @@ export default async () => { // 初始化文件 if (!confFileExist) { - await fs.writeFile(confFile, await fs.readFile(sampleConfigFile)); + await writeFileWithLock(confFile, await fs.readFile(sampleConfigFile)); } - await fs.writeFile(jsNotifyFile, await fs.readFile(sampleNotifyJsFile)); - await fs.writeFile(pyNotifyFile, await fs.readFile(sampleNotifyPyFile)); + await writeFileWithLock(jsNotifyFile, await fs.readFile(sampleNotifyJsFile)); + await writeFileWithLock(pyNotifyFile, await fs.readFile(sampleNotifyPyFile)); if (!scriptNotifyJsFileExist) { - await fs.writeFile( + await writeFileWithLock( scriptNotifyJsFile, await fs.readFile(sampleNotifyJsFile), ); } if (!scriptNotifyPyFileExist) { - await fs.writeFile( + await writeFileWithLock( scriptNotifyPyFile, await fs.readFile(sampleNotifyPyFile), ); } if (!TaskBeforeFileExist) { - await fs.writeFile(TaskBeforeFile, await fs.readFile(sampleTaskShellFile)); + await writeFileWithLock(TaskBeforeFile, await fs.readFile(sampleTaskShellFile)); } if (!TaskBeforeJsFileExist) { - await fs.writeFile( + await writeFileWithLock( TaskBeforeJsFile, '// The JavaScript code that executes before the JavaScript task execution will execute.', ); } if (!TaskBeforePyFileExist) { - await fs.writeFile( + await writeFileWithLock( TaskBeforePyFile, '# The Python code that executes before the Python task execution will execute.', ); } if (!TaskAfterFileExist) { - await fs.writeFile(TaskAfterFile, await fs.readFile(sampleTaskShellFile)); + await writeFileWithLock(TaskAfterFile, await fs.readFile(sampleTaskShellFile)); } Logger.info('✌️ Init file down'); diff --git a/back/services/cron.ts b/back/services/cron.ts index 9b9b9852..e7ed12e7 100644 --- a/back/services/cron.ts +++ b/back/services/cron.ts @@ -21,6 +21,7 @@ import { spawn } from 'cross-spawn'; import dayjs from 'dayjs'; import pickBy from 'lodash/pickBy'; import omit from 'lodash/omit'; +import { writeFileWithLock } from '../shared/utils'; @Service() export default class CronService { @@ -601,7 +602,7 @@ export default class CronService { } }); - await fs.writeFile(config.crontabFile, crontab_string); + await writeFileWithLock(config.crontabFile, crontab_string); execSync(`crontab ${config.crontabFile}`); await CrontabModel.update({ saved: true }, { where: {} }); diff --git a/back/services/env.ts b/back/services/env.ts index 345bb065..fe209d6a 100644 --- a/back/services/env.ts +++ b/back/services/env.ts @@ -13,6 +13,7 @@ import { } from '../data/env'; import groupBy from 'lodash/groupBy'; import { FindOptions, Op } from 'sequelize'; +import { writeFileWithLock } from '../shared/utils'; @Service() export default class EnvService { @@ -225,8 +226,8 @@ export default class EnvService { } } } - await fs.writeFile(config.envFile, env_string); - await fs.writeFile(config.jsEnvFile, js_env_string); - await fs.writeFile(config.pyEnvFile, py_env_string); + await writeFileWithLock(config.envFile, env_string); + await writeFileWithLock(config.jsEnvFile, js_env_string); + await writeFileWithLock(config.pyEnvFile, py_env_string); } } diff --git a/back/services/sshKey.ts b/back/services/sshKey.ts index 62244d2b..0a2824c9 100644 --- a/back/services/sshKey.ts +++ b/back/services/sshKey.ts @@ -7,6 +7,7 @@ import { Subscription } from '../data/subscription'; import { formatUrl } from '../config/subscription'; import config from '../config'; import { fileExist, rmPath } from '../config/util'; +import { writeFileWithLock } from '../shared/utils'; @Service() export default class SshKeyService { @@ -25,13 +26,12 @@ export default class SshKeyService { if (_exist) { config = await fs.readFile(this.sshConfigFilePath, { encoding: 'utf-8' }); } else { - await fs.writeFile(this.sshConfigFilePath, ''); + await writeFileWithLock(this.sshConfigFilePath, ''); } if (!config.includes(this.sshConfigHeader)) { - await fs.writeFile( + await writeFileWithLock( this.sshConfigFilePath, `${this.sshConfigHeader}\n\n${config}`, - { encoding: 'utf-8' }, ); } } @@ -41,10 +41,14 @@ export default class SshKeyService { key: string, ): Promise { try { - await fs.writeFile(path.join(this.sshPath, alias), `${key}${os.EOL}`, { - encoding: 'utf8', - mode: '400', - }); + await writeFileWithLock( + path.join(this.sshPath, alias), + `${key}${os.EOL}`, + { + encoding: 'utf8', + mode: '400', + }, + ); } catch (error) { this.logger.error('生成私钥文件失败', error); } @@ -74,12 +78,9 @@ export default class SshKeyService { this.sshPath, alias, )}\n StrictHostKeyChecking no\n${proxyStr}`; - await fs.writeFile( + await writeFileWithLock( `${path.join(this.sshPath, `${alias}.config`)}`, config, - { - encoding: 'utf8', - }, ); } @@ -102,7 +103,11 @@ export default class SshKeyService { await this.generateSingleSshConfig(alias, host, proxy); } - public async removeSSHKey(alias: string, host: string, proxy?: string): Promise { + public async removeSSHKey( + alias: string, + host: string, + proxy?: string, + ): Promise { await this.removePrivateKeyFile(alias); await this.removeSshConfig(alias); } diff --git a/back/shared/utils.ts b/back/shared/utils.ts new file mode 100644 index 00000000..2753476b --- /dev/null +++ b/back/shared/utils.ts @@ -0,0 +1,27 @@ +import { lock } from 'proper-lockfile'; +import { writeFile, open } from 'fs/promises'; +import { fileExist } from '../config/util'; + +export async function writeFileWithLock( + path: string, + content: string | Buffer, + options: Parameters[2] = {}, +) { + if (typeof options === 'string') { + options = { encoding: options }; + } + if (!(await fileExist(path))) { + const fileHandle = await open(path, 'w'); + fileHandle.close(); + } + const release = await lock(path, { + retries: { + retries: 10, + factor: 2, + minTimeout: 100, + maxTimeout: 3000, + }, + }); + await writeFile(path, content, { encoding: 'utf8', ...options }); + await release(); +} diff --git a/back/token.ts b/back/token.ts index b3711a87..6aeed6a4 100755 --- a/back/token.ts +++ b/back/token.ts @@ -6,6 +6,7 @@ import fs from 'fs'; import config from './config'; import path from 'path'; import os from 'os'; +import { writeFileWithLock } from './shared/utils'; const tokenFile = path.join(config.configPath, 'token.json'); @@ -25,16 +26,7 @@ async function getToken() { } async function writeFile(data: any) { - return new Promise((resolve, reject) => { - fs.writeFile( - tokenFile, - `${JSON.stringify(data)}${os.EOL}`, - { encoding: 'utf8' }, - () => { - resolve(); - }, - ); - }); + await writeFileWithLock(tokenFile, `${JSON.stringify(data)}${os.EOL}`); } getToken(); diff --git a/package.json b/package.json index 5fc9ae6f..aad1bf67 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,8 @@ "request-ip": "3.3.0", "ip2region": "2.3.0", "keyv": "^5.2.3", - "@keyv/sqlite": "^4.0.1" + "@keyv/sqlite": "^4.0.1", + "proper-lockfile": "^4.1.2" }, "devDependencies": { "moment": "2.30.1", @@ -138,6 +139,7 @@ "@types/sockjs-client": "^1.5.1", "@types/uuid": "^8.3.4", "@types/request-ip": "0.0.41", + "@types/proper-lockfile": "^4.1.4", "@uiw/codemirror-extensions-langs": "^4.21.9", "@uiw/react-codemirror": "^4.21.9", "@umijs/max": "^4.3.36", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf0ed1bc..04322a31 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,5 +1,9 @@ lockfileVersion: '6.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + overrides: sqlite3: git+https://github.com/whyour/node-sqlite3.git#v1.0.3 @@ -97,6 +101,9 @@ dependencies: p-queue-cjs: specifier: 7.3.4 version: 7.3.4 + proper-lockfile: + specifier: ^4.1.2 + version: 4.1.2 protobufjs: specifier: ^7.4.0 version: 7.4.0 @@ -201,6 +208,9 @@ devDependencies: '@types/nodemailer': specifier: ^6.4.4 version: 6.4.17 + '@types/proper-lockfile': + specifier: ^4.1.4 + version: 4.1.4 '@types/qrcode.react': specifier: ^1.0.2 version: 1.0.5 @@ -4634,6 +4644,12 @@ packages: resolution: {integrity: sha512-hCZTSvwbzWGvhqxp/RqVqwU999pBf2vp7hzIjiYOsl8wqOmUxkQ6ddw1cV3l8811+kdUFus/q4d1Y3E3SyEifA==} dev: true + /@types/proper-lockfile@4.1.4: + resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==} + dependencies: + '@types/retry': 0.12.5 + dev: true + /@types/qrcode.react@1.0.5: resolution: {integrity: sha512-BghPtnlwvrvq8QkGa1H25YnN+5OIgCKFuQruncGWLGJYOzeSKiix/4+B9BtfKF2wf5ja8yfyWYA3OXju995G8w==} dependencies: @@ -4683,6 +4699,10 @@ packages: '@types/node': 17.0.45 dev: false + /@types/retry@0.12.5: + resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + dev: true + /@types/semver@7.5.8: resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} dev: true @@ -12466,6 +12486,14 @@ packages: react-is: 16.13.1 dev: true + /proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + dev: false + /protobufjs@7.4.0: resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} engines: {node: '>=12.0.0'} @@ -14041,7 +14069,6 @@ packages: engines: {node: '>= 4'} requiresBuild: true dev: false - optional: true /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} @@ -16174,7 +16201,3 @@ packages: - encoding - supports-color dev: false - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false diff --git a/tsconfig.back.json b/tsconfig.back.json index 007d4a85..18c7c610 100644 --- a/tsconfig.back.json +++ b/tsconfig.back.json @@ -3,9 +3,6 @@ "target": "es2017", "lib": ["ESNext"], "typeRoots": ["./node_modules/celebrate/lib", "./node_modules/@types"], - "paths": { - "@/*": ["./back/*"] - }, "allowSyntheticDefaultImports": true, "experimentalDecorators": true, "emitDecoratorMetadata": true, diff --git a/tsconfig.json b/tsconfig.json index fb34aa5c..a0a5399d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,10 +22,6 @@ "allowJs": true, "noEmit": false }, - "include": ["src/**/*", ".umirc.ts", "typings.d.ts", "back/**/*"], - "exclude": [ - "node_modules", - "static", - "data", - ] + "include": ["src/**/*", ".umirc.ts", "typings.d.ts"], + "exclude": ["node_modules", "static", "data"] }