fix(notify): audit system and JS/Python script notification results

This commit is contained in:
whyour
2026-10-13 02:20:59 +08:00
30 changed files with 2038 additions and 335 deletions
+61
View File
@@ -0,0 +1,61 @@
const test = require('node:test'),
assert = require('node:assert/strict'),
fs = require('node:fs'),
os = require('node:os'),
path = require('node:path'),
express = require('express');
const load = require('../helpers/load-security-module.cjs');
test('unknown API requests return JSON 404 while env.js and SPA deep links work', async (t) => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-fallback-'));
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
fs.mkdirSync(path.join(dir, 'static/dist'), { recursive: true });
fs.writeFileSync(path.join(dir, 'static/dist/index.html'), 'SPA');
const app = express();
load('back/loaders/express.ts', {
'../api': () => express.Router(),
'../config': {
rootPath: dir,
uploadPath: dir,
api: { prefix: '/api' },
apiWhiteList: ['/api/env.js'],
jwt: { secret: 'test' },
},
'express-jwt': {
expressjwt: () => ({ unless: () => (_q, _r, next) => next() }),
UnauthorizedError: class extends Error {},
},
'../config/util': {
getPlatform: () => 'desktop',
getToken: () => 'stored',
},
'../config/serverEnv': {
serveEnv: (_q, r) => r.type('js').send('window.QL_CONFIG={};'),
},
'../shared/store': {
shareStore: { getAuthInfo: async () => ({}), getApps: async () => [] },
},
'../shared/auth': {
isValidToken: () => true,
isDefaultAuthInfo: () => false,
},
'../shared/i18n': { t: (s) => s },
'../middlewares/uniqueConstraintError': (_q, _r, next) => next(),
'../middlewares/protectedPathCase': (_q, _r, next) => next(),
'../data/system': {},
'../data/open': {},
}).default({ app });
const server = await new Promise((r) => {
const s = app.listen(0, '127.0.0.1', () => r(s));
});
t.after(() => new Promise((r) => server.close(r)));
const url = `http://127.0.0.1:${server.address().port}`;
for (const p of ['/api', '/api/missing', '/open/missing']) {
const res = await fetch(url + p);
assert.equal(res.status, 404, p);
assert.match(res.headers.get('content-type'), /json/);
assert.equal((await res.json()).code, 404);
}
const env = await fetch(url + '/api/env.js');
assert.match(await env.text(), /QL_CONFIG/);
assert.equal(await (await fetch(url + '/script')).text(), 'SPA');
});
+253
View File
@@ -0,0 +1,253 @@
const assert = require('node:assert/strict');
const test = require('node:test');
const fs = require('node:fs/promises');
const path = require('node:path');
const os = require('node:os');
const { spawnSync } = require('node:child_process');
const load = require('../helpers/load-security-module.cjs');
const hasApt = spawnSync('apt-get', ['--version']).status === 0;
const {
rewriteMirrorSources,
replaceMirrorUrl,
normalizeMirror,
mirrorWriteCommand,
} = require('../../back/shared/linuxMirror');
async function fixture(t) {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-issue-fixes-'));
t.after(() => fs.rm(root, { force: true, recursive: true }));
return root;
}
test('#2902/#2871 mirror writes are idempotent, preserve suites, security/ports/version paths and quote URLs', async (t) => {
const original = `# http://leave.example/debian\nTypes: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm bookworm-updates\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n\nTypes: deb\nURIs: http://security.debian.org/debian-security\nSuites: bookworm-security\n`;
const url = 'https://mirror.example/mirrors/debian/';
const changed = rewriteMirrorSources(original, url);
assert.equal(rewriteMirrorSources(changed, url), changed);
assert.match(
changed,
/URIs: https:\/\/mirror.example\/mirrors\/debian-security/,
);
assert.match(changed, /Suites: bookworm-security/);
assert.match(changed, /# http:\/\/leave.example\/debian/);
assert.equal(
replaceMirrorUrl('http://old/debian/debian/debian', url),
'https://mirror.example/mirrors/debian',
);
assert.equal(
replaceMirrorUrl('http://old/armbian/armbian', 'https://new/armbian'),
'https://new/armbian',
);
assert.equal(
replaceMirrorUrl('http://old/ubuntu-ports', 'https://new/ubuntu'),
'https://new/ubuntu-ports',
);
assert.equal(
replaceMirrorUrl('http://old/alpine/v3.22/main', 'https://new/alpine'),
'https://new/alpine/v3.22/main',
);
assert.equal(
replaceMirrorUrl('http://old/debian', 'https://new'),
'https://new/debian',
);
for (const bad of [
'file:///tmp/a',
'http://u:p@host',
'https://host?x=1',
'https://host/\nanything',
])
assert.throws(() => normalizeMirror(bad));
const root = await fixture(t);
const file = path.join(root, "quote's", 'sources');
const command = mirrorWriteCommand(file, changed, '', 'true');
const result = spawnSync('bash', ['-c', command], { encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
assert.equal(await fs.readFile(file, 'utf8'), changed);
});
test('#2701 shell API serializes CR/LF, quotes, backslashes and colons without corrupting JSON', async () => {
const result = spawnSync(
'bash',
[
'-c',
`source shell/api.sh
ql_api_request() { while [[ $# -gt 0 ]]; do if [[ "$1" == --data-raw ]]; then printf '%s\\n' "$2" >&3; break; fi; shift; done; printf '{"code":200}'; }
t() { :; }
add_cron_api '0 0 1 1 *' 'task a:b.js' $'quote"slash\\\\colon:\\r\\n\\t中文' 23
update_cron_api '0 0 1 1 *' 'task a:b.js' $'quote"slash\\\\colon:\\r\\n\\t中文' 27
update_cron_command_api $'task path\\\\with"quotes.js' 27
`,
],
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe', 'pipe'] },
);
assert.equal(result.status, 0, result.stderr);
const payloads = result.output[3].trim().split('\n').map(JSON.parse);
assert.equal(payloads[0].name, 'quote"slash\\colon:\r\n\t中文');
assert.equal(payloads[0].sub_id, 23);
assert.equal(payloads[1].id, '27');
assert.equal(payloads[2].command, 'task path\\with"quotes.js');
});
for (const distribution of ['Debian', 'Ubuntu']) {
for (const legacyContent of ['', ' \t\r\n']) {
test(`${distribution}: empty legacy sources.list falls back to deb822 .sources`, async (t) => {
const root = await fixture(t);
const legacy = path.join(root, 'sources.list');
const sources = path.join(
root,
'sources.list.d',
`${distribution.toLowerCase()}.sources`,
);
await fs.writeFile(legacy, legacyContent);
const util = load('back/config/util.ts', {
'fs/promises': {
readFile: async (file) => {
if (file === '/etc/os-release')
return `ID=${distribution.toLowerCase()}\nVERSION_CODENAME=${
distribution === 'Debian' ? 'bookworm' : 'noble'
}\n`;
if (file === '/etc/apt/sources.list')
return fs.readFile(legacy, 'utf8');
throw Object.assign(new Error('missing fixture source'), {
code: 'ENOENT',
});
},
},
os: { platform: () => 'linux' },
'./index': {},
'./const': {},
'./share': {},
'./container': { isInContainer: () => false },
'../loaders/logger': {},
'../shared/utils': {},
'../shared/fileAccess': {},
'../data/dependence': {},
});
const previousOS = process.env.QL_OS_TYPE;
let command;
try {
process.env.QL_OS_TYPE = distribution.toLowerCase();
command = await util.updateLinuxMirrorFile('https://mirror.example');
} finally {
if (previousOS === undefined) delete process.env.QL_OS_TYPE;
else process.env.QL_OS_TYPE = previousOS;
}
// Execute the entire generated writer, redirecting its APT paths into
// the fixture and replacing only the package-index refresh.
const isolated = command
.replaceAll('/etc/apt', root)
.replace('apt-get update --error-on=any', 'true');
const result = spawnSync('bash', ['-c', isolated], { encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
assert.equal(await fs.readFile(legacy, 'utf8'), legacyContent);
const generated = await fs.readFile(sources, 'utf8');
assert.match(generated, /^Types: deb\nURIs: https:\/\/mirror\.example\//);
assert.match(generated, /Signed-By: \/usr\/share\/keyrings\//);
assert.match(
generated,
distribution === 'Debian' ? /bookworm-security/ : /noble-backports/,
);
if (hasApt) {
// Parse both source locations with real APT, without refreshing indexes
// or touching the host's configuration/cache.
const parsed = spawnSync(
'apt-get',
[
'indextargets',
'-o',
`Dir::Etc::sourcelist=${legacy}`,
'-o',
`Dir::Etc::sourceparts=${path.dirname(sources)}`,
'-o',
`Dir::State::lists=${path.join(root, 'lists')}`,
],
{ encoding: 'utf8' },
);
assert.equal(parsed.status, 0, parsed.stderr);
}
});
}
}
test('deb822 mirror changes preserve vendor archives, signing keys and comments across repeated saves', () => {
const system =
'Types: deb\nURIs: https://deb.debian.org/debian\nSuites: bookworm\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n';
const vendors = [
'Types: deb\nURIs: https://download.docker.com/linux/debian\nSuites: bookworm\nComponents: stable\nSigned-By: /etc/apt/keyrings/docker.asc\n',
'Types: deb\nURIs: https://download.docker.com/linux/debian\nSuites: bookworm\nComponents: stable\n',
'Types: deb\nURIs: https://packages.microsoft.com/repos/code\nSuites: stable\nComponents: main\n',
'Types: deb\nURIs: https://vendor.example/debian\nSuites: bookworm\nComponents: main\nSigned-By: /etc/apt/keyrings/vendor.gpg\n',
];
const comment = '# Keep https://vendor.example/debian unchanged\n';
const original = [system, ...vendors, comment].join('\n');
const first = rewriteMirrorSources(
original,
'https://mirror.example/mirrors/debian',
);
assert.ok(first.includes('URIs: https://mirror.example/mirrors/debian\n'));
for (const unchanged of [...vendors, comment])
assert.ok(first.includes(unchanged));
assert.equal(
rewriteMirrorSources(first, 'https://mirror.example/mirrors/debian'),
first,
);
const second = rewriteMirrorSources(first, 'https://second.example/debian');
assert.ok(second.includes('URIs: https://second.example/debian\n'));
for (const unchanged of [...vendors, comment])
assert.ok(second.includes(unchanged));
});
test('legacy mirror changes preserve third-party lines, inline comments and source options', () => {
const system =
'deb [arch=amd64 signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] https://deb.debian.org/debian bookworm main # https://leave.example/debian\n';
const security =
'deb-src https://security.debian.org/debian-security bookworm-security main\n';
const vendors = [
'deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable\n',
'deb https://download.docker.com/linux/debian bookworm stable\n',
'deb [signed-by=/etc/apt/keyrings/vendor.gpg] https://vendor.example/debian bookworm main\n',
'deb https://packages.microsoft.com/repos/code stable main\n',
'# deb https://deb.debian.org/debian bookworm main\n',
];
const original = system + security + vendors.join('');
const changed = rewriteMirrorSources(
original,
'https://mirror.example/debian',
);
assert.ok(
changed.includes(
system.replace(
'https://deb.debian.org/debian',
'https://mirror.example/debian',
),
),
);
assert.ok(
changed.includes(
security.replace(
'https://security.debian.org/debian-security',
'https://mirror.example/debian-security',
),
),
);
for (const unchanged of vendors) assert.ok(changed.includes(unchanged));
assert.equal(
rewriteMirrorSources(changed, 'https://mirror.example/debian'),
changed,
);
});
test('Ubuntu ports, Armbian and Alpine roots still switch mirrors without altering repository metadata', () => {
for (const url of [
'https://ports.ubuntu.com/ubuntu-ports',
'https://apt.armbian.com/armbian',
'https://old.example/alpine/v3.22/main',
'https://old.example/alpine/v3.22/community',
]) {
const original = `URIs: ${url}\nSuites: noble\nComponents: main\n`;
assert.equal(
rewriteMirrorSources(original, 'https://mirror.example'),
original.replace(url, replaceMirrorUrl(url, 'https://mirror.example')),
);
}
});
+1 -1
View File
@@ -129,7 +129,7 @@ Acquire::http::Timeout "1";
'fs/promises': {
readFile: async (file) =>
file === '/etc/os-release'
? distribution
? `${distribution}\nVERSION_CODENAME=bookworm\n`
: existingMirror
? 'URIs: http://old.example/debian\n'
: '',
+34 -2
View File
@@ -38,8 +38,22 @@ async function fixture(t) {
}).default;
const service = Object.assign(Object.create(S.prototype), {
logger,
notificationService: { notify: async () => true },
notificationService: new (load('back/services/notify.ts', {
typedi: decorators,
'../config/util': {},
'../data/system': {
SystemModel: rows,
AuthDataType: { notifyLog: 'notifyLog' },
NotifyStatus: { success: 0, fail: 1 },
},
'../shared/i18n': { t: (s) => s },
'./user': {},
}).default)(),
});
service.notificationService.userService = {
getNotificationMode: async () => ({ type: 'email' }),
};
service.notificationService.modeMap.set('email', async () => true);
service.getDb = async () => ({ info: { type: 'email' } });
return { rows, service };
}
@@ -72,7 +86,7 @@ test('audit DB failure does not turn a successful notification into failure', as
});
test('failed sends are recorded and message size is bounded', async (t) => {
const { service } = await fixture(t);
service.notificationService.notify = async () => false;
service.notificationService.modeMap.set('gotify', async () => false);
assert.equal(
(
await service.notify({
@@ -89,3 +103,21 @@ test('failed sends are recorded and message size is bounded', async (t) => {
assert.equal(row.content.length, 8192);
assert.equal(row.notifyType, 'gotify');
});
test('script reports share retention and preserve source without sending twice', async (t) => {
const { service } = await fixture(t);
await service.recordNotifyLog({
title: 'task result',
content: 'body',
status: 1,
notifyType: 'smtp',
source: 'script',
scriptName: 'job.py',
taskId: '42',
});
const [row] = await service.getNotifyLog();
assert.equal(row.source, 'script');
assert.equal(row.scriptName, 'job.py');
assert.equal(row.taskId, '42');
assert.equal(row.status, 1);
});
@@ -0,0 +1,152 @@
const test = require('node:test'),
assert = require('node:assert/strict'),
fs = require('node:fs'),
path = require('node:path'),
vm = require('node:vm');
const load = require('../helpers/load-security-module.cjs');
function script(config = {}, response = {}, auditError = false) {
const logs = [],
source = fs.readFileSync('sample/notify.js', 'utf8');
const transport = async () => ({
statusCode: response.httpStatus || 200,
body: { text: async () => JSON.stringify(response) },
});
const context = {
module: { exports: {} },
console: { log() {}, info() {} },
process: {
env: { QL_DIR: '/fixture', HITOKOTO: 'false', ...config },
argv: ['node', '/fixture/scripts/test-job.js'],
},
Buffer,
URL,
setTimeout,
clearTimeout,
};
context.require = (name) => {
if (name === 'undici')
return { request: transport, ProxyAgent: class {}, FormData };
if (name === '/fixture/shell/preload/notificationAudit.js')
return async (x) => {
if (auditError) throw Error('offline');
logs.push(x);
};
if (name === 'nodemailer')
return {
createTransport: () => ({
sendMail: async () => {
if (response.smtpFailure) throw Error('auth rejected');
return { messageId: 'fixture' };
},
close() {},
}),
};
return require(name);
};
vm.runInNewContext(source, context, {
filename: path.resolve('sample/notify.js'),
});
return { send: context.module.exports.sendNotify, logs };
}
test('JS helper logs only configured channels, success/failure and script/task provenance', async () => {
for (const fail of [false, true]) {
const s = script(
{
SMTP_EMAIL: 'sender@example.test',
SMTP_PASSWORD: 'private-password',
SMTP_SERVICE: 'fixture',
ID: '17',
},
{ smtpFailure: fail },
);
await s.send('标题', '正文');
assert.equal(s.logs.length, 1);
assert.equal(s.logs[0].success, !fail);
assert.equal(s.logs[0].notifyType, 'smtp');
assert.equal(s.logs[0].scriptName, 'test-job.js');
assert.equal(s.logs[0].taskId, '17');
assert.ok(!JSON.stringify(s.logs).includes('private-password'));
}
const s = script();
await s.send('none', 'none');
assert.equal(s.logs.length, 0);
});
test('HTTP business rejection is audited as failure and logging errors do not change sends', async () => {
for (const [code, success] of [
[200, true],
[500, false],
]) {
const s = script({ BARK_PUSH: 'https://fixture.invalid' }, { code });
await s.send('title', 'body');
assert.equal(s.logs.length, 1);
assert.equal(s.logs[0].success, success);
}
const offline = script(
{
SMTP_EMAIL: 'sender@example.test',
SMTP_PASSWORD: 'fixture',
SMTP_SERVICE: 'fixture',
},
{},
true,
);
await offline.send('still sends', 'body');
});
test('multi-channel helpers write one result per actual channel', async () => {
const s = script(
{
GOTIFY_URL: 'https://fixture.invalid',
GOTIFY_TOKEN: 'private-token',
SMTP_EMAIL: 'sender@example.test',
SMTP_PASSWORD: 'fixture',
SMTP_SERVICE: 'fixture',
},
{ id: 123 },
);
await s.send('title', 'body');
assert.deepEqual(s.logs.map((x) => x.notifyType).sort(), ['gotify', 'smtp']);
assert.ok(s.logs.every((x) => x.success));
});
test('script audit protobuf round trips without notification credentials', () => {
const p = load('back/protos/api.ts');
const value = p.RecordNotifyLogRequest.create({
title: '中文',
content: '内容',
success: false,
notifyType: 'smtp',
scriptName: 'job.py',
taskId: '12',
});
assert.deepEqual(
p.RecordNotifyLogRequest.decode(
p.RecordNotifyLogRequest.encode(value).finish(),
),
value,
);
assert.ok(p.ApiService.recordNotifyLog);
});
test('multi-target Chronocat records one failure if any recipient fails', async () => {
let calls = 0;
const response = {};
Object.defineProperty(response, 'httpStatus', {
get: () => (++calls === 1 ? 200 : 500),
});
const s = script(
{
CHRONOCAT_URL: 'https://fixture.invalid',
CHRONOCAT_TOKEN: 'private-token',
CHRONOCAT_QQ: 'user_id=123&group_id=456',
},
response,
);
await s.send('title', 'body');
assert.equal(calls, 2);
assert.equal(s.logs.length, 1);
assert.equal(s.logs[0].success, false);
});
+114
View File
@@ -0,0 +1,114 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const os = require('node:os');
const { promisify } = require('node:util');
const { execFile } = require('node:child_process');
const grpc = require('@grpc/grpc-js');
const loader = require('@grpc/proto-loader');
const load = require('../helpers/load-security-module.cjs');
const run = promisify(execFile);
test('JS and Python script reports reach the same authenticated RPC with bounded failure', async (t) => {
const data = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-audit-rpc-'));
t.after(() => fs.rmSync(data, { recursive: true, force: true }));
fs.mkdirSync(path.join(data, 'config'));
const certs = await load('back/config/grpcCerts.ts', {
'./index': { configPath: path.join(data, 'config') },
'./util': { fileExist: async (p) => fs.existsSync(p) },
'../loaders/logger': { info() {}, error() {} },
}).initGrpcCerts();
const definition = loader.loadSync(path.resolve('back/protos/api.proto'), {
keepCase: true,
defaults: true,
});
const Api = grpc.loadPackageDefinition(definition).com.ql.api.Api;
const received = [];
const server = new grpc.Server();
server.addService(Api.service, {
RecordNotifyLog: (call, callback) => {
received.push(call.request);
callback(null, { code: 200 });
},
});
const credentials = grpc.ServerCredentials.createSsl(
Buffer.from(certs.caCert),
[
{
private_key: Buffer.from(certs.serverKey),
cert_chain: Buffer.from(certs.serverCert),
},
],
true,
);
const port = await new Promise((resolve, reject) =>
server.bindAsync('127.0.0.1:0', credentials, (error, port) =>
error ? reject(error) : resolve(port),
),
);
t.after(() => server.forceShutdown());
const env = {
...process.env,
QL_DIR: process.cwd(),
QL_DATA_DIR: data,
GRPC_PORT: String(port),
ID: '9',
HITOKOTO: 'false',
};
await run(
process.execPath,
[
'-e',
`require('./shell/preload/notificationAudit.js')({title:'JS task',content:'body',success:true,notifyType:'smtp',scriptName:'task.js',taskId:process.env.ID})`,
],
{ env, timeout: 8000 },
);
const python = `import importlib.util,sys,types,smtplib,os
sys.modules['requests']=types.SimpleNamespace()
spec=importlib.util.spec_from_file_location('notify',os.path.join(os.environ['QL_DIR'],'sample/notify.py'))
n=importlib.util.module_from_spec(spec);spec.loader.exec_module(n)
class Mail:
def __init__(self,*a,**kw): pass
def login(self,*a): pass
def sendmail(self,*a): return {}
def close(self): pass
smtplib.SMTP_SSL=Mail
n.push_config={'SMTP_SERVER':'fixture','SMTP_SSL':'true','SMTP_EMAIL':'sender@example.test','SMTP_PASSWORD':'private-password','SMTP_NAME':'fixture','HITOKOTO':False}
n.send('Python success','body')
def rejected(*a,**kw): raise RuntimeError('private-password')
smtplib.SMTP_SSL=rejected
n.send('Python failure','body')
`;
await run('python3', ['-c', python], { env, timeout: 12000 });
assert.deepEqual(
received.map((x) => [x.title, x.success]),
[
['JS task', true],
['Python success', true],
['Python failure', false],
],
);
assert.ok(received.every((x) => x.taskId === '9'));
assert.ok(!JSON.stringify(received).includes('private-password'));
// Missing certificates and an older/unavailable backend must not kill scripts.
await run(
process.execPath,
[
'-e',
`require('./shell/preload/notificationAudit.js')({title:'offline'}).then(()=>console.log('done'))`,
],
{ env: { ...env, QL_DATA_DIR: path.join(data, 'missing') }, timeout: 8000 },
);
server.forceShutdown();
const started = Date.now();
await run(
process.execPath,
[
'-e',
`require('./shell/preload/notificationAudit.js')({title:'offline'})`,
],
{ env, timeout: 8000 },
);
assert.ok(Date.now() - started < 5000);
});