From 95b3ba6121e351536ff2ddb81859eb13dd08b82b Mon Sep 17 00:00:00 2001 From: wuyouMaster Date: Sun, 20 Sep 2026 20:40:10 +0800 Subject: [PATCH] fix: auto retry scheduled reports after send recovery --- src/main/services/scheduled-report-service.ts | 100 +++-- tests/unit/scheduled-report-service.test.ts | 408 ++++++++++++++++++ 2 files changed, 481 insertions(+), 27 deletions(-) diff --git a/src/main/services/scheduled-report-service.ts b/src/main/services/scheduled-report-service.ts index 37342fb..6e406be 100644 --- a/src/main/services/scheduled-report-service.ts +++ b/src/main/services/scheduled-report-service.ts @@ -46,6 +46,9 @@ const EXECUTIONS_FILE = 'executions.json' const NOTIFICATIONS_FILE = 'notifications.json' const SETTINGS_FILE = 'settings.json' const TICK_MS = 15_000 +const AUTO_RESEND_MAX_ATTEMPTS = 6 +const AUTO_RESEND_BASE_BACKOFF_MS = 5 * 60_000 +const AUTO_RESEND_MAX_BACKOFF_MS = 60 * 60_000 const NOTIFICATION_TEST_MESSAGE = `✅ TraceMemo 定时日报通知已开启 以后定时日报生成或发送出现异常时， @@ -77,28 +80,6 @@ export interface ScheduledReportSendActionInput { taskId?: string } -function buildScheduledReportActionRequest( - input: ScheduledReportSendActionInput -): WechatActionRequest { - return { - idempotencyKey: - input.retryCount && input.retryCount > 0 - ? `scheduled_report:${input.executionId}:retry:${input.retryCount}` - : `scheduled_report:${input.executionId}`, - origin: 'scheduled_report', - purpose: 'scheduled_report', - triggerType: input.triggerType === 'scheduled' ? 'automation' : 'user', - sourceId: input.executionId, - executionId: input.executionId, - recipient: { type: 'group', id: input.target }, - content: { type: 'image', path: input.filePath }, - metadata: { - taskId: input.taskId, - retryCount: input.retryCount || 0 - } - } -} - const defaultDependencies = (): ScheduledReportDependencies => ({ getCapability: () => personalWechatCapabilityService.getPersonalWechatSendCapability(), generateReport: generateAgentGroupReport, @@ -161,6 +142,29 @@ interface ScheduledReportNotificationPayload { suggestedAction?: string } +export function buildScheduledReportActionRequest( + input: ScheduledReportSendActionInput +): WechatActionRequest { + return { + idempotencyKey: + input.retryCount && input.retryCount > 0 + ? `scheduled_report:${input.executionId}:retry:${input.retryCount}` + : `scheduled_report:${input.executionId}`, + origin: 'scheduled_report', + purpose: 'scheduled_report', + triggerType: input.triggerType === 'scheduled' ? 'automation' : 'user', + sourceId: input.executionId, + executionId: input.executionId, + // filehelper 是文件传输助手：唯一的真实发送验收对象，走 contact 而不是群。 + recipient: { type: input.target === 'filehelper' ? 'contact' : 'group', id: input.target }, + content: { type: 'image', path: input.filePath }, + metadata: { + taskId: input.taskId, + retryCount: input.retryCount || 0 + } + } +} + export function validateScheduleTime(value: string): boolean { return /^(?:[01]\d|2[0-3]):[0-5]\d$/.test(String(value || '').trim()) } @@ -498,11 +502,7 @@ export class ScheduledReportService { await this.load() const execution = this.executions!.find((item) => item.id === executionId) if (!execution) return { success: false, error: '未找到定时日报执行记录' } - const existing = this.retrying.get(executionId) - if (existing) return { success: true, data: await existing } - const promise = this.retrySend(execution).finally(() => this.retrying.delete(executionId)) - this.retrying.set(executionId, promise) - const result = await promise + const result = await this.startRetry(execution) return { success: result.status !== 'failed', data: result, @@ -568,6 +568,7 @@ export class ScheduledReportService { async tick(at = this.deps.now?.() || new Date()): Promise { await this.load() await this.flushNotifications() + await this.flushPendingSends(at) if (!this.deps.isDatabaseReady()) return const nowMs = at.getTime() for (const task of [...this.tasks!]) { @@ -959,6 +960,15 @@ export class ScheduledReportService { return completed } + /** 为同一执行记录复用唯一的重发 Promise，合并自动重发和手动重试。 */ + private startRetry(execution: ScheduledReportExecution): Promise { + const existing = this.retrying.get(execution.id) + if (existing) return existing + const promise = this.retrySend(execution).finally(() => this.retrying.delete(execution.id)) + this.retrying.set(execution.id, promise) + return promise + } + private clearFailureFields( execution: ScheduledReportExecution, patch: Partial @@ -1190,6 +1200,40 @@ export class ScheduledReportService { await Promise.all([this.saveNotifications(), this.saveExecutions()]) } + /** waiting_to_send 执行的下一次自动重发时间：自 finishedAt 起按指数退避。 */ + private autoResendDueAt(execution: ScheduledReportExecution): number { + const base = Date.parse(execution.finishedAt || execution.startedAt) + const attempt = Math.max(execution.retryCount || 0, 0) + return base + Math.min(AUTO_RESEND_BASE_BACKOFF_MS * 2 ** attempt, AUTO_RESEND_MAX_BACKOFF_MS) + } + + /** 能力恢复后自动重发挂起的日报；重试有上限，超限后仍可手动重试。 */ + private async flushPendingSends(at: Date): Promise { + await this.load() + const nowMs = at.getTime() + const candidates = this.executions!.filter( + (item) => + item.status === 'waiting_to_send' && + item.retryable !== false && + Boolean(item.pngPath) && + (item.retryCount || 0) < AUTO_RESEND_MAX_ATTEMPTS && + !this.retrying.has(item.id) && + this.autoResendDueAt(item) <= nowMs + ) + const retries: Promise[] = [] + for (const execution of candidates) { + const task = this.tasks!.find((item) => item.id === execution.taskId) + if (!task || !task.enabled) continue + retries.push(this.startRetry(execution)) + } + const results = await Promise.allSettled(retries) + for (const result of results) { + if (result.status === 'rejected') { + console.warn('[ScheduledReport] auto resend failed:', result.reason) + } + } + } + private notificationCapabilityReasonForSend( result: AgentHubNotificationResult ): ScheduledReportNotificationCapabilityReason { @@ -1219,6 +1263,8 @@ export class ScheduledReportService { private resolveTarget(task: ScheduledReportTask): string | undefined { const explicit = String(task.target || '').trim() if (explicit.endsWith('@chatroom')) return explicit + // 文件传输助手是唯一允许真实发送验收的个人目标。 + if (explicit.toLowerCase() === 'filehelper') return 'filehelper' const contact = resolveMd5(task.group || explicit) return contact?.m_nsUsrName?.endsWith('@chatroom') ? contact.m_nsUsrName : undefined } diff --git a/tests/unit/scheduled-report-service.test.ts b/tests/unit/scheduled-report-service.test.ts index 6f1793e..92e8474 100644 --- a/tests/unit/scheduled-report-service.test.ts +++ b/tests/unit/scheduled-report-service.test.ts @@ -4,6 +4,7 @@ import { join } from 'path' import { describe, expect, it, vi } from 'vitest' vi.mock('electron', () => ({ app: { getPath: () => '/tmp/tracememo-test-user-data' } })) import { + buildScheduledReportActionRequest, calculateNextRunAt, ScheduledReportService, validateScheduleTime @@ -973,3 +974,410 @@ describe('scheduled report scheduling', () => { ).toHaveLength(1) }) }) + +describe('scheduled report auto resend', () => { + const latestExecution = async ( + service: ScheduledReportService, + taskId: string + ): Promise => (await service.listExecutions(taskId))[0] + + const waitForAutoResend = async ( + predicate: () => Promise, + timeoutMs = 3_000 + ): Promise => { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (await predicate()) return + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error('auto resend state did not settle in time') + } + + const sentAction = { + actionId: 'send-success-action', + status: 'sent' as const, + decision: 'allow' as const, + startedAt: '2026-08-27T01:06:00.000Z', + finishedAt: '2026-08-27T01:06:01.000Z' + } + + const unavailableSendAction = () => async () => ({ + actionId: 'send-unavailable-action', + status: 'failed' as const, + decision: 'allow' as const, + errorCode: 'SEND_CAPABILITY_UNAVAILABLE' as const, + reason: '当前微信发送能力不可用', + startedAt: '2026-08-27T01:00:00.000Z', + finishedAt: '2026-08-27T01:00:01.000Z' + }) + + it('auto resends a waiting report after backoff and notifies recovery', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-auto-resend-')) + await enableNotifications(storageDir) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + const sendNotification = vi.fn(async () => ({ + success: true as const, + status: 'sent' as const + })) + const sendAction = vi + .fn() + .mockResolvedValueOnce({ + actionId: 'send-1', + status: 'failed' as const, + decision: 'allow' as const, + errorCode: 'SEND_CAPABILITY_UNAVAILABLE' as const, + reason: '当前微信发送能力不可用', + startedAt: '2026-08-27T01:00:00.000Z', + finishedAt: '2026-08-27T01:00:01.000Z' + }) + .mockResolvedValueOnce({ + actionId: 'send-2', + status: 'sent' as const, + decision: 'allow' as const, + startedAt: '2026-08-27T01:06:00.000Z', + finishedAt: '2026-08-27T01:06:01.000Z' + }) + const service = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction, sendNotification }) + }) + const created = await service.createTask({ + name: '自动重发日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + + const first = await runScheduled(service, created.data!.id) + expect(first).toMatchObject({ + status: 'waiting_to_send', + retryCount: 0, + pngPath: '/tmp/saved.png', + notificationStatus: 'not_needed' + }) + + // retryCount 0 的退避是 5 分钟：推进到到期时刻再 tick。 + clock.now = Date.parse(first.finishedAt || first.startedAt) + 5 * 60_000 + 1_000 + await service.tick(new Date(clock.now)) + await waitForAutoResend(async () => sendAction.mock.calls.length >= 2) + + const recovered = await latestExecution(service, created.data!.id) + expect(recovered).toMatchObject({ + status: 'success', + retryCount: 1, + sendStatus: 'success', + notificationStatus: 'sent' + }) + expect(sendAction).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + target: '研发群@chatroom', + filePath: '/tmp/saved.png', + executionId: first.id, + triggerType: 'scheduled', + retryCount: 1, + taskId: created.data!.id + }) + ) + expect(sendNotification).toHaveBeenCalledTimes(1) + expect(await service.listNotifications()).toEqual([ + expect.objectContaining({ + type: 'recovery', + status: 'sent', + title: expect.stringContaining('已恢复') + }) + ]) + }) + + it('does not auto resend before the backoff elapses', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-resend-early-')) + await enableNotifications(storageDir) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + const sendAction = vi.fn(unavailableSendAction()) + const sendNotification = vi.fn() + const service = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction, sendNotification }) + }) + const created = await service.createTask({ + name: '退避未到期日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + const first = await runScheduled(service, created.data!.id) + expect(first).toMatchObject({ status: 'waiting_to_send', retryCount: 0 }) + + // 仅推进 1 分钟，尚未达到 5 分钟退避。 + clock.now = Date.parse(first.finishedAt || first.startedAt) + 60_000 + await service.tick(new Date(clock.now)) + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(sendAction).toHaveBeenCalledTimes(1) + expect(await latestExecution(service, created.data!.id)).toMatchObject({ + status: 'waiting_to_send', + retryCount: 0 + }) + }) + + it('stops auto resending after the attempt limit but keeps manual retry', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-resend-limit-')) + await enableNotifications(storageDir) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + const sendAction = vi.fn(unavailableSendAction()) + const service = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction }) + }) + const created = await service.createTask({ + name: '重发上限日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + let current = await runScheduled(service, created.data!.id) + expect(current).toMatchObject({ status: 'waiting_to_send', retryCount: 0 }) + + for (let attempt = 1; attempt <= 6; attempt += 1) { + const backoff = Math.min(5 * 60_000 * 2 ** (current.retryCount || 0), 60 * 60_000) + clock.now = Date.parse(current.finishedAt || current.startedAt) + backoff + 1_000 + await service.tick(new Date(clock.now)) + await waitForAutoResend(async () => { + const execution = await latestExecution(service, created.data!.id) + return sendAction.mock.calls.length === attempt + 1 && execution.retryCount === attempt + }) + // 等待后台 promise 从 retrying map 移除，避免影响下一次 tick 判定。 + await new Promise((resolve) => setTimeout(resolve, 20)) + current = await latestExecution(service, created.data!.id) + } + + // 已达 6 次自动重试上限：继续 tick 不再自动发送。 + clock.now += 60 * 60_000 + 1_000 + await service.tick(new Date(clock.now)) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(sendAction).toHaveBeenCalledTimes(7) + expect(await latestExecution(service, created.data!.id)).toMatchObject({ + status: 'waiting_to_send', + retryCount: 6 + }) + + // 手动重试不受自动上限限制。 + const manual = await service.retryScheduledReportSend(current.id) + expect(manual.success).toBe(true) + expect(sendAction).toHaveBeenCalledTimes(8) + expect(manual.data).toMatchObject({ status: 'waiting_to_send', retryCount: 7 }) + }) + + it('skips auto resend when the task is disabled', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-resend-disabled-')) + await enableNotifications(storageDir) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + const sendAction = vi.fn(unavailableSendAction()) + const service = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction }) + }) + const created = await service.createTask({ + name: '任务禁用日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + const first = await runScheduled(service, created.data!.id) + expect(first).toMatchObject({ status: 'waiting_to_send', retryCount: 0 }) + + await service.setTaskEnabled(created.data!.id, false) + clock.now = Date.parse(first.finishedAt || first.startedAt) + 10 * 60_000 + await service.tick(new Date(clock.now)) + await new Promise((resolve) => setTimeout(resolve, 50)) + + expect(sendAction).toHaveBeenCalledTimes(1) + expect(await latestExecution(service, created.data!.id)).toMatchObject({ + status: 'waiting_to_send', + retryCount: 0 + }) + }) + + it('sends an expired execution only once when ticks overlap', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-resend-overlap-')) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + let releaseRetry: ((value: typeof sentAction) => void) | undefined + const retryPending = new Promise((resolve) => { + releaseRetry = resolve + }) + const sendAction = vi + .fn() + .mockResolvedValueOnce(unavailableSendAction()()) + .mockReturnValueOnce(retryPending) + const service = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction }) + }) + const created = await service.createTask({ + name: '并发重发日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + const first = await runScheduled(service, created.data!.id) + const dueAt = Date.parse(first.finishedAt || first.startedAt) + 5 * 60_000 + 1_000 + + const firstTick = service.tick(new Date(dueAt)) + const secondTick = service.tick(new Date(dueAt)) + await waitForAutoResend(async () => sendAction.mock.calls.length === 2) + expect(sendAction).toHaveBeenCalledTimes(2) + + releaseRetry!(sentAction) + await Promise.all([firstTick, secondTick]) + expect(sendAction).toHaveBeenCalledTimes(2) + expect(await latestExecution(service, created.data!.id)).toMatchObject({ + status: 'success', + retryCount: 1 + }) + }) + + it('shares the automatic resend with a concurrent manual retry', async () => { + const storageDir = await mkdtemp( + join(tmpdir(), 'tracememo-scheduled-report-resend-manual-race-') + ) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + let releaseRetry: ((value: typeof sentAction) => void) | undefined + const retryPending = new Promise((resolve) => { + releaseRetry = resolve + }) + const sendAction = vi + .fn() + .mockResolvedValueOnce(unavailableSendAction()()) + .mockReturnValueOnce(retryPending) + const service = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction }) + }) + const created = await service.createTask({ + name: '手动竞争日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + const first = await runScheduled(service, created.data!.id) + const dueAt = Date.parse(first.finishedAt || first.startedAt) + 5 * 60_000 + 1_000 + + const automatic = service.tick(new Date(dueAt)) + await waitForAutoResend(async () => sendAction.mock.calls.length === 2) + const manual = service.retryScheduledReportSend(first.id) + expect(sendAction).toHaveBeenCalledTimes(2) + + releaseRetry!(sentAction) + await automatic + const manualResult = await manual + expect(sendAction).toHaveBeenCalledTimes(2) + expect(manualResult).toMatchObject({ + success: true, + data: { status: 'success', retryCount: 1 } + }) + expect(await latestExecution(service, created.data!.id)).toMatchObject(manualResult.data) + }) + + it('resumes an expired waiting execution after service recreation', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-resend-restart-')) + const clock = { now: Date.parse('2026-08-27T01:00:00.000Z') } + const firstSendAction = vi.fn(unavailableSendAction()) + const firstService = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction: firstSendAction }) + }) + const created = await firstService.createTask({ + name: '重启恢复日报', + group: '研发群', + target: '研发群@chatroom', + scheduleTime: '09:00' + }) + const first = await runScheduled(firstService, created.data!.id) + expect(first.status).toBe('waiting_to_send') + + const resumedSendAction = vi.fn().mockResolvedValue(sentAction) + const secondService = new ScheduledReportService({ + storageDir, + now: () => new Date(clock.now), + ...makeDependencies({ sendAction: resumedSendAction }) + }) + const dueAt = Date.parse(first.finishedAt || first.startedAt) + 5 * 60_000 + 1_000 + await secondService.tick(new Date(dueAt)) + + expect(resumedSendAction).toHaveBeenCalledTimes(1) + expect(await latestExecution(secondService, created.data!.id)).toMatchObject({ + status: 'success', + retryCount: 1, + sendStatus: 'success' + }) + const restored = new ScheduledReportService({ storageDir }) + expect(await latestExecution(restored, created.data!.id)).toMatchObject({ + status: 'success', + retryCount: 1 + }) + }) +}) + +describe('scheduled report filehelper target', () => { + it('builds contact recipients for filehelper and group recipients for chatrooms', () => { + expect( + buildScheduledReportActionRequest({ + target: 'filehelper', + filePath: '/tmp/report.png', + executionId: 'exec-1', + triggerType: 'scheduled', + retryCount: 1, + taskId: 'task-1' + }) + ).toMatchObject({ recipient: { type: 'contact', id: 'filehelper' } }) + expect( + buildScheduledReportActionRequest({ + target: '12345@chatroom', + filePath: '/tmp/report.png', + executionId: 'exec-2', + triggerType: 'scheduled' + }) + ).toMatchObject({ recipient: { type: 'group', id: '12345@chatroom' } }) + }) + + it('sends a waiting report to filehelper when the target is the transfer assistant', async () => { + const storageDir = await mkdtemp(join(tmpdir(), 'tracememo-scheduled-report-filehelper-')) + const sendAction = vi.fn(async () => ({ + actionId: 'send-filehelper', + status: 'failed' as const, + decision: 'allow' as const, + errorCode: 'SEND_CAPABILITY_UNAVAILABLE' as const, + reason: '当前微信发送能力不可用', + startedAt: '2026-08-27T01:00:00.000Z', + finishedAt: '2026-08-27T01:00:01.000Z' + })) + const service = new ScheduledReportService({ + storageDir, + ...makeDependencies({ sendAction }) + }) + const created = await service.createTask({ + name: '文件传输助手日报', + group: '文件传输助手', + target: 'filehelper', + scheduleTime: '09:00' + }) + + const execution = await runScheduled(service, created.data!.id) + + expect(execution).toMatchObject({ + status: 'waiting_to_send', + sendTarget: 'filehelper', + pngPath: '/tmp/saved.png' + }) + expect(sendAction).toHaveBeenCalledWith( + expect.objectContaining({ target: 'filehelper', filePath: '/tmp/saved.png' }) + ) + }) +})