Compare commits

...
382 Commits
Author SHA1 Message Date
whyour 0118be41f9 test(ql3): isolate unavailable capture tools under noexec mounts 2026-09-04 05:45:57 +08:00
whyour efb7379fa9 fix(ql3): stream sparse process logs within exact byte quotas 2026-09-04 05:32:25 +08:00
whyour ed4ba9b817 fix(ci): retain audited local cancellation gate marker 2026-09-04 04:59:48 +08:00
whyour eaf1370cd5 test(ci): remove minute-boundary race from scheduler replica fixture 2026-09-04 04:51:52 +08:00
whyour e684a6e64a test(ql3): exercise actual panel client in Linux runtime gate 2026-09-04 04:49:45 +08:00
whyour e4ba5d405f feat(ql3): manage canonical task runs from the existing panel 2026-09-04 04:27:42 +08:00
whyour 2005cb6ba9 fix(ci): ignore terminating CloudNativePG rollout remnants 2026-09-04 04:21:17 +08:00
whyour 03afd7e899 fix(ql3): isolate native console asynchronous sessions 2026-09-04 03:56:18 +08:00
whyour 9708a37154 docs(ql3): record delivered native log navigation artifacts 2026-09-04 03:31:51 +08:00
whyour ce8c3a7d2a feat(ql3): paginate native task trigger and run ledgers 2026-09-04 03:23:21 +08:00
whyour 8e1f1ed7ae docs(ql3): distinguish delivered console features from candidates 2026-09-04 03:02:14 +08:00
whyour 12ed38b7d7 feat(ql3): navigate bounded native console log windows 2026-09-04 02:52:27 +08:00
whyour 7016bb91b0 docs(ql3): record delivered canonical log bridge milestone 2026-09-04 02:37:19 +08:00
whyour 09ef174522 feat(ql3): bridge legacy panel to bounded run logs 2026-09-04 01:55:44 +08:00
whyour fa730da091 docs(ql3): align local milestone schema guidance 2026-09-02 14:13:04 +08:00
whyour a1b21443ba test(ql3): harden local console trial on Docker Desktop 2026-09-02 13:35:13 +08:00
whyour 1b223ff2ad feat(ql3): ship bounded legacy panel console 2026-09-02 13:34:46 +08:00
whyour ca41640794 test(ql3): sync package boundary snapshot 2026-09-02 11:41:43 +08:00
whyour 882ce07d7e feat(ql3): add bounded panel bootstrap 2026-09-02 11:30:42 +08:00
whyour 83966a1287 fix(ql3): seal ordered reconciliation evidence 2026-09-02 08:45:43 +08:00
whyour cbdddffc91 fix(ql3): bind evolved reconciliation target 2026-09-02 08:19:44 +08:00
whyour a8ca739fe8 chore(ql3): add opt-in deployment diagnostics 2026-09-02 07:54:30 +08:00
whyour f76e5f16da fix(ql3): bound secret config auth namespaces 2026-09-02 07:18:07 +08:00
whyour 53672c468b fix(ql3): make completion fixture migration-ready 2026-09-02 06:46:36 +08:00
whyour 3df73261ab chore(ql3): expose reconciliation plan summary 2026-09-02 06:31:19 +08:00
whyour 15d970be41 feat(ql3): add bounded legacy panel cron adapter 2026-09-02 06:18:44 +08:00
whyour d6595912a2 fix(ql3): admit secret planning snapshot authority 2026-09-02 05:45:18 +08:00
whyour f8ec9fccec fix(ql3): bind secret planning to automation target 2026-09-02 05:35:39 +08:00
whyour 1cd298ebaf fix(ql3): classify retained legacy target tables 2026-09-02 04:33:08 +08:00
whyour 6717a3dfa6 fix(ql3): preserve legacy running instances 2026-09-02 04:06:13 +08:00
whyour 950f638998 fix(ql3): admit external secret reconciliation review 2026-09-02 03:25:19 +08:00
whyour a95ed67fe8 feat(ql3): rehearse cross-domain reconciliation completion 2026-09-02 02:37:39 +08:00
whyour 37801d2f30 docs(ql3): record automation rollback artifact 2026-09-02 00:25:43 +08:00
whyour c8d9eed95d test(ql3): isolate legacy API compatibility fixture 2026-09-01 23:35:42 +08:00
whyour 8580b25ecc feat(ql3): rehearse reviewed automation rollback 2026-09-01 23:20:13 +08:00
whyour 7ed7a08211 docs(ql3): record reconciliation artifact delivery 2026-09-01 19:02:44 +08:00
whyour 0235973c9b fix(ql3): preserve adopted baseline during capture 2026-09-01 12:18:28 +08:00
whyour 367ec49e4a feat(ql3): capture post-cutover reconciliation evidence 2026-09-01 11:46:40 +08:00
whyour 45879ef0a7 docs(ql3): record exact Console cutover artifact 2026-09-01 11:04:38 +08:00
whyour 229c3cb4e8 fix(ql3): align Local API artifact CLI contract 2026-09-01 10:20:05 +08:00
whyour b3421ac9cf feat(ql3): prove Console adopted cutover entry 2026-09-01 10:12:32 +08:00
whyour 50e6742b72 docs(ql3): record exact cutover alpha milestone 2026-08-31 02:15:30 +08:00
whyour 79045a0d43 refactor(ql3): keep application CLI boundary thin 2026-08-31 01:43:41 +08:00
whyour a76618e176 test(ql3): authorize read-only cutover probe boundary 2026-08-31 01:32:28 +08:00
whyour 2215a29afb test(ql3): audit cutover probe executable contract 2026-08-31 01:15:24 +08:00
whyour fe8dfc3379 feat(ql3): add frozen-admission cutover probe 2026-08-31 01:07:14 +08:00
whyour 652653c7f6 test(ql3): compact cutover evidence output 2026-08-31 00:50:14 +08:00
whyour c0a4901186 test(ql3): expose cutover reconciliation evidence 2026-08-31 00:40:28 +08:00
whyour b172ff5e02 test(ql3): expose cutover reconciliation result 2026-08-31 00:25:30 +08:00
whyour 4b0241f992 fix(ql3): require read-only legacy target mount 2026-08-31 00:09:41 +08:00
whyour 32c0143f5d fix(ql3): align legacy stop cutover request 2026-08-30 23:52:49 +08:00
whyour a03c94dc05 fix(ql3): canonicalize cutover docker socket 2026-08-30 23:33:59 +08:00
whyour 1b6523e93c fix(ql3): preserve transform no-replace boundary 2026-08-30 23:12:08 +08:00
whyour 6edf6ff3c5 fix(ql3): bind upgrade plans to canonical paths 2026-08-30 22:46:51 +08:00
whyour 93b45ca62d feat(ql3): rehearse exact local upgrade cutover 2026-08-30 22:05:04 +08:00
whyour 784e77a971 fix(ql3): bind post-apply target baseline 2026-08-30 21:11:47 +08:00
whyour 419e1bc141 test(ql3): update owner cli boundary snapshot 2026-08-30 20:07:54 +08:00
whyour 0f4b6b43fb feat(ql3): bind offline adopted target images 2026-08-30 19:50:24 +08:00
whyour 38d85952e7 docs(ql3): record upgrade staging artifacts 2026-08-30 18:51:43 +08:00
whyour 7a8acacb6c feat(ql3): add reviewed upgrade staging 2026-08-30 18:14:30 +08:00
whyour 157b1d9afd docs(ql3): record D-425 alpha artifacts 2026-08-30 17:22:47 +08:00
whyour d6571e4b89 feat(ql3): add downloadable upgrade readiness 2026-08-30 14:00:26 +08:00
whyour 6e5bfccdfc docs(ql3): record full alpha stage delivery 2026-08-30 11:16:05 +08:00
whyour 97333da34c test(ql3): bound provider convergence retries 2026-08-30 01:22:55 +08:00
whyour be7a3fea41 docs(ql3): record headless alpha milestone 2026-08-29 23:48:13 +08:00
whyour d459c3b45c docs(ql3): record D-424 alpha milestone 2026-08-29 22:26:01 +08:00
whyour f46fb44ac9 feat(ql3): add secret-backed console automation 2026-08-29 19:44:04 +08:00
whyour 53d65abd8d docs(ql3): record D-423 console milestone 2026-08-29 18:01:43 +08:00
whyour b970e2aede feat(ql3): add secure console cron scheduling 2026-08-29 17:13:09 +08:00
whyour 951de26ffd docs(ql3): record D-422 console milestone 2026-08-29 14:13:03 +08:00
whyour f28bf74d1b test(ql3): refresh local api boundary snapshot 2026-08-29 13:27:05 +08:00
whyour 6239c4d698 feat(ql3): add secure console task editing 2026-08-29 13:18:43 +08:00
whyour 9089c5a8e6 docs(ql3): record console alpha milestone 2026-08-29 11:27:25 +08:00
whyour dc1686bd6f fix(ql3): keep console task authority lazy 2026-08-29 10:41:36 +08:00
whyour 884912d1c8 feat(ql3): add request-scoped console task creation 2026-08-29 03:51:35 +08:00
whyour 53b7072370 docs(ql3): record first usable console alpha 2026-08-28 21:40:34 +08:00
whyour 37abfa160d fix(ql3): install alpha finalizer dependencies 2026-08-28 20:47:48 +08:00
whyour b80dc56398 docs(ql3): record D-420 acceptance evidence 2026-08-28 16:30:58 +08:00
whyour 57953ec80b feat(ql3): surface bounded console run logs 2026-08-28 12:27:19 +08:00
whyour 920a31fb66 fix(ql3): ship local process launcher asset 2026-08-28 08:31:14 +08:00
whyour ce4284ff76 fix(ql3): stabilize first automation live gate 2026-08-28 08:17:33 +08:00
whyour 6342e11a10 feat(ql3): prove first usable automation journey 2026-08-28 08:00:11 +08:00
whyour c60ab7e48a feat(ql3): ship selectable local console trial kit 2026-08-28 06:50:37 +08:00
whyour 84d1a61372 feat(ql3): add bounded local web console 2026-08-28 05:21:43 +08:00
whyour b7a8104d9f feat(ql3): ship runnable local alpha quickstart 2026-08-28 03:56:54 +08:00
whyour 82d74cf56f feat(ql3): close cross-profile alpha stage index 2026-08-28 01:07:04 +08:00
whyour 07d1dd97c1 feat(ql3): close cluster alpha milestone 2026-08-28 00:25:04 +08:00
whyour a7308c78f9 feat(ql3): close local alpha milestone 2026-08-27 20:44:33 +08:00
whyour 78c261b556 feat(ql3): close public local release pair 2026-08-27 14:51:33 +08:00
whyour 238df17fdf fix(ql3): bind alpha artifacts to CI evidence 2026-08-27 06:57:08 +08:00
whyour 4239464af6 fix(ql3): patch Alpine OpenSSL runtime 2026-08-27 00:41:33 +08:00
whyour 218521994a docs(ql3): record arm64 trial kit evidence 2026-08-26 15:42:34 +08:00
whyour 2620be0587 feat(ql3): materialize auditable local trial kits 2026-08-26 09:58:29 +08:00
whyour 2253b99066 feat(ql3): add downloadable local alpha operator 2026-08-26 09:20:23 +08:00
whyour 3731ae6051 docs(ql3): record alpha.2 candidate evidence 2026-08-26 06:19:20 +08:00
whyour e3c05862b8 chore(ql3): advance release identity to alpha.2 2026-08-26 05:56:38 +08:00
whyour b48be7f058 docs(ql3): record D-407 live acceptance 2026-08-26 05:19:28 +08:00
whyour f8934b401d fix(ql3): project cnpg runtime trust into private files 2026-08-26 04:58:09 +08:00
whyour eb34662d01 fix(ql3): preserve materialization in live fixtures 2026-08-26 04:32:53 +08:00
whyour beb490c48c fix(ql3): materialize projected runtime files 2026-08-26 04:09:47 +08:00
whyour 9c58b15d2b test(ql3): report control rollout failure facts 2026-08-26 03:48:12 +08:00
whyour ac0992081e feat(ql3): validate cluster credential key rotation 2026-08-26 03:31:13 +08:00
whyour b45a5e04b7 fix(ql3): align live control fixture keyring 2026-08-26 02:08:27 +08:00
whyour 2c107d1e9d fix(ql3): align HA fixture with pepper keyring 2026-08-26 01:59:41 +08:00
whyour 07cdc76bae feat(ql3): cut alpha.1 candidate milestone 2026-08-26 01:41:20 +08:00
whyour c2df0c7215 ci(ql3): wait for managed postgres resources 2026-08-25 23:38:21 +08:00
whyour d2ebce85fa ci(ql3): close network policy startup race 2026-08-25 22:38:40 +08:00
whyour 309694ebde ci(ql3): load bridge netfilter for nested k3s 2026-08-25 10:15:32 +08:00
whyour a027c92f26 ci(ql3): enable bridge netfilter for security live 2026-08-25 10:01:13 +08:00
whyour 9afcf9413c ci(ql3): expose content-free migration failure facts 2026-08-25 09:51:00 +08:00
whyour 82348e9f91 ci(ql3): build live helper dependency closure 2026-08-25 09:28:38 +08:00
whyour 52ebc850c8 ci(ql3): build postgres contract before live tests 2026-08-25 09:24:35 +08:00
whyour 02a5a4712d ci(ql3): run security administration gate on next 2026-08-25 09:20:34 +08:00
whyour c6b9e1504c feat(ql3): prove security administration custody live 2026-08-25 09:15:40 +08:00
whyour 2eb8b29e51 fix(ql3): tolerate provider readiness convergence 2026-08-25 03:30:34 +08:00
whyour 931f9eee38 feat(ql3): add opt-in security administration job 2026-08-25 03:15:37 +08:00
whyour fbedae0116 test(ql3): harden resource rollback CI watchdog 2026-08-25 02:18:58 +08:00
whyour 7b3d49acda feat(ql3): add short-lived security administration command 2026-08-25 02:09:06 +08:00
whyour 9c736f0943 fix(ci): make Vault live fixture capability-free 2026-08-25 00:31:55 +08:00
whyour b0ab255a68 fix(ci): build Vault adapter workspace closure 2026-08-25 00:22:18 +08:00
whyour 686873f8ec feat(ql3): add direct Vault KV worker secret custody 2026-08-25 00:16:59 +08:00
whyour a64d08bda1 test(ql3): prove legacy env replay across promotion 2026-08-24 22:33:15 +08:00
whyour 4cf806b4b2 fix(ql3): preserve automation manager least privilege 2026-08-24 21:47:43 +08:00
whyour bd8678a888 feat(ql3): atomically apply cluster legacy env migration 2026-08-24 21:13:15 +08:00
whyour 4abf125ce9 feat(ql3): add opaque cluster environment bundle delivery 2026-08-24 19:31:03 +08:00
whyour cf2c0ec7b3 test(ql3): tolerate initial postgres cron boundary 2026-08-24 18:27:53 +08:00
whyour 784d9b21a0 feat(ql3): add cluster legacy env migration plan ledger 2026-08-24 18:05:06 +08:00
whyour 54056e8bab fix(ci): build workspace before mounted secret gate 2026-08-24 16:24:30 +08:00
whyour b731b434f7 feat(ql3): prove mounted secret provider rotation 2026-08-24 16:19:51 +08:00
whyour fd10f72aba fix(ql3): preserve target-native reconciliation domains 2026-08-24 13:55:54 +08:00
whyour d94ca02e0d feat(ql3): add secret config completion evidence 2026-08-24 11:54:25 +08:00
whyour 8ad8ed96bd feat(ql3): complete local secret config apply lifecycle 2026-08-24 10:38:04 +08:00
whyour 7142395897 fix(ci): align local OCI contract v52 2026-08-24 01:48:27 +08:00
whyour f822e92b02 feat(local): publish secret config application atomically 2026-08-24 00:11:48 +08:00
whyour e261d4130d fix(ci): bind plugin recovery completion evidence 2026-08-23 20:46:05 +08:00
whyour 45f8218247 fix(ci): align local image contract v51 2026-08-23 20:33:40 +08:00
whyour 31a9bcbe83 feat(local): bind legacy adoption provenance 2026-08-23 20:26:01 +08:00
whyour 74ae891070 feat(local): authorize secret config reconciliation decisions 2026-08-23 18:52:10 +08:00
whyour 5dd8678d7e feat(local): publish secret config reconciliation plans 2026-08-23 17:27:01 +08:00
whyour fbb67ff5c8 feat(local): plan secret config reconciliation 2026-08-23 16:13:47 +08:00
whyour dbd219bedc feat(local): classify legacy secret config migration 2026-08-23 15:29:00 +08:00
whyour f2b198ff05 docs(rfc): accept run history preservation 2026-08-23 14:47:32 +08:00
whyour 6001174e23 fix(ci): bind provider evidence to rollout generation 2026-08-23 14:29:19 +08:00
whyour 85e244f602 test(postgres): stabilize expired claim takeover 2026-08-23 13:59:41 +08:00
whyour 42262094cc feat(local): preserve terminal run history 2026-08-23 13:30:18 +08:00
whyour 5f62f2f724 feat(local): bind service completion restart 2026-08-23 00:34:46 +08:00
whyour 4d71dfb937 feat(local): fence reconciliation completion 2026-08-22 22:36:50 +08:00
whyour ab589e9ecb feat(local): seal reconciliation apply storage 2026-08-22 21:39:10 +08:00
whyour 4541d142d3 fix(ci): authorize reconciliation apply boundaries 2026-08-22 20:48:22 +08:00
whyour 546b02a0bc feat(local): recover reconciliation automation apply 2026-08-22 20:41:54 +08:00
whyour 2af3ccc158 feat(local): apply signed automation decisions atomically 2026-08-22 19:34:16 +08:00
whyour e6089fb77c docs(rfc): record reconciliation decision gate 2026-08-22 19:24:37 +08:00
whyour a82b7fb98b fix(test): use database clock for claim takeover 2026-08-22 19:05:54 +08:00
whyour 2638df29c8 fix(ci): refresh ql3 package boundary snapshot 2026-08-22 18:55:34 +08:00
whyour e6615c7088 feat(local): authorize reconciliation automation decisions 2026-08-22 18:01:24 +08:00
whyour 35efa1f33e fix(ci): wait for live service readiness 2026-08-22 17:32:47 +08:00
whyour 1250555881 fix(ci): audit recovery image roles 2026-08-22 17:12:20 +08:00
whyour 9bf3472c14 fix(ci): count both OCI recovery reads 2026-08-22 17:02:20 +08:00
whyour d55933d781 fix(ci): track restarted provider evidence 2026-08-22 16:42:36 +08:00
whyour a1549b3a24 fix(ci): bind recovery evidence to migration stream 2026-08-22 16:28:10 +08:00
whyour 3e0fc8b46f fix(ci): bind provider evidence to ready pod 2026-08-22 16:19:19 +08:00
whyour 42c1bfbb56 fix(ci): align recovery gate with durable admission 2026-08-22 16:16:16 +08:00
whyour eaf9bf486a fix(ci): seed durable package trust in recovery gate 2026-08-22 15:28:09 +08:00
whyour 47118e13fe fix(ci): close remaining clean-runner gaps 2026-08-22 15:09:58 +08:00
whyour 3fb02f4377 fix(ci): refresh local profile envelopes 2026-08-22 14:36:00 +08:00
whyour c0cd94f50d fix(ci): align kubernetes live evidence 2026-08-22 14:34:39 +08:00
whyour 5efaca3d5a test(postgres): isolate backend fault injection 2026-08-22 14:26:57 +08:00
whyour 66668ffdcd fix(ci): preserve console capacity helper input 2026-08-22 14:11:35 +08:00
whyour c8ad5ea368 fix(ci): restore ql3 multi-profile gates 2026-08-22 13:44:19 +08:00
whyour f1cca9a3ed fix(images): minimize ql3 runtime base 2026-08-22 12:44:49 +08:00
whyour ca8ddf34b8 fix(ci): close clean runner gaps 2026-08-22 12:17:11 +08:00
whyour 1a4acaa56f fix(ci): make ql3 gates reproducible 2026-08-22 11:51:17 +08:00
whyour 1834c00ff7 feat(ql3): plan automation reconciliation rows 2026-08-22 00:26:04 +08:00
whyour c218f71250 feat(ql3): plan reconciliation application 2026-08-21 20:24:41 +08:00
whyour 22f58b1eff feat(ql3): authorize reconciliation reviews 2026-08-21 19:34:29 +08:00
whyour 827ba5294f feat(ql3): prepare private reconciliation reviews 2026-08-21 18:35:28 +08:00
whyour 473a535815 docs(ql3): freeze private reconciliation review 2026-08-21 18:13:42 +08:00
whyour 46a7dc7496 feat(ql3): plan sealed reconciliation bundles 2026-08-21 18:06:27 +08:00
whyour 4012f54a50 feat(ql3): seal reconciliation capture bundles 2026-08-21 17:20:06 +08:00
whyour f527d83c42 docs(ql3): freeze sealed reconciliation planning 2026-08-21 17:02:12 +08:00
whyour c341a92a18 feat(ql3): capture stopped reconciliation bundles 2026-08-21 16:52:22 +08:00
whyour a789c4a4d6 feat(ql3): fence reconciliation capture preparation 2026-08-21 15:54:49 +08:00
whyour 1bf14f807c docs(ql3): freeze stopped reconciliation capture 2026-08-21 15:40:59 +08:00
whyour b714f2bf18 fix(ql3): close adopted lineage acceptance gates 2026-08-21 15:35:48 +08:00
whyour 4f6be5bc2c feat(ql3): preserve adopted compose lineage 2026-08-21 15:18:07 +08:00
whyour 190117a662 feat(ql3): prepare adopted deployment bundles 2026-08-21 14:09:59 +08:00
whyour 07cb5ea0a8 feat(ql3): bind service lineage to legacy data receipt 2026-08-21 13:27:40 +08:00
whyour f1fc316be1 feat(ql3): gate startup on legacy data receipt 2026-08-21 12:49:45 +08:00
whyour 3a725cc8e8 docs(ql3): freeze legacy deployment lineage contract 2026-08-21 12:24:14 +08:00
whyour 744a948de1 docs(ql3): record legacy application gates 2026-08-21 12:09:19 +08:00
whyour d4813e48a4 feat(ql3): apply legacy data transformation 2026-08-21 12:06:41 +08:00
whyour 69c322fa8d feat(ql3): prepare legacy data transformation 2026-08-21 10:56:24 +08:00
whyour 19bb09faa3 feat(ql3): stage legacy data directory 2026-08-21 10:09:34 +08:00
whyour 878a360b09 feat(ql3): inventory legacy data directory 2026-08-21 01:34:18 +08:00
whyour c9e41812cb feat(ql3): rehearse legacy sqlite upgrade 2026-08-21 00:52:58 +08:00
whyour 4ea156f189 fix(ql3): harden legacy api compatibility 2026-08-20 19:09:21 +08:00
whyour 3c02ae2020 feat(ql3): prove legacy rollback readiness 2026-08-20 18:24:52 +08:00
whyour 2cd6ea1d55 feat(ql3): commit service manager legacy rollback 2026-08-20 17:40:29 +08:00
whyour ad291e514d feat(ql3): prepare service manager legacy rollback 2026-08-20 16:34:30 +08:00
whyour 0c503932cc test(ql3): lock legacy core api contracts 2026-08-20 15:41:39 +08:00
whyour c0ef9a969e feat(ql3): fence console capabilities by session 2026-08-20 15:15:42 +08:00
whyour ffa4b4e7cb feat(ql3): observe package installations in console 2026-08-20 14:42:54 +08:00
whyour 344680d64a feat(ql3): expose worker observations in console 2026-08-20 14:02:01 +08:00
whyour af5d5bfc0b feat(ql3): add generic worker management entry 2026-08-20 13:30:06 +08:00
whyour 4a4fa85f51 feat(ql3): expose bounded worker session observation 2026-08-20 12:55:44 +08:00
whyour 6265e31dce feat(ql3): version worker support tier admission 2026-08-20 12:02:15 +08:00
whyour 6e11258d45 feat(ql3): enforce architecture support tiers 2026-08-20 11:02:17 +08:00
whyour 3979707f6d feat(ql3): gate console capacity evidence 2026-08-20 10:04:21 +08:00
whyour 0a5f1448f1 feat(ql3): add optional console run drilldown 2026-08-20 09:23:19 +08:00
whyour cf21e984cb feat(ql3): page blocked cancellations 2026-08-19 23:51:14 +08:00
whyour 095683a4cb feat(ql3): expose cancellation status card 2026-08-19 09:21:50 +08:00
whyour 5a979e510e feat(ql3): expose cancellation availability summary 2026-08-19 08:29:45 +08:00
whyour 5261c41828 feat(ql3): operationalize cancellation rearm 2026-08-19 07:54:02 +08:00
whyour 0b5f3bcb39 feat(ql3): gate remote cancellation delivery 2026-08-19 06:55:36 +08:00
whyour 1809fbb8d3 feat(ql3): add postgres cancellation dispatch 2026-08-19 06:24:11 +08:00
whyour 36035ac43e feat(ql3): persist manual primary activation 2026-08-19 05:13:58 +08:00
whyour 9981f47851 feat(ql3): operationalize manual primary canary 2026-08-19 04:29:15 +08:00
whyour be98d6753b feat(ql3): gate primary on shadow capture evidence 2026-08-19 03:27:32 +08:00
whyour c60a86d1bf feat(ql3): prove shadow resource rollback 2026-08-19 02:36:06 +08:00
whyour b739d92347 feat(ql3): audit legacy shadow terminal windows 2026-08-19 00:07:04 +08:00
whyour d38f624262 feat(ql3): report legacy shadow startup differences 2026-08-18 09:58:16 +08:00
whyour 6cb9a075d7 feat(ql3): reconcile legacy shadow runs on startup 2026-08-18 09:26:12 +08:00
whyour 1ff5aed844 feat(ql3): observe boot runs without false origins 2026-08-18 06:54:25 +08:00
whyour 0ad96d38a6 feat(ql3): observe system crond runs idempotently 2026-08-18 06:33:23 +08:00
whyour 6831ea3de5 feat(ql3): observe legacy schedule service runs 2026-08-18 05:57:19 +08:00
whyour ca19423f1d feat(ql3): validate release tags before registry mutation 2026-08-18 05:23:48 +08:00
whyour 1c4e3269b4 feat(ql3): gate release tags on deployment readiness 2026-08-18 04:54:10 +08:00
whyour a99f1508f7 feat(ql3): close release tags after catalog verification 2026-08-18 03:53:25 +08:00
whyour dbd57b27d0 feat(ql3): prevent release catalog tag overwrite 2026-08-18 03:16:39 +08:00
whyour 7ac5e678ba feat(ql3): revalidate release evidence at closure 2026-08-18 02:41:41 +08:00
whyour cc6d6b7b48 feat(ql3): make release evidence replay deterministic 2026-08-18 02:17:26 +08:00
whyour 3e7bfa8087 feat(ql3): bind private release evidence receipts 2026-08-18 01:48:27 +08:00
whyour fd433c41cd feat(ql3): gate local releases through published catalogs 2026-08-18 00:56:12 +08:00
whyour 9cd42ff454 feat(ql3): gate cluster releases through published catalogs 2026-08-18 00:24:13 +08:00
whyour 8496b56bed feat(ql3): fence Kubernetes resource retirement 2026-08-17 23:45:34 +08:00
whyour 2e848ef7c0 feat(ql3): fence cluster deployment transitions 2026-08-17 01:41:10 +08:00
whyour abf2d25d63 feat(ql3): add catalog-bound cluster deployment ceremony 2026-08-16 17:12:37 +08:00
whyour dce72c800f feat(ql3): bind local compose revisions to release catalogs 2026-08-16 16:02:53 +08:00
whyour 2ccdd71851 feat(ql3): bind deployment locks to release catalogs 2026-08-16 15:08:15 +08:00
whyour 00a65360cf feat(ql3): verify release catalog consumption 2026-08-16 14:28:54 +08:00
whyour 8c09850249 feat(ql3): materialize offline deployment locks 2026-08-16 13:39:50 +08:00
whyour a44c213be1 feat(ql3): publish durable release catalog 2026-08-16 10:50:15 +08:00
whyour 41549524d4 feat(ql3): publish complete release sets 2026-08-16 10:06:34 +08:00
whyour bad8399cc3 feat(ql3): govern release version transitions 2026-08-16 09:32:27 +08:00
whyour 7130d77a76 feat(ql3): scope deployment-family release candidates 2026-08-16 08:54:51 +08:00
whyour d991deb7ee feat(ql3): add external release workstation ceremony 2026-08-16 08:17:11 +08:00
whyour 33b9eb2f5e feat(ql3): verify cluster evidence offline 2026-08-16 07:40:55 +08:00
whyour 4f62a86d15 feat(ql3): export redacted cluster evidence bundle 2026-08-16 07:07:32 +08:00
whyour 7955d55629 feat(ql3): add explicit cluster observation console 2026-08-16 06:09:54 +08:00
whyour fba8dfb602 feat(ql3): distribute copilot console via signed admin image 2026-08-16 05:14:17 +08:00
whyour c4a1238a92 feat(ql3): add read-only cluster copilot console 2026-08-16 04:35:28 +08:00
whyour da4e7cf688 feat(ql3): harden copilot mcp host deployment 2026-08-16 03:41:38 +08:00
whyour 58025ede55 feat(ql3): add bounded copilot mcp surface 2026-08-16 03:04:12 +08:00
whyour 0e85cbdeb6 feat(ql3): add bounded copilot product client 2026-08-16 02:15:21 +08:00
whyour e5a3d83a8d feat(ql3): add fenced copilot diagnosis cancellation 2026-08-16 01:34:55 +08:00
whyour f77b19e615 feat(ql3): expose copilot diagnosis read model 2026-08-16 00:41:34 +08:00
whyour 774af4a4c9 feat(ql3): expose fenced copilot diagnosis API 2026-08-15 23:45:13 +08:00
whyour 0a31c3364b feat(ql3): terminalize copilot diagnosis failures 2026-08-15 23:16:26 +08:00
whyour 5fc70010f2 feat(ql3): compose cluster copilot diagnosis 2026-08-15 21:59:59 +08:00
whyour c3bd6d40bb feat(ql3): add copilot diagnosis output keyring 2026-08-15 21:04:21 +08:00
whyour 453a7679b2 feat(ql3): execute copilot diagnosis models 2026-08-15 20:38:17 +08:00
whyour 9c90c6ce82 feat(ql3): execute copilot diagnosis tools 2026-08-15 16:39:19 +08:00
whyour 0163c94be7 feat(ql3): admit copilot diagnosis runs 2026-08-15 15:07:49 +08:00
whyour aae96390cc feat(ql3): add cluster tool result key authority 2026-08-15 00:40:56 +08:00
whyour 93690dd921 feat(ql3): add bounded failure diagnosis prompt 2026-08-14 21:00:39 +08:00
whyour 82adb9ace2 feat(ql3): expose redacted log tails over local mcp 2026-08-14 18:59:29 +08:00
whyour fef0fe2bd1 feat(ql3): add bounded redacted log tail tool 2026-08-14 18:03:18 +08:00
whyour 828370c60b feat(ql3): compare bounded task run outcomes 2026-08-14 17:18:25 +08:00
whyour a55613afaa feat(ql3): expose run comparison over local mcp 2026-08-14 16:23:22 +08:00
whyour 3b0aa97303 feat(ql3): add bounded trusted run comparison 2026-08-14 15:58:14 +08:00
whyour 1d83d1d6bb feat(ql3): attest physical edge release archives 2026-08-14 15:29:33 +08:00
whyour 51145a8028 test(ql3): qualify failed upgrades on edge devices 2026-08-14 14:53:44 +08:00
whyour 1c75ac0f45 test(ql3): audit plugin recovery evidence 2026-08-14 14:24:29 +08:00
whyour 2485d61e8a test(ql3): gate failed plugin upgrades before activation 2026-08-14 13:57:16 +08:00
whyour 056eebef94 feat(ql3): qualify plugin upgrades before activation 2026-08-14 13:01:26 +08:00
whyour 3ca9901055 feat(ql3): resolve secret action recovery manually 2026-08-14 12:11:50 +08:00
whyour d8489d8a0b feat(ql3): recover terminal secret action jobs 2026-08-14 10:48:37 +08:00
whyour 7f5339e047 feat(ql3): admit plugin secret action jobs 2026-08-14 10:22:14 +08:00
whyour 75dbe7a4c4 feat(ql3): isolate plugin secret action execution 2026-08-14 01:49:54 +08:00
whyour 53c150c4ec test(ql3): prove kubernetes secret revoke rollout 2026-08-14 01:08:18 +08:00
whyour 9d7431c2cd feat(ql3): project plugin secrets into kubernetes 2026-08-14 00:34:49 +08:00
whyour d41c821ce0 feat(ql3): separate cluster secret transition authority 2026-08-13 22:56:02 +08:00
whyour 5f93e0b5d2 feat(ql3): persist cluster secret transition receipts 2026-08-13 21:48:57 +08:00
whyour c877147d0f feat(ql3): gate local secret transitions before activation 2026-08-13 20:29:47 +08:00
whyour 326783ff14 feat(ql3): enforce staged secret binding persistence 2026-08-13 19:20:39 +08:00
whyour f111d8b10c fix(ql3): allow transitions without prior secret binding 2026-08-13 18:20:14 +08:00
whyour d8c4503adf feat(ql3): define secret binding generation transitions 2026-08-13 18:12:57 +08:00
whyour 22eea3ac71 test(ql3): prove Kubernetes secret binding projection 2026-08-13 17:44:05 +08:00
whyour 73eeaed4de feat(ql3): expose cluster secret binding management 2026-08-13 15:41:59 +08:00
whyour 7016903fba feat(ql3): complete cluster secret binding authority 2026-08-13 14:23:41 +08:00
whyour 56d06bd6cc test(ql3): fail closed on PostgreSQL HA gates 2026-08-13 12:45:45 +08:00
whyour 3a58e1508e feat(ql3): persist package secret approval plans 2026-08-13 12:43:43 +08:00
whyour e43fd607aa feat(ql3): define package secret approval plans 2026-08-13 11:54:18 +08:00
whyour 5e70b89f55 feat(ql3): harden cluster secret projection checks 2026-08-13 11:45:42 +08:00
whyour 8ba27dc148 feat(ql3): add owner-confirmed package secret binding 2026-08-13 11:28:06 +08:00
whyour d72631a2b0 feat(ql3): materialize package secret bindings 2026-08-13 10:31:06 +08:00
whyour 55ef085a6e feat(ql3): bind package secrets to generations 2026-08-13 04:49:19 +08:00
whyour 4c2a0b6adf feat(ql3): atomically withdraw quarantined automation 2026-08-13 03:42:38 +08:00
whyour a6ad636251 feat(ql3): add read-only cluster context readiness 2026-08-13 02:14:17 +08:00
whyour a83b4c5e8d feat(ql3): add offline cluster context preflight 2026-08-13 01:19:23 +08:00
whyour ea3bbe35c3 feat(ql3): add owner-private cluster operator context 2026-08-13 00:08:42 +08:00
whyour 2934123155 feat(ql3): add unified cluster operator cli 2026-08-12 21:42:41 +08:00
whyour bd0621c9f2 test(ql3): accept local cancellation live gate 2026-08-12 20:22:41 +08:00
whyour d85cd88c2d test(ql3): accept cluster run management live gate 2026-08-12 18:47:36 +08:00
whyour 5c89e3bc60 feat(ql3): add cluster run management live gate 2026-08-12 11:51:14 +08:00
whyour 86eb0f1eb3 feat(ql3): add strong local run stop 2026-08-12 10:36:38 +08:00
whyour dd370b2842 feat(ql3): add strong cluster run stop 2026-08-12 08:21:28 +08:00
whyour c0ab62e64a feat(ql3): add strong cluster run management 2026-08-12 07:28:43 +08:00
whyour e38b143dbb feat(ql3): add cluster manual run retry authority 2026-08-12 06:30:52 +08:00
whyour 6b7f8913b5 feat(ql3): add strong local manual run retry 2026-08-12 05:52:18 +08:00
whyour 42e8992818 feat(ql3): reconcile local lost run retries 2026-08-12 05:11:46 +08:00
whyour 5d3b40ce3b feat(ql3): complete cluster log retention lifecycle 2026-08-12 04:41:59 +08:00
whyour 40628c843d feat(ql3): add cluster log retention authority 2026-08-12 03:38:27 +08:00
whyour 2bfa8ca279 feat(ql3): add local run log retention 2026-08-12 02:57:43 +08:00
whyour 308aa75d89 feat(ql3): add profile-aware run log reads 2026-08-12 01:43:14 +08:00
whyour c699c32461 feat(ql3): establish 3.0 incubation baseline 2026-08-12 00:25:26 +08:00
whyour 4bf92dcfeb 新版本 v2.21.0 2026-07-18 11:02:15 +08:00
whyour 214e23a63d fix: 恢复 linux/s390x 镜像支持 2026-07-18 11:02:07 +08:00
whyour ddb3ab7ac3 Add configurable browser tab title for multi-instance QingLong deployments (#3041) 2026-07-10 02:07:11 +08:00
whyour da6dac8bf1 ts-node 异常,锁定 typescript 版本 2026-07-10 01:36:31 +08:00
whyour 9b17a41987 修复 dashboard 接口地址 2026-07-08 00:28:35 +08:00
whyour cdfa6c3c3c 增加启动错误日志 2026-07-04 21:36:11 +08:00
whyour 1953022b99 修复识别系统默认语言 2026-07-04 01:13:59 +08:00
whyour 38d84f9f31 修复订阅链接正则判断 2026-07-02 00:08:39 +08:00
whyour e92ddb0aed 修复添加任务有可能失败 2026-07-01 00:37:42 +08:00
Copilotandcopilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> a0d57c78c5 Fix "Validation failed" when editing env variables with null labels (#3026)
* Initial plan

* Fix env variable edit validation error when labels is null

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-06-30 23:51:21 +08:00
Martha Ramirez a942a5686f 修复删除定时任务后 crontab.list 残留导致订阅更新无法重新注册 (#3021)
* 修复删除定时任务后 crontab.list 未同步导致订阅更新无法重新注册脚本

crontab.list 作为数据库的文件镜像,其同步 setCrontab() 此前串行在尽力而为
的 gRPC addCron/delCron 调用之后:当调度 worker 重启等导致 gRPC 短暂不可用、
addCron/delCron reject 时,setCrontab() 会被跳过,crontab.list 与数据库脱节。
而 shell/update.sh 的 gen_list_repo() 将 crontab.list 作为"已有任务"真源,
于是订阅更新误判脚本已存在、跳过新增注册(前端再也看不到这些脚本)。

将 create/update/remove/disabled/enabled/autosave_crontab 中的 gRPC 调用改为
尽力而为(try/catch + 告警日志),保证 setCrontab() 总是执行;autosave_crontab
额外将 setCrontab 提前到 addCron 之前,确保启动/调度器重启时文件总是同步,
并避免 gRPC 失败导致应用启动崩溃。

调度器内存中的任务注册与 crontab.list/系统 crontab 同步是两个相互独立的关注点:
gRPC 失败时内存调度可能在调度器重启后由 autosave_crontab 重新注册,但文件同步
不应因此被阻断。修复后既有的 crontab.list 残留会在下次重启或任意增删改时自愈。

可能与 #2422(订阅更新重复添加任务)同根因。

* 过滤 delCron 中 scheduleStacks 的 null job

nodeSchedule.scheduleJob() 对无效 cron 表达式(如 "0 * /6 * * *",
空格分开了 * /6)会返回 null,原代码原样存入 scheduleStacks 数组。
delCron 取消时对 null 调 cancel() 抛出 UNKNOWN 错误,导致 gRPC
handler 中断、HTTP 端 cronService.remove() 跳过 setCrontab(),
crontab.list 残留已删任务的记录,导致订阅更新时 gen_list_repo()
误判脚本已存在、不再重新注册。

过滤 null 并对每个 cancel 调用加 try/catch 容错。
2026-06-30 23:48:51 +08:00
magicJie a1ccd9947d Add Kubernetes deployment manifests (#3025) 2026-06-30 22:24:03 +08:00
zjiecodeandzjiecode 327c3a317b Add WxPusher simple push support,Enhance user experience and reduce configuration complexity (#3023)
* 新增对WxPusher极简推送的支持,减小用户使用成本

* 新增脚本对WxPusher极简推送的支持

* 修复通知配置的枚举定义

---------

Co-authored-by: zjiecode <zjiecode@gmail.com>
2026-06-30 14:30:31 +08:00
whyour d3016431ce 定时任务显示完整运行历史 2026-06-28 15:48:58 +08:00
whyour 5fbff0e1c8 修复 openapi 鉴权提示 2026-06-26 15:31:25 +08:00
whyour 3044f63f03 修复国际化文案 2026-06-21 23:53:32 +08:00
whyour 369dd13212 修复 deps 目录依赖逻辑 2026-06-21 12:12:29 +08:00
whyour b5a5fb3be6 shell 增加国际化 2026-06-14 21:19:06 +08:00
whyour 0ee0b83207 延迟增加运行时间提示 2026-06-14 15:21:29 +08:00
whyour 5f2d5bb24a 修复 esm 依赖查询路径 2026-06-13 21:08:25 +08:00
whyour 949d956aef grpc 服务增加证书校验 2026-06-13 20:16:49 +08:00
whyour 96b4c90398 增加 sudo 命令判断 2026-06-13 00:53:41 +08:00
whyour 7d8feadc78 修复获取定时任务参数 2026-06-13 00:09:31 +08:00
whyour d1dfde3ca9 修复配置文件路径可能越权 2026-06-12 23:45:40 +08:00
whyour 6796068523 更新国际化文案 2026-06-11 23:18:45 +08:00
whyour 05f8fd3805 接口提示信息国际化 2026-06-11 02:19:04 +08:00
whyour 946731ac8d 增加运行实例 2026-06-10 01:53:10 +08:00
whyour 617cf7e5b4 更新 stop emoji 2026-06-09 00:48:03 +08:00
whyour 662adc0f9c 修复 dashboard title 2026-06-07 23:45:43 +08:00
whyour 23b41893a3 修复获取任务退出码 2026-06-07 23:28:21 +08:00
whyour 34d4526413 修复 work_dir 目录判断 2026-06-07 22:03:48 +08:00
whyour ad92e0c2c0 修复任务统计日志 2026-06-07 15:15:12 +08:00
whyour c14390cf41 修复 work_dir 验证 2026-06-07 13:56:06 +08:00
whyour fd6b4e4cde 定时任务增加 work_dir 设置 2026-06-07 13:19:01 +08:00
whyour 865e3035b9 开机运行任务同时开始运行 2026-06-01 23:24:10 +08:00
whyour d8d30c3619 修复 initFile 2026-06-01 19:07:15 +08:00
whyour 4e803df51e 修复内部服务 ip 地址 2026-06-01 18:28:54 +08:00
whyour e8ac195c96 增加任务统计 2026-06-01 18:20:18 +08:00
whyour c0b7527148 修复路径穿越 2026-06-01 13:49:32 +08:00
whyour ca347c5854 修复非 root 用户更新时区和 linux 配置 2026-05-31 16:48:13 +08:00
whyour 4bee3fbdf4 修复 getCronById api status 2026-05-31 15:53:24 +08:00
whyour b6537e27ef update mirror action 2026-05-31 15:04:10 +08:00
Copilotwhyourcopilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
0e5b8bef4b Fix HITOKOTO parameter boolean/string type handling (#2918)
* Initial plan

* Fix HITOKOTO parameter type mismatch in notify scripts

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

* Refactor JavaScript HITOKOTO check for consistency with Python

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>
2026-05-31 14:26:57 +08:00
Copilotwhyourcopilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
e06c3571a8 Fix undefined error in serverNotify when Server酱 returns error responses (#2917)
* Initial plan

* Fix: Replace data.data.errno with data.code to prevent undefined error

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>
2026-05-31 14:25:51 +08:00
whyour c2eac8b8fd 移除未使用配置 2026-05-31 00:36:54 +08:00
jmclulu 3aab1233bb fix: correct typos in source code and locales (#3003)
- Fix Countrys -> Countries in comment
- Fix Scrolldown -> ScrollDown in variable name
- Fix completeTowFactor -> completeTwoFactor (3x)
- Fix deactiveTowFactor -> deactivateTwoFactor (3x)
- Fix activeOrDeactiveTwoFactor -> activeOrDeactivateTwoFactor
- Fix API route /two-factor/deactive -> /two-factor/deactivate
- Fix elment -> element in function param
- Fix synolog -> synology in comment
- Fix Chinese comment 制定 -> 指定
- Fix swapped BARK English translations on lines 360-361
2026-05-31 00:32:04 +08:00
whyour 2fe9470ff0 fix: gRPC extra_schedules 为空时序列化报错 not iterable 2026-05-31 00:14:22 +08:00
whyour abad29cbf9 修复非 root debian dockerfile 2026-05-30 23:55:57 +08:00
whyouranddream10201 84d730d510 统一 Alpine/Debian 分支,QL_SCHEDULER 参数化调度
* 修改获取示例文件 api path

* 增加 debian-slim 基础镜像

* 修复 debian apt 命令,支持 qinglong 命令

* 更新 npm 版本 0.7.7

* 更新 npm v0.8.4

* 修复linux依赖检测 (#2082)

* 修复拉取私有仓库

* 修复 shell check_server

* 修复 qinglong 命令

* 更新 npm 版本 v0.13.2

* 增加 debian 开发版本

* 修改切换 linux 镜像源

* 修复 qinglong 命令

* 移除 qinglong 命令 npm 默认镜像源

* 修复 workflow

* 更新 npm 版本 v0.14.5

* 增加 npx 命令

* 更新 workflow action 版本

* 更新 npm 版本 v0.16.0

* 修复 linux 镜像源

* 更新 npm 版本 v0.17.0

* 更新 npm 版本 v0.18.0

* 修改 npm 安装启动命令

* 更新 npm 版本 v0.19.9

* 修复 debian netcat 包名

* 更新 npm 版本 v0.20.4

* 安装 linux 依赖自动识别 alpine 和 debian

* 修改 apt 命令

* 更新 npm 版本 v0.21.2

* 修改 ts 文件执行依赖

* npm 启动增加 reload 逻辑

* 更新 npm 版本 v2.17.8

* 修复 qinglong 命令

* 更新 npm 版本 v2.17.9

* 更新 npm 版本 v2.17.10

* 更新 npm 版本 v2.17.11

* 修改 debian 版本为 12 bookworm

* 更新 npm 版本 v2.17.12

* 修改本地服务启动提示

* 更新 npm 版本 v2.17.13

* 写入文件增加文件锁

* 修复系统安装依赖提示

* 更新 npm 版本 v2.18.2-6

* 更新 nodejs 版本

* 更新 npm 版本 v2.18.3-3

* 修复 command 变量

* 移除自动清除 deb

* 修复 npm 启动脚本

* 修复发布 npm包依赖文件

* 修改 linux 启动文件逻辑

* 更新 npm 版本 v2.19.0-10

* 修复 apt 命令

* 更新 npm 版本 v2.19.1-0

* 更新 npm 版本 v2.19.2-2

* 增加 packageManager

* 增加用户 qinglong

* 更新 pipeline

* 移除 init_nginx

* 更新 npm 版本 v2.20.0

* 更新 npm 版本 2.20.1

* 更新 npm 版本 2.20.2

* fix: 修复非 root 用户启动

* chore: 合并 debian 和 alpine 逻辑

---------

Co-authored-by: dream10201 <xiuxiu10201@gmail.com>
2026-05-30 18:03:51 +08:00
whyour 57d58c871e fix: isolate task node dependencies 2026-05-24 01:40:07 +08:00
whyour bb6d436c19 fix: run scripts from their own directory 2026-05-24 00:15:24 +08:00
whyour 8bc0906949 feat: add environment variable labels 2026-05-23 23:21:38 +08:00
Copilotwhyourcopilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
7a8917f8e4 Fix CodeMirror "multiple instances of @codemirror/state" crash on page navigation (#2969)
* Initial plan

* Fix CodeMirror multiple instances error by pinning to single versions

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>
2026-05-23 20:48:10 +08:00
whyour 66f9457be8 ci: replace hub-mirror-action with direct git push for Gitee sync
Replace Yikun/hub-mirror-action with manual git push for Gitee mirrors. The action HTTPS API call to gitee.com timed out (60s). Use set +e with explicit notice/warning status. Also add .deepseek/ to .gitignore.
2026-05-18 01:05:46 +08:00
whyour 0c5e3b5d04 ci: update workflow actions 2026-05-18 00:12:22 +08:00
whyour 400e4770de 修复环境变量 position 数据类型可能异常 2026-05-17 23:53:43 +08:00
whyour 40d4de9017 增加 localhost 检测 2026-05-17 23:43:09 +08:00
whyour 3464c4da61 fix IPv6 connectivity 2026-05-06 01:29:01 +08:00
Max 1315578878 支持自定义接收邮箱地址 (#2973)
* 支持自定义接收邮箱地址

* 新增支持多个接收邮箱,同步到node和系统内置版本
2026-05-06 00:34:26 +08:00
TengDream a1ae08da58 fix: create root .tmp directory on init for data export (#2993)
The data export feature (system backup) writes data.tgz to
`config.tmpPath` which resolves to `<rootPath>/.tmp/`. However,
`initFile.ts` only created `<dataPath>/log/.tmp/` (used for crontab
list temp files), never the root-level `.tmp/` directory.

In Docker deployments, `shell/share.sh`'s `fix_config()` creates
`$dir_root/.tmp` during shell initialization, but local/non-Docker
deployments that start the Node service directly skip the shell init,
causing a 404 ENOENT error when attempting to export/backup data.

Add `rootTmpPath` (`<rootPath>/.tmp/`) to the directories array in
`initFile.ts` so it is created during Node service startup regardless
of deployment method.
2026-05-06 00:32:29 +08:00
Copilotwhyourcopilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
66700ebe1a Add OpeniLink notification channel (#2988)
* Initial plan

* Add OpeniLink notification channel support

Agent-Logs-Url: https://github.com/whyour/qinglong/sessions/c80b4882-1bd7-4ffe-9180-cd3220da5986

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

* Add context_token and hub_url to OpeniLink notification channel

Agent-Logs-Url: https://github.com/whyour/qinglong/sessions/a5e66f5a-dab8-4a65-96ca-960d35fa9d50

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>
2026-04-25 16:01:36 +08:00
Copilotandwhyour 07bf0c705b fix: respect QlPort env var in Docker health check (#2963)
* Initial plan

* fix: use QlPort env variable in health check with fallback to default 5700

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>
2026-03-11 20:43:56 +08:00
whyour fd516977e3 chore: upgrade nodemailer 2026-03-07 22:35:18 +08:00
whyour c39f4ef846 chore: 更新 multer,解决 cve 漏洞 2026-03-07 21:31:24 +08:00
whyour 275d8af4e2 更新版本 v2.20.2 2026-03-01 20:35:25 +08:00
whyour 544c432f49 修复 PATH 环境变量 2026-03-01 20:35:19 +08:00
3781 changed files with 1075632 additions and 4427 deletions
+5
View File
@@ -1,6 +1,11 @@
GRPC_PORT=5500
BACK_PORT=5700
# 服务绑定地址,默认 ::(IPv6 通配,双栈系统同时支持 IPv4/IPv6)
# 纯 IPv4 环境自动 fallback 到 0.0.0.0,也可手动指定
# BIND_HOST=0.0.0.0
# BIND_HOST_GRPC=0.0.0.0
LOG_LEVEL='info'
JWT_SECRET=
+276 -91
View File
@@ -11,6 +11,9 @@ on:
- "v*"
workflow_dispatch:
permissions:
contents: read
jobs:
code_gitlab:
runs-on: ubuntu-latest
@@ -18,14 +21,16 @@ jobs:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: Yikun/hub-mirror-action@master
with:
src: github/whyour
dst: gitlab/whyour
dst_key: ${{ secrets.GITLAB_SSH_PK }}
dst_token: ${{ secrets.GITLAB_TOKEN }}
static_list: "qinglong"
force_update: true
- name: Push to GitLab
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
git remote add gitlab git@gitlab.com:whyour/qinglong.git 2>/dev/null
git push --force --all gitlab 2>&1 || echo "::warning::GitLab push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab tags push failed"
code_gitee:
runs-on: ubuntu-latest
@@ -33,26 +38,42 @@ jobs:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: Yikun/hub-mirror-action@master
with:
src: github/whyour
dst: gitee/whyour
dst_key: ${{ secrets.GITLAB_SSH_PK }}
dst_token: ${{ secrets.GITEE_TOKEN }}
static_list: "qinglong"
force_update: true
- name: Push to Gitee
env:
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git remote add gitee git@gitee.com:whyour/qinglong.git 2>/dev/null
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee push --all succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee push succeeded after repo creation" || echo "::warning::Gitee push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed"
build-static:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@v6
with:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
- name: build front and back
run: |
@@ -64,9 +85,6 @@ jobs:
env:
GITHUB_REPO: github.com/${{ github.repository_owner }}/qinglong-static
GITHUB_BRANCH: ${{ github.ref_name }}
REPO_GITEE: git@gitee.com:whyour/qinglong-static.git
REPO_GITLAB: git@gitlab.com:whyour/qinglong-static.git
PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PK }}
run: |
mkdir -p tmp
cd ./tmp
@@ -81,73 +99,83 @@ jobs:
needs: build-static
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: Yikun/hub-mirror-action@master
with:
src: github/whyour
dst: gitlab/whyour
dst_key: ${{ secrets.GITLAB_SSH_PK }}
dst_token: ${{ secrets.GITLAB_TOKEN }}
static_list: "qinglong-static"
force_update: true
- name: Push qinglong-static to GitLab
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitlab.com >> ~/.ssh/known_hosts 2>/dev/null
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
cd static-mirror
git remote add gitlab git@gitlab.com:whyour/qinglong-static.git 2>/dev/null
git push --force --all gitlab 2>&1 || echo "::warning::GitLab static push failed"
git push --force --tags gitlab 2>&1 || echo "::warning::GitLab static tags push failed"
static_gitee:
needs: build-static
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: Yikun/hub-mirror-action@master
with:
src: github/whyour
dst: gitee/whyour
dst_key: ${{ secrets.GITLAB_SSH_PK }}
dst_token: ${{ secrets.GITEE_TOKEN }}
static_list: "qinglong-static"
force_update: true
- name: Push qinglong-static to Gitee
env:
GITEE_TOKEN: ${{ secrets.GITEE_TOKEN }}
run: |
set +e
mkdir -p ~/.ssh
printf '%s\n' "${{ secrets.GITLAB_SSH_PK }}" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -T 10 gitee.com >> ~/.ssh/known_hosts 2>/dev/null
git clone --depth=1 --single-branch https://github.com/whyour/qinglong-static.git static-mirror
cd static-mirror
git remote add gitee git@gitee.com:whyour/qinglong-static.git 2>/dev/null
build:
if git push --force --all gitee 2>&1; then
echo "::notice::Gitee static push succeeded"
else
echo "::warning::Push failed, trying to create repo via API..."
curl -sS --connect-timeout 30 --max-time 60 \
-X POST "https://gitee.com/api/v5/user/repos" \
-H "Content-Type: application/json" \
-d '{"name":"qinglong-static","private":"false"}' \
"?access_token=$GITEE_TOKEN" 2>/dev/null
git push --force --all gitee 2>&1 && echo "::notice::Gitee static push succeeded after repo creation" || echo "::warning::Gitee static push failed after retry"
fi
git push --force --tags gitee 2>&1 || echo "::warning::Gitee static tags push failed"
build-alpine:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
needs: build-static
runs-on: ubuntu-22.04
permissions:
packages: write
contents: read
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@v6
with:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
- name: Read version from version.yaml
id: version
run: |
VERSION=$(grep '^version:' version.yaml | awk '{print $2}')
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "Version: $VERSION"
- name: Setup timezone
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: Asia/Shanghai
run: sudo timedatectl set-timezone Asia/Shanghai
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v3
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@@ -155,7 +183,7 @@ jobs:
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v6
with:
images: |
${{ github.repository }}
@@ -169,104 +197,261 @@ jobs:
type=semver,pattern={{version}}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
uses: docker/setup-qemu-action@v4
with:
cache-image: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@v4
- name: Build and push
id: docker_build
uses: docker/build-push-action@v6
- name: Build and push (Alpine)
uses: docker/build-push-action@v7
with:
build-args: |
MAINTAINER=${{ github.repository_owner }}
QL_BRANCH=${{ github.ref_name }}
SOURCE_COMMIT=${{ github.sha }}
network: host
# linux/s390x npm 暂不可用
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/386
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
context: .
file: ./docker/Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=whyour/qinglong:cache
cache-to: type=registry,ref=whyour/qinglong:cache,mode=max
cache-from: type=registry,ref=whyour/qinglong:cache-alpine
cache-to: type=registry,ref=whyour/qinglong:cache-alpine,mode=max
- name: Image digest
run: |
echo ${{ steps.docker_build.outputs.digest }}
build310:
if: ${{ github.ref_name == 'master' }}
build-debian:
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
needs: build-static
runs-on: ubuntu-22.04
permissions:
packages: write
contents: read
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v4
- uses: pnpm/action-setup@v6
with:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
- name: Read version from version.yaml
id: version
run: |
VERSION=$(grep '^version:' version.yaml | awk '{print $2}')
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "Version: $VERSION"
- name: Setup timezone
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: Asia/Shanghai
run: sudo timedatectl set-timezone Asia/Shanghai
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v3
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
with:
images: |
${{ github.repository }}
ghcr.io/${{ github.repository }}
flavor: |
latest=false
tags: |
type=raw,value=debian-dev,enable=${{ github.ref == format('refs/heads/{0}', 'develop') }}
type=raw,value=debian,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
type=raw,value=${{ steps.version.outputs.version }}-debian,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
uses: docker/setup-qemu-action@v4
with:
cache-image: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@v4
- name: Build and push python3.10
id: docker_build_310
uses: docker/build-push-action@v6
- name: Build and push (Debian)
uses: docker/build-push-action@v7
with:
build-args: |
MAINTAINER=${{ github.repository_owner }}
QL_BRANCH=${{ github.ref_name }}
SOURCE_COMMIT=${{ github.sha }}
network: host
# linux/s390x npm 暂不可用
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/386
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
context: .
file: ./docker/310.Dockerfile
file: ./docker/Dockerfile.debian
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=whyour/qinglong:cache-debian
cache-to: type=registry,ref=whyour/qinglong:cache-debian,mode=max
build-alpine310:
if: ${{ github.ref_name == 'master' }}
needs: build-static
runs-on: ubuntu-22.04
permissions:
packages: write
contents: read
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
- name: Read version from version.yaml
id: version
run: |
VERSION=$(grep '^version:' version.yaml | awk '{print $2}')
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Setup timezone
run: sudo timedatectl set-timezone Asia/Shanghai
- name: Login to DockerHub
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
with:
cache-image: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build and push (Alpine Python 3.10)
uses: docker/build-push-action@v7
with:
build-args: |
MAINTAINER=${{ github.repository_owner }}
QL_BRANCH=${{ github.ref_name }}
SOURCE_COMMIT=${{ github.sha }}
network: host
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x,linux/386
context: .
file: ./docker/Dockerfile.310
push: true
tags: |
whyour/qinglong:python3.10
whyour/qinglong:${{ steps.version.outputs.version }}-python3.10
cache-from: type=registry,ref=whyour/qinglong:cache-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-python3.10,mode=max
cache-from: type=registry,ref=whyour/qinglong:cache-alpine-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-alpine-python3.10,mode=max
- name: Image digest
build-debian310:
if: ${{ github.ref_name == 'master' }}
needs: build-static
runs-on: ubuntu-22.04
permissions:
packages: write
contents: read
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: "8.3.1"
- uses: actions/setup-node@v6
with:
cache: "pnpm"
cache-dependency-path: pnpm-lock.yaml
- name: Read version from version.yaml
id: version
run: |
echo ${{ steps.docker_build_310.outputs.digest }}
VERSION=$(grep '^version:' version.yaml | awk '{print $2}')
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Setup timezone
run: sudo timedatectl set-timezone Asia/Shanghai
- name: Login to DockerHub
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
with:
cache-image: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build and push (Debian Python 3.10)
uses: docker/build-push-action@v7
with:
build-args: |
MAINTAINER=${{ github.repository_owner }}
QL_BRANCH=${{ github.ref_name }}
SOURCE_COMMIT=${{ github.sha }}
network: host
# Keep s390x on Debian: npm can hang under QEMU with Alpine (nodejs/docker-node#1973).
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
context: .
file: ./docker/Dockerfile.debian310
push: true
tags: |
whyour/qinglong:debian-python3.10
whyour/qinglong:${{ steps.version.outputs.version }}-debian-python3.10
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
publish:
if: ${{ github.ref_name == 'master' }}
needs: [build-alpine, build-debian]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
with:
version: "8.3.1"
- uses: actions/setup-node@v3
with:
cache: "pnpm"
- name: build front and back
run: |
pnpm install --frozen-lockfile
pnpm build:front
pnpm build:back
- name: publish npm package
run: |
echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> ~/.npmrc
npm publish
@@ -0,0 +1,122 @@
name: QingLong 3.0 Approval Management Kubernetes live evidence
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-approval-management-live-${{ github.ref }}
cancel-in-progress: false
jobs:
approval-management-live:
name: Three-node Approval, OIDC, mTLS and CloudNativePG
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Require enough ephemeral disk for three isolated K3s nodes
shell: bash
run: |
set -euo pipefail
available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')"
minimum_kib="$((25 * 1024 * 1024))"
if (( available_kib < minimum_kib )); then
echo "Approval live gate requires at least 25 GiB free; found ${available_kib} KiB" >&2
exit 1
fi
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.before"
docker system df
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.32.8
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Fetch checksum-locked CloudNativePG operator manifest
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/cloudnative-pg.yaml" \
https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml
echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88 ${RUNNER_TEMP}/cloudnative-pg.yaml" | sha256sum --check
- name: Recheck static Approval and evidence contracts
run: |
pnpm audit:cluster-deployment:ql3
node --test \
test/back/ql3ApprovalManagementKubernetesLiveContract.test.cjs \
test/back/ql3ApprovalManagementKubernetesLiveAudit.test.cjs
pnpm --filter @qinglong/runtime-core build
- name: Preload the digest-bound K3s distribution
run: docker pull rancher/k3s:v1.34.3-k3s1
- name: Prove Approval over three nodes, mTLS, OIDC and CloudNativePG
env:
QL3_APPROVAL_MANAGEMENT_KUBERNETES_LIVE: '1'
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_CNPG_OPERATOR_MANIFEST_FILE: ${{ runner.temp }}/cloudnative-pg.yaml
QL3_APPROVAL_REPORT: ${{ runner.temp }}/ql3-approval/report.json
shell: bash
run: |
set -euo pipefail
umask 077
mkdir -m 0700 "$(dirname "${QL3_APPROVAL_REPORT}")"
node scripts/ql3-approval-management-kubernetes-live-contract.cjs \
"--report=${QL3_APPROVAL_REPORT}"
- name: Re-audit private report and isolated cleanup
env:
QL3_APPROVAL_REPORT: ${{ runner.temp }}/ql3-approval/report.json
shell: bash
run: |
set -euo pipefail
test "$(stat -c '%a' "${QL3_APPROVAL_REPORT}")" = '600'
pnpm audit:approval-management-kubernetes-live:ql3 \
"--report=${QL3_APPROVAL_REPORT}"
sha256sum "${QL3_APPROVAL_REPORT}"
test -z "$(docker ps -aq --filter name=ql3-approval-live-)"
test -z "$(docker network ls -q --filter name=ql3-approval-live-)"
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
diff --unified \
"${RUNNER_TEMP}/ql3-dangling-volumes.before" \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
docker system df
- name: Upload audited content-free Approval evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-approval-management-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-approval/report.json
if-no-files-found: error
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,134 @@
name: QingLong 3.0 CloudNativePG DR live evidence
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-cloudnativepg-dr-live-${{ github.ref }}
cancel-in-progress: false
jobs:
cloudnativepg-dr-live:
name: Barman WAL, latest and PITR recovery
runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Require enough ephemeral disk for four isolated K3s nodes
shell: bash
run: |
set -euo pipefail
available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')"
minimum_kib="$((35 * 1024 * 1024))"
if (( available_kib < minimum_kib )); then
echo "CloudNativePG DR live gate requires at least 35 GiB free; found ${available_kib} KiB" >&2
exit 1
fi
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.before"
docker system df
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.32.8
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Fetch checksum-locked operator manifests
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/cert-manager.yaml" \
https://github.com/cert-manager/cert-manager/releases/download/v1.20.3/cert-manager.yaml
echo "7ee74ba06845213e96d8ceaff3d20dd51e682765c1418eddda4e8780ba082261 ${RUNNER_TEMP}/cert-manager.yaml" | sha256sum --check
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/barman-cloud.yaml" \
https://github.com/cloudnative-pg/plugin-barman-cloud/releases/download/v0.13.0/manifest.yaml
echo "d2e71e7b06822448f1a421f05781846cfdb9cc621e7ef32eef5e20c5133213b0 ${RUNNER_TEMP}/barman-cloud.yaml" | sha256sum --check
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/cloudnative-pg.yaml" \
https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml
echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88 ${RUNNER_TEMP}/cloudnative-pg.yaml" | sha256sum --check
- name: Recheck the static backup and supply-chain contracts
run: |
pnpm audit:cloudnativepg-backup:ql3
pnpm audit:barman-cloud-supply-chain:ql3
pnpm audit:cert-manager-selection:ql3
node --test \
test/back/ql3CloudNativePgBarmanLiveContract.test.cjs \
test/back/ql3CloudNativePgDrEvidenceAudit.test.cjs \
test/back/ql3CloudNativePgBarmanWorkflow.test.cjs
- name: Prove TLS object storage, continuous WAL, latest restore and PITR
env:
QL3_CLOUDNATIVEPG_BARMAN_LIVE: '1'
QL3_SOURCE_REVISION: ${{ github.sha }}
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_CERT_MANAGER_MANIFEST_FILE: ${{ runner.temp }}/cert-manager.yaml
QL3_BARMAN_MANIFEST_FILE: ${{ runner.temp }}/barman-cloud.yaml
QL3_CNPG_OPERATOR_MANIFEST_FILE: ${{ runner.temp }}/cloudnative-pg.yaml
QL3_DR_REPORT: ${{ runner.temp }}/ql3-cloudnativepg-dr/report.json
shell: bash
run: |
set -euo pipefail
umask 077
mkdir -m 0700 "$(dirname "${QL3_DR_REPORT}")"
pnpm test:cloudnativepg-barman-live:ql3 -- \
"--report=${QL3_DR_REPORT}"
- name: Re-audit the exact private report and isolated cleanup
env:
QL3_DR_REPORT: ${{ runner.temp }}/ql3-cloudnativepg-dr/report.json
shell: bash
run: |
set -euo pipefail
test "$(stat -c '%a' "${QL3_DR_REPORT}")" = '600'
pnpm audit:cloudnativepg-dr-evidence:ql3 -- \
"--report=${QL3_DR_REPORT}"
sha256sum "${QL3_DR_REPORT}"
test -z "$(docker ps -aq --filter name=ql3-barman-dr-)"
test -z "$(docker network ls -q --filter name=ql3-barman-dr-)"
test -z "$(docker ps -aq --filter label=io.qinglong.ql3.live=cloudnativepg-barman-disaster-recovery)"
test -z "$(docker network ls -q --filter label=io.qinglong.ql3.live=cloudnativepg-barman-disaster-recovery)"
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
diff --unified \
"${RUNNER_TEMP}/ql3-dangling-volumes.before" \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
docker system df
- name: Upload the audited content-free DR evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-cloudnativepg-dr-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-cloudnativepg-dr/report.json
if-no-files-found: error
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,90 @@
name: QingLong 3.0 Kubernetes deployment live contract
on:
push:
branches:
- next
pull_request:
branches:
- next
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-kubernetes-deployment-live-${{ github.ref }}
cancel-in-progress: true
jobs:
kubernetes-deployment-live:
name: Deployment Head and fenced resource retirement
runs-on: ubuntu-24.04
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.34.3
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Preload the digest-reviewed K3s distribution
run: docker pull rancher/k3s:v1.34.3-k3s1
- name: Recheck the closed deployment ceremony contract
run: node --test test/back/ql3KubernetesDeploymentCeremony.test.cjs
- name: Prove install, UID/resourceVersion retirement and Head convergence
env:
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_DEPLOYMENT_LIVE_REPORT: ${{ runner.temp }}/ql3-kubernetes-deployment/report.json
shell: bash
run: |
set -euo pipefail
umask 077
install -d -m 0700 "$(dirname "${QL3_DEPLOYMENT_LIVE_REPORT}")"
node scripts/ql3-kubernetes-deployment-live-contract.cjs > \
"${QL3_DEPLOYMENT_LIVE_REPORT}"
node -e \
"const fs=require('node:fs');const report=JSON.parse(fs.readFileSync(process.env.QL3_DEPLOYMENT_LIVE_REPORT,'utf8'));if(report.cleanupComplete!==true||report.receiptAuditCompatible!==true||report.retirement?.receiptAuditCompatible!==true||report.retirement?.targetAbsent!==true||report.retirement?.uidResourceVersionDeletePreconditions!==true||report.retirement?.deploymentHeadCas!==true||report.retirement?.unixSocketProxy!==true)process.exit(1)"
- name: Verify isolated cleanup
if: always()
shell: bash
run: |
set -euo pipefail
test -z "$(docker ps -aq --filter name=ql3-deploy-live-)"
test -z "$(docker network ls -q --filter name=ql3-deploy-live-)"
- name: Upload content-free deployment evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-kubernetes-deployment-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-kubernetes-deployment/report.json
if-no-files-found: warn
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
@@ -0,0 +1,124 @@
name: QingLong 3.0 Run Management Kubernetes live evidence
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-run-management-live-${{ github.ref }}
cancel-in-progress: false
jobs:
run-management-live:
name: Three-node Run retry, stop, OIDC, mTLS and CloudNativePG
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Require enough ephemeral disk for three isolated K3s nodes
shell: bash
run: |
set -euo pipefail
available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')"
minimum_kib="$((25 * 1024 * 1024))"
if (( available_kib < minimum_kib )); then
echo "Run management live gate requires at least 25 GiB free; found ${available_kib} KiB" >&2
exit 1
fi
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.before"
docker system df
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.32.8
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Fetch checksum-locked CloudNativePG operator manifest
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/cloudnative-pg.yaml" \
https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml
echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88 ${RUNNER_TEMP}/cloudnative-pg.yaml" | sha256sum --check
- name: Recheck static Run management and evidence contracts
run: |
pnpm audit:cluster-deployment:ql3
node --test \
test/back/ql3RunManagementDeployment.test.cjs \
test/back/ql3RunManagementKubernetesLiveContract.test.cjs \
test/back/ql3RunManagementKubernetesLiveAudit.test.cjs
pnpm --filter @qinglong/cluster-admin build
pnpm --filter @qinglong/cluster-control build
- name: Preload the digest-bound K3s distribution
run: docker pull rancher/k3s:v1.34.3-k3s1
- name: Prove Run retry and stop over three nodes, mTLS, OIDC and CloudNativePG
env:
QL3_RUN_MANAGEMENT_KUBERNETES_LIVE: '1'
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_CNPG_OPERATOR_MANIFEST_FILE: ${{ runner.temp }}/cloudnative-pg.yaml
QL3_RUN_MANAGEMENT_REPORT: ${{ runner.temp }}/ql3-run-management/report.json
shell: bash
run: |
set -euo pipefail
umask 077
mkdir -m 0700 "$(dirname "${QL3_RUN_MANAGEMENT_REPORT}")"
node scripts/ql3-run-management-kubernetes-live-contract.cjs \
"--report=${QL3_RUN_MANAGEMENT_REPORT}"
- name: Re-audit private report and isolated cleanup
env:
QL3_RUN_MANAGEMENT_REPORT: ${{ runner.temp }}/ql3-run-management/report.json
shell: bash
run: |
set -euo pipefail
test "$(stat -c '%a' "${QL3_RUN_MANAGEMENT_REPORT}")" = '600'
pnpm audit:run-management-kubernetes-live:ql3 \
"--report=${QL3_RUN_MANAGEMENT_REPORT}"
sha256sum "${QL3_RUN_MANAGEMENT_REPORT}"
test -z "$(docker ps -aq --filter name=ql3-run-live-)"
test -z "$(docker network ls -q --filter name=ql3-run-live-)"
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
diff --unified \
"${RUNNER_TEMP}/ql3-dangling-volumes.before" \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
docker system df
- name: Upload audited content-free Run management evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-run-management-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-run-management/report.json
if-no-files-found: error
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
@@ -0,0 +1,151 @@
name: QingLong 3.0 Security Administration Kubernetes live evidence
on:
push:
branches:
- next
paths:
- .github/workflows/ql3-security-administration-live.yml
- deploy/containers/ql3-cluster-admin/**
- deploy/containers/ql3-cluster-control/**
- deploy/kubernetes/ql3-cluster/base/**
- deploy/kubernetes/ql3-cluster/operations/cloudnative-pg/**
- deploy/kubernetes/ql3-cluster/operations/security-administration/**
- package.json
- packages/ql3-cluster-admin/**
- packages/ql3-cluster-control/**
- packages/ql3-cluster-postgres/**
- packages/ql3-runtime-core/**
- pnpm-lock.yaml
- scripts/lib/ql3-k3s-docker-live.cjs
- scripts/ql3-security-administration-kubernetes-*.cjs
- test/back/ql3SecurityAdministrationKubernetes*.test.cjs
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-security-administration-live-${{ github.ref }}
cancel-in-progress: false
jobs:
security-administration-live:
name: Three-node Security Administration, PostgreSQL and PVC custody
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Require enough ephemeral disk for three isolated K3s nodes
shell: bash
run: |
set -euo pipefail
available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')"
minimum_kib="$((25 * 1024 * 1024))"
if (( available_kib < minimum_kib )); then
echo "Security Administration live gate requires at least 25 GiB free; found ${available_kib} KiB" >&2
exit 1
fi
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.before"
docker system df
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.34.3
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Fetch checksum-locked CloudNativePG operator manifest
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/cloudnative-pg.yaml" \
https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml
echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88 ${RUNNER_TEMP}/cloudnative-pg.yaml" | sha256sum --check
- name: Recheck static Security Administration and evidence contracts
run: |
pnpm --filter @qinglong/runtime-core build
pnpm --filter @qinglong/cluster-postgres build
pnpm audit:security-administration-kubernetes:ql3
node --test \
test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs \
test/back/ql3SecurityAdministrationKubernetesLiveAudit.test.cjs
- name: Preload the digest-bound K3s distribution
run: docker pull rancher/k3s:v1.34.3-k3s1
- name: Prepare Linux bridge netfilter for nested K3s
shell: bash
run: |
set -euo pipefail
sudo modprobe br_netfilter
test -e /proc/sys/net/bridge/bridge-nf-call-iptables
sudo sysctl -w net.ipv4.ip_forward=1
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
- name: Prove Security Administration through K3s, PostgreSQL and PVC
env:
QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE: '1'
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_CNPG_OPERATOR_MANIFEST_FILE: ${{ runner.temp }}/cloudnative-pg.yaml
QL3_SECURITY_ADMINISTRATION_REPORT: ${{ runner.temp }}/ql3-security-administration/report.json
shell: bash
run: |
set -euo pipefail
umask 077
mkdir -m 0700 "$(dirname "${QL3_SECURITY_ADMINISTRATION_REPORT}")"
pnpm test:security-administration-kubernetes-live:ql3 \
"--report=${QL3_SECURITY_ADMINISTRATION_REPORT}"
- name: Re-audit content-free report and isolated cleanup
env:
QL3_SECURITY_ADMINISTRATION_REPORT: ${{ runner.temp }}/ql3-security-administration/report.json
shell: bash
run: |
set -euo pipefail
test "$(stat -c '%a' "${QL3_SECURITY_ADMINISTRATION_REPORT}")" = '600'
pnpm audit:security-administration-kubernetes-live:ql3 \
"--report=${QL3_SECURITY_ADMINISTRATION_REPORT}"
sha256sum "${QL3_SECURITY_ADMINISTRATION_REPORT}"
test -z "$(docker ps -aq --filter name=ql3-security-live-)"
test -z "$(docker network ls -q --filter name=ql3-security-live-)"
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
diff --unified \
"${RUNNER_TEMP}/ql3-dangling-volumes.before" \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
docker system df
- name: Upload audited content-free Security Administration evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-security-administration-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-security-administration/report.json
if-no-files-found: error
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
@@ -0,0 +1,122 @@
name: QingLong 3.0 Worker Kubernetes rollout live evidence
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ql3-worker-kubernetes-rollout-live-${{ github.ref }}
cancel-in-progress: false
jobs:
worker-kubernetes-rollout-live:
name: Production Worker, Recreate, mTLS, PVC and Session drain
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Require isolated fixture capacity
shell: bash
run: |
set -euo pipefail
available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')"
minimum_kib="$((15 * 1024 * 1024))"
if (( available_kib < minimum_kib )); then
echo "Worker rollout live gate requires at least 15 GiB free; found ${available_kib} KiB" >&2
exit 1
fi
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.before"
docker system df
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.34.3
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Recheck Worker runtime and evidence contracts
run: |
pnpm --filter @qinglong/worker-runtime test
node --test test/back/ql3WorkerKubernetesRolloutLiveAudit.test.cjs
pnpm audit:worker-deployment:ql3
- name: Preload digest-bound fixture images
shell: bash
run: |
set -euo pipefail
docker pull \
rancher/k3s@sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c
docker tag \
rancher/k3s@sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c \
rancher/k3s:v1.34.3-k3s1
docker pull \
postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296
docker tag \
postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296 \
postgres:18.4-bookworm
- name: Prove production Worker Kubernetes lifecycle
env:
QL3_WORKER_KUBERNETES_ROLLOUT_LIVE: '1'
QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
QL3_WORKER_KUBERNETES_REPORT: ${{ runner.temp }}/ql3-worker/report.json
shell: bash
run: |
set -euo pipefail
umask 077
mkdir -m 0700 "$(dirname "${QL3_WORKER_KUBERNETES_REPORT}")"
pnpm test:worker-kubernetes-rollout-live:ql3 \
"--report=${QL3_WORKER_KUBERNETES_REPORT}"
- name: Re-audit private report and isolated cleanup
env:
QL3_WORKER_KUBERNETES_REPORT: ${{ runner.temp }}/ql3-worker/report.json
shell: bash
run: |
set -euo pipefail
test "$(stat -c '%a' "${QL3_WORKER_KUBERNETES_REPORT}")" = '600'
pnpm audit:worker-kubernetes-rollout-live:ql3 \
"--report=${QL3_WORKER_KUBERNETES_REPORT}"
sha256sum "${QL3_WORKER_KUBERNETES_REPORT}"
test -z "$(docker ps -aq --filter name=ql3-worker-rollout-live-)"
test -z "$(docker ps -aq --filter name=ql3-worker-rollout-postgres-live-)"
docker volume ls --quiet --filter dangling=true | sort > \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
diff --unified \
"${RUNNER_TEMP}/ql3-dangling-volumes.before" \
"${RUNNER_TEMP}/ql3-dangling-volumes.after"
docker system df
- name: Upload audited content-free Worker evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-worker-kubernetes-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-worker/report.json
if-no-files-found: error
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
+10
View File
@@ -2,6 +2,8 @@
# dependencies
/node_modules
/packages/*/node_modules
/packages/*/dist
/npm-debug.log*
/yarn-error.log
/yarn.lock
@@ -9,6 +11,7 @@
# production
/static
/.ql3-panel-dist
/data
# misc
@@ -28,3 +31,10 @@ __pycache__
/shell/preload/notify.*
/shell/preload/*-notify.json
/shell/preload/__ql_notify__.*
/shell/preload/lang_env.sh
.deepseek/
.claude/
# local Kubernetes overlays
/deploy/kubernetes/overlays/local/
+22
View File
@@ -0,0 +1,22 @@
/.tmp/
/.github/
/.vscode/
/.history/
/back/**/*.ts
/back/**/*.json
/cli/
/data/
/src/
/static/**/*.js.map
/static/**/*.gz
/.editorconfig
/.gitignore
/.prettierignore
/.prettierrc
/.umirc.ts
/nodemon.json
/pnpm-lock.yaml
/tsconfig.back.json
/tsconfig.json
/typings.d.ts
/.env
+2 -1
View File
@@ -1 +1,2 @@
strict-peer-dependencies=false
strict-peer-dependencies=false
resolve-peers-from-workspace-root=false
+43
View File
@@ -0,0 +1,43 @@
<!-- gitnexus:start -->
# GitNexus — Code Intelligence
This project is indexed by GitNexus as **qinglong** (2778 symbols, 6698 relationships, 233 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
> Index stale? Run `node .gitnexus/run.cjs analyze` from the project root — it auto-selects an available runner. No `.gitnexus/run.cjs` yet? `npx gitnexus analyze` (npm 11 crash → `npm i -g gitnexus`; #1939).
## Always Do
- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user.
- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: `detect_changes({scope: "compare", base_ref: "develop"})`.
- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits.
- When exploring unfamiliar code, use `query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance.
- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`.
## Never Do
- NEVER edit a function, class, or method without first running `impact` on it.
- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis.
- NEVER rename symbols with find-and-replace — use `rename` which understands the call graph.
- NEVER commit changes without running `detect_changes()` to check affected scope.
## Resources
| Resource | Use for |
|----------|---------|
| `gitnexus://repo/qinglong/context` | Codebase overview, check index freshness |
| `gitnexus://repo/qinglong/clusters` | All functional areas |
| `gitnexus://repo/qinglong/processes` | All execution flows |
| `gitnexus://repo/qinglong/process/{name}` | Step-by-step execution trace |
## CLI
| Task | Read this skill file |
|------|---------------------|
| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
<!-- gitnexus:end -->
+11
View File
@@ -48,6 +48,17 @@ docker pull whyour/qinglong:latest
docker pull whyour/qinglong:debian
```
When running the `debian` image as a non-root user, specify `--user qinglong`:
```bash
docker run -d \
-v /path/to/ql/data:/ql/data \
-p 5700:5700 \
--user qinglong \
--name qinglong \
whyour/qinglong:debian
```
### npm
The npm version supports `debian/ubuntu/alpine` systems and requires `node/npm/python3/pip3/pnpm` to be installed.
+11
View File
@@ -50,6 +50,17 @@ docker pull whyour/qinglong:latest
docker pull whyour/qinglong:debian
```
使用 `debian` 镜像以非 root 用户运行时,需指定 `--user qinglong`:
```bash
docker run -d \
-v /path/to/ql/data:/ql/data \
-p 5700:5700 \
--user qinglong \
--name qinglong \
whyour/qinglong:debian
```
### npm
npm 版本支持 `debian/ubuntu/alpine` 系统,需要自行安装 `node/npm/python3/pip3/pnpm`
+19 -9
View File
@@ -4,8 +4,9 @@ import { Logger } from 'winston';
import config from '../config';
import * as fs from 'fs/promises';
import { celebrate, Joi } from 'celebrate';
import { join } from 'path';
import { join, basename } from 'path';
import { SAMPLE_FILES } from '../config/const';
import { t } from '../shared/i18n';
import ConfigService from '../services/config';
import { writeFileWithLock } from '../shared/utils';
const route = Router();
@@ -14,7 +15,7 @@ export default (app: Router) => {
app.use('/configs', route);
route.get(
'/sample',
'/samples',
async (req: Request, res: Response, next: NextFunction) => {
try {
res.send({
@@ -71,15 +72,24 @@ export default (app: Router) => {
const logger: Logger = Container.get('logger');
try {
const { name, content } = req.body;
if (config.blackFileList.includes(name)) {
res.send({ code: 403, message: '文件无法访问' });
}
let path = join(config.configPath, name);
// Resolve path first to prevent traversal attacks
let basePath = config.configPath;
if (name.startsWith('data/scripts/')) {
path = join(config.rootPath, name);
basePath = join(config.rootPath, 'data/scripts');
}
await writeFileWithLock(path, content);
res.send({ code: 200, message: '保存成功' });
const cleanName = name.replace(/^data\/scripts\//, '');
const resolvedPath = join(basePath, cleanName);
const normalized = join(resolvedPath);
// Verify the resolved path stays within allowed directory
if (!normalized.startsWith(basePath)) {
return res.send({ code: 403, message: t('文件路径无效') });
}
// Check blacklist on actual filename (not user input)
if (config.blackFileList.includes(basename(normalized))) {
return res.send({ code: 403, message: t('文件无法访问') });
}
await writeFileWithLock(normalized, content);
res.send({ code: 200, message: t('保存成功') });
} catch (e) {
return next(e);
}
+61 -3
View File
@@ -5,6 +5,8 @@ import CronService from '../services/cron';
import CronViewService from '../services/cronView';
import { celebrate, Joi } from 'celebrate';
import { commonCronSchema } from '../validation/schedule';
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
import { t } from '../shared/i18n';
const route = Router();
@@ -60,7 +62,7 @@ export default (app: Router) => {
try {
const cronViewService = Container.get(CronViewService);
if (req.body.type === 1) {
return res.send({ code: 400, message: '参数错误' });
return res.send({ code: 400, message: t('参数错误') });
} else {
const data = await cronViewService.update(req.body);
return res.send({ code: 200, data });
@@ -307,8 +309,12 @@ export default (app: Router) => {
const logger: Logger = Container.get('logger');
try {
const cronService = Container.get(CronService);
const data = await cronService.log(req.params.id);
return res.send({ code: 200, data });
const result = await cronService.log(req.params.id);
return res.send({
code: 200,
data: result.content,
logStatus: result.status,
});
} catch (e) {
return next(e);
}
@@ -429,6 +435,12 @@ export default (app: Router) => {
log_path: Joi.string().optional().allow(null),
last_running_time: Joi.number().optional().allow(null),
last_execution_time: Joi.number().optional().allow(null),
exit_code: Joi.number().optional().allow(null),
execution_id: Joi.string()
.pattern(
/^legacy-system:[1-9][0-9]{0,12}:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u,
)
.optional(),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
@@ -446,6 +458,52 @@ export default (app: Router) => {
},
);
route.get(
'/:id/instances',
celebrate({
params: Joi.object({
id: Joi.number().required(),
}),
}),
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
try {
const instances = await RunningInstanceModel.findAll({
where: {
cron_id: req.params.id,
},
order: [['started_at', 'DESC']],
raw: true,
});
return res.send({ code: 200, data: instances });
} catch (e) {
return next(e);
}
},
);
route.post(
'/:id/instances/:instanceId/stop',
celebrate({
params: Joi.object({
id: Joi.number().required(),
instanceId: Joi.number().required(),
}),
}),
async (
req: Request<{ id: number; instanceId: number }>,
res: Response,
next: NextFunction,
) => {
try {
const cronService = Container.get(CronService);
const data = await cronService.stopInstance(req.params.instanceId);
return res.send(data);
} catch (e) {
return next(e);
}
},
);
route.get(
'/:id/logs',
celebrate({
+397
View File
@@ -0,0 +1,397 @@
import { Router, Request, Response, NextFunction } from 'express';
import { Container } from 'typedi';
import { fn, col, where, Op } from 'sequelize';
import { CrontabModel } from '../data/cron';
import { CrontabStatModel } from '../data/cronStats';
import {
RunningInstanceModel,
InstanceStatus,
} from '../data/runningInstance';
import dayjs from 'dayjs';
import os from 'os';
import { isEmpty } from 'lodash';
import { t, tf } from '../shared/i18n';
const route = Router();
export default (app: Router) => {
app.use('/dashboard', route);
route.post('/record', async (req: Request, res: Response) => {
try {
const { ref_id, code, elapsed } = req.body;
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
const today = dayjs().format('YYYY-MM-DD');
const isSuccess = code === 0 ? 1 : 0;
const isFail = code !== 0 ? 1 : 0;
const elapsedMs = (Number(elapsed) || 0) * 1000;
const existing = await CrontabStatModel.findOne({
where: { ref_id: Number(ref_id), date: today },
});
if (existing) {
await CrontabStatModel.update(
{
run_count: (existing.run_count || 0) + 1,
success_count: (existing.success_count || 0) + isSuccess,
fail_count: (existing.fail_count || 0) + isFail,
total_time: (existing.total_time || 0) + elapsedMs,
max_time: Math.max(existing.max_time || 0, elapsedMs),
},
{ where: { id: existing.id } },
);
} else {
await CrontabStatModel.create({
ref_id: Number(ref_id),
date: today,
run_count: 1,
success_count: isSuccess,
fail_count: isFail,
total_time: elapsedMs,
max_time: elapsedMs,
});
}
res.send({ code: 200 });
} catch (e) {
res.send({ code: 500 });
}
});
route.get(
'/overview',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [total, enabled, disabled, stats] = await Promise.all([
CrontabModel.count(),
CrontabModel.count({ where: { isDisabled: 0 } }),
CrontabModel.count({ where: { isDisabled: 1 } }),
CrontabStatModel.findOne({
attributes: [
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
[fn('SUM', col('total_time')), 'total_time'],
],
where: { date: today },
raw: true,
}),
]);
const row = stats as any;
const totalRuns = Number(row?.total_runs) || 0;
const totalSuccess = Number(row?.total_success) || 0;
const totalFail = Number(row?.total_fail) || 0;
const totalTime = Number(row?.total_time) || 0;
res.send({
code: 200,
data: {
total,
enabled,
disabled,
todayRuns: totalRuns,
todaySuccess: totalSuccess,
todayFail: totalFail,
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
},
});
} catch (e) {
next(e);
}
},
);
route.get(
'/trend',
async (req: Request, res: Response, next: NextFunction) => {
try {
const days = parseInt(req.query.days as string) || 7;
const dates: string[] = [];
for (let i = days - 1; i >= 0; i--) {
dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD'));
}
const rows = (await CrontabStatModel.findAll({
attributes: [
'date',
[fn('SUM', col('run_count')), 'total_runs'],
[fn('SUM', col('success_count')), 'total_success'],
[fn('SUM', col('fail_count')), 'total_fail'],
],
where: {
date: { [Op.in]: dates },
},
group: ['date'],
order: [['date', 'ASC']],
raw: true,
})) as any[];
const dataMap: Record<string, any> = {};
rows.forEach((r: any) => {
dataMap[r.date] = {
total: Number(r.total_runs) || 0,
success: Number(r.total_success) || 0,
fail: Number(r.total_fail) || 0,
};
});
const data = dates.map((d) => ({
date: dayjs(d).format('MM-DD'),
...(dataMap[d] || { total: 0, success: 0, fail: 0 }),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-time',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('run_count')), 'run_count'],
[fn('MAX', col('max_time')), 'max_time'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('total_time')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
maxTime: Number(r.max_time),
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/top-count',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const rows = (await CrontabStatModel.findAll({
attributes: [
'ref_id',
[fn('SUM', col('run_count')), 'run_count'],
[fn('SUM', col('total_time')), 'total_time'],
[fn('SUM', col('success_count')), 'success_count'],
],
where: { date: today, run_count: { [Op.gt]: 0 } },
group: ['ref_id'],
order: [[fn('SUM', col('run_count')), 'DESC']],
limit: 5,
raw: true,
})) as any[];
const ids = rows.map((r) => Number(r.ref_id));
const crons = await CrontabModel.findAll({
where: { id: { [Op.in]: ids } },
raw: true,
});
const nameMap: Record<number, string> = {};
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
const data = rows.map((r: any, i) => ({
rank: i + 1,
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
runCount: Number(r.run_count),
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
successRate:
Number(r.run_count) > 0
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
: '0',
}));
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/runtime',
async (req: Request, res: Response, next: NextFunction) => {
try {
const runningInstances = await RunningInstanceModel.findAll({
where: {
status: InstanceStatus.running,
},
raw: true,
});
const queuedCrons = await CrontabModel.findAll({
where: {
status: 3, // queued
},
raw: true,
});
// Fetch cron names for running instances
const cronIds = [
...new Set(runningInstances.map((i: any) => i.cron_id)),
];
const crons =
cronIds.length > 0
? await CrontabModel.findAll({
where: { id: cronIds },
raw: true,
})
: [];
const cronMap = new Map(crons.map((c: any) => [c.id, c]));
const now = dayjs().unix();
const running = runningInstances.map((inst: any) => {
const cron = cronMap.get(inst.cron_id);
return {
instanceId: inst.id,
id: inst.cron_id,
name: cron?.name || cron?.command || tf('任务#%s', inst.cron_id),
pid: inst.pid,
elapsed: inst.started_at ? now - inst.started_at : 0,
logPath: inst.log_path,
};
});
const dayAgo = dayjs().subtract(24, 'hour').unix();
const idleTasks = await CrontabModel.findAll({
where: {
isDisabled: 0,
status: 1,
last_execution_time: { [Op.lt]: dayAgo },
},
order: [['last_execution_time', 'ASC']],
limit: 5,
raw: true,
});
res.send({
code: 200,
data: {
runningCount: running.length,
queuedCount: queuedCrons.length,
running,
idleTasks: idleTasks.map((c: any) => ({
id: c.id,
name: c.name || c.command || tf('任务#%s', c.id),
lastRun: c.last_execution_time
? dayjs.unix(c.last_execution_time).format('MM-DD HH:mm')
: '-',
})),
},
});
} catch (e) {
next(e);
}
},
);
route.get(
'/labels',
async (req: Request, res: Response, next: NextFunction) => {
try {
const today = dayjs().format('YYYY-MM-DD');
const [crons, stats] = (await Promise.all([
CrontabModel.findAll({ where: { isDisabled: 0 }, raw: true }),
CrontabStatModel.findAll({ where: { date: today }, raw: true }),
]));
const statMap: Record<number, any> = {};
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {};
crons.forEach((c) => {
let rawLabels = c.labels;
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
const labels: string[] = Array.isArray(rawLabels)
? [...new Set((rawLabels as string[]).filter((l: string) => !isEmpty(l)))]
: [];
if (labels.length === 0) {
labels.push(t('未分类'));
}
const st = statMap[c.id!];
labels.forEach((label: string) => {
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
labelMap[label].count += 1;
if (st) {
labelMap[label].runs += Number(st.run_count) || 0;
labelMap[label].success += Number(st.success_count) || 0;
labelMap[label].totalTime += Number(st.total_time) || 0;
}
});
});
const data = Object.entries(labelMap)
.map(([label, v]) => ({
label,
count: v.count,
todayRuns: v.runs,
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
}))
.sort((a, b) => b.todayRuns - a.todayRuns);
res.send({ code: 200, data });
} catch (e) {
next(e);
}
},
);
route.get(
'/system',
async (req: Request, res: Response, next: NextFunction) => {
try {
const memUsage = process.memoryUsage();
res.send({
code: 200,
data: {
platform: os.platform(),
uptime: Math.floor(process.uptime()),
memTotal: os.totalmem(),
memFree: os.freemem(),
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1),
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
cpus: os.cpus().length,
},
});
} catch (e) {
next(e);
}
},
);
};
+48 -2
View File
@@ -6,6 +6,7 @@ import { Container } from 'typedi';
import { Logger } from 'winston';
import config from '../config';
import { safeJSONParse } from '../config/util';
import { t } from '../shared/i18n';
import EnvService from '../services/env';
const route = Router();
@@ -18,6 +19,10 @@ const storage = multer.diskStorage({
},
});
const upload = multer({ storage: storage });
const labelSchema = Joi.array()
.items(Joi.string().trim().required())
.min(1)
.required();
export default (app: Router) => {
app.use('/envs', route);
@@ -44,6 +49,7 @@ export default (app: Router) => {
.required()
.pattern(/^[a-zA-Z_][0-9a-zA-Z_]*$/),
remarks: Joi.string().optional().allow(''),
labels: Joi.array().items(Joi.string().trim()).optional().allow(null),
}),
),
}),
@@ -52,7 +58,7 @@ export default (app: Router) => {
try {
const envService = Container.get(EnvService);
if (!req.body?.length) {
return res.send({ code: 400, message: '参数不正确' });
return res.send({ code: 400, message: t('参数不正确') });
}
const data = await envService.create(req.body);
return res.send({ code: 200, data });
@@ -70,6 +76,7 @@ export default (app: Router) => {
name: Joi.string().required(),
remarks: Joi.string().optional().allow('').allow(null),
id: Joi.number().required(),
labels: Joi.array().items(Joi.string().trim()).optional().allow(null),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
@@ -230,6 +237,44 @@ export default (app: Router) => {
},
);
route.post(
'/labels',
celebrate({
body: Joi.object({
ids: Joi.array().items(Joi.number().required()).min(1).required(),
labels: labelSchema,
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const envService = Container.get(EnvService);
const data = await envService.addLabels(req.body.ids, req.body.labels);
return res.send({ code: 200, data });
} catch (e) {
return next(e);
}
},
);
route.delete(
'/labels',
celebrate({
body: Joi.object({
ids: Joi.array().items(Joi.number().required()).min(1).required(),
labels: labelSchema,
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const envService = Container.get(EnvService);
const data = await envService.removeLabels(req.body.ids, req.body.labels);
return res.send({ code: 200, data });
} catch (e) {
return next(e);
}
},
);
route.post(
'/upload',
upload.single('env'),
@@ -248,13 +293,14 @@ export default (app: Router) => {
name: x.name,
value: x.value,
remarks: x.remarks,
labels: x.labels,
})),
);
return res.send({ code: 200, data: result });
} else {
return res.send({
code: 400,
message: '每条数据 name 或者 value 字段不能为空,参考导出文件格式',
message: t('每条数据 name 或者 value 字段不能为空,参考导出文件格式'),
});
}
} catch (e) {
+2
View File
@@ -10,6 +10,7 @@ import dependence from './dependence';
import system from './system';
import subscription from './subscription';
import update from './update';
import dashboard from './dashboard';
import health from './health';
export default () => {
@@ -25,6 +26,7 @@ export default () => {
system(app);
subscription(app);
update(app);
dashboard(app);
health(app);
return app;
+16 -5
View File
@@ -3,6 +3,7 @@ import { NextFunction, Request, Response, Router } from 'express';
import { Container } from 'typedi';
import { Logger } from 'winston';
import config from '../config';
import { t } from '../shared/i18n';
import {
getFileContentByName,
readDirs,
@@ -10,6 +11,7 @@ import {
rmPath,
} from '../config/util';
import LogService from '../services/log';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
const route = Router();
const blacklist = ['.tmp'];
@@ -42,11 +44,20 @@ export default (app: Router) => {
if (!finalPath || blacklist.includes(req.query.path as string)) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
const logPath = `${req.query.path as string}/${req.query.file as string}`;
const runningInstance = await RunningInstanceModel.findOne({
where: { log_path: logPath, status: InstanceStatus.running },
});
const content = await getFileContentByName(finalPath);
res.send({ code: 200, data: removeAnsi(content) });
res.send({
code: 200,
data: removeAnsi(content),
logStatus: runningInstance ? 'running' : undefined,
});
} catch (e) {
return next(e);
}
@@ -65,7 +76,7 @@ export default (app: Router) => {
if (!finalPath || blacklist.includes(req.query.path as string)) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
const content = await getFileContentByName(finalPath);
@@ -96,7 +107,7 @@ export default (app: Router) => {
if (!finalPath || blacklist.includes(path)) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
await rmPath(finalPath);
@@ -126,7 +137,7 @@ export default (app: Router) => {
if (!filePath) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
return res.download(filePath, filename, (err) => {
+49 -13
View File
@@ -1,4 +1,11 @@
import { fileExist, readDirs, readDir, rmPath, IFile } from '../config/util';
import {
fileExist,
isPathInside,
readDirs,
readDir,
rmPath,
IFile,
} from '../config/util';
import { Router, Request, Response, NextFunction } from 'express';
import { Container } from 'typedi';
import { Logger } from 'winston';
@@ -7,10 +14,18 @@ import * as fs from 'fs/promises';
import { celebrate, Joi } from 'celebrate';
import path, { join, parse } from 'path';
import ScriptService from '../services/script';
import { t } from '../shared/i18n';
import multer from 'multer';
import { writeFileWithLock } from '../shared/utils';
const route = Router();
function isPathAllowed(targetPath: string): boolean {
const resolved = path.resolve(targetPath);
return config.writePathList.some((rootPath) =>
isPathInside(rootPath, resolved),
);
}
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, config.scriptPath);
@@ -152,32 +167,44 @@ export default (app: Router) => {
if (!path.startsWith('/')) {
path = join(config.scriptPath, path);
}
if (config.writePathList.every((x) => !path.startsWith(x))) {
if (
config.writePathList.every(
(rootPath) => !isPathInside(rootPath, path),
)
) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
if (req.file) {
await fs.rename(req.file.path, join(path, filename));
const uploadPath = join(path, filename);
if (!isPathAllowed(uploadPath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
await fs.rename(req.file.path, uploadPath);
return res.send({ code: 200 });
}
if (directory) {
await fs.mkdir(join(path, directory), { recursive: true });
const dirPath = join(path, directory);
if (!isPathAllowed(dirPath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
await fs.mkdir(dirPath, { recursive: true });
return res.send({ code: 200 });
}
if (!originFilename) {
originFilename = filename;
}
const originFilePath = join(
path,
`${originFilename.replace(/\//g, '')}`,
);
const originFilePath = join(path, originFilename);
const filePath = join(path, filename);
if (!isPathAllowed(filePath) || !isPathAllowed(originFilePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
await fs.mkdir(path, { recursive: true });
const filePath = join(path, `${filename.replace(/\//g, '')}`);
const fileExists = await fileExist(filePath);
if (fileExists) {
await fs.copyFile(
@@ -217,7 +244,7 @@ export default (app: Router) => {
if (!filePath) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
await writeFileWithLock(filePath, content);
@@ -251,7 +278,7 @@ export default (app: Router) => {
if (!filePath) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
await rmPath(filePath);
@@ -284,7 +311,7 @@ export default (app: Router) => {
if (!filePath) {
return res.send({
code: 403,
message: '暂无权限',
message: t('暂无权限'),
});
}
return res.download(filePath, filename, (err) => {
@@ -316,6 +343,9 @@ export default (app: Router) => {
}
const { name, ext } = parse(filename);
const filePath = join(config.scriptPath, path, `${name}.swap${ext}`);
if (!isPathAllowed(filePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
await writeFileWithLock(filePath, content || '');
const scriptService = Container.get(ScriptService);
@@ -344,6 +374,9 @@ export default (app: Router) => {
}
const { name, ext } = parse(filename);
const filePath = join(config.scriptPath, path, `${name}.swap${ext}`);
if (!isPathAllowed(filePath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
const logPath = join(config.logPath, path, `${name}.swap`);
const scriptService = Container.get(ScriptService);
@@ -379,6 +412,9 @@ export default (app: Router) => {
}
const filePath = join(config.scriptPath, path, filename);
const newPath = join(config.scriptPath, path, newFilename);
if (!isPathAllowed(filePath) || !isPathAllowed(newPath)) {
return res.send({ code: 403, message: t('暂无权限') });
}
await fs.rename(filePath, newPath);
res.send({ code: 200 });
} catch (e) {
+44 -10
View File
@@ -6,6 +6,8 @@ import config from '../config';
import SystemService from '../services/system';
import { celebrate, Joi } from 'celebrate';
import UserService from '../services/user';
import { t } from '../shared/i18n';
import { isDefaultAuthInfo } from '../shared/auth';
import {
getUniqPath,
handleLogPath,
@@ -38,14 +40,7 @@ export default (app: Router) => {
const { version, changeLog, changeLogLink, publishTime } =
await parseVersion(config.versionFile);
let isInitialized = true;
if (
Object.keys(authInfo).length === 2 &&
authInfo.username === 'admin' &&
authInfo.password === 'admin'
) {
isInitialized = false;
}
const isInitialized = !isDefaultAuthInfo(authInfo);
res.send({
code: 200,
data: {
@@ -400,8 +395,11 @@ export default (app: Router) => {
async (req: Request, res: Response, next: NextFunction) => {
try {
const userService = Container.get(UserService);
await userService.resetAuthInfo(req.body);
res.send({ code: 200, message: '更新成功' });
const result = await userService.resetAuthInfo(req.body);
if (result) {
return res.send(result);
}
res.send({ code: 200, message: t('更新成功') });
} catch (e) {
return next(e);
}
@@ -426,6 +424,42 @@ export default (app: Router) => {
},
);
route.put(
'/config/lang',
celebrate({
body: Joi.object({
lang: Joi.string().allow('').allow(null),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const systemService = Container.get(SystemService);
const result = await systemService.updateLanguage(req.body);
res.send(result);
} catch (e) {
return next(e);
}
},
);
route.put(
'/config/panel-title',
celebrate({
body: Joi.object({
panelTitle: Joi.string().max(100).allow('').allow(null),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const systemService = Container.get(SystemService);
const result = await systemService.updatePanelTitle(req.body);
res.send(result);
} catch (e) {
return next(e);
}
},
);
route.put(
'/config/global-ssh-key',
celebrate({
+6 -5
View File
@@ -8,6 +8,7 @@ import path from 'path';
import { v4 as uuidV4 } from 'uuid';
import rateLimit from 'express-rate-limit';
import config from '../config';
import { t } from '../shared/i18n';
import { isDemoEnv, getToken } from '../config/util';
const route = Router();
@@ -76,11 +77,11 @@ export default (app: Router) => {
async (req: Request, res: Response, next: NextFunction) => {
try {
if (isDemoEnv()) {
return res.send({ code: 450, message: '未知错误' });
return res.send({ code: 450, message: t('未知错误') });
}
const userService = Container.get(UserService);
await userService.updateUsernameAndPassword(req.body);
res.send({ code: 200, message: '更新成功' });
res.send({ code: 200, message: t('更新成功') });
} catch (e) {
return next(e);
}
@@ -140,12 +141,12 @@ export default (app: Router) => {
);
route.put(
'/two-factor/deactive',
'/two-factor/deactivate',
async (req: Request, res: Response, next: NextFunction) => {
const logger: Logger = Container.get('logger');
try {
const userService = Container.get(UserService);
const data = await userService.deactiveTwoFactor();
const data = await userService.deactivateTwoFactor();
res.send({ code: 200, data });
} catch (e) {
return next(e);
@@ -229,7 +230,7 @@ export default (app: Router) => {
try {
const userService = Container.get(UserService);
await userService.updateUsernameAndPassword(req.body);
res.send({ code: 200, message: '更新成功' });
res.send({ code: 200, message: t('更新成功') });
} catch (e) {
return next(e);
}
+81 -27
View File
@@ -8,6 +8,7 @@ import { Container } from 'typedi';
import config from './config';
import Logger from './loaders/logger';
import { monitoringMiddleware } from './middlewares/monitoring';
import { errStack } from './config/util';
import { type GrpcServerService } from './services/grpc';
import { type HttpServerService } from './services/http';
@@ -22,6 +23,12 @@ class Application {
private app: express.Application;
private httpServerService?: HttpServerService;
private grpcServerService?: GrpcServerService;
private manualPrimaryRuntime?: {
stop(): Promise<'drained' | 'timed_out'>;
};
private legacyShadowCaptureEvidence?: {
close(): Promise<unknown>;
};
private isShuttingDown = false;
private workerMetadataMap = new Map<number, WorkerMetadata>();
private httpWorker?: Worker;
@@ -48,7 +55,7 @@ class Application {
await this.startWorkerProcess();
}
} catch (error) {
Logger.error('Failed to start application:', error);
Logger.error(`Failed to start application:\n${errStack(error)}`);
process.exit(1);
}
}
@@ -60,11 +67,11 @@ class Application {
// Wait for gRPC worker to signal it's ready before starting HTTP worker
this.waitForWorkerReady(grpcWorker, 30000)
.then(() => {
Logger.info('✌️ gRPC worker is ready, starting HTTP worker');
Logger.info('[boot] gRPC worker is ready, starting HTTP worker');
this.httpWorker = this.forkWorker('http');
})
.catch((error) => {
Logger.error('✌️ Failed to wait for gRPC worker:', error);
Logger.error(`[boot] Failed to wait for gRPC worker:\n${errStack(error)}`);
process.exit(1);
});
@@ -73,7 +80,7 @@ class Application {
if (metadata) {
if (!this.isShuttingDown) {
Logger.error(
`✌️ ${metadata.serviceType} worker ${worker.process.pid} died (${signal || code
`${metadata.serviceType} worker ${worker.process.pid} died (${signal || code
}). Restarting...`,
);
// If gRPC worker died, restart it and wait for it to be ready
@@ -81,26 +88,26 @@ class Application {
const newGrpcWorker = this.forkWorker('grpc');
this.waitForWorkerReady(newGrpcWorker, 30000)
.then(() => {
Logger.info('✌️ gRPC worker restarted and ready');
Logger.info('gRPC worker restarted and ready');
// Re-register cron jobs by notifying the HTTP worker
if (this.httpWorker) {
try {
this.httpWorker.send('reregister-crons');
Logger.info('✌️ Sent reregister-crons message to HTTP worker');
Logger.info('Sent reregister-crons message to HTTP worker');
} catch (error) {
Logger.error('✌️ Failed to send reregister-crons message:', error);
Logger.error(`Failed to send reregister-crons message:\n${errStack(error)}`);
}
}
})
.catch((error) => {
Logger.error('✌️ Failed to restart gRPC worker:', error);
Logger.error(`Failed to restart gRPC worker:\n${errStack(error)}`);
process.exit(1);
});
} else {
// For HTTP worker, just restart it
const newWorker = this.forkWorker(metadata.serviceType);
this.httpWorker = newWorker;
Logger.info(`✌️ Restarted ${metadata.serviceType} worker (PID: ${newWorker.process.pid})`);
Logger.info(`Restarted ${metadata.serviceType} worker (PID: ${newWorker.process.pid})`);
}
}
@@ -169,17 +176,14 @@ class Application {
if (worker) {
const exitPromise = new Promise<void>((resolve) => {
worker.once('exit', () => {
Logger.info(`✌️ Worker ${worker.process.pid} exited`);
Logger.info(`Worker ${worker.process.pid} exited`);
resolve();
});
try {
worker.send('shutdown');
} catch (error) {
Logger.warn(
`✌️ Failed to send shutdown to worker ${worker.process.pid}:`,
error,
);
Logger.warn(`Failed to send shutdown to worker ${worker.process.pid}:\n${errStack(error)}`);
}
});
@@ -192,14 +196,14 @@ class Application {
Promise.all(workerPromises),
new Promise<void>((resolve) => {
setTimeout(() => {
Logger.warn('✌️ Worker shutdown timeout reached');
Logger.warn('Worker shutdown timeout reached');
resolve();
}, 10000);
}),
]);
process.exit(0);
} catch (error) {
Logger.error('✌️ Error during worker shutdown:', error);
Logger.error(`Error during worker shutdown:\n${errStack(error)}`);
process.exit(1);
}
};
@@ -211,11 +215,11 @@ class Application {
private async startWorkerProcess() {
const serviceType = process.env.SERVICE_TYPE;
if (!serviceType || !['http', 'grpc'].includes(serviceType)) {
Logger.error('✌️ Invalid SERVICE_TYPE:', serviceType);
Logger.error('[boot] Invalid SERVICE_TYPE:', serviceType);
process.exit(1);
}
Logger.info(`✌️ ${serviceType} worker started (PID: ${process.pid})`);
Logger.info(`[boot] ${serviceType} worker started (PID: ${process.pid})`);
try {
if (serviceType === 'http') {
@@ -226,12 +230,16 @@ class Application {
process.send?.('ready');
} catch (error) {
Logger.error(`✌️ ${serviceType} worker failed:`, error);
Logger.error(`[boot] ${serviceType} worker failed:\n${errStack(error)}`);
process.exit(1);
}
}
private async startHttpService() {
// 在导入任何 gRPC 客户端模块之前初始化 mTLS 证书
const { initGrpcCerts } = await import('./config/grpcCerts');
await initGrpcCerts();
this.setupMiddlewares();
const { HttpServerService } = await import('./services/http');
@@ -240,10 +248,33 @@ class Application {
const appLoader = await import('./loaders/app');
await appLoader.default({ app: this.app });
const server = await this.httpServerService.initialize(
this.app,
config.port,
const { bootstrapLegacyShadowStartupReconciliation } = await import(
'./runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation'
);
const legacyShadowStartup =
await bootstrapLegacyShadowStartupReconciliation();
const { bootstrapLegacyShadowCaptureEvidence } = await import(
'./runtime/adapters/legacy/bootstrapLegacyShadowCaptureEvidence'
);
this.legacyShadowCaptureEvidence =
await bootstrapLegacyShadowCaptureEvidence({
startup: legacyShadowStartup,
});
const { bootstrapDefaultManualPrimaryRuntime } = await import(
'./runtime/adapters/legacy/bootstrapDefaultManualPrimaryRuntime'
);
this.manualPrimaryRuntime = await bootstrapDefaultManualPrimaryRuntime();
let server;
try {
server = await this.httpServerService.initialize(this.app, config.port);
} catch (error) {
await this.stopManualPrimaryRuntime();
await this.closeLegacyShadowCaptureEvidence();
throw error;
}
const serverLoader = await import('./loaders/server');
await (serverLoader.default as any)({ server });
@@ -265,13 +296,13 @@ class Application {
} else if (msg === 'reregister-crons' && serviceType === 'http') {
// Re-register cron jobs when gRPC worker restarts
try {
Logger.info('✌️ Received reregister-crons message, re-registering cron jobs...');
Logger.info('[boot] Received reregister-crons message, re-registering cron jobs...');
const CronService = (await import('./services/cron')).default;
const cronService = Container.get(CronService);
await cronService.autosave_crontab();
Logger.info('✌️ Cron jobs re-registered successfully');
Logger.info('[boot] Cron jobs re-registered successfully');
} catch (error) {
Logger.error('✌️ Failed to re-register cron jobs:', error);
Logger.error(`[boot] Failed to re-register cron jobs:\n${errStack(error)}`);
}
}
});
@@ -287,20 +318,43 @@ class Application {
try {
if (serviceType === 'http') {
await this.stopManualPrimaryRuntime();
await this.httpServerService?.shutdown();
await this.closeLegacyShadowCaptureEvidence();
} else {
await this.grpcServerService?.shutdown();
}
process.exit(0);
} catch (error) {
Logger.error(`✌️ [${serviceType}] Error during shutdown:`, error);
Logger.error(`[${serviceType}] Error during shutdown:\n${errStack(error)}`);
process.exit(1);
}
}
private async stopManualPrimaryRuntime(): Promise<void> {
const runtime = this.manualPrimaryRuntime;
this.manualPrimaryRuntime = undefined;
if (!runtime) return;
try {
const result = await runtime.stop();
if (result === 'timed_out') {
Logger.warn('[runtime-activation] shutdown timed out');
}
} catch {
Logger.error('[runtime-activation] shutdown failed');
}
}
private async closeLegacyShadowCaptureEvidence(): Promise<void> {
const evidence = this.legacyShadowCaptureEvidence;
this.legacyShadowCaptureEvidence = undefined;
if (!evidence) return;
await evidence.close();
}
}
const app = new Application();
app.start().catch((error) => {
Logger.error('🙅‍♀️ Application failed to start:', error);
Logger.error(`🙅‍♀️ Application failed to start:\n${errStack(error)}`);
process.exit(1);
});
+38
View File
@@ -1,3 +1,5 @@
import { maybeSudo } from './container';
export const LOG_END_SYMBOL = '     ';
export const TASK_COMMAND = 'task';
@@ -48,4 +50,40 @@ export const NotificationModeStringMap = {
18: 'chronocat',
19: 'ntfy',
20: 'wxPusherBot',
21: 'wxPusherSpt',
} as const;
export const LINUX_DEPENDENCE_COMMAND: Record<
'Debian' | 'Ubuntu' | 'Alpine',
{
install: string;
uninstall: string;
info: string;
check(info: string): boolean;
}
> = {
Debian: {
install: maybeSudo('apt-get install -y'),
uninstall: maybeSudo('apt-get remove -y'),
info: maybeSudo('dpkg-query -s'),
check(info: string) {
return info.includes('install ok installed');
},
},
Ubuntu: {
install: maybeSudo('apt-get install -y'),
uninstall: maybeSudo('apt-get remove -y'),
info: maybeSudo('dpkg-query -s'),
check(info: string) {
return info.includes('install ok installed');
},
},
Alpine: {
install: 'apk add --no-check-certificate',
uninstall: 'apk del',
info: 'apk info -es',
check(info: string) {
return info.includes('installed');
},
},
};
+7
View File
@@ -0,0 +1,7 @@
export function isInContainer(): boolean {
return process.env.QL_CONTAINER === 'true';
}
export function maybeSudo(cmd: string): string {
return isInContainer() ? `sudo ${cmd}` : cmd;
}
+150
View File
@@ -0,0 +1,150 @@
import { execSync } from 'child_process';
import * as fs from 'fs/promises';
import * as os from 'os';
import path from 'path';
import config from './index';
import { fileExist } from './util';
import Logger from '../loaders/logger';
export interface GrpcTlsConfig {
caCert: string;
serverCert: string;
serverKey: string;
clientCert: string;
clientKey: string;
}
const certDir = path.join(config.configPath, 'grpc');
const caKeyPath = path.join(certDir, 'ca.key');
const caCertPath = path.join(certDir, 'ca.crt');
const serverKeyPath = path.join(certDir, 'server.key');
const serverCertPath = path.join(certDir, 'server.crt');
const clientKeyPath = path.join(certDir, 'client.key');
const clientCertPath = path.join(certDir, 'client.crt');
let cachedConfig: GrpcTlsConfig | null = null;
function run(cmd: string, execOpts?: Record<string, unknown>): string {
const opts = { stdio: 'pipe', timeout: 30000, encoding: 'utf-8', ...execOpts } as any;
return (execSync(cmd, opts) as string).trim();
}
async function tmpFile(prefix: string): Promise<string> {
const dir = (await fileExist(certDir)) ? certDir : os.tmpdir();
await fs.mkdir(dir, { recursive: true });
return path.join(dir, `.${prefix}_${Date.now()}_${Math.random().toString(36).slice(2)}.pem`);
}
async function generateAllCerts(): Promise<GrpcTlsConfig> {
Logger.info('[boot] Generating gRPC mTLS certificates...');
const caKeyTmp = await tmpFile('ca_key');
const caCertTmp = await tmpFile('ca_cert');
const serverKeyTmp = await tmpFile('server_key');
const serverCsrTmp = await tmpFile('server_csr');
const serverExtTmp = await tmpFile('server_ext');
const clientKeyTmp = await tmpFile('client_key');
const clientCsrTmp = await tmpFile('client_csr');
const clientExtTmp = await tmpFile('client_ext');
const srlTmp = path.join(path.dirname(caKeyTmp), '.grpc_ca.srl');
const cleanup = async () => {
for (const f of [caKeyTmp, caCertTmp, serverKeyTmp, serverCsrTmp, serverExtTmp,
clientKeyTmp, clientCsrTmp, clientExtTmp, srlTmp]) {
try { await fs.unlink(f); } catch {}
}
};
try {
// 1. CA(私钥直接存盘,证书写入临时文件供签发使用)
run(`openssl genrsa -out '${caKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -x509 -days 3650 -key '${caKeyTmp}' -out '${caCertTmp}' -subj '/CN=qinglong-ca/O=qinglong/C=CN' 2>/dev/null`);
const caKey = await fs.readFile(caKeyTmp, 'utf-8');
const caCert = await fs.readFile(caCertTmp, 'utf-8');
await fs.mkdir(certDir, { recursive: true });
await fs.writeFile(caKeyPath, caKey, { mode: 0o600 });
// 2. 服务端
run(`openssl genrsa -out '${serverKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -key '${serverKeyTmp}' -out '${serverCsrTmp}' -subj '/CN=grpc-server' 2>/dev/null`);
await fs.writeFile(serverExtTmp, 'subjectAltName=DNS:localhost,IP:127.0.0.1,IP:::1\n');
const serverCert = run(
`openssl x509 -req -days 3650 -in '${serverCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${serverExtTmp}' 2>/dev/null`,
);
const serverKey = await fs.readFile(serverKeyTmp, 'utf-8');
// 3. 客户端
run(`openssl genrsa -out '${clientKeyTmp}' 2048 2>/dev/null`);
run(`openssl req -new -key '${clientKeyTmp}' -out '${clientCsrTmp}' -subj '/CN=grpc-client' 2>/dev/null`);
await fs.writeFile(clientExtTmp, 'extendedKeyUsage=clientAuth\n');
const clientCert = run(
`openssl x509 -req -days 3650 -in '${clientCsrTmp}' -CA '${caCertTmp}' -CAkey '${caKeyTmp}' -CAcreateserial -extfile '${clientExtTmp}' 2>/dev/null`,
);
const clientKey = await fs.readFile(clientKeyTmp, 'utf-8');
await cleanup();
Logger.info('[boot] gRPC mTLS certificates generated successfully');
return { caCert, serverCert, serverKey, clientCert, clientKey };
} catch (e) {
await cleanup();
throw e;
}
}
async function saveCerts(tlsConfig: GrpcTlsConfig): Promise<void> {
await fs.mkdir(certDir, { recursive: true });
await fs.writeFile(caCertPath, tlsConfig.caCert, { mode: 0o644 });
await fs.writeFile(serverCertPath, tlsConfig.serverCert, { mode: 0o644 });
await fs.writeFile(serverKeyPath, tlsConfig.serverKey, { mode: 0o600 });
await fs.writeFile(clientCertPath, tlsConfig.clientCert, { mode: 0o644 });
await fs.writeFile(clientKeyPath, tlsConfig.clientKey, { mode: 0o600 });
Logger.info(`[boot] gRPC mTLS certificates saved to ${certDir}`);
}
async function loadExistingCerts(): Promise<GrpcTlsConfig | null> {
const exists = await Promise.all([
fileExist(caCertPath),
fileExist(serverCertPath),
fileExist(serverKeyPath),
fileExist(clientCertPath),
fileExist(clientKeyPath),
]);
if (exists.some((e) => !e)) {
return null;
}
const [caCert, serverCert, serverKey, clientCert, clientKey] = await Promise.all([
fs.readFile(caCertPath, 'utf-8'),
fs.readFile(serverCertPath, 'utf-8'),
fs.readFile(serverKeyPath, 'utf-8'),
fs.readFile(clientCertPath, 'utf-8'),
fs.readFile(clientKeyPath, 'utf-8'),
]);
Logger.info('[boot] Loaded existing gRPC mTLS certificates from disk');
return { caCert, serverCert, serverKey, clientCert, clientKey };
}
export async function initGrpcCerts(): Promise<GrpcTlsConfig> {
if (cachedConfig) {
return cachedConfig;
}
let tlsConfig = await loadExistingCerts();
if (!tlsConfig) {
tlsConfig = await generateAllCerts();
await saveCerts(tlsConfig);
}
cachedConfig = tlsConfig;
return tlsConfig;
}
export function getGrpcCerts(): GrpcTlsConfig | null {
return cachedConfig;
}
+8 -5
View File
@@ -1,6 +1,5 @@
import dotenv from 'dotenv';
import path from 'path';
import { createRandomString } from './share';
dotenv.config({
path: path.join(__dirname, '../../.env'),
@@ -9,6 +8,8 @@ dotenv.config({
interface Config {
port: number;
grpcPort: number;
bindHost: string;
bindHostGrpc: string;
nodeEnv: string;
isDevelopment: boolean;
isProduction: boolean;
@@ -31,6 +32,8 @@ interface Config {
const config: Config = {
port: parseInt(process.env.BACK_PORT || '5700', 10),
grpcPort: parseInt(process.env.GRPC_PORT || '5500', 10),
bindHost: process.env.BIND_HOST || '::',
bindHostGrpc: process.env.BIND_HOST_GRPC || '::',
nodeEnv: process.env.NODE_ENV || 'development',
isDevelopment: process.env.NODE_ENV === 'development',
isProduction: process.env.NODE_ENV === 'production',
@@ -106,6 +109,7 @@ const jsEnvFile = path.join(preloadPath, 'env.js');
const pyEnvFile = path.join(preloadPath, 'env.py');
const jsNotifyFile = path.join(preloadPath, '__ql_notify__.js');
const pyNotifyFile = path.join(preloadPath, '__ql_notify__.py');
const langEnvFile = path.join(preloadPath, 'lang_env.sh');
const confFile = path.join(configPath, 'config.sh');
const crontabFile = path.join(configPath, 'crontab.list');
const authConfigFile = path.join(configPath, 'auth.json');
@@ -114,8 +118,6 @@ const confBakDir = path.join(dataPath, 'config/bak/');
const sampleFile = path.join(samplePath, 'config.sample.sh');
const sqliteFile = path.join(samplePath, 'database.sqlite');
const authError = '错误的用户名密码,请重试';
const loginFaild = '请先登录!';
const configString = 'config sample crontab shareCode diy';
const versionFile = path.join(rootPath, 'version.yaml');
const dataTgzFile = path.join(tmpPath, 'data.tgz');
@@ -137,8 +139,6 @@ export default {
shareShellFile,
dependenceProxyFile,
configString,
loginFaild,
authError,
logPath,
extraFile,
authConfigFile,
@@ -151,6 +151,7 @@ export default {
pyEnvFile,
jsNotifyFile,
pyNotifyFile,
langEnvFile,
dbPath,
uploadPath,
configPath,
@@ -167,6 +168,8 @@ export default {
'env.js',
'env.py',
'token.json',
'grpc',
'__pycache__',
],
writePathList: [configPath, scriptPath],
bakPath,
+226 -12
View File
@@ -1,6 +1,6 @@
import * as fs from 'fs/promises';
import * as path from 'path';
import { exec } from 'child_process';
import { exec, execSync } from 'child_process';
import psTreeFun from 'ps-tree';
import { promisify } from 'util';
import { load } from 'js-yaml';
@@ -10,9 +10,39 @@ import Logger from '../loaders/logger';
import { writeFileWithLock } from '../shared/utils';
import { DependenceTypes } from '../data/dependence';
import { FormData } from 'undici';
import os from 'os';
import { maybeSudo, isInContainer } from './container';
export * from './share';
let osType: 'Debian' | 'Ubuntu' | 'Alpine' | undefined;
function getOsTypeSync(): 'Debian' | 'Ubuntu' | 'Alpine' | undefined {
// 1. 环境变量覆盖
const envOs = process.env.QL_OS_TYPE?.toLowerCase();
if (envOs === 'alpine') return 'Alpine';
if (envOs === 'debian') return 'Debian';
if (envOs === 'ubuntu') return 'Ubuntu';
// 2. 模块缓存(由 detectOS 设置)
if (osType) return osType;
// 3. 能力检测:检查包管理器二进制
try {
execSync('which apt-get', { stdio: 'ignore' });
return 'Debian';
} catch {
try {
execSync('which apk', { stdio: 'ignore' });
return 'Alpine';
} catch {
// macOS / 未知系统
}
}
return undefined;
}
export async function getFileContentByName(fileName: string) {
const _exsit = await fileExist(fileName);
if (_exsit) {
@@ -233,10 +263,26 @@ export async function readDir(
baseDir: string = '',
blacklist: string[] = [],
): Promise<IFile[]> {
const absoluteDir = path.join(baseDir, dir);
const relativePath = path.relative(baseDir, absoluteDir);
const absoluteBaseDir = path.resolve(baseDir);
const absoluteDir = path.resolve(baseDir, dir);
if (!isPathInside(absoluteBaseDir, absoluteDir)) {
return [];
}
try {
const [realBaseDir, realDirectory] = await Promise.all([
fs.realpath(absoluteBaseDir),
fs.realpath(absoluteDir),
]);
const directoryStat = await fs.lstat(absoluteDir);
if (
!isPathInside(realBaseDir, realDirectory) ||
!directoryStat.isDirectory() ||
directoryStat.isSymbolicLink()
) {
return [];
}
const relativePath = path.relative(absoluteBaseDir, absoluteDir);
const files = await fs.readdir(absoluteDir);
const result: IFile[] = [];
@@ -279,6 +325,19 @@ export async function readDir(
}
}
export function isPathInside(rootPath: string, targetPath: string): boolean {
const relative = path.relative(
path.resolve(rootPath),
path.resolve(targetPath),
);
return (
relative === '' ||
(relative !== '..' &&
!relative.startsWith(`..${path.sep}`) &&
!path.isAbsolute(relative))
);
}
export async function promiseExec(command: string): Promise<string> {
try {
const { stderr, stdout } = await promisify(exec)(command, {
@@ -405,7 +464,7 @@ export async function killTask(pid: number) {
[pid, ...pids].reverse().forEach((x) => {
process.kill(x, 15);
});
} catch (error) {}
} catch (error) { }
} else {
process.kill(pid, 2);
}
@@ -503,11 +562,17 @@ export function safeJSONParse(value?: string) {
try {
return JSON.parse(value);
} catch (error) {
Logger.error('[safeJSONParse失败]', error);
Logger.error('[safeJSONParse error]', error);
return {};
}
}
export function errStack(error: unknown): string {
return error instanceof Error && error.stack
? error.stack
: String(error);
}
export async function rmPath(path: string) {
try {
const _exsit = await fileExist(path);
@@ -515,7 +580,7 @@ export async function rmPath(path: string) {
await fs.rm(path, { force: true, recursive: true, maxRetries: 5 });
}
} catch (error) {
Logger.error('[rmPath失败]', error);
Logger.error('[rmPath error]', error);
}
}
@@ -525,12 +590,12 @@ export async function setSystemTimezone(timezone: string): Promise<boolean> {
throw new Error('Invalid timezone');
}
await promiseExec(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`);
await promiseExec(`echo "${timezone}" > /etc/timezone`);
await promiseExec(maybeSudo(`ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`));
await promiseExec(`echo "${timezone}" | ${maybeSudo('tee /etc/timezone')}`);
return true;
} catch (error) {
Logger.error('[setSystemTimezone失败]', error);
Logger.error('[setSystemTimezone error]', error);
return false;
}
}
@@ -550,7 +615,9 @@ except:
spec=u.find_spec(name)
print(name if spec else '')
''')"`,
[DependenceTypes.linux]: `apk info -es ${name}`,
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
? `apk info -es ${name}`
: maybeSudo(`dpkg-query -s ${name}`),
};
return baseCommands[type];
@@ -561,7 +628,9 @@ export function getInstallCommand(type: DependenceTypes, name: string): string {
[DependenceTypes.nodejs]: 'pnpm add -g',
[DependenceTypes.python3]:
'pip3 install --disable-pip-version-check --root-user-action=ignore',
[DependenceTypes.linux]: 'apk add --no-check-certificate',
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
? 'apk add --no-check-certificate'
: maybeSudo('apt-get install -y'),
};
let command = baseCommands[type];
@@ -581,7 +650,9 @@ export function getUninstallCommand(
[DependenceTypes.nodejs]: 'pnpm remove -g',
[DependenceTypes.python3]:
'pip3 uninstall --disable-pip-version-check --root-user-action=ignore -y',
[DependenceTypes.linux]: 'apk del',
[DependenceTypes.linux]: getOsTypeSync() === 'Alpine'
? 'apk del'
: maybeSudo('apt-get remove -y'),
};
return `${baseCommands[type]} ${name.trim()}`;
@@ -590,3 +661,146 @@ export function getUninstallCommand(
export function isDemoEnv() {
return process.env.DeployEnv === 'demo';
}
async function getOSReleaseInfo(): Promise<string> {
const osRelease = await fs.readFile('/etc/os-release', 'utf8');
return osRelease;
}
function isDebian(osReleaseInfo: string): boolean {
return osReleaseInfo.includes('Debian');
}
function isUbuntu(osReleaseInfo: string): boolean {
return osReleaseInfo.includes('Ubuntu');
}
function isCentOS(osReleaseInfo: string): boolean {
return osReleaseInfo.includes('CentOS') || osReleaseInfo.includes('Red Hat');
}
function isAlpine(osReleaseInfo: string): boolean {
return osReleaseInfo.includes('Alpine');
}
export async function detectOS(): Promise<
'Debian' | 'Ubuntu' | 'Alpine' | undefined
> {
if (osType) return osType;
const envOs = process.env.QL_OS_TYPE?.toLowerCase();
if (envOs === 'alpine') {
osType = 'Alpine';
return osType;
}
if (envOs === 'debian') {
osType = 'Debian';
return osType;
}
if (envOs === 'ubuntu') {
osType = 'Ubuntu';
return osType;
}
const platform = os.platform();
if (platform === 'linux') {
const osReleaseInfo = await getOSReleaseInfo();
if (isDebian(osReleaseInfo)) {
osType = 'Debian';
} else if (isUbuntu(osReleaseInfo)) {
osType = 'Ubuntu';
} else if (isAlpine(osReleaseInfo)) {
osType = 'Alpine';
} else {
Logger.error(`Unknown Linux Distribution: ${osReleaseInfo}`);
console.error(`Unknown Linux Distribution: ${osReleaseInfo}`);
}
} else if (platform === 'darwin') {
osType = undefined;
} else {
Logger.error(`Unsupported platform: ${platform}`);
console.error(`Unsupported platform: ${platform}`);
}
return osType;
}
async function getCurrentMirrorDomain(
filePath: string,
): Promise<string | null> {
const fileContent = await fs.readFile(filePath, 'utf8');
const lines = fileContent.split('\n');
for (const line of lines) {
if (line.trim().startsWith('#')) {
continue;
}
const match = line.match(/https?:\/\/[^\/]+/);
if (match) {
return match[0];
}
}
return null;
}
async function replaceDomainInFile(
filePath: string,
oldDomainWithScheme: string,
newDomainWithScheme: string,
): Promise<void> {
let fileContent = await fs.readFile(filePath, 'utf8');
let updatedContent = fileContent.replace(
new RegExp(oldDomainWithScheme, 'g'),
newDomainWithScheme,
);
if (!newDomainWithScheme.endsWith('/')) {
newDomainWithScheme += '/';
}
await writeFileWithLock(filePath, updatedContent);
}
async function _updateLinuxMirror(
osType: string,
mirrorDomainWithScheme: string,
): Promise<string> {
const S = isInContainer() ? 'sudo ' : '';
let filePath: string, currentDomainWithScheme: string | null;
switch (osType) {
case 'Debian':
filePath = '/etc/apt/sources.list.d/debian.sources';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://deb.debian.org'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
} else {
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://deb.debian.org'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates\\nComponents: main\\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
}
case 'Ubuntu':
filePath = '/etc/apt/sources.list.d/ubuntu.sources';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `${S}sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}|g' ${filePath} || (${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath}) && ${S}apt-get update`;
} else {
return `${S}mkdir -p /etc/apt/sources.list.d && echo -e "Types: deb\\nURIs: ${mirrorDomainWithScheme || 'http://archive.ubuntu.com'}\\nSuites: \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-updates \\$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2)-backports\\nComponents: main restricted universe multiverse\\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" | ${S}tee ${filePath} && ${S}apt-get update`;
}
case 'Alpine':
filePath = '/etc/apk/repositories';
currentDomainWithScheme = await getCurrentMirrorDomain(filePath);
if (currentDomainWithScheme) {
return `sed -i 's|${currentDomainWithScheme}|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}|g' ${filePath} || (mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}/alpine/v|" | tee ${filePath}) && apk update`;
} else {
return `mkdir -p /etc/apk && echo -e "\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/main\\n\\$(grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1,2)/community" | sed "s|^|${mirrorDomainWithScheme || 'http://dl-cdn.alpinelinux.org'}/alpine/v|" | tee ${filePath} && apk update`;
}
default:
throw Error('Unsupported OS type for updating mirrors.');
}
}
export async function updateLinuxMirrorFile(mirror: string): Promise<string> {
const detectedOS = await detectOS();
if (!detectedOS) {
throw Error(`Unknown Linux Distribution`);
}
return await _updateLinuxMirror(detectedOS, mirror);
}
+5 -2
View File
@@ -23,6 +23,7 @@ export class Crontab {
task_after?: string;
log_name?: string;
allow_multiple_instances?: 1 | 0;
work_dir?: string;
constructor(options: Crontab) {
this.name = options.name;
@@ -49,14 +50,15 @@ export class Crontab {
this.task_after = options.task_after;
this.log_name = options.log_name;
this.allow_multiple_instances = options.allow_multiple_instances || 0;
this.work_dir = options.work_dir;
}
}
export enum CrontabStatus {
'running' = 0,
'queued' = 0.5,
'idle' = 1,
'disabled',
'disabled' = 2,
'queued' = 3,
}
export interface CronInstance extends Model<Crontab, Crontab>, Crontab {}
@@ -90,4 +92,5 @@ export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
task_after: DataTypes.STRING,
log_name: DataTypes.STRING,
allow_multiple_instances: DataTypes.NUMBER,
work_dir: DataTypes.STRING,
});
+71
View File
@@ -0,0 +1,71 @@
import { DataTypes, Model } from 'sequelize';
import { sequelize } from '.';
export class CrontabStat {
id?: number;
ref_id!: number;
date!: string;
run_count?: number;
success_count?: number;
fail_count?: number;
total_time?: number;
max_time?: number;
constructor(options: CrontabStat) {
this.id = options.id;
this.ref_id = options.ref_id;
this.date = options.date;
this.run_count = options.run_count || 0;
this.success_count = options.success_count || 0;
this.fail_count = options.fail_count || 0;
this.total_time = options.total_time || 0;
this.max_time = options.max_time || 0;
}
}
export interface CrontabStatInstance extends Model<CrontabStat, CrontabStat>, CrontabStat {}
export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
'CrontabStat',
{
id: {
type: DataTypes.INTEGER,
primaryKey: true,
autoIncrement: true,
},
ref_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
date: {
type: DataTypes.STRING,
allowNull: false,
},
run_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
success_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
fail_count: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
total_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
max_time: {
type: DataTypes.NUMBER,
defaultValue: 0,
},
},
{
indexes: [
{ unique: true, fields: ['ref_id', 'date'] },
{ fields: ['date'] },
],
},
);
+3
View File
@@ -10,6 +10,7 @@ export class Env {
name?: string;
remarks?: string;
isPinned?: 1 | 0;
labels?: string[];
constructor(options: Env) {
this.value = options.value;
@@ -23,6 +24,7 @@ export class Env {
this.name = options.name;
this.remarks = options.remarks || '';
this.isPinned = options.isPinned || 0;
this.labels = options.labels || [];
}
}
@@ -45,4 +47,5 @@ export const EnvModel = sequelize.define<EnvInstance>('Env', {
name: { type: DataTypes.STRING, unique: 'compositeIndex' },
remarks: DataTypes.STRING,
isPinned: DataTypes.NUMBER,
labels: DataTypes.JSON,
});
+14 -1
View File
@@ -20,6 +20,7 @@ export enum NotificationMode {
'chronocat' = 'Chronocat',
'ntfy' = 'ntfy',
'wxPusherBot' = 'wxPusherBot',
'openiLink' = 'openiLink',
}
abstract class NotificationBaseInfo {
@@ -161,6 +162,16 @@ export class WxPusherBotNotification extends NotificationBaseInfo {
public wxPusherBotUids = '';
}
export class WxPusherSptNotification extends NotificationBaseInfo {
public wxPusherSptList = '';
}
export class OpeniLinkNotification extends NotificationBaseInfo {
public openiLinkAppToken = '';
public openiLinkHubUrl = '';
public openiLinkContextToken = '';
}
export interface NotificationInfo
extends GoCqHttpBotNotification,
GotifyNotification,
@@ -182,4 +193,6 @@ export interface NotificationInfo
ChronocatNotification,
LarkNotification,
NtfyNotification,
WxPusherBotNotification {}
WxPusherBotNotification,
WxPusherSptNotification,
OpeniLinkNotification {}
+1 -1
View File
@@ -24,7 +24,7 @@ export interface AppToken {
expiration: number;
}
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system';
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard';
export interface AppInstance extends Model<App, App>, App {}
export const AppModel = sequelize.define<AppInstance>('App', {
+96
View File
@@ -0,0 +1,96 @@
import { sequelize } from '.';
import { DataTypes, Model } from 'sequelize';
export enum InstanceStatus {
'running' = 0,
'finished' = 1,
'stopped' = 2,
'error' = 3,
}
export interface RunningInstanceAttributes {
id?: number;
cron_id: number;
run_id?: string | null;
attempt_id?: string | null;
pid?: number;
log_path?: string;
started_at: number;
finished_at?: number;
status: InstanceStatus;
exit_code?: number;
}
export class RunningInstance {
id?: number;
cron_id!: number;
run_id?: string | null;
attempt_id?: string | null;
pid?: number;
log_path?: string;
started_at!: number;
finished_at?: number;
status!: InstanceStatus;
exit_code?: number;
constructor(options: RunningInstanceAttributes) {
this.id = options.id;
this.cron_id = options.cron_id;
this.run_id = options.run_id;
this.attempt_id = options.attempt_id;
this.pid = options.pid;
this.log_path = options.log_path;
this.started_at = options.started_at;
this.finished_at = options.finished_at;
this.status = options.status;
this.exit_code = options.exit_code;
}
}
export interface RunningInstanceModel
extends Model<RunningInstanceAttributes, RunningInstanceAttributes>,
RunningInstanceAttributes {}
export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
'RunningInstance',
{
cron_id: {
type: DataTypes.NUMBER,
allowNull: false,
},
run_id: {
type: DataTypes.STRING(36),
allowNull: true,
},
attempt_id: {
type: DataTypes.STRING(36),
allowNull: true,
unique: 'running_instances_attempt_uidx',
},
pid: {
type: DataTypes.NUMBER,
allowNull: true,
},
log_path: {
type: DataTypes.STRING,
allowNull: true,
},
started_at: {
type: DataTypes.NUMBER,
allowNull: false,
},
finished_at: {
type: DataTypes.NUMBER,
allowNull: true,
},
status: {
type: DataTypes.NUMBER,
allowNull: false,
defaultValue: InstanceStatus.running,
},
exit_code: {
type: DataTypes.NUMBER,
allowNull: true,
},
},
);
+40
View File
@@ -0,0 +1,40 @@
import { DataTypes, Model, ModelStatic, Sequelize } from 'sequelize';
import { sequelize } from '.';
export interface SchemaMigrationAttributes {
id: string;
checksum: string;
applied_at: number;
}
export interface SchemaMigrationInstance
extends Model<SchemaMigrationAttributes, SchemaMigrationAttributes>,
SchemaMigrationAttributes {}
export function defineSchemaMigrationModel(
database: Sequelize,
): ModelStatic<SchemaMigrationInstance> {
return database.define<SchemaMigrationInstance>(
'SchemaMigration',
{
id: {
type: DataTypes.STRING,
primaryKey: true,
},
checksum: {
type: DataTypes.STRING,
allowNull: false,
},
applied_at: {
type: DataTypes.BIGINT,
allowNull: false,
},
},
{
tableName: 'SchemaMigrations',
timestamps: false,
},
);
}
export const SchemaMigrationModel = defineSchemaMigrationModel(sequelize);
+2
View File
@@ -2,11 +2,13 @@ export class SockMessage {
message?: string;
type?: SockMessageType;
references?: number[];
status?: number | string;
constructor(options: SockMessage) {
this.type = options.type;
this.message = options.message;
this.references = options.references;
this.status = options.status;
}
}
+2
View File
@@ -31,6 +31,8 @@ export enum AuthDataType {
}
export interface SystemConfigInfo {
lang?: string;
panelTitle?: string;
logRemoveFrequency?: number;
cronConcurrency?: number;
dependenceProxy?: string;
+7 -7
View File
@@ -9,20 +9,20 @@ import initFile from './initFile';
export default async ({ app }: { app: Application }) => {
depInjectorLoader();
Logger.info('✌️ Dependency loaded');
Logger.info('[boot] Dependency loaded');
await linkDeps();
Logger.info('✌️ Link deps loaded');
Logger.info('[boot] Link deps loaded');
initFile();
Logger.info('✌️ Init file loaded');
await initFile();
Logger.info('[boot] Init file loaded');
await initData();
Logger.info('✌️ Init data loaded');
Logger.info('[boot] Init data loaded');
initTask();
Logger.info('✌️ Init task loaded');
Logger.info('[boot] Init task loaded');
expressLoader({ app });
Logger.info('✌️ Express loaded');
Logger.info('[boot] Express loaded');
};
+9 -37
View File
@@ -6,7 +6,9 @@ import { AppModel } from '../data/open';
import { SystemModel } from '../data/system';
import { SubscriptionModel } from '../data/subscription';
import { CrontabViewModel } from '../data/cronView';
import { sequelize } from '../data';
import { CrontabStatModel } from '../data/cronStats';
import { RunningInstanceModel } from '../data/runningInstance';
import { runMigrations } from '../migrations/runner';
export default async () => {
try {
@@ -17,43 +19,13 @@ export default async () => {
await EnvModel.sync();
await SubscriptionModel.sync();
await CrontabViewModel.sync();
await CrontabStatModel.sync();
await RunningInstanceModel.sync();
await runMigrations();
// 初始化新增字段
const migrations = [
{
table: 'CrontabViews',
column: 'filterRelation',
type: 'VARCHAR(255)',
},
{ table: 'Subscriptions', column: 'proxy', type: 'VARCHAR(255)' },
{ table: 'CrontabViews', column: 'type', type: 'NUMBER' },
{ table: 'Subscriptions', column: 'autoAddCron', type: 'NUMBER' },
{ table: 'Subscriptions', column: 'autoDelCron', type: 'NUMBER' },
{ table: 'Crontabs', column: 'sub_id', type: 'NUMBER' },
{ table: 'Crontabs', column: 'extra_schedules', type: 'JSON' },
{ table: 'Crontabs', column: 'task_before', type: 'TEXT' },
{ table: 'Crontabs', column: 'task_after', type: 'TEXT' },
{ table: 'Crontabs', column: 'log_name', type: 'VARCHAR(255)' },
{
table: 'Crontabs',
column: 'allow_multiple_instances',
type: 'NUMBER',
},
{ table: 'Envs', column: 'isPinned', type: 'NUMBER' },
];
for (const migration of migrations) {
try {
await sequelize.query(
`alter table ${migration.table} add column ${migration.column} ${migration.type}`,
);
} catch (error) {
// Column already exists or other error, continue
}
}
Logger.info('✌️ DB loaded');
Logger.info('[boot] DB loaded');
} catch (error) {
Logger.error('✌️ DB load failed', error);
Logger.error('[boot] DB load failed', error);
throw error;
}
};
+1 -25
View File
@@ -1,22 +1,9 @@
import path from 'path';
import fs from 'fs/promises';
import os from 'os';
import chokidar from 'chokidar';
import config from '../config/index';
import Logger from './logger';
async function linkToNodeModule(src: string, dst?: string) {
const target = path.join(config.rootPath, 'node_modules', dst || src);
const source = path.join(config.rootPath, src);
try {
const stats = await fs.lstat(target);
if (!stats) {
await fs.symlink(source, target, 'dir');
}
} catch (error) { }
}
async function linkCommand() {
const homeDir = os.homedir();
let userBinDir = path.join(homeDir, 'bin');
@@ -59,17 +46,6 @@ async function linkCommandToDir(commandDir: string) {
}
}
export default async (src: string = 'deps') => {
export default async () => {
await linkCommand();
await linkToNodeModule(src);
const source = path.join(config.rootPath, src);
const watcher = chokidar.watch(source, {
ignored: /(^|[\/\\])\../, // ignore dotfiles
persistent: true,
});
watcher
.on('add', () => linkToNodeModule(src))
.on('change', () => linkToNodeModule(src));
};
+34 -29
View File
@@ -8,9 +8,12 @@ import { getPlatform, getToken } from '../config/util';
import rewrite from 'express-urlrewrite';
import { errors } from 'celebrate';
import { serveEnv } from '../config/serverEnv';
import { IKeyvStore, shareStore } from '../shared/store';
import { isValidToken } from '../shared/auth';
import { shareStore } from '../shared/store';
import { isValidToken, isDefaultAuthInfo } from '../shared/auth';
import { AuthInfo } from '../data/system';
import path from 'path';
import { t } from '../shared/i18n';
import { AppScope } from '../data/open';
export default ({ app }: { app: Application }) => {
// Security: Enable strict routing to prevent case-insensitive path bypass
@@ -18,30 +21,30 @@ export default ({ app }: { app: Application }) => {
app.set('strict routing', true);
app.set('trust proxy', 'loopback');
app.use(cors());
// Security: Path normalization middleware to prevent case variation attacks
app.use((req, res, next) => {
const originalPath = req.path;
const normalizedPath = originalPath.toLowerCase();
// Block requests with case variations on protected paths
if (originalPath !== normalizedPath &&
(normalizedPath.startsWith('/api/') || normalizedPath.startsWith('/open/'))) {
if (originalPath !== normalizedPath &&
(normalizedPath.startsWith('/api/') || normalizedPath.startsWith('/open/'))) {
return res.status(400).json({
code: 400,
message: 'Invalid path format'
});
}
next();
});
// Rewrite URLs to strip baseUrl prefix if configured
// This allows the rest of the app to work without baseUrl awareness
if (config.baseUrl) {
app.use(rewrite(`${config.baseUrl}/*`, '/$1'));
}
app.get(`${config.api.prefix}/env.js`, serveEnv);
app.use(`${config.api.prefix}/static`, express.static(config.uploadPath));
@@ -86,17 +89,26 @@ export default ({ app }: { app: Application }) => {
const currentToken = doc.tokens.find((x) => x.value === headerToken);
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
const key = keyMatch && keyMatch[1];
if (
doc.scopes.includes(key as any) &&
currentToken &&
currentToken.expiration >= Math.round(Date.now() / 1000)
) {
return next();
if (!doc.scopes.includes(key as AppScope)) {
const err = new UnauthorizedError('credentials_bad_scheme', {
message: t('暂无权限'),
});
return next(err);
}
if (!currentToken || currentToken.expiration < Math.round(Date.now() / 1000)) {
const err = new UnauthorizedError('invalid_token', {
message: t('Token 已失效'),
});
return next(err);
}
return next();
}
}
const originPath = `${req.baseUrl}${req.path === '/' ? '' : req.path}`;
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
if (
!headerToken &&
originPath &&
@@ -112,8 +124,8 @@ export default ({ app }: { app: Application }) => {
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
const errorMessage = headerToken
? 'jwt malformed'
: 'No authorization token was found';
? t('Token 已失效')
: t('请先登录');
const err = new UnauthorizedError(errorCode, { message: errorMessage });
next(err);
});
@@ -126,24 +138,17 @@ export default ({ app }: { app: Application }) => {
'/api/user/notification/init',
'/open/user/init',
'/open/user/notification/init',
].includes(req.path)
].includes(pathLower)
) {
return next();
}
const authInfo =
(await shareStore.getAuthInfo()) || ({} as IKeyvStore['authInfo']);
(await shareStore.getAuthInfo()) || ({} as AuthInfo);
let isInitialized = true;
if (
Object.keys(authInfo).length === 2 &&
authInfo.username === 'admin' &&
authInfo.password === 'admin'
) {
isInitialized = false;
}
let isInitialized = !isDefaultAuthInfo(authInfo);
if (isInitialized) {
return res.send({ code: 450, message: '未知错误' });
return res.send({ code: 450, message: t('未知错误') });
} else {
return next();
}
+29 -3
View File
@@ -13,11 +13,13 @@ import { AuthDataType, SystemModel } from '../data/system';
import SystemService from '../services/system';
import UserService from '../services/user';
import { writeFile, readFile } from 'fs/promises';
import { createRandomString, fileExist, safeJSONParse } from '../config/util';
import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../config/util';
import OpenService from '../services/open';
import { shareStore } from '../shared/store';
import Logger from './logger';
import { AppModel } from '../data/open';
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
import { setLang, systemLang } from '../shared/i18n';
export default async () => {
const cronService = Container.get(CronService);
@@ -36,10 +38,15 @@ export default async () => {
if (!systemApp) {
systemApp = await AppModel.create({
name: 'system',
scopes: ['crons', 'system'],
scopes: ['crons', 'system', 'dashboard'],
client_id: createRandomString(12, 12),
client_secret: createRandomString(24, 24),
});
} else if (!systemApp.scopes.includes('dashboard')) {
await AppModel.update(
{ scopes: [...systemApp.scopes, 'dashboard'] },
{ where: { name: 'system' } },
);
}
const [systemConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.systemConfig },
@@ -50,7 +57,7 @@ export default async () => {
const [authConfig] = await SystemModel.findOrCreate({
where: { type: AuthDataType.authConfig },
});
if (!authConfig?.info) {
if (!authConfig?.info || isDemoEnv()) {
let authInfo = {
username: 'admin',
password: 'admin',
@@ -134,6 +141,12 @@ export default async () => {
// 初始化更新所有任务状态为空闲
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} });
// 清空所有运行中的实例记录(服务重启后进程已不存在)
await RunningInstanceModel.update(
{ status: InstanceStatus.stopped },
{ where: { status: InstanceStatus.running } },
);
// 初始化时执行一次所有的 ql repo 任务
CrontabModel.findAll({
where: {
@@ -210,8 +223,21 @@ export default async () => {
}
});
// 初始化语言(必须在 autosave_crontab 之前)
const lang = systemConfig.info?.lang || systemLang();
setLang(lang);
// 确保 lang_env.sh 存在
try {
const langEnvExist = await fileExist(config.langEnvFile);
if (!langEnvExist) {
await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`);
}
} catch { }
// 初始化保存一次ck和定时任务数据
await cronService.autosave_crontab();
await envService.set_envs();
const authInfo = await userService.getAuthInfo();
+3 -1
View File
@@ -20,6 +20,7 @@ const uploadPath = path.join(dataPath, 'upload/');
const bakPath = path.join(dataPath, 'bak/');
const samplePath = path.join(rootPath, 'sample/');
const tmpPath = path.join(logPath, '.tmp/');
const rootTmpPath = path.join(rootPath, '.tmp/');
const confFile = path.join(configPath, 'config.sh');
const sampleConfigFile = path.join(samplePath, 'config.sample.sh');
const sampleTaskShellFile = path.join(samplePath, 'task.sample.sh');
@@ -44,6 +45,7 @@ const directories = [
preloadPath,
logPath,
tmpPath,
rootTmpPath,
uploadPath,
sshPath,
bakPath,
@@ -123,5 +125,5 @@ export default async () => {
}
}
Logger.info('✌️ Init file down');
Logger.info('[boot] Init file down');
};
+3 -2
View File
@@ -6,6 +6,7 @@ import SshKeyService from '../services/sshKey';
import config from '../config';
import { fileExist } from '../config/util';
import { join } from 'path';
import { t } from '../shared/i18n';
export default async () => {
const systemService = Container.get(SystemService);
@@ -25,7 +26,7 @@ export default async () => {
}
const cron = {
id: NaN,
name: '生成token',
name: t('生成token'),
command: tokenCommand,
runOrigin: 'system',
} as ScheduleTaskType;
@@ -44,7 +45,7 @@ export default async () => {
if (data.info.logRemoveFrequency) {
const rmlogCron = {
id: data.id as number,
name: '删除日志',
name: t('删除日志'),
command: `ql rmlog ${data.info.logRemoveFrequency}`,
runOrigin: 'system' as const,
};
+1 -1
View File
@@ -4,7 +4,7 @@ import Sock from './sock';
export default async ({ server }: { server: Server }) => {
await Sock({ server });
Logger.info('✌️ Sock loaded');
Logger.info('[boot] Sock loaded');
process.on('uncaughtException', (error) => {
Logger.error('Uncaught exception:', error);
+84
View File
@@ -0,0 +1,84 @@
import { createHash } from 'crypto';
import { DataTypes } from 'sequelize';
import type { Migration } from './types';
type ColumnType = 'string' | 'integer' | 'text' | 'json';
interface LegacyColumn {
table: string;
column: string;
type: ColumnType;
}
const columns: LegacyColumn[] = [
{ table: 'CrontabViews', column: 'filterRelation', type: 'string' },
{ table: 'Subscriptions', column: 'proxy', type: 'string' },
{ table: 'CrontabViews', column: 'type', type: 'integer' },
{ table: 'Subscriptions', column: 'autoAddCron', type: 'integer' },
{ table: 'Subscriptions', column: 'autoDelCron', type: 'integer' },
{ table: 'Crontabs', column: 'sub_id', type: 'integer' },
{ table: 'Crontabs', column: 'extra_schedules', type: 'json' },
{ table: 'Crontabs', column: 'task_before', type: 'text' },
{ table: 'Crontabs', column: 'task_after', type: 'text' },
{ table: 'Crontabs', column: 'log_name', type: 'string' },
{
table: 'Crontabs',
column: 'allow_multiple_instances',
type: 'integer',
},
{ table: 'Crontabs', column: 'work_dir', type: 'string' },
{ table: 'Envs', column: 'isPinned', type: 'integer' },
{ table: 'Envs', column: 'labels', type: 'json' },
];
export const legacyColumnOwnership: readonly {
table: string;
column: string;
type: ColumnType;
}[] = columns;
function dataType(type: ColumnType) {
switch (type) {
case 'integer':
return DataTypes.INTEGER;
case 'text':
return DataTypes.TEXT;
case 'json':
return DataTypes.JSON;
default:
return DataTypes.STRING;
}
}
export const legacyColumnsMigration: Migration = {
id: '0001-legacy-columns',
checksum: createHash('sha256').update(JSON.stringify(columns)).digest('hex'),
async up({ queryInterface, transaction }) {
const descriptions = new Map<string, Record<string, unknown>>();
for (const definition of columns) {
let description = descriptions.get(definition.table);
if (!description) {
description = await queryInterface.describeTable(definition.table);
descriptions.set(definition.table, description);
}
if (
Object.prototype.hasOwnProperty.call(description, definition.column)
) {
continue;
}
await queryInterface.addColumn(
definition.table,
definition.column,
{
type: dataType(definition.type),
allowNull: true,
},
{ transaction },
);
description[definition.column] = {};
}
},
};
+319
View File
@@ -0,0 +1,319 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import type { Migration } from './types';
export const RUN_TABLE = 'Runs';
export const RUN_ATTEMPT_TABLE = 'RunAttempts';
export const RUN_EVENT_TABLE = 'RunEvents';
const schemaManifest = {
version: 1,
tables: {
Runs: [
'id',
'project_id',
'task_id',
'task_revision',
'task_name',
'task_snapshot_ref',
'legacy_cron_id',
'parent_run_id',
'retry_of_run_id',
'trigger_id',
'trigger_type',
'execution_origin',
'execution_owner',
'triggered_by',
'request_id',
'scheduled_for_ms',
'status',
'version',
'event_sequence',
'priority',
'idempotency_key',
'input_ref',
'output_ref',
'created_at_ms',
'queued_at_ms',
'started_at_ms',
'finished_at_ms',
'error_code',
'error_summary',
],
RunAttempts: [
'id',
'run_id',
'step_run_id',
'attempt',
'status',
'executor_type',
'worker_id',
'executor_handle',
'pid',
'log_artifact_id',
'lease_token',
'lease_expires_at_ms',
'callback_token_hash',
'callback_sequence',
'created_at_ms',
'started_at_ms',
'finished_at_ms',
'exit_code',
'error_code',
'error_summary',
],
RunEvents: [
'id',
'run_id',
'sequence',
'type',
'dedupe_key',
'actor_type',
'actor_id',
'attempt_id',
'step_run_id',
'payload',
'created_at_ms',
],
},
indexes: [
'runs_project_created_idx',
'runs_task_created_idx',
'runs_status_queued_idx',
'runs_legacy_cron_created_idx',
'runs_project_idempotency_uidx',
'run_attempts_run_attempt_uidx',
'run_attempts_run_status_idx',
'run_attempts_status_created_idx',
'run_attempts_lease_idx',
'run_events_run_sequence_uidx',
'run_events_run_dedupe_uidx',
'run_events_run_created_idx',
],
constraints: [
'runs_version_nonnegative_check',
'runs_event_sequence_nonnegative_check',
'run_attempts_attempt_positive_check',
'run_attempts_callback_sequence_nonnegative_check',
'run_events_sequence_positive_check',
],
};
export const runSchemaManifest = schemaManifest;
export const runSchemaMigration: Migration = {
id: '0002-run-schema',
checksum: createHash('sha256')
.update(JSON.stringify(schemaManifest))
.digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
RUN_TABLE,
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
project_id: { type: DataTypes.STRING(128), allowNull: false },
task_id: { type: DataTypes.STRING(255), allowNull: false },
task_revision: { type: DataTypes.STRING(128), allowNull: false },
task_name: { type: DataTypes.STRING(255), allowNull: true },
task_snapshot_ref: { type: DataTypes.STRING(512), allowNull: true },
legacy_cron_id: { type: DataTypes.INTEGER, allowNull: true },
parent_run_id: { type: DataTypes.STRING(36), allowNull: true },
retry_of_run_id: { type: DataTypes.STRING(36), allowNull: true },
trigger_id: { type: DataTypes.STRING(36), allowNull: true },
trigger_type: { type: DataTypes.STRING(64), allowNull: false },
execution_origin: { type: DataTypes.STRING(64), allowNull: false },
execution_owner: { type: DataTypes.STRING(16), allowNull: false },
triggered_by: { type: DataTypes.STRING(255), allowNull: true },
request_id: { type: DataTypes.STRING(128), allowNull: true },
scheduled_for_ms: { type: DataTypes.BIGINT, allowNull: true },
status: { type: DataTypes.STRING(32), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0 },
event_sequence: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0,
},
priority: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0,
},
idempotency_key: { type: DataTypes.STRING(255), allowNull: true },
input_ref: { type: DataTypes.STRING(512), allowNull: true },
output_ref: { type: DataTypes.STRING(512), allowNull: true },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
queued_at_ms: { type: DataTypes.BIGINT, allowNull: true },
started_at_ms: { type: DataTypes.BIGINT, allowNull: true },
finished_at_ms: { type: DataTypes.BIGINT, allowNull: true },
error_code: { type: DataTypes.STRING(128), allowNull: true },
error_summary: { type: DataTypes.STRING(1024), allowNull: true },
},
{ transaction },
);
await queryInterface.createTable(
RUN_ATTEMPT_TABLE,
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
run_id: {
type: DataTypes.STRING(36),
allowNull: false,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
step_run_id: { type: DataTypes.STRING(36), allowNull: true },
attempt: { type: DataTypes.INTEGER, allowNull: false },
status: { type: DataTypes.STRING(32), allowNull: false },
executor_type: { type: DataTypes.STRING(64), allowNull: false },
worker_id: { type: DataTypes.STRING(128), allowNull: true },
executor_handle: { type: DataTypes.TEXT, allowNull: true },
pid: { type: DataTypes.INTEGER, allowNull: true },
log_artifact_id: { type: DataTypes.STRING(36), allowNull: true },
lease_token: { type: DataTypes.STRING(128), allowNull: true },
lease_expires_at_ms: { type: DataTypes.BIGINT, allowNull: true },
callback_token_hash: { type: DataTypes.STRING(128), allowNull: true },
callback_sequence: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0,
},
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
started_at_ms: { type: DataTypes.BIGINT, allowNull: true },
finished_at_ms: { type: DataTypes.BIGINT, allowNull: true },
exit_code: { type: DataTypes.INTEGER, allowNull: true },
error_code: { type: DataTypes.STRING(128), allowNull: true },
error_summary: { type: DataTypes.STRING(1024), allowNull: true },
},
{ transaction },
);
await queryInterface.createTable(
RUN_EVENT_TABLE,
{
id: { type: DataTypes.STRING(36), allowNull: false, primaryKey: true },
run_id: {
type: DataTypes.STRING(36),
allowNull: false,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
sequence: { type: DataTypes.INTEGER, allowNull: false },
type: { type: DataTypes.STRING(128), allowNull: false },
dedupe_key: { type: DataTypes.STRING(255), allowNull: true },
actor_type: { type: DataTypes.STRING(64), allowNull: false },
actor_id: { type: DataTypes.STRING(255), allowNull: true },
attempt_id: {
type: DataTypes.STRING(36),
allowNull: true,
references: { model: RUN_ATTEMPT_TABLE, key: 'id' },
onDelete: 'SET NULL',
onUpdate: 'CASCADE',
},
step_run_id: { type: DataTypes.STRING(36), allowNull: true },
payload: { type: DataTypes.JSON, allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(RUN_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 0 } },
name: 'runs_version_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(RUN_TABLE, {
fields: ['event_sequence'],
type: 'check',
where: { event_sequence: { [Op.gte]: 0 } },
name: 'runs_event_sequence_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(RUN_ATTEMPT_TABLE, {
fields: ['attempt'],
type: 'check',
where: { attempt: { [Op.gte]: 1 } },
name: 'run_attempts_attempt_positive_check',
transaction,
});
await queryInterface.addConstraint(RUN_ATTEMPT_TABLE, {
fields: ['callback_sequence'],
type: 'check',
where: { callback_sequence: { [Op.gte]: 0 } },
name: 'run_attempts_callback_sequence_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(RUN_EVENT_TABLE, {
fields: ['sequence'],
type: 'check',
where: { sequence: { [Op.gte]: 1 } },
name: 'run_events_sequence_positive_check',
transaction,
});
await queryInterface.addIndex(RUN_TABLE, ['project_id', 'created_at_ms'], {
name: 'runs_project_created_idx',
transaction,
});
await queryInterface.addIndex(RUN_TABLE, ['task_id', 'created_at_ms'], {
name: 'runs_task_created_idx',
transaction,
});
await queryInterface.addIndex(RUN_TABLE, ['status', 'queued_at_ms'], {
name: 'runs_status_queued_idx',
transaction,
});
await queryInterface.addIndex(
RUN_TABLE,
['legacy_cron_id', 'created_at_ms'],
{ name: 'runs_legacy_cron_created_idx', transaction },
);
await queryInterface.addIndex(
RUN_TABLE,
['project_id', 'idempotency_key'],
{
name: 'runs_project_idempotency_uidx',
unique: true,
transaction,
},
);
await queryInterface.addIndex(RUN_ATTEMPT_TABLE, ['run_id', 'attempt'], {
name: 'run_attempts_run_attempt_uidx',
unique: true,
transaction,
});
await queryInterface.addIndex(RUN_ATTEMPT_TABLE, ['run_id', 'status'], {
name: 'run_attempts_run_status_idx',
transaction,
});
await queryInterface.addIndex(
RUN_ATTEMPT_TABLE,
['status', 'created_at_ms'],
{ name: 'run_attempts_status_created_idx', transaction },
);
await queryInterface.addIndex(RUN_ATTEMPT_TABLE, ['lease_expires_at_ms'], {
name: 'run_attempts_lease_idx',
transaction,
});
await queryInterface.addIndex(RUN_EVENT_TABLE, ['run_id', 'sequence'], {
name: 'run_events_run_sequence_uidx',
unique: true,
transaction,
});
await queryInterface.addIndex(RUN_EVENT_TABLE, ['run_id', 'dedupe_key'], {
name: 'run_events_run_dedupe_uidx',
unique: true,
transaction,
});
await queryInterface.addIndex(
RUN_EVENT_TABLE,
['run_id', 'created_at_ms'],
{ name: 'run_events_run_created_idx', transaction },
);
},
};
@@ -0,0 +1,71 @@
import { createHash } from 'crypto';
import { DataTypes } from 'sequelize';
import type { Migration } from './types';
export const RUNNING_INSTANCE_TABLE = 'RunningInstances';
export const RUNNING_INSTANCE_RUN_INDEX = 'running_instances_run_started_idx';
export const RUNNING_INSTANCE_ATTEMPT_INDEX = 'running_instances_attempt_uidx';
const manifest = {
table: RUNNING_INSTANCE_TABLE,
columns: {
run_id: 'varchar(36) null',
attempt_id: 'varchar(36) null',
},
indexes: [
`${RUNNING_INSTANCE_RUN_INDEX}(run_id,started_at)`,
`${RUNNING_INSTANCE_ATTEMPT_INDEX}(attempt_id) unique`,
],
};
export const runningInstanceRunReferenceManifest = manifest;
export const runningInstanceRunReferenceMigration: Migration = {
id: '0003-running-instance-run-reference',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
const tables = new Set(await queryInterface.showAllTables());
if (!tables.has(RUNNING_INSTANCE_TABLE)) return;
const columns = await queryInterface.describeTable(RUNNING_INSTANCE_TABLE);
if (!columns.run_id) {
await queryInterface.addColumn(
RUNNING_INSTANCE_TABLE,
'run_id',
{ type: DataTypes.STRING(36), allowNull: true },
{ transaction },
);
}
if (!columns.attempt_id) {
await queryInterface.addColumn(
RUNNING_INSTANCE_TABLE,
'attempt_id',
{ type: DataTypes.STRING(36), allowNull: true },
{ transaction },
);
}
const currentIndexes = (await queryInterface.showIndex(
RUNNING_INSTANCE_TABLE,
{ transaction },
)) as Array<{ name: string }>;
const indexes = new Set(currentIndexes.map((index) => index.name));
if (!indexes.has(RUNNING_INSTANCE_RUN_INDEX)) {
await queryInterface.addIndex(
RUNNING_INSTANCE_TABLE,
['run_id', 'started_at'],
{
name: RUNNING_INSTANCE_RUN_INDEX,
transaction,
},
);
}
if (!indexes.has(RUNNING_INSTANCE_ATTEMPT_INDEX)) {
await queryInterface.addIndex(RUNNING_INSTANCE_TABLE, ['attempt_id'], {
name: RUNNING_INSTANCE_ATTEMPT_INDEX,
unique: true,
transaction,
});
}
},
};
@@ -0,0 +1,60 @@
import { createHash } from 'crypto';
import { DataTypes } from 'sequelize';
import { RUN_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const RUN_CANCELLATION_REQUEST_INDEX =
'runs_status_cancel_requested_idx';
const manifest = {
table: RUN_TABLE,
columns: {
cancel_requested_at_ms: 'bigint null',
cancel_reason: 'varchar(32) null',
},
indexes: [`${RUN_CANCELLATION_REQUEST_INDEX}(status,cancel_requested_at_ms)`],
};
export const runCancellationRequestManifest = manifest;
export const runCancellationRequestMigration: Migration = {
id: '0004-run-cancellation-request',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
const tables = new Set(await queryInterface.showAllTables());
if (!tables.has(RUN_TABLE)) return;
const columns = await queryInterface.describeTable(RUN_TABLE);
if (!columns.cancel_requested_at_ms) {
await queryInterface.addColumn(
RUN_TABLE,
'cancel_requested_at_ms',
{ type: DataTypes.BIGINT, allowNull: true },
{ transaction },
);
}
if (!columns.cancel_reason) {
await queryInterface.addColumn(
RUN_TABLE,
'cancel_reason',
{ type: DataTypes.STRING(32), allowNull: true },
{ transaction },
);
}
const currentIndexes = (await queryInterface.showIndex(RUN_TABLE, {
transaction,
})) as Array<{ name: string }>;
if (
!currentIndexes.some(
(index) => index.name === RUN_CANCELLATION_REQUEST_INDEX,
)
) {
await queryInterface.addIndex(
RUN_TABLE,
['status', 'cancel_requested_at_ms'],
{ name: RUN_CANCELLATION_REQUEST_INDEX, transaction },
);
}
},
};
@@ -0,0 +1,106 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import { RUN_ATTEMPT_TABLE, RUN_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const RUN_CANCELLATION_DISPATCH_TABLE = 'RunCancellationDispatches';
export const RUN_CANCELLATION_DISPATCH_DUE_INDEX =
'run_cancel_dispatch_due_idx';
export const RUN_CANCELLATION_DISPATCH_LEASE_INDEX =
'run_cancel_dispatch_lease_idx';
const manifest = {
table: RUN_CANCELLATION_DISPATCH_TABLE,
columns: [
'run_id',
'attempt_id',
'status',
'version',
'dispatch_count',
'next_attempt_at_ms',
'lease_owner',
'lease_token',
'lease_expires_at_ms',
'last_result',
'last_dispatched_at_ms',
'created_at_ms',
'updated_at_ms',
],
indexes: [
`${RUN_CANCELLATION_DISPATCH_DUE_INDEX}(status,next_attempt_at_ms)`,
`${RUN_CANCELLATION_DISPATCH_LEASE_INDEX}(lease_expires_at_ms)`,
],
constraints: [
'run_cancel_dispatch_version_nonnegative_check',
'run_cancel_dispatch_count_nonnegative_check',
],
};
export const runCancellationDispatchManifest = manifest;
export const runCancellationDispatchMigration: Migration = {
id: '0005-run-cancellation-dispatch',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
RUN_CANCELLATION_DISPATCH_TABLE,
{
run_id: {
type: DataTypes.STRING(36),
allowNull: false,
primaryKey: true,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
attempt_id: {
type: DataTypes.STRING(36),
allowNull: false,
references: { model: RUN_ATTEMPT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
status: { type: DataTypes.STRING(32), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false, defaultValue: 0 },
dispatch_count: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0,
},
next_attempt_at_ms: { type: DataTypes.BIGINT, allowNull: true },
lease_owner: { type: DataTypes.STRING(128), allowNull: true },
lease_token: { type: DataTypes.STRING(128), allowNull: true },
lease_expires_at_ms: { type: DataTypes.BIGINT, allowNull: true },
last_result: { type: DataTypes.STRING(64), allowNull: true },
last_dispatched_at_ms: { type: DataTypes.BIGINT, allowNull: true },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(RUN_CANCELLATION_DISPATCH_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 0 } },
name: 'run_cancel_dispatch_version_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(RUN_CANCELLATION_DISPATCH_TABLE, {
fields: ['dispatch_count'],
type: 'check',
where: { dispatch_count: { [Op.gte]: 0 } },
name: 'run_cancel_dispatch_count_nonnegative_check',
transaction,
});
await queryInterface.addIndex(
RUN_CANCELLATION_DISPATCH_TABLE,
['status', 'next_attempt_at_ms'],
{ name: RUN_CANCELLATION_DISPATCH_DUE_INDEX, transaction },
);
await queryInterface.addIndex(
RUN_CANCELLATION_DISPATCH_TABLE,
['lease_expires_at_ms'],
{ name: RUN_CANCELLATION_DISPATCH_LEASE_INDEX, transaction },
);
},
};
@@ -0,0 +1,48 @@
import { createHash } from 'crypto';
import { DataTypes } from 'sequelize';
import { RUN_ATTEMPT_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const RUN_ATTEMPT_DEADLINE_INDEX = 'run_attempt_status_deadline_idx';
const manifest = {
table: RUN_ATTEMPT_TABLE,
columns: {
deadline_at_ms: 'bigint null',
},
indexes: [`${RUN_ATTEMPT_DEADLINE_INDEX}(status,deadline_at_ms,id)`],
};
export const runAttemptDeadlineManifest = manifest;
export const runAttemptDeadlineMigration: Migration = {
id: '0006-run-attempt-deadline',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
const tables = new Set(await queryInterface.showAllTables());
if (!tables.has(RUN_ATTEMPT_TABLE)) return;
const columns = await queryInterface.describeTable(RUN_ATTEMPT_TABLE);
if (!columns.deadline_at_ms) {
await queryInterface.addColumn(
RUN_ATTEMPT_TABLE,
'deadline_at_ms',
{ type: DataTypes.BIGINT, allowNull: true },
{ transaction },
);
}
const currentIndexes = (await queryInterface.showIndex(RUN_ATTEMPT_TABLE, {
transaction,
})) as Array<{ name: string }>;
if (
!currentIndexes.some((index) => index.name === RUN_ATTEMPT_DEADLINE_INDEX)
) {
await queryInterface.addIndex(
RUN_ATTEMPT_TABLE,
['status', 'deadline_at_ms', 'id'],
{ name: RUN_ATTEMPT_DEADLINE_INDEX, transaction },
);
}
},
};
@@ -0,0 +1,102 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import { RUN_ATTEMPT_TABLE, RUN_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const COMPLETION_RECEIPT_JOURNAL_TABLE = 'CompletionReceiptJournals';
export const COMPLETION_RECEIPT_JOURNAL_SCAN_INDEX =
'completion_receipt_journal_scan_idx';
export const COMPLETION_RECEIPT_JOURNAL_PURGE_INDEX =
'completion_receipt_journal_purge_idx';
const manifest = {
table: COMPLETION_RECEIPT_JOURNAL_TABLE,
columns: [
'attempt_id',
'run_id',
'state',
'quarantine_ref',
'purge_after_ms',
'registered_at_ms',
'updated_at_ms',
],
indexes: [
`${COMPLETION_RECEIPT_JOURNAL_SCAN_INDEX}(state,updated_at_ms,attempt_id)`,
`${COMPLETION_RECEIPT_JOURNAL_PURGE_INDEX}(state,purge_after_ms,attempt_id)`,
],
constraints: [
'completion_receipt_journal_state_check',
'completion_receipt_journal_registered_nonnegative_check',
'completion_receipt_journal_updated_nonnegative_check',
],
};
export const completionReceiptJournalManifest = manifest;
export const completionReceiptJournalMigration: Migration = {
id: '0007-completion-receipt-journal',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
COMPLETION_RECEIPT_JOURNAL_TABLE,
{
attempt_id: {
type: DataTypes.STRING(36),
allowNull: false,
primaryKey: true,
references: { model: RUN_ATTEMPT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
run_id: {
type: DataTypes.STRING(36),
allowNull: false,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
state: {
type: DataTypes.STRING(16),
allowNull: false,
defaultValue: 'pending',
},
quarantine_ref: { type: DataTypes.STRING(255), allowNull: true },
purge_after_ms: { type: DataTypes.BIGINT, allowNull: true },
registered_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(COMPLETION_RECEIPT_JOURNAL_TABLE, {
fields: ['state'],
type: 'check',
where: { state: { [Op.in]: ['pending', 'quarantined'] } },
name: 'completion_receipt_journal_state_check',
transaction,
});
await queryInterface.addConstraint(COMPLETION_RECEIPT_JOURNAL_TABLE, {
fields: ['registered_at_ms'],
type: 'check',
where: { registered_at_ms: { [Op.gte]: 0 } },
name: 'completion_receipt_journal_registered_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(COMPLETION_RECEIPT_JOURNAL_TABLE, {
fields: ['updated_at_ms'],
type: 'check',
where: { updated_at_ms: { [Op.gte]: 0 } },
name: 'completion_receipt_journal_updated_nonnegative_check',
transaction,
});
await queryInterface.addIndex(
COMPLETION_RECEIPT_JOURNAL_TABLE,
['state', 'updated_at_ms', 'attempt_id'],
{ name: COMPLETION_RECEIPT_JOURNAL_SCAN_INDEX, transaction },
);
await queryInterface.addIndex(
COMPLETION_RECEIPT_JOURNAL_TABLE,
['state', 'purge_after_ms', 'attempt_id'],
{ name: COMPLETION_RECEIPT_JOURNAL_PURGE_INDEX, transaction },
);
},
};
+110
View File
@@ -0,0 +1,110 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import type { Migration } from './types';
export const WORKER_REGISTRY_TABLE = 'Workers';
export const WORKER_REGISTRY_LEASE_INDEX = 'workers_lease_idx';
export const WORKER_REGISTRY_CAPACITY_INDEX = 'workers_capacity_idx';
const manifest = {
table: WORKER_REGISTRY_TABLE,
columns: [
'id',
'session_id',
'generation',
'status',
'version',
'capabilities_json',
'capabilities_hash',
'max_concurrent_runs',
'available_slots',
'registered_at_ms',
'last_heartbeat_at_ms',
'lease_expires_at_ms',
'updated_at_ms',
],
indexes: [
`${WORKER_REGISTRY_LEASE_INDEX}(status,lease_expires_at_ms,id)`,
`${WORKER_REGISTRY_CAPACITY_INDEX}(status,available_slots,lease_expires_at_ms,id)`,
],
constraints: [
'workers_generation_positive_check',
'workers_version_nonnegative_check',
'workers_max_concurrency_positive_check',
'workers_available_slots_nonnegative_check',
'workers_status_check',
],
};
export const workerRegistryManifest = manifest;
export const workerRegistryMigration: Migration = {
id: '0008-worker-registry',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
WORKER_REGISTRY_TABLE,
{
id: { type: DataTypes.STRING(128), allowNull: false, primaryKey: true },
session_id: { type: DataTypes.STRING(36), allowNull: false },
generation: { type: DataTypes.INTEGER, allowNull: false },
status: { type: DataTypes.STRING(16), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
capabilities_json: { type: DataTypes.TEXT, allowNull: false },
capabilities_hash: { type: DataTypes.STRING(64), allowNull: false },
max_concurrent_runs: { type: DataTypes.INTEGER, allowNull: false },
available_slots: { type: DataTypes.INTEGER, allowNull: false },
registered_at_ms: { type: DataTypes.BIGINT, allowNull: false },
last_heartbeat_at_ms: { type: DataTypes.BIGINT, allowNull: false },
lease_expires_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(WORKER_REGISTRY_TABLE, {
fields: ['generation'],
type: 'check',
where: { generation: { [Op.gt]: 0 } },
name: 'workers_generation_positive_check',
transaction,
});
await queryInterface.addConstraint(WORKER_REGISTRY_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 0 } },
name: 'workers_version_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(WORKER_REGISTRY_TABLE, {
fields: ['max_concurrent_runs'],
type: 'check',
where: { max_concurrent_runs: { [Op.gt]: 0 } },
name: 'workers_max_concurrency_positive_check',
transaction,
});
await queryInterface.addConstraint(WORKER_REGISTRY_TABLE, {
fields: ['available_slots'],
type: 'check',
where: { available_slots: { [Op.gte]: 0 } },
name: 'workers_available_slots_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(WORKER_REGISTRY_TABLE, {
fields: ['status'],
type: 'check',
where: { status: { [Op.in]: ['online', 'draining', 'offline'] } },
name: 'workers_status_check',
transaction,
});
await queryInterface.addIndex(
WORKER_REGISTRY_TABLE,
['status', 'lease_expires_at_ms', 'id'],
{ name: WORKER_REGISTRY_LEASE_INDEX, transaction },
);
await queryInterface.addIndex(
WORKER_REGISTRY_TABLE,
['status', 'available_slots', 'lease_expires_at_ms', 'id'],
{ name: WORKER_REGISTRY_CAPACITY_INDEX, transaction },
);
},
};
+143
View File
@@ -0,0 +1,143 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import { RUN_ATTEMPT_TABLE, RUN_TABLE } from './0002-run-schema';
import { WORKER_REGISTRY_TABLE } from './0008-worker-registry';
import type { Migration } from './types';
export const RUN_DISPATCH_LEASE_TABLE = 'RunDispatchLeases';
export const RUN_DISPATCH_LEASE_EXPIRY_INDEX = 'run_dispatch_leases_expiry_idx';
export const RUN_DISPATCH_LEASE_WORKER_INDEX = 'run_dispatch_leases_worker_idx';
export const RUN_DISPATCH_LEASE_TOKEN_INDEX = 'run_dispatch_leases_token_uidx';
const manifest = {
table: RUN_DISPATCH_LEASE_TABLE,
columns: [
'attempt_id',
'run_id',
'status',
'version',
'lease_generation',
'worker_id',
'worker_session_id',
'worker_generation',
'lease_token',
'acquired_at_ms',
'renewed_at_ms',
'expires_at_ms',
'released_at_ms',
'release_reason',
'completed_at_ms',
'updated_at_ms',
],
indexes: [
`${RUN_DISPATCH_LEASE_EXPIRY_INDEX}(status,expires_at_ms,attempt_id)`,
`${RUN_DISPATCH_LEASE_WORKER_INDEX}(worker_id,worker_session_id,worker_generation,status,expires_at_ms,attempt_id)`,
`${RUN_DISPATCH_LEASE_TOKEN_INDEX}(lease_token)`,
],
constraints: [
'run_dispatch_leases_status_check',
'run_dispatch_leases_version_nonnegative_check',
'run_dispatch_leases_generation_positive_check',
'run_dispatch_leases_worker_generation_positive_check',
],
};
export const runDispatchLeaseManifest = manifest;
export const runDispatchLeaseMigration: Migration = {
id: '0009-run-dispatch-lease',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
RUN_DISPATCH_LEASE_TABLE,
{
attempt_id: {
type: DataTypes.STRING(36),
allowNull: false,
primaryKey: true,
references: { model: RUN_ATTEMPT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
run_id: {
type: DataTypes.STRING(36),
allowNull: false,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
status: { type: DataTypes.STRING(16), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
lease_generation: { type: DataTypes.INTEGER, allowNull: false },
worker_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: WORKER_REGISTRY_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
worker_session_id: { type: DataTypes.STRING(36), allowNull: false },
worker_generation: { type: DataTypes.INTEGER, allowNull: false },
lease_token: { type: DataTypes.STRING(128), allowNull: false },
acquired_at_ms: { type: DataTypes.BIGINT, allowNull: false },
renewed_at_ms: { type: DataTypes.BIGINT, allowNull: false },
expires_at_ms: { type: DataTypes.BIGINT, allowNull: false },
released_at_ms: { type: DataTypes.BIGINT, allowNull: true },
release_reason: { type: DataTypes.STRING(32), allowNull: true },
completed_at_ms: { type: DataTypes.BIGINT, allowNull: true },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(RUN_DISPATCH_LEASE_TABLE, {
fields: ['status'],
type: 'check',
where: { status: { [Op.in]: ['leased', 'released', 'completed'] } },
name: 'run_dispatch_leases_status_check',
transaction,
});
await queryInterface.addConstraint(RUN_DISPATCH_LEASE_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 0 } },
name: 'run_dispatch_leases_version_nonnegative_check',
transaction,
});
await queryInterface.addConstraint(RUN_DISPATCH_LEASE_TABLE, {
fields: ['lease_generation'],
type: 'check',
where: { lease_generation: { [Op.gt]: 0 } },
name: 'run_dispatch_leases_generation_positive_check',
transaction,
});
await queryInterface.addConstraint(RUN_DISPATCH_LEASE_TABLE, {
fields: ['worker_generation'],
type: 'check',
where: { worker_generation: { [Op.gt]: 0 } },
name: 'run_dispatch_leases_worker_generation_positive_check',
transaction,
});
await queryInterface.addIndex(
RUN_DISPATCH_LEASE_TABLE,
['status', 'expires_at_ms', 'attempt_id'],
{ name: RUN_DISPATCH_LEASE_EXPIRY_INDEX, transaction },
);
await queryInterface.addIndex(
RUN_DISPATCH_LEASE_TABLE,
[
'worker_id',
'worker_session_id',
'worker_generation',
'status',
'expires_at_ms',
'attempt_id',
],
{ name: RUN_DISPATCH_LEASE_WORKER_INDEX, transaction },
);
await queryInterface.addIndex(RUN_DISPATCH_LEASE_TABLE, ['lease_token'], {
name: RUN_DISPATCH_LEASE_TOKEN_INDEX,
unique: true,
transaction,
});
},
};
@@ -0,0 +1,40 @@
import { createHash } from 'crypto';
import { Op } from 'sequelize';
import { RUN_ATTEMPT_TABLE, RUN_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const RUN_DISPATCH_CANDIDATE_RUN_INDEX = 'runs_dispatch_candidates_idx';
export const RUN_DISPATCH_CANDIDATE_ATTEMPT_INDEX =
'run_attempts_dispatch_candidates_idx';
const manifest = {
indexes: [
`${RUN_DISPATCH_CANDIDATE_RUN_INDEX}(priority DESC,queued_at_ms,id) WHERE runtime queued/dispatching uncancelled`,
`${RUN_DISPATCH_CANDIDATE_ATTEMPT_INDEX}(status,run_id,created_at_ms,id)`,
],
};
export const runDispatchCandidateManifest = manifest;
export const runDispatchCandidateMigration: Migration = {
id: '0010-run-dispatch-candidates',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.addIndex(RUN_TABLE, {
fields: [{ name: 'priority', order: 'DESC' }, 'queued_at_ms', 'id'],
name: RUN_DISPATCH_CANDIDATE_RUN_INDEX,
where: {
execution_owner: 'runtime',
status: { [Op.in]: ['queued', 'dispatching'] },
cancel_requested_at_ms: null,
queued_at_ms: { [Op.ne]: null },
},
transaction,
});
await queryInterface.addIndex(
RUN_ATTEMPT_TABLE,
['status', 'run_id', 'created_at_ms', 'id'],
{ name: RUN_DISPATCH_CANDIDATE_ATTEMPT_INDEX, transaction },
);
},
};
+115
View File
@@ -0,0 +1,115 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import { RUN_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const RUN_RETRY_POLICY_TABLE = 'RunRetryPolicies';
export const RUN_RETRY_POLICY_DUE_INDEX = 'run_retry_policies_due_idx';
export const RUN_LOST_RETRY_INDEX = 'runs_lost_retry_idx';
const manifest = {
table: RUN_RETRY_POLICY_TABLE,
columns: [
'run_id',
'max_attempts',
'retry_on_lost',
'safety',
'backoff_base_ms',
'backoff_max_ms',
'next_attempt_at_ms',
'version',
'created_at_ms',
'updated_at_ms',
],
indexes: [
`${RUN_LOST_RETRY_INDEX}(execution_owner,status,id)`,
`${RUN_RETRY_POLICY_DUE_INDEX}(next_attempt_at_ms,run_id) WHERE next_attempt_at_ms IS NOT NULL`,
],
constraints: [
'run_retry_policies_max_attempts_check',
'run_retry_policies_backoff_base_check',
'run_retry_policies_backoff_max_check',
'run_retry_policies_version_check',
],
};
export const runRetryPolicyManifest = manifest;
export const runRetryPolicyMigration: Migration = {
id: '0011-run-retry-policy',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
RUN_RETRY_POLICY_TABLE,
{
run_id: {
type: DataTypes.STRING(36),
allowNull: false,
primaryKey: true,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
max_attempts: { type: DataTypes.INTEGER, allowNull: false },
retry_on_lost: { type: DataTypes.BOOLEAN, allowNull: false },
safety: { type: DataTypes.STRING(16), allowNull: false },
backoff_base_ms: { type: DataTypes.BIGINT, allowNull: false },
backoff_max_ms: { type: DataTypes.BIGINT, allowNull: false },
next_attempt_at_ms: { type: DataTypes.BIGINT, allowNull: true },
version: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: 0,
},
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(RUN_RETRY_POLICY_TABLE, {
fields: ['max_attempts'],
type: 'check',
where: { max_attempts: { [Op.between]: [1, 16] } },
name: 'run_retry_policies_max_attempts_check',
transaction,
});
await queryInterface.addConstraint(RUN_RETRY_POLICY_TABLE, {
fields: ['backoff_base_ms'],
type: 'check',
where: { backoff_base_ms: { [Op.gte]: 0 } },
name: 'run_retry_policies_backoff_base_check',
transaction,
});
await queryInterface.addConstraint(RUN_RETRY_POLICY_TABLE, {
fields: ['backoff_max_ms'],
type: 'check',
where: { backoff_max_ms: { [Op.gte]: 0 } },
name: 'run_retry_policies_backoff_max_check',
transaction,
});
await queryInterface.addConstraint(RUN_RETRY_POLICY_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 0 } },
name: 'run_retry_policies_version_check',
transaction,
});
await queryInterface.addIndex(
RUN_TABLE,
['execution_owner', 'status', 'id'],
{
name: RUN_LOST_RETRY_INDEX,
transaction,
},
);
await queryInterface.addIndex(
RUN_RETRY_POLICY_TABLE,
['next_attempt_at_ms', 'run_id'],
{
name: RUN_RETRY_POLICY_DUE_INDEX,
where: { next_attempt_at_ms: { [Op.ne]: null } },
transaction,
},
);
},
};
@@ -0,0 +1,72 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import type { Migration } from './types';
export const TASK_EXECUTION_REVISION_TABLE = 'TaskExecutionRevisions';
export const TASK_EXECUTION_REVISION_CREATED_INDEX =
'task_execution_revisions_created_idx';
const manifest = {
table: TASK_EXECUTION_REVISION_TABLE,
columns: [
'project_id',
'task_id',
'task_revision',
'executor_type',
'execution_template',
'context_ref',
'content_digest',
'created_at_ms',
],
indexes: [
`${TASK_EXECUTION_REVISION_CREATED_INDEX}(project_id,created_at_ms,task_id,task_revision)`,
],
constraints: ['task_execution_revisions_created_at_check'],
};
export const taskExecutionRevisionManifest = manifest;
export const taskExecutionRevisionMigration: Migration = {
id: '0012-task-execution-revisions',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
TASK_EXECUTION_REVISION_TABLE,
{
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
task_id: {
type: DataTypes.STRING(255),
allowNull: false,
primaryKey: true,
},
task_revision: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
executor_type: { type: DataTypes.STRING(64), allowNull: false },
execution_template: { type: DataTypes.TEXT, allowNull: false },
context_ref: { type: DataTypes.STRING(512), allowNull: false },
content_digest: { type: DataTypes.STRING(64), allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(TASK_EXECUTION_REVISION_TABLE, {
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'task_execution_revisions_created_at_check',
transaction,
});
await queryInterface.addIndex(
TASK_EXECUTION_REVISION_TABLE,
['project_id', 'created_at_ms', 'task_id', 'task_revision'],
{ name: TASK_EXECUTION_REVISION_CREATED_INDEX, transaction },
);
},
};
@@ -0,0 +1,57 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import type { Migration } from './types';
export const LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE =
'LocalExecutionContextRecipes';
export const LOCAL_EXECUTION_CONTEXT_RECIPE_CREATED_INDEX =
'local_execution_context_recipes_created_idx';
const manifest = {
table: LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
columns: [
'context_ref',
'environment_recipe',
'content_digest',
'created_at_ms',
],
indexes: [
`${LOCAL_EXECUTION_CONTEXT_RECIPE_CREATED_INDEX}(created_at_ms,context_ref)`,
],
constraints: ['local_execution_context_recipes_created_at_check'],
};
export const localExecutionContextRecipeManifest = manifest;
export const localExecutionContextRecipeMigration: Migration = {
id: '0013-local-execution-context-recipes',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
{
context_ref: {
type: DataTypes.STRING(512),
allowNull: false,
primaryKey: true,
},
environment_recipe: { type: DataTypes.TEXT, allowNull: false },
content_digest: { type: DataTypes.STRING(64), allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE, {
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'local_execution_context_recipes_created_at_check',
transaction,
});
await queryInterface.addIndex(
LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
['created_at_ms', 'context_ref'],
{ name: LOCAL_EXECUTION_CONTEXT_RECIPE_CREATED_INDEX, transaction },
);
},
};
@@ -0,0 +1,99 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import type { Migration } from './types';
export const LOCAL_SECRET_ENVELOPE_TABLE = 'LocalSecretEnvelopes';
export const LOCAL_SECRET_MUTATION_INDEX = 'local_secret_mutation_idx';
export const LOCAL_SECRET_CURRENT_INDEX = 'local_secret_current_idx';
export const LOCAL_SECRET_KEY_INDEX = 'local_secret_key_idx';
const manifest = {
table: LOCAL_SECRET_ENVELOPE_TABLE,
columns: [
'project_id',
'secret_name',
'version',
'mutation_id',
'key_id',
'algorithm',
'nonce',
'ciphertext',
'auth_tag',
'created_at_ms',
],
indexes: [
`${LOCAL_SECRET_MUTATION_INDEX}(project_id,secret_name,mutation_id) UNIQUE`,
`${LOCAL_SECRET_CURRENT_INDEX}(project_id,secret_name,version DESC)`,
`${LOCAL_SECRET_KEY_INDEX}(key_id,project_id,secret_name,version)`,
],
constraints: [
'local_secret_envelopes_version_check',
'local_secret_envelopes_created_at_check',
],
};
export const localSecretEnvelopeManifest = manifest;
export const localSecretEnvelopeMigration: Migration = {
id: '0014-local-secret-envelopes',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
LOCAL_SECRET_ENVELOPE_TABLE,
{
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
secret_name: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
version: {
type: DataTypes.INTEGER,
allowNull: false,
primaryKey: true,
},
mutation_id: { type: DataTypes.STRING(64), allowNull: false },
key_id: { type: DataTypes.STRING(128), allowNull: false },
algorithm: { type: DataTypes.STRING(32), allowNull: false },
nonce: { type: DataTypes.BLOB, allowNull: false },
ciphertext: { type: DataTypes.BLOB, allowNull: false },
auth_tag: { type: DataTypes.BLOB, allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(LOCAL_SECRET_ENVELOPE_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 1 } },
name: 'local_secret_envelopes_version_check',
transaction,
});
await queryInterface.addConstraint(LOCAL_SECRET_ENVELOPE_TABLE, {
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'local_secret_envelopes_created_at_check',
transaction,
});
await queryInterface.addIndex(
LOCAL_SECRET_ENVELOPE_TABLE,
['project_id', 'secret_name', 'mutation_id'],
{ name: LOCAL_SECRET_MUTATION_INDEX, unique: true, transaction },
);
await queryInterface.addIndex(LOCAL_SECRET_ENVELOPE_TABLE, {
fields: ['project_id', 'secret_name', { name: 'version', order: 'DESC' }],
name: LOCAL_SECRET_CURRENT_INDEX,
transaction,
});
await queryInterface.addIndex(
LOCAL_SECRET_ENVELOPE_TABLE,
['key_id', 'project_id', 'secret_name', 'version'],
{ name: LOCAL_SECRET_KEY_INDEX, transaction },
);
},
};
@@ -0,0 +1,94 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import { RUN_ATTEMPT_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const LOCAL_ARTIFACT_RETENTION_TABLE = 'LocalArtifactRetentions';
export const LOCAL_ARTIFACT_RETENTION_RECORDED_INDEX =
'local_artifact_retention_recorded_idx';
export const RUN_ATTEMPT_ARTIFACT_RETENTION_INDEX =
'run_attempts_artifact_retention_idx';
const manifest = {
table: LOCAL_ARTIFACT_RETENTION_TABLE,
columns: [
'attempt_id',
'log_artifact_id',
'finished_at_ms',
'eligible_at_ms',
'disposition',
'bytes_reclaimed',
'recorded_at_ms',
],
indexes: [
`${LOCAL_ARTIFACT_RETENTION_RECORDED_INDEX}(recorded_at_ms,attempt_id)`,
`${RUN_ATTEMPT_ARTIFACT_RETENTION_INDEX}(status,finished_at_ms,id)`,
],
constraints: [
'local_artifact_retention_disposition_check',
'local_artifact_retention_finished_check',
'local_artifact_retention_eligible_check',
'local_artifact_retention_bytes_check',
'local_artifact_retention_recorded_check',
],
};
export const localArtifactRetentionManifest = manifest;
export const localArtifactRetentionMigration: Migration = {
id: '0015-local-artifact-retention',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
LOCAL_ARTIFACT_RETENTION_TABLE,
{
attempt_id: {
type: DataTypes.STRING(36),
allowNull: false,
primaryKey: true,
references: { model: RUN_ATTEMPT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
log_artifact_id: { type: DataTypes.STRING(36), allowNull: false },
finished_at_ms: { type: DataTypes.BIGINT, allowNull: false },
eligible_at_ms: { type: DataTypes.BIGINT, allowNull: false },
disposition: { type: DataTypes.STRING(16), allowNull: false },
bytes_reclaimed: { type: DataTypes.BIGINT, allowNull: false },
recorded_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(LOCAL_ARTIFACT_RETENTION_TABLE, {
fields: ['disposition'],
type: 'check',
where: { disposition: { [Op.in]: ['deleted', 'already_absent'] } },
name: 'local_artifact_retention_disposition_check',
transaction,
});
for (const [field, name] of [
['finished_at_ms', 'local_artifact_retention_finished_check'],
['eligible_at_ms', 'local_artifact_retention_eligible_check'],
['bytes_reclaimed', 'local_artifact_retention_bytes_check'],
['recorded_at_ms', 'local_artifact_retention_recorded_check'],
] as const) {
await queryInterface.addConstraint(LOCAL_ARTIFACT_RETENTION_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.gte]: 0 } },
name,
transaction,
});
}
await queryInterface.addIndex(
LOCAL_ARTIFACT_RETENTION_TABLE,
['recorded_at_ms', 'attempt_id'],
{ name: LOCAL_ARTIFACT_RETENTION_RECORDED_INDEX, transaction },
);
await queryInterface.addIndex(
RUN_ATTEMPT_TABLE,
['status', 'finished_at_ms', 'id'],
{ name: RUN_ATTEMPT_ARTIFACT_RETENTION_INDEX, transaction },
);
},
};
@@ -0,0 +1,96 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import type { Migration } from './types';
export const LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE =
'LocalArtifactMaintenanceCursors';
const manifest = {
table: LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
columns: [
'scope',
'cursor_finished_at_ms',
'cursor_attempt_id',
'version',
'updated_at_ms',
],
indexes: [] as string[],
constraints: [
'local_artifact_cursor_scope_check',
'local_artifact_cursor_pair_check',
'local_artifact_cursor_version_check',
'local_artifact_cursor_updated_check',
],
};
export const localArtifactMaintenanceCursorManifest = manifest;
export const localArtifactMaintenanceCursorMigration: Migration = {
id: '0016-local-artifact-maintenance-cursor',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
{
scope: {
type: DataTypes.STRING(32),
allowNull: false,
primaryKey: true,
},
cursor_finished_at_ms: { type: DataTypes.BIGINT, allowNull: true },
cursor_attempt_id: { type: DataTypes.STRING(36), allowNull: true },
version: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
{
fields: ['scope'],
type: 'check',
where: { scope: 'retention' },
name: 'local_artifact_cursor_scope_check',
transaction,
},
);
await queryInterface.addConstraint(
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
{
fields: ['cursor_finished_at_ms', 'cursor_attempt_id'],
type: 'check',
where: {
[Op.or]: [
{ cursor_finished_at_ms: null, cursor_attempt_id: null },
{
cursor_finished_at_ms: { [Op.not]: null },
cursor_attempt_id: { [Op.not]: null },
},
],
},
name: 'local_artifact_cursor_pair_check',
transaction,
},
);
await queryInterface.addConstraint(
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
{
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 1 } },
name: 'local_artifact_cursor_version_check',
transaction,
},
);
await queryInterface.addConstraint(
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
{
fields: ['updated_at_ms'],
type: 'check',
where: { updated_at_ms: { [Op.gte]: 0 } },
name: 'local_artifact_cursor_updated_check',
transaction,
},
);
},
};
+227
View File
@@ -0,0 +1,227 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
POLICY_SUBJECT_TYPES,
PROJECT_ROLES,
PROJECT_ROLE_BINDING_STATES,
PROJECT_STATUSES,
} from '../runtime/domain/projectPolicy';
import type { Migration } from './types';
export const PROJECT_TABLE = 'Projects';
export const PROJECT_ROLE_BINDING_TABLE = 'ProjectRoleBindings';
export const PROJECT_SLUG_INDEX = 'projects_slug_uidx';
export const PROJECT_ROLE_BINDING_CURRENT_INDEX =
'project_role_binding_current_idx';
export const PROJECT_ROLE_BINDING_MUTATION_INDEX =
'project_role_binding_mutation_uidx';
export const PROJECT_ROLE_BINDING_SUBJECT_INDEX =
'project_role_binding_subject_idx';
const manifest = {
tables: {
Projects: [
'id',
'name',
'slug',
'status',
'version',
'created_at_ms',
'updated_at_ms',
],
ProjectRoleBindings: [
'project_id',
'subject_type',
'subject_id',
'version',
'state',
'role',
'mutation_id',
'changed_by_type',
'changed_by_id',
'created_at_ms',
],
},
indexes: [
`${PROJECT_SLUG_INDEX}(slug) UNIQUE`,
`${PROJECT_ROLE_BINDING_CURRENT_INDEX}(project_id,subject_type,subject_id,version DESC)`,
`${PROJECT_ROLE_BINDING_MUTATION_INDEX}(project_id,mutation_id) UNIQUE`,
`${PROJECT_ROLE_BINDING_SUBJECT_INDEX}(subject_type,subject_id,project_id,version DESC)`,
],
constraints: [
'projects_status_check',
'projects_version_check',
'projects_created_at_check',
'projects_updated_at_check',
'project_role_bindings_subject_type_check',
'project_role_bindings_version_check',
'project_role_bindings_state_check',
'project_role_bindings_role_check',
'project_role_bindings_changed_by_type_check',
'project_role_bindings_created_at_check',
],
baseline: {
id: 'default',
name: 'Default',
slug: 'default',
status: 'active',
version: 1,
created_at_ms: 0,
updated_at_ms: 0,
},
};
export const projectPolicyManifest = manifest;
export const projectPolicyMigration: Migration = {
id: '0017-project-policy',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
PROJECT_TABLE,
{
id: { type: DataTypes.STRING(128), allowNull: false, primaryKey: true },
name: { type: DataTypes.STRING(255), allowNull: false },
slug: { type: DataTypes.STRING(128), allowNull: false },
status: { type: DataTypes.STRING(16), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(PROJECT_TABLE, {
fields: ['status'],
type: 'check',
where: { status: { [Op.in]: PROJECT_STATUSES } },
name: 'projects_status_check',
transaction,
});
for (const [field, name, minimum] of [
['version', 'projects_version_check', 1],
['created_at_ms', 'projects_created_at_check', 0],
['updated_at_ms', 'projects_updated_at_check', 0],
] as const) {
await queryInterface.addConstraint(PROJECT_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.gte]: minimum } },
name,
transaction,
});
}
await queryInterface.addIndex(PROJECT_TABLE, ['slug'], {
name: PROJECT_SLUG_INDEX,
unique: true,
transaction,
});
await queryInterface.bulkInsert(PROJECT_TABLE, [manifest.baseline], {
transaction,
});
await queryInterface.createTable(
PROJECT_ROLE_BINDING_TABLE,
{
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
subject_type: {
type: DataTypes.STRING(32),
allowNull: false,
primaryKey: true,
},
subject_id: {
type: DataTypes.STRING(255),
allowNull: false,
primaryKey: true,
},
version: {
type: DataTypes.INTEGER,
allowNull: false,
primaryKey: true,
},
state: { type: DataTypes.STRING(16), allowNull: false },
role: { type: DataTypes.STRING(16), allowNull: true },
mutation_id: { type: DataTypes.STRING(64), allowNull: false },
changed_by_type: { type: DataTypes.STRING(32), allowNull: false },
changed_by_id: { type: DataTypes.STRING(255), allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
for (const [field, values, name] of [
[
'subject_type',
POLICY_SUBJECT_TYPES,
'project_role_bindings_subject_type_check',
],
[
'state',
PROJECT_ROLE_BINDING_STATES,
'project_role_bindings_state_check',
],
['role', PROJECT_ROLES, 'project_role_bindings_role_check'],
[
'changed_by_type',
POLICY_SUBJECT_TYPES,
'project_role_bindings_changed_by_type_check',
],
] as const) {
await queryInterface.addConstraint(PROJECT_ROLE_BINDING_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.in]: values } },
name,
transaction,
});
}
await queryInterface.addConstraint(PROJECT_ROLE_BINDING_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 1 } },
name: 'project_role_bindings_version_check',
transaction,
});
await queryInterface.addConstraint(PROJECT_ROLE_BINDING_TABLE, {
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'project_role_bindings_created_at_check',
transaction,
});
await queryInterface.addIndex(PROJECT_ROLE_BINDING_TABLE, {
fields: [
'project_id',
'subject_type',
'subject_id',
{ name: 'version', order: 'DESC' },
],
name: PROJECT_ROLE_BINDING_CURRENT_INDEX,
transaction,
});
await queryInterface.addIndex(
PROJECT_ROLE_BINDING_TABLE,
['project_id', 'mutation_id'],
{
name: PROJECT_ROLE_BINDING_MUTATION_INDEX,
unique: true,
transaction,
},
);
await queryInterface.addIndex(PROJECT_ROLE_BINDING_TABLE, {
fields: [
'subject_type',
'subject_id',
'project_id',
{ name: 'version', order: 'DESC' },
],
name: PROJECT_ROLE_BINDING_SUBJECT_INDEX,
transaction,
});
},
};
@@ -0,0 +1,154 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import { POLICY_SUBJECT_TYPES } from '../runtime/domain/projectPolicy';
import { PROJECT_TABLE } from './0017-project-policy';
import type { Migration } from './types';
export const PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE =
'ProjectOwnerBootstrapChallenges';
export const PROJECT_OWNER_BOOTSTRAP_CHALLENGE_ID_INDEX =
'project_owner_bootstrap_challenge_id_uidx';
export const PROJECT_OWNER_BOOTSTRAP_CURRENT_INDEX =
'project_owner_bootstrap_current_idx';
const manifest = {
table: PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
columns: [
'project_id',
'version',
'challenge_id',
'token_digest',
'issued_at_ms',
'expires_at_ms',
'consumed_at_ms',
'claimed_subject_type',
'claimed_subject_id',
],
indexes: [
`${PROJECT_OWNER_BOOTSTRAP_CHALLENGE_ID_INDEX}(challenge_id) UNIQUE`,
`${PROJECT_OWNER_BOOTSTRAP_CURRENT_INDEX}(project_id,version DESC)`,
],
constraints: [
'project_owner_bootstrap_version_check',
'project_owner_bootstrap_issued_at_check',
'project_owner_bootstrap_lifetime_check',
'project_owner_bootstrap_claim_tuple_check',
'project_owner_bootstrap_claimed_subject_type_check',
],
};
export const projectOwnerBootstrapManifest = manifest;
export const projectOwnerBootstrapMigration: Migration = {
id: '0018-project-owner-bootstrap',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
{
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
version: {
type: DataTypes.INTEGER,
allowNull: false,
primaryKey: true,
},
challenge_id: { type: DataTypes.STRING(22), allowNull: false },
token_digest: { type: DataTypes.STRING(64), allowNull: false },
issued_at_ms: { type: DataTypes.BIGINT, allowNull: false },
expires_at_ms: { type: DataTypes.BIGINT, allowNull: false },
consumed_at_ms: { type: DataTypes.BIGINT, allowNull: true },
claimed_subject_type: { type: DataTypes.STRING(32), allowNull: true },
claimed_subject_id: { type: DataTypes.STRING(255), allowNull: true },
},
{ transaction },
);
await queryInterface.addConstraint(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
{
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 1 } },
name: 'project_owner_bootstrap_version_check',
transaction,
},
);
await queryInterface.addConstraint(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
{
fields: ['issued_at_ms'],
type: 'check',
where: { issued_at_ms: { [Op.gte]: 0 } },
name: 'project_owner_bootstrap_issued_at_check',
transaction,
},
);
await queryInterface.addConstraint(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
{
fields: ['expires_at_ms', 'issued_at_ms'],
type: 'check',
where: { expires_at_ms: { [Op.gt]: { [Op.col]: 'issued_at_ms' } } },
name: 'project_owner_bootstrap_lifetime_check',
transaction,
},
);
await queryInterface.addConstraint(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
{
fields: [
'consumed_at_ms',
'claimed_subject_type',
'claimed_subject_id',
],
type: 'check',
where: {
[Op.or]: [
{
consumed_at_ms: null,
claimed_subject_type: null,
claimed_subject_id: null,
},
{
consumed_at_ms: { [Op.not]: null },
claimed_subject_type: { [Op.not]: null },
claimed_subject_id: { [Op.not]: null },
},
],
},
name: 'project_owner_bootstrap_claim_tuple_check',
transaction,
},
);
await queryInterface.addConstraint(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
{
fields: ['claimed_subject_type'],
type: 'check',
where: { claimed_subject_type: { [Op.in]: POLICY_SUBJECT_TYPES } },
name: 'project_owner_bootstrap_claimed_subject_type_check',
transaction,
},
);
await queryInterface.addIndex(
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
['challenge_id'],
{
name: PROJECT_OWNER_BOOTSTRAP_CHALLENGE_ID_INDEX,
unique: true,
transaction,
},
);
await queryInterface.addIndex(PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE, {
fields: ['project_id', { name: 'version', order: 'DESC' }],
name: PROJECT_OWNER_BOOTSTRAP_CURRENT_INDEX,
transaction,
});
},
};
+208
View File
@@ -0,0 +1,208 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
IDENTITY_AUTHENTICATION_BINDING_STATES,
IDENTITY_SUBJECT_STATUSES,
LEGACY_PANEL_IDENTITY_PROVIDER,
LEGACY_PANEL_PROVIDER_SUBJECT,
LEGACY_PRIMARY_USER_SUBJECT_ID,
} from '../runtime/domain/identityDirectory';
import { POLICY_SUBJECT_TYPES } from '../runtime/domain/projectPolicy';
import type { Migration } from './types';
export const IDENTITY_SUBJECT_TABLE = 'IdentitySubjects';
export const IDENTITY_AUTHENTICATION_BINDING_TABLE =
'IdentityAuthenticationBindings';
export const IDENTITY_SUBJECT_STATUS_INDEX = 'identity_subject_status_idx';
export const IDENTITY_AUTHENTICATION_BINDING_CURRENT_INDEX =
'identity_auth_binding_current_idx';
export const IDENTITY_AUTHENTICATION_BINDING_SUBJECT_INDEX =
'identity_auth_binding_subject_idx';
const manifest = {
tables: {
IdentitySubjects: [
'id',
'type',
'status',
'version',
'created_at_ms',
'updated_at_ms',
],
IdentityAuthenticationBindings: [
'provider',
'provider_subject',
'version',
'state',
'subject_id',
'created_at_ms',
],
},
indexes: [
`${IDENTITY_SUBJECT_STATUS_INDEX}(type,status,id)`,
`${IDENTITY_AUTHENTICATION_BINDING_CURRENT_INDEX}(provider,provider_subject,version DESC)`,
`${IDENTITY_AUTHENTICATION_BINDING_SUBJECT_INDEX}(subject_id,provider,provider_subject,version DESC)`,
],
constraints: [
'identity_subject_type_check',
'identity_subject_status_check',
'identity_subject_version_check',
'identity_subject_created_at_check',
'identity_subject_updated_at_check',
'identity_auth_binding_version_check',
'identity_auth_binding_state_check',
'identity_auth_binding_created_at_check',
],
baseline: {
subject: {
id: LEGACY_PRIMARY_USER_SUBJECT_ID,
type: 'user',
status: 'active',
version: 1,
created_at_ms: 0,
updated_at_ms: 0,
},
binding: {
provider: LEGACY_PANEL_IDENTITY_PROVIDER,
provider_subject: LEGACY_PANEL_PROVIDER_SUBJECT,
version: 1,
state: 'active',
subject_id: LEGACY_PRIMARY_USER_SUBJECT_ID,
created_at_ms: 0,
},
},
};
export const identityDirectoryManifest = manifest;
export const identityDirectoryMigration: Migration = {
id: '0019-identity-directory',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
IDENTITY_SUBJECT_TABLE,
{
id: { type: DataTypes.STRING(255), allowNull: false, primaryKey: true },
type: { type: DataTypes.STRING(32), allowNull: false },
status: { type: DataTypes.STRING(16), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
for (const [field, values, name] of [
['type', POLICY_SUBJECT_TYPES, 'identity_subject_type_check'],
['status', IDENTITY_SUBJECT_STATUSES, 'identity_subject_status_check'],
] as const) {
await queryInterface.addConstraint(IDENTITY_SUBJECT_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.in]: values } },
name,
transaction,
});
}
for (const [field, name, minimum] of [
['version', 'identity_subject_version_check', 1],
['created_at_ms', 'identity_subject_created_at_check', 0],
['updated_at_ms', 'identity_subject_updated_at_check', 0],
] as const) {
await queryInterface.addConstraint(IDENTITY_SUBJECT_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.gte]: minimum } },
name,
transaction,
});
}
await queryInterface.addIndex(IDENTITY_SUBJECT_TABLE, {
fields: ['type', 'status', 'id'],
name: IDENTITY_SUBJECT_STATUS_INDEX,
transaction,
});
await queryInterface.bulkInsert(
IDENTITY_SUBJECT_TABLE,
[manifest.baseline.subject],
{ transaction },
);
await queryInterface.createTable(
IDENTITY_AUTHENTICATION_BINDING_TABLE,
{
provider: {
type: DataTypes.STRING(64),
allowNull: false,
primaryKey: true,
},
provider_subject: {
type: DataTypes.STRING(128),
allowNull: false,
primaryKey: true,
},
version: {
type: DataTypes.INTEGER,
allowNull: false,
primaryKey: true,
},
state: { type: DataTypes.STRING(16), allowNull: false },
subject_id: {
type: DataTypes.STRING(255),
allowNull: false,
references: { model: IDENTITY_SUBJECT_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(IDENTITY_AUTHENTICATION_BINDING_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.gte]: 1 } },
name: 'identity_auth_binding_version_check',
transaction,
});
await queryInterface.addConstraint(IDENTITY_AUTHENTICATION_BINDING_TABLE, {
fields: ['state'],
type: 'check',
where: {
state: { [Op.in]: IDENTITY_AUTHENTICATION_BINDING_STATES },
},
name: 'identity_auth_binding_state_check',
transaction,
});
await queryInterface.addConstraint(IDENTITY_AUTHENTICATION_BINDING_TABLE, {
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'identity_auth_binding_created_at_check',
transaction,
});
await queryInterface.addIndex(IDENTITY_AUTHENTICATION_BINDING_TABLE, {
fields: [
'provider',
'provider_subject',
{ name: 'version', order: 'DESC' },
],
name: IDENTITY_AUTHENTICATION_BINDING_CURRENT_INDEX,
transaction,
});
await queryInterface.addIndex(IDENTITY_AUTHENTICATION_BINDING_TABLE, {
fields: [
'subject_id',
'provider',
'provider_subject',
{ name: 'version', order: 'DESC' },
],
name: IDENTITY_AUTHENTICATION_BINDING_SUBJECT_INDEX,
transaction,
});
await queryInterface.bulkInsert(
IDENTITY_AUTHENTICATION_BINDING_TABLE,
[manifest.baseline.binding],
{ transaction },
);
},
};
+436
View File
@@ -0,0 +1,436 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
APPROVAL_DECISIONS,
APPROVAL_REQUEST_STATES,
APPROVAL_RISKS,
APPROVED_ACTION_DISPATCH_STATES,
} from '../runtime/domain/approvalRequest';
import { POLICY_SUBJECT_TYPES } from '../runtime/domain/projectPolicy';
import { PROJECT_TABLE } from './0017-project-policy';
import type { Migration } from './types';
export const APPROVAL_REQUEST_TABLE = 'ApprovalRequests';
export const APPROVED_ACTION_DISPATCH_TABLE = 'ApprovedActionDispatches';
export const APPROVAL_REQUEST_DECISION_ID_INDEX =
'approval_request_decision_id_uidx';
export const APPROVAL_REQUEST_CONSUMPTION_ID_INDEX =
'approval_request_consumption_id_uidx';
export const APPROVAL_REQUEST_PENDING_INDEX = 'approval_request_pending_idx';
export const APPROVAL_REQUEST_REQUESTER_INDEX =
'approval_request_requester_idx';
export const APPROVED_ACTION_DISPATCH_REQUEST_INDEX =
'approved_action_dispatch_request_uidx';
export const APPROVED_ACTION_DISPATCH_PENDING_INDEX =
'approved_action_dispatch_pending_idx';
const approvalRequestColumns = [
'id',
'project_id',
'version',
'state',
'permission',
'action_type',
'action_ref',
'action_digest',
'preview_digest',
'risk',
'requested_by_type',
'requested_by_id',
'requested_at_ms',
'expires_at_ms',
'decision_id',
'decision',
'decision_reason_code',
'decided_by_type',
'decided_by_id',
'decided_at_ms',
'consumption_id',
'dispatch_id',
'consumed_by_type',
'consumed_by_id',
'consumed_at_ms',
];
const approvedActionDispatchColumns = [
'id',
'approval_request_id',
'approval_request_version',
'project_id',
'state',
'permission',
'action_type',
'action_ref',
'action_digest',
'preview_digest',
'requested_by_type',
'requested_by_id',
'consumed_by_type',
'consumed_by_id',
'created_at_ms',
];
const constraints = [
'approval_request_version_check',
'approval_request_state_check',
'approval_request_risk_check',
'approval_request_requester_type_check',
'approval_request_lifetime_check',
'approval_request_decision_tuple_check',
'approval_request_decision_value_check',
'approval_request_decided_by_type_check',
'approval_request_decision_time_check',
'approval_request_consumption_tuple_check',
'approval_request_consumed_by_type_check',
'approval_request_consumption_time_check',
'approval_request_state_tuple_check',
'approved_action_dispatch_version_check',
'approved_action_dispatch_state_check',
'approved_action_dispatch_requester_type_check',
'approved_action_dispatch_consumer_type_check',
'approved_action_dispatch_created_at_check',
];
const manifest = {
tables: {
ApprovalRequests: approvalRequestColumns,
ApprovedActionDispatches: approvedActionDispatchColumns,
},
indexes: [
`${APPROVAL_REQUEST_DECISION_ID_INDEX}(decision_id) UNIQUE`,
`${APPROVAL_REQUEST_CONSUMPTION_ID_INDEX}(consumption_id) UNIQUE`,
`${APPROVAL_REQUEST_PENDING_INDEX}(project_id,state,expires_at_ms,id)`,
`${APPROVAL_REQUEST_REQUESTER_INDEX}(project_id,requested_by_type,requested_by_id,requested_at_ms DESC,id)`,
`${APPROVED_ACTION_DISPATCH_REQUEST_INDEX}(approval_request_id) UNIQUE`,
`${APPROVED_ACTION_DISPATCH_PENDING_INDEX}(project_id,state,created_at_ms,id)`,
],
constraints,
};
export const approvalRequestManifest = manifest;
function nullableTuple(...fields: string[]) {
return {
[Op.or]: [
Object.fromEntries(fields.map((field) => [field, null])),
Object.fromEntries(fields.map((field) => [field, { [Op.not]: null }])),
],
};
}
export const approvalRequestMigration: Migration = {
id: '0020-approval-requests',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
APPROVAL_REQUEST_TABLE,
{
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
version: { type: DataTypes.INTEGER, allowNull: false },
state: { type: DataTypes.STRING(16), allowNull: false },
permission: { type: DataTypes.STRING(255), allowNull: false },
action_type: { type: DataTypes.STRING(64), allowNull: false },
action_ref: { type: DataTypes.STRING(255), allowNull: false },
action_digest: { type: DataTypes.STRING(64), allowNull: false },
preview_digest: { type: DataTypes.STRING(64), allowNull: false },
risk: { type: DataTypes.STRING(16), allowNull: false },
requested_by_type: { type: DataTypes.STRING(32), allowNull: false },
requested_by_id: { type: DataTypes.STRING(255), allowNull: false },
requested_at_ms: { type: DataTypes.BIGINT, allowNull: false },
expires_at_ms: { type: DataTypes.BIGINT, allowNull: false },
decision_id: { type: DataTypes.STRING(64), allowNull: true },
decision: { type: DataTypes.STRING(16), allowNull: true },
decision_reason_code: { type: DataTypes.STRING(64), allowNull: true },
decided_by_type: { type: DataTypes.STRING(32), allowNull: true },
decided_by_id: { type: DataTypes.STRING(255), allowNull: true },
decided_at_ms: { type: DataTypes.BIGINT, allowNull: true },
consumption_id: { type: DataTypes.STRING(64), allowNull: true },
dispatch_id: { type: DataTypes.STRING(64), allowNull: true },
consumed_by_type: { type: DataTypes.STRING(32), allowNull: true },
consumed_by_id: { type: DataTypes.STRING(255), allowNull: true },
consumed_at_ms: { type: DataTypes.BIGINT, allowNull: true },
},
{ transaction },
);
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: ['version'],
type: 'check',
where: { version: { [Op.between]: [1, 3] } },
name: 'approval_request_version_check',
transaction,
});
for (const [field, values, name] of [
['state', APPROVAL_REQUEST_STATES, 'approval_request_state_check'],
['risk', APPROVAL_RISKS, 'approval_request_risk_check'],
[
'requested_by_type',
POLICY_SUBJECT_TYPES,
'approval_request_requester_type_check',
],
[
'decided_by_type',
POLICY_SUBJECT_TYPES,
'approval_request_decided_by_type_check',
],
[
'consumed_by_type',
POLICY_SUBJECT_TYPES,
'approval_request_consumed_by_type_check',
],
] as const) {
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.in]: values } },
name,
transaction,
});
}
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: ['requested_at_ms', 'expires_at_ms'],
type: 'check',
where: {
requested_at_ms: { [Op.gte]: 0 },
expires_at_ms: { [Op.gt]: { [Op.col]: 'requested_at_ms' } },
},
name: 'approval_request_lifetime_check',
transaction,
});
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: [
'decision_id',
'decision',
'decision_reason_code',
'decided_by_type',
'decided_by_id',
'decided_at_ms',
],
type: 'check',
where: nullableTuple(
'decision_id',
'decision',
'decision_reason_code',
'decided_by_type',
'decided_by_id',
'decided_at_ms',
),
name: 'approval_request_decision_tuple_check',
transaction,
});
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: ['decision'],
type: 'check',
where: { decision: { [Op.in]: APPROVAL_DECISIONS } },
name: 'approval_request_decision_value_check',
transaction,
});
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: ['decided_at_ms', 'requested_at_ms', 'expires_at_ms'],
type: 'check',
where: {
[Op.or]: [
{ decided_at_ms: null },
{
decided_at_ms: {
[Op.gte]: { [Op.col]: 'requested_at_ms' },
[Op.lt]: { [Op.col]: 'expires_at_ms' },
},
},
],
},
name: 'approval_request_decision_time_check',
transaction,
});
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: [
'consumption_id',
'dispatch_id',
'consumed_by_type',
'consumed_by_id',
'consumed_at_ms',
],
type: 'check',
where: nullableTuple(
'consumption_id',
'dispatch_id',
'consumed_by_type',
'consumed_by_id',
'consumed_at_ms',
),
name: 'approval_request_consumption_tuple_check',
transaction,
});
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: ['consumed_at_ms', 'decided_at_ms', 'expires_at_ms'],
type: 'check',
where: {
[Op.or]: [
{ consumed_at_ms: null },
{
consumed_at_ms: {
[Op.gte]: { [Op.col]: 'decided_at_ms' },
[Op.lt]: { [Op.col]: 'expires_at_ms' },
},
},
],
},
name: 'approval_request_consumption_time_check',
transaction,
});
await queryInterface.addConstraint(APPROVAL_REQUEST_TABLE, {
fields: ['state', 'version', 'decision', 'decision_id', 'consumption_id'],
type: 'check',
where: {
[Op.or]: [
{
state: 'pending',
version: 1,
decision_id: null,
consumption_id: null,
},
{
state: 'approved',
version: 2,
decision: 'approved',
consumption_id: null,
},
{
state: 'rejected',
version: 2,
decision: 'rejected',
consumption_id: null,
},
{
state: 'consumed',
version: 3,
decision: 'approved',
consumption_id: { [Op.not]: null },
},
],
},
name: 'approval_request_state_tuple_check',
transaction,
});
await queryInterface.addIndex(APPROVAL_REQUEST_TABLE, ['decision_id'], {
name: APPROVAL_REQUEST_DECISION_ID_INDEX,
unique: true,
transaction,
});
await queryInterface.addIndex(APPROVAL_REQUEST_TABLE, ['consumption_id'], {
name: APPROVAL_REQUEST_CONSUMPTION_ID_INDEX,
unique: true,
transaction,
});
await queryInterface.addIndex(APPROVAL_REQUEST_TABLE, {
fields: ['project_id', 'state', 'expires_at_ms', 'id'],
name: APPROVAL_REQUEST_PENDING_INDEX,
transaction,
});
await queryInterface.addIndex(APPROVAL_REQUEST_TABLE, {
fields: [
'project_id',
'requested_by_type',
'requested_by_id',
{ name: 'requested_at_ms', order: 'DESC' },
'id',
],
name: APPROVAL_REQUEST_REQUESTER_INDEX,
transaction,
});
await queryInterface.createTable(
APPROVED_ACTION_DISPATCH_TABLE,
{
id: { type: DataTypes.STRING(64), allowNull: false, primaryKey: true },
approval_request_id: {
type: DataTypes.STRING(64),
allowNull: false,
references: { model: APPROVAL_REQUEST_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
approval_request_version: { type: DataTypes.INTEGER, allowNull: false },
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
state: { type: DataTypes.STRING(16), allowNull: false },
permission: { type: DataTypes.STRING(255), allowNull: false },
action_type: { type: DataTypes.STRING(64), allowNull: false },
action_ref: { type: DataTypes.STRING(255), allowNull: false },
action_digest: { type: DataTypes.STRING(64), allowNull: false },
preview_digest: { type: DataTypes.STRING(64), allowNull: false },
requested_by_type: { type: DataTypes.STRING(32), allowNull: false },
requested_by_id: { type: DataTypes.STRING(255), allowNull: false },
consumed_by_type: { type: DataTypes.STRING(32), allowNull: false },
consumed_by_id: { type: DataTypes.STRING(255), allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(APPROVED_ACTION_DISPATCH_TABLE, {
fields: ['approval_request_version'],
type: 'check',
where: { approval_request_version: 3 },
name: 'approved_action_dispatch_version_check',
transaction,
});
for (const [field, values, name] of [
[
'state',
APPROVED_ACTION_DISPATCH_STATES,
'approved_action_dispatch_state_check',
],
[
'requested_by_type',
POLICY_SUBJECT_TYPES,
'approved_action_dispatch_requester_type_check',
],
[
'consumed_by_type',
POLICY_SUBJECT_TYPES,
'approved_action_dispatch_consumer_type_check',
],
] as const) {
await queryInterface.addConstraint(APPROVED_ACTION_DISPATCH_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.in]: values } },
name,
transaction,
});
}
await queryInterface.addConstraint(APPROVED_ACTION_DISPATCH_TABLE, {
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'approved_action_dispatch_created_at_check',
transaction,
});
await queryInterface.addIndex(
APPROVED_ACTION_DISPATCH_TABLE,
['approval_request_id'],
{
name: APPROVED_ACTION_DISPATCH_REQUEST_INDEX,
unique: true,
transaction,
},
);
await queryInterface.addIndex(APPROVED_ACTION_DISPATCH_TABLE, {
fields: ['project_id', 'state', 'created_at_ms', 'id'],
name: APPROVED_ACTION_DISPATCH_PENDING_INDEX,
transaction,
});
},
};
@@ -0,0 +1,227 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
APPROVED_ACTION_EXECUTION_STATUSES,
DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS,
MAX_APPROVED_ACTION_ATTEMPTS,
} from '../runtime/domain/approvedActionDispatchExecution';
import { PROJECT_TABLE } from './0017-project-policy';
import { APPROVED_ACTION_DISPATCH_TABLE } from './0020-approval-requests';
import type { Migration } from './types';
export const APPROVED_ACTION_DISPATCH_EXECUTION_TABLE =
'ApprovedActionDispatchExecutions';
export const APPROVED_ACTION_DISPATCH_EXECUTION_DUE_INDEX =
'approved_action_execution_due_idx';
export const APPROVED_ACTION_DISPATCH_EXECUTION_PROJECT_INDEX =
'approved_action_execution_project_idx';
export const APPROVED_ACTION_DISPATCH_EXECUTION_LEASE_INDEX =
'approved_action_execution_lease_idx';
const columns = [
'dispatch_id',
'project_id',
'status',
'version',
'attempt_count',
'max_attempts',
'eligible_at_ms',
'next_attempt_at_ms',
'lease_owner',
'lease_token',
'lease_expires_at_ms',
'started_at_ms',
'result_mutation_id',
'last_result_code',
'completed_at_ms',
'created_at_ms',
'updated_at_ms',
];
const manifest = {
table: APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
columns,
indexes: [
`${APPROVED_ACTION_DISPATCH_EXECUTION_DUE_INDEX}(eligible_at_ms,dispatch_id)`,
`${APPROVED_ACTION_DISPATCH_EXECUTION_PROJECT_INDEX}(project_id,status,created_at_ms,dispatch_id)`,
`${APPROVED_ACTION_DISPATCH_EXECUTION_LEASE_INDEX}(status,lease_expires_at_ms,dispatch_id)`,
],
constraints: [
'approved_action_execution_status_check',
'approved_action_execution_version_check',
'approved_action_execution_attempt_count_check',
'approved_action_execution_max_attempts_check',
'approved_action_execution_attempt_budget_check',
'approved_action_execution_lease_tuple_check',
'approved_action_execution_created_at_check',
'approved_action_execution_updated_at_check',
],
baselineMaxAttempts: DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS,
};
export const approvedActionDispatchExecutionManifest = manifest;
export const approvedActionDispatchExecutionMigration: Migration = {
id: '0021-approved-action-dispatch-executions',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
dispatch_id: {
type: DataTypes.STRING(64),
allowNull: false,
primaryKey: true,
references: { model: APPROVED_ACTION_DISPATCH_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
status: { type: DataTypes.STRING(16), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
attempt_count: { type: DataTypes.INTEGER, allowNull: false },
max_attempts: { type: DataTypes.INTEGER, allowNull: false },
eligible_at_ms: { type: DataTypes.BIGINT, allowNull: true },
next_attempt_at_ms: { type: DataTypes.BIGINT, allowNull: true },
lease_owner: { type: DataTypes.STRING(128), allowNull: true },
lease_token: { type: DataTypes.STRING(128), allowNull: true },
lease_expires_at_ms: { type: DataTypes.BIGINT, allowNull: true },
started_at_ms: { type: DataTypes.BIGINT, allowNull: true },
result_mutation_id: { type: DataTypes.STRING(64), allowNull: true },
last_result_code: { type: DataTypes.STRING(64), allowNull: true },
completed_at_ms: { type: DataTypes.BIGINT, allowNull: true },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
fields: ['status'],
type: 'check',
where: { status: { [Op.in]: APPROVED_ACTION_EXECUTION_STATUSES } },
name: 'approved_action_execution_status_check',
transaction,
},
);
for (const [field, minimum, maximum, name] of [
['version', 0, 2_147_483_647, 'approved_action_execution_version_check'],
[
'attempt_count',
0,
MAX_APPROVED_ACTION_ATTEMPTS,
'approved_action_execution_attempt_count_check',
],
[
'max_attempts',
1,
MAX_APPROVED_ACTION_ATTEMPTS,
'approved_action_execution_max_attempts_check',
],
] as const) {
await queryInterface.addConstraint(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
fields: [field],
type: 'check',
where: { [field]: { [Op.between]: [minimum, maximum] } },
name,
transaction,
},
);
}
await queryInterface.addConstraint(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
fields: ['attempt_count', 'max_attempts'],
type: 'check',
where: { attempt_count: { [Op.lte]: { [Op.col]: 'max_attempts' } } },
name: 'approved_action_execution_attempt_budget_check',
transaction,
},
);
await queryInterface.addConstraint(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
fields: ['lease_owner', 'lease_token', 'lease_expires_at_ms'],
type: 'check',
where: {
[Op.or]: [
{
lease_owner: null,
lease_token: null,
lease_expires_at_ms: null,
},
{
lease_owner: { [Op.not]: null },
lease_token: { [Op.not]: null },
lease_expires_at_ms: { [Op.not]: null },
},
],
},
name: 'approved_action_execution_lease_tuple_check',
transaction,
},
);
await queryInterface.addConstraint(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
fields: ['created_at_ms'],
type: 'check',
where: { created_at_ms: { [Op.gte]: 0 } },
name: 'approved_action_execution_created_at_check',
transaction,
},
);
await queryInterface.addConstraint(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
{
fields: ['updated_at_ms', 'created_at_ms'],
type: 'check',
where: { updated_at_ms: { [Op.gte]: { [Op.col]: 'created_at_ms' } } },
name: 'approved_action_execution_updated_at_check',
transaction,
},
);
await queryInterface.addIndex(
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
['eligible_at_ms', 'dispatch_id'],
{
name: APPROVED_ACTION_DISPATCH_EXECUTION_DUE_INDEX,
transaction,
},
);
await queryInterface.addIndex(APPROVED_ACTION_DISPATCH_EXECUTION_TABLE, {
fields: ['project_id', 'status', 'created_at_ms', 'dispatch_id'],
name: APPROVED_ACTION_DISPATCH_EXECUTION_PROJECT_INDEX,
transaction,
});
await queryInterface.addIndex(APPROVED_ACTION_DISPATCH_EXECUTION_TABLE, {
fields: ['status', 'lease_expires_at_ms', 'dispatch_id'],
name: APPROVED_ACTION_DISPATCH_EXECUTION_LEASE_INDEX,
transaction,
});
await queryInterface.sequelize.query(
`INSERT INTO "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
(dispatch_id, project_id, status, version, attempt_count, max_attempts,
eligible_at_ms, next_attempt_at_ms, lease_owner, lease_token,
lease_expires_at_ms, started_at_ms, result_mutation_id,
last_result_code, completed_at_ms, created_at_ms, updated_at_ms)
SELECT id, project_id, 'pending', 0, 0, :maxAttempts,
created_at_ms, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
created_at_ms, created_at_ms
FROM "${APPROVED_ACTION_DISPATCH_TABLE}"`,
{
replacements: { maxAttempts: DEFAULT_APPROVED_ACTION_MAX_ATTEMPTS },
transaction,
},
);
},
};
@@ -0,0 +1,447 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
APPROVED_ACTION_RECOVERY_CONTROL_STATUSES,
APPROVED_ACTION_RECOVERY_DECISIONS,
APPROVED_ACTION_RECOVERY_FINDINGS,
APPROVED_ACTION_RECOVERY_SOURCES,
MAX_APPROVED_ACTION_RECOVERY_FINDINGS,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
} from '../runtime/domain/approvedActionRecovery';
import { PROJECT_TABLE } from './0017-project-policy';
import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from './0021-approved-action-dispatch-executions';
import type { Migration } from './types';
export const APPROVED_ACTION_RECOVERY_CONTROL_TABLE =
'ApprovedActionRecoveryControls';
export const APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE =
'ApprovedActionRecoveryResolutions';
export const APPROVED_ACTION_RECOVERY_DUE_INDEX =
'approved_action_recovery_due_idx';
export const APPROVED_ACTION_RECOVERY_PROJECT_INDEX =
'approved_action_recovery_project_idx';
export const APPROVED_ACTION_RECOVERY_LEASE_INDEX =
'approved_action_recovery_lease_idx';
export const APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX =
'approved_action_recovery_resolution_mutation_uidx';
export const APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX =
'approved_action_recovery_resolution_project_idx';
const controlColumns = [
'dispatch_id',
'project_id',
'execution_version',
'status',
'version',
'next_scan_at_ms',
'lease_owner',
'lease_token',
'lease_expires_at_ms',
'finding_count',
'last_finding_mutation_id',
'last_finding',
'last_result_code',
'last_evidence_digest',
'resolution_mutation_id',
'created_at_ms',
'updated_at_ms',
];
const resolutionColumns = [
'dispatch_id',
'project_id',
'execution_version',
'mutation_id',
'source',
'decision',
'evidence_digest',
'reason_code',
'resolved_by_type',
'resolved_by_id',
'resolved_at_ms',
];
const constraints = [
'approved_action_recovery_status_check',
'approved_action_recovery_execution_version_check',
'approved_action_recovery_version_check',
'approved_action_recovery_finding_count_check',
'approved_action_recovery_lease_tuple_check',
'approved_action_recovery_finding_tuple_check',
'approved_action_recovery_finding_value_check',
'approved_action_recovery_evidence_digest_check',
'approved_action_recovery_resolution_mutation_tuple_check',
'approved_action_recovery_timestamps_check',
'approved_action_recovery_resolution_execution_version_check',
'approved_action_recovery_resolution_source_check',
'approved_action_recovery_resolution_decision_check',
'approved_action_recovery_resolution_actor_tuple_check',
'approved_action_recovery_resolution_source_tuple_check',
'approved_action_recovery_resolution_evidence_digest_check',
'approved_action_recovery_resolution_time_check',
];
const manifest = {
tables: {
ApprovedActionRecoveryControls: controlColumns,
ApprovedActionRecoveryResolutions: resolutionColumns,
},
indexes: [
`${APPROVED_ACTION_RECOVERY_DUE_INDEX}(status,next_scan_at_ms,dispatch_id)`,
`${APPROVED_ACTION_RECOVERY_PROJECT_INDEX}(project_id,status,created_at_ms,dispatch_id)`,
`${APPROVED_ACTION_RECOVERY_LEASE_INDEX}(status,lease_expires_at_ms,dispatch_id)`,
`${APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX}(mutation_id) UNIQUE`,
`${APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX}(project_id,resolved_at_ms,dispatch_id)`,
],
constraints,
};
export const approvedActionRecoveryManifest = manifest;
export const approvedActionRecoveryMigration: Migration = {
id: '0022-approved-action-recovery',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
{
dispatch_id: {
type: DataTypes.STRING(64),
allowNull: false,
primaryKey: true,
references: {
model: APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
key: 'dispatch_id',
},
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
execution_version: { type: DataTypes.INTEGER, allowNull: false },
status: { type: DataTypes.STRING(24), allowNull: false },
version: { type: DataTypes.INTEGER, allowNull: false },
next_scan_at_ms: { type: DataTypes.BIGINT, allowNull: true },
lease_owner: { type: DataTypes.STRING(128), allowNull: true },
lease_token: { type: DataTypes.STRING(128), allowNull: true },
lease_expires_at_ms: { type: DataTypes.BIGINT, allowNull: true },
finding_count: { type: DataTypes.INTEGER, allowNull: false },
last_finding_mutation_id: {
type: DataTypes.STRING(64),
allowNull: true,
},
last_finding: { type: DataTypes.STRING(24), allowNull: true },
last_result_code: { type: DataTypes.STRING(64), allowNull: true },
last_evidence_digest: { type: DataTypes.STRING(64), allowNull: true },
resolution_mutation_id: {
type: DataTypes.STRING(64),
allowNull: true,
},
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
updated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
await queryInterface.createTable(
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
{
dispatch_id: {
type: DataTypes.STRING(64),
allowNull: false,
primaryKey: true,
references: {
model: APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
key: 'dispatch_id',
},
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'CASCADE',
onUpdate: 'CASCADE',
},
execution_version: { type: DataTypes.INTEGER, allowNull: false },
mutation_id: { type: DataTypes.STRING(64), allowNull: false },
source: { type: DataTypes.STRING(24), allowNull: false },
decision: { type: DataTypes.STRING(24), allowNull: false },
evidence_digest: { type: DataTypes.STRING(64), allowNull: true },
reason_code: { type: DataTypes.STRING(64), allowNull: false },
resolved_by_type: { type: DataTypes.STRING(32), allowNull: true },
resolved_by_id: { type: DataTypes.STRING(255), allowNull: true },
resolved_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
for (const [table, field, minimum, maximum, name] of [
[
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
'execution_version',
1,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
'approved_action_recovery_execution_version_check',
],
[
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
'version',
0,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
'approved_action_recovery_version_check',
],
[
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
'finding_count',
0,
MAX_APPROVED_ACTION_RECOVERY_FINDINGS,
'approved_action_recovery_finding_count_check',
],
[
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
'execution_version',
1,
MAX_APPROVED_ACTION_RECOVERY_VERSION,
'approved_action_recovery_resolution_execution_version_check',
],
] as const) {
await queryInterface.addConstraint(table, {
fields: [field],
type: 'check',
where: { [field]: { [Op.between]: [minimum, maximum] } },
name,
transaction,
});
}
for (const [table, field, values, name] of [
[
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
'status',
APPROVED_ACTION_RECOVERY_CONTROL_STATUSES,
'approved_action_recovery_status_check',
],
[
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
'last_finding',
APPROVED_ACTION_RECOVERY_FINDINGS,
'approved_action_recovery_finding_value_check',
],
[
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
'source',
APPROVED_ACTION_RECOVERY_SOURCES,
'approved_action_recovery_resolution_source_check',
],
[
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
'decision',
APPROVED_ACTION_RECOVERY_DECISIONS,
'approved_action_recovery_resolution_decision_check',
],
] as const) {
await queryInterface.addConstraint(table, {
fields: [field],
type: 'check',
where: { [field]: { [Op.in]: values } },
name,
transaction,
});
}
await queryInterface.addConstraint(APPROVED_ACTION_RECOVERY_CONTROL_TABLE, {
fields: ['lease_owner', 'lease_token', 'lease_expires_at_ms'],
type: 'check',
where: {
[Op.or]: [
{ lease_owner: null, lease_token: null, lease_expires_at_ms: null },
{
lease_owner: { [Op.not]: null },
lease_token: { [Op.not]: null },
lease_expires_at_ms: { [Op.not]: null },
},
],
},
name: 'approved_action_recovery_lease_tuple_check',
transaction,
});
await queryInterface.addConstraint(APPROVED_ACTION_RECOVERY_CONTROL_TABLE, {
fields: ['last_finding_mutation_id', 'last_finding', 'last_result_code'],
type: 'check',
where: {
[Op.or]: [
{
last_finding_mutation_id: null,
last_finding: null,
last_result_code: null,
},
{
last_finding_mutation_id: { [Op.not]: null },
last_finding: { [Op.not]: null },
last_result_code: { [Op.not]: null },
},
],
},
name: 'approved_action_recovery_finding_tuple_check',
transaction,
});
for (const [table, field, name] of [
[
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
'last_evidence_digest',
'approved_action_recovery_evidence_digest_check',
],
[
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
'evidence_digest',
'approved_action_recovery_resolution_evidence_digest_check',
],
] as const) {
await queryInterface.addConstraint(table, {
fields: [field],
type: 'check',
where: {
[Op.or]: [
{ [field]: null },
queryInterface.sequelize.where(
queryInterface.sequelize.fn(
'length',
queryInterface.sequelize.col(field),
),
64,
),
],
},
name,
transaction,
});
}
await queryInterface.addConstraint(APPROVED_ACTION_RECOVERY_CONTROL_TABLE, {
fields: ['status', 'resolution_mutation_id'],
type: 'check',
where: {
[Op.or]: [
{ status: 'resolved', resolution_mutation_id: { [Op.not]: null } },
{
status: { [Op.not]: 'resolved' },
resolution_mutation_id: null,
},
],
},
name: 'approved_action_recovery_resolution_mutation_tuple_check',
transaction,
});
await queryInterface.addConstraint(APPROVED_ACTION_RECOVERY_CONTROL_TABLE, {
fields: ['created_at_ms', 'updated_at_ms'],
type: 'check',
where: {
created_at_ms: { [Op.gte]: 0 },
updated_at_ms: { [Op.gte]: { [Op.col]: 'created_at_ms' } },
},
name: 'approved_action_recovery_timestamps_check',
transaction,
});
await queryInterface.addConstraint(
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
{
fields: ['resolved_by_type', 'resolved_by_id'],
type: 'check',
where: {
[Op.or]: [
{ resolved_by_type: null, resolved_by_id: null },
{
resolved_by_type: 'user',
resolved_by_id: { [Op.not]: null },
},
],
},
name: 'approved_action_recovery_resolution_actor_tuple_check',
transaction,
},
);
await queryInterface.addConstraint(
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
{
fields: ['source', 'decision', 'evidence_digest', 'resolved_by_type'],
type: 'check',
where: {
[Op.or]: [
{
source: 'automatic_evidence',
decision: {
[Op.in]: ['confirm_succeeded', 'confirm_failed'],
},
evidence_digest: { [Op.not]: null },
resolved_by_type: null,
},
{ source: 'human', resolved_by_type: 'user' },
],
},
name: 'approved_action_recovery_resolution_source_tuple_check',
transaction,
},
);
await queryInterface.addConstraint(
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
{
fields: ['resolved_at_ms'],
type: 'check',
where: { resolved_at_ms: { [Op.gte]: 0 } },
name: 'approved_action_recovery_resolution_time_check',
transaction,
},
);
await queryInterface.addIndex(
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
['status', 'next_scan_at_ms', 'dispatch_id'],
{ name: APPROVED_ACTION_RECOVERY_DUE_INDEX, transaction },
);
await queryInterface.addIndex(APPROVED_ACTION_RECOVERY_CONTROL_TABLE, {
fields: ['project_id', 'status', 'created_at_ms', 'dispatch_id'],
name: APPROVED_ACTION_RECOVERY_PROJECT_INDEX,
transaction,
});
await queryInterface.addIndex(APPROVED_ACTION_RECOVERY_CONTROL_TABLE, {
fields: ['status', 'lease_expires_at_ms', 'dispatch_id'],
name: APPROVED_ACTION_RECOVERY_LEASE_INDEX,
transaction,
});
await queryInterface.addIndex(
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
['mutation_id'],
{
name: APPROVED_ACTION_RECOVERY_RESOLUTION_MUTATION_INDEX,
unique: true,
transaction,
},
);
await queryInterface.addIndex(
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
['project_id', 'resolved_at_ms', 'dispatch_id'],
{
name: APPROVED_ACTION_RECOVERY_RESOLUTION_PROJECT_INDEX,
transaction,
},
);
await queryInterface.sequelize.query(
`INSERT INTO "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}"
(dispatch_id, project_id, execution_version, status, version,
next_scan_at_ms, lease_owner, lease_token, lease_expires_at_ms,
finding_count, last_finding_mutation_id, last_finding,
last_result_code, last_evidence_digest,
resolution_mutation_id, created_at_ms, updated_at_ms)
SELECT dispatch_id, project_id, version, 'armed', 0,
lease_expires_at_ms, NULL, NULL, NULL,
0, NULL, NULL, NULL, NULL, NULL, started_at_ms, started_at_ms
FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}"
WHERE status = 'executing'`,
{ transaction },
);
},
};
@@ -0,0 +1,188 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
APPROVED_RUN_ACTION_TYPE,
APPROVED_RUN_RECEIPT_RESULT_CODE,
APPROVED_RUN_RECEIPT_SCHEMA_VERSION,
} from '../runtime/domain/approvedRunAction';
import { PROJECT_TABLE } from './0017-project-policy';
import {
APPROVAL_REQUEST_TABLE,
APPROVED_ACTION_DISPATCH_TABLE,
} from './0020-approval-requests';
import { RUN_TABLE } from './0002-run-schema';
import type { Migration } from './types';
export const APPROVED_RUN_ACTION_RECEIPT_TABLE = 'ApprovedRunActionReceipts';
export const APPROVED_RUN_ACTION_RECEIPT_PROJECT_INDEX =
'approved_run_receipt_project_idx';
export const APPROVED_RUN_ACTION_RECEIPT_RESOURCE_UNIQUE_INDEX =
'approved_run_receipt_resource_uidx';
const columns = [
'dispatch_id',
'approval_request_id',
'project_id',
'schema_version',
'action_type',
'action_digest',
'execution_attempt',
'execution_version',
'started_at_ms',
'idempotency_key',
'outcome',
'result_code',
'resource_type',
'resource_id',
'finished_at_ms',
'evidence_digest',
'created_at_ms',
];
const manifest = {
table: APPROVED_RUN_ACTION_RECEIPT_TABLE,
columns,
indexes: [
`${APPROVED_RUN_ACTION_RECEIPT_PROJECT_INDEX}(project_id,created_at_ms,dispatch_id)`,
`${APPROVED_RUN_ACTION_RECEIPT_RESOURCE_UNIQUE_INDEX}(resource_type,resource_id)`,
],
constraints: [
'approved_run_receipt_schema_version_check',
'approved_run_receipt_action_type_check',
'approved_run_receipt_execution_attempt_check',
'approved_run_receipt_execution_version_check',
'approved_run_receipt_idempotency_check',
'approved_run_receipt_outcome_check',
'approved_run_receipt_result_code_check',
'approved_run_receipt_resource_type_check',
'approved_run_receipt_timestamps_check',
],
};
export const approvedRunActionReceiptManifest = manifest;
export const approvedRunActionReceiptMigration: Migration = {
id: '0023-approved-run-action-receipts',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
APPROVED_RUN_ACTION_RECEIPT_TABLE,
{
dispatch_id: {
type: DataTypes.STRING(64),
allowNull: false,
primaryKey: true,
references: { model: APPROVED_ACTION_DISPATCH_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
approval_request_id: {
type: DataTypes.STRING(64),
allowNull: false,
references: { model: APPROVAL_REQUEST_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
schema_version: { type: DataTypes.INTEGER, allowNull: false },
action_type: { type: DataTypes.STRING(64), allowNull: false },
action_digest: { type: DataTypes.STRING(64), allowNull: false },
execution_attempt: { type: DataTypes.INTEGER, allowNull: false },
execution_version: { type: DataTypes.INTEGER, allowNull: false },
started_at_ms: { type: DataTypes.BIGINT, allowNull: false },
idempotency_key: { type: DataTypes.STRING(64), allowNull: false },
outcome: { type: DataTypes.STRING(16), allowNull: false },
result_code: { type: DataTypes.STRING(64), allowNull: false },
resource_type: { type: DataTypes.STRING(16), allowNull: false },
resource_id: {
type: DataTypes.STRING(64),
allowNull: false,
references: { model: RUN_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
finished_at_ms: { type: DataTypes.BIGINT, allowNull: false },
evidence_digest: { type: DataTypes.STRING(64), allowNull: false },
created_at_ms: { type: DataTypes.BIGINT, allowNull: false },
},
{ transaction },
);
for (const [field, value, name] of [
[
'schema_version',
APPROVED_RUN_RECEIPT_SCHEMA_VERSION,
'approved_run_receipt_schema_version_check',
],
[
'action_type',
APPROVED_RUN_ACTION_TYPE,
'approved_run_receipt_action_type_check',
],
['outcome', 'succeeded', 'approved_run_receipt_outcome_check'],
[
'result_code',
APPROVED_RUN_RECEIPT_RESULT_CODE,
'approved_run_receipt_result_code_check',
],
['resource_type', 'run', 'approved_run_receipt_resource_type_check'],
] as const) {
await queryInterface.addConstraint(APPROVED_RUN_ACTION_RECEIPT_TABLE, {
fields: [field],
type: 'check',
where: { [field]: value },
name,
transaction,
});
}
for (const [field, maximum, name] of [
['execution_attempt', 16, 'approved_run_receipt_execution_attempt_check'],
[
'execution_version',
2_147_483_647,
'approved_run_receipt_execution_version_check',
],
] as const) {
await queryInterface.addConstraint(APPROVED_RUN_ACTION_RECEIPT_TABLE, {
fields: [field],
type: 'check',
where: { [field]: { [Op.between]: [1, maximum] } },
name,
transaction,
});
}
await queryInterface.addConstraint(APPROVED_RUN_ACTION_RECEIPT_TABLE, {
fields: ['dispatch_id', 'idempotency_key'],
type: 'check',
where: { idempotency_key: { [Op.eq]: { [Op.col]: 'dispatch_id' } } },
name: 'approved_run_receipt_idempotency_check',
transaction,
});
await queryInterface.addConstraint(APPROVED_RUN_ACTION_RECEIPT_TABLE, {
fields: ['started_at_ms', 'finished_at_ms', 'created_at_ms'],
type: 'check',
where: {
finished_at_ms: { [Op.gte]: { [Op.col]: 'started_at_ms' } },
created_at_ms: { [Op.eq]: { [Op.col]: 'finished_at_ms' } },
},
name: 'approved_run_receipt_timestamps_check',
transaction,
});
await queryInterface.addIndex(APPROVED_RUN_ACTION_RECEIPT_TABLE, {
fields: ['project_id', 'created_at_ms', 'dispatch_id'],
name: APPROVED_RUN_ACTION_RECEIPT_PROJECT_INDEX,
transaction,
});
await queryInterface.addIndex(APPROVED_RUN_ACTION_RECEIPT_TABLE, {
fields: ['resource_type', 'resource_id'],
unique: true,
name: APPROVED_RUN_ACTION_RECEIPT_RESOURCE_UNIQUE_INDEX,
transaction,
});
},
};
@@ -0,0 +1,166 @@
import { createHash } from 'crypto';
import { DataTypes, Op } from 'sequelize';
import {
APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES,
MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS,
} from '../runtime/domain/approvedActionRecoveryAuthorization';
import { PROJECT_TABLE } from './0017-project-policy';
import { APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE } from './0022-approved-action-recovery';
import type { Migration } from './types';
export const APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE =
'ApprovedActionRecoveryAuthorizationFacts';
export const APPROVED_ACTION_RECOVERY_AUTHORIZATION_MUTATION_INDEX =
'approved_action_recovery_authorization_mutation_uidx';
export const APPROVED_ACTION_RECOVERY_AUTHORIZATION_PROJECT_INDEX =
'approved_action_recovery_authorization_project_idx';
export const APPROVED_ACTION_RECOVERY_AUTHORIZATION_AUTH_INDEX =
'approved_action_recovery_authorization_auth_idx';
const columns = [
'dispatch_id',
'project_id',
'mutation_id',
'resolved_by_id',
'authentication_id',
'assurance',
'authenticated_at_ms',
'project_version',
'binding_version',
'authorized_at_ms',
'fact_digest',
];
const manifest = {
table: APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
columns,
indexes: [
`${APPROVED_ACTION_RECOVERY_AUTHORIZATION_MUTATION_INDEX}(mutation_id) UNIQUE`,
`${APPROVED_ACTION_RECOVERY_AUTHORIZATION_PROJECT_INDEX}(project_id,authorized_at_ms,dispatch_id)`,
`${APPROVED_ACTION_RECOVERY_AUTHORIZATION_AUTH_INDEX}(authentication_id,authorized_at_ms,dispatch_id)`,
],
constraints: [
'approved_action_recovery_authorization_assurance_check',
'approved_action_recovery_authorization_project_version_check',
'approved_action_recovery_authorization_binding_version_check',
'approved_action_recovery_authorization_recency_check',
],
maxAuthenticationAgeMs: MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS,
};
export const approvedActionRecoveryAuthorizationManifest = manifest;
export const approvedActionRecoveryAuthorizationMigration: Migration = {
id: '0024-approved-action-recovery-authorization',
checksum: createHash('sha256').update(JSON.stringify(manifest)).digest('hex'),
async up({ queryInterface, transaction }) {
await queryInterface.createTable(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
{
dispatch_id: {
type: DataTypes.STRING(64),
allowNull: false,
primaryKey: true,
references: {
model: APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
key: 'dispatch_id',
},
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
project_id: {
type: DataTypes.STRING(128),
allowNull: false,
references: { model: PROJECT_TABLE, key: 'id' },
onDelete: 'RESTRICT',
onUpdate: 'CASCADE',
},
mutation_id: { type: DataTypes.STRING(64), allowNull: false },
resolved_by_id: { type: DataTypes.STRING(255), allowNull: false },
authentication_id: { type: DataTypes.STRING(128), allowNull: false },
assurance: { type: DataTypes.STRING(32), allowNull: false },
authenticated_at_ms: { type: DataTypes.BIGINT, allowNull: false },
project_version: { type: DataTypes.INTEGER, allowNull: false },
binding_version: { type: DataTypes.INTEGER, allowNull: false },
authorized_at_ms: { type: DataTypes.BIGINT, allowNull: false },
fact_digest: { type: DataTypes.STRING(64), allowNull: false },
},
{ transaction },
);
await queryInterface.addConstraint(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
{
fields: ['assurance'],
type: 'check',
where: {
assurance: { [Op.in]: APPROVED_ACTION_RECOVERY_STRONG_ASSURANCES },
},
name: 'approved_action_recovery_authorization_assurance_check',
transaction,
},
);
for (const [field, name] of [
[
'project_version',
'approved_action_recovery_authorization_project_version_check',
],
[
'binding_version',
'approved_action_recovery_authorization_binding_version_check',
],
] as const) {
await queryInterface.addConstraint(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
{
fields: [field],
type: 'check',
where: { [field]: { [Op.between]: [1, 2_147_483_647] } },
name,
transaction,
},
);
}
await queryInterface.addConstraint(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
{
fields: ['authenticated_at_ms', 'authorized_at_ms'],
type: 'check',
where: {
authorized_at_ms: {
[Op.gte]: { [Op.col]: 'authenticated_at_ms' },
[Op.lte]: queryInterface.sequelize.literal(
`authenticated_at_ms + ${MAX_APPROVED_ACTION_RECOVERY_AUTH_AGE_MS}`,
),
},
},
name: 'approved_action_recovery_authorization_recency_check',
transaction,
},
);
await queryInterface.addIndex(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
['mutation_id'],
{
name: APPROVED_ACTION_RECOVERY_AUTHORIZATION_MUTATION_INDEX,
unique: true,
transaction,
},
);
await queryInterface.addIndex(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
['project_id', 'authorized_at_ms', 'dispatch_id'],
{
name: APPROVED_ACTION_RECOVERY_AUTHORIZATION_PROJECT_INDEX,
transaction,
},
);
await queryInterface.addIndex(
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
['authentication_id', 'authorized_at_ms', 'dispatch_id'],
{
name: APPROVED_ACTION_RECOVERY_AUTHORIZATION_AUTH_INDEX,
transaction,
},
);
},
};
@@ -0,0 +1,262 @@
import {
MigrationStreamHistoryCorruptionError,
type MigrationStreamRecord,
type MigrationStreamStore,
type MigrationStreamTransaction,
} from '../core/migrationStream';
import type {
PostgresClient as PostgresMigrationClient,
PostgresPool as PostgresMigrationPool,
PostgresQueryable as PostgresMigrationQueryable,
PostgresQueryResult as PostgresMigrationQueryResult,
} from '@qinglong/runtime-core';
export type {
PostgresClient as PostgresMigrationClient,
PostgresPool as PostgresMigrationPool,
PostgresQueryable as PostgresMigrationQueryable,
PostgresQueryResult as PostgresMigrationQueryResult,
} from '@qinglong/runtime-core';
export const POSTGRESQL_MAIN_MIGRATION_STREAM_ID = 'postgresql-main';
export const POSTGRESQL_MIGRATION_SCHEMA = 'ql3';
export const POSTGRESQL_MIGRATION_HISTORY_TABLE = 'schema_migrations';
const POSTGRESQL_MIGRATION_LOCK_KEY = [0x514c, 0x0300] as const;
const POSTGRESQL_MIGRATION_STATEMENT_TIMEOUT_MS = 15_000;
const POSTGRESQL_MIGRATION_LOCK_TIMEOUT_MS = 5_000;
const POSTGRESQL_MIGRATION_IDLE_TRANSACTION_TIMEOUT_MS = 15_000;
export type PostgresMigrationContext = PostgresMigrationQueryable;
export class PostgresMigrationLeaderUnavailableError extends Error {
constructor() {
super('PostgreSQL migration leader lock is unavailable');
this.name = 'PostgresMigrationLeaderUnavailableError';
}
}
interface HistoryRow extends Record<string, unknown> {
streamId: unknown;
dialect: unknown;
migrationId: unknown;
checksum: unknown;
appliedAtMs: unknown;
}
interface AdvisoryLockRow extends Record<string, unknown> {
acquired: unknown;
}
const CREATE_HISTORY_SQL = `
CREATE TABLE IF NOT EXISTS "${POSTGRESQL_MIGRATION_SCHEMA}"."${POSTGRESQL_MIGRATION_HISTORY_TABLE}" (
migration_id varchar(128) PRIMARY KEY,
stream_id varchar(64) NOT NULL,
dialect varchar(16) NOT NULL
CONSTRAINT ql3_schema_migrations_dialect_check
CHECK (dialect = 'postgresql'),
checksum char(64) NOT NULL
CONSTRAINT ql3_schema_migrations_checksum_check
CHECK (checksum ~ '^[0-9a-f]{64}$'),
applied_at_ms bigint NOT NULL
CONSTRAINT ql3_schema_migrations_applied_at_check
CHECK (applied_at_ms >= 0)
)`.trim();
const SELECT_HISTORY_SQL = `
SELECT
stream_id AS "streamId",
dialect,
migration_id AS "migrationId",
checksum,
applied_at_ms AS "appliedAtMs"
FROM "${POSTGRESQL_MIGRATION_SCHEMA}"."${POSTGRESQL_MIGRATION_HISTORY_TABLE}"
WHERE migration_id = $1
`.trim();
const SELECT_ALL_HISTORY_SQL = `
SELECT
stream_id AS "streamId",
dialect,
migration_id AS "migrationId",
checksum,
applied_at_ms AS "appliedAtMs"
FROM "${POSTGRESQL_MIGRATION_SCHEMA}"."${POSTGRESQL_MIGRATION_HISTORY_TABLE}"
ORDER BY applied_at_ms, migration_id
`.trim();
const INSERT_HISTORY_SQL = `
INSERT INTO "${POSTGRESQL_MIGRATION_SCHEMA}"."${POSTGRESQL_MIGRATION_HISTORY_TABLE}" (
migration_id,
stream_id,
dialect,
checksum,
applied_at_ms
)
VALUES ($1, $2, $3, $4, $5)
`.trim();
function parseAppliedAtMs(value: unknown, migrationId: string): number {
if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) {
return value;
}
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
const parsed = Number(value);
if (Number.isSafeInteger(parsed)) return parsed;
}
throw new MigrationStreamHistoryCorruptionError(migrationId);
}
async function findHistoryRecord(
queryable: PostgresMigrationQueryable,
migrationId: string,
): Promise<MigrationStreamRecord | null> {
const result = await queryable.query<HistoryRow>(SELECT_HISTORY_SQL, [
migrationId,
]);
if (result.rows.length === 0) return null;
if (result.rows.length !== 1) {
throw new MigrationStreamHistoryCorruptionError(migrationId);
}
const row = result.rows[0];
if (!row) {
throw new MigrationStreamHistoryCorruptionError(migrationId);
}
return {
streamId: row.streamId as string,
dialect: row.dialect as 'postgresql',
migrationId: row.migrationId as string,
checksum: row.checksum as string,
appliedAtMs: parseAppliedAtMs(row.appliedAtMs, migrationId),
};
}
export async function readPostgresMigrationHistory(
queryable: PostgresMigrationQueryable,
): Promise<readonly MigrationStreamRecord[]> {
const result = await queryable.query<HistoryRow>(SELECT_ALL_HISTORY_SQL);
return result.rows.map((row) => ({
streamId: row.streamId as string,
dialect: row.dialect as 'postgresql',
migrationId: row.migrationId as string,
checksum: row.checksum as string,
appliedAtMs: parseAppliedAtMs(
row.appliedAtMs,
typeof row.migrationId === 'string' ? row.migrationId : 'unknown',
),
}));
}
async function configureTransaction(
client: PostgresMigrationClient,
): Promise<void> {
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
`${POSTGRESQL_MIGRATION_STATEMENT_TIMEOUT_MS}ms`,
]);
await client.query(`SELECT set_config('lock_timeout', $1, true)`, [
`${POSTGRESQL_MIGRATION_LOCK_TIMEOUT_MS}ms`,
]);
await client.query(
`SELECT set_config('idle_in_transaction_session_timeout', $1, true)`,
[`${POSTGRESQL_MIGRATION_IDLE_TRANSACTION_TIMEOUT_MS}ms`],
);
}
async function acquireLeaderLock(
client: PostgresMigrationClient,
): Promise<void> {
const result = await client.query<AdvisoryLockRow>(
'SELECT pg_try_advisory_xact_lock($1, $2) AS acquired',
POSTGRESQL_MIGRATION_LOCK_KEY,
);
if (result.rows.length !== 1 || result.rows[0]?.acquired !== true) {
throw new PostgresMigrationLeaderUnavailableError();
}
}
/**
* PostgreSQL migration history adapter without a runtime dependency on `pg`.
* The cluster-only package owns the concrete Pool binding; edge and standalone
* builds can compile this contract without installing or importing the driver.
*/
export class PostgresMigrationStreamStore
implements MigrationStreamStore<PostgresMigrationContext>
{
constructor(private readonly pool: PostgresMigrationPool) {}
async ensureHistory(): Promise<void> {
await this.withLeaderTransaction(async (client) => {
await client.query(
`CREATE SCHEMA IF NOT EXISTS "${POSTGRESQL_MIGRATION_SCHEMA}"`,
);
await client.query(CREATE_HISTORY_SQL);
});
}
async findById(migrationId: string): Promise<MigrationStreamRecord | null> {
return findHistoryRecord(this.pool, migrationId);
}
async listAll(): Promise<readonly MigrationStreamRecord[]> {
return readPostgresMigrationHistory(this.pool);
}
async transaction<T>(
work: (
transaction: MigrationStreamTransaction<PostgresMigrationContext>,
) => Promise<T>,
): Promise<T> {
return this.withLeaderTransaction(async (client) =>
work({
context: client,
findById: (migrationId) => findHistoryRecord(client, migrationId),
insert: async (record) => {
if (
record.streamId !== POSTGRESQL_MAIN_MIGRATION_STREAM_ID ||
record.dialect !== 'postgresql'
) {
throw new TypeError(
'PostgreSQL migration history record has the wrong stream or dialect',
);
}
await client.query(INSERT_HISTORY_SQL, [
record.migrationId,
record.streamId,
record.dialect,
record.checksum,
record.appliedAtMs,
]);
},
}),
);
}
private async withLeaderTransaction<T>(
work: (client: PostgresMigrationClient) => Promise<T>,
): Promise<T> {
const client = await this.pool.connect();
let began = false;
try {
await client.query('BEGIN');
began = true;
await configureTransaction(client);
await acquireLeaderLock(client);
const result = await work(client);
await client.query('COMMIT');
began = false;
return result;
} catch (error) {
if (began) {
try {
await client.query('ROLLBACK');
} catch {
// Preserve the migration error. A broken connection is discarded by
// the concrete driver when release() runs.
}
}
throw error;
} finally {
client.release();
}
}
}
@@ -0,0 +1,97 @@
import type { ModelStatic, Sequelize, Transaction } from 'sequelize';
import type {
SchemaMigrationInstance,
SchemaMigrationAttributes,
} from '../../data/schemaMigration';
import type {
MigrationStreamRecord,
MigrationStreamStore,
MigrationStreamTransaction,
} from '../core/migrationStream';
import type { MigrationContext } from '../types';
export const SQLITE_MAIN_MIGRATION_STREAM_ID = 'sqlite-main';
function toRecord(instance: SchemaMigrationInstance): MigrationStreamRecord {
return {
streamId: SQLITE_MAIN_MIGRATION_STREAM_ID,
dialect: 'sqlite',
migrationId: instance.id,
checksum: instance.checksum,
appliedAtMs: Number(instance.applied_at),
};
}
/**
* Compatibility adapter for the existing Sequelize-owned SQLite connection.
* It preserves the legacy SchemaMigrations row shape while the generic core
* owns ordering, replay and checksum semantics.
*/
export class SequelizeSqliteMigrationStreamStore
implements MigrationStreamStore<MigrationContext>
{
constructor(
private readonly database: Sequelize,
private readonly migrationModel: ModelStatic<SchemaMigrationInstance>,
) {
if (database.getDialect() !== 'sqlite') {
throw new TypeError(
'SequelizeSqliteMigrationStreamStore requires SQLite',
);
}
}
async ensureHistory(): Promise<void> {
await this.migrationModel.sync();
}
async listAll(): Promise<readonly MigrationStreamRecord[]> {
return (await this.migrationModel.findAll()).map(toRecord);
}
async findById(migrationId: string): Promise<MigrationStreamRecord | null> {
const applied = await this.migrationModel.findByPk(migrationId);
return applied ? toRecord(applied) : null;
}
async transaction<T>(
work: (
transaction: MigrationStreamTransaction<MigrationContext>,
) => Promise<T>,
): Promise<T> {
return this.database.transaction(async (transaction) =>
work(this.createTransaction(transaction)),
);
}
private createTransaction(
transaction: Transaction,
): MigrationStreamTransaction<MigrationContext> {
return {
context: {
queryInterface: this.database.getQueryInterface(),
transaction,
},
findById: async (migrationId) => {
const applied = await this.migrationModel.findByPk(migrationId, {
transaction,
});
return applied ? toRecord(applied) : null;
},
insert: async (record) => {
if (
record.streamId !== SQLITE_MAIN_MIGRATION_STREAM_ID ||
record.dialect !== 'sqlite'
) {
throw new TypeError('SQLite migration record identity is invalid');
}
const attributes: SchemaMigrationAttributes = {
id: record.migrationId,
checksum: record.checksum,
applied_at: record.appliedAtMs,
};
await this.migrationModel.create(attributes, { transaction });
},
};
}
}
+361
View File
@@ -0,0 +1,361 @@
export const MIGRATION_STREAM_DIALECTS = ['sqlite', 'postgresql'] as const;
export type MigrationStreamDialect = (typeof MIGRATION_STREAM_DIALECTS)[number];
export const MIGRATION_ID_SCHEMES = [
'sqlite-numbered',
'postgres-prefixed',
] as const;
export type MigrationIdScheme = (typeof MIGRATION_ID_SCHEMES)[number];
export const MIGRATION_CHECKSUM_SCHEMES = ['sha256', 'legacy-opaque'] as const;
export type MigrationChecksumScheme =
(typeof MIGRATION_CHECKSUM_SCHEMES)[number];
export interface MigrationStreamRecord {
streamId: string;
dialect: MigrationStreamDialect;
migrationId: string;
checksum: string;
appliedAtMs: number;
}
export interface MigrationStreamStep<TContext> {
id: string;
checksum: string;
up(context: TContext): Promise<void>;
}
export interface MigrationStreamTransaction<TContext> {
readonly context: TContext;
findById(migrationId: string): Promise<MigrationStreamRecord | null>;
insert(record: MigrationStreamRecord): Promise<void>;
}
export interface MigrationStreamStore<TContext> {
ensureHistory(): Promise<void>;
listAll(): Promise<readonly MigrationStreamRecord[]>;
findById(migrationId: string): Promise<MigrationStreamRecord | null>;
transaction<T>(
work: (transaction: MigrationStreamTransaction<TContext>) => Promise<T>,
): Promise<T>;
}
export interface MigrationStreamDefinition<TContext> {
id: string;
dialect: MigrationStreamDialect;
migrationIdScheme: MigrationIdScheme;
checksumScheme: MigrationChecksumScheme;
migrations: readonly MigrationStreamStep<TContext>[];
}
export interface RunMigrationStreamOptions<TContext> {
stream: MigrationStreamDefinition<TContext>;
store: MigrationStreamStore<TContext>;
clock?: () => number;
logger?: { info(message: string): unknown };
}
const STREAM_ID_PATTERN = /^[a-z][a-z0-9-]{0,63}$/;
const SQLITE_MIGRATION_ID_PATTERN = /^\d{4}-[a-z0-9][a-z0-9-]{0,122}$/;
const POSTGRES_MIGRATION_ID_PATTERN = /^pg-\d{4}-[a-z0-9][a-z0-9-]{0,119}$/;
const CHECKSUM_PATTERN = /^[0-9a-f]{64}$/;
const MAX_LEGACY_CHECKSUM_LENGTH = 255;
export class InvalidMigrationStreamError extends TypeError {
constructor(message: string) {
super(`Migration stream is invalid: ${message}`);
this.name = 'InvalidMigrationStreamError';
}
}
export class MigrationStreamChecksumMismatchError extends Error {
constructor(
readonly migrationId: string,
readonly databaseChecksum: string,
readonly codeChecksum: string,
) {
super(
`Migration checksum mismatch: ${migrationId} ` +
`(database=${databaseChecksum}, code=${codeChecksum})`,
);
this.name = 'MigrationStreamChecksumMismatchError';
}
}
export class MigrationStreamHistoryCorruptionError extends Error {
constructor(readonly migrationId: string) {
super(`Migration history is corrupt: ${migrationId}`);
this.name = 'MigrationStreamHistoryCorruptionError';
}
}
export class MigrationStreamAheadOfCodeError extends Error {
constructor(readonly migrationId: string) {
super(`Migration history is ahead of this code: ${migrationId}`);
this.name = 'MigrationStreamAheadOfCodeError';
}
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const keys = Object.keys(value).sort();
const canonical = [...expected].sort();
return (
keys.length === canonical.length &&
keys.every((key, index) => key === canonical[index])
);
}
function checksumIsValid(
scheme: MigrationChecksumScheme,
value: unknown,
): value is string {
if (typeof value !== 'string') return false;
return scheme === 'sha256'
? CHECKSUM_PATTERN.test(value)
: value.length <= MAX_LEGACY_CHECKSUM_LENGTH;
}
function validateDefinition<TContext>(
stream: MigrationStreamDefinition<TContext>,
): void {
if (!stream || typeof stream !== 'object' || Array.isArray(stream)) {
throw new InvalidMigrationStreamError('definition must be an object');
}
if (
!exactKeys(stream, [
'id',
'dialect',
'migrationIdScheme',
'checksumScheme',
'migrations',
])
) {
throw new InvalidMigrationStreamError('definition shape is invalid');
}
if (!STREAM_ID_PATTERN.test(stream.id)) {
throw new InvalidMigrationStreamError('id is invalid');
}
if (!MIGRATION_STREAM_DIALECTS.includes(stream.dialect)) {
throw new InvalidMigrationStreamError('dialect is invalid');
}
if (!MIGRATION_ID_SCHEMES.includes(stream.migrationIdScheme)) {
throw new InvalidMigrationStreamError('migrationIdScheme is invalid');
}
if (!MIGRATION_CHECKSUM_SCHEMES.includes(stream.checksumScheme)) {
throw new InvalidMigrationStreamError('checksumScheme is invalid');
}
if (
(stream.dialect === 'sqlite' &&
stream.migrationIdScheme !== 'sqlite-numbered') ||
(stream.dialect === 'postgresql' &&
stream.migrationIdScheme !== 'postgres-prefixed')
) {
throw new InvalidMigrationStreamError(
'migrationIdScheme does not match dialect',
);
}
if (!Array.isArray(stream.migrations)) {
throw new InvalidMigrationStreamError('migrations must be an array');
}
const ids = new Set<string>();
for (const migration of stream.migrations) {
if (
!migration ||
typeof migration !== 'object' ||
Array.isArray(migration) ||
!exactKeys(migration, ['id', 'checksum', 'up']) ||
!(
stream.migrationIdScheme === 'sqlite-numbered'
? SQLITE_MIGRATION_ID_PATTERN
: POSTGRES_MIGRATION_ID_PATTERN
).test(migration.id) ||
!checksumIsValid(stream.checksumScheme, migration.checksum) ||
typeof migration.up !== 'function'
) {
throw new InvalidMigrationStreamError('migration is invalid');
}
if (ids.has(migration.id)) {
throw new InvalidMigrationStreamError(
`duplicate migration id: ${migration.id}`,
);
}
ids.add(migration.id);
}
}
function validateRecord(
value: MigrationStreamRecord,
stream: MigrationStreamDefinition<unknown>,
migrationId: string,
): void {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'streamId',
'dialect',
'migrationId',
'checksum',
'appliedAtMs',
]) ||
value.streamId !== stream.id ||
value.dialect !== stream.dialect ||
value.migrationId !== migrationId ||
!checksumIsValid(stream.checksumScheme, value.checksum) ||
!Number.isSafeInteger(value.appliedAtMs) ||
value.appliedAtMs < 0
) {
throw new MigrationStreamHistoryCorruptionError(migrationId);
}
}
function assertChecksum(
record: MigrationStreamRecord,
migration: MigrationStreamStep<unknown>,
): void {
if (record.checksum !== migration.checksum) {
throw new MigrationStreamChecksumMismatchError(
migration.id,
record.checksum,
migration.checksum,
);
}
}
export function auditMigrationStreamHistory<TContext>(
history: readonly MigrationStreamRecord[],
stream: MigrationStreamDefinition<TContext>,
): ReadonlySet<string> {
if (!Array.isArray(history)) {
throw new MigrationStreamHistoryCorruptionError('history');
}
const migrationsById = new Map(
stream.migrations.map((migration, index) => [
migration.id,
{ migration, index },
]),
);
const appliedIds = new Set<string>();
const appliedIndexes = new Set<number>();
for (const record of history) {
const expected = migrationsById.get(record?.migrationId);
if (!expected) {
throw new MigrationStreamAheadOfCodeError(
record?.migrationId ?? 'unknown',
);
}
if (appliedIds.has(record.migrationId)) {
throw new MigrationStreamHistoryCorruptionError(record.migrationId);
}
validateRecord(
record,
stream as MigrationStreamDefinition<unknown>,
expected.migration.id,
);
assertChecksum(record, expected.migration as MigrationStreamStep<unknown>);
appliedIds.add(record.migrationId);
appliedIndexes.add(expected.index);
}
for (let index = 0; index < appliedIndexes.size; index += 1) {
if (!appliedIndexes.has(index)) {
throw new MigrationStreamHistoryCorruptionError(
stream.migrations[index].id,
);
}
}
return appliedIds;
}
/**
* Dialect-neutral migration ordering and history semantics. Concrete stores own
* SQL, leader election and transaction APIs; this core never imports a driver.
*/
export async function runMigrationStream<TContext>(
options: RunMigrationStreamOptions<TContext>,
): Promise<void> {
if (!options || typeof options !== 'object' || Array.isArray(options)) {
throw new InvalidMigrationStreamError('options must be an object');
}
const expectedKeys = [
'stream',
'store',
...(options.clock === undefined ? [] : ['clock']),
...(options.logger === undefined ? [] : ['logger']),
];
if (!exactKeys(options, expectedKeys)) {
throw new InvalidMigrationStreamError('options shape is invalid');
}
validateDefinition(options.stream);
if (
!options.store ||
typeof options.store !== 'object' ||
typeof options.store.ensureHistory !== 'function' ||
typeof options.store.listAll !== 'function' ||
typeof options.store.findById !== 'function' ||
typeof options.store.transaction !== 'function'
) {
throw new InvalidMigrationStreamError('store is invalid');
}
const clock = options.clock ?? Date.now;
if (typeof clock !== 'function') {
throw new InvalidMigrationStreamError('clock is invalid');
}
await options.store.ensureHistory();
const appliedAtStart = auditMigrationStreamHistory(
await options.store.listAll(),
options.stream,
);
for (const migration of options.stream.migrations) {
if (appliedAtStart.has(migration.id)) continue;
const applied = await options.store.findById(migration.id);
if (applied) {
validateRecord(
applied,
options.stream as MigrationStreamDefinition<unknown>,
migration.id,
);
assertChecksum(applied, migration as MigrationStreamStep<unknown>);
continue;
}
let appliedNow = false;
await options.store.transaction(async (transaction) => {
const appliedInsideTransaction = await transaction.findById(migration.id);
if (appliedInsideTransaction) {
validateRecord(
appliedInsideTransaction,
options.stream as MigrationStreamDefinition<unknown>,
migration.id,
);
assertChecksum(
appliedInsideTransaction,
migration as MigrationStreamStep<unknown>,
);
return;
}
await migration.up(transaction.context);
const appliedAtMs = clock();
if (!Number.isSafeInteger(appliedAtMs) || appliedAtMs < 0) {
throw new InvalidMigrationStreamError(
'clock must return a non-negative safe integer',
);
}
await transaction.insert({
streamId: options.stream.id,
dialect: options.stream.dialect,
migrationId: migration.id,
checksum: migration.checksum,
appliedAtMs,
});
appliedNow = true;
});
if (appliedNow) {
options.logger?.info(
`[migration:${options.stream.id}] Applied ${migration.id}`,
);
}
}
}
+52
View File
@@ -0,0 +1,52 @@
import { runSchemaMigration } from './0002-run-schema';
import { runningInstanceRunReferenceMigration } from './0003-running-instance-run-reference';
import { runCancellationRequestMigration } from './0004-run-cancellation-request';
import { runCancellationDispatchMigration } from './0005-run-cancellation-dispatch';
import { runAttemptDeadlineMigration } from './0006-run-attempt-deadline';
import { completionReceiptJournalMigration } from './0007-completion-receipt-journal';
import { workerRegistryMigration } from './0008-worker-registry';
import { runDispatchLeaseMigration } from './0009-run-dispatch-lease';
import { runDispatchCandidateMigration } from './0010-run-dispatch-candidates';
import { runRetryPolicyMigration } from './0011-run-retry-policy';
import { taskExecutionRevisionMigration } from './0012-task-execution-revisions';
import { localExecutionContextRecipeMigration } from './0013-local-execution-context-recipes';
import { localSecretEnvelopeMigration } from './0014-local-secret-envelopes';
import { localArtifactRetentionMigration } from './0015-local-artifact-retention';
import { localArtifactMaintenanceCursorMigration } from './0016-local-artifact-maintenance-cursor';
import { projectPolicyMigration } from './0017-project-policy';
import { projectOwnerBootstrapMigration } from './0018-project-owner-bootstrap';
import { identityDirectoryMigration } from './0019-identity-directory';
import { approvalRequestMigration } from './0020-approval-requests';
import { approvedActionDispatchExecutionMigration } from './0021-approved-action-dispatch-executions';
import { approvedActionRecoveryMigration } from './0022-approved-action-recovery';
import { approvedRunActionReceiptMigration } from './0023-approved-run-action-receipts';
import { approvedActionRecoveryAuthorizationMigration } from './0024-approved-action-recovery-authorization';
import { legacyColumnsMigration } from './0001-legacy-columns';
import type { Migration } from './types';
export const migrations: Migration[] = [
legacyColumnsMigration,
runSchemaMigration,
runningInstanceRunReferenceMigration,
runCancellationRequestMigration,
runCancellationDispatchMigration,
runAttemptDeadlineMigration,
completionReceiptJournalMigration,
workerRegistryMigration,
runDispatchLeaseMigration,
runDispatchCandidateMigration,
runRetryPolicyMigration,
taskExecutionRevisionMigration,
localExecutionContextRecipeMigration,
localSecretEnvelopeMigration,
localArtifactRetentionMigration,
localArtifactMaintenanceCursorMigration,
projectPolicyMigration,
projectOwnerBootstrapMigration,
identityDirectoryMigration,
approvalRequestMigration,
approvedActionDispatchExecutionMigration,
approvedActionRecoveryMigration,
approvedRunActionReceiptMigration,
approvedActionRecoveryAuthorizationMigration,
];
+21
View File
@@ -0,0 +1,21 @@
import {
POSTGRESQL_MAIN_MIGRATION_STREAM_ID,
type PostgresMigrationContext,
} from '../adapters/postgresMigrationStreamStore';
import type { MigrationStreamDefinition } from '../core/migrationStream';
import { pg0001SchemaCapabilityMigration } from './pg-0001-schema-capability';
import { pg0002RunCoreMigration } from './pg-0002-run-core';
import { pg0003RunRetryPolicyMigration } from './pg-0003-run-retry-policy';
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
Object.freeze({
id: POSTGRESQL_MAIN_MIGRATION_STREAM_ID,
dialect: 'postgresql',
migrationIdScheme: 'postgres-prefixed',
checksumScheme: 'sha256',
migrations: Object.freeze([
pg0001SchemaCapabilityMigration,
pg0002RunCoreMigration,
pg0003RunRetryPolicyMigration,
]),
});
@@ -0,0 +1,28 @@
import { definePostgresSqlMigration } from './sqlMigration';
export const POSTGRESQL_SCHEMA_CAPABILITY_TABLE = 'schema_capabilities';
export const pg0001SchemaCapabilityMigration = definePostgresSqlMigration({
id: 'pg-0001-schema-capability',
statements: [
`
CREATE TABLE "ql3"."${POSTGRESQL_SCHEMA_CAPABILITY_TABLE}" (
contract_name varchar(64) PRIMARY KEY,
contract_version integer NOT NULL
CONSTRAINT ql3_schema_capabilities_version_check
CHECK (contract_version >= 0),
migration_id varchar(128) NOT NULL,
capabilities jsonb NOT NULL
CONSTRAINT ql3_schema_capabilities_payload_check
CHECK (jsonb_typeof(capabilities) = 'object'),
updated_at_ms bigint NOT NULL
CONSTRAINT ql3_schema_capabilities_updated_at_check
CHECK (updated_at_ms >= 0),
CONSTRAINT ql3_schema_capabilities_migration_fk
FOREIGN KEY (migration_id)
REFERENCES "ql3"."schema_migrations" (migration_id)
DEFERRABLE INITIALLY DEFERRED
)
`.trim(),
],
});
@@ -0,0 +1,224 @@
import { definePostgresSqlMigration } from './sqlMigration';
export const POSTGRESQL_RUN_TABLE = 'runs';
export const POSTGRESQL_RUN_ATTEMPT_TABLE = 'run_attempts';
export const POSTGRESQL_RUN_EVENT_TABLE = 'run_events';
export const pg0002RunCoreMigration = definePostgresSqlMigration({
id: 'pg-0002-run-core',
statements: [
`
CREATE TABLE "ql3"."${POSTGRESQL_RUN_TABLE}" (
id varchar(36) PRIMARY KEY,
project_id varchar(128) NOT NULL,
task_id varchar(255) NOT NULL,
task_revision varchar(128) NOT NULL,
task_name varchar(255),
task_snapshot_ref varchar(512),
parent_run_id varchar(36),
retry_of_run_id varchar(36),
trigger_id varchar(36),
trigger_type varchar(64) NOT NULL,
execution_origin varchar(64) NOT NULL,
execution_owner varchar(16) NOT NULL
CONSTRAINT ql3_runs_execution_owner_check
CHECK (execution_owner = 'runtime'),
triggered_by varchar(255),
request_id varchar(128),
scheduled_for_ms bigint
CONSTRAINT ql3_runs_scheduled_for_check
CHECK (scheduled_for_ms IS NULL OR scheduled_for_ms >= 0),
status varchar(32) NOT NULL
CONSTRAINT ql3_runs_status_check
CHECK (status IN (
'created', 'queued', 'dispatching', 'running', 'waiting_approval',
'retry_wait', 'lost', 'succeeded', 'failed', 'cancelled', 'timed_out'
)),
version integer NOT NULL DEFAULT 0
CONSTRAINT ql3_runs_version_check
CHECK (version >= 0),
event_sequence integer NOT NULL DEFAULT 0
CONSTRAINT ql3_runs_event_sequence_check
CHECK (event_sequence >= 0),
priority integer NOT NULL DEFAULT 0,
idempotency_key varchar(255),
input_ref varchar(512),
output_ref varchar(512),
created_at_ms bigint NOT NULL
CONSTRAINT ql3_runs_created_at_check
CHECK (created_at_ms >= 0),
queued_at_ms bigint
CONSTRAINT ql3_runs_queued_at_check
CHECK (queued_at_ms IS NULL OR queued_at_ms >= 0),
started_at_ms bigint
CONSTRAINT ql3_runs_started_at_check
CHECK (started_at_ms IS NULL OR started_at_ms >= 0),
finished_at_ms bigint
CONSTRAINT ql3_runs_finished_at_check
CHECK (finished_at_ms IS NULL OR finished_at_ms >= 0),
cancel_requested_at_ms bigint
CONSTRAINT ql3_runs_cancel_requested_at_check
CHECK (cancel_requested_at_ms IS NULL OR cancel_requested_at_ms >= 0),
cancel_reason varchar(16)
CONSTRAINT ql3_runs_cancel_reason_check
CHECK (cancel_reason IS NULL OR cancel_reason IN (
'user', 'policy', 'shutdown', 'reconcile', 'timeout'
)),
error_code varchar(128),
error_summary varchar(1024),
CONSTRAINT ql3_runs_parent_fk
FOREIGN KEY (parent_run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id),
CONSTRAINT ql3_runs_retry_of_fk
FOREIGN KEY (retry_of_run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id)
)
`.trim(),
`
CREATE UNIQUE INDEX ql3_runs_project_idempotency_uidx
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (project_id, idempotency_key)
WHERE idempotency_key IS NOT NULL
`.trim(),
`
CREATE INDEX ql3_runs_project_created_idx
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (project_id, created_at_ms, id)
`.trim(),
`
CREATE INDEX ql3_runs_task_created_idx
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (task_id, created_at_ms, id)
`.trim(),
`
CREATE INDEX ql3_runs_dispatch_candidates_idx
ON "ql3"."${POSTGRESQL_RUN_TABLE}" (priority DESC, queued_at_ms, id)
WHERE execution_owner = 'runtime'
AND status IN ('queued', 'dispatching')
AND cancel_requested_at_ms IS NULL
AND queued_at_ms IS NOT NULL
`.trim(),
`
CREATE TABLE "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (
id varchar(36) PRIMARY KEY,
run_id varchar(36) NOT NULL,
step_run_id varchar(36),
attempt integer NOT NULL
CONSTRAINT ql3_run_attempts_attempt_check
CHECK (attempt >= 1),
status varchar(32) NOT NULL
CONSTRAINT ql3_run_attempts_status_check
CHECK (status IN (
'claimed', 'starting', 'running', 'succeeded', 'failed',
'cancelled', 'timed_out', 'lost'
)),
executor_type varchar(64) NOT NULL,
worker_id varchar(128),
executor_handle varchar(2048),
pid integer
CONSTRAINT ql3_run_attempts_pid_check
CHECK (pid IS NULL OR pid >= 1),
log_artifact_id varchar(36),
lease_token varchar(128),
lease_expires_at_ms bigint
CONSTRAINT ql3_run_attempts_lease_expiry_check
CHECK (lease_expires_at_ms IS NULL OR lease_expires_at_ms >= 0),
deadline_at_ms bigint
CONSTRAINT ql3_run_attempts_deadline_check
CHECK (deadline_at_ms IS NULL OR deadline_at_ms >= 0),
callback_token_hash varchar(128),
callback_sequence integer NOT NULL DEFAULT 0
CONSTRAINT ql3_run_attempts_callback_sequence_check
CHECK (callback_sequence >= 0),
created_at_ms bigint NOT NULL
CONSTRAINT ql3_run_attempts_created_at_check
CHECK (created_at_ms >= 0),
started_at_ms bigint
CONSTRAINT ql3_run_attempts_started_at_check
CHECK (started_at_ms IS NULL OR started_at_ms >= 0),
finished_at_ms bigint
CONSTRAINT ql3_run_attempts_finished_at_check
CHECK (finished_at_ms IS NULL OR finished_at_ms >= 0),
exit_code integer,
error_code varchar(128),
error_summary varchar(1024),
CONSTRAINT ql3_run_attempts_run_fk
FOREIGN KEY (run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id)
)
`.trim(),
`
CREATE UNIQUE INDEX ql3_run_attempts_run_attempt_uidx
ON "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (run_id, attempt)
`.trim(),
`
CREATE INDEX ql3_run_attempts_dispatch_candidates_idx
ON "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (status, run_id, created_at_ms, id)
`.trim(),
`
CREATE INDEX ql3_run_attempts_lease_idx
ON "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (lease_expires_at_ms, id)
WHERE lease_expires_at_ms IS NOT NULL
`.trim(),
`
CREATE TABLE "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (
id varchar(36) PRIMARY KEY,
run_id varchar(36) NOT NULL,
sequence integer NOT NULL
CONSTRAINT ql3_run_events_sequence_check
CHECK (sequence >= 1),
type varchar(128) NOT NULL,
dedupe_key varchar(255),
actor_type varchar(64) NOT NULL
CONSTRAINT ql3_run_events_actor_type_check
CHECK (actor_type IN (
'user', 'api_app', 'trigger', 'agent', 'mcp_client', 'worker',
'executor', 'system', 'legacy_shell', 'scheduler', 'reconciler',
'compatibility'
)),
actor_id varchar(255),
attempt_id varchar(36),
step_run_id varchar(36),
payload jsonb NOT NULL
CONSTRAINT ql3_run_events_payload_check
CHECK (jsonb_typeof(payload) = 'object'),
created_at_ms bigint NOT NULL
CONSTRAINT ql3_run_events_created_at_check
CHECK (created_at_ms >= 0),
CONSTRAINT ql3_run_events_run_fk
FOREIGN KEY (run_id) REFERENCES "ql3"."${POSTGRESQL_RUN_TABLE}" (id),
CONSTRAINT ql3_run_events_attempt_fk
FOREIGN KEY (attempt_id) REFERENCES "ql3"."${POSTGRESQL_RUN_ATTEMPT_TABLE}" (id)
)
`.trim(),
`
CREATE UNIQUE INDEX ql3_run_events_run_sequence_uidx
ON "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (run_id, sequence)
`.trim(),
`
CREATE UNIQUE INDEX ql3_run_events_run_dedupe_uidx
ON "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (run_id, dedupe_key)
WHERE dedupe_key IS NOT NULL
`.trim(),
`
CREATE INDEX ql3_run_events_run_created_idx
ON "ql3"."${POSTGRESQL_RUN_EVENT_TABLE}" (run_id, created_at_ms, id)
`.trim(),
`
INSERT INTO "ql3"."schema_capabilities" (
contract_name,
contract_version,
migration_id,
capabilities,
updated_at_ms
)
VALUES (
'control-core',
1,
'pg-0002-run-core',
'{"run_core":1}'::jsonb,
floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
)
ON CONFLICT (contract_name) DO UPDATE
SET contract_version = EXCLUDED.contract_version,
migration_id = EXCLUDED.migration_id,
capabilities = EXCLUDED.capabilities,
updated_at_ms = EXCLUDED.updated_at_ms
WHERE "schema_capabilities".contract_version < EXCLUDED.contract_version
`.trim(),
],
});
@@ -0,0 +1,78 @@
import { definePostgresSqlMigration } from './sqlMigration';
export const POSTGRESQL_RUN_RETRY_POLICY_TABLE = 'run_retry_policies';
export const pg0003RunRetryPolicyMigration = definePostgresSqlMigration({
id: 'pg-0003-run-retry-policy',
statements: [
`
ALTER TABLE "ql3"."runs"
ADD COLUMN legacy_cron_id integer
CONSTRAINT ql3_runs_legacy_cron_id_check
CHECK (legacy_cron_id IS NULL OR legacy_cron_id >= 1)
`.trim(),
`
CREATE TABLE "ql3"."${POSTGRESQL_RUN_RETRY_POLICY_TABLE}" (
run_id varchar(36) PRIMARY KEY,
max_attempts integer NOT NULL
CONSTRAINT ql3_run_retry_policies_max_attempts_check
CHECK (max_attempts BETWEEN 1 AND 16),
retry_on_lost boolean NOT NULL,
safety varchar(16) NOT NULL
CONSTRAINT ql3_run_retry_policies_safety_check
CHECK (safety IN ('unknown', 'idempotent', 'deduplicated')),
backoff_base_ms bigint NOT NULL
CONSTRAINT ql3_run_retry_policies_backoff_base_check
CHECK (backoff_base_ms BETWEEN 0 AND 86400000),
backoff_max_ms bigint NOT NULL
CONSTRAINT ql3_run_retry_policies_backoff_max_check
CHECK (backoff_max_ms BETWEEN backoff_base_ms AND 86400000),
next_attempt_at_ms bigint
CONSTRAINT ql3_run_retry_policies_next_attempt_check
CHECK (next_attempt_at_ms IS NULL OR next_attempt_at_ms >= 0),
version integer NOT NULL DEFAULT 0
CONSTRAINT ql3_run_retry_policies_version_check
CHECK (version >= 0),
created_at_ms bigint NOT NULL
CONSTRAINT ql3_run_retry_policies_created_at_check
CHECK (created_at_ms >= 0),
updated_at_ms bigint NOT NULL
CONSTRAINT ql3_run_retry_policies_updated_at_check
CHECK (updated_at_ms >= created_at_ms),
CONSTRAINT ql3_run_retry_policies_run_fk
FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE CASCADE
)
`.trim(),
`
CREATE INDEX ql3_run_retry_policies_due_idx
ON "ql3"."${POSTGRESQL_RUN_RETRY_POLICY_TABLE}" (next_attempt_at_ms, run_id)
WHERE next_attempt_at_ms IS NOT NULL
`.trim(),
`
CREATE INDEX ql3_runs_lost_retry_idx
ON "ql3"."runs" (execution_owner, status, id)
`.trim(),
`
DO $ql3$
BEGIN
UPDATE "ql3"."schema_capabilities"
SET contract_version = 2,
migration_id = 'pg-0003-run-retry-policy',
capabilities = '{"run_core":1,"run_retry_policy":1}'::jsonb,
updated_at_ms = floor(
extract(epoch FROM transaction_timestamp()) * 1000
)::bigint
WHERE contract_name = 'control-core'
AND contract_version = 1
AND migration_id = 'pg-0002-run-core'
AND capabilities = '{"run_core":1}'::jsonb;
IF NOT FOUND THEN
RAISE EXCEPTION 'control-core capability is not at version 1'
USING ERRCODE = 'check_violation';
END IF;
END
$ql3$
`.trim(),
],
});
@@ -0,0 +1,206 @@
export interface PostgresSchemaContractTable {
readonly name: string;
readonly columns: readonly string[];
}
export interface PostgresSchemaContract {
readonly schema: 'ql3';
readonly contractName: 'control-core';
readonly contractVersion: 2;
readonly migrationId: 'pg-0003-run-retry-policy';
readonly minimumServerMajor: 16;
readonly maximumServerMajor: 18;
readonly capabilities: Readonly<{
run_core: 1;
run_retry_policy: 1;
}>;
readonly tables: readonly PostgresSchemaContractTable[];
readonly indexes: readonly string[];
readonly checks: readonly string[];
readonly foreignKeys: readonly string[];
}
function table(
name: string,
columns: readonly string[],
): PostgresSchemaContractTable {
return Object.freeze({ name, columns: Object.freeze([...columns]) });
}
export const postgresqlControlSchemaContract: PostgresSchemaContract =
Object.freeze({
schema: 'ql3',
contractName: 'control-core',
contractVersion: 2,
migrationId: 'pg-0003-run-retry-policy',
minimumServerMajor: 16,
maximumServerMajor: 18,
capabilities: Object.freeze({ run_core: 1, run_retry_policy: 1 }),
tables: Object.freeze([
table('schema_migrations', [
'migration_id',
'stream_id',
'dialect',
'checksum',
'applied_at_ms',
]),
table('schema_capabilities', [
'contract_name',
'contract_version',
'migration_id',
'capabilities',
'updated_at_ms',
]),
table('runs', [
'id',
'project_id',
'task_id',
'task_revision',
'task_name',
'task_snapshot_ref',
'legacy_cron_id',
'parent_run_id',
'retry_of_run_id',
'trigger_id',
'trigger_type',
'execution_origin',
'execution_owner',
'triggered_by',
'request_id',
'scheduled_for_ms',
'status',
'version',
'event_sequence',
'priority',
'idempotency_key',
'input_ref',
'output_ref',
'created_at_ms',
'queued_at_ms',
'started_at_ms',
'finished_at_ms',
'cancel_requested_at_ms',
'cancel_reason',
'error_code',
'error_summary',
]),
table('run_attempts', [
'id',
'run_id',
'step_run_id',
'attempt',
'status',
'executor_type',
'worker_id',
'executor_handle',
'pid',
'log_artifact_id',
'lease_token',
'lease_expires_at_ms',
'deadline_at_ms',
'callback_token_hash',
'callback_sequence',
'created_at_ms',
'started_at_ms',
'finished_at_ms',
'exit_code',
'error_code',
'error_summary',
]),
table('run_events', [
'id',
'run_id',
'sequence',
'type',
'dedupe_key',
'actor_type',
'actor_id',
'attempt_id',
'step_run_id',
'payload',
'created_at_ms',
]),
table('run_retry_policies', [
'run_id',
'max_attempts',
'retry_on_lost',
'safety',
'backoff_base_ms',
'backoff_max_ms',
'next_attempt_at_ms',
'version',
'created_at_ms',
'updated_at_ms',
]),
]),
indexes: Object.freeze([
'schema_migrations_pkey',
'schema_capabilities_pkey',
'runs_pkey',
'ql3_runs_project_idempotency_uidx',
'ql3_runs_project_created_idx',
'ql3_runs_task_created_idx',
'ql3_runs_dispatch_candidates_idx',
'run_attempts_pkey',
'ql3_run_attempts_run_attempt_uidx',
'ql3_run_attempts_dispatch_candidates_idx',
'ql3_run_attempts_lease_idx',
'run_events_pkey',
'ql3_run_events_run_sequence_uidx',
'ql3_run_events_run_dedupe_uidx',
'ql3_run_events_run_created_idx',
'run_retry_policies_pkey',
'ql3_run_retry_policies_due_idx',
'ql3_runs_lost_retry_idx',
]),
checks: Object.freeze([
'ql3_schema_migrations_dialect_check',
'ql3_schema_migrations_checksum_check',
'ql3_schema_migrations_applied_at_check',
'ql3_schema_capabilities_version_check',
'ql3_schema_capabilities_payload_check',
'ql3_schema_capabilities_updated_at_check',
'ql3_runs_legacy_cron_id_check',
'ql3_runs_execution_owner_check',
'ql3_runs_scheduled_for_check',
'ql3_runs_status_check',
'ql3_runs_version_check',
'ql3_runs_event_sequence_check',
'ql3_runs_created_at_check',
'ql3_runs_queued_at_check',
'ql3_runs_started_at_check',
'ql3_runs_finished_at_check',
'ql3_runs_cancel_requested_at_check',
'ql3_runs_cancel_reason_check',
'ql3_run_attempts_attempt_check',
'ql3_run_attempts_status_check',
'ql3_run_attempts_pid_check',
'ql3_run_attempts_lease_expiry_check',
'ql3_run_attempts_deadline_check',
'ql3_run_attempts_callback_sequence_check',
'ql3_run_attempts_created_at_check',
'ql3_run_attempts_started_at_check',
'ql3_run_attempts_finished_at_check',
'ql3_run_events_sequence_check',
'ql3_run_events_actor_type_check',
'ql3_run_events_payload_check',
'ql3_run_events_created_at_check',
'ql3_run_retry_policies_max_attempts_check',
'ql3_run_retry_policies_safety_check',
'ql3_run_retry_policies_backoff_base_check',
'ql3_run_retry_policies_backoff_max_check',
'ql3_run_retry_policies_next_attempt_check',
'ql3_run_retry_policies_version_check',
'ql3_run_retry_policies_created_at_check',
'ql3_run_retry_policies_updated_at_check',
]),
foreignKeys: Object.freeze([
'ql3_schema_capabilities_migration_fk',
'ql3_runs_parent_fk',
'ql3_runs_retry_of_fk',
'ql3_run_attempts_run_fk',
'ql3_run_events_run_fk',
'ql3_run_events_attempt_fk',
'ql3_run_retry_policies_run_fk',
]),
});
@@ -0,0 +1,470 @@
import {
readPostgresMigrationHistory,
type PostgresMigrationQueryable,
} from '../adapters/postgresMigrationStreamStore';
import { auditMigrationStreamHistory } from '../core/migrationStream';
import { postgresqlMainMigrationStream } from '.';
import {
postgresqlControlSchemaContract,
type PostgresSchemaContract,
} from './schemaContract';
export const POSTGRES_SCHEMA_READINESS_ERROR_CODES = [
'server_version_unsupported',
'server_not_writable_primary',
'migration_history_invalid',
'capability_invalid',
'schema_contract_invalid',
'runtime_role_invalid',
] as const;
export type PostgresSchemaReadinessErrorCode =
(typeof POSTGRES_SCHEMA_READINESS_ERROR_CODES)[number];
export class PostgresSchemaReadinessError extends Error {
constructor(
readonly code: PostgresSchemaReadinessErrorCode,
readonly facts: readonly string[] = [],
) {
super(`PostgreSQL schema is not ready: ${code}`);
this.name = 'PostgresSchemaReadinessError';
}
}
export interface PostgresSchemaReadinessReport {
readonly ready: true;
readonly writablePrimary: true;
readonly serverVersionNum: number;
readonly serverMajor: number;
readonly currentUser: string;
readonly contractName: string;
readonly contractVersion: number;
readonly migrationIds: readonly string[];
}
interface ServerRow extends Record<string, unknown> {
serverVersionNum: unknown;
currentUser: unknown;
inRecovery: unknown;
transactionReadOnly: unknown;
}
interface CapabilityRow extends Record<string, unknown> {
contractName: unknown;
contractVersion: unknown;
migrationId: unknown;
capabilities: unknown;
}
interface ColumnRow extends Record<string, unknown> {
tableName: unknown;
columnName: unknown;
}
interface IndexRow extends Record<string, unknown> {
indexName: unknown;
}
interface ConstraintRow extends Record<string, unknown> {
constraintName: unknown;
constraintType: unknown;
}
interface SchemaPrivilegeRow extends Record<string, unknown> {
schemaUsage: unknown;
schemaCreate: unknown;
}
interface TablePrivilegeRow extends Record<string, unknown> {
tableName: unknown;
selectAllowed: unknown;
insertAllowed: unknown;
updateAllowed: unknown;
deleteAllowed: unknown;
isOwner: unknown;
}
const REQUIRED_RUNTIME_PRIVILEGES = Object.freeze({
schema_migrations: Object.freeze({
select: true,
insert: false,
update: false,
delete: false,
}),
schema_capabilities: Object.freeze({
select: true,
insert: false,
update: false,
delete: false,
}),
runs: Object.freeze({
select: true,
insert: true,
update: true,
delete: false,
}),
run_attempts: Object.freeze({
select: true,
insert: true,
update: true,
delete: false,
}),
run_events: Object.freeze({
select: true,
insert: true,
update: false,
delete: false,
}),
run_retry_policies: Object.freeze({
select: true,
insert: true,
update: true,
delete: false,
}),
});
function safeInteger(value: unknown): number | null {
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
const parsed = Number(value);
return Number.isSafeInteger(parsed) ? parsed : null;
}
return null;
}
function sorted(values: Iterable<string>): string[] {
return [...values].sort((left, right) => left.localeCompare(right));
}
function exactJsonObject(
actual: unknown,
expected: Readonly<Record<string, unknown>>,
): boolean {
if (!actual || typeof actual !== 'object' || Array.isArray(actual)) {
return false;
}
const actualObject = actual as Record<string, unknown>;
const actualKeys = sorted(Object.keys(actualObject));
const expectedKeys = sorted(Object.keys(expected));
return (
actualKeys.length === expectedKeys.length &&
actualKeys.every(
(key, index) =>
key === expectedKeys[index] && actualObject[key] === expected[key],
)
);
}
async function readServer(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract,
): Promise<{
writablePrimary: true;
serverVersionNum: number;
serverMajor: number;
currentUser: string;
}> {
const result = await queryable.query<ServerRow>(
`
SELECT
current_setting('server_version_num') AS "serverVersionNum",
current_user AS "currentUser",
pg_is_in_recovery() AS "inRecovery",
current_setting('transaction_read_only') AS "transactionReadOnly"
`.trim(),
);
const row = result.rows[0];
const serverVersionNum = safeInteger(row?.serverVersionNum);
const currentUser = row?.currentUser;
const inRecovery = row?.inRecovery;
const transactionReadOnly = row?.transactionReadOnly;
const serverMajor =
serverVersionNum === null ? null : Math.floor(serverVersionNum / 10_000);
if (
result.rows.length !== 1 ||
serverVersionNum === null ||
serverMajor === null ||
serverMajor < contract.minimumServerMajor ||
serverMajor > contract.maximumServerMajor ||
typeof currentUser !== 'string' ||
currentUser.length === 0
) {
throw new PostgresSchemaReadinessError('server_version_unsupported', [
String(serverVersionNum ?? 'invalid'),
]);
}
if (inRecovery !== false || transactionReadOnly !== 'off') {
throw new PostgresSchemaReadinessError('server_not_writable_primary', [
`in-recovery:${String(inRecovery)}`,
`transaction-read-only:${String(transactionReadOnly)}`,
]);
}
return {
writablePrimary: true,
serverVersionNum,
serverMajor,
currentUser,
};
}
async function assertHistory(
queryable: PostgresMigrationQueryable,
): Promise<readonly string[]> {
const history = await readPostgresMigrationHistory(queryable);
try {
auditMigrationStreamHistory(history, postgresqlMainMigrationStream);
return Object.freeze(history.map(({ migrationId }) => migrationId));
} catch (error) {
throw new PostgresSchemaReadinessError('migration_history_invalid', [
error instanceof Error ? error.name : 'UnknownError',
]);
}
}
async function assertCapability(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract,
): Promise<void> {
const result = await queryable.query<CapabilityRow>(
`
SELECT
contract_name AS "contractName",
contract_version AS "contractVersion",
migration_id AS "migrationId",
capabilities
FROM "${contract.schema}"."schema_capabilities"
WHERE contract_name = $1
`.trim(),
[contract.contractName],
);
const row = result.rows[0];
if (
result.rows.length !== 1 ||
!row ||
row.contractName !== contract.contractName ||
safeInteger(row.contractVersion) !== contract.contractVersion ||
row.migrationId !== contract.migrationId ||
!exactJsonObject(row.capabilities, contract.capabilities)
) {
throw new PostgresSchemaReadinessError('capability_invalid');
}
}
async function assertSchemaContract(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract,
): Promise<void> {
const [columnsResult, indexesResult, constraintsResult] = await Promise.all([
queryable.query<ColumnRow>(
`
SELECT table_name AS "tableName", column_name AS "columnName"
FROM information_schema.columns
WHERE table_schema = $1
ORDER BY table_name, ordinal_position
`.trim(),
[contract.schema],
),
queryable.query<IndexRow>(
`
SELECT indexname AS "indexName"
FROM pg_indexes
WHERE schemaname = $1
ORDER BY indexname
`.trim(),
[contract.schema],
),
queryable.query<ConstraintRow>(
`
SELECT
constraints.conname AS "constraintName",
CASE constraints.contype
WHEN 'c' THEN 'check'
WHEN 'f' THEN 'foreign_key'
END AS "constraintType"
FROM pg_constraint constraints
JOIN pg_class tables ON tables.oid = constraints.conrelid
JOIN pg_namespace schemas ON schemas.oid = tables.relnamespace
WHERE schemas.nspname = $1
AND constraints.contype IN ('c', 'f')
ORDER BY constraints.contype, constraints.conname
`.trim(),
[contract.schema],
),
]);
const actualTables = new Map<string, Set<string>>();
for (const row of columnsResult.rows) {
if (
typeof row.tableName !== 'string' ||
typeof row.columnName !== 'string'
) {
throw new PostgresSchemaReadinessError('schema_contract_invalid');
}
const columns = actualTables.get(row.tableName) ?? new Set<string>();
columns.add(row.columnName);
actualTables.set(row.tableName, columns);
}
const expectedTables = new Map(
contract.tables.map((table) => [table.name, new Set(table.columns)]),
);
const findings: string[] = [];
for (const [tableName, expectedColumns] of expectedTables) {
const actualColumns = actualTables.get(tableName);
if (!actualColumns) {
findings.push(`missing-table:${tableName}`);
continue;
}
for (const column of expectedColumns) {
if (!actualColumns.has(column)) {
findings.push(`missing-column:${tableName}.${column}`);
}
}
for (const column of actualColumns) {
if (!expectedColumns.has(column)) {
findings.push(`unknown-column:${tableName}.${column}`);
}
}
}
for (const tableName of actualTables.keys()) {
if (!expectedTables.has(tableName))
findings.push(`unknown-table:${tableName}`);
}
const actualIndexes = new Set<string>();
for (const row of indexesResult.rows) {
if (typeof row.indexName !== 'string') {
throw new PostgresSchemaReadinessError('schema_contract_invalid');
}
actualIndexes.add(row.indexName);
}
const expectedIndexes = new Set(contract.indexes);
for (const index of expectedIndexes) {
if (!actualIndexes.has(index)) findings.push(`missing-index:${index}`);
}
for (const index of actualIndexes) {
if (!expectedIndexes.has(index)) findings.push(`unknown-index:${index}`);
}
const actualChecks = new Set<string>();
const actualForeignKeys = new Set<string>();
for (const row of constraintsResult.rows) {
if (
typeof row.constraintName !== 'string' ||
(row.constraintType !== 'check' && row.constraintType !== 'foreign_key')
) {
throw new PostgresSchemaReadinessError('schema_contract_invalid');
}
const target =
row.constraintType === 'check' ? actualChecks : actualForeignKeys;
target.add(row.constraintName);
}
for (const check of contract.checks) {
if (!actualChecks.has(check)) findings.push(`missing-check:${check}`);
}
for (const check of actualChecks) {
if (!contract.checks.includes(check))
findings.push(`unknown-check:${check}`);
}
for (const foreignKey of contract.foreignKeys) {
if (!actualForeignKeys.has(foreignKey)) {
findings.push(`missing-foreign-key:${foreignKey}`);
}
}
for (const foreignKey of actualForeignKeys) {
if (!contract.foreignKeys.includes(foreignKey)) {
findings.push(`unknown-foreign-key:${foreignKey}`);
}
}
if (findings.length > 0) {
throw new PostgresSchemaReadinessError(
'schema_contract_invalid',
sorted(findings),
);
}
}
async function assertRuntimeRole(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract,
): Promise<void> {
const schemaResult = await queryable.query<SchemaPrivilegeRow>(
`
SELECT
has_schema_privilege(current_user, $1, 'USAGE') AS "schemaUsage",
has_schema_privilege(current_user, $1, 'CREATE') AS "schemaCreate"
`.trim(),
[contract.schema],
);
const schema = schemaResult.rows[0];
const tableNames = Object.keys(REQUIRED_RUNTIME_PRIVILEGES);
const tableResult = await queryable.query<TablePrivilegeRow>(
`
SELECT
requested.table_name AS "tableName",
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'SELECT') AS "selectAllowed",
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'INSERT') AS "insertAllowed",
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'UPDATE') AS "updateAllowed",
has_table_privilege(current_user, format('%I.%I', $1, requested.table_name), 'DELETE') AS "deleteAllowed",
pg_get_userbyid(classes.relowner) = current_user AS "isOwner"
FROM unnest($2::text[]) AS requested(table_name)
JOIN pg_namespace namespaces ON namespaces.nspname = $1
JOIN pg_class classes
ON classes.relnamespace = namespaces.oid
AND classes.relname = requested.table_name
ORDER BY requested.table_name
`.trim(),
[contract.schema, tableNames],
);
const findings: string[] = [];
if (
schemaResult.rows.length !== 1 ||
schema?.schemaUsage !== true ||
schema?.schemaCreate !== false
) {
findings.push('schema-privileges');
}
const privilegesByTable = new Map(
tableResult.rows.map((row) => [row.tableName, row]),
);
for (const tableName of tableNames) {
const expected =
REQUIRED_RUNTIME_PRIVILEGES[
tableName as keyof typeof REQUIRED_RUNTIME_PRIVILEGES
];
const actual = privilegesByTable.get(tableName);
if (
!actual ||
actual.selectAllowed !== expected.select ||
actual.insertAllowed !== expected.insert ||
actual.updateAllowed !== expected.update ||
actual.deleteAllowed !== expected.delete ||
actual.isOwner !== false
) {
findings.push(`table-privileges:${tableName}`);
}
}
if (privilegesByTable.size !== tableNames.length) {
findings.push('table-privilege-row-count');
}
if (findings.length > 0) {
throw new PostgresSchemaReadinessError(
'runtime_role_invalid',
sorted(findings),
);
}
}
export async function assertPostgresSchemaReady(
queryable: PostgresMigrationQueryable,
contract: PostgresSchemaContract = postgresqlControlSchemaContract,
): Promise<PostgresSchemaReadinessReport> {
const server = await readServer(queryable, contract);
const migrationIds = await assertHistory(queryable);
await assertCapability(queryable, contract);
await assertSchemaContract(queryable, contract);
await assertRuntimeRole(queryable, contract);
return Object.freeze({
ready: true,
...server,
contractName: contract.contractName,
contractVersion: contract.contractVersion,
migrationIds,
});
}
@@ -0,0 +1,34 @@
import { createHash } from 'crypto';
import type { MigrationStreamStep } from '../core/migrationStream';
import type { PostgresMigrationContext } from '../adapters/postgresMigrationStreamStore';
export interface PostgresSqlMigrationDefinition {
readonly id: string;
readonly statements: readonly string[];
}
export function checksumPostgresStatements(
statements: readonly string[],
): string {
return createHash('sha256')
.update(
JSON.stringify({
format: 1,
statements,
}),
)
.digest('hex');
}
export function definePostgresSqlMigration(
definition: PostgresSqlMigrationDefinition,
): MigrationStreamStep<PostgresMigrationContext> {
const statements = Object.freeze([...definition.statements]);
return Object.freeze({
id: definition.id,
checksum: checksumPostgresStatements(statements),
async up(context: PostgresMigrationContext) {
for (const statement of statements) await context.query(statement);
},
});
}
+91
View File
@@ -0,0 +1,91 @@
import type { ModelStatic, Sequelize } from 'sequelize';
import Logger from '../loaders/logger';
import {
SchemaMigrationInstance,
SchemaMigrationModel,
} from '../data/schemaMigration';
import { sequelize } from '../data';
import { migrations as registeredMigrations } from '.';
import {
SQLITE_MAIN_MIGRATION_STREAM_ID,
SequelizeSqliteMigrationStreamStore,
} from './adapters/sequelizeSqliteMigrationStreamStore';
import {
runMigrationStream,
type MigrationStreamStore,
} from './core/migrationStream';
import type { Migration, MigrationContext } from './types';
interface MigrationLogger {
info(message: string): unknown;
}
export interface RunMigrationsOptions {
database?: Sequelize;
migrationModel?: ModelStatic<SchemaMigrationInstance>;
migrations?: Migration[];
logger?: MigrationLogger;
}
function validateMigrations(migrations: Migration[]) {
const ids = new Set<string>();
for (const migration of migrations) {
if (ids.has(migration.id)) {
throw new Error(`Duplicate migration id: ${migration.id}`);
}
ids.add(migration.id);
}
}
function scopeHistoryToCustomMigrations(
store: MigrationStreamStore<MigrationContext>,
migrations: readonly Migration[],
): MigrationStreamStore<MigrationContext> {
const ids = new Set(migrations.map(({ id }) => id));
return {
ensureHistory: () => store.ensureHistory(),
listAll: async () =>
(await store.listAll()).filter(({ migrationId }) => ids.has(migrationId)),
findById: (migrationId) => store.findById(migrationId),
transaction: (work) => store.transaction(work),
};
}
export async function runMigrations(
options: RunMigrationsOptions = {},
): Promise<void> {
const database = options.database || sequelize;
const migrationModel = options.migrationModel || SchemaMigrationModel;
const migrations = options.migrations || registeredMigrations;
const logger = options.logger || Logger;
validateMigrations(migrations);
const store = new SequelizeSqliteMigrationStreamStore(
database,
migrationModel,
);
await runMigrationStream({
stream: {
id: SQLITE_MAIN_MIGRATION_STREAM_ID,
dialect: 'sqlite',
migrationIdScheme: 'sqlite-numbered',
checksumScheme: 'legacy-opaque',
migrations,
},
store:
options.migrations === undefined
? store
: scopeHistoryToCustomMigrations(store, migrations),
clock: Date.now,
logger: {
info(message) {
logger.info(
message.replace(
`[migration:${SQLITE_MAIN_MIGRATION_STREAM_ID}]`,
'[migration]',
),
);
},
},
});
}
+141
View File
@@ -0,0 +1,141 @@
import {
SqliteSchemaOwnershipManifest,
sqliteSchemaOwnership,
} from './schemaOwnership';
export interface SqliteSchemaSnapshotTable {
name: string;
columns: readonly string[];
indexes: readonly string[];
}
export interface SqliteSchemaSnapshot {
tables: readonly SqliteSchemaSnapshotTable[];
migrationIds: readonly string[];
}
export interface SchemaColumnFinding {
table: string;
column: string;
}
export interface SqliteSchemaAuditReport {
compatible: boolean;
driftDetected: boolean;
missingTables: string[];
missingColumns: SchemaColumnFinding[];
missingIndexes: string[];
missingMigrationIds: string[];
unknownTables: string[];
unknownColumns: SchemaColumnFinding[];
unknownIndexes: string[];
extraMigrationIds: string[];
}
function sortedUnique(values: readonly string[]): string[] {
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
}
function sortColumns(values: SchemaColumnFinding[]): SchemaColumnFinding[] {
return values.sort(
(left, right) =>
left.table.localeCompare(right.table) ||
left.column.localeCompare(right.column),
);
}
export function auditSqliteSchema(
snapshot: SqliteSchemaSnapshot,
manifest: SqliteSchemaOwnershipManifest = sqliteSchemaOwnership,
): SqliteSchemaAuditReport {
const actualTables = new Map(
snapshot.tables.map((table) => [
table.name,
{
columns: new Set(table.columns),
indexes: new Set(table.indexes),
},
]),
);
const ownedTables = new Map(
manifest.tables.map((table) => [table.name, table]),
);
const actualIndexes = new Set(
snapshot.tables.flatMap((table) => [...table.indexes]),
);
const ownedIndexes = new Set(manifest.indexes.map((index) => index.name));
const actualMigrations = new Set(snapshot.migrationIds);
const ownedMigrations = new Set(manifest.migrationIds);
const missingTables: string[] = [];
const missingColumns: SchemaColumnFinding[] = [];
const unknownColumns: SchemaColumnFinding[] = [];
for (const table of manifest.tables) {
const actual = actualTables.get(table.name);
if (!actual) {
if (table.mode === 'unmanaged-legacy') continue;
missingTables.push(table.name);
continue;
}
for (const column of table.requiredColumns) {
if (!actual.columns.has(column)) {
missingColumns.push({ table: table.name, column });
}
}
if (table.mode === 'full') {
const ownedColumns = new Set(table.requiredColumns);
for (const column of actual.columns) {
if (!ownedColumns.has(column)) {
unknownColumns.push({ table: table.name, column });
}
}
}
}
const missingIndexes = manifest.indexes
.map((index) => index.name)
.filter((index) => !actualIndexes.has(index));
const missingMigrationIds = manifest.migrationIds.filter(
(migrationId) => !actualMigrations.has(migrationId),
);
const unknownTables = snapshot.tables
.map((table) => table.name)
.filter(
(table) => !table.startsWith('sqlite_') && !ownedTables.has(table),
);
const reportableIndexes = new Set(
snapshot.tables.flatMap((table) => {
const ownership = ownedTables.get(table.name);
return !ownership || ownership.mode === 'full' ? [...table.indexes] : [];
}),
);
const unknownIndexes = [...reportableIndexes].filter(
(index) => !index.startsWith('sqlite_autoindex_') && !ownedIndexes.has(index),
);
const extraMigrationIds = snapshot.migrationIds.filter(
(migrationId) => !ownedMigrations.has(migrationId),
);
const compatible =
missingTables.length === 0 &&
missingColumns.length === 0 &&
missingIndexes.length === 0 &&
missingMigrationIds.length === 0;
const driftDetected =
unknownTables.length > 0 ||
unknownColumns.length > 0 ||
unknownIndexes.length > 0 ||
extraMigrationIds.length > 0;
return {
compatible,
driftDetected,
missingTables: sortedUnique(missingTables),
missingColumns: sortColumns(missingColumns),
missingIndexes: sortedUnique(missingIndexes),
missingMigrationIds: sortedUnique(missingMigrationIds),
unknownTables: sortedUnique(unknownTables),
unknownColumns: sortColumns(unknownColumns),
unknownIndexes: sortedUnique(unknownIndexes),
extraMigrationIds: sortedUnique(extraMigrationIds),
};
}
+391
View File
@@ -0,0 +1,391 @@
import { legacyColumnOwnership } from './0001-legacy-columns';
import {
RUN_ATTEMPT_TABLE,
RUN_EVENT_TABLE,
RUN_TABLE,
runSchemaManifest,
} from './0002-run-schema';
import {
RUNNING_INSTANCE_TABLE,
runningInstanceRunReferenceManifest,
} from './0003-running-instance-run-reference';
import { runCancellationRequestManifest } from './0004-run-cancellation-request';
import {
RUN_CANCELLATION_DISPATCH_TABLE,
runCancellationDispatchManifest,
} from './0005-run-cancellation-dispatch';
import { runAttemptDeadlineManifest } from './0006-run-attempt-deadline';
import {
COMPLETION_RECEIPT_JOURNAL_TABLE,
completionReceiptJournalManifest,
} from './0007-completion-receipt-journal';
import {
WORKER_REGISTRY_TABLE,
workerRegistryManifest,
} from './0008-worker-registry';
import {
RUN_DISPATCH_LEASE_TABLE,
runDispatchLeaseManifest,
} from './0009-run-dispatch-lease';
import { runDispatchCandidateManifest } from './0010-run-dispatch-candidates';
import {
RUN_RETRY_POLICY_TABLE,
runRetryPolicyManifest,
} from './0011-run-retry-policy';
import {
TASK_EXECUTION_REVISION_TABLE,
taskExecutionRevisionManifest,
} from './0012-task-execution-revisions';
import {
LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
localExecutionContextRecipeManifest,
} from './0013-local-execution-context-recipes';
import {
LOCAL_SECRET_ENVELOPE_TABLE,
localSecretEnvelopeManifest,
} from './0014-local-secret-envelopes';
import {
LOCAL_ARTIFACT_RETENTION_TABLE,
localArtifactRetentionManifest,
} from './0015-local-artifact-retention';
import {
LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
localArtifactMaintenanceCursorManifest,
} from './0016-local-artifact-maintenance-cursor';
import {
PROJECT_ROLE_BINDING_TABLE,
PROJECT_TABLE,
projectPolicyManifest,
} from './0017-project-policy';
import {
PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
projectOwnerBootstrapManifest,
} from './0018-project-owner-bootstrap';
import {
IDENTITY_AUTHENTICATION_BINDING_TABLE,
IDENTITY_SUBJECT_TABLE,
identityDirectoryManifest,
} from './0019-identity-directory';
import {
APPROVAL_REQUEST_TABLE,
APPROVED_ACTION_DISPATCH_TABLE,
approvalRequestManifest,
} from './0020-approval-requests';
import {
APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
approvedActionDispatchExecutionManifest,
} from './0021-approved-action-dispatch-executions';
import {
APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
approvedActionRecoveryManifest,
} from './0022-approved-action-recovery';
import {
APPROVED_RUN_ACTION_RECEIPT_TABLE,
approvedRunActionReceiptManifest,
} from './0023-approved-run-action-receipts';
import {
APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
approvedActionRecoveryAuthorizationManifest,
} from './0024-approved-action-recovery-authorization';
export type SchemaOwnershipMode = 'full' | 'extension' | 'unmanaged-legacy';
export interface SqliteTableOwnership {
name: string;
mode: SchemaOwnershipMode;
requiredColumns: readonly string[];
}
export interface SqliteIndexOwnership {
name: string;
}
export interface SqliteSchemaOwnershipManifest {
version: 1;
database: 'database.sqlite';
migrationIds: readonly string[];
tables: readonly SqliteTableOwnership[];
indexes: readonly SqliteIndexOwnership[];
constraints: readonly string[];
unknownObjectPolicy: 'preserve-and-report';
}
function unique(values: readonly string[]): string[] {
return [...new Set(values)];
}
function indexName(definition: string): string {
const boundary = definition.indexOf('(');
return boundary === -1 ? definition : definition.slice(0, boundary);
}
function legacyColumns(table: string): string[] {
return legacyColumnOwnership
.filter((definition) => definition.table === table)
.map((definition) => definition.column);
}
export const sqliteSchemaOwnership: SqliteSchemaOwnershipManifest = {
version: 1,
database: 'database.sqlite',
migrationIds: [
'0001-legacy-columns',
'0002-run-schema',
'0003-running-instance-run-reference',
'0004-run-cancellation-request',
'0005-run-cancellation-dispatch',
'0006-run-attempt-deadline',
'0007-completion-receipt-journal',
'0008-worker-registry',
'0009-run-dispatch-lease',
'0010-run-dispatch-candidates',
'0011-run-retry-policy',
'0012-task-execution-revisions',
'0013-local-execution-context-recipes',
'0014-local-secret-envelopes',
'0015-local-artifact-retention',
'0016-local-artifact-maintenance-cursor',
'0017-project-policy',
'0018-project-owner-bootstrap',
'0019-identity-directory',
'0020-approval-requests',
'0021-approved-action-dispatch-executions',
'0022-approved-action-recovery',
'0023-approved-run-action-receipts',
'0024-approved-action-recovery-authorization',
],
tables: [
{
name: 'SchemaMigrations',
mode: 'full',
requiredColumns: ['id', 'checksum', 'applied_at'],
},
{
name: 'Apps',
mode: 'unmanaged-legacy',
requiredColumns: [],
},
{
name: 'Auths',
mode: 'unmanaged-legacy',
requiredColumns: [],
},
{
name: 'CrontabStats',
mode: 'unmanaged-legacy',
requiredColumns: [],
},
{
name: 'Dependences',
mode: 'unmanaged-legacy',
requiredColumns: [],
},
{
name: 'CrontabViews',
mode: 'extension',
requiredColumns: legacyColumns('CrontabViews'),
},
{
name: 'Subscriptions',
mode: 'extension',
requiredColumns: legacyColumns('Subscriptions'),
},
{
name: 'Crontabs',
mode: 'extension',
requiredColumns: legacyColumns('Crontabs'),
},
{
name: 'Envs',
mode: 'extension',
requiredColumns: legacyColumns('Envs'),
},
{
name: RUN_TABLE,
mode: 'full',
requiredColumns: unique([
...runSchemaManifest.tables.Runs,
...Object.keys(runCancellationRequestManifest.columns),
]),
},
{
name: RUN_ATTEMPT_TABLE,
mode: 'full',
requiredColumns: unique([
...runSchemaManifest.tables.RunAttempts,
...Object.keys(runAttemptDeadlineManifest.columns),
]),
},
{
name: RUN_EVENT_TABLE,
mode: 'full',
requiredColumns: runSchemaManifest.tables.RunEvents,
},
{
name: RUNNING_INSTANCE_TABLE,
mode: 'extension',
requiredColumns: Object.keys(runningInstanceRunReferenceManifest.columns),
},
{
name: RUN_CANCELLATION_DISPATCH_TABLE,
mode: 'full',
requiredColumns: runCancellationDispatchManifest.columns,
},
{
name: COMPLETION_RECEIPT_JOURNAL_TABLE,
mode: 'full',
requiredColumns: completionReceiptJournalManifest.columns,
},
{
name: WORKER_REGISTRY_TABLE,
mode: 'full',
requiredColumns: workerRegistryManifest.columns,
},
{
name: RUN_DISPATCH_LEASE_TABLE,
mode: 'full',
requiredColumns: runDispatchLeaseManifest.columns,
},
{
name: RUN_RETRY_POLICY_TABLE,
mode: 'full',
requiredColumns: runRetryPolicyManifest.columns,
},
{
name: TASK_EXECUTION_REVISION_TABLE,
mode: 'full',
requiredColumns: taskExecutionRevisionManifest.columns,
},
{
name: LOCAL_EXECUTION_CONTEXT_RECIPE_TABLE,
mode: 'full',
requiredColumns: localExecutionContextRecipeManifest.columns,
},
{
name: LOCAL_SECRET_ENVELOPE_TABLE,
mode: 'full',
requiredColumns: localSecretEnvelopeManifest.columns,
},
{
name: LOCAL_ARTIFACT_RETENTION_TABLE,
mode: 'full',
requiredColumns: localArtifactRetentionManifest.columns,
},
{
name: LOCAL_ARTIFACT_MAINTENANCE_CURSOR_TABLE,
mode: 'full',
requiredColumns: localArtifactMaintenanceCursorManifest.columns,
},
{
name: PROJECT_TABLE,
mode: 'full',
requiredColumns: projectPolicyManifest.tables.Projects,
},
{
name: PROJECT_ROLE_BINDING_TABLE,
mode: 'full',
requiredColumns: projectPolicyManifest.tables.ProjectRoleBindings,
},
{
name: PROJECT_OWNER_BOOTSTRAP_CHALLENGE_TABLE,
mode: 'full',
requiredColumns: projectOwnerBootstrapManifest.columns,
},
{
name: IDENTITY_SUBJECT_TABLE,
mode: 'full',
requiredColumns: identityDirectoryManifest.tables.IdentitySubjects,
},
{
name: IDENTITY_AUTHENTICATION_BINDING_TABLE,
mode: 'full',
requiredColumns:
identityDirectoryManifest.tables.IdentityAuthenticationBindings,
},
{
name: APPROVAL_REQUEST_TABLE,
mode: 'full',
requiredColumns: approvalRequestManifest.tables.ApprovalRequests,
},
{
name: APPROVED_ACTION_DISPATCH_TABLE,
mode: 'full',
requiredColumns: approvalRequestManifest.tables.ApprovedActionDispatches,
},
{
name: APPROVED_ACTION_DISPATCH_EXECUTION_TABLE,
mode: 'full',
requiredColumns: approvedActionDispatchExecutionManifest.columns,
},
{
name: APPROVED_ACTION_RECOVERY_CONTROL_TABLE,
mode: 'full',
requiredColumns:
approvedActionRecoveryManifest.tables.ApprovedActionRecoveryControls,
},
{
name: APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE,
mode: 'full',
requiredColumns:
approvedActionRecoveryManifest.tables.ApprovedActionRecoveryResolutions,
},
{
name: APPROVED_RUN_ACTION_RECEIPT_TABLE,
mode: 'full',
requiredColumns: approvedRunActionReceiptManifest.columns,
},
{
name: APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE,
mode: 'full',
requiredColumns: approvedActionRecoveryAuthorizationManifest.columns,
},
],
indexes: unique([
...runSchemaManifest.indexes,
...runningInstanceRunReferenceManifest.indexes.map(indexName),
...runCancellationRequestManifest.indexes.map(indexName),
...runCancellationDispatchManifest.indexes.map(indexName),
...runAttemptDeadlineManifest.indexes.map(indexName),
...completionReceiptJournalManifest.indexes.map(indexName),
...workerRegistryManifest.indexes.map(indexName),
...runDispatchLeaseManifest.indexes.map(indexName),
...runDispatchCandidateManifest.indexes.map(indexName),
...runRetryPolicyManifest.indexes.map(indexName),
...taskExecutionRevisionManifest.indexes.map(indexName),
...localExecutionContextRecipeManifest.indexes.map(indexName),
...localSecretEnvelopeManifest.indexes.map(indexName),
...localArtifactRetentionManifest.indexes.map(indexName),
...localArtifactMaintenanceCursorManifest.indexes.map(indexName),
...projectPolicyManifest.indexes.map(indexName),
...projectOwnerBootstrapManifest.indexes.map(indexName),
...identityDirectoryManifest.indexes.map(indexName),
...approvalRequestManifest.indexes.map(indexName),
...approvedActionDispatchExecutionManifest.indexes.map(indexName),
...approvedActionRecoveryManifest.indexes.map(indexName),
...approvedRunActionReceiptManifest.indexes.map(indexName),
...approvedActionRecoveryAuthorizationManifest.indexes.map(indexName),
]).map((name) => ({ name })),
constraints: unique([
...runSchemaManifest.constraints,
...runCancellationDispatchManifest.constraints,
...completionReceiptJournalManifest.constraints,
...workerRegistryManifest.constraints,
...runDispatchLeaseManifest.constraints,
...runRetryPolicyManifest.constraints,
...taskExecutionRevisionManifest.constraints,
...localExecutionContextRecipeManifest.constraints,
...localSecretEnvelopeManifest.constraints,
...localArtifactRetentionManifest.constraints,
...localArtifactMaintenanceCursorManifest.constraints,
...projectPolicyManifest.constraints,
...projectOwnerBootstrapManifest.constraints,
...identityDirectoryManifest.constraints,
...approvalRequestManifest.constraints,
...approvedActionDispatchExecutionManifest.constraints,
...approvedActionRecoveryManifest.constraints,
...approvedRunActionReceiptManifest.constraints,
...approvedActionRecoveryAuthorizationManifest.constraints,
]),
unknownObjectPolicy: 'preserve-and-report',
};
+12
View File
@@ -0,0 +1,12 @@
import type { QueryInterface, Transaction } from 'sequelize';
export interface MigrationContext {
queryInterface: QueryInterface;
transaction: Transaction;
}
export interface Migration {
id: string;
checksum: string;
up(context: MigrationContext): Promise<void>;
}
+3
View File
@@ -151,6 +151,7 @@ enum NotificationMode {
chronocat = 18;
ntfy = 19;
wxPusherBot = 20;
wxPusherSpt = 21;
}
message NotificationInfo {
@@ -244,6 +245,8 @@ message NotificationInfo {
optional string wxPusherBotAppToken = 66;
optional string wxPusherBotTopicIds = 67;
optional string wxPusherBotUids = 68;
optional string wxPusherSptList = 70;
}
message SystemNotifyRequest {
+24
View File
@@ -43,6 +43,7 @@ export enum NotificationMode {
chronocat = 18,
ntfy = 19,
wxPusherBot = 20,
wxPusherSpt = 21,
UNRECOGNIZED = -1,
}
@@ -111,6 +112,9 @@ export function notificationModeFromJSON(object: any): NotificationMode {
case 20:
case "wxPusherBot":
return NotificationMode.wxPusherBot;
case 21:
case "wxPusherSpt":
return NotificationMode.wxPusherSpt;
case -1:
case "UNRECOGNIZED":
default:
@@ -162,6 +166,8 @@ export function notificationModeToJSON(object: NotificationMode): string {
return "ntfy";
case NotificationMode.wxPusherBot:
return "wxPusherBot";
case NotificationMode.wxPusherSpt:
return "wxPusherSpt";
case NotificationMode.UNRECOGNIZED:
default:
return "UNRECOGNIZED";
@@ -393,6 +399,7 @@ export interface NotificationInfo {
wxPusherBotAppToken?: string | undefined;
wxPusherBotTopicIds?: string | undefined;
wxPusherBotUids?: string | undefined;
wxPusherSptList?: string | undefined;
}
export interface SystemNotifyRequest {
@@ -2959,6 +2966,7 @@ function createBaseNotificationInfo(): NotificationInfo {
wxPusherBotAppToken: undefined,
wxPusherBotTopicIds: undefined,
wxPusherBotUids: undefined,
wxPusherSptList: undefined,
};
}
@@ -3171,6 +3179,9 @@ export const NotificationInfo: MessageFns<NotificationInfo> = {
if (message.wxPusherBotUids !== undefined) {
writer.uint32(546).string(message.wxPusherBotUids);
}
if (message.wxPusherSptList !== undefined) {
writer.uint32(562).string(message.wxPusherSptList);
}
return writer;
},
@@ -3733,6 +3744,14 @@ export const NotificationInfo: MessageFns<NotificationInfo> = {
message.wxPusherBotUids = reader.string();
continue;
}
case 70: {
if (tag !== 562) {
break;
}
message.wxPusherSptList = reader.string();
continue;
}
}
if ((tag & 7) === 4 || tag === 0) {
break;
@@ -3825,6 +3844,7 @@ export const NotificationInfo: MessageFns<NotificationInfo> = {
? globalThis.String(object.wxPusherBotTopicIds)
: undefined,
wxPusherBotUids: isSet(object.wxPusherBotUids) ? globalThis.String(object.wxPusherBotUids) : undefined,
wxPusherSptList: isSet(object.wxPusherSptList) ? globalThis.String(object.wxPusherSptList) : undefined,
};
},
@@ -4037,6 +4057,9 @@ export const NotificationInfo: MessageFns<NotificationInfo> = {
if (message.wxPusherBotUids !== undefined) {
obj.wxPusherBotUids = message.wxPusherBotUids;
}
if (message.wxPusherSptList !== undefined) {
obj.wxPusherSptList = message.wxPusherSptList;
}
return obj;
},
@@ -4114,6 +4137,7 @@ export const NotificationInfo: MessageFns<NotificationInfo> = {
message.wxPusherBotAppToken = object.wxPusherBotAppToken ?? undefined;
message.wxPusherBotTopicIds = object.wxPusherBotTopicIds ?? undefined;
message.wxPusherBotUids = object.wxPusherBotUids ?? undefined;
message.wxPusherSptList = object.wxPusherSptList ?? undefined;
return message;
},
};
@@ -0,0 +1,110 @@
import { createHash, timingSafeEqual } from 'crypto';
import type {
LegacyPanelAuthSnapshot,
LegacyPanelAuthSnapshotReader,
LegacyPanelPlatform,
LegacyPanelSessionSource,
} from '../../ports/legacyPanelSessionSource';
export const MAX_LEGACY_PANEL_TOKEN_LENGTH = 4096;
export const MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM = 64;
export class LegacyPanelSessionUnavailableError extends Error {
readonly code = 'LEGACY_PANEL_SESSION_UNAVAILABLE';
constructor() {
super('Legacy panel session state is unavailable');
this.name = 'LegacyPanelSessionUnavailableError';
}
}
function assertToken(value: string): void {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > MAX_LEGACY_PANEL_TOKEN_LENGTH
) {
throw new LegacyPanelSessionUnavailableError();
}
}
function tokenMatches(left: string, right: string): boolean {
assertToken(left);
assertToken(right);
return timingSafeEqual(
createHash('sha256').update(left, 'utf8').digest(),
createHash('sha256').update(right, 'utf8').digest(),
);
}
function candidates(
snapshot: Readonly<LegacyPanelAuthSnapshot>,
platform: LegacyPanelPlatform,
): string[] {
const result: string[] = [];
if (snapshot.token !== undefined && snapshot.token !== '') {
assertToken(snapshot.token);
result.push(snapshot.token);
}
if (snapshot.tokens === undefined) return result;
if (
!snapshot.tokens ||
typeof snapshot.tokens !== 'object' ||
Array.isArray(snapshot.tokens)
) {
throw new LegacyPanelSessionUnavailableError();
}
const platformTokens = snapshot.tokens[platform];
if (platformTokens === null || platformTokens === undefined) return result;
if (typeof platformTokens === 'string') {
if (platformTokens === '') return result;
assertToken(platformTokens);
result.push(platformTokens);
return result;
}
if (
!Array.isArray(platformTokens) ||
platformTokens.length > MAX_LEGACY_PANEL_TOKENS_PER_PLATFORM
) {
throw new LegacyPanelSessionUnavailableError();
}
for (const item of platformTokens) {
if (!item || typeof item !== 'object' || Array.isArray(item)) {
throw new LegacyPanelSessionUnavailableError();
}
assertToken(item.value);
result.push(item.value);
}
return result;
}
export class LegacyAuthInfoSessionSource implements LegacyPanelSessionSource {
constructor(private readonly read: LegacyPanelAuthSnapshotReader) {
if (typeof read !== 'function') {
throw new TypeError('Legacy panel auth snapshot reader is invalid');
}
}
async isActive(
token: string,
platform: LegacyPanelPlatform,
): Promise<boolean> {
assertToken(token);
if (platform !== 'desktop' && platform !== 'mobile') {
throw new TypeError('Legacy panel platform is invalid');
}
let snapshot: Readonly<LegacyPanelAuthSnapshot> | null;
try {
snapshot = await this.read();
} catch {
throw new LegacyPanelSessionUnavailableError();
}
if (!snapshot) return false;
if (typeof snapshot !== 'object' || Array.isArray(snapshot)) {
throw new LegacyPanelSessionUnavailableError();
}
return candidates(snapshot, platform).some((candidate) =>
tokenMatches(candidate, token),
);
}
}
@@ -0,0 +1,94 @@
import { createCipheriv, createDecipheriv, randomBytes } from 'crypto';
import { TextDecoder } from 'util';
import {
LOCAL_SECRET_ALGORITHM,
LocalSecretUnavailableError,
localSecretBinary,
localSecretEnvelopeAad,
normalizeLocalSecretEnvelope,
type LocalSecretEnvelope,
} from '../../domain/localSecret';
const SECRET_KEY_BYTES = 32;
const SECRET_NONCE_BYTES = 12;
const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
export type LocalSecretNonceFactory = () => Uint8Array;
function ownedSecretKey(key: Uint8Array): Buffer {
if (!(key instanceof Uint8Array) || key.byteLength !== SECRET_KEY_BYTES) {
throw new LocalSecretUnavailableError();
}
return Buffer.from(key);
}
export function encryptLocalSecretEnvelope(
metadata: Omit<LocalSecretEnvelope, 'nonce' | 'ciphertext' | 'authTag'>,
plaintext: string,
key: Uint8Array,
nonceFactory: LocalSecretNonceFactory = () => randomBytes(SECRET_NONCE_BYTES),
): LocalSecretEnvelope {
const ownedKey = ownedSecretKey(key);
const plaintextBuffer = Buffer.from(plaintext, 'utf8');
let nonce: Buffer | undefined;
try {
nonce = Buffer.from(nonceFactory());
if (nonce.length !== SECRET_NONCE_BYTES) {
throw new LocalSecretUnavailableError();
}
const cipher = createCipheriv(LOCAL_SECRET_ALGORITHM, ownedKey, nonce, {
authTagLength: 16,
});
cipher.setAAD(localSecretEnvelopeAad(metadata));
const ciphertext = Buffer.concat([
cipher.update(plaintextBuffer),
cipher.final(),
]);
return normalizeLocalSecretEnvelope({
...metadata,
nonce: nonce.toString('base64url'),
ciphertext: ciphertext.toString('base64url'),
authTag: cipher.getAuthTag().toString('base64url'),
});
} catch (error) {
if (error instanceof LocalSecretUnavailableError) throw error;
throw new LocalSecretUnavailableError();
} finally {
ownedKey.fill(0);
plaintextBuffer.fill(0);
nonce?.fill(0);
}
}
export function decryptLocalSecretEnvelopeToBuffer(
envelope: LocalSecretEnvelope,
key: Uint8Array,
): Buffer {
const normalized = normalizeLocalSecretEnvelope(envelope);
const ownedKey = ownedSecretKey(key);
const nonce = localSecretBinary('nonce', normalized.nonce);
const ciphertext = localSecretBinary('ciphertext', normalized.ciphertext);
const authTag = localSecretBinary('authTag', normalized.authTag);
try {
const decipher = createDecipheriv(LOCAL_SECRET_ALGORITHM, ownedKey, nonce, {
authTagLength: 16,
});
decipher.setAAD(localSecretEnvelopeAad(normalized));
decipher.setAuthTag(authTag);
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
} catch {
throw new LocalSecretUnavailableError();
} finally {
ownedKey.fill(0);
nonce.fill(0);
authTag.fill(0);
}
}
export function decodeLocalSecretPlaintext(plaintext: Buffer): string {
try {
return UTF8_DECODER.decode(plaintext);
} catch {
throw new LocalSecretUnavailableError();
}
}
@@ -0,0 +1,24 @@
import type { CompletionReceipt } from '../../domain/completionReceipt';
import { matchesWorkerExecutionCompletionReceiptAuthentication } from '../../domain/workerExecutionCompletionReceiptAuthentication';
import type { WorkerExecutionOfferJournalRecord } from '../../domain/workerExecutionOffer';
import type { WorkerExecutionCompletionReceiptAuthenticator } from '../../ports/workerExecutionCompletionReceiptAuthenticator';
export class Sha256WorkerExecutionCompletionReceiptAuthenticator
implements WorkerExecutionCompletionReceiptAuthenticator
{
authenticate(
receipt: CompletionReceipt,
record: WorkerExecutionOfferJournalRecord,
): boolean {
if (
record.completionReceiptCallbackSequence === undefined ||
record.completionReceiptTokenDigest === undefined
) {
return false;
}
return matchesWorkerExecutionCompletionReceiptAuthentication(receipt, {
callbackSequence: record.completionReceiptCallbackSequence,
tokenDigest: record.completionReceiptTokenDigest,
});
}
}
@@ -0,0 +1,226 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { randomBytes } from 'crypto';
import {
assertCompletionReceiptId,
InvalidCompletionReceiptError,
MAX_COMPLETION_RECEIPT_BYTES,
parseCompletionReceipt,
serializeCompletionReceipt,
type CompletionReceipt,
} from '../../domain/completionReceipt';
import type { CompletionReceiptStore } from '../../ports/completionReceiptStore';
export class CompletionReceiptAlreadyExistsError extends Error {
constructor(readonly attemptId: string) {
super(`Completion receipt already exists for Attempt ${attemptId}`);
this.name = 'CompletionReceiptAlreadyExistsError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
/**
* A bounded local journal. It does not scan directories or own a lifecycle;
* callers must discover active Attempts from the database.
*/
export class CompletionReceiptFileStore implements CompletionReceiptStore {
constructor(private readonly root: string) {
if (!path.isAbsolute(root)) {
throw new RangeError('Completion receipt root must be absolute');
}
}
async publish(receipt: CompletionReceipt): Promise<void> {
const serialized = serializeCompletionReceipt(receipt);
const directory = this.directory(receipt.attemptId);
const target = this.target(receipt.attemptId);
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
const temporary = path.join(
directory,
`.${receipt.attemptId}.${randomBytes(16).toString('hex')}.tmp`,
);
let handle: fs.FileHandle | undefined;
try {
handle = await fs.open(temporary, 'wx', 0o600);
await handle.writeFile(serialized, 'utf8');
await handle.sync();
await handle.close();
handle = undefined;
// A hard link publishes the fully written inode atomically and, unlike
// plain rename(), never replaces an existing completion fact.
try {
await fs.link(temporary, target);
} catch (error) {
if (isCode(error, 'EEXIST')) {
throw new CompletionReceiptAlreadyExistsError(receipt.attemptId);
}
throw error;
}
await this.bestEffortUnlink(temporary);
await this.bestEffortSyncDirectory(directory);
} catch (error) {
await handle?.close().catch(() => undefined);
await this.bestEffortUnlink(temporary);
throw error;
}
}
async read(attemptId: string): Promise<CompletionReceipt | undefined> {
const target = this.target(attemptId);
let handle: fs.FileHandle;
try {
handle = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
if (isCode(error, 'ELOOP')) {
throw new InvalidCompletionReceiptError(
'Completion receipt must not be a symbolic link',
);
}
throw error;
}
try {
const stat = await handle.stat();
if (!stat.isFile()) {
throw new InvalidCompletionReceiptError(
'Completion receipt must be a regular file',
);
}
const bytes = Buffer.allocUnsafe(MAX_COMPLETION_RECEIPT_BYTES + 1);
const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
if (bytesRead > MAX_COMPLETION_RECEIPT_BYTES) {
throw new InvalidCompletionReceiptError(
'Completion receipt exceeds the byte limit',
);
}
const receipt = parseCompletionReceipt(bytes.subarray(0, bytesRead));
if (receipt.attemptId !== attemptId) {
throw new InvalidCompletionReceiptError(
'Completion receipt path and Attempt do not match',
);
}
return receipt;
} finally {
await handle.close();
}
}
async remove(attemptId: string): Promise<boolean> {
const target = this.target(attemptId);
try {
await fs.unlink(target);
await this.bestEffortSyncDirectory(path.dirname(target));
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
async quarantine(attemptId: string): Promise<string | undefined> {
const target = this.target(attemptId);
const reference = this.quarantineReference(attemptId);
const relativeDirectory = path.dirname(reference);
const directory = path.join(this.root, relativeDirectory);
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
await fs.chmod(directory, 0o700);
const quarantined = path.join(this.root, reference);
try {
await fs.link(target, quarantined);
} catch (error) {
if (isCode(error, 'ENOENT')) {
return (await this.pathExists(quarantined)) ? reference : undefined;
}
if (!isCode(error, 'EEXIST')) throw error;
}
await this.unlinkIfPresent(target);
await this.bestEffortSyncDirectory(path.dirname(target));
await this.bestEffortSyncDirectory(directory);
return reference;
}
quarantineReference(attemptId: string): string {
assertCompletionReceiptId(attemptId, 'attemptId');
return path.posix.join(
'.quarantine',
attemptId.slice(0, 2),
`${attemptId}.json`,
);
}
async purgeQuarantine(attemptId: string): Promise<boolean> {
const target = path.join(this.root, this.quarantineReference(attemptId));
try {
await fs.unlink(target);
await this.bestEffortSyncDirectory(path.dirname(target));
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
private directory(attemptId: string): string {
assertCompletionReceiptId(attemptId, 'attemptId');
return path.join(this.root, attemptId.slice(0, 2));
}
private target(attemptId: string): string {
return path.join(this.directory(attemptId), `${attemptId}.json`);
}
private async bestEffortUnlink(value: string): Promise<void> {
try {
await fs.unlink(value);
} catch (error) {
if (!isCode(error, 'ENOENT')) {
// Temp cleanup is diagnostic-only; the immutable final fact wins.
}
}
}
private async unlinkIfPresent(value: string): Promise<void> {
try {
await fs.unlink(value);
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
}
}
private async pathExists(value: string): Promise<boolean> {
try {
await fs.lstat(value);
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
throw error;
}
}
private async bestEffortSyncDirectory(directory: string): Promise<void> {
try {
const handle = await fs.open(directory, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
} catch {
// Some supported filesystems cannot fsync directories. The receipt
// remains restart-safe, while power-loss durability is best effort.
}
}
}
@@ -0,0 +1,302 @@
import { createHash } from 'crypto';
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { assertCompletionReceiptId } from '../../domain/completionReceipt';
import type {
CompletionReceiptDirectoryEntry,
CompletionReceiptDirectoryEntryKind,
CompletionReceiptOrphanDirectory,
CompletionReceiptOrphanQuarantineResult,
CompletionReceiptShardSnapshot,
} from '../../ports/completionReceiptOrphanMaintenance';
const SHARD_PATTERN = /^[0-9a-f]{2}$/;
const TEMPORARY_PATTERN = /^\.([0-9a-f-]{36})\.[0-9a-f]{32}\.tmp$/;
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
function receiptAttemptId(name: string): string | undefined {
if (!name.endsWith('.json')) return undefined;
const attemptId = name.slice(0, -'.json'.length);
try {
assertCompletionReceiptId(attemptId, 'attemptId');
return attemptId;
} catch {
return undefined;
}
}
function temporaryAttemptId(name: string): string | undefined {
const match = TEMPORARY_PATTERN.exec(name);
if (!match) return undefined;
try {
assertCompletionReceiptId(match[1], 'attemptId');
return match[1];
} catch {
return undefined;
}
}
function entryKind(
shard: string,
name: string,
regularFile: boolean,
): { kind: CompletionReceiptDirectoryEntryKind; attemptId?: string } {
if (!regularFile) return { kind: 'unsafe' };
const finalAttemptId = receiptAttemptId(name);
if (finalAttemptId && finalAttemptId.startsWith(shard)) {
return { kind: 'receipt', attemptId: finalAttemptId };
}
const tempAttemptId = temporaryAttemptId(name);
if (tempAttemptId && tempAttemptId.startsWith(shard)) {
return { kind: 'temporary', attemptId: tempAttemptId };
}
return { kind: 'unknown' };
}
function filesystemIdentity(
stat: Awaited<ReturnType<typeof fs.lstat>>,
): string {
return [stat.dev, stat.ino, stat.size, stat.mtimeMs].join(':');
}
export class CompletionReceiptOrphanFileDirectory
implements CompletionReceiptOrphanDirectory
{
constructor(private readonly root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new RangeError(
'Completion receipt orphan root must be an absolute path containing no NUL',
);
}
if (path.resolve(root) === path.parse(path.resolve(root)).root) {
throw new RangeError(
'Completion receipt orphan root must not be a filesystem root',
);
}
}
async inspectShard(
shard: string,
maxEntries: number,
): Promise<CompletionReceiptShardSnapshot> {
if (!SHARD_PATTERN.test(shard)) {
throw new RangeError(
'Completion receipt shard must be two lowercase hex digits',
);
}
if (
!Number.isSafeInteger(maxEntries) ||
maxEntries < 1 ||
maxEntries > 64
) {
throw new RangeError('maxEntries must be between 1 and 64');
}
const directoryPath = await this.resolveShardDirectory(shard);
if (!directoryPath) return { shard, entries: [], overflow: false };
const directory = await fs.opendir(directoryPath, { bufferSize: 1 });
const entries: CompletionReceiptDirectoryEntry[] = [];
let overflow = false;
try {
for await (const dirent of directory) {
if (entries.length === maxEntries) {
overflow = true;
break;
}
const entryPath = path.join(directoryPath, dirent.name);
let stat: Awaited<ReturnType<typeof fs.lstat>>;
try {
stat = await fs.lstat(entryPath);
} catch (error) {
if (isCode(error, 'ENOENT')) continue;
throw error;
}
const classified = entryKind(shard, dirent.name, stat.isFile());
entries.push({
shard,
name: dirent.name,
kind: classified.kind,
modifiedAtMs: Math.max(0, Math.trunc(stat.mtimeMs)),
sizeBytes: stat.size,
filesystemIdentity: filesystemIdentity(stat),
...(classified.attemptId ? { attemptId: classified.attemptId } : {}),
});
}
} finally {
await directory.close().catch((error) => {
if (!isCode(error, 'ERR_DIR_CLOSED')) throw error;
});
}
return { shard, entries, overflow };
}
async quarantine(
entry: CompletionReceiptDirectoryEntry,
): Promise<CompletionReceiptOrphanQuarantineResult> {
if (
!SHARD_PATTERN.test(entry.shard) ||
path.basename(entry.name) !== entry.name
) {
throw new RangeError('Completion receipt orphan entry path is invalid');
}
if (entry.kind === 'unsafe') return { status: 'changed' };
const shardDirectory = await this.resolveShardDirectory(entry.shard);
if (!shardDirectory) return { status: 'changed' };
const source = path.join(shardDirectory, entry.name);
let current: Awaited<ReturnType<typeof fs.lstat>>;
try {
current = await fs.lstat(source);
} catch (error) {
if (isCode(error, 'ENOENT')) return { status: 'changed' };
throw error;
}
if (
!current.isFile() ||
filesystemIdentity(current) !== entry.filesystemIdentity
) {
return { status: 'changed' };
}
const digest = createHash('sha256')
.update(`${entry.shard}/${entry.name}\0${entry.filesystemIdentity}`)
.digest('hex');
const reference = path.posix.join(
'.orphan-quarantine',
entry.shard,
`${digest}.entry`,
);
const canonicalRoot = path.dirname(shardDirectory);
const directory = await this.ensureQuarantineDirectory(
canonicalRoot,
entry.shard,
);
const target = path.join(directory, `${digest}.entry`);
let linkedByThisCall = false;
try {
await fs.link(source, target);
linkedByThisCall = true;
} catch (error) {
if (!isCode(error, 'EEXIST')) {
if (isCode(error, 'ENOENT')) return { status: 'changed' };
throw error;
}
const targetStat = await fs.lstat(target);
const sourceStat = await fs.lstat(source).catch(() => undefined);
if (
!sourceStat ||
targetStat.dev !== sourceStat.dev ||
targetStat.ino !== sourceStat.ino
) {
return { status: 'changed' };
}
}
let verifiedShardDirectory: string | undefined;
try {
verifiedShardDirectory = await this.resolveShardDirectory(entry.shard);
} catch (error) {
if (linkedByThisCall) await fs.unlink(target).catch(() => undefined);
throw error;
}
const verifiedSource = await fs.lstat(source).catch(() => undefined);
if (
verifiedShardDirectory !== shardDirectory ||
!verifiedSource ||
filesystemIdentity(verifiedSource) !== entry.filesystemIdentity
) {
if (linkedByThisCall) await fs.unlink(target).catch(() => undefined);
return { status: 'changed' };
}
await fs.unlink(source);
await this.bestEffortSync(path.dirname(source));
await this.bestEffortSync(directory);
return { status: 'quarantined', reference };
}
private async resolveShardDirectory(
shard: string,
): Promise<string | undefined> {
let canonicalRoot: string;
try {
canonicalRoot = await fs.realpath(this.root);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw error;
}
const candidate = path.join(canonicalRoot, shard);
let stat: Awaited<ReturnType<typeof fs.lstat>>;
try {
stat = await fs.lstat(candidate);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw error;
}
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new Error(
`Completion receipt shard ${shard} is not a safe directory`,
);
}
const canonicalDirectory = await fs.realpath(candidate);
if (
path.dirname(canonicalDirectory) !== canonicalRoot ||
path.basename(canonicalDirectory) !== shard
) {
throw new Error(`Completion receipt shard ${shard} escapes its root`);
}
return canonicalDirectory;
}
private async ensureQuarantineDirectory(
canonicalRoot: string,
shard: string,
): Promise<string> {
const quarantineRoot = path.join(canonicalRoot, '.orphan-quarantine');
await this.ensurePrivateDirectory(quarantineRoot, canonicalRoot);
const directory = path.join(quarantineRoot, shard);
await this.ensurePrivateDirectory(directory, quarantineRoot);
return directory;
}
private async ensurePrivateDirectory(
directory: string,
expectedParent: string,
): Promise<void> {
try {
await fs.mkdir(directory, { mode: 0o700 });
} catch (error) {
if (!isCode(error, 'EEXIST')) throw error;
}
const stat = await fs.lstat(directory);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new Error(
'Completion receipt quarantine path is not a safe directory',
);
}
const canonicalDirectory = await fs.realpath(directory);
if (path.dirname(canonicalDirectory) !== expectedParent) {
throw new Error('Completion receipt quarantine path escapes its root');
}
await fs.chmod(directory, 0o700);
}
private async bestEffortSync(directory: string): Promise<void> {
try {
const handle = await fs.open(directory, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
} catch {
// Directory fsync is unavailable on some supported filesystems.
}
}
}
@@ -0,0 +1,122 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { normalizeLocalArtifactReadRange } from '../../domain/artifactRead';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type {
AvailableLocalArtifactByteRange,
LocalArtifactByteRangeReadResult,
LocalArtifactByteRangeReader as LocalArtifactByteRangeReaderPort,
} from '../../ports/localArtifactByteRangeReader';
export class UnsafeLocalArtifactReadTargetError extends Error {
constructor() {
super('Local Artifact read target is unsafe');
this.name = 'UnsafeLocalArtifactReadTargetError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class LocalArtifactByteRangeReader
implements LocalArtifactByteRangeReaderPort
{
private readonly root: string;
constructor(root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact read root must be absolute');
}
this.root = path.resolve(root);
}
async read(
logArtifactId: string,
requestedRange: Parameters<LocalArtifactByteRangeReaderPort['read']>[1],
): Promise<LocalArtifactByteRangeReadResult> {
assertLocalExecutionArtifactId(logArtifactId);
const range = normalizeLocalArtifactReadRange(requestedRange);
const directory = path.join(this.root, logArtifactId.slice(6, 8));
await this.assertDirectory(this.root);
if (!(await this.optionalDirectory(directory)))
return { status: 'missing' };
const target = path.join(directory, `${logArtifactId}.log`);
let handle;
try {
handle = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return { status: 'missing' };
throw new UnsafeLocalArtifactReadTargetError();
}
try {
const stat = await handle.stat();
if (!stat.isFile() || !Number.isSafeInteger(stat.size) || stat.size < 0) {
throw new UnsafeLocalArtifactReadTargetError();
}
const start = Math.min(range.offset, stat.size);
const expected = Math.min(range.length, stat.size - start);
const content = Buffer.allocUnsafe(expected);
let read = 0;
while (read < expected) {
const result = await handle.read(
content,
read,
expected - read,
start + read,
);
if (result.bytesRead < 1) {
throw new UnsafeLocalArtifactReadTargetError();
}
read += result.bytesRead;
}
const endExclusive = start + expected;
const result: AvailableLocalArtifactByteRange = {
status: 'available',
content,
start,
endExclusive,
totalBytes: stat.size,
...(endExclusive < stat.size ? { nextOffset: endExclusive } : {}),
};
return Object.freeze(result);
} finally {
await handle.close();
}
}
private async assertDirectory(value: string): Promise<void> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactReadTargetError();
}
} catch (error) {
if (error instanceof UnsafeLocalArtifactReadTargetError) throw error;
throw new UnsafeLocalArtifactReadTargetError();
}
}
private async optionalDirectory(value: string): Promise<boolean> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactReadTargetError();
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof UnsafeLocalArtifactReadTargetError) throw error;
throw new UnsafeLocalArtifactReadTargetError();
}
}
}
@@ -0,0 +1,179 @@
import { constants, type Stats } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import { MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES } from '../../domain/localArtifactTruncation';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type {
LocalArtifactFileRetirementResult,
LocalArtifactFileRetirementStore as LocalArtifactFileRetirementStorePort,
} from '../../ports/localArtifactFileRetirementStore';
import { localArtifactTruncationFactFileName } from './localArtifactTruncationFactStore';
export class UnsafeLocalArtifactRetirementError extends Error {
constructor() {
super('Local Artifact retirement target is unsafe');
this.name = 'UnsafeLocalArtifactRetirementError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class LocalArtifactFileRetirementStore
implements LocalArtifactFileRetirementStorePort
{
private readonly root: string;
constructor(root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact retirement root must be absolute');
}
this.root = path.resolve(root);
}
async retire(
logArtifactId: string,
): Promise<LocalArtifactFileRetirementResult> {
assertLocalExecutionArtifactId(logArtifactId);
const directory = path.join(this.root, logArtifactId.slice(6, 8));
await this.assertDirectory(this.root);
if (!(await this.optionalDirectory(directory))) {
return Object.freeze({
disposition: 'already_absent',
bytesReclaimed: 0,
});
}
const target = path.join(directory, `${logArtifactId}.log`);
const fifo = path.join(directory, `.${logArtifactId}.log.fifo`);
const truncation = path.join(
directory,
localArtifactTruncationFactFileName(logArtifactId),
);
const truncationTemporary = path.join(
directory,
`.${logArtifactId}.log.truncated.tmp`,
);
const [targetStat, fifoStat, truncationStat, truncationTemporaryStat] =
await Promise.all([
this.lstat(target),
this.lstat(fifo),
this.lstat(truncation),
this.lstat(truncationTemporary),
]);
if (targetStat && (!targetStat.isFile() || targetStat.isSymbolicLink())) {
throw new UnsafeLocalArtifactRetirementError();
}
if (
targetStat &&
(!Number.isSafeInteger(targetStat.size) || targetStat.size < 0)
) {
throw new UnsafeLocalArtifactRetirementError();
}
if (fifoStat && (!fifoStat.isFIFO() || fifoStat.isSymbolicLink())) {
throw new UnsafeLocalArtifactRetirementError();
}
for (const stat of [truncationStat, truncationTemporaryStat]) {
if (
stat &&
(!stat.isFile() ||
stat.isSymbolicLink() ||
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES)
) {
throw new UnsafeLocalArtifactRetirementError();
}
}
let targetDeleted = false;
let bytesReclaimed = 0;
if (targetStat) {
try {
await fs.unlink(target);
targetDeleted = true;
bytesReclaimed = targetStat.size;
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
}
}
let auxiliaryRemoved = false;
for (const [auxiliary, stat] of [
[fifo, fifoStat],
[truncation, truncationStat],
[truncationTemporary, truncationTemporaryStat],
] as const) {
if (!stat) continue;
try {
await fs.unlink(auxiliary);
auxiliaryRemoved = true;
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
}
}
if (targetDeleted || auxiliaryRemoved) {
await this.syncDirectory(directory);
}
if (!targetDeleted) {
return Object.freeze({
disposition: 'already_absent',
bytesReclaimed: 0,
});
}
return Object.freeze({
disposition: 'deleted',
bytesReclaimed,
});
}
private async assertDirectory(value: string): Promise<void> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactRetirementError();
}
} catch (error) {
if (error instanceof UnsafeLocalArtifactRetirementError) throw error;
throw new UnsafeLocalArtifactRetirementError();
}
}
private async optionalDirectory(value: string): Promise<boolean> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactRetirementError();
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof UnsafeLocalArtifactRetirementError) throw error;
throw new UnsafeLocalArtifactRetirementError();
}
}
private async lstat(value: string): Promise<Stats | null> {
try {
return await fs.lstat(value);
} catch (error) {
if (isCode(error, 'ENOENT')) return null;
throw error;
}
}
private async syncDirectory(directory: string): Promise<void> {
const handle = await fs.open(
directory,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
try {
await handle.sync();
} finally {
await handle.close();
}
}
}
@@ -0,0 +1,113 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import {
decodeLocalArtifactTruncationFact,
MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES,
type LocalArtifactTruncationFact,
} from '../../domain/localArtifactTruncation';
import { assertLocalExecutionArtifactId } from '../../domain/localExecutionArtifact';
import type { LocalArtifactTruncationFactStore as LocalArtifactTruncationFactStorePort } from '../../ports/localArtifactTruncationFactStore';
export function localArtifactTruncationFactFileName(
logArtifactId: string,
): string {
assertLocalExecutionArtifactId(logArtifactId);
return `.${logArtifactId}.log.truncated.json`;
}
export class UnsafeLocalArtifactTruncationFactError extends Error {
constructor() {
super('Local Artifact truncation fact target is unsafe');
this.name = 'UnsafeLocalArtifactTruncationFactError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class LocalArtifactTruncationFactStore
implements LocalArtifactTruncationFactStorePort
{
private readonly root: string;
constructor(root: string) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact truncation root must be absolute');
}
this.root = path.resolve(root);
}
async read(
logArtifactId: string,
): Promise<Readonly<LocalArtifactTruncationFact> | null> {
assertLocalExecutionArtifactId(logArtifactId);
const directory = path.join(this.root, logArtifactId.slice(6, 8));
await this.assertDirectory(this.root);
if (!(await this.optionalDirectory(directory))) return null;
const target = path.join(
directory,
localArtifactTruncationFactFileName(logArtifactId),
);
let handle;
try {
handle = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return null;
throw new UnsafeLocalArtifactTruncationFactError();
}
try {
const stat = await handle.stat();
if (
!stat.isFile() ||
!Number.isSafeInteger(stat.size) ||
stat.size < 1 ||
stat.size > MAX_LOCAL_ARTIFACT_TRUNCATION_FACT_BYTES
) {
throw new UnsafeLocalArtifactTruncationFactError();
}
const fact = decodeLocalArtifactTruncationFact(await handle.readFile());
if (fact.logArtifactId !== logArtifactId) {
throw new UnsafeLocalArtifactTruncationFactError();
}
return fact;
} finally {
await handle.close();
}
}
private async assertDirectory(value: string): Promise<void> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactTruncationFactError();
}
} catch (error) {
if (error instanceof UnsafeLocalArtifactTruncationFactError) throw error;
throw new UnsafeLocalArtifactTruncationFactError();
}
}
private async optionalDirectory(value: string): Promise<boolean> {
try {
const stat = await fs.lstat(value);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new UnsafeLocalArtifactTruncationFactError();
}
return true;
} catch (error) {
if (isCode(error, 'ENOENT')) return false;
if (error instanceof UnsafeLocalArtifactTruncationFactError) throw error;
throw new UnsafeLocalArtifactTruncationFactError();
}
}
}
@@ -0,0 +1,175 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import type {
ExecutionOutputChunk,
ExecutionOutputSink,
} from '../../domain/execution';
import {
LocalArtifactCapacityUnavailableError,
LocalArtifactQuotaExceededError,
normalizeLocalArtifactCapacityPolicy,
type LocalArtifactCapacityPolicy,
} from '../../domain/localArtifactCapacity';
import {
localExecutionArtifactId,
assertLocalExecutionArtifactId,
} from '../../domain/localExecutionArtifact';
import type { RunDispatchCandidate } from '../../domain/runDispatchCandidate';
import type {
LocalExecutionArtifactAllocator,
PreparedLocalExecutionArtifact,
} from '../../ports/localExecutionArtifactAllocator';
import type { LocalArtifactCapacityProbe } from '../../ports/localArtifactCapacityProbe';
import { LocalFileSystemCapacityProbe } from './localFileSystemCapacityProbe';
import { enableDurableLocalProcessOutput } from '../local-process/durableLocalProcessOutput';
async function privateDirectory(directory: string): Promise<void> {
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
const stat = await fs.lstat(directory);
if (!stat.isDirectory() || stat.isSymbolicLink()) {
throw new TypeError('Local execution artifact directory is unsafe');
}
await fs.chmod(directory, 0o700);
}
class LocalFileExecutionOutput implements ExecutionOutputSink {
private pending: Promise<unknown> = Promise.resolve();
private closed = false;
private remainingBytes: number;
constructor(
private readonly file: fs.FileHandle,
maximumBytes: number,
existingBytes: number,
) {
this.remainingBytes = maximumBytes - existingBytes;
}
write(output: ExecutionOutputChunk): Promise<void> {
if (this.closed) {
return Promise.reject(new Error('Local execution artifact is closed'));
}
const chunk = Buffer.from(output.chunk);
const operation = this.pending.then(async () => {
if (this.remainingBytes <= 0) {
throw new LocalArtifactQuotaExceededError();
}
const accepted = chunk.subarray(
0,
Math.min(chunk.length, this.remainingBytes),
);
let written = 0;
while (written < accepted.length) {
const result = await this.file.write(accepted.subarray(written));
if (result.bytesWritten < 1) {
throw new Error('Local execution artifact write made no progress');
}
written += result.bytesWritten;
this.remainingBytes -= result.bytesWritten;
}
if (accepted.length !== chunk.length) {
throw new LocalArtifactQuotaExceededError();
}
});
this.pending = operation.catch(() => undefined);
return operation;
}
async close(): Promise<void> {
if (this.closed) return;
this.closed = true;
await this.pending.catch(() => undefined);
await this.file.close();
}
}
export class LocalFileExecutionArtifactAllocator
implements LocalExecutionArtifactAllocator
{
private readonly artifactRoot: string;
private readonly completionReceiptRoot: string;
private readonly policy: Readonly<LocalArtifactCapacityPolicy>;
private readonly capacity: LocalArtifactCapacityProbe;
constructor(
artifactRoot: string,
completionReceiptRoot: string,
policy: LocalArtifactCapacityPolicy,
capacity: LocalArtifactCapacityProbe = new LocalFileSystemCapacityProbe(),
) {
if (
!path.isAbsolute(artifactRoot) ||
artifactRoot.includes('\0') ||
!path.isAbsolute(completionReceiptRoot) ||
completionReceiptRoot.includes('\0')
) {
throw new TypeError('Local execution artifact roots must be absolute');
}
this.artifactRoot = path.resolve(artifactRoot);
this.completionReceiptRoot = path.resolve(completionReceiptRoot);
this.policy = normalizeLocalArtifactCapacityPolicy(policy);
this.capacity = capacity;
}
async prepare(
candidate: Readonly<RunDispatchCandidate>,
): Promise<PreparedLocalExecutionArtifact> {
const logArtifactId = localExecutionArtifactId(candidate);
assertLocalExecutionArtifactId(logArtifactId);
const shard = logArtifactId.slice('local-'.length, 'local-'.length + 2);
const directory = path.join(this.artifactRoot, shard);
await privateDirectory(this.artifactRoot);
const capacity = await this.capacity.inspect(this.artifactRoot);
const requiredBytes =
BigInt(this.policy.minimumFreeBytes) +
BigInt(this.policy.maximumAttemptBytes);
if (capacity.availableBytes < requiredBytes) {
throw new LocalArtifactCapacityUnavailableError();
}
await privateDirectory(directory);
await privateDirectory(this.completionReceiptRoot);
const outputFilePath = path.join(directory, `${logArtifactId}.log`);
const file = await fs.open(
outputFilePath,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_APPEND |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
try {
const stat = await file.stat();
if (!stat.isFile()) {
throw new TypeError('Local execution artifact target is unsafe');
}
if (
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > this.policy.maximumAttemptBytes
) {
throw new LocalArtifactQuotaExceededError();
}
await file.chmod(0o600);
const output = new LocalFileExecutionOutput(
file,
this.policy.maximumAttemptBytes,
stat.size,
);
return {
logArtifactId,
output: enableDurableLocalProcessOutput(output, {
outputFilePath,
completionReceiptRoot: this.completionReceiptRoot,
maximumBytes: this.policy.maximumAttemptBytes,
logArtifactId,
}),
dispose: () => output.close(),
};
} catch (error) {
await file.close().catch(() => undefined);
throw error;
}
}
}
@@ -0,0 +1,68 @@
import fs from 'fs/promises';
import path from 'path';
import { LocalArtifactCapacityUnavailableError } from '../../domain/localArtifactCapacity';
import type {
LocalArtifactCapacityProbe,
LocalArtifactCapacitySnapshot,
LocalArtifactCapacitySource,
} from '../../ports/localArtifactCapacityProbe';
interface BigIntStatFs {
bavail: bigint;
blocks: bigint;
bsize: bigint;
}
export class RootedLocalFileSystemCapacitySource
implements LocalArtifactCapacitySource
{
private readonly root: string;
constructor(
root: string,
private readonly probe: LocalArtifactCapacityProbe = new LocalFileSystemCapacityProbe(),
) {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new TypeError('Local Artifact capacity root must be absolute');
}
this.root = path.resolve(root);
}
inspect(): Promise<LocalArtifactCapacitySnapshot> {
return this.probe.inspect(this.root);
}
}
interface StatFsPromises {
statfs(value: string, options: { bigint: true }): Promise<BigIntStatFs>;
}
export class LocalFileSystemCapacityProbe
implements LocalArtifactCapacityProbe
{
async inspect(root: string): Promise<LocalArtifactCapacitySnapshot> {
if (!path.isAbsolute(root) || root.includes('\0')) {
throw new LocalArtifactCapacityUnavailableError();
}
try {
const statfs = (fs as unknown as StatFsPromises).statfs;
if (typeof statfs !== 'function') {
throw new LocalArtifactCapacityUnavailableError();
}
const stat = await statfs.call(fs, root, { bigint: true });
const availableBytes = stat.bavail * stat.bsize;
const totalBytes = stat.blocks * stat.bsize;
if (
availableBytes < BigInt(0) ||
totalBytes < BigInt(1) ||
availableBytes > totalBytes
) {
throw new LocalArtifactCapacityUnavailableError();
}
return Object.freeze({ availableBytes, totalBytes });
} catch (error) {
if (error instanceof LocalArtifactCapacityUnavailableError) throw error;
throw new LocalArtifactCapacityUnavailableError();
}
}
}
@@ -0,0 +1,160 @@
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import {
LocalSecretUnavailableError,
assertLocalSecretKeyId,
} from '../../domain/localSecret';
import type {
LocalSecretKeyMaterial,
LocalSecretKeyProvider,
} from '../../ports/localSecretKeyProvider';
const MAX_KEYRING_BYTES = 16 * 1024;
const MAX_KEY_COUNT = 16;
const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/;
interface LocalSecretKeyringManifest {
version: 1;
activeKeyId: string;
keys: Record<string, string>;
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const actual = Object.keys(value).sort();
return (
actual.length === expected.length &&
actual.every((key, index) => key === expected[index])
);
}
function parseManifest(value: Buffer): LocalSecretKeyringManifest {
let parsed: unknown;
try {
parsed = JSON.parse(value.toString('utf8'));
} catch {
throw new LocalSecretUnavailableError();
}
if (
!parsed ||
typeof parsed !== 'object' ||
Array.isArray(parsed) ||
!exactKeys(parsed, ['activeKeyId', 'keys', 'version'])
) {
throw new LocalSecretUnavailableError();
}
const record = parsed as Record<string, unknown>;
if (
record.version !== 1 ||
!record.keys ||
typeof record.keys !== 'object' ||
Array.isArray(record.keys)
) {
throw new LocalSecretUnavailableError();
}
assertLocalSecretKeyId(record.activeKeyId as string);
const entries = Object.entries(record.keys as Record<string, unknown>);
if (entries.length < 1 || entries.length > MAX_KEY_COUNT) {
throw new LocalSecretUnavailableError();
}
const keys: Record<string, string> = Object.create(null);
for (const [keyId, encoded] of entries) {
assertLocalSecretKeyId(keyId);
const decoded =
typeof encoded === 'string'
? Buffer.from(encoded, 'base64url')
: Buffer.alloc(0);
if (
typeof encoded !== 'string' ||
!BASE64URL_PATTERN.test(encoded) ||
decoded.length !== 32 ||
decoded.toString('base64url') !== encoded
) {
decoded.fill(0);
throw new LocalSecretUnavailableError();
}
decoded.fill(0);
keys[keyId] = encoded;
}
if (!keys[record.activeKeyId as string]) {
throw new LocalSecretUnavailableError();
}
return {
version: 1,
activeKeyId: record.activeKeyId as string,
keys,
};
}
export class LocalSecretKeyringFileProvider implements LocalSecretKeyProvider {
private readonly filePath: string;
constructor(filePath: string) {
if (!path.isAbsolute(filePath) || filePath.includes('\0')) {
throw new TypeError('Local Secret keyring path must be absolute');
}
this.filePath = path.resolve(filePath);
}
async active(): Promise<LocalSecretKeyMaterial> {
const manifest = await this.read();
return this.material(
manifest,
manifest.activeKeyId,
) as LocalSecretKeyMaterial;
}
async resolve(keyId: string): Promise<LocalSecretKeyMaterial | null> {
try {
assertLocalSecretKeyId(keyId);
} catch {
throw new LocalSecretUnavailableError();
}
const manifest = await this.read();
return this.material(manifest, keyId);
}
private material(
manifest: LocalSecretKeyringManifest,
keyId: string,
): LocalSecretKeyMaterial | null {
const encoded = manifest.keys[keyId];
return encoded
? Object.freeze({
keyId,
key: Uint8Array.from(Buffer.from(encoded, 'base64url')),
})
: null;
}
private async read(): Promise<LocalSecretKeyringManifest> {
let file: fs.FileHandle | undefined;
let contents: Buffer | undefined;
try {
file = await fs.open(
this.filePath,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
const stat = await file.stat();
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
(stat.mode & 0o077) !== 0 ||
stat.size < 1 ||
stat.size > MAX_KEYRING_BYTES
) {
throw new LocalSecretUnavailableError();
}
contents = await file.readFile();
if (contents.length !== stat.size) {
throw new LocalSecretUnavailableError();
}
return parseManifest(contents);
} catch {
throw new LocalSecretUnavailableError();
} finally {
contents?.fill(0);
await file?.close().catch(() => undefined);
}
}
}

Some files were not shown because too many files have changed in this diff Show More