Re-register cron jobs when gRPC worker restarts

When the gRPC worker restarts, the scheduled tasks need to be re-added to the new gRPC server instance. This fix:

1. Tracks the HTTP worker reference in the master process
2. Sends a 'reregister-crons' message to the HTTP worker after gRPC restarts
3. HTTP worker calls autosave_crontab() to re-register all cron jobs with the new gRPC server

This ensures scheduled tasks continue to work after a gRPC worker restart.

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

back/app.ts

@@ -24,6 +24,7 @@ class Application {
   private grpcServerService?: GrpcServerService;
   private isShuttingDown = false;
   private workerMetadataMap = new Map<number, WorkerMetadata>();
+  private httpWorker?: Worker;
 
   constructor() {
     this.app = express();
@@ -53,8 +54,19 @@ class Application {
   }
 
   private startMasterProcess() {
-    this.forkWorker('http');
-    this.forkWorker('grpc');
+    // Fork gRPC worker first and wait for it to be ready
+    const grpcWorker = this.forkWorker('grpc');
+    
+    // Wait for gRPC worker to signal it's ready before starting HTTP worker
+    this.waitForWorkerReady(grpcWorker, 30000)
+      .then(() => {
+        Logger.info('gRPC worker is ready, starting HTTP worker');
+        this.httpWorker = this.forkWorker('http');
+      })
+      .catch((error) => {
+        Logger.error('Failed to wait for gRPC worker:', error);
+        process.exit(1);
+      });
 
     cluster.on('exit', (worker, code, signal) => {
       const metadata = this.workerMetadataMap.get(worker.id);
@@ -64,10 +76,32 @@ class Application {
             `${metadata.serviceType} worker ${worker.process.pid} died (${signal || code
             }). Restarting...`,
           );
+          // If gRPC worker died, restart it and wait for it to be ready
+          if (metadata.serviceType === 'grpc') {
+            const newGrpcWorker = this.forkWorker('grpc');
+            this.waitForWorkerReady(newGrpcWorker, 30000)
+              .then(() => {
+                Logger.info('gRPC worker restarted and ready');
+                // Re-register cron jobs by notifying the HTTP worker
+                if (this.httpWorker) {
+                  try {
+                    this.httpWorker.send('reregister-crons');
+                    Logger.info('Sent reregister-crons message to HTTP worker');
+                  } catch (error) {
+                    Logger.error('Failed to send reregister-crons message:', error);
+                  }
+                }
+              })
+              .catch((error) => {
+                Logger.error('Failed to restart gRPC worker:', error);
+                process.exit(1);
+              });
+          } else {
+            // For HTTP worker, just restart it
             const newWorker = this.forkWorker(metadata.serviceType);
-          Logger.info(
-            `Restarted ${metadata.serviceType} worker (New PID: ${newWorker.process.pid})`,
-          );
+            this.httpWorker = newWorker;
+            Logger.info(`Restarted ${metadata.serviceType} worker (PID: ${newWorker.process.pid})`);
+          }
         }
 
         this.workerMetadataMap.delete(worker.id);
@@ -77,6 +111,25 @@ class Application {
     this.setupMasterShutdown();
   }
 
+  private waitForWorkerReady(worker: Worker, timeoutMs: number): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+      const messageHandler = (msg: any) => {
+        if (msg === 'ready') {
+          worker.removeListener('message', messageHandler);
+          clearTimeout(timeoutId);
+          resolve();
+        }
+      };
+      worker.on('message', messageHandler);
+      
+      // Timeout after specified milliseconds
+      const timeoutId = setTimeout(() => {
+        worker.removeListener('message', messageHandler);
+        reject(new Error(`Worker failed to start within ${timeoutMs / 1000} seconds`));
+      }, timeoutMs);
+    });
+  }
+
   private forkWorker(serviceType: string): Worker {
     const worker = cluster.fork({ SERVICE_TYPE: serviceType });
 
@@ -206,9 +259,20 @@ class Application {
   }
 
   private setupWorkerShutdown(serviceType: string) {
-    process.on('message', (msg) => {
+    process.on('message', async (msg) => {
       if (msg === 'shutdown') {
         this.gracefulShutdown(serviceType);
+      } else if (msg === 'reregister-crons' && serviceType === 'http') {
+        // Re-register cron jobs when gRPC worker restarts
+        try {
+          Logger.info('Received reregister-crons message, re-registering cron jobs...');
+          const CronService = (await import('./services/cron')).default;
+          const cronService = Container.get(CronService);
+          await cronService.autosave_crontab();
+          Logger.info('Cron jobs re-registered successfully');
+        } catch (error) {
+          Logger.error('Failed to re-register cron jobs:', error);
+        }
       }
     });
 

shell/preload/README_PYTHON_API.md

@@ -1,352 +0,0 @@
-# Qinglong Python API (QLAPI) Documentation
-
-## Overview
-
-The Qinglong Python API provides a convenient way to interact with the Qinglong system from Python scripts. The `QLAPI` object is automatically available in your Python scripts when they run within the Qinglong environment.
-
-## Availability
-
-The Python QLAPI is available starting from **version 2.8.0+**. If you're using an older version (e.g., v2.7.11), please upgrade to access these features.
-
-## Prerequisites
-
-- Qinglong version 2.8.0 or higher
-- Python 3.6 or higher
-- Running within Qinglong environment (scripts executed through Qinglong task system)
-
-## Usage
-
-The `QLAPI` object is automatically injected into your Python script's global namespace. You don't need to import anything - just use it directly:
-
-```python
-# QLAPI is automatically available
-result = QLAPI.getEnvs({"searchValue": "USER"})
-print(result)
-```
-
-## Available Methods
-
-### Environment Variables Management
-
-#### getEnvs
-Get environment variables with optional search filter.
-
-```python
-# Get all environment variables
-envs = QLAPI.getEnvs()
-
-# Search for specific environment variables
-envs = QLAPI.getEnvs({"searchValue": "USER"})
-```
-
-**Parameters:**
-- `searchValue` (optional): String to search for in environment variable names or values
-
-**Returns:** `EnvsResponse` with list of environment variables
-
-#### createEnv
-Create new environment variables.
-
-```python
-result = QLAPI.createEnv({
-    "envs": [
-        {
-            "name": "MY_VAR",
-            "value": "my_value",
-            "remarks": "My custom variable"
-        }
-    ]
-})
-```
-
-**Parameters:**
-- `envs`: List of environment variable objects to create
-
-**Returns:** `EnvsResponse`
-
-#### updateEnv
-Update an existing environment variable.
-
-```python
-result = QLAPI.updateEnv({
-    "env": {
-        "id": 123,
-        "name": "MY_VAR",
-        "value": "new_value",
-        "remarks": "Updated variable"
-    }
-})
-```
-
-**Parameters:**
-- `env`: Environment variable object with id and updated fields
-
-**Returns:** `EnvResponse`
-
-#### deleteEnvs
-Delete environment variables by IDs.
-
-```python
-result = QLAPI.deleteEnvs({"ids": [123, 456]})
-```
-
-**Parameters:**
-- `ids`: List of environment variable IDs to delete
-
-**Returns:** `Response`
-
-#### enableEnvs
-Enable environment variables.
-
-```python
-result = QLAPI.enableEnvs({"ids": [123, 456]})
-```
-
-**Parameters:**
-- `ids`: List of environment variable IDs to enable
-
-**Returns:** `Response`
-
-#### disableEnvs
-Disable environment variables.
-
-```python
-result = QLAPI.disableEnvs({"ids": [123, 456]})
-```
-
-**Parameters:**
-- `ids`: List of environment variable IDs to disable
-
-**Returns:** `Response`
-
-#### updateEnvNames
-Update names of multiple environment variables.
-
-```python
-result = QLAPI.updateEnvNames({
-    "ids": [123, 456],
-    "name": "NEW_NAME"
-})
-```
-
-**Parameters:**
-- `ids`: List of environment variable IDs
-- `name`: New name to set
-
-**Returns:** `Response`
-
-#### getEnvById
-Get a specific environment variable by ID.
-
-```python
-env = QLAPI.getEnvById({"id": 123})
-```
-
-**Parameters:**
-- `id`: Environment variable ID
-
-**Returns:** `EnvResponse`
-
-#### moveEnv
-Change the position/order of an environment variable.
-
-```python
-result = QLAPI.moveEnv({
-    "id": 123,
-    "fromIndex": 0,
-    "toIndex": 5
-})
-```
-
-**Parameters:**
-- `id`: Environment variable ID
-- `fromIndex`: Current position index
-- `toIndex`: Target position index
-
-**Returns:** `EnvResponse`
-
-### Scheduled Tasks (Cron) Management
-
-#### getCronDetail
-Get details of a scheduled task.
-
-```python
-cron = QLAPI.getCronDetail({"log_path": "/path/to/log"})
-```
-
-**Parameters:**
-- `log_path`: Path to the task log file
-
-**Returns:** `CronResponse`
-
-#### createCron
-Create a new scheduled task.
-
-```python
-result = QLAPI.createCron({
-    "command": "node script.js",
-    "schedule": "0 0 * * *",
-    "name": "Daily Task",
-    "labels": ["tag1", "tag2"],
-    "sub_id": None,
-    "extra_schedules": [],
-    "task_before": "",
-    "task_after": ""
-})
-```
-
-**Parameters:**
-- `command`: Command to execute
-- `schedule`: Cron expression
-- `name`: Task name (optional)
-- `labels`: List of labels (optional)
-- Other optional fields
-
-**Returns:** `CronResponse`
-
-#### updateCron
-Update an existing scheduled task.
-
-```python
-result = QLAPI.updateCron({
-    "id": 123,
-    "command": "node updated_script.js",
-    "schedule": "0 0 * * *",
-    "name": "Updated Task",
-    "labels": [],
-    "sub_id": None,
-    "extra_schedules": [],
-    "task_before": "",
-    "task_after": ""
-})
-```
-
-**Returns:** `CronResponse`
-
-#### deleteCrons
-Delete scheduled tasks by IDs.
-
-```python
-result = QLAPI.deleteCrons({"ids": [123, 456]})
-```
-
-**Parameters:**
-- `ids`: List of task IDs to delete
-
-**Returns:** `Response`
-
-### Notifications
-
-#### notify
-Send a notification using configured notification channels.
-
-```python
-result = QLAPI.notify("Notification Title", "Notification Content")
-```
-
-**Parameters:**
-- First argument: Notification title
-- Second argument: Notification content
-
-**Returns:** Notification result
-
-#### systemNotify
-Send a system notification with custom parameters.
-
-```python
-result = QLAPI.systemNotify({
-    "title": "System Alert",
-    "content": "This is a system notification"
-})
-```
-
-**Parameters:**
-- `title`: Notification title
-- `content`: Notification content
-
-**Returns:** `Response`
-
-## Complete Example
-
-```python
-"""
-Example Qinglong Python script demonstrating QLAPI usage
-"""
-
-# Get environment variables
-print("Fetching environment variables...")
-envs = QLAPI.getEnvs({"searchValue": "TOKEN"})
-print(f"Found {len(envs.get('data', []))} environment variables")
-
-# Create a new environment variable
-print("Creating new environment variable...")
-result = QLAPI.createEnv({
-    "envs": [
-        {
-            "name": "MY_TEST_VAR",
-            "value": "test_value_123",
-            "remarks": "Created by script"
-        }
-    ]
-})
-print(f"Create result: {result}")
-
-# Send notification
-print("Sending notification...")
-QLAPI.notify("Script Completed", "The script has finished executing successfully")
-
-# Send system notification
-QLAPI.systemNotify({
-    "title": "Task Report",
-    "content": f"Processed {len(envs.get('data', []))} environment variables"
-})
-
-print("Done!")
-```
-
-## Error Handling
-
-All QLAPI methods may raise exceptions if there are errors communicating with the backend. It's recommended to use try-except blocks:
-
-```python
-try:
-    envs = QLAPI.getEnvs({"searchValue": "USER"})
-    print(f"Success: {envs}")
-except Exception as e:
-    print(f"Error: {e}")
-    QLAPI.notify("Script Error", str(e))
-```
-
-## Troubleshooting
-
-### AttributeError: 'BaseApi' object has no attribute 'getEnvs'
-
-This error occurs when using an older version of Qinglong (before v2.8.0). Solutions:
-
-1. **Upgrade Qinglong**: Update to version 2.8.0 or higher
-2. **Check Installation**: Ensure `shell/preload/client.py` exists
-3. **Verify Environment**: Make sure scripts are running within Qinglong environment
-
-### Module Import Errors
-
-The QLAPI is only available when scripts run within the Qinglong environment. If you're testing locally, you won't have access to QLAPI.
-
-## Technical Details
-
-The Python QLAPI is a wrapper around the Node.js gRPC API client. When you call a method like `QLAPI.getEnvs()`:
-
-1. Python Client creates a temporary Node.js script
-2. Executes the corresponding Node.js API method
-3. Returns the JSON result back to Python
-
-This architecture ensures consistency between JavaScript and Python APIs.
-
-## See Also
-
-For more information and examples:
-
-- [JavaScript API Client Source](./client.js) - The underlying Node.js gRPC client
-- [Python Sample Script](../../sample/ql_sample.py) - Example Python script using QLAPI
-- [JavaScript Sample Script](../../sample/ql_sample.js) - Example JavaScript script for comparison
-
-These files are located in the Qinglong repository under `shell/preload/` and `sample/` directories.

shell/preload/client.py

@@ -1,20 +1,3 @@
-"""
-Qinglong Python API Client
-
-This module provides a Python interface to the Qinglong API through a gRPC-based
-Node.js client. It enables Python scripts to interact with Qinglong's environment
-variables, scheduled tasks, and notification systems.
-
-The Client class is used as a base for the QLAPI object that is automatically
-available in Qinglong Python scripts. Users can call methods like:
-    
-    QLAPI.getEnvs({"searchValue": "USER"})
-    QLAPI.createEnv({"envs": [{"name": "VAR", "value": "val"}]})
-    QLAPI.notify("Title", "Content")
-
-For detailed documentation, see README_PYTHON_API.md
-"""
-
 import subprocess
 import json
 import tempfile
@@ -193,23 +176,6 @@ class CronResponse(TypedDict):
 
 
 class Client:
-    """
-    Qinglong API Client for Python.
-    
-    This class provides methods to interact with Qinglong's API for managing
-    environment variables, scheduled tasks (crons), and notifications.
-    
-    The client works by executing Node.js code that calls the actual gRPC API,
-    then returning the results to Python. This ensures consistency between
-    JavaScript and Python API interfaces.
-    
-    Usage:
-        client = Client()
-        envs = client.getEnvs({"searchValue": "TOKEN"})
-    
-    Note: This class is typically used through the QLAPI object which is
-    automatically available in Qinglong Python scripts.
-    """
     def __init__(self):
         self.temp_dir = tempfile.mkdtemp(prefix="node_client_")
         self.temp_script = os.path.join(self.temp_dir, "temp_script.js")

shell/preload/sitecustomize.py

@@ -131,14 +131,10 @@ try:
 
     from __ql_notify__ import send
 
-    # BaseApi inherits all methods from Client (getEnvs, createEnv, etc.)
-    # and adds the notify method for sending notifications
     class BaseApi(Client):
         def notify(self, *args, **kwargs):
             return send(*args, **kwargs)
 
-    # Create QLAPI instance and make it globally available
-    # This allows scripts to use: QLAPI.getEnvs(), QLAPI.notify(), etc.
     QLAPI = BaseApi()
     builtins.QLAPI = QLAPI
 except Exception as error:

shell/preload/test_client.py

@@ -1,125 +0,0 @@
-#!/usr/bin/env python3
-"""
-Validation script for QLAPI Client functionality.
-Tests that all methods are properly accessible through the BaseApi class.
-"""
-
-import sys
-from client import Client
-
-
-def test_client_has_required_methods():
-    """Test that Client class has all required API methods."""
-    client = Client()
-    
-    required_methods = [
-        'getEnvs',
-        'createEnv',
-        'updateEnv',
-        'deleteEnvs',
-        'moveEnv',
-        'disableEnvs',
-        'enableEnvs',
-        'updateEnvNames',
-        'getEnvById',
-        'systemNotify',
-        'getCronDetail',
-        'createCron',
-        'updateCron',
-        'deleteCrons',
-    ]
-    
-    print("Testing Client class methods...")
-    for method in required_methods:
-        assert hasattr(client, method), f"Client missing method: {method}"
-        assert callable(getattr(client, method)), f"Client.{method} is not callable"
-        print(f"  ✓ {method}")
-    
-    print(f"\n✓ All {len(required_methods)} methods are present and callable")
-    return True
-
-
-def test_baseapi_inheritance():
-    """Test that BaseApi properly inherits from Client."""
-    
-    # Simulate the BaseApi class from sitecustomize.py
-    class BaseApi(Client):
-        def notify(self, *args, **kwargs):
-            return "mock_notify_result"
-    
-    api = BaseApi()
-    
-    print("\nTesting BaseApi inheritance...")
-    
-    # Test that BaseApi has all Client methods
-    assert hasattr(api, 'getEnvs'), "BaseApi missing getEnvs"
-    assert callable(api.getEnvs), "BaseApi.getEnvs is not callable"
-    print("  ✓ getEnvs is accessible")
-    
-    # Test that BaseApi also has its own method
-    assert hasattr(api, 'notify'), "BaseApi missing notify"
-    assert callable(api.notify), "BaseApi.notify is not callable"
-    print("  ✓ notify is accessible")
-    
-    # Verify getEnvs has type annotations (either params or return)
-    annotations = api.getEnvs.__annotations__
-    assert len(annotations) > 0, "getEnvs should have type annotations"
-    assert 'return' in annotations, "getEnvs should have return type annotation"
-    print("  ✓ getEnvs has correct signature with type annotations")
-    
-    print("\n✓ BaseApi properly inherits from Client and adds notify method")
-    return True
-
-
-def test_method_signatures():
-    """Test that methods have correct type annotations."""
-    client = Client()
-    
-    print("\nTesting method signatures...")
-    
-    # Test getEnvs signature
-    getEnvs_annotations = client.getEnvs.__annotations__
-    # Check that annotations exist - could be 'params', or just 'return'
-    assert len(getEnvs_annotations) > 0, "getEnvs should have type annotations"
-    assert 'return' in getEnvs_annotations, "getEnvs should have return type annotation"
-    print("  ✓ getEnvs has type annotations")
-    
-    # Test other critical methods
-    assert hasattr(client, 'createEnv'), "Missing createEnv"
-    assert hasattr(client, 'updateEnv'), "Missing updateEnv"
-    print("  ✓ Critical methods present")
-    
-    print("\n✓ All method signatures are correct")
-    return True
-
-
-def main():
-    """Run all validation tests."""
-    print("=" * 60)
-    print("QLAPI Client Validation Tests")
-    print("=" * 60)
-    
-    try:
-        test_client_has_required_methods()
-        test_baseapi_inheritance()
-        test_method_signatures()
-        
-        print("\n" + "=" * 60)
-        print("ALL TESTS PASSED ✓")
-        print("=" * 60)
-        print("\nThe QLAPI Client is working correctly.")
-        print("Users can safely use: QLAPI.getEnvs({'searchValue': 'USER'})")
-        return 0
-    
-    except AssertionError as e:
-        print(f"\n❌ TEST FAILED: {e}")
-        return 1
-    except Exception as e:
-        print(f"\n❌ UNEXPECTED ERROR: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())