v0.3.8

internal/backend/forwarder/codebase_index_store.go

@@ -0,0 +1,606 @@
+package forwarder
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const (
+	codebaseIndexStatusReady              = "ready"
+	codebaseIndexFileStage                = "indexed"
+	codebaseIndexEventKindRegister        = "register"
+	codebaseIndexSubscriberSignalCapacity = 1
+	codebaseIndexMaxPersistedEvents       = 2000
+	codebaseIndexSourceRepositoryService  = "repository_service"
+	repositoryIndexStatusReady            = "ready"
+	repositoryIndexSyncStatusCompleted    = "sync_completed"
+	repositoryIndexCopyStatusCompleted    = "copy_completed"
+)
+
+type CodebaseSearchBackend interface {
+	EnsureIndexed(record CodebaseIndexRecord) (CodebaseIndexRecord, error)
+	RegisterFile(indexID string, file CodebaseIndexFileRecord) (CodebaseIndexFileRecord, error)
+	ListFiles(indexID string) ([]CodebaseIndexFileRecord, error)
+}
+
+type CodebaseIndexStore struct {
+	mu          sync.Mutex
+	root        string
+	path        string
+	loaded      bool
+	nextEventID int64
+	state       codebaseIndexState
+	subscribers map[string]map[string]chan struct{}
+}
+
+type codebaseIndexState struct {
+	SchemaVersion int                            `json:"schema_version"`
+	Indexes       map[string]CodebaseIndexRecord `json:"indexes"`
+	Events        []CodebaseIndexEventRecord     `json:"events,omitempty"`
+}
+
+type CodebaseIndexRecord struct {
+	IndexID           string                             `json:"index_id"`
+	Repo              string                             `json:"repo,omitempty"`
+	TargetDir         string                             `json:"target_dir,omitempty"`
+	Files             []string                           `json:"files,omitempty"`
+	Status            string                             `json:"status"`
+	Source            string                             `json:"source,omitempty"`
+	DependenciesReady bool                               `json:"dependencies_ready,omitempty"`
+	TopoSortReady     bool                               `json:"topo_sort_ready,omitempty"`
+	RepositoryState   *RepositoryIndexStateRecord        `json:"repository_state,omitempty"`
+	CreatedAt         time.Time                          `json:"created_at"`
+	UpdatedAt         time.Time                          `json:"updated_at"`
+	RegisteredFiles   map[string]CodebaseIndexFileRecord `json:"registered_files,omitempty"`
+}
+
+type RepositoryIndexStateRecord struct {
+	CodebaseID       string    `json:"codebase_id"`
+	Repo             string    `json:"repo,omitempty"`
+	TargetDir        string    `json:"target_dir,omitempty"`
+	Status           string    `json:"status"`
+	CopyStatus       string    `json:"copy_status,omitempty"`
+	SyncStatus       string    `json:"sync_status,omitempty"`
+	CopyTaskHandle   string    `json:"copy_task_handle,omitempty"`
+	PathKeyHash      string    `json:"path_key_hash,omitempty"`
+	Source           string    `json:"source"`
+	LastHandshakeAt  time.Time `json:"last_handshake_at,omitempty"`
+	LastSyncAt       time.Time `json:"last_sync_at,omitempty"`
+	LastCopyStatusAt time.Time `json:"last_copy_status_at,omitempty"`
+	CreatedAt        time.Time `json:"created_at"`
+	UpdatedAt        time.Time `json:"updated_at"`
+}
+
+type CodebaseIndexFileRecord struct {
+	WorkspaceRelativePath string    `json:"workspace_relative_path"`
+	ContentHash           string    `json:"content_hash,omitempty"`
+	Stage                 string    `json:"stage"`
+	Order                 int32     `json:"order,omitempty"`
+	NodeCount             int32     `json:"node_count,omitempty"`
+	CreatedAt             time.Time `json:"created_at"`
+	UpdatedAt             time.Time `json:"updated_at"`
+}
+
+type CodebaseIndexEventRecord struct {
+	ID      int64                   `json:"id"`
+	IndexID string                  `json:"index_id"`
+	Kind    string                  `json:"kind"`
+	ReqUUID string                  `json:"req_uuid,omitempty"`
+	File    CodebaseIndexFileRecord `json:"file,omitempty"`
+	At      time.Time               `json:"at"`
+}
+
+type CodebaseIndexSubscription struct {
+	IndexID      string
+	SubscriberID string
+	Signal       <-chan struct{}
+}
+
+func NewCodebaseIndexStore(root string) *CodebaseIndexStore {
+	trimmed := strings.TrimSpace(root)
+	if trimmed == "" {
+		trimmed = "codebase-index"
+	}
+	return &CodebaseIndexStore{
+		root:        trimmed,
+		path:        filepath.Join(trimmed, "index.json"),
+		subscribers: make(map[string]map[string]chan struct{}),
+		state: codebaseIndexState{
+			SchemaVersion: 1,
+			Indexes:       make(map[string]CodebaseIndexRecord),
+		},
+	}
+}
+
+func (store *CodebaseIndexStore) EnsureIndexed(record CodebaseIndexRecord) (CodebaseIndexRecord, error) {
+	if store == nil {
+		return CodebaseIndexRecord{}, fmt.Errorf("codebase index store is nil")
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	if err := store.loadLocked(); err != nil {
+		return CodebaseIndexRecord{}, err
+	}
+	now := time.Now().UTC()
+	indexID := strings.TrimSpace(record.IndexID)
+	if indexID == "" {
+		indexID = stableCodebaseIndexID(record.Repo, record.TargetDir, record.Files)
+	}
+	existing, ok := store.state.Indexes[indexID]
+	if ok {
+		existing.Repo = firstNonEmptyCodebaseIndex(record.Repo, existing.Repo)
+		existing.TargetDir = firstNonEmptyCodebaseIndex(record.TargetDir, existing.TargetDir)
+		if len(record.Files) > 0 {
+			existing.Files = append([]string(nil), record.Files...)
+		}
+		if strings.TrimSpace(record.Source) != "" {
+			existing.Source = strings.TrimSpace(record.Source)
+		}
+		if record.RepositoryState != nil {
+			existing.RepositoryState = mergeRepositoryIndexState(existing.RepositoryState, record.RepositoryState, now)
+		}
+		existing.Status = codebaseIndexStatusReady
+		existing.UpdatedAt = now
+		if existing.RegisteredFiles == nil {
+			existing.RegisteredFiles = make(map[string]CodebaseIndexFileRecord)
+		}
+		store.state.Indexes[indexID] = existing
+		return existing, store.saveLocked()
+	}
+	record.IndexID = indexID
+	record.Status = codebaseIndexStatusReady
+	record.Source = strings.TrimSpace(record.Source)
+	record.CreatedAt = now
+	record.UpdatedAt = now
+	if record.RepositoryState != nil {
+		record.RepositoryState = mergeRepositoryIndexState(nil, record.RepositoryState, now)
+	}
+	record.Files = append([]string(nil), record.Files...)
+	record.RegisteredFiles = make(map[string]CodebaseIndexFileRecord)
+	store.state.Indexes[indexID] = record
+	return record, store.saveLocked()
+}
+
+func (store *CodebaseIndexStore) EnsureRepositoryIndexed(state RepositoryIndexStateRecord) (CodebaseIndexRecord, error) {
+	if store == nil {
+		return CodebaseIndexRecord{}, fmt.Errorf("codebase index store is nil")
+	}
+	codebaseID := strings.TrimSpace(state.CodebaseID)
+	if codebaseID == "" {
+		return CodebaseIndexRecord{}, fmt.Errorf("codebase_id is required")
+	}
+	state.CodebaseID = codebaseID
+	state.Source = codebaseIndexSourceRepositoryService
+	if strings.TrimSpace(state.Status) == "" {
+		state.Status = repositoryIndexStatusReady
+	}
+	return store.EnsureIndexed(CodebaseIndexRecord{
+		IndexID:         codebaseID,
+		Repo:            strings.TrimSpace(state.Repo),
+		TargetDir:       strings.TrimSpace(state.TargetDir),
+		Source:          codebaseIndexSourceRepositoryService,
+		RepositoryState: &state,
+	})
+}
+
+func (store *CodebaseIndexStore) MarkRepositorySyncComplete(codebaseID string, pathKeyHash string) error {
+	_, err := store.EnsureRepositoryIndexed(RepositoryIndexStateRecord{
+		CodebaseID:  strings.TrimSpace(codebaseID),
+		Status:      repositoryIndexStatusReady,
+		SyncStatus:  repositoryIndexSyncStatusCompleted,
+		PathKeyHash: strings.TrimSpace(pathKeyHash),
+		LastSyncAt:  time.Now().UTC(),
+	})
+	return err
+}
+
+func (store *CodebaseIndexStore) MarkRepositoryCopyComplete(codebaseID string, copyTaskHandle string) error {
+	_, err := store.EnsureRepositoryIndexed(RepositoryIndexStateRecord{
+		CodebaseID:       strings.TrimSpace(codebaseID),
+		Status:           repositoryIndexStatusReady,
+		CopyStatus:       repositoryIndexCopyStatusCompleted,
+		CopyTaskHandle:   strings.TrimSpace(copyTaskHandle),
+		LastCopyStatusAt: time.Now().UTC(),
+	})
+	return err
+}
+
+func (store *CodebaseIndexStore) RegisterFile(indexID string, file CodebaseIndexFileRecord) (CodebaseIndexFileRecord, error) {
+	registered, _, err := store.RegisterFileWithRequest(indexID, "", file)
+	return registered, err
+}
+
+func (store *CodebaseIndexStore) RegisterFileWithRequest(indexID string, reqUUID string, file CodebaseIndexFileRecord) (CodebaseIndexFileRecord, CodebaseIndexEventRecord, error) {
+	if store == nil {
+		return CodebaseIndexFileRecord{}, CodebaseIndexEventRecord{}, fmt.Errorf("codebase index store is nil")
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	if err := store.loadLocked(); err != nil {
+		return CodebaseIndexFileRecord{}, CodebaseIndexEventRecord{}, err
+	}
+	trimmedIndexID := strings.TrimSpace(indexID)
+	if trimmedIndexID == "" {
+		return CodebaseIndexFileRecord{}, CodebaseIndexEventRecord{}, fmt.Errorf("index_id is required")
+	}
+	record, ok := store.state.Indexes[trimmedIndexID]
+	if !ok {
+		now := time.Now().UTC()
+		record = CodebaseIndexRecord{
+			IndexID:         trimmedIndexID,
+			Status:          codebaseIndexStatusReady,
+			CreatedAt:       now,
+			UpdatedAt:       now,
+			RegisteredFiles: make(map[string]CodebaseIndexFileRecord),
+		}
+	}
+	if record.RegisteredFiles == nil {
+		record.RegisteredFiles = make(map[string]CodebaseIndexFileRecord)
+	}
+	path := strings.TrimSpace(file.WorkspaceRelativePath)
+	if path == "" {
+		return CodebaseIndexFileRecord{}, CodebaseIndexEventRecord{}, fmt.Errorf("workspace_relative_path is required")
+	}
+	now := time.Now().UTC()
+	current, exists := record.RegisteredFiles[path]
+	if exists {
+		file.CreatedAt = current.CreatedAt
+	} else {
+		file.CreatedAt = now
+	}
+	file.WorkspaceRelativePath = path
+	if strings.TrimSpace(file.Stage) == "" {
+		file.Stage = codebaseIndexFileStage
+	}
+	if file.Order == 0 {
+		if exists && current.Order != 0 {
+			file.Order = current.Order
+		} else {
+			file.Order = int32(len(record.RegisteredFiles) + 1)
+		}
+	}
+	file.UpdatedAt = now
+	record.RegisteredFiles[path] = file
+	record.Status = codebaseIndexStatusReady
+	record.UpdatedAt = now
+	store.state.Indexes[trimmedIndexID] = record
+	event := CodebaseIndexEventRecord{
+		ID:      store.nextCodebaseIndexEventIDLocked(),
+		IndexID: trimmedIndexID,
+		Kind:    codebaseIndexEventKindRegister,
+		ReqUUID: strings.TrimSpace(reqUUID),
+		File:    file,
+		At:      now,
+	}
+	store.state.Events = append(store.state.Events, event)
+	store.compactEventsLocked()
+	if err := store.saveLocked(); err != nil {
+		return CodebaseIndexFileRecord{}, CodebaseIndexEventRecord{}, err
+	}
+	store.notifyIndexSubscribersLocked(trimmedIndexID)
+	return file, event, nil
+}
+
+func (store *CodebaseIndexStore) ListFiles(indexID string) ([]CodebaseIndexFileRecord, error) {
+	if store == nil {
+		return nil, fmt.Errorf("codebase index store is nil")
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	if err := store.loadLocked(); err != nil {
+		return nil, err
+	}
+	record, ok := store.state.Indexes[strings.TrimSpace(indexID)]
+	if !ok || len(record.RegisteredFiles) == 0 {
+		return nil, nil
+	}
+	files := make([]CodebaseIndexFileRecord, 0, len(record.RegisteredFiles))
+	for _, file := range record.RegisteredFiles {
+		files = append(files, file)
+	}
+	sort.Slice(files, func(i, j int) bool {
+		if files[i].Order != files[j].Order {
+			return files[i].Order < files[j].Order
+		}
+		return files[i].WorkspaceRelativePath < files[j].WorkspaceRelativePath
+	})
+	return files, nil
+}
+
+func (store *CodebaseIndexStore) ListEvents(indexID string, afterEventID int64) ([]CodebaseIndexEventRecord, error) {
+	if store == nil {
+		return nil, fmt.Errorf("codebase index store is nil")
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	if err := store.loadLocked(); err != nil {
+		return nil, err
+	}
+	trimmedIndexID := strings.TrimSpace(indexID)
+	if trimmedIndexID == "" {
+		return nil, fmt.Errorf("index_id is required")
+	}
+	events := make([]CodebaseIndexEventRecord, 0)
+	for _, event := range store.state.Events {
+		if event.ID <= afterEventID || event.IndexID != trimmedIndexID {
+			continue
+		}
+		events = append(events, event)
+	}
+	return events, nil
+}
+
+func (store *CodebaseIndexStore) Subscribe(indexID string) (CodebaseIndexSubscription, error) {
+	if store == nil {
+		return CodebaseIndexSubscription{}, fmt.Errorf("codebase index store is nil")
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	if err := store.loadLocked(); err != nil {
+		return CodebaseIndexSubscription{}, err
+	}
+	trimmedIndexID := strings.TrimSpace(indexID)
+	if trimmedIndexID == "" {
+		return CodebaseIndexSubscription{}, fmt.Errorf("index_id is required")
+	}
+	if store.subscribers == nil {
+		store.subscribers = make(map[string]map[string]chan struct{})
+	}
+	subscriberID := uuid.NewString()
+	signal := make(chan struct{}, codebaseIndexSubscriberSignalCapacity)
+	if store.subscribers[trimmedIndexID] == nil {
+		store.subscribers[trimmedIndexID] = make(map[string]chan struct{})
+	}
+	store.subscribers[trimmedIndexID][subscriberID] = signal
+	return CodebaseIndexSubscription{
+		IndexID:      trimmedIndexID,
+		SubscriberID: subscriberID,
+		Signal:       signal,
+	}, nil
+}
+
+func (store *CodebaseIndexStore) Unsubscribe(subscription CodebaseIndexSubscription) {
+	if store == nil {
+		return
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	indexID := strings.TrimSpace(subscription.IndexID)
+	subscriberID := strings.TrimSpace(subscription.SubscriberID)
+	if indexID == "" || subscriberID == "" {
+		return
+	}
+	byIndex := store.subscribers[indexID]
+	if byIndex == nil {
+		return
+	}
+	delete(byIndex, subscriberID)
+	if len(byIndex) == 0 {
+		delete(store.subscribers, indexID)
+	}
+}
+
+func (store *CodebaseIndexStore) MarkDependenciesReady(indexID string) error {
+	return store.updateIndex(indexID, func(record *CodebaseIndexRecord) {
+		record.DependenciesReady = true
+	})
+}
+
+func (store *CodebaseIndexStore) MarkTopoSortReady(indexID string) error {
+	return store.updateIndex(indexID, func(record *CodebaseIndexRecord) {
+		record.TopoSortReady = true
+	})
+}
+
+func (store *CodebaseIndexStore) updateIndex(indexID string, update func(*CodebaseIndexRecord)) error {
+	if store == nil {
+		return fmt.Errorf("codebase index store is nil")
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	if err := store.loadLocked(); err != nil {
+		return err
+	}
+	trimmedIndexID := strings.TrimSpace(indexID)
+	if trimmedIndexID == "" {
+		return fmt.Errorf("index_id is required")
+	}
+	now := time.Now().UTC()
+	record, ok := store.state.Indexes[trimmedIndexID]
+	if !ok {
+		record = CodebaseIndexRecord{
+			IndexID:         trimmedIndexID,
+			Status:          codebaseIndexStatusReady,
+			CreatedAt:       now,
+			RegisteredFiles: make(map[string]CodebaseIndexFileRecord),
+		}
+	}
+	if record.RegisteredFiles == nil {
+		record.RegisteredFiles = make(map[string]CodebaseIndexFileRecord)
+	}
+	update(&record)
+	record.Status = codebaseIndexStatusReady
+	record.UpdatedAt = now
+	store.state.Indexes[trimmedIndexID] = record
+	return store.saveLocked()
+}
+
+func (store *CodebaseIndexStore) loadLocked() error {
+	if store.loaded {
+		return nil
+	}
+	state := codebaseIndexState{SchemaVersion: 1, Indexes: make(map[string]CodebaseIndexRecord)}
+	data, err := os.ReadFile(store.path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			store.state = state
+			store.nextEventID = 1
+			store.loaded = true
+			return nil
+		}
+		return err
+	}
+	if len(data) != 0 {
+		if err := json.Unmarshal(data, &state); err != nil {
+			return err
+		}
+	}
+	if state.SchemaVersion <= 0 {
+		state.SchemaVersion = 1
+	}
+	if state.Indexes == nil {
+		state.Indexes = make(map[string]CodebaseIndexRecord)
+	}
+	var maxEventID int64
+	for id, record := range state.Indexes {
+		if record.RegisteredFiles == nil {
+			record.RegisteredFiles = make(map[string]CodebaseIndexFileRecord)
+			state.Indexes[id] = record
+		}
+	}
+	for _, event := range state.Events {
+		if event.ID > maxEventID {
+			maxEventID = event.ID
+		}
+	}
+	store.state = state
+	store.nextEventID = maxEventID + 1
+	store.loaded = true
+	return nil
+}
+
+func (store *CodebaseIndexStore) saveLocked() error {
+	if err := os.MkdirAll(store.root, 0o755); err != nil {
+		return err
+	}
+	data, err := json.MarshalIndent(store.state, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(store.root, ".index-*.json.tmp")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	defer func() {
+		_ = os.Remove(tmpPath)
+	}()
+	if _, err := tmp.Write(data); err != nil {
+		_ = tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	if err := os.Chmod(tmpPath, 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, store.path)
+}
+
+func (store *CodebaseIndexStore) nextCodebaseIndexEventIDLocked() int64 {
+	if store.nextEventID <= 0 {
+		store.nextEventID = 1
+	}
+	eventID := store.nextEventID
+	store.nextEventID++
+	return eventID
+}
+
+func (store *CodebaseIndexStore) notifyIndexSubscribersLocked(indexID string) {
+	if store.subscribers == nil {
+		return
+	}
+	for _, signal := range store.subscribers[strings.TrimSpace(indexID)] {
+		select {
+		case signal <- struct{}{}:
+		default:
+		}
+	}
+}
+
+func (store *CodebaseIndexStore) compactEventsLocked() {
+	if len(store.state.Events) <= codebaseIndexMaxPersistedEvents {
+		return
+	}
+	store.state.Events = append([]CodebaseIndexEventRecord(nil), store.state.Events[len(store.state.Events)-codebaseIndexMaxPersistedEvents:]...)
+}
+
+func stableCodebaseIndexID(repo string, targetDir string, files []string) string {
+	parts := append([]string{strings.TrimSpace(repo), strings.TrimSpace(targetDir)}, files...)
+	normalized := make([]string, 0, len(parts))
+	for _, part := range parts {
+		trimmed := strings.TrimSpace(part)
+		if trimmed != "" {
+			normalized = append(normalized, trimmed)
+		}
+	}
+	if len(normalized) == 0 {
+		return "idx_local"
+	}
+	sort.Strings(normalized)
+	sum := sha256.Sum256([]byte(strings.Join(normalized, "\x00")))
+	return "idx_" + hex.EncodeToString(sum[:])[:24]
+}
+
+func codebaseFileContentHash(content string) string {
+	if content == "" {
+		return ""
+	}
+	sum := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(sum[:])
+}
+
+func firstNonEmptyCodebaseIndex(values ...string) string {
+	for _, value := range values {
+		trimmed := strings.TrimSpace(value)
+		if trimmed != "" {
+			return trimmed
+		}
+	}
+	return ""
+}
+
+func mergeRepositoryIndexState(existing *RepositoryIndexStateRecord, next *RepositoryIndexStateRecord, now time.Time) *RepositoryIndexStateRecord {
+	if next == nil {
+		return existing
+	}
+	merged := RepositoryIndexStateRecord{}
+	if existing != nil {
+		merged = *existing
+	}
+	merged.CodebaseID = firstNonEmptyCodebaseIndex(next.CodebaseID, merged.CodebaseID)
+	merged.Repo = firstNonEmptyCodebaseIndex(next.Repo, merged.Repo)
+	merged.TargetDir = firstNonEmptyCodebaseIndex(next.TargetDir, merged.TargetDir)
+	merged.Status = firstNonEmptyCodebaseIndex(next.Status, merged.Status, repositoryIndexStatusReady)
+	merged.CopyStatus = firstNonEmptyCodebaseIndex(next.CopyStatus, merged.CopyStatus)
+	merged.SyncStatus = firstNonEmptyCodebaseIndex(next.SyncStatus, merged.SyncStatus)
+	merged.CopyTaskHandle = firstNonEmptyCodebaseIndex(next.CopyTaskHandle, merged.CopyTaskHandle)
+	merged.PathKeyHash = firstNonEmptyCodebaseIndex(next.PathKeyHash, merged.PathKeyHash)
+	merged.Source = firstNonEmptyCodebaseIndex(next.Source, merged.Source, codebaseIndexSourceRepositoryService)
+	merged.LastHandshakeAt = firstNonZeroCodebaseIndexTime(next.LastHandshakeAt, merged.LastHandshakeAt)
+	merged.LastSyncAt = firstNonZeroCodebaseIndexTime(next.LastSyncAt, merged.LastSyncAt)
+	merged.LastCopyStatusAt = firstNonZeroCodebaseIndexTime(next.LastCopyStatusAt, merged.LastCopyStatusAt)
+	merged.CreatedAt = firstNonZeroCodebaseIndexTime(merged.CreatedAt, next.CreatedAt, now)
+	merged.UpdatedAt = now
+	return &merged
+}
+
+func firstNonZeroCodebaseIndexTime(values ...time.Time) time.Time {
+	for _, value := range values {
+		if !value.IsZero() {
+			return value.UTC()
+		}
+	}
+	return time.Time{}
+}