mirror of
https://wget.la/https://github.com/leookun/cursor-byok
synced 2026-08-20 13:07:00 +08:00
v0.3.8
This commit is contained in:
leokun
@@ -0,0 +1,241 @@
|
||||
package forwarder
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
docsIndexStatusIndexed = "indexed"
|
||||
docsIndexSourceLocal = "local_docs"
|
||||
docsIndexSourceAdditional = "additional_docs"
|
||||
)
|
||||
|
||||
type DocsIndexStore struct {
|
||||
mu sync.Mutex
|
||||
root string
|
||||
path string
|
||||
loaded bool
|
||||
state docsIndexState
|
||||
}
|
||||
|
||||
type docsIndexState struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Docs map[string]DocsIndexRecord `json:"docs"`
|
||||
}
|
||||
|
||||
type DocsIndexRecord struct {
|
||||
ID string `json:"id"`
|
||||
Identifier string `json:"identifier"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
GitOrigin string `json:"git_origin,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Source string `json:"source"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LastCrawlAt time.Time `json:"last_crawl_at,omitempty"`
|
||||
LastIndexAt time.Time `json:"last_index_at,omitempty"`
|
||||
}
|
||||
|
||||
func NewDocsIndexStore(root string) *DocsIndexStore {
|
||||
trimmed := strings.TrimSpace(root)
|
||||
if trimmed == "" {
|
||||
trimmed = "docs-index"
|
||||
}
|
||||
return &DocsIndexStore{
|
||||
root: trimmed,
|
||||
path: filepath.Join(trimmed, "index.json"),
|
||||
state: docsIndexState{
|
||||
SchemaVersion: 1,
|
||||
Docs: make(map[string]DocsIndexRecord),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (store *DocsIndexStore) Upsert(record DocsIndexRecord) (DocsIndexRecord, error) {
|
||||
if store == nil {
|
||||
return DocsIndexRecord{}, fmt.Errorf("docs index store is nil")
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if err := store.loadLocked(); err != nil {
|
||||
return DocsIndexRecord{}, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
identifier := strings.TrimSpace(record.Identifier)
|
||||
if identifier == "" {
|
||||
identifier = stableDocsIdentifier(record.Title, record.URL, record.Content, record.GitOrigin)
|
||||
}
|
||||
existing, ok := store.state.Docs[identifier]
|
||||
if ok {
|
||||
existing.Title = firstNonEmptyDocs(record.Title, existing.Title, identifier)
|
||||
existing.URL = firstNonEmptyDocs(record.URL, existing.URL)
|
||||
existing.Content = firstNonEmptyDocs(record.Content, existing.Content)
|
||||
existing.GitOrigin = firstNonEmptyDocs(record.GitOrigin, existing.GitOrigin)
|
||||
existing.Status = docsIndexStatusIndexed
|
||||
existing.Source = firstNonEmptyDocs(record.Source, existing.Source, docsIndexSourceLocal)
|
||||
existing.UpdatedAt = now
|
||||
existing.LastCrawlAt = now
|
||||
existing.LastIndexAt = now
|
||||
store.state.Docs[identifier] = existing
|
||||
return existing, store.saveLocked()
|
||||
}
|
||||
record.ID = firstNonEmptyDocs(record.ID, identifier)
|
||||
record.Identifier = identifier
|
||||
record.Title = firstNonEmptyDocs(record.Title, identifier)
|
||||
record.Status = docsIndexStatusIndexed
|
||||
record.Source = firstNonEmptyDocs(record.Source, docsIndexSourceLocal)
|
||||
record.CreatedAt = now
|
||||
record.UpdatedAt = now
|
||||
record.LastCrawlAt = now
|
||||
record.LastIndexAt = now
|
||||
store.state.Docs[identifier] = record
|
||||
return record, store.saveLocked()
|
||||
}
|
||||
|
||||
func (store *DocsIndexStore) List(gitOrigin string, limit int32) ([]DocsIndexRecord, error) {
|
||||
if store == nil {
|
||||
return nil, fmt.Errorf("docs index store is nil")
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if err := store.loadLocked(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
origin := strings.TrimSpace(gitOrigin)
|
||||
records := make([]DocsIndexRecord, 0, len(store.state.Docs))
|
||||
for _, record := range store.state.Docs {
|
||||
if origin != "" && strings.TrimSpace(record.GitOrigin) != origin {
|
||||
continue
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
sort.SliceStable(records, func(i, j int) bool {
|
||||
if records[i].UpdatedAt.Equal(records[j].UpdatedAt) {
|
||||
return records[i].Identifier < records[j].Identifier
|
||||
}
|
||||
return records[i].UpdatedAt.After(records[j].UpdatedAt)
|
||||
})
|
||||
if limit > 0 && len(records) > int(limit) {
|
||||
records = records[:limit]
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (store *DocsIndexStore) Get(identifier string) (DocsIndexRecord, bool, error) {
|
||||
if store == nil {
|
||||
return DocsIndexRecord{}, false, fmt.Errorf("docs index store is nil")
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if err := store.loadLocked(); err != nil {
|
||||
return DocsIndexRecord{}, false, err
|
||||
}
|
||||
record, ok := store.state.Docs[strings.TrimSpace(identifier)]
|
||||
return record, ok, nil
|
||||
}
|
||||
|
||||
func (store *DocsIndexStore) Remove(identifier string) error {
|
||||
if store == nil {
|
||||
return fmt.Errorf("docs index store is nil")
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if err := store.loadLocked(); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(store.state.Docs, strings.TrimSpace(identifier))
|
||||
return store.saveLocked()
|
||||
}
|
||||
|
||||
func (store *DocsIndexStore) loadLocked() error {
|
||||
if store.loaded {
|
||||
return nil
|
||||
}
|
||||
state := docsIndexState{SchemaVersion: 1, Docs: make(map[string]DocsIndexRecord)}
|
||||
data, err := os.ReadFile(store.path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
store.state = state
|
||||
store.loaded = true
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if len(data) != 0 {
|
||||
if err := json.Unmarshal(data, &state); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if state.SchemaVersion <= 0 {
|
||||
state.SchemaVersion = 1
|
||||
}
|
||||
if state.Docs == nil {
|
||||
state.Docs = make(map[string]DocsIndexRecord)
|
||||
}
|
||||
store.state = state
|
||||
store.loaded = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *DocsIndexStore) saveLocked() error {
|
||||
if err := os.MkdirAll(store.root, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(store.state, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(store.root, ".index-*.json.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
defer func() { _ = os.Remove(tmpPath) }()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Chmod(tmpPath, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmpPath, store.path)
|
||||
}
|
||||
|
||||
func stableDocsIdentifier(values ...string) string {
|
||||
parts := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed != "" {
|
||||
parts = append(parts, trimmed)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "doc_local"
|
||||
}
|
||||
sum := sha256.Sum256([]byte(strings.Join(parts, "\x00")))
|
||||
return "doc_" + hex.EncodeToString(sum[:])[:24]
|
||||
}
|
||||
|
||||
func firstNonEmptyDocs(values ...string) string {
|
||||
for _, value := range values {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user