cursor-byok/internal/backend/agent/bridge/interaction/baidusearch.go

package interaction

import (
	"net/http"
	neturl "net/url"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"cursor/gen/agentv1"
)

const (
	baiduWebSearchBaseURL     = "https://www.baidu.com/s?ie=utf-8&tn=baidu&wd="
	baiduWebSearchHostURL     = "https://www.baidu.com"
	baiduSearchAbstractLimit  = 300
	baiduSearchReferenceLimit = 8
)

// extractBaiduWebSearchReferences 从百度搜索结果页 HTML 中解析出搜索结果列表。
func extractBaiduWebSearchReferences(body string) []*agentv1.WebSearchReference {
	document, err := goquery.NewDocumentFromReader(strings.NewReader(body))
	if err != nil {
		return nil
	}
	references := make([]*agentv1.WebSearchReference, 0, baiduSearchReferenceLimit)
	document.Find("#content_left > div").EachWithBreak(func(_ int, selection *goquery.Selection) bool {
		if len(references) >= baiduSearchReferenceLimit {
			return false
		}
		if !selection.HasClass("c-container") {
			return true
		}
		title, resultURL, abstract := extractBaiduSearchResult(selection)
		if title == "" || resultURL == "" {
			return true
		}
		references = append(references, &agentv1.WebSearchReference{
			Title: title,
			Url:   normalizeBaiduSearchURL(resultURL),
			Chunk: truncateBaiduSearchAbstract(abstract),
		})
		return true
	})
	return references
}

// extractBaiduSearchResult 从单条百度搜索结果节点中提取标题、链接和摘要。
func extractBaiduSearchResult(selection *goquery.Selection) (string, string, string) {
	title := cleanBaiduSearchText(selection.Find("h3").First().Text())
	resultURL, _ := selection.Find("h3 a").First().Attr("href")
	if title == "" {
		title = firstBaiduSearchLine(selection.Text())
	}
	if resultURL == "" {
		resultURL, _ = selection.Find("a").First().Attr("href")
	}
	abstract := cleanBaiduSearchText(selection.Find(".c-abstract").First().Text())
	if abstract == "" {
		abstract = cleanBaiduSearchText(selection.ChildrenFiltered("div").First().Text())
	}
	if abstract == "" {
		abstract = baiduSearchTextAfterFirstLine(selection.Text())
	}
	return title, strings.TrimSpace(resultURL), abstract
}

// normalizeBaiduSearchURL 把百度返回的相对或协议省略链接归一化为绝对 URL。
func normalizeBaiduSearchURL(rawURL string) string {
	rawURL = strings.TrimSpace(rawURL)
	if strings.HasPrefix(rawURL, "//") {
		return "https:" + rawURL
	}
	if strings.HasPrefix(rawURL, "/") {
		return baiduWebSearchHostURL + rawURL
	}
	return rawURL
}

// resolveBaiduWebSearchRedirects 把百度跳转链接解析为最终目标地址，就地更新引用列表。
func resolveBaiduWebSearchRedirects(client *http.Client, references []*agentv1.WebSearchReference) {
	for _, reference := range references {
		if reference == nil {
			continue
		}
		reference.Url = resolveBaiduRedirectURL(client, reference.GetUrl())
	}
}

// resolveBaiduRedirectURL 判断链接是否是百度跳转链接，并尝试解析出真实目标。
func resolveBaiduRedirectURL(client *http.Client, rawURL string) string {
	resultURL := normalizeBaiduSearchURL(rawURL)
	if !isBaiduRedirectURL(resultURL) {
		return resultURL
	}
	redirectClient := baiduRedirectHTTPClient(client)
	if location := requestBaiduRedirectLocation(redirectClient, http.MethodHead, resultURL); location != "" {
		return location
	}
	if location := requestBaiduRedirectLocation(redirectClient, http.MethodGet, resultURL); location != "" {
		return location
	}
	return resultURL
}

// baiduRedirectHTTPClient 基于基础 client 构造一个不自动跟随重定向的短超时客户端。
func baiduRedirectHTTPClient(base *http.Client) *http.Client {
	if base == nil {
		base = http.DefaultClient
	}
	client := *base
	if client.Timeout == 0 || client.Timeout > 6*time.Second {
		client.Timeout = 6 * time.Second
	}
	client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
		return http.ErrUseLastResponse
	}
	return &client
}

// requestBaiduRedirectLocation 发起一次请求并读取响应头中的重定向目标地址。
func requestBaiduRedirectLocation(client *http.Client, method string, rawURL string) string {
	request, err := http.NewRequest(method, rawURL, nil)
	if err != nil {
		return ""
	}
	request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0 Safari/537.36")
	request.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
	request.Header.Set("Referer", baiduWebSearchHostURL+"/")
	response, err := client.Do(request)
	if err != nil {
		return ""
	}
	defer response.Body.Close()
	location := strings.TrimSpace(response.Header.Get("Location"))
	if location == "" {
		return ""
	}
	return resolveBaiduLocationURL(rawURL, location)
}

// resolveBaiduLocationURL 把响应头里的相对重定向地址解析为绝对地址。
func resolveBaiduLocationURL(baseURL string, location string) string {
	parsedLocation, err := neturl.Parse(location)
	if err != nil {
		return location
	}
	if parsedLocation.IsAbs() {
		return parsedLocation.String()
	}
	parsedBase, err := neturl.Parse(baseURL)
	if err != nil {
		return location
	}
	return parsedBase.ResolveReference(parsedLocation).String()
}

// isBaiduRedirectURL 判断给定地址是否是百度域名下的跳转链接。
func isBaiduRedirectURL(rawURL string) bool {
	parsedURL, err := neturl.Parse(strings.TrimSpace(rawURL))
	if err != nil {
		return false
	}
	host := strings.ToLower(parsedURL.Hostname())
	path := strings.ToLower(parsedURL.EscapedPath())
	return (host == "baidu.com" || strings.HasSuffix(host, ".baidu.com")) && strings.HasPrefix(path, "/link")
}

// truncateBaiduSearchAbstract 按字符数截断摘要文本，避免结果过长。
func truncateBaiduSearchAbstract(value string) string {
	value = cleanBaiduSearchText(value)
	if baiduSearchAbstractLimit <= 0 {
		return value
	}
	runes := []rune(value)
	if len(runes) <= baiduSearchAbstractLimit {
		return value
	}
	return string(runes[:baiduSearchAbstractLimit])
}

// firstBaiduSearchLine 返回文本中第一个非空行。
func firstBaiduSearchLine(value string) string {
	for _, line := range strings.Split(strings.ReplaceAll(value, "\r", "\n"), "\n") {
		line = cleanBaiduSearchText(line)
		if line != "" {
			return line
		}
	}
	return ""
}

// baiduSearchTextAfterFirstLine 返回除第一个非空行外剩余文本的拼接结果。
func baiduSearchTextAfterFirstLine(value string) string {
	nonEmpty := make([]string, 0, 8)
	for _, line := range strings.Split(strings.ReplaceAll(value, "\r", "\n"), "\n") {
		line = cleanBaiduSearchText(line)
		if line != "" {
			nonEmpty = append(nonEmpty, line)
		}
	}
	if len(nonEmpty) <= 1 {
		return ""
	}
	return cleanBaiduSearchText(strings.Join(nonEmpty[1:], " "))
}

// cleanBaiduSearchText 折叠多余空白并去除首尾空格。
func cleanBaiduSearchText(value string) string {
	return strings.Join(strings.Fields(strings.TrimSpace(value)), " ")
}