From b9cd1d53cbc36a92a608f70f344bb650dac005f4 Mon Sep 17 00:00:00 2001 From: wuchulonly <1746825356@QQ.COM> Date: Mon, 11 May 2026 01:50:47 +0000 Subject: [PATCH 1/3] Add examples/cases/match_detail demonstrating MatchDetail usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A minimal, copy-pasteable case showing how to obtain matcher details (matcher type/value, rule index, send_data) and the matched resource URL from a fingerprint hit. Key points the case illustrates: * EnableMatchDetail() must be called AFTER NewEngine(), because engine.Compile() resets each finger's EnableMatchDetail to the engine field's default value (false). * common.Framework.MatchDetail is read directly — no extra wrapper type is introduced. * match_url uses MatchDetail.SendData's "url=" segment when present (only populated by upstream consumers that do active probing), and falls back to resp.Request.URL for passive matches via DetectContent / MatchHTTP. Also introduces examples/cases/ as the home for cookbook-style minimal snippets, separate from the four top-level CLI tools. Co-Authored-By: Claude Opus 4.7 (1M context) --- examples/README.md | 24 +++++- examples/cases/match_detail/main.go | 113 ++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 examples/cases/match_detail/main.go diff --git a/examples/README.md b/examples/README.md index 3f14afc..ab71983 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,7 +9,9 @@ examples/ ├── fingers/ # 指纹识别工具 ├── neutron/ # POC 扫描工具 ├── gogo/ # 端口扫描和指纹识别工具 -└── spray/ # HTTP 批量探测工具 +├── spray/ # HTTP 批量探测工具 +└── cases/ # 小颗粒度使用案例（cookbook） + └── match_detail/ # 获取指纹命中的 matcher 详情和命中资源 URL ``` ## 快速开始 @@ -257,6 +259,26 @@ echo "http://127.0.0.1:8080" >> test_urls.txt --- +## Cases - 小颗粒度使用案例 + +`examples/cases/` 下放的是 cookbook 风格的最小可运行片段，每个 case 只演示一个 API 或一个用法要点，复制即可融入到自己的工程里。 + +### match_detail - 获取 matcher 详情和命中资源 URL + +演示如何让指纹引擎在命中后输出 `MatchDetail`（matcher 类型/值、rule_index、send_data）以及如何拿到命中的资源 URL。 + +```bash +go run ./cases/match_detail -url http://127.0.0.1:8080 -key your_api_key -target http://127.0.0.1:3000 +``` + +要点： + +- 必须在 `fingers.NewEngine()` 之后调用 `eng.GetFingersEngine().EnableMatchDetail()`。`NewEngine` 内部会触发 `engine.Compile()`，把每条 finger 的 `EnableMatchDetail` 重置为 engine 字段默认值 (false)。 +- 命中后直接读 `framework.MatchDetail`，不需要任何额外封装。 +- `match_url` 取值优先级：`MatchDetail.SendData` 中的 `url=` > 当前请求 URL（`resp.Request.URL`，已处理重定向）。SDK 自带的被动匹配（`DetectContent`/`MatchHTTP`）不会主动发包，所以 `SendData` 通常为空，必须由调用方用请求 URL 兜底。 + +--- + ## 常见问题 ### Q: 如何获取 Cyberhub API Key? diff --git a/examples/cases/match_detail/main.go b/examples/cases/match_detail/main.go new file mode 100644 index 0000000..a872472 --- /dev/null +++ b/examples/cases/match_detail/main.go @@ -0,0 +1,113 @@ +// match_detail 演示：在 sdk/fingers 上拿到指纹命中的 matcher 详情 + 命中的资源 URL。 +// +// 关键点： +// 1. SDK 不需要任何改造，只在 NewEngine() 之后翻开 MatchDetail 开关即可。 +// 原因：NewEngine 内部会调用 engine.Compile()，把每条 finger 的 +// EnableMatchDetail 重置回 engine 字段的默认值 (false)。 +// 2. 命中后直接读 common.Framework.MatchDetail，没有额外封装。 +// 3. MatchDetail.SendData 在 active 探测增强后的链路下形如 +// "scope=... method=... url=..."；SDK 自带的被动匹配下通常为空， +// 此时 match_url 用最终请求 URL 兜底。 +// +// 用法： +// go run ./examples/cases/match_detail -url http://127.0.0.1:8080 -key -target http://example.com +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "strings" + + "github.com/chainreactors/sdk/fingers" + "github.com/chainreactors/utils/httputils" +) + +func main() { + cyberhubURL := flag.String("url", "", "Cyberhub URL (optional, local fingers used if empty)") + apiKey := flag.String("key", "", "Cyberhub API Key") + target := flag.String("target", "", "Target URL to match (required)") + flag.Parse() + if *target == "" { + flag.Usage() + os.Exit(1) + } + + // 1. 构建 engine + cfg := fingers.NewConfig() + if *cyberhubURL != "" { + cfg.WithCyberhub(*cyberhubURL, *apiKey) + } + eng, err := fingers.NewEngine(cfg) + if err != nil { + fmt.Printf("engine init failed: %v\n", err) + os.Exit(1) + } + + // 2. ★ 关键：NewEngine 之后翻开 MatchDetail 开关 + if fe, _ := eng.GetFingersEngine(); fe != nil { + fe.EnableMatchDetail() + } + + // 3. 抓 + 匹配 + resp, err := http.Get(*target) + if err != nil { + fmt.Printf("http get failed: %v\n", err) + os.Exit(1) + } + defer resp.Body.Close() + + frameworks, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + if err != nil { + fmt.Printf("match failed: %v\n", err) + os.Exit(1) + } + + // 4. 直接读 common.Framework.MatchDetail；match_url 用最终请求 URL 兜底 + finalURL := *target + if resp.Request != nil && resp.Request.URL != nil { + finalURL = resp.Request.URL.String() + } + if len(frameworks) == 0 { + fmt.Println("no fingerprints matched") + return + } + for _, fw := range frameworks { + fmt.Printf("[%s]\n", fw.Name) + d := fw.MatchDetail + if d == nil { + fmt.Printf(" match_url : %s (MatchDetail empty)\n", finalURL) + continue + } + matchURL := finalURL + if u := extractURL(d.SendData); u != "" { + matchURL = u + } + fmt.Printf(" match_url : %s\n", matchURL) + fmt.Printf(" matcher_type : %s\n", d.MatcherType) + fmt.Printf(" matcher_value : %s\n", d.MatcherValue) + fmt.Printf(" rule_index : %d\n", d.RuleIndex) + if d.SendData != "" { + fmt.Printf(" send_data : %s\n", d.SendData) + } + } +} + +// extractURL 从 "scope=... method=... url=<...>" 里取 url= 后整段。 +// 词边界判断避免 value 里出现 "url=" 子串时误匹配。 +func extractURL(s string) string { + const tag = "url=" + for start := 0; start < len(s); { + i := strings.Index(s[start:], tag) + if i < 0 { + return "" + } + i += start + if i == 0 || s[i-1] == ' ' { + return strings.TrimSpace(s[i+len(tag):]) + } + start = i + len(tag) + } + return "" +} From 10b71d78f7c5b136848a12cce69eda21f37debd0 Mon Sep 17 00:00:00 2001 From: wuchulonly <1746825356@QQ.COM> Date: Mon, 11 May 2026 02:03:01 +0000 Subject: [PATCH 2/3] Add test-based demo and FingerMatch helper variant for match_detail Adds two complementary case variants alongside the existing cmd-style example so users can pick the form that best fits their integration: * examples/cases/match_detail/match_detail_test.go Test-based minimal demo using an inline Finger + httptest.Server. Verifies EnableMatchDetail() propagation end-to-end and serves as a reverse-control test (without EnableMatchDetail, MatchDetail is nil). Run with: go test ./examples/cases/match_detail -v * examples/cases/match_detail_helper/ Wrapper-style variant exposing a flat FingerMatch struct plus helpers (EnableMatchDetail, FlattenMatches, ExtractURL) and two cookbook functions (DetectFingersDetail, SprayWithCrawlAndFingerDetail). Comes with unit + e2e tests. Run with: go test ./examples/cases/match_detail_helper -v README updated to describe both styles and clarify that match_url must fall back to the request URL (or SprayResult.UrlString) for passive matchers, since MatchDetail.SendData is only populated by upstream active-probe flows. Co-Authored-By: Claude Opus 4.7 (1M context) --- examples/README.md | 25 +- .../cases/match_detail/match_detail_test.go | 124 ++++++++++ examples/cases/match_detail_helper/helper.go | 216 ++++++++++++++++++ .../cases/match_detail_helper/helper_test.go | 128 +++++++++++ 4 files changed, 488 insertions(+), 5 deletions(-) create mode 100644 examples/cases/match_detail/match_detail_test.go create mode 100644 examples/cases/match_detail_helper/helper.go create mode 100644 examples/cases/match_detail_helper/helper_test.go diff --git a/examples/README.md b/examples/README.md index ab71983..8bd5f42 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,7 +11,8 @@ examples/ ├── gogo/ # 端口扫描和指纹识别工具 ├── spray/ # HTTP 批量探测工具 └── cases/ # 小颗粒度使用案例（cookbook） - └── match_detail/ # 获取指纹命中的 matcher 详情和命中资源 URL + ├── match_detail/ # 获取 matcher 详情和命中资源 URL（cmd + test） + └── match_detail_helper/ # 同上，FingerMatch 封装风格（library + test） ``` ## 快速开始 @@ -265,17 +266,31 @@ echo "http://127.0.0.1:8080" >> test_urls.txt ### match_detail - 获取 matcher 详情和命中资源 URL -演示如何让指纹引擎在命中后输出 `MatchDetail`（matcher 类型/值、rule_index、send_data）以及如何拿到命中的资源 URL。 +演示如何让指纹引擎在命中后输出 `MatchDetail`（matcher 类型/值、rule_index、send_data）以及如何拿到命中的资源 URL。提供两种风格，二选一即可： + +**风格 ① 直接用 SDK 原生类型（推荐）** —— `cases/match_detail/` ```bash +# 跑命令行版（被动匹配真实 target） go run ./cases/match_detail -url http://127.0.0.1:8080 -key your_api_key -target http://127.0.0.1:3000 + +# 跑测试版（inline finger + httptest，离线可跑） +go test ./cases/match_detail -v +``` + +**风格 ② 封装一层调用方友好结构** —— `cases/match_detail_helper/` + +`FingerMatch` 把 `MatchDetail` 拍平成可直接 JSON 序列化的结构，并把 `match_url` 兜底逻辑封装好；同时给出 `DetectFingersDetail`（被动）和 `SprayWithCrawlAndFingerDetail`（spray + 静态爬虫）两段示例代码。 + +```bash +go test ./cases/match_detail_helper -v ``` -要点： +**两种风格共通的要点：** - 必须在 `fingers.NewEngine()` 之后调用 `eng.GetFingersEngine().EnableMatchDetail()`。`NewEngine` 内部会触发 `engine.Compile()`，把每条 finger 的 `EnableMatchDetail` 重置为 engine 字段默认值 (false)。 -- 命中后直接读 `framework.MatchDetail`，不需要任何额外封装。 -- `match_url` 取值优先级：`MatchDetail.SendData` 中的 `url=` > 当前请求 URL（`resp.Request.URL`，已处理重定向）。SDK 自带的被动匹配（`DetectContent`/`MatchHTTP`）不会主动发包，所以 `SendData` 通常为空，必须由调用方用请求 URL 兜底。 +- 命中后直接读 `framework.MatchDetail`，SDK 没有额外封装类型；上面风格 ② 的 `FingerMatch` 只是调用方 ergonomics，可选。 +- `match_url` 取值优先级：`MatchDetail.SendData` 中的 `url=` > 当前请求 URL（`resp.Request.URL`，已处理重定向）/ `SprayResult.UrlString`（spray 链路）。SDK 自带的被动匹配（`DetectContent`/`MatchHTTP`）不会主动发包，所以 `SendData` 通常为空，必须由调用方用请求 URL 兜底。 --- diff --git a/examples/cases/match_detail/match_detail_test.go b/examples/cases/match_detail/match_detail_test.go new file mode 100644 index 0000000..1583ff8 --- /dev/null +++ b/examples/cases/match_detail/match_detail_test.go @@ -0,0 +1,124 @@ +// Test-based 演示：直接用 go test 跑通整条链路，证明 +// EnableMatchDetail() 之后 framework.MatchDetail 真的会被填进去。 +// +// Run with: +// go test ./examples/cases/match_detail -v +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + fingersEngine "github.com/chainreactors/fingers/fingers" + sdkfingers "github.com/chainreactors/sdk/fingers" + "github.com/chainreactors/utils/httputils" +) + +// TestMatchDetailUsage 演示获取 matcher 详情的最小调用流程： +// +// 1. 构造 Engine +// 2. ★ NewEngine() 之后立刻翻开 MatchDetail +// (engine.Compile() 会重置每条 finger 的开关，所以必须这里调) +// 3. 跑 DetectContent，从 framework.MatchDetail 读 matcher 详情 +// 4. match_url 取值：MatchDetail.SendData 的 "url=" > 当前请求 URL（兜底） +func TestMatchDetailUsage(t *testing.T) { + // —— 演示用：inline finger + httptest。 + // 实际工程里换成 WithCyberhub / WithLocalFile 即可。 + finger := &fingersEngine.Finger{ + Name: "demo-app", + Protocol: "http", + Rules: fingersEngine.Rules{ + {Regexps: &fingersEngine.Regexps{Body: []string{"DemoMarker"}}}, + }, + } + eng, err := sdkfingers.NewEngine( + sdkfingers.NewConfig().WithFingers(fingersEngine.Fingers{finger}), + ) + if err != nil { + t.Fatal(err) + } + + // ★ STEP 1：必须在 NewEngine 之后调用 + if fe, _ := eng.GetFingersEngine(); fe != nil { + fe.EnableMatchDetail() + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("response with DemoMarker")) + })) + defer srv.Close() + + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + // ★ STEP 2：跑匹配 + frames, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + if err != nil { + t.Fatal(err) + } + + // ★ STEP 3：读 MatchDetail，match_url 用兜底 URL + fallbackURL := srv.URL + if resp.Request != nil && resp.Request.URL != nil { + fallbackURL = resp.Request.URL.String() + } + for _, fw := range frames { + d := fw.MatchDetail + if d == nil { + continue + } + matchURL := fallbackURL + if u := extractURL(d.SendData); u != "" { + matchURL = u + } + t.Logf("[%s] match_url=%s matcher_type=%s matcher_value=%s rule_index=%d", + fw.Name, matchURL, d.MatcherType, d.MatcherValue, d.RuleIndex) + } + + // —— 把演示也当一条回归断言 —— + fw, ok := frames["demo-app"] + if !ok { + t.Fatalf("expected match for demo-app, got: %v", frames) + } + if fw.MatchDetail == nil { + t.Fatal("MatchDetail is nil — EnableMatchDetail() must be called after NewEngine()") + } + if fw.MatchDetail.MatcherType == "" || fw.MatchDetail.MatcherValue == "" { + t.Fatalf("expected non-empty matcher fields, got %+v", *fw.MatchDetail) + } +} + +// TestMatchDetailRequiresEnable 反向对照：不调用 EnableMatchDetail() 时 +// MatchDetail 为 nil，证明这一步是必需的不是可有可无。 +func TestMatchDetailRequiresEnable(t *testing.T) { + finger := &fingersEngine.Finger{ + Name: "demo-app-disabled", + Protocol: "http", + Rules: fingersEngine.Rules{ + {Regexps: &fingersEngine.Regexps{Body: []string{"DemoMarker2"}}}, + }, + } + eng, _ := sdkfingers.NewEngine( + sdkfingers.NewConfig().WithFingers(fingersEngine.Fingers{finger}), + ) + // 故意不调 EnableMatchDetail() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("DemoMarker2")) + })) + defer srv.Close() + resp, _ := http.Get(srv.URL) + defer resp.Body.Close() + + frames, _ := eng.Get().DetectContent(httputils.ReadRaw(resp)) + if fw, ok := frames["demo-app-disabled"]; ok && fw.MatchDetail != nil { + t.Fatalf("expected MatchDetail=nil without EnableMatchDetail(), got %+v", *fw.MatchDetail) + } +} + +// 注意：extractURL 在 main.go 里已经定义，本文件直接调用，不再重复。 +// 同一 package main 下编译时两文件共享所有顶层符号。 diff --git a/examples/cases/match_detail_helper/helper.go b/examples/cases/match_detail_helper/helper.go new file mode 100644 index 0000000..0e8c595 --- /dev/null +++ b/examples/cases/match_detail_helper/helper.go @@ -0,0 +1,216 @@ +// match_detail_helper 演示一种把 common.MatchDetail 拍平成调用方友好结构 +// (FingerMatch) 的写法，覆盖被动 + 主动两种链路的 match_url 兜底。 +// +// 关键点和 examples/cases/match_detail 一致： +// 1. NewEngine() 之后必须调用 GetFingersEngine().EnableMatchDetail()。 +// 2. common.Framework.MatchDetail 是数据源，FingerMatch 只是 ergonomics 封装。 +// 3. match_url 取值优先级： +// MatchDetail.SendData 中的 "url=" > 当前请求 URL / SprayResult.UrlString +// +// 这一份和 cases/match_detail/main.go 的区别只是封装风格，二选一即可。 +// 用 go test ./examples/cases/match_detail_helper 运行演示。 +package main + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/chainreactors/fingers/common" + sdkfingers "github.com/chainreactors/sdk/fingers" + "github.com/chainreactors/sdk/spray" + "github.com/chainreactors/utils/httputils" +) + +func main() { + fmt.Println("This example is exercised via `go test ./examples/cases/match_detail_helper`.") +} + +// FingerMatch 是给上层的扁平、易序列化结构。 +type FingerMatch struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Tags []string `json:"tags,omitempty"` + Attributes *common.Attributes `json:"attributes,omitempty"` + MatchURL string `json:"match_url,omitempty"` + MatcherType string `json:"matcher_type,omitempty"` + MatcherValue string `json:"matcher_value,omitempty"` + RuleIndex int `json:"rule_index,omitempty"` + SendData string `json:"send_data,omitempty"` +} + +// EnableMatchDetail 翻开底层 fingers 引擎的 matcher 详情开关。 +// 必须在 sdkfingers.NewEngine() 之后调用，因为 NewEngine 内部会触发 +// engine.Compile()，把每条 finger 的 EnableMatchDetail 重置回 engine +// 字段的默认值 (false)。 +func EnableMatchDetail(eng *sdkfingers.Engine) error { + if eng == nil { + return nil + } + fe, err := eng.GetFingersEngine() + if err != nil { + return err + } + if fe != nil { + fe.EnableMatchDetail() + } + return nil +} + +// FlattenMatches 把 common.Frameworks 拍平成 FingerMatch 切片。 +// fallbackURL：MatchDetail.SendData 不含 url= 时回填用 (DetectFingers 传请求 URL, +// spray 传 SprayResult.UrlString)。 +func FlattenMatches(frames common.Frameworks, fallbackURL string) []FingerMatch { + out := make([]FingerMatch, 0, len(frames)) + for _, f := range frames { + if f == nil { + continue + } + fm := FingerMatch{ + Name: f.Name, + Tags: f.Tags, + Attributes: f.Attributes, + MatchURL: fallbackURL, + } + if f.Attributes != nil { + fm.Version = f.Attributes.Version + } + if d := f.MatchDetail; d != nil { + fm.MatcherType = d.MatcherType + fm.MatcherValue = d.MatcherValue + fm.RuleIndex = d.RuleIndex + fm.SendData = d.SendData + if u := ExtractURL(d.SendData); u != "" { + fm.MatchURL = u + } + } + out = append(out, fm) + } + return out +} + +// ExtractURL 从 "scope=... method=... url=<...>" 中取 url= 后整段。 +// 词边界判断避免 value 内出现 url= 子串时误匹配。 +func ExtractURL(sendData string) string { + const tag = "url=" + for start := 0; start < len(sendData); { + i := strings.Index(sendData[start:], tag) + if i < 0 { + return "" + } + i += start + if i == 0 || sendData[i-1] == ' ' { + return strings.TrimSpace(sendData[i+len(tag):]) + } + start = i + len(tag) + } + return "" +} + +// DetectFingersDetail 演示 ① ：单次 HTTP 被动匹配，返回 FingerMatch 列表。 +func DetectFingersDetail(target, cyberhubURL, apiKey string) ([]FingerMatch, error) { + cfg := sdkfingers.NewConfig() + if cyberhubURL != "" { + cfg.WithCyberhub(cyberhubURL, apiKey) + } + eng, err := sdkfingers.NewEngine(cfg) + if err != nil { + return nil, err + } + if err := EnableMatchDetail(eng); err != nil { + return nil, err + } + resp, err := http.Get(target) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + fallbackURL := target + if resp.Request != nil && resp.Request.URL != nil { + fallbackURL = resp.Request.URL.String() + } + frames, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + if err != nil { + return nil, err + } + return FlattenMatches(frames, fallbackURL), nil +} + +// SprayDetailResult spray 命中的资源 + 该资源上识别出的指纹列表。 +type SprayDetailResult struct { + URL string `json:"url"` + Path string `json:"path"` + Status int `json:"status"` + Title string `json:"title"` + Matches []FingerMatch `json:"matches"` +} + +// SprayWithCrawlAndFingerDetail 演示 ② ：spray + 静态爬虫 + 指纹联动。 +// max <= 0 表示不限；建议给个上限，否则 crawl 在大站会无限扩张。 +func SprayWithCrawlAndFingerDetail(target string, seeds []string, depth, max int, cyberhubURL, apiKey string) ([]SprayDetailResult, error) { + if depth >= 3 { + depth = 2 + } + if len(seeds) == 0 { + seeds = []string{""} + } + cfg := spray.NewConfig() + if cyberhubURL != "" && apiKey != "" { + fEng, err := sdkfingers.NewEngine(sdkfingers.NewConfig().WithCyberhub(cyberhubURL, apiKey)) + if err != nil { + return nil, err + } + if err := EnableMatchDetail(fEng); err != nil { + return nil, err + } + if fEng.Get() != nil { + cfg = cfg.WithFingersEngine(fEng) + } + } + se := spray.NewEngine(cfg) + if err := se.Init(); err != nil { + return nil, err + } + defer se.Close() + + innerCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx := spray.NewContext(). + WithContext(innerCtx). + SetThreads(50). + SetTimeout(10). + SetFinger(true). + SetCrawlPlugin(true). + SetCrawlDepth(depth) + + ch, err := se.Execute(ctx, spray.NewBruteTask(target, seeds)) + if err != nil { + return nil, err + } + + var out []SprayDetailResult + for r := range ch { + sr, ok := r.(*spray.Result) + if !ok || !sr.Success() { + continue + } + data := sr.SprayResult() + if data == nil || len(data.Frameworks) == 0 { + continue + } + out = append(out, SprayDetailResult{ + URL: data.UrlString, + Path: data.Path, + Status: data.Status, + Title: data.Title, + Matches: FlattenMatches(data.Frameworks, data.UrlString), + }) + if max > 0 && len(out) >= max { + cancel() + break + } + } + return out, nil +} diff --git a/examples/cases/match_detail_helper/helper_test.go b/examples/cases/match_detail_helper/helper_test.go new file mode 100644 index 0000000..81337a4 --- /dev/null +++ b/examples/cases/match_detail_helper/helper_test.go @@ -0,0 +1,128 @@ +// helper_test.go 既是 helper.go 的回归测试，也是它最小的使用演示。 +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/chainreactors/fingers/common" + fingersEngine "github.com/chainreactors/fingers/fingers" + sdkfingers "github.com/chainreactors/sdk/fingers" + "github.com/chainreactors/utils/httputils" +) + +func detectInlineForTest(t *testing.T, target, marker, fingerName string) []FingerMatch { + t.Helper() + finger := &fingersEngine.Finger{ + Name: fingerName, + Protocol: "http", + Rules: fingersEngine.Rules{{Regexps: &fingersEngine.Regexps{Body: []string{marker}}}}, + } + eng, err := sdkfingers.NewEngine( + sdkfingers.NewConfig().WithFingers(fingersEngine.Fingers{finger}), + ) + if err != nil { + t.Fatal(err) + } + if err := EnableMatchDetail(eng); err != nil { + t.Fatal(err) + } + resp, err := http.Get(target) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + fallback := target + if resp.Request != nil && resp.Request.URL != nil { + fallback = resp.Request.URL.String() + } + frames, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + if err != nil { + t.Fatal(err) + } + return FlattenMatches(frames, fallback) +} + +func TestExtractURL(t *testing.T) { + cases := []struct{ name, in, want string }{ + {"empty", "", ""}, + {"only-url", "url=https://x.test/a", "https://x.test/a"}, + {"scope-method-url", "scope=currentpath method=GET url=https://x.test/p", "https://x.test/p"}, + {"url-value-with-eq", "scope=cp method=GET url=https://x.test/api?next=https://y/foo=1", "https://x.test/api?next=https://y/foo=1"}, + {"no-url", "scope=cp method=GET", ""}, + {"url-substring-in-value-only", "scope=anonymous_url=x method=GET", ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := ExtractURL(c.in); got != c.want { + t.Fatalf("ExtractURL(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +func TestFlattenMatches_FallbackURL(t *testing.T) { + frames := common.Frameworks{ + "WordPress": &common.Framework{ + Name: "WordPress", + Tags: []string{"cms"}, + MatchDetail: &common.MatchDetail{MatcherType: "word", MatcherValue: "wp-content", RuleIndex: 3}, + }, + "Pentaho": &common.Framework{ + Name: "Pentaho", + MatchDetail: &common.MatchDetail{ + MatcherType: "word", + MatcherValue: "Pentaho", + SendData: "scope=currentpath method=GET url=https://x.test/pentaho/Login", + }, + }, + } + out := FlattenMatches(frames, "https://x.test/") + if len(out) != 2 { + t.Fatalf("expected 2 matches, got %d", len(out)) + } + got := map[string]FingerMatch{} + for _, m := range out { + got[m.Name] = m + } + if got["WordPress"].MatchURL != "https://x.test/" { + t.Fatalf("WordPress fallback failed: %q", got["WordPress"].MatchURL) + } + if got["Pentaho"].MatchURL != "https://x.test/pentaho/Login" { + t.Fatalf("Pentaho should pick MatchURL from SendData, got %q", got["Pentaho"].MatchURL) + } +} + +func TestEnableMatchDetail_NilSafe(t *testing.T) { + if err := EnableMatchDetail(nil); err != nil { + t.Fatalf("expected no-op on nil, got %v", err) + } +} + +// TestDetectFingersDetail_E2E 端到端：用 inline finger + httptest 跑通 +// DetectFingersDetail 的完整链路，证明 FingerMatch.MatchURL/MatcherType 都被填上。 +func TestDetectFingersDetail_E2E(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("HelperDemoMarker in body")) + })) + defer srv.Close() + + // DetectFingersDetail 当前实现走 cyberhub 加载，这里测离线路径绕一下: + // 直接构造 engine + inline finger，复用 FlattenMatches 验证封装层逻辑。 + matches := detectInlineForTest(t, srv.URL, "HelperDemoMarker", "helper-demo") + if len(matches) != 1 { + t.Fatalf("expected 1 match, got %d", len(matches)) + } + m := matches[0] + if m.Name != "helper-demo" { + t.Fatalf("expected helper-demo, got %q", m.Name) + } + if m.MatchURL == "" { + t.Fatal("MatchURL should fall back to request URL, got empty") + } + if m.MatcherType == "" || m.MatcherValue == "" { + t.Fatalf("matcher fields empty: %+v", m) + } + t.Logf("OK: %+v", m) +} From 7170e486838bfac0e35041e246bb2f33ba8d6faf Mon Sep 17 00:00:00 2001 From: root Date: Mon, 11 May 2026 02:43:37 +0000 Subject: [PATCH 3/3] Expose match detail results in fingers SDK --- examples/README.md | 14 +- examples/cases/match_detail/main.go | 78 ++----- .../cases/match_detail/match_detail_test.go | 71 ++----- examples/cases/match_detail_helper/helper.go | 197 +++--------------- .../cases/match_detail_helper/helper_test.go | 98 +++------ fingers/match_detail.go | 143 +++++++++++++ fingers/match_detail_test.go | 156 ++++++++++++++ 7 files changed, 416 insertions(+), 341 deletions(-) create mode 100644 fingers/match_detail.go create mode 100644 fingers/match_detail_test.go diff --git a/examples/README.md b/examples/README.md index 8bd5f42..8fa62f3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -266,7 +266,7 @@ echo "http://127.0.0.1:8080" >> test_urls.txt ### match_detail - 获取 matcher 详情和命中资源 URL -演示如何让指纹引擎在命中后输出 `MatchDetail`（matcher 类型/值、rule_index、send_data）以及如何拿到命中的资源 URL。提供两种风格，二选一即可： +演示如何通过 SDK public API 输出 `MatchDetail`（matcher 类型/值、rule_index、send_data）以及命中的资源 URL。提供两种风格，二选一即可： **风格 ① 直接用 SDK 原生类型（推荐）** —— `cases/match_detail/` @@ -280,17 +280,17 @@ go test ./cases/match_detail -v **风格 ② 封装一层调用方友好结构** —— `cases/match_detail_helper/` -`FingerMatch` 把 `MatchDetail` 拍平成可直接 JSON 序列化的结构，并把 `match_url` 兜底逻辑封装好；同时给出 `DetectFingersDetail`（被动）和 `SprayWithCrawlAndFingerDetail`（spray + 静态爬虫）两段示例代码。 +`FingerMatch` 演示如何把 SDK 返回的 `fingers.MatchResult` 转成业务自己的 JSON DTO。`match_url`、matcher detail、fallback URL 都来自 SDK，不需要调用方复制解析逻辑。 ```bash go test ./cases/match_detail_helper -v ``` -**两种风格共通的要点：** - -- 必须在 `fingers.NewEngine()` 之后调用 `eng.GetFingersEngine().EnableMatchDetail()`。`NewEngine` 内部会触发 `engine.Compile()`，把每条 finger 的 `EnableMatchDetail` 重置为 engine 字段默认值 (false)。 -- 命中后直接读 `framework.MatchDetail`，SDK 没有额外封装类型；上面风格 ② 的 `FingerMatch` 只是调用方 ergonomics，可选。 -- `match_url` 取值优先级：`MatchDetail.SendData` 中的 `url=` > 当前请求 URL（`resp.Request.URL`，已处理重定向）/ `SprayResult.UrlString`（spray 链路）。SDK 自带的被动匹配（`DetectContent`/`MatchHTTP`）不会主动发包，所以 `SendData` 通常为空，必须由调用方用请求 URL 兜底。 +**两种风格共通的要点：** + +- 推荐直接调用 `eng.MatchHTTPWithDetail(resp)`，返回 `[]fingers.MatchResult`。 +- `MatchHTTPWithDetail` 会自动打开底层 `MatchDetail` 开关，调用方不需要手动调用 `GetFingersEngine().EnableMatchDetail()`。 +- `match_url` 取值优先级：`MatchDetail.SendData` 中的 `url=` > 当前请求 URL（`resp.Request.URL`，已处理重定向）。 --- diff --git a/examples/cases/match_detail/main.go b/examples/cases/match_detail/main.go index a872472..dbe5bbb 100644 --- a/examples/cases/match_detail/main.go +++ b/examples/cases/match_detail/main.go @@ -1,16 +1,13 @@ -// match_detail 演示：在 sdk/fingers 上拿到指纹命中的 matcher 详情 + 命中的资源 URL。 +// match_detail 演示：在 sdk/fingers 上直接拿到指纹命中的 matcher 详情 + 命中的资源 URL。 // // 关键点： -// 1. SDK 不需要任何改造，只在 NewEngine() 之后翻开 MatchDetail 开关即可。 -// 原因：NewEngine 内部会调用 engine.Compile()，把每条 finger 的 -// EnableMatchDetail 重置回 engine 字段的默认值 (false)。 -// 2. 命中后直接读 common.Framework.MatchDetail，没有额外封装。 -// 3. MatchDetail.SendData 在 active 探测增强后的链路下形如 -// "scope=... method=... url=..."；SDK 自带的被动匹配下通常为空， -// 此时 match_url 用最终请求 URL 兜底。 +// 1. 调用 MatchHTTPWithDetail(resp)，SDK 会自动打开 MatchDetail。 +// 2. 结果里直接读 MatchResult.MatchURL / MatcherType / MatcherValue。 +// 3. match_url 取值优先级：MatchDetail.SendData 中的 "url=" > 当前请求 URL。 // // 用法： -// go run ./examples/cases/match_detail -url http://127.0.0.1:8080 -key -target http://example.com +// +// go run ./examples/cases/match_detail -url http://127.0.0.1:8080 -key -target http://example.com package main import ( @@ -18,10 +15,8 @@ import ( "fmt" "net/http" "os" - "strings" "github.com/chainreactors/sdk/fingers" - "github.com/chainreactors/utils/httputils" ) func main() { @@ -45,12 +40,7 @@ func main() { os.Exit(1) } - // 2. ★ 关键：NewEngine 之后翻开 MatchDetail 开关 - if fe, _ := eng.GetFingersEngine(); fe != nil { - fe.EnableMatchDetail() - } - - // 3. 抓 + 匹配 + // 2. 抓 + 匹配。MatchHTTPWithDetail 会自动打开 MatchDetail。 resp, err := http.Get(*target) if err != nil { fmt.Printf("http get failed: %v\n", err) @@ -58,56 +48,28 @@ func main() { } defer resp.Body.Close() - frameworks, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + results, err := eng.MatchHTTPWithDetail(resp) if err != nil { fmt.Printf("match failed: %v\n", err) os.Exit(1) } - // 4. 直接读 common.Framework.MatchDetail；match_url 用最终请求 URL 兜底 - finalURL := *target - if resp.Request != nil && resp.Request.URL != nil { - finalURL = resp.Request.URL.String() - } - if len(frameworks) == 0 { + if len(results) == 0 { fmt.Println("no fingerprints matched") return } - for _, fw := range frameworks { - fmt.Printf("[%s]\n", fw.Name) - d := fw.MatchDetail - if d == nil { - fmt.Printf(" match_url : %s (MatchDetail empty)\n", finalURL) - continue - } - matchURL := finalURL - if u := extractURL(d.SendData); u != "" { - matchURL = u - } - fmt.Printf(" match_url : %s\n", matchURL) - fmt.Printf(" matcher_type : %s\n", d.MatcherType) - fmt.Printf(" matcher_value : %s\n", d.MatcherValue) - fmt.Printf(" rule_index : %d\n", d.RuleIndex) - if d.SendData != "" { - fmt.Printf(" send_data : %s\n", d.SendData) - } - } -} - -// extractURL 从 "scope=... method=... url=<...>" 里取 url= 后整段。 -// 词边界判断避免 value 里出现 "url=" 子串时误匹配。 -func extractURL(s string) string { - const tag = "url=" - for start := 0; start < len(s); { - i := strings.Index(s[start:], tag) - if i < 0 { - return "" + for _, r := range results { + name := "" + if r.Framework != nil { + name = r.Framework.Name } - i += start - if i == 0 || s[i-1] == ' ' { - return strings.TrimSpace(s[i+len(tag):]) + fmt.Printf("[%s]\n", name) + fmt.Printf(" match_url : %s\n", r.MatchURL) + fmt.Printf(" matcher_type : %s\n", r.MatcherType) + fmt.Printf(" matcher_value : %s\n", r.MatcherValue) + fmt.Printf(" rule_index : %d\n", r.RuleIndex) + if r.SendData != "" { + fmt.Printf(" send_data : %s\n", r.SendData) } - start = i + len(tag) } - return "" } diff --git a/examples/cases/match_detail/match_detail_test.go b/examples/cases/match_detail/match_detail_test.go index 1583ff8..1985033 100644 --- a/examples/cases/match_detail/match_detail_test.go +++ b/examples/cases/match_detail/match_detail_test.go @@ -1,8 +1,9 @@ // Test-based 演示：直接用 go test 跑通整条链路，证明 -// EnableMatchDetail() 之后 framework.MatchDetail 真的会被填进去。 +// MatchHTTPWithDetail 会返回 match_url 和 matcher 详情。 // // Run with: -// go test ./examples/cases/match_detail -v +// +// go test ./examples/cases/match_detail -v package main import ( @@ -12,16 +13,13 @@ import ( fingersEngine "github.com/chainreactors/fingers/fingers" sdkfingers "github.com/chainreactors/sdk/fingers" - "github.com/chainreactors/utils/httputils" ) // TestMatchDetailUsage 演示获取 matcher 详情的最小调用流程： // // 1. 构造 Engine -// 2. ★ NewEngine() 之后立刻翻开 MatchDetail -// (engine.Compile() 会重置每条 finger 的开关，所以必须这里调) -// 3. 跑 DetectContent，从 framework.MatchDetail 读 matcher 详情 -// 4. match_url 取值：MatchDetail.SendData 的 "url=" > 当前请求 URL（兜底） +// 2. 调 MatchHTTPWithDetail(resp) +// 3. 直接读 MatchResult.MatchURL / MatcherType / MatcherValue func TestMatchDetailUsage(t *testing.T) { // —— 演示用：inline finger + httptest。 // 实际工程里换成 WithCyberhub / WithLocalFile 即可。 @@ -39,11 +37,6 @@ func TestMatchDetailUsage(t *testing.T) { t.Fatal(err) } - // ★ STEP 1：必须在 NewEngine 之后调用 - if fe, _ := eng.GetFingersEngine(); fe != nil { - fe.EnableMatchDetail() - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("response with DemoMarker")) })) @@ -55,46 +48,31 @@ func TestMatchDetailUsage(t *testing.T) { } defer resp.Body.Close() - // ★ STEP 2：跑匹配 - frames, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + // MatchHTTPWithDetail 会自动打开 MatchDetail，并返回扁平结果。 + results, err := eng.MatchHTTPWithDetail(resp) if err != nil { t.Fatal(err) } - - // ★ STEP 3：读 MatchDetail，match_url 用兜底 URL - fallbackURL := srv.URL - if resp.Request != nil && resp.Request.URL != nil { - fallbackURL = resp.Request.URL.String() + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) } - for _, fw := range frames { - d := fw.MatchDetail - if d == nil { - continue - } - matchURL := fallbackURL - if u := extractURL(d.SendData); u != "" { - matchURL = u - } - t.Logf("[%s] match_url=%s matcher_type=%s matcher_value=%s rule_index=%d", - fw.Name, matchURL, d.MatcherType, d.MatcherValue, d.RuleIndex) + r := results[0] + if r.Framework == nil || r.Framework.Name != "demo-app" { + t.Fatalf("Framework not preserved: %+v", r) } - - // —— 把演示也当一条回归断言 —— - fw, ok := frames["demo-app"] - if !ok { - t.Fatalf("expected match for demo-app, got: %v", frames) - } - if fw.MatchDetail == nil { - t.Fatal("MatchDetail is nil — EnableMatchDetail() must be called after NewEngine()") + if r.MatchURL != srv.URL { + t.Fatalf("MatchURL should fall back to request URL %q, got %q", srv.URL, r.MatchURL) } - if fw.MatchDetail.MatcherType == "" || fw.MatchDetail.MatcherValue == "" { - t.Fatalf("expected non-empty matcher fields, got %+v", *fw.MatchDetail) + if r.MatcherType == "" || r.MatcherValue == "" { + t.Fatalf("expected matcher fields, got %+v", r) } + t.Logf("[%s] match_url=%s matcher_type=%s matcher_value=%s rule_index=%d", + r.Framework.Name, r.MatchURL, r.MatcherType, r.MatcherValue, r.RuleIndex) } -// TestMatchDetailRequiresEnable 反向对照：不调用 EnableMatchDetail() 时 -// MatchDetail 为 nil，证明这一步是必需的不是可有可无。 -func TestMatchDetailRequiresEnable(t *testing.T) { +// TestPlainMatchWithoutDetail 反向对照：plain MatchHTTP 仍保持兼容， +// 不会自动填 MatchDetail。 +func TestPlainMatchWithoutDetail(t *testing.T) { finger := &fingersEngine.Finger{ Name: "demo-app-disabled", Protocol: "http", @@ -114,11 +92,8 @@ func TestMatchDetailRequiresEnable(t *testing.T) { resp, _ := http.Get(srv.URL) defer resp.Body.Close() - frames, _ := eng.Get().DetectContent(httputils.ReadRaw(resp)) + frames, _ := eng.MatchHTTP(resp) if fw, ok := frames["demo-app-disabled"]; ok && fw.MatchDetail != nil { - t.Fatalf("expected MatchDetail=nil without EnableMatchDetail(), got %+v", *fw.MatchDetail) + t.Fatalf("expected MatchDetail=nil for plain MatchHTTP, got %+v", *fw.MatchDetail) } } - -// 注意：extractURL 在 main.go 里已经定义，本文件直接调用，不再重复。 -// 同一 package main 下编译时两文件共享所有顶层符号。 diff --git a/examples/cases/match_detail_helper/helper.go b/examples/cases/match_detail_helper/helper.go index 0e8c595..d5a6de3 100644 --- a/examples/cases/match_detail_helper/helper.go +++ b/examples/cases/match_detail_helper/helper.go @@ -1,26 +1,19 @@ -// match_detail_helper 演示一种把 common.MatchDetail 拍平成调用方友好结构 -// (FingerMatch) 的写法，覆盖被动 + 主动两种链路的 match_url 兜底。 +// match_detail_helper 演示调用方如何把 SDK 的 MatchResult 转成自己的 DTO。 // -// 关键点和 examples/cases/match_detail 一致： -// 1. NewEngine() 之后必须调用 GetFingersEngine().EnableMatchDetail()。 -// 2. common.Framework.MatchDetail 是数据源，FingerMatch 只是 ergonomics 封装。 -// 3. match_url 取值优先级： -// MatchDetail.SendData 中的 "url=" > 当前请求 URL / SprayResult.UrlString +// 核心 matcher detail / match_url 逻辑已经在 sdk/fingers 公共 API 中： // -// 这一份和 cases/match_detail/main.go 的区别只是封装风格，二选一即可。 -// 用 go test ./examples/cases/match_detail_helper 运行演示。 +// eng.MatchHTTPWithDetail(resp) -> []fingers.MatchResult +// +// 这一份只保留应用层字段映射，不再让调用方复制 EnableMatchDetail、 +// SendData 解析、fallback URL 等 SDK 内部细节。 package main import ( - "context" "fmt" "net/http" - "strings" "github.com/chainreactors/fingers/common" sdkfingers "github.com/chainreactors/sdk/fingers" - "github.com/chainreactors/sdk/spray" - "github.com/chainreactors/utils/httputils" ) func main() { @@ -36,79 +29,41 @@ type FingerMatch struct { MatchURL string `json:"match_url,omitempty"` MatcherType string `json:"matcher_type,omitempty"` MatcherValue string `json:"matcher_value,omitempty"` - RuleIndex int `json:"rule_index,omitempty"` + RuleIndex int `json:"rule_index"` SendData string `json:"send_data,omitempty"` } -// EnableMatchDetail 翻开底层 fingers 引擎的 matcher 详情开关。 -// 必须在 sdkfingers.NewEngine() 之后调用，因为 NewEngine 内部会触发 -// engine.Compile()，把每条 finger 的 EnableMatchDetail 重置回 engine -// 字段的默认值 (false)。 -func EnableMatchDetail(eng *sdkfingers.Engine) error { - if eng == nil { - return nil - } - fe, err := eng.GetFingersEngine() - if err != nil { - return err - } - if fe != nil { - fe.EnableMatchDetail() - } - return nil -} - -// FlattenMatches 把 common.Frameworks 拍平成 FingerMatch 切片。 -// fallbackURL：MatchDetail.SendData 不含 url= 时回填用 (DetectFingers 传请求 URL, -// spray 传 SprayResult.UrlString)。 -func FlattenMatches(frames common.Frameworks, fallbackURL string) []FingerMatch { - out := make([]FingerMatch, 0, len(frames)) - for _, f := range frames { - if f == nil { - continue - } - fm := FingerMatch{ - Name: f.Name, - Tags: f.Tags, - Attributes: f.Attributes, - MatchURL: fallbackURL, - } - if f.Attributes != nil { - fm.Version = f.Attributes.Version - } - if d := f.MatchDetail; d != nil { - fm.MatcherType = d.MatcherType - fm.MatcherValue = d.MatcherValue - fm.RuleIndex = d.RuleIndex - fm.SendData = d.SendData - if u := ExtractURL(d.SendData); u != "" { - fm.MatchURL = u - } - } - out = append(out, fm) +// FromMatchResults 把 SDK MatchResult 拍平成调用方自己的结构。 +func FromMatchResults(results []sdkfingers.MatchResult) []FingerMatch { + out := make([]FingerMatch, 0, len(results)) + for _, r := range results { + out = append(out, FromMatchResult(r)) } return out } -// ExtractURL 从 "scope=... method=... url=<...>" 中取 url= 后整段。 -// 词边界判断避免 value 内出现 url= 子串时误匹配。 -func ExtractURL(sendData string) string { - const tag = "url=" - for start := 0; start < len(sendData); { - i := strings.Index(sendData[start:], tag) - if i < 0 { - return "" - } - i += start - if i == 0 || sendData[i-1] == ' ' { - return strings.TrimSpace(sendData[i+len(tag):]) - } - start = i + len(tag) - } - return "" +// FromMatchResult 转换单条 SDK MatchResult。 +func FromMatchResult(r sdkfingers.MatchResult) FingerMatch { + fm := FingerMatch{ + MatchURL: r.MatchURL, + MatcherType: r.MatcherType, + MatcherValue: r.MatcherValue, + RuleIndex: r.RuleIndex, + SendData: r.SendData, + } + if r.Framework == nil { + return fm + } + fm.Name = r.Framework.Name + fm.Tags = r.Framework.Tags + fm.Attributes = r.Framework.Attributes + if r.Framework.Attributes != nil { + fm.Version = r.Framework.Attributes.Version + } + return fm } -// DetectFingersDetail 演示 ① ：单次 HTTP 被动匹配，返回 FingerMatch 列表。 +// DetectFingersDetail 演示单次 HTTP 被动匹配，返回调用方自己的 FingerMatch 列表。 func DetectFingersDetail(target, cyberhubURL, apiKey string) ([]FingerMatch, error) { cfg := sdkfingers.NewConfig() if cyberhubURL != "" { @@ -118,99 +73,15 @@ func DetectFingersDetail(target, cyberhubURL, apiKey string) ([]FingerMatch, err if err != nil { return nil, err } - if err := EnableMatchDetail(eng); err != nil { - return nil, err - } resp, err := http.Get(target) if err != nil { return nil, err } defer resp.Body.Close() - fallbackURL := target - if resp.Request != nil && resp.Request.URL != nil { - fallbackURL = resp.Request.URL.String() - } - frames, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + results, err := eng.MatchHTTPWithDetail(resp) if err != nil { return nil, err } - return FlattenMatches(frames, fallbackURL), nil -} - -// SprayDetailResult spray 命中的资源 + 该资源上识别出的指纹列表。 -type SprayDetailResult struct { - URL string `json:"url"` - Path string `json:"path"` - Status int `json:"status"` - Title string `json:"title"` - Matches []FingerMatch `json:"matches"` -} - -// SprayWithCrawlAndFingerDetail 演示 ② ：spray + 静态爬虫 + 指纹联动。 -// max <= 0 表示不限；建议给个上限，否则 crawl 在大站会无限扩张。 -func SprayWithCrawlAndFingerDetail(target string, seeds []string, depth, max int, cyberhubURL, apiKey string) ([]SprayDetailResult, error) { - if depth >= 3 { - depth = 2 - } - if len(seeds) == 0 { - seeds = []string{""} - } - cfg := spray.NewConfig() - if cyberhubURL != "" && apiKey != "" { - fEng, err := sdkfingers.NewEngine(sdkfingers.NewConfig().WithCyberhub(cyberhubURL, apiKey)) - if err != nil { - return nil, err - } - if err := EnableMatchDetail(fEng); err != nil { - return nil, err - } - if fEng.Get() != nil { - cfg = cfg.WithFingersEngine(fEng) - } - } - se := spray.NewEngine(cfg) - if err := se.Init(); err != nil { - return nil, err - } - defer se.Close() - - innerCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - ctx := spray.NewContext(). - WithContext(innerCtx). - SetThreads(50). - SetTimeout(10). - SetFinger(true). - SetCrawlPlugin(true). - SetCrawlDepth(depth) - - ch, err := se.Execute(ctx, spray.NewBruteTask(target, seeds)) - if err != nil { - return nil, err - } - - var out []SprayDetailResult - for r := range ch { - sr, ok := r.(*spray.Result) - if !ok || !sr.Success() { - continue - } - data := sr.SprayResult() - if data == nil || len(data.Frameworks) == 0 { - continue - } - out = append(out, SprayDetailResult{ - URL: data.UrlString, - Path: data.Path, - Status: data.Status, - Title: data.Title, - Matches: FlattenMatches(data.Frameworks, data.UrlString), - }) - if max > 0 && len(out) >= max { - cancel() - break - } - } - return out, nil + return FromMatchResults(results), nil } diff --git a/examples/cases/match_detail_helper/helper_test.go b/examples/cases/match_detail_helper/helper_test.go index 81337a4..c1e0eee 100644 --- a/examples/cases/match_detail_helper/helper_test.go +++ b/examples/cases/match_detail_helper/helper_test.go @@ -9,7 +9,6 @@ import ( "github.com/chainreactors/fingers/common" fingersEngine "github.com/chainreactors/fingers/fingers" sdkfingers "github.com/chainreactors/sdk/fingers" - "github.com/chainreactors/utils/httputils" ) func detectInlineForTest(t *testing.T, target, marker, fingerName string) []FingerMatch { @@ -25,91 +24,60 @@ func detectInlineForTest(t *testing.T, target, marker, fingerName string) []Fing if err != nil { t.Fatal(err) } - if err := EnableMatchDetail(eng); err != nil { - t.Fatal(err) - } resp, err := http.Get(target) if err != nil { t.Fatal(err) } defer resp.Body.Close() - fallback := target - if resp.Request != nil && resp.Request.URL != nil { - fallback = resp.Request.URL.String() - } - frames, err := eng.Get().DetectContent(httputils.ReadRaw(resp)) + + results, err := eng.MatchHTTPWithDetail(resp) if err != nil { t.Fatal(err) } - return FlattenMatches(frames, fallback) -} - -func TestExtractURL(t *testing.T) { - cases := []struct{ name, in, want string }{ - {"empty", "", ""}, - {"only-url", "url=https://x.test/a", "https://x.test/a"}, - {"scope-method-url", "scope=currentpath method=GET url=https://x.test/p", "https://x.test/p"}, - {"url-value-with-eq", "scope=cp method=GET url=https://x.test/api?next=https://y/foo=1", "https://x.test/api?next=https://y/foo=1"}, - {"no-url", "scope=cp method=GET", ""}, - {"url-substring-in-value-only", "scope=anonymous_url=x method=GET", ""}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - if got := ExtractURL(c.in); got != c.want { - t.Fatalf("ExtractURL(%q) = %q, want %q", c.in, got, c.want) - } - }) - } + return FromMatchResults(results) } -func TestFlattenMatches_FallbackURL(t *testing.T) { - frames := common.Frameworks{ - "WordPress": &common.Framework{ - Name: "WordPress", - Tags: []string{"cms"}, - MatchDetail: &common.MatchDetail{MatcherType: "word", MatcherValue: "wp-content", RuleIndex: 3}, - }, - "Pentaho": &common.Framework{ - Name: "Pentaho", - MatchDetail: &common.MatchDetail{ - MatcherType: "word", - MatcherValue: "Pentaho", - SendData: "scope=currentpath method=GET url=https://x.test/pentaho/Login", - }, - }, - } - out := FlattenMatches(frames, "https://x.test/") - if len(out) != 2 { - t.Fatalf("expected 2 matches, got %d", len(out)) - } - got := map[string]FingerMatch{} - for _, m := range out { - got[m.Name] = m - } - if got["WordPress"].MatchURL != "https://x.test/" { - t.Fatalf("WordPress fallback failed: %q", got["WordPress"].MatchURL) - } - if got["Pentaho"].MatchURL != "https://x.test/pentaho/Login" { - t.Fatalf("Pentaho should pick MatchURL from SendData, got %q", got["Pentaho"].MatchURL) +func TestFromMatchResults(t *testing.T) { + framework := &common.Framework{ + Name: "WordPress", + Tags: []string{"cms"}, + Attributes: &common.Attributes{Version: "6.4"}, + } + out := FromMatchResults([]sdkfingers.MatchResult{{ + Framework: framework, + MatchURL: "https://x.test/", + MatcherType: "word", + MatcherValue: "wp-content", + RuleIndex: 3, + SendData: "scope=currentpath method=GET url=https://x.test/", + }}) + if len(out) != 1 { + t.Fatalf("expected 1 match, got %d", len(out)) + } + m := out[0] + if m.Name != "WordPress" || m.Version != "6.4" { + t.Fatalf("framework fields not mapped: %+v", m) + } + if m.MatchURL != "https://x.test/" || m.MatcherType != "word" || m.MatcherValue != "wp-content" || m.RuleIndex != 3 { + t.Fatalf("detail fields not mapped: %+v", m) } } -func TestEnableMatchDetail_NilSafe(t *testing.T) { - if err := EnableMatchDetail(nil); err != nil { - t.Fatalf("expected no-op on nil, got %v", err) +func TestFromMatchResult_NilFramework(t *testing.T) { + m := FromMatchResult(sdkfingers.MatchResult{MatchURL: "https://x.test/"}) + if m.MatchURL != "https://x.test/" { + t.Fatalf("MatchURL should survive nil framework, got %+v", m) } } // TestDetectFingersDetail_E2E 端到端：用 inline finger + httptest 跑通 -// DetectFingersDetail 的完整链路，证明 FingerMatch.MatchURL/MatcherType 都被填上。 +// MatchHTTPWithDetail -> FingerMatch 的完整链路。 func TestDetectFingersDetail_E2E(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("HelperDemoMarker in body")) })) defer srv.Close() - // DetectFingersDetail 当前实现走 cyberhub 加载，这里测离线路径绕一下: - // 直接构造 engine + inline finger，复用 FlattenMatches 验证封装层逻辑。 matches := detectInlineForTest(t, srv.URL, "HelperDemoMarker", "helper-demo") if len(matches) != 1 { t.Fatalf("expected 1 match, got %d", len(matches)) @@ -118,8 +86,8 @@ func TestDetectFingersDetail_E2E(t *testing.T) { if m.Name != "helper-demo" { t.Fatalf("expected helper-demo, got %q", m.Name) } - if m.MatchURL == "" { - t.Fatal("MatchURL should fall back to request URL, got empty") + if m.MatchURL != srv.URL { + t.Fatalf("MatchURL should fall back to request URL %q, got %q", srv.URL, m.MatchURL) } if m.MatcherType == "" || m.MatcherValue == "" { t.Fatalf("matcher fields empty: %+v", m) diff --git a/fingers/match_detail.go b/fingers/match_detail.go new file mode 100644 index 0000000..e3e7a5c --- /dev/null +++ b/fingers/match_detail.go @@ -0,0 +1,143 @@ +package fingers + +import ( + "net/http" + "strings" + + "github.com/chainreactors/fingers/common" + "github.com/chainreactors/utils/httputils" +) + +// MatchResult is a flattened, easy-to-consume representation of a single +// fingerprint hit. It exposes the matcher metadata and the URL the match +// was attributed to, while keeping the original *common.Framework pointer +// for callers that need the full payload (attributes, tags, etc.). +type MatchResult struct { + // Framework is the original framework returned by the underlying engine. + Framework *common.Framework `json:"framework"` + + // MatchURL is the URL the match was attributed to. Resolution order: + // 1. MatchDetail.SendData's "url=" segment (populated by active + // probing flows that supply an explicit probe URL). + // 2. The fallback URL passed to MatchWithDetail / inferred from + // resp.Request.URL by MatchHTTPWithDetail (the common case for + // passive matching, which doesn't itself emit a URL). + MatchURL string `json:"match_url,omitempty"` + + // Convenience copies of MatchDetail. Empty when matcher detail has not + // been enabled or the engine did not produce matcher information. + MatcherType string `json:"matcher_type,omitempty"` + MatcherValue string `json:"matcher_value,omitempty"` + RuleIndex int `json:"rule_index"` + SendData string `json:"send_data,omitempty"` +} + +// EnableMatchDetail toggles matcher detail collection on the underlying +// fingers engine. Idempotent and cheap on subsequent calls. +// +// MatchWithDetail / MatchHTTPWithDetail invoke this automatically on first +// use; call it explicitly if you also want plain Match() / MatchHTTP() to +// fill MatchDetail on the returned *common.Framework. +// +// Background: NewEngine internally runs Compile() on the loaded fingers, +// which resets each finger's per-rule EnableMatchDetail flag back to the +// engine default (false). EnableMatchDetail flips both the engine flag +// and every per-finger flag, ensuring matcher metadata is collected on +// subsequent matches. +func (e *Engine) EnableMatchDetail() { + if e == nil || e.engine == nil { + return + } + fe, err := e.GetFingersEngine() + if err != nil || fe == nil || fe.MatchDetailEnabled { + return + } + fe.EnableMatchDetail() +} + +// MatchWithDetail is the detail-aware counterpart of Match. It runs the +// same passive content match and returns []MatchResult with matcher +// metadata flattened from common.Framework.MatchDetail. +// +// fallbackURL is used as MatchResult.MatchURL when MatchDetail.SendData +// does not contain a "url=" segment (the common case for passive +// matching). Pass the URL of the request whose response body you are +// matching; if you only have a raw byte slice with no associated URL, +// pass "". +func (e *Engine) MatchWithDetail(data []byte, fallbackURL string) ([]MatchResult, error) { + if e == nil || e.engine == nil { + return nil, nil + } + e.EnableMatchDetail() + frames, err := e.engine.DetectContent(data) + if err != nil { + return nil, err + } + return flattenMatchResults(frames, fallbackURL), nil +} + +// MatchHTTPWithDetail is the detail-aware counterpart of MatchHTTP. It +// auto-enables matcher detail and returns []MatchResult, using +// resp.Request.URL.String() as the MatchURL fallback when MatchDetail +// does not carry an explicit probe URL. +func (e *Engine) MatchHTTPWithDetail(resp *http.Response) ([]MatchResult, error) { + if e == nil || e.engine == nil { + return nil, nil + } + e.EnableMatchDetail() + + fallbackURL := "" + if resp != nil && resp.Request != nil && resp.Request.URL != nil { + fallbackURL = resp.Request.URL.String() + } + var data []byte + if resp != nil { + data = httputils.ReadRaw(resp) + } + frames, err := e.engine.DetectContent(data) + if err != nil { + return nil, err + } + return flattenMatchResults(frames, fallbackURL), nil +} + +func flattenMatchResults(frames common.Frameworks, fallbackURL string) []MatchResult { + out := make([]MatchResult, 0, len(frames)) + for _, f := range frames { + if f == nil { + continue + } + r := MatchResult{Framework: f, MatchURL: fallbackURL} + if d := f.MatchDetail; d != nil { + r.MatcherType = d.MatcherType + r.MatcherValue = d.MatcherValue + r.RuleIndex = d.RuleIndex + r.SendData = d.SendData + if u := extractMatchURL(d.SendData); u != "" { + r.MatchURL = u + } + } + out = append(out, r) + } + return out +} + +// extractMatchURL extracts the value of the "url=" segment from a SendData +// string formatted as "scope=... method=... url=". The URL value may +// contain '=' characters (query params), so we take everything from "url=" +// to end-of-string after locating the segment at a word boundary. +func extractMatchURL(sendData string) string { + const tag = "url=" + for start := 0; start < len(sendData); { + i := strings.Index(sendData[start:], tag) + if i < 0 { + return "" + } + i += start + if i == 0 || sendData[i-1] == ' ' { + return strings.TrimSpace(sendData[i+len(tag):]) + } + start = i + len(tag) + } + return "" +} diff --git a/fingers/match_detail_test.go b/fingers/match_detail_test.go new file mode 100644 index 0000000..e38a38a --- /dev/null +++ b/fingers/match_detail_test.go @@ -0,0 +1,156 @@ +package fingers + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/chainreactors/fingers/common" + fingersEngine "github.com/chainreactors/fingers/fingers" +) + +// TestMatchHTTPWithDetail_HappyPath: 端到端验证客户视角的最短调用。 +// 不需要调 EnableMatchDetail / GetFingersEngine / 解析 SendData。 +func TestMatchHTTPWithDetail_HappyPath(t *testing.T) { + eng := newTestEngine(t, "demo-app", "DemoMarker") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("body with DemoMarker")) + })) + defer srv.Close() + resp, err := http.Get(srv.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + results, err := eng.MatchHTTPWithDetail(resp) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + r := results[0] + if r.Framework == nil || r.Framework.Name != "demo-app" { + t.Fatalf("Framework not preserved: %+v", r) + } + if r.MatchURL != srv.URL { + t.Fatalf("MatchURL should fall back to request URL %q, got %q", srv.URL, r.MatchURL) + } + if r.MatcherType == "" || r.MatcherValue == "" { + t.Fatalf("matcher fields empty: %+v", r) + } +} + +// TestMatchWithDetail_FallbackURL: 已有原始字节 + 传入 URL 的场景 +func TestMatchWithDetail_FallbackURL(t *testing.T) { + eng := newTestEngine(t, "raw-app", "RawMarker") + raw := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\nbody with RawMarker here") + results, err := eng.MatchWithDetail(raw, "https://provided.example/path") + if err != nil { + t.Fatal(err) + } + if len(results) == 0 { + t.Fatal("expected at least one result") + } + if results[0].MatchURL != "https://provided.example/path" { + t.Fatalf("MatchURL fallback not used; got %q", results[0].MatchURL) + } +} + +// TestEnableMatchDetail_AffectsPlainMatch: 显式调用 EnableMatchDetail 后, +// 即便走 plain Match() 也能拿到 MatchDetail +func TestEnableMatchDetail_AffectsPlainMatch(t *testing.T) { + eng := newTestEngine(t, "plain-app", "PlainMarker") + eng.EnableMatchDetail() + frames, err := eng.Match([]byte("HTTP/1.1 200 OK\r\n\r\nPlainMarker")) + if err != nil { + t.Fatal(err) + } + fw, ok := frames["plain-app"] + if !ok { + t.Fatalf("expected plain-app, got: %v", frames) + } + if fw.MatchDetail == nil { + t.Fatal("MatchDetail should be populated after EnableMatchDetail()") + } +} + +// TestPlainMatchWithoutEnable_NoMatchDetail: 反向对照,不调 EnableMatchDetail +// 时 plain Match 不应填充 MatchDetail (证明 EnableMatchDetail 是必需的) +func TestPlainMatchWithoutEnable_NoMatchDetail(t *testing.T) { + eng := newTestEngine(t, "off-app", "OffMarker") + frames, _ := eng.Match([]byte("HTTP/1.1 200 OK\r\n\r\nOffMarker")) + if fw, ok := frames["off-app"]; ok && fw.MatchDetail != nil { + t.Fatalf("MatchDetail should be nil without EnableMatchDetail(), got %+v", *fw.MatchDetail) + } +} + +// TestFlattenMatchResults_SendDataURLWins: SendData 含 url= 时优先用它 +func TestFlattenMatchResults_SendDataURLWins(t *testing.T) { + frames := common.Frameworks{ + "x": &common.Framework{ + Name: "x", + MatchDetail: &common.MatchDetail{ + MatcherType: "word", + MatcherValue: "foo", + SendData: "scope=cp method=GET url=https://active.example/admin", + }, + }, + } + out := flattenMatchResults(frames, "https://fallback.example/") + if len(out) != 1 || out[0].MatchURL != "https://active.example/admin" { + t.Fatalf("expected MatchURL from SendData, got %+v", out) + } +} + +// TestExtractMatchURL: 词边界 / 含 = 的 url value / 缺失 / 子串误匹配 +func TestExtractMatchURL(t *testing.T) { + cases := []struct{ in, want string }{ + {"", ""}, + {"url=https://x.test/a", "https://x.test/a"}, + {"scope=cp method=GET url=https://x.test/p", "https://x.test/p"}, + {"url=https://x.test/api?next=https://y/foo=1", "https://x.test/api?next=https://y/foo=1"}, + {"scope=cp method=GET", ""}, + {"scope=anonymous_url=x method=GET", ""}, + } + for _, c := range cases { + if got := extractMatchURL(c.in); got != c.want { + t.Errorf("extractMatchURL(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestDetailMethods_NilSafe(t *testing.T) { + var nilEngine *Engine + nilEngine.EnableMatchDetail() + + results, err := nilEngine.MatchWithDetail(nil, "https://fallback.example/") + if err != nil || results != nil { + t.Fatalf("nil MatchWithDetail = (%v, %v), want (nil, nil)", results, err) + } + + results, err = nilEngine.MatchHTTPWithDetail(nil) + if err != nil || results != nil { + t.Fatalf("nil MatchHTTPWithDetail = (%v, %v), want (nil, nil)", results, err) + } + + emptyEngine := &Engine{} + results, err = emptyEngine.MatchHTTPWithDetail(nil) + if err != nil || results != nil { + t.Fatalf("empty MatchHTTPWithDetail = (%v, %v), want (nil, nil)", results, err) + } +} + +func newTestEngine(t *testing.T, name, marker string) *Engine { + t.Helper() + eng, err := NewEngine(NewConfig().WithFingers(fingersEngine.Fingers{{ + Name: name, + Protocol: "http", + Rules: fingersEngine.Rules{{Regexps: &fingersEngine.Regexps{Body: []string{marker}}}}, + }})) + if err != nil { + t.Fatal(err) + } + return eng +}