From 9519fe70178f820af742008e377b5eb9ea3f8a2e Mon Sep 17 00:00:00 2001 From: Andrii Iudin Date: Wed, 21 Jan 2026 16:31:13 +0000 Subject: [PATCH 1/2] Fix nested ref resolution context for remote docs --- datamodel/low/extraction_functions.go | 39 ++++++++--- index/index_model.go | 4 ++ index/resolver.go | 55 ++++++++++++--- index/resolver_test.go | 99 +++++++++++++++++++++++++++ utils/utils.go | 42 +++++++++--- utils/utils_test.go | 32 +++++++++ 6 files changed, 244 insertions(+), 27 deletions(-) diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index 72302e52..e81af9bd 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -126,27 +126,50 @@ func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.S for _, collection := range collections { found = collection() if found != nil && found[rv] != nil { + foundRef := found[rv] + foundIndex := idx + if foundRef.Index != nil { + foundIndex = foundRef.Index + } + if foundIndex != nil && foundRef.RemoteLocation != "" && + foundIndex.GetSpecAbsolutePath() != foundRef.RemoteLocation { + if rolo := foundIndex.GetRolodex(); rolo != nil { + for _, candidate := range append(rolo.GetIndexes(), rolo.GetRootIndex()) { + if candidate == nil { + continue + } + if candidate.GetSpecAbsolutePath() == foundRef.RemoteLocation { + foundIndex = candidate + break + } + } + } + } + foundCtx := ctx + if foundRef.RemoteLocation != "" { + foundCtx = context.WithValue(foundCtx, index.CurrentPathKey, foundRef.RemoteLocation) + } // if this is a ref node, we need to keep diving // until we hit something that isn't a ref. - if jh, _, _ := utils.IsNodeRefValue(found[rv].Node); jh { + if jh, _, _ := utils.IsNodeRefValue(foundRef.Node); jh { // if this node is circular, stop drop and roll. - if !IsCircular(found[rv].Node, idx) && found[rv].Node != root { - return LocateRefNodeWithContext(ctx, found[rv].Node, idx) + if !IsCircular(foundRef.Node, foundIndex) && foundRef.Node != root { + return LocateRefNodeWithContext(foundCtx, foundRef.Node, foundIndex) } else { - crr := GetCircularReferenceResult(found[rv].Node, idx) + crr := GetCircularReferenceResult(foundRef.Node, foundIndex) jp := "" if crr != nil { jp = crr.GenerateJourneyPath() } - return found[rv].Node, idx, fmt.Errorf("circular reference '%s' found during lookup at line "+ + return foundRef.Node, foundIndex, fmt.Errorf("circular reference '%s' found during lookup at line "+ "%d, column %d, It cannot be resolved", jp, - found[rv].Node.Line, - found[rv].Node.Column), ctx + foundRef.Node.Line, + foundRef.Node.Column), foundCtx } } - return utils.NodeAlias(found[rv].Node), idx, nil, ctx + return utils.NodeAlias(foundRef.Node), foundIndex, nil, foundCtx } } diff --git a/index/index_model.go b/index/index_model.go index 2739e978..c09eabf5 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -211,6 +211,10 @@ type SpecIndexConfig struct { // When enabled, properties from referenced schemas will be merged with local sibling properties. MergeReferencedProperties bool + // ResolveNestedRefsWithDocumentContext uses the referenced document's path/index as the base for any nested refs. + // This is disabled by default to preserve historical resolver behavior. + ResolveNestedRefsWithDocumentContext bool + // PropertyMergeStrategy defines how to handle conflicts when merging properties. PropertyMergeStrategy datamodel.PropertyMergeStrategy diff --git a/index/resolver.go b/index/resolver.go index 89d14c58..fb8f9e39 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -4,6 +4,7 @@ package index import ( + "context" "errors" "fmt" "net/url" @@ -348,6 +349,35 @@ func visitIndex(res *Resolver, idx *SpecIndex) { } } +// searchReferenceWithContext resolves a reference using document context when enabled in the config. +func (resolver *Resolver) searchReferenceWithContext(sourceRef, searchRef *Reference) (*Reference, *SpecIndex, context.Context) { + if resolver.specIndex == nil || resolver.specIndex.config == nil || !resolver.specIndex.config.ResolveNestedRefsWithDocumentContext { + ref, idx := resolver.specIndex.SearchIndexForReferenceByReference(searchRef) + return ref, idx, context.Background() + } + + searchIndex := resolver.specIndex + if searchRef != nil && searchRef.Index != nil { + searchIndex = searchRef.Index + } else if sourceRef != nil && sourceRef.Index != nil { + searchIndex = sourceRef.Index + } + + ctx := context.Background() + currentPath := "" + if sourceRef != nil { + currentPath = sourceRef.RemoteLocation + } + if currentPath == "" && searchIndex != nil { + currentPath = searchIndex.specAbsolutePath + } + if currentPath != "" { + ctx = context.WithValue(ctx, CurrentPathKey, currentPath) + } + + return searchIndex.SearchIndexForReferenceByReferenceWithContext(ctx, searchRef) +} + // VisitReference will visit a reference as part of a journey and will return resolved nodes. func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { resolver.referencesVisited++ @@ -374,7 +404,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j if j.FullDefinition == r.FullDefinition { var foundDup *Reference - foundRef, _ := resolver.specIndex.SearchIndexForReferenceByReference(r) + foundRef, _, _ := resolver.searchReferenceWithContext(ref, r) if foundRef != nil { foundDup = foundRef } @@ -421,7 +451,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j if !skip { var original *Reference - foundRef, _ := resolver.specIndex.SearchIndexForReferenceByReference(r) + foundRef, _, _ := resolver.searchReferenceWithContext(ref, r) if foundRef != nil { original = foundRef } @@ -530,7 +560,7 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No depth++ var foundRef *Reference - foundRef, _ = resolver.specIndex.SearchIndexForReferenceByReference(ref) + foundRef, _, _ = resolver.searchReferenceWithContext(ref, ref) if foundRef != nil && !foundRef.Circular { found = append(found, resolver.extractRelatives(foundRef, n, node, foundRelatives, journey, seen, resolve, depth)...) depth-- @@ -591,10 +621,14 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No } } else { // local component, full def is based on passed in ref - if strings.HasPrefix(ref.FullDefinition, "http") { + baseLocation := ref.FullDefinition + if ref.RemoteLocation != "" { + baseLocation = ref.RemoteLocation + } + if strings.HasPrefix(baseLocation, "http") { // split the http URI into parts - httpExp := strings.Split(ref.FullDefinition, "#/") + httpExp := strings.Split(baseLocation, "#/") // parse a URL from the full def u, _ := url.Parse(httpExp[0]) @@ -604,7 +638,7 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No } else { // split the full def into parts - fileDef := strings.Split(ref.FullDefinition, "#/") + fileDef := strings.Split(baseLocation, "#/") fullDef = fmt.Sprintf("%s#/%s", fileDef[0], exp[1]) } } @@ -618,7 +652,11 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No } else { // split the full def into parts - fileDef := strings.Split(ref.FullDefinition, "#/") + baseLocation := ref.FullDefinition + if ref.RemoteLocation != "" { + baseLocation = ref.RemoteLocation + } + fileDef := strings.Split(baseLocation, "#/") // is the file def a http link? if strings.HasPrefix(fileDef[0], "http") { @@ -639,9 +677,10 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No FullDefinition: fullDef, RemoteLocation: ref.RemoteLocation, IsRemote: true, + Index: ref.Index, } - locatedRef, _ = resolver.specIndex.SearchIndexForReferenceByReference(searchRef) + locatedRef, _, _ = resolver.searchReferenceWithContext(ref, searchRef) if locatedRef == nil { _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) diff --git a/index/resolver_test.go b/index/resolver_test.go index d8e876ee..b4aabd02 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "net/http" + "net/http/httptest" "net/url" "os" "path/filepath" @@ -674,6 +675,104 @@ func TestResolver_ResolveThroughPaths(t *testing.T) { assert.Len(t, err, 0) } +func TestResolver_ResolveComponents_RemoteNestedRefs(t *testing.T) { + remoteSpec := `openapi: 3.0.0 +info: + title: Remote + version: 1.0.0 +components: + schemas: + landingPage: + type: object + properties: + extra: + $ref: "schemas/extra.yaml#/components/schemas/Extra" + responses: + LandingPage: + description: landing + content: + application/json: + schema: + $ref: "#/components/schemas/landingPage"` + + extraSpec := `openapi: 3.0.0 +info: + title: Extra + version: 1.0.0 +components: + schemas: + Extra: + type: object + properties: + name: + type: string` + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/remote.yaml": + _, _ = rw.Write([]byte(remoteSpec)) + case "/schemas/extra.yaml": + _, _ = rw.Write([]byte(extraSpec)) + default: + rw.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + rootSpec := fmt.Sprintf(`openapi: 3.0.0 +info: + title: Root + version: 1.0.0 +paths: + /landing: + get: + responses: + "200": + $ref: "%s/remote.yaml#/components/responses/LandingPage"`, server.URL) + + tempDir := t.TempDir() + rootPath := filepath.Join(tempDir, "root.yaml") + writeErr := os.WriteFile(rootPath, []byte(rootSpec), 0o600) + assert.NoError(t, writeErr) + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(rootSpec), &rootNode) + + config := CreateOpenAPIIndexConfig() + config.AllowRemoteLookup = true + config.AvoidBuildIndex = true + config.AvoidCircularReferenceCheck = true + config.ResolveNestedRefsWithDocumentContext = true + config.BasePath = tempDir + config.SpecAbsolutePath = rootPath + config.ExtractRefsSequentially = true + + rolo := NewRolodex(config) + rolo.SetRootNode(&rootNode) + + remoteFS, _ := NewRemoteFSWithRootURL(server.URL) + remoteFS.SetIndexConfig(config) + remoteFS.RemoteHandlerFunc = (&http.Client{}).Get + + fsCfg := LocalFSConfig{ + BaseDirectory: config.BasePath, + IndexConfig: config, + } + fileFS, err := NewLocalFSWithConfig(&fsCfg) + assert.NoError(t, err) + + rolo.AddLocalFS(config.BasePath, fileFS) + rolo.AddRemoteFS(server.URL, remoteFS) + + indexedErr := rolo.IndexTheRolodex(context.Background()) + assert.NoError(t, indexedErr) + + rolo.Resolve() + resolver := rolo.GetRootIndex().GetResolver() + assert.NotNil(t, resolver) + assert.Len(t, resolver.GetResolvingErrors(), 0) +} + func TestResolver_ResolveComponents_MixedRef(t *testing.T) { mixedref, _ := os.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") var rootNode yaml.Node diff --git a/utils/utils.go b/utils/utils.go index 91c5b4f1..7dbc105c 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,7 +1,6 @@ package utils import ( - "context" "crypto/rand" "encoding/json" "fmt" @@ -12,6 +11,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/pb33f/jsonpath/pkg/jsonpath" @@ -42,14 +42,36 @@ const ( UnknownCase ) +type cachedJSONPath struct { + path *jsonpath.JSONPath + err error +} + +// jsonPathCache stores compiled JSONPath expressions keyed by normalized string. +var jsonPathCache sync.Map + +// getJSONPath returns a cached JSONPath when available, compiling and caching otherwise. +func getJSONPath(rawPath string) (*jsonpath.JSONPath, error) { + cleaned := FixContext(rawPath) + if cached, ok := jsonPathCache.Load(cleaned); ok { + entry := cached.(cachedJSONPath) + return entry.path, entry.err + } + + path, err := jsonpath.NewPath(cleaned, jsonpathconfig.WithPropertyNameExtension()) + jsonPathCache.Store(cleaned, cachedJSONPath{ + path: path, + err: err, + }) + return path, err +} + // FindNodes will find a node based on JSONPath, it accepts raw yaml/json as input. func FindNodes(yamlData []byte, jsonPath string) ([]*yaml.Node, error) { - jsonPath = FixContext(jsonPath) - var node yaml.Node yaml.Unmarshal(yamlData, &node) - path, err := jsonpath.NewPath(jsonPath, jsonpathconfig.WithPropertyNameExtension()) + path, err := getJSONPath(jsonPath) if err != nil { return nil, err } @@ -116,18 +138,16 @@ func FindNodesWithoutDeserializing(node *yaml.Node, jsonPath string) ([]*yaml.No // FindNodesWithoutDeserializingWithTimeout will find a node based on JSONPath, without deserializing from yaml/json // This function can be customized with a timeout. func FindNodesWithoutDeserializingWithTimeout(node *yaml.Node, jsonPath string, timeout time.Duration) ([]*yaml.Node, error) { - jsonPath = FixContext(jsonPath) - - path, err := jsonpath.NewPath(jsonPath, jsonpathconfig.WithPropertyNameExtension()) + path, err := getJSONPath(jsonPath) if err != nil { return nil, err } // this can spin out, to lets gatekeep it. - done := make(chan struct{}) + done := make(chan struct{}, 1) var results []*yaml.Node - to, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() + timer := time.NewTimer(timeout) + defer timer.Stop() go func() { results = path.Query(node) done <- struct{}{} @@ -136,7 +156,7 @@ func FindNodesWithoutDeserializingWithTimeout(node *yaml.Node, jsonPath string, select { case <-done: return results, nil - case <-to.Done(): + case <-timer.C: return nil, fmt.Errorf("node lookup timeout exceeded (%v)", timeout) } } diff --git a/utils/utils_test.go b/utils/utils_test.go index 31726e48..1ed76d2c 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -56,6 +56,30 @@ func TestFindNodes_BadPath(t *testing.T) { assert.Nil(t, nodes) } +func TestGetJSONPath_CacheHit(t *testing.T) { + jsonPathCache = sync.Map{} + + path1, err := getJSONPath("$.info.contact") + assert.NoError(t, err) + assert.NotNil(t, path1) + + path2, err := getJSONPath("$.info.contact") + assert.NoError(t, err) + assert.Equal(t, path1, path2) +} + +func TestGetJSONPath_CacheHit_Invalid(t *testing.T) { + jsonPathCache = sync.Map{} + + path1, err := getJSONPath("I am not valid") + assert.Error(t, err) + assert.Nil(t, path1) + + path2, err := getJSONPath("I am not valid") + assert.Error(t, err) + assert.Equal(t, path1, path2) +} + func TestFindLastChildNode(t *testing.T) { nodes, _ := FindNodes(getPetstore(), "$.info") lastNode := FindLastChildNode(nodes[0]) @@ -1548,6 +1572,14 @@ func TestFindNodesWithoutDeserializingWithTimeout(t *testing.T) { assert.Error(t, err) } +func TestFindNodesWithoutDeserializingWithTimeout_Success(t *testing.T) { + root, _ := FindNodes(getPetstore(), "$") + nodes, err := FindNodesWithoutDeserializingWithTimeout(root[0], "$.info.contact", 100*time.Millisecond) + assert.NoError(t, err) + assert.NotNil(t, nodes) + assert.Len(t, nodes, 1) +} + func TestGenerateAlphanumericString(t *testing.T) { reg := regexp.MustCompile("^[0-9A-Za-z]{1,4}$") assert.NotNil(t, reg.MatchString(GenerateAlphanumericString(4))) From 5da73a0cba024153721b77151bd5368ddfb50803 Mon Sep 17 00:00:00 2001 From: Andrii Iudin Date: Thu, 22 Jan 2026 09:59:11 +0000 Subject: [PATCH 2/2] Add coverage for nested ref context changes --- datamodel/low/extraction_functions_test.go | 75 ++++++++++++++++++++++ index/resolver_test.go | 53 +++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/datamodel/low/extraction_functions_test.go b/datamodel/low/extraction_functions_test.go index f0959d23..cdbb62f1 100644 --- a/datamodel/low/extraction_functions_test.go +++ b/datamodel/low/extraction_functions_test.go @@ -1731,6 +1731,81 @@ func TestLocateRefNode_CurrentPathKey_HttpLink_RemoteCtx_WithPath(t *testing.T) assert.NotNil(t, c) } +func TestLocateRefNodeWithContext_RemoteIndexLookup(t *testing.T) { + tempDir := t.TempDir() + rootPath := filepath.Join(tempDir, "root.yaml") + externalPath := filepath.Join(tempDir, "external.yaml") + + rootCfg := index.CreateClosedAPIIndexConfig() + rootCfg.SpecAbsolutePath = rootPath + rootNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + rootIdx := index.NewSpecIndexWithConfig(rootNode, rootCfg) + + externalCfg := index.CreateClosedAPIIndexConfig() + externalCfg.SpecAbsolutePath = externalPath + externalNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + externalIdx := index.NewSpecIndexWithConfig(externalNode, externalCfg) + + rolo := index.NewRolodex(rootCfg) + rolo.AddExternalIndex(externalIdx, externalPath) + + rootIdx.SetRolodex(rolo) + externalIdx.SetRolodex(rolo) + + refValue := "external.yaml#/components/schemas/Thing" + refNode := utils.CreateRefNode(refValue) + + rootIdx.SetMappedReferences(map[string]*index.Reference{ + refValue: { + FullDefinition: refValue, + Node: utils.CreateStringNode("value"), + RemoteLocation: externalPath, + Index: rootIdx, + }, + }) + + _, foundIdx, _, foundCtx := LocateRefNodeWithContext(context.Background(), refNode, rootIdx) + assert.Equal(t, externalIdx, foundIdx) + assert.Equal(t, externalPath, foundCtx.Value(index.CurrentPathKey)) +} + +func TestLocateRefNodeWithContext_RolodexNilCandidate(t *testing.T) { + tempDir := t.TempDir() + rootPath := filepath.Join(tempDir, "root.yaml") + dummyPath := filepath.Join(tempDir, "dummy.yaml") + + rootCfg := index.CreateClosedAPIIndexConfig() + rootCfg.SpecAbsolutePath = rootPath + rootNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + rootIdx := index.NewSpecIndexWithConfig(rootNode, rootCfg) + + dummyCfg := index.CreateClosedAPIIndexConfig() + dummyCfg.SpecAbsolutePath = dummyPath + dummyNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + dummyIdx := index.NewSpecIndexWithConfig(dummyNode, dummyCfg) + + rolo := index.NewRolodex(rootCfg) + rolo.AddExternalIndex(dummyIdx, dummyPath) + + rootIdx.SetRolodex(rolo) + dummyIdx.SetRolodex(rolo) + + refValue := "missing.yaml#/components/schemas/Thing" + refNode := utils.CreateRefNode(refValue) + + rootIdx.SetMappedReferences(map[string]*index.Reference{ + refValue: { + FullDefinition: refValue, + Node: utils.CreateStringNode("value"), + RemoteLocation: "not-matching.yaml", + Index: rootIdx, + }, + }) + + _, foundIdx, _, _ := LocateRefNodeWithContext(context.Background(), refNode, rootIdx) + assert.Equal(t, rootIdx, foundIdx) +} + func TestLocateRefNode_CurrentPathKey_Path_Link(t *testing.T) { no := yaml.Node{ Kind: yaml.MappingNode, diff --git a/index/resolver_test.go b/index/resolver_test.go index b4aabd02..b63fcb8c 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -773,6 +773,59 @@ paths: assert.Len(t, resolver.GetResolvingErrors(), 0) } +func TestResolver_SearchReferenceWithContext_SourceIndex(t *testing.T) { + rootCfg := CreateClosedAPIIndexConfig() + rootCfg.ResolveNestedRefsWithDocumentContext = true + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte("openapi: 3.0.0"), &rootNode) + rootIdx := NewSpecIndexWithConfig(&rootNode, rootCfg) + + otherCfg := CreateClosedAPIIndexConfig() + otherCfg.SpecAbsolutePath = filepath.Join(t.TempDir(), "other.yaml") + var otherNode yaml.Node + _ = yaml.Unmarshal([]byte("openapi: 3.0.0"), &otherNode) + otherIdx := NewSpecIndexWithConfig(&otherNode, otherCfg) + + searchRef := &Reference{ + FullDefinition: "#/components/schemas/Thing", + } + otherIdx.SetMappedReferences(map[string]*Reference{ + searchRef.FullDefinition: { + FullDefinition: searchRef.FullDefinition, + Node: utils.CreateStringNode("value"), + Index: otherIdx, + RemoteLocation: otherCfg.SpecAbsolutePath, + }, + }) + + resolver := NewResolver(rootIdx) + sourceRef := &Reference{ + Index: otherIdx, + RemoteLocation: otherCfg.SpecAbsolutePath, + } + + foundRef, foundIdx, _ := resolver.searchReferenceWithContext(sourceRef, searchRef) + assert.NotNil(t, foundRef) + assert.Equal(t, otherIdx, foundIdx) +} + +func TestResolver_ExtractRelatives_HttpFullDefinition(t *testing.T) { + refNode := utils.CreateRefNode("#/components/schemas/Root") + ref := &Reference{ + FullDefinition: "#/components/schemas/Root", + Node: refNode, + } + + targetNode := utils.CreateRefNode("http://example.com/other.yaml#/components/schemas/Thing") + + idx := NewSpecIndexWithConfig(refNode, CreateClosedAPIIndexConfig()) + resolver := NewResolver(idx) + ref.Index = idx + + _ = resolver.extractRelatives(ref, targetNode, nil, map[string]bool{}, []*Reference{}, map[int]bool{}, false, 0) + assert.NotEmpty(t, resolver.GetResolvingErrors()) +} + func TestResolver_ResolveComponents_MixedRef(t *testing.T) { mixedref, _ := os.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") var rootNode yaml.Node