diff --git a/external/.golangci.yaml b/external/.golangci.yaml new file mode 100644 index 00000000000..c66c97b37a0 --- /dev/null +++ b/external/.golangci.yaml @@ -0,0 +1,144 @@ +version: "2" + +linters: + default: all + # prettier-ignore + disable: + - forbidigo + - paralleltest + - tparallel + - cyclop # + - depguard # Too annoying + - err113 # will re-add later (another-rex) + - exhaustruct # overkill (g-rath) + - forcetypeassert # too hard (g-rath) + - funlen # + - funcorder # + - gochecknoglobals # disagree with, for non changing variables (another-rex) + - gocognit # + - goconst # not everything should be a constant + - gocyclo # + - godot # comments are fine without full stops (g-rath) + - godox # to-do comments are fine (g-rath) + - ireturn # disagree with, sort of (g-rath) + - lll # line length is hard (g-rath) + - maintidx # + - mnd # not every number is magic (g-rath) + - nestif # + - noctx # Most of these don't need a context + - noinlineerr # + - nonamedreturns # disagree with, for now (another-rex) + - tagliatelle # we're parsing data from external sources (g-rath) + - testpackage # will re-add later (another-rex) + - varnamelen # maybe later (g-rath) + - wrapcheck # too difficult, will re-add later (another-rex) + - wsl # disagree with, for now (g-rath) + - wsl_v5 # disagree with, for now (g-rath) + settings: + exhaustive: + default-signifies-exhaustive: true + gocritic: + disabled-checks: + - ifElseChain + nlreturn: + block-size: 2 + revive: + rules: + - name: increment-decrement + disabled: true + - name: blank-imports + disabled: false + - name: context-as-argument + disabled: false + - name: context-keys-type + disabled: false + - name: dot-imports + disabled: false + - name: empty-block + disabled: false + - name: error-naming + disabled: false + - name: error-return + disabled: false + - name: error-strings + disabled: false + - name: errorf + disabled: false + - name: exported + disabled: false + arguments: + # TODO: get these all enabled + - "check-private-receivers" + # - "check-public-interface" + - "disable-checks-on-constants" + - "disable-checks-on-functions" + - "disable-checks-on-methods" + - "disable-checks-on-types" + - "disable-checks-on-variables" + - name: import-alias-naming + disabled: false + - name: import-shadowing + disabled: false + - name: indent-error-flow + disabled: false + - name: package-comments + disabled: false + - name: range + disabled: false + - name: receiver-naming + disabled: false + - name: redefines-builtin-id + disabled: false + - name: redundant-test-main-exit + disabled: false + - name: superfluous-else + disabled: false + - name: time-naming + disabled: false + - name: unexported-return + disabled: false + - name: unreachable-code + disabled: false + - name: unused-parameter + disabled: false + - name: use-any + disabled: false + - name: var-declaration + disabled: false + - name: var-naming + disabled: false + arguments: + - [] # AllowList + - [] # DenyList + - - skip-package-name-checks: true + exclusions: + generated: lax + presets: + - common-false-positives + - legacy + - std-error-handling + rules: + - path: _test\.go + linters: + - dupl + - path-except: _test\.go + text: use `testutility.GetCurrentWorkingDirectory` + paths: + - third_party$ + - builtin$ + - examples$ + +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/external/cmd/ids/README.md b/external/cmd/ids/README.md new file mode 100644 index 00000000000..4fe4620bd02 --- /dev/null +++ b/external/cmd/ids/README.md @@ -0,0 +1,25 @@ +# IDs Tool + +This utility assigns IDs to OSV records in a directory. It ensures that IDs are unique and follow a specified prefix and year format. + +It is predominately used by [PYSEC](https://github.com/pypa/advisory-database/blob/main/.github/workflows/automation.yaml) and [Malicious Packages](https://github.com/ossf/malicious-packages/blob/main/.github/workflows/assign-osv-ids.yml). + +## Usage + +```bash +go run main.go [flags] +``` + +### Flags + +- `-prefix`: Vulnerability prefix (e.g., "PYSEC"). Required field. +- `-dir`: Path to vulnerabilities. Required field. +- `-format`: Format of OSV reports in the repository. Must be "json" or "yaml" (default: "yaml"). + +## Description + +The tool performs the following steps: +1. Walks the specified directory to find unassigned vulnerabilities (files starting with `PREFIX-0000-`). +2. Determines the maximum allocated ID for each year. +3. Assigns new IDs to unassigned vulnerabilities, incrementing the counter for the respective year. +4. Renames the files to match the new IDs. diff --git a/external/cmd/ids/main.go b/external/cmd/ids/main.go new file mode 100644 index 00000000000..7555b107f37 --- /dev/null +++ b/external/cmd/ids/main.go @@ -0,0 +1,244 @@ +// package main contains a utility for assigning IDs to OSV records. +package main + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/goccy/go-yaml" + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/google/osv/vulnfeeds/vulns" + "github.com/ossf/osv-schema/bindings/go/osvschema" + "google.golang.org/protobuf/encoding/protojson" +) + +const ( + conflictFile = ".id-allocator" + conflictMarkerSize = 32 +) + +type fileFormat string + +const ( + fileFormatJSON = fileFormat("json") + fileFormatYAML = fileFormat("yaml") +) + +var ( + validFormats = []fileFormat{fileFormatYAML, fileFormatJSON} + formatToExtension = map[fileFormat]string{ + fileFormatYAML: ".yaml", + fileFormatJSON: ".json", + } +) + +func main() { + prefix := flag.String("prefix", "", "Vulnerability prefix (e.g. \"PYSEC\".") + dir := flag.String("dir", "", "Path to vulnerabilities.") + format := flag.String("format", string(fileFormatYAML), "Format of OSV reports in the repository. Must be \"json\" or \"yaml\".") + + flag.Parse() + + logger.InitGlobalLogger() + defer logger.Close() + + if *prefix == "" || *dir == "" { + flag.Usage() + return + } + + if !slices.Contains(validFormats, fileFormat(*format)) { + flag.Usage() + return + } + + if err := assignIDs(*prefix, *dir, fileFormat(*format)); err != nil { + logger.Info("Failed to assign IDs", slog.Any("err", err)) + logger.Close() // os.Exit() doesn't call deferred functions + os.Exit(1) //nolint:gocritic + } +} + +func extractYearAndNum(prefix, filename string, format fileFormat) (int, int) { + // Extract year and num from "PREFIX-YEAR-NUM" + parts := strings.Split(strings.TrimSuffix(filename, formatToExtension[format]), "-") + if len(parts) != 3 { + return 0, 0 + } + + if parts[0] != prefix { + return 0, 0 + } + + year, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, 0 + } + + num, err := strconv.Atoi(parts[2]) + if err != nil { + return 0, 0 + } + + return year, num +} + +func isUnassigned(prefix, filename string) bool { + return strings.HasPrefix(filename, prefix+"-0000-") +} + +func assignID(prefix, path string, format fileFormat, yearCounters map[int]int, defaultYear int) error { + // Parse the existing vulnerability. + readf, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open %s: %w", path, err) + } + defer readf.Close() + + vuln, err := readVulnWithFormat(readf, format) + if err != nil { + return fmt.Errorf("failed to parse %s: %w", format, err) + } + readf.Close() + + // If the vulnerability has a published date, use the year from that. + // Otherwise, just default to the current year. + year := defaultYear + if vuln.GetPublished() != nil { + year = vuln.GetPublished().AsTime().Year() + } + + // Allocate a new ID and write the new file. + id := yearCounters[year] + 1 + yearCounters[year] = id + + vuln.Id = fmt.Sprintf("%s-%d-%d", prefix, year, id) + newPath := filepath.Join(filepath.Dir(path), vuln.GetId()+formatToExtension[format]) + + writef, err := os.Create(newPath) + if err != nil { + return fmt.Errorf("failed to create %s: %w", newPath, err) + } + defer writef.Close() + + if err := writeVulnWithFormat(vuln.Vulnerability, writef, format); err != nil { + return fmt.Errorf("failed to serialize: %w", err) + } + + logger.Info("Assigning", slog.String("path", path), slog.String("newPath", newPath)) + + return os.Remove(path) +} + +func assignIDs(prefix, dir string, format fileFormat) error { + defaultYear := time.Now().Year() + var unassigned []string + yearCounters := map[int]int{} + + // Look for unassigned vulnerabilities, as well as the maximum allocated IDs for every year. + err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("failed to access %s: %w", path, err) + } + if info.IsDir() { + return nil + } + + filename := filepath.Base(path) + if isUnassigned(prefix, filename) { + unassigned = append(unassigned, path) + return nil + } + + year, num := extractYearAndNum(prefix, filename, format) + if year == 0 || num == 0 { + return nil + } + + if num > yearCounters[year] { + yearCounters[year] = num + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk: %w", err) + } + + if len(unassigned) == 0 { + logger.Info("Nothing to allocate") + return nil + } + + logger.Info("Assigning IDs using detected maximums", slog.Any("counters", yearCounters)) + for _, path := range unassigned { + if err := assignID(prefix, path, format, yearCounters, defaultYear); err != nil { + return fmt.Errorf("failed to assign ID for %s: %w", path, err) + } + } + + b := make([]byte, conflictMarkerSize) + if _, err := rand.Read(b); err != nil { + return fmt.Errorf("failed to generate random string: %w", err) + } + + return os.WriteFile(filepath.Join(dir, conflictFile), []byte(hex.EncodeToString(b)), 0600) +} + +func readVulnWithFormat(r io.Reader, format fileFormat) (*vulns.Vulnerability, error) { + switch format { + case fileFormatJSON: + return vulns.FromJSON(r) + case fileFormatYAML: + return vulns.FromYAML(r) + default: + return nil, fmt.Errorf("unknown file format: %v", format) + } +} + +func writeVulnWithFormat(v *osvschema.Vulnerability, w io.Writer, format fileFormat) error { + jsonBytes, err := protojson.Marshal(v) + if err != nil { + return err + } + + var data []byte + switch format { + case fileFormatJSON: + var buf bytes.Buffer + if err := json.Indent(&buf, jsonBytes, "", " "); err != nil { + return err + } + data = buf.Bytes() + case fileFormatYAML: + data, err = yaml.JSONToYAML(jsonBytes) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown file format: %v", format) + } + + // Ensure the output has a trailing newline to match the behavior of + // json.Encoder, which was previously used. + if len(data) > 0 && data[len(data)-1] != '\n' { + data = append(data, '\n') + } + + _, err = w.Write(data) + + return err +} diff --git a/external/cmd/ids/main_test.go b/external/cmd/ids/main_test.go new file mode 100644 index 00000000000..cecfa50cd4f --- /dev/null +++ b/external/cmd/ids/main_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAssignIDs(t *testing.T) { + tests := []struct { + name string + prefix string + format fileFormat + templateName string + existingName string + expectedID string + }{ + { + name: "OSV YAML", + prefix: "OSV", + format: fileFormatYAML, + templateName: "OSV-0000-abc.yaml", + existingName: "OSV-2026-10.yaml", + expectedID: "OSV-2026-11", + }, + { + name: "TEST JSON", + prefix: "TEST", + format: fileFormatJSON, + templateName: "TEST-0000-def.json", + existingName: "TEST-2026-20.json", + expectedID: "TEST-2026-21", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + + // 1. Copy existing assigned vulnerability from testdata to set counter. + existingPath := filepath.Join("testdata", tt.existingName) + existingData, err := os.ReadFile(existingPath) + if err != nil { + t.Fatalf("failed to read existing ID: %v", err) + } + err = os.WriteFile(filepath.Join(tmpDir, tt.existingName), existingData, 0600) + if err != nil { + t.Fatalf("failed to copy existing ID: %v", err) + } + + // 2. Setup unassigned vulnerability using template. + templatePath := filepath.Join("testdata", tt.templateName) + templateData, err := os.ReadFile(templatePath) + if err != nil { + t.Fatalf("failed to read template %s: %v", templatePath, err) + } + destPath := filepath.Join(tmpDir, tt.templateName) + if err := os.WriteFile(destPath, templateData, 0600); err != nil { + t.Fatalf("failed to setup unassigned vuln: %v", err) + } + + // 3. Run assignIDs. + if err := assignIDs(tt.prefix, tmpDir, tt.format); err != nil { + t.Fatalf("assignIDs failed: %v", err) + } + + // 4. Verify results. + ext := formatToExtension[tt.format] + expectedFilename := tt.expectedID + ext + gotPath := filepath.Join(tmpDir, expectedFilename) + if _, err := os.Stat(gotPath); os.IsNotExist(err) { + t.Errorf("Expected %s to exist", gotPath) + return + } + + if _, err := os.Stat(destPath); !os.IsNotExist(err) { + t.Errorf("Expected old file %s to be removed", tt.templateName) + } + + gotData, err := os.ReadFile(gotPath) + if err != nil { + t.Fatalf("failed to read assigned vuln %s: %v", gotPath, err) + } + + wantData, err := os.ReadFile(filepath.Join("testdata", expectedFilename)) + if err != nil { + t.Fatalf("failed to read expected vuln: %v", err) + } + + // Trim space to be robust against minor formatting differences, + // for example trailing newlines. + if !bytes.Equal(bytes.TrimSpace(gotData), bytes.TrimSpace(wantData)) { + t.Errorf("Data fidelity mismatch for %s:\nGot:\n%s\nWant:\n%s", tt.expectedID, string(gotData), string(wantData)) + } + + // Verify .id-allocator was created. + if _, err := os.Stat(filepath.Join(tmpDir, ".id-allocator")); os.IsNotExist(err) { + t.Errorf("Expected .id-allocator to exist") + } + }) + } +} + +func TestAssignIDsError(t *testing.T) { + tmpDir := t.TempDir() + invalidFile := filepath.Join(tmpDir, "OSV-0000-invalid.yaml") + if err := os.WriteFile(invalidFile, []byte("invalid yaml content"), 0600); err != nil { + t.Fatalf("failed to setup invalid vuln: %v", err) + } + + err := assignIDs("OSV", tmpDir, fileFormatYAML) + if err == nil { + t.Fatal("expected error but got nil") + } + + expectedErr := "failed to assign ID for " + invalidFile + if !strings.Contains(err.Error(), expectedErr) { + t.Errorf("Expected error message to contain %q, but got %q", expectedErr, err.Error()) + } +} diff --git a/external/cmd/ids/testdata/OSV-0000-abc.yaml b/external/cmd/ids/testdata/OSV-0000-abc.yaml new file mode 100644 index 00000000000..ed9f7e7648d --- /dev/null +++ b/external/cmd/ids/testdata/OSV-0000-abc.yaml @@ -0,0 +1,31 @@ +id: +published: "2026-01-02T00:00:00Z" +modified: "2026-01-02T00:00:00Z" +summary: A vulnerability +details: | + Blah blah blah + Blah +affected: +- package: + name: blah.com/package + ecosystem: Go + ranges: + - type: GIT + repo: https://osv-test/repo/url + events: + - introduced: eefe8ec3f1f90d0e684890e810f3f21e8500a4cd + - fixed: 8d8242f545e9cec3e6d0d2e3f5bde8be1c659735 + versions: + - branch-v0.1.1 +references: +- type: WEB + url: https://ref.com/ref +database_specific: + specific: 1337 +severity: +- type: CVSS_V3 + score: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L +credits: +- name: Foo bar + contact: + - mailto:foo@bar.com diff --git a/external/cmd/ids/testdata/OSV-2026-10.yaml b/external/cmd/ids/testdata/OSV-2026-10.yaml new file mode 100644 index 00000000000..6a9e3529ed8 --- /dev/null +++ b/external/cmd/ids/testdata/OSV-2026-10.yaml @@ -0,0 +1,2 @@ +id: OSV-2026-10 +modified: '2026-01-01T00:00:00Z' diff --git a/external/cmd/ids/testdata/OSV-2026-11.yaml b/external/cmd/ids/testdata/OSV-2026-11.yaml new file mode 100644 index 00000000000..981a997166c --- /dev/null +++ b/external/cmd/ids/testdata/OSV-2026-11.yaml @@ -0,0 +1,31 @@ +id: OSV-2026-11 +published: "2026-01-02T00:00:00Z" +modified: "2026-01-02T00:00:00Z" +summary: A vulnerability +details: | + Blah blah blah + Blah +affected: +- package: + name: blah.com/package + ecosystem: Go + ranges: + - type: GIT + repo: https://osv-test/repo/url + events: + - introduced: eefe8ec3f1f90d0e684890e810f3f21e8500a4cd + - fixed: 8d8242f545e9cec3e6d0d2e3f5bde8be1c659735 + versions: + - branch-v0.1.1 +references: +- type: WEB + url: https://ref.com/ref +database_specific: + specific: 1337 +severity: +- type: CVSS_V3 + score: CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L +credits: +- name: Foo bar + contact: + - mailto:foo@bar.com diff --git a/external/cmd/ids/testdata/TEST-0000-def.json b/external/cmd/ids/testdata/TEST-0000-def.json new file mode 100644 index 00000000000..3fec72bd2b0 --- /dev/null +++ b/external/cmd/ids/testdata/TEST-0000-def.json @@ -0,0 +1,55 @@ +{ + "id": "", + "published": "2026-01-02T00:00:00Z", + "modified": "2026-01-02T00:00:00Z", + "summary": "A vulnerability", + "details": "Blah blah blah\nBlah\n", + "affected": [ + { + "package": { + "name": "blah.com/package", + "ecosystem": "Go" + }, + "ranges": [ + { + "type": "GIT", + "repo": "https://osv-test/repo/url", + "events": [ + { + "introduced": "eefe8ec3f1f90d0e684890e810f3f21e8500a4cd" + }, + { + "fixed": "8d8242f545e9cec3e6d0d2e3f5bde8be1c659735" + } + ] + } + ], + "versions": [ + "branch-v0.1.1" + ] + } + ], + "references": [ + { + "type": "WEB", + "url": "https://ref.com/ref" + } + ], + "database_specific": { + "specific": 1337 + }, + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L" + } + ], + "credits": [ + { + "name": "Foo bar", + "contact": [ + "mailto:foo@bar.com" + ] + } + ] +} diff --git a/external/cmd/ids/testdata/TEST-2026-20.json b/external/cmd/ids/testdata/TEST-2026-20.json new file mode 100644 index 00000000000..6da85fc064b --- /dev/null +++ b/external/cmd/ids/testdata/TEST-2026-20.json @@ -0,0 +1,4 @@ +{ + "id": "TEST-2026-20", + "modified": "2026-01-01T00:00:00Z" +} diff --git a/external/cmd/ids/testdata/TEST-2026-21.json b/external/cmd/ids/testdata/TEST-2026-21.json new file mode 100644 index 00000000000..765756d5323 --- /dev/null +++ b/external/cmd/ids/testdata/TEST-2026-21.json @@ -0,0 +1,55 @@ +{ + "id": "TEST-2026-21", + "published": "2026-01-02T00:00:00Z", + "modified": "2026-01-02T00:00:00Z", + "summary": "A vulnerability", + "details": "Blah blah blah\nBlah\n", + "affected": [ + { + "package": { + "name": "blah.com/package", + "ecosystem": "Go" + }, + "ranges": [ + { + "type": "GIT", + "repo": "https://osv-test/repo/url", + "events": [ + { + "introduced": "eefe8ec3f1f90d0e684890e810f3f21e8500a4cd" + }, + { + "fixed": "8d8242f545e9cec3e6d0d2e3f5bde8be1c659735" + } + ] + } + ], + "versions": [ + "branch-v0.1.1" + ] + } + ], + "references": [ + { + "type": "WEB", + "url": "https://ref.com/ref" + } + ], + "database_specific": { + "specific": 1337 + }, + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:L" + } + ], + "credits": [ + { + "name": "Foo bar", + "contact": [ + "mailto:foo@bar.com" + ] + } + ] +} diff --git a/external/cmd/pypi/main.go b/external/cmd/pypi/main.go new file mode 100644 index 00000000000..5848389340a --- /dev/null +++ b/external/cmd/pypi/main.go @@ -0,0 +1,235 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// package main contains a utility to generate PyPI OSV records. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/fs" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/google/osv/external/pypi" + "github.com/google/osv/vulnfeeds/conversion" + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/triage" + "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/google/osv/vulnfeeds/vulns" +) + +const ( + extension = ".yaml" +) + +func loadExisting(vulnsDir string) (map[string]bool, error) { + ids := map[string]bool{} + err := filepath.Walk(vulnsDir, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("failed to access %s: %w", path, err) + } + if info.IsDir() { + return nil + } + + if !strings.HasSuffix(path, extension) { + return nil + } + + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open %s: %w", path, err) + } + defer f.Close() + + vuln, err := vulns.FromYAML(f) + if err != nil { + return fmt.Errorf("failed to parse %s: %w", path, err) + } + + for _, affected := range vuln.GetAffected() { + pkgName := affected.GetPackage().GetName() + if pkgName == "" { + continue + } + ids[vuln.GetId()+"/"+pkgName] = true + for _, alias := range vuln.GetAliases() { + ids[alias+"/"+pkgName] = true + } + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk: %w", err) + } + + return ids, nil +} + +func anyUnbounded(v *vulns.Vulnerability) bool { + for _, affected := range v.Affected { + for _, ranges := range affected.GetRanges() { + hasFixed := false + hasLastAffected := false + for _, event := range ranges.GetEvents() { + if event.GetFixed() != "" { + hasFixed = true + } + if event.GetLastAffected() != "" { + hasLastAffected = true + } + } + if !hasFixed && !hasLastAffected { + return true + } + } + } + + return false +} + +func main() { + jsonPath := flag.String("nvd_json", "", "Path to NVD CVE JSON.") + pypiLinksPath := flag.String("pypi_links", "", "Path to pypi_links.json.") + pypiVersionsPath := flag.String("pypi_versions", "", "Path to pypi_versions.json.") + falsePositivesPath := flag.String("false_positives", "", "Path to false positives file.") + withoutNotes := flag.Bool("without_notes", false, "Output vulnerabilities without notes only.") + excludeUnbounded := flag.Bool("exclude_unbounded", false, "Exclude vulnerabilities with unbounded affected ranges.") + outDir := flag.String("out_dir", "", "Path to output results.") + + flag.Parse() + + logger.InitGlobalLogger() + defer logger.Close() + + data, err := os.ReadFile(*jsonPath) + if err != nil { + logger.Fatal("Failed to open file", slog.Any("err", err)) + } + var parsed models.CVEAPIJSON20Schema + err = json.Unmarshal(data, &parsed) + if err != nil { + logger.Fatal("Failed to parse NVD CVE JSON", slog.Any("err", err)) + } + + falsePositives, err := triage.LoadFalsePositives(*falsePositivesPath) + if err != nil { + logger.Fatal("Failed to load false positives file", slog.String("path", *falsePositivesPath), slog.Any("err", err)) + } + + ecosystem := pypi.New(*pypiLinksPath, *pypiVersionsPath) + existingIDs, err := loadExisting(*outDir) + if err != nil { + logger.Fatal("Failed to load existing IDs", slog.Any("err", err)) + } + + for _, cve := range parsed.Vulnerabilities { + if falsePositives.CheckID(string(cve.CVE.ID)) { + logger.Info("Skipping as a false positive", slog.String("cve", string(cve.CVE.ID))) + continue + } + + pkgs := ecosystem.Matches(cve.CVE, falsePositives) + if len(pkgs) == 0 { + continue + } + + for _, pkg := range pkgs { + if _, exists := existingIDs[string(cve.CVE.ID)+"/"+pkg]; exists { + logger.Info("Skipping match as it already exists", slog.String("cve", string(cve.CVE.ID)), slog.String("package", pkg)) + continue + } + + logger.Info("Matched CVE to package", slog.String("cve", string(cve.CVE.ID)), slog.String("package", pkg)) + validVersions := ecosystem.Versions(pkg) + if validVersions == nil { + logger.Info("Package does not have valid versions, skipping", slog.String("package", pkg)) + continue + } + logger.Info("Got valid versions", slog.Any("versions", validVersions)) + + purl := ecosystem.PackageURL(pkg) + metrics := &models.ConversionMetrics{ + CVEID: cve.CVE.ID, + } + v := generatePyPIAffected(cve.CVE, pkg, validVersions, purl, metrics) + if len(v.Affected) == 0 || len(v.Affected[0].GetRanges()) == 0 { + logger.Info("No affected versions detected") + } + + if *excludeUnbounded && anyUnbounded(v) { + logger.Info("Skipping as we could not find an upperbound version", slog.String("cve", string(cve.CVE.ID))) + continue + } + + pkgDir := filepath.Join(*outDir, pkg) + err = os.MkdirAll(pkgDir, 0755) + if err != nil { + logger.Fatal("Failed to create dir", slog.Any("err", err)) + } + + vulnPath := filepath.Join(pkgDir, v.Id+extension) + if _, err := os.Stat(vulnPath); err == nil { + logger.Info("Skipping as it already exists", slog.String("path", vulnPath)) + continue + } + + if len(metrics.Notes) > 0 && *withoutNotes { + logger.Info("Skipping as there are notes associated with it", slog.String("path", vulnPath)) + continue + } + + f, err := os.Create(vulnPath) + if err != nil { + logger.Fatal("Failed to open for writing", slog.String("path", vulnPath), slog.Any("err", err)) + } + defer f.Close() + err = v.ToYAML(f) + if err != nil { + logger.Panic("Failed to write", slog.String("path", vulnPath), slog.Any("err", err)) + } + + // If there are notes that require human intervention, write them to the end of the YAML. + if len(metrics.Notes) > 0 { + notesPath := filepath.Join(pkgDir, v.Id+".notes") + _, err = f.WriteString("\n# \n# " + strings.Join(metrics.Notes, "\n# ")) + if err != nil { + logger.Panic("Failed to write", slog.String("path", notesPath), slog.Any("err", err)) + } + } + } + } +} + +func generatePyPIAffected(cve models.NVDCVE, pkg string, validVersions []string, purl string, metrics *models.ConversionMetrics) *vulns.Vulnerability { + id := "PYSEC-0000-" + cve.ID + versions := conversion.ExtractVersionInfo(cve, validVersions, http.DefaultClient, metrics, nil) + + pkgInfo := vulns.PackageInfo{ + PkgName: pkg, + Ecosystem: "PyPI", + PURL: purl, + VersionInfo: versions, + } + v := vulns.FromNVDCVE(id, cve) + v.AddPkgInfo(pkgInfo) + + return v +} diff --git a/external/cmd/pypi/main_test.go b/external/cmd/pypi/main_test.go new file mode 100644 index 00000000000..d88c71aaaed --- /dev/null +++ b/external/cmd/pypi/main_test.go @@ -0,0 +1,208 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/google/osv/vulnfeeds/models" + "github.com/ossf/osv-schema/bindings/go/osvschema" +) + +func TestLoadExisting(t *testing.T) { + tmpDir := t.TempDir() + + // 1. Write a valid vulnerability YAML + validYaml := ` +id: PYSEC-2021-123 +affected: + - package: + name: foo-pkg + ecosystem: PyPI +aliases: + - CVE-2021-12345 +` + if err := os.WriteFile(filepath.Join(tmpDir, "valid.yaml"), []byte(validYaml), 0600); err != nil { + t.Fatalf("failed to write valid YAML: %v", err) + } + + // 2. Write a vulnerability YAML with empty/missing affected block + missingAffectedYaml := ` +id: PYSEC-2021-456 +aliases: + - CVE-2021-67890 +` + if err := os.WriteFile(filepath.Join(tmpDir, "missing_affected.yaml"), []byte(missingAffectedYaml), 0600); err != nil { + t.Fatalf("failed to write YAML with missing affected: %v", err) + } + + // 3. Write a vulnerability YAML with affected block but missing package + missingPackageYaml := ` +id: PYSEC-2021-789 +affected: + - {} +` + if err := os.WriteFile(filepath.Join(tmpDir, "missing_package.yaml"), []byte(missingPackageYaml), 0600); err != nil { + t.Fatalf("failed to write YAML with missing package: %v", err) + } + + // Run loadExisting + got, err := loadExisting(tmpDir) + if err != nil { + t.Fatalf("loadExisting failed: %v", err) + } + + // Check expected IDs + expected := map[string]bool{ + "PYSEC-2021-123/foo-pkg": true, + "CVE-2021-12345/foo-pkg": true, + } + + if len(got) != len(expected) { + t.Errorf("loadExisting returned map of size %d, want %d", len(got), len(expected)) + } + + for k, v := range expected { + if !got[k] { + t.Errorf("Expected map to contain %s", k) + } + if got[k] != v { + t.Errorf("Expected map[%s] to be %t, got %t", k, v, got[k]) + } + } + + // Ensure invalid/skipped entries are NOT present + unexpectedKeys := []string{ + "PYSEC-2021-456/foo-pkg", + "CVE-2021-67890/foo-pkg", + "PYSEC-2021-789/foo-pkg", + } + for _, k := range unexpectedKeys { + if got[k] { + t.Errorf("Map unexpectedly contains %s", k) + } + } +} + +func strPtr(s string) *string { + return &s +} + +func TestGeneratePyPIAffected(t *testing.T) { + cve := models.NVDCVE{ + ID: "CVE-2022-29194", + Metrics: &models.CVEItemMetrics{}, + Configurations: []models.Config{ + { + Nodes: []models.Node{ + { + Operator: "OR", + CPEMatch: []models.CPEMatch{ + { + Vulnerable: true, + Criteria: "cpe:2.3:a:google:tensorflow:*:*:*:*:*:*:*:*", + VersionEndExcluding: strPtr("2.6.4"), + }, + { + Vulnerable: true, + Criteria: "cpe:2.3:a:google:tensorflow:*:*:*:*:*:*:*:*", + VersionStartIncluding: strPtr("2.7.0"), + VersionEndExcluding: strPtr("2.7.2"), + }, + }, + }, + }, + }, + }, + } + + validVersions := []string{ + "2.5.0", "2.6.0", "2.6.1", "2.6.2", "2.6.3", "2.6.4", + "2.7.0", "2.7.1", "2.7.2", + } + pkg := "tensorflow" + purl := "pkg:pypi/tensorflow" + metrics := &models.ConversionMetrics{ + CVEID: cve.ID, + } + + v := generatePyPIAffected(cve, pkg, validVersions, purl, metrics) + + // Verify ID + expectedID := "PYSEC-0000-CVE-2022-29194" + if v.Id != expectedID { + t.Errorf("expected ID %q, got %q", expectedID, v.Id) + } + + // Verify Affected + if len(v.Affected) != 1 { + t.Fatalf("expected exactly 1 affected entry, got %d", len(v.Affected)) + } + + affected := v.Affected[0] + + // Verify Package + if affected.GetPackage() == nil { + t.Fatal("expected non-nil affected.Package") + } + if affected.GetPackage().GetName() != pkg { + t.Errorf("expected package name %q, got %q", pkg, affected.GetPackage().GetName()) + } + if affected.GetPackage().GetEcosystem() != "PyPI" { + t.Errorf("expected package ecosystem %q, got %q", "PyPI", affected.GetPackage().GetEcosystem()) + } + if affected.GetPackage().GetPurl() != purl { + t.Errorf("expected package PURL %q, got %q", purl, affected.GetPackage().GetPurl()) + } + + // Verify Ranges + if len(affected.GetRanges()) != 1 { + t.Fatalf("expected exactly 1 range entry, got %d", len(affected.GetRanges())) + } + + r := affected.GetRanges()[0] + if r.GetType() != osvschema.Range_ECOSYSTEM { + t.Errorf("expected range type %v, got %v", osvschema.Range_ECOSYSTEM, r.GetType()) + } + + // Log all generated events for debugging + for i, ev := range r.GetEvents() { + t.Logf("Generated Event[%d]: %+v", i, ev) + } + + // Verify Events + expectedEvents := []*osvschema.Event{ + {Introduced: "0"}, + {Fixed: "2.6.4"}, + {Introduced: "2.7.0"}, + {Fixed: "2.7.2"}, + } + + if len(r.GetEvents()) != len(expectedEvents) { + t.Fatalf("expected %d events, got %d", len(expectedEvents), len(r.GetEvents())) + } + + for i, ev := range r.GetEvents() { + expectedEv := expectedEvents[i] + if ev.GetIntroduced() != expectedEv.GetIntroduced() { + t.Errorf("event[%d]: expected introduced %q, got %q", i, expectedEv.GetIntroduced(), ev.GetIntroduced()) + } + if ev.GetFixed() != expectedEv.GetFixed() { + t.Errorf("event[%d]: expected fixed %q, got %q", i, expectedEv.GetFixed(), ev.GetFixed()) + } + } +} diff --git a/external/go.mod b/external/go.mod new file mode 100644 index 00000000000..8afe428a435 --- /dev/null +++ b/external/go.mod @@ -0,0 +1,92 @@ +module github.com/google/osv/external + +go 1.26.3 + +require ( + github.com/aquasecurity/go-pep440-version v0.0.1 + github.com/goccy/go-yaml v1.19.2 + github.com/google/osv/vulnfeeds v0.0.0-20260522014743-9e7bf2973465 + github.com/ossf/osv-schema/bindings/go v0.0.0-20260525004216-afe0bddbf893 + google.golang.org/protobuf v1.36.11 +) + +require ( + charm.land/lipgloss/v2 v2.0.3 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/secretmanager v1.20.0 // indirect + cloud.google.com/go/trace v1.11.7 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/aquasecurity/go-version v0.0.1 // indirect + github.com/atombender/go-jsonschema v0.23.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect + github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect + github.com/charmbracelet/x/ansi v0.11.7 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/charmbracelet/x/termios v0.1.1 // indirect + github.com/charmbracelet/x/windows v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect + github.com/go-git/go-git/v5 v5.19.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/knqyf263/go-cpe v0.0.0-20230627041855-cb0794d06872 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/redis/go-redis/v9 v9.19.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/api v0.279.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/grpc v1.80.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/external/go.sum b/external/go.sum new file mode 100644 index 00000000000..6be7e71ac88 --- /dev/null +++ b/external/go.sum @@ -0,0 +1,264 @@ +charm.land/lipgloss/v2 v2.0.3 h1:yM2zJ4Cf5Y51b7RHIwioil4ApI/aypFXXVHSwlM6RzU= +charm.land/lipgloss/v2 v2.0.3/go.mod h1:7myLU9iG/3xluAWzpY/fSxYYHCgoKTie7laxk6ATwXA= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= +cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8= +cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/secretmanager v1.20.0 h1:GjE3NoyFXo7ipRPy26PMmg4oRX1Ra8fswH45r16rWV0= +cloud.google.com/go/secretmanager v1.20.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.32.0 h1:ftVmySBwuOJafpEJnnZvco+iV3p6Lokgu2sd89/qY7M= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.32.0/go.mod h1:nikqFGPI5OGwEsdxXzd3f58sB3tzkjqpqwYOV/S1rmo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0 h1:ZIT85vKP7LBS84XJ0WdJ3dPOX3iz4j3c0+lpajGQMyo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.56.0/go.mod h1:rqP9UEhOXv9WhQ7Gjz+G5y/pf8+BJZW5/Ts0AhE0PwE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 h1:0YP0+/ixwu+Uqeu/FGiBZNQ19huiUxxiPXIc9WsLKuQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0/go.mod h1:6ZZMQhZKDvUvkJw2rc+oDP90tMMzuU/J+5HG1ZmPOmE= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/aquasecurity/go-pep440-version v0.0.1 h1:8VKKQtH2aV61+0hovZS3T//rUF+6GDn18paFTVS0h0M= +github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4= +github.com/aquasecurity/go-version v0.0.1 h1:4cNl516agK0TCn5F7mmYN+xVs1E3S45LkgZk3cbaW2E= +github.com/aquasecurity/go-version v0.0.1/go.mod h1:s1UU6/v2hctXcOa3OLwfj5d9yoXHa3ahf+ipSwEvGT0= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/atombender/go-jsonschema v0.23.0 h1:1W586wlGS2Zup69szfgJpQ/NKZcjuMEocAtYvEcPyzw= +github.com/atombender/go-jsonschema v0.23.0/go.mod h1:KAi1zDASp4e0FvlFM5QvLyU3k7+DsX+7hCq98G34gtg= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= +github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 h1:OqDqxQZliC7C8adA7KjelW3OjtAxREfeHkNcd66wpeI= +github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318/go.mod h1:Y6kE2GzHfkyQQVCSL9r2hwokSrIlHGzZG+71+wDYSZI= +github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= +github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= +github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= +github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/osv/vulnfeeds v0.0.0-20260522014743-9e7bf2973465 h1:xDhvAidL8Bw3wDc2fzDmXZ1j9o9rMOMVQPbbH2+gQHY= +github.com/google/osv/vulnfeeds v0.0.0-20260522014743-9e7bf2973465/go.mod h1:RBtfa+wl0FZqZvxzE/ziS72a4zHYcN/vqoqaLFNtBJo= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/knqyf263/go-cpe v0.0.0-20230627041855-cb0794d06872 h1:snH0nDYi3kizy9vxYBhZm5KXkGt9VXdGEtr6/1SGUqY= +github.com/knqyf263/go-cpe v0.0.0-20230627041855-cb0794d06872/go.mod h1:4cVhzV/TndScEg4xMtSo3TTz3cMFhEAvhAA4igAyXZY= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/ossf/osv-schema/bindings/go v0.0.0-20260525004216-afe0bddbf893 h1:JHJyNXqj54lVkR4hznbi3xq+3VgE1H8w7G6znTesWKE= +github.com/ossf/osv-schema/bindings/go v0.0.0-20260525004216-afe0bddbf893/go.mod h1:IrUa4QzZUi03J3WXDzZYXVawYipHownNfqqZrqeGXfg= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= +github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= +go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.279.0 h1:hsx2M2OaRcaKtVYK6vXEUnQvdjnend7ZYES+lYaot74= +google.golang.org/api v0.279.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/dnaeon/go-vcr.v4 v4.0.6 h1:PiJkrakkmzc5s7EfBnZOnyiLwi7o7A9fwPzN0X2uwe0= +gopkg.in/dnaeon/go-vcr.v4 v4.0.6/go.mod h1:sbq5oMEcM4PXngbcNbHhzfCP9OdZodLhrbRYoyg09HY= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/external/pypi/README.md b/external/pypi/README.md new file mode 100644 index 00000000000..8ceb299efa8 --- /dev/null +++ b/external/pypi/README.md @@ -0,0 +1,26 @@ +# PyPI + +## Reference matching +For PyPI, we find package reference URLs by doing a BigQuery query on +the public PyPI dataset. + +```bash +bq query --max_rows=10000000 --format=json --nouse_legacy_sql < pypi_links.sql > pypi_links.json +``` + +This is also continuously updated and available at + + +However this includes packages that no longer exist or were deleted, so we check +against the [pypi simple API](https://docs.pypi.org/api/index-api/) +to make sure any matches actually exist. + +## Version matching +We also extract all valid versions by doing: + +```bash +bq query --max_rows=10000000 --format=json --nouse_legacy_sql < pypi_versions.sql > pypi_versions.json +``` + +This is also continuously updated and available at + diff --git a/external/pypi/cloudbuild.yaml b/external/pypi/cloudbuild.yaml new file mode 100644 index 00000000000..ef4eb12fdac --- /dev/null +++ b/external/pypi/cloudbuild.yaml @@ -0,0 +1,30 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These steps generate pypi_links.sql and pypi_versions.sql as described on +# https://github.com/google/osv/tree/master/vulnfeeds/pypi. +steps: +- name: gcr.io/cloud-builders/gcloud + entrypoint: bash + args: + - -c + - bq query --max_rows=10000000 --format=json --nouse_legacy_sql < vulnfeeds/pypi/pypi_links.sql > pypi_links.json +- name: gcr.io/cloud-builders/gcloud + entrypoint: bash + args: + - -c + - bq query --max_rows=10000000 --format=json --nouse_legacy_sql < vulnfeeds/pypi/pypi_versions.sql > pypi_versions.json +- name: 'gcr.io/google.com/cloudsdktool/google-cloud-cli' + entrypoint: 'gcloud' + args: ['storage', 'cp', 'pypi_links.json', 'pypi_versions.json', 'gs://pypa-advisory-db/triage/'] diff --git a/external/pypi/pypi.go b/external/pypi/pypi.go new file mode 100644 index 00000000000..e84fd770f91 --- /dev/null +++ b/external/pypi/pypi.go @@ -0,0 +1,444 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pypi contains helpers for generating PyPI OSV records. +package pypi + +import ( + "encoding/json" + "log" + "log/slog" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + + version "github.com/aquasecurity/go-pep440-version" + c "github.com/google/osv/vulnfeeds/conversion" + "github.com/google/osv/vulnfeeds/models" + "github.com/google/osv/vulnfeeds/triage" + "github.com/google/osv/vulnfeeds/utility/logger" +) + +type pypiLinks struct { + Name string `json:"name"` + Links []string `json:"links"` +} + +type pypiVersions struct { + Name string `json:"name"` + Versions []string `json:"versions"` +} + +type PyPI struct { + // links is a map of link -> array of packages with that link referenced somewhere on PyPI. + links map[string][]string + // versions is a map of package name -> array of versions. + versions map[string][]string + // vendorProductToPkg is a map of "vendor/product" to package names. + vendorProductToPkg map[string][]string + // checkedPackages is a cache that stores whether a package still exists on PyPI. + checkedPackages map[string]bool +} + +const ( + pypiSimple = "https://pypi.org/simple/" +) + +// linkBlocklist is a set of reference links to reject. +var linkBlocklist = map[string]bool{ + "https://github.com": true, + "https://gitlab.com": true, + "https://bitbucket.com": true, + "https://bitbucket.org": true, + "https://twitter.com": true, + "https://pypi.org": true, + "https://pypi.org/project": true, + "https://jira.atlassian.com": true, + "https://www.python.org": true, + "https://gitee.com": true, + "http://github.com": true, + "http://www.cisco.com": true, + "http://www.redhat.com": true, + "http://www.hp.com": true, + "http://www.oracle.com": true, + "https://www.oracle.com": true, + "http://www.python.org": true, + "http://dev.mysql.com": true, + "https://aws.amazon.com": true, + "https://github.com/aws": true, + "unknown": true, +} + +func readOrPanic(path string) []byte { + data, err := os.ReadFile(path) + if err != nil { + log.Fatalf("Failed to read %s: %v", path, err) + } + + return data +} + +func loadLinks(path string) []pypiLinks { + data := readOrPanic(path) + + var links []pypiLinks + err := json.Unmarshal(data, &links) + if err != nil { + log.Fatalf("Failed to parse %s: %v", data, err) + } + + return links +} + +func loadVersions(path string) []pypiVersions { + data := readOrPanic(path) + + var versions []pypiVersions + err := json.Unmarshal(data, &versions) + if err != nil { + log.Fatalf("Failed to parse %s: %v", data, err) + } + + return versions +} + +// NormalizePackageName normalizes a PyPI package name. +func NormalizePackageName(name string) string { + // Per https://www.python.org/dev/peps/pep-0503/#normalized-names + re := regexp.MustCompile(`[-_.]+`) + return strings.ToLower(re.ReplaceAllString(name, "-")) +} + +func hasPrefix(list []string, item string) bool { + for _, candidate := range list { + if item == candidate { + // Don't count exact matches. + continue + } + + if strings.HasPrefix(candidate, item) { + return true + } + } + + return false +} + +func processMatches(names []string) []string { + // Normalize all PyPI package names. + normalized := make([]string, 0, len(names)) + encountered := map[string]bool{} + for _, name := range names { + normalizedName := NormalizePackageName(name) + + if _, exists := encountered[normalizedName]; !exists { + encountered[normalizedName] = true + normalized = append(normalized, normalizedName) + } + } + + // Then filter out package names which are a prefix of another. + // It's very likely it's a false positive and we should take the longest match. + filtered := make([]string, 0, len(names)) + for _, name := range normalized { + if !hasPrefix(normalized, name) { + filtered = append(filtered, name) + } + } + + return filtered +} + +// extractVendorProduct takes a link and extracts the "vendor/product" from it +// if the link is a VCS link. +func extractVendorProduct(link string) string { + // Example: https://github.com/vendor/product + u, err := url.Parse(link) + if err != nil { + return "" + } + + if u.Host != "github.com" && u.Host != "bitbucket.org" && u.Host != "gitlab.com" { + return "" + } + + parts := strings.Split(u.Path, "/") + if len(parts) < 3 { + return "" + } + + return strings.ToLower(parts[1]) + "/" + strings.TrimSuffix(strings.ToLower(parts[2]), ".git") +} + +// processLinks takes a pypi_links.json and returns a map of links to list of +// packages and a map of "vendor/product" to packages. +func processLinks(linksSource []pypiLinks) (map[string][]string, map[string][]string) { + vendorProductToPkg := map[string][]string{} + links := map[string]map[string]bool{} + for _, pkg := range linksSource { + for _, link := range pkg.Links { + link = strings.ToLower(strings.TrimSuffix(strings.TrimRight(link, "/"), ".git")) + if _, exists := linkBlocklist[link]; exists { + continue + } + + normalizedName := NormalizePackageName(pkg.Name) + + if _, exists := links[link]; !exists { + links[link] = make(map[string]bool) + } + links[link][normalizedName] = true + + if vendorProduct := extractVendorProduct(link); vendorProduct != "" { + vendorProductToPkg[vendorProduct] = append(vendorProductToPkg[vendorProduct], normalizedName) + } + } + } + + processedLinks := map[string][]string{} + for link, pkgs := range links { + processedLinks[link] = make([]string, 0, len(pkgs)) + for pkg := range pkgs { + processedLinks[link] = append(processedLinks[link], pkg) + } + } + + return processedLinks, vendorProductToPkg +} + +// processVersions takes a pypi_versions.json and returns a map of packages to versions. +func processVersions(versionsSource []pypiVersions) map[string][]string { + versions := map[string][]string{} + for _, data := range versionsSource { + versions[NormalizePackageName(data.Name)] = data.Versions + } + + return versions +} + +func New(pypiLinksPath string, pypiVersionsPath string) *PyPI { + linksSource := loadLinks(pypiLinksPath) + versionsSource := loadVersions(pypiVersionsPath) + + links, vendorProductToPkg := processLinks(linksSource) + + return &PyPI{ + links: links, + versions: processVersions(versionsSource), + checkedPackages: map[string]bool{}, + vendorProductToPkg: vendorProductToPkg, + } +} + +func (p *PyPI) Matches(cve models.NVDCVE, falsePositives *triage.FalsePositives) []string { + matches := []string{} + for _, reference := range cve.References { + // If there is a PyPI link, it must be a Python package. These take precedence. + if pkg := extractPyPIProject(reference.URL); pkg != "" { + logger.Info("Matched via PyPI link", slog.String("url", reference.URL)) + matches = append(matches, pkg) + } + } + if len(matches) != 0 { + return processMatches(matches) + } + + for _, reference := range cve.References { + // Otherwise try to cross-reference the link against our set of known links. + pkgs := p.matchesPackage(reference.URL, cve, falsePositives) + matches = append(matches, pkgs...) + } + if len(matches) != 0 { + return processMatches(matches) + } + + // As a last resort, extract the vendor and product from the CPE and try to match that + // against vendor/product combinations extracted from e.g. GitHub links. + cpes := c.CPEs(cve) + if len(cpes) == 0 { + return processMatches(matches) + } + + cpe := strings.Split(cpes[0], ":") + if len(cpe) < 5 { + return processMatches(matches) + } + + vendorProduct := cpe[3] + "/" + cpe[4] + if pkgs, exists := p.vendorProductToPkg[vendorProduct]; exists { + for _, pkg := range pkgs { + if p.finalPkgCheck(cve, pkg, falsePositives) { + matches = append(matches, pkg) + } + } + } + + return processMatches(matches) +} + +func (p *PyPI) PackageURL(pkg string) string { + // https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst#pypi + // Example: pkg:pypi/django-allauth + normalizedName := NormalizePackageName(pkg) + return "pkg:pypi/" + normalizedName +} + +func filterVersions(versions []string) []string { + var filtered []string + for _, v := range versions { + if _, err := version.Parse(v); err == nil { + filtered = append(filtered, v) + } + } + + return filtered +} + +func (p *PyPI) Versions(pkg string) []string { + versions := filterVersions(p.versions[pkg]) + if versions == nil { + return nil + } + + sort.Slice(versions, func(i, j int) bool { + versionI, err := version.Parse(versions[i]) + if err != nil { + log.Panicf("Failed to parse version %s: %v", versions[i], err) + } + + versionJ, err := version.Parse(versions[j]) + if err != nil { + log.Panicf("Failed to parse version %s: %v", versions[j], err) + } + + return versionI.LessThan(versionJ) + }) + + return versions +} + +func (p *PyPI) packageExists(pkg string) bool { + if result, exists := p.checkedPackages[pkg]; exists { + return result + } + + resp, err := http.Get(pypiSimple + pkg + "/") + if err != nil { + log.Panicf("Failed to call create request: %v", err) + } + defer resp.Body.Close() + + result := resp.StatusCode == http.StatusOK + p.checkedPackages[pkg] = result + + return result +} + +func (p *PyPI) finalPkgCheck(cve models.NVDCVE, pkg string, falsePositives *triage.FalsePositives) bool { + // To avoid false positives, check that the pkg name is mentioned in the description. + desc := strings.ToLower(models.EnglishDescription(cve.Descriptions)) + pkgNameParts := strings.Split(pkg, "-") + + for _, part := range pkgNameParts { + // Python packages can commonly be py or -py. + // Remove this to be a bit more lenient when matching against the description. + part = strings.TrimPrefix(part, "py") + part = strings.TrimSuffix(part, "py") + if !strings.Contains(desc, strings.ToLower(part)) { + return false + } + } + logger.Info("Matched description", slog.String("package", pkg)) + + if falsePositives.CheckPackage(pkg) && !strings.Contains(desc, "python") { + // If this package is listed as a false positive, and the description does not + // mention "python" anywhere, it's most likely a true false positive. + return false + } + + // Finally check that the package still exists. + return p.packageExists(pkg) +} + +// matchesPackage checks if a given reference link matches a PyPI package. +func (p *PyPI) matchesPackage(link string, cve models.NVDCVE, falsePositives *triage.FalsePositives) []string { + pkgs := []string{} + u, err := url.Parse(strings.ToLower(link)) + if err != nil { + return pkgs + } + + // Repeatedly strip the last component in the URL. + pathParts := strings.Split(u.Path, "/") + for i := len(pathParts); i > 0; i-- { + u.Path = strings.Join(pathParts[0:i], "/") + fullURL := strings.TrimSuffix(u.String(), ".git") + + candidates, exists := p.links[fullURL] + if !exists { + candidates, exists = p.links[fullURL+"/"] + } + if !exists { + continue + } + + // Check that the package still exists on PyPI. + for _, pkg := range candidates { + logger.Info("Got potential match", slog.String("url", fullURL), slog.String("package", pkg)) + if p.finalPkgCheck(cve, pkg, falsePositives) { + pkgs = append(pkgs, pkg) + } + } + } + + return pkgs +} + +func extractPyPIProject(link string) string { + // Example: https://pypi.org/project/tensorflow + u, err := url.Parse(link) + if err != nil { + return "" + } + + parts := strings.Split(u.Path, "/") + + switch u.Host { + // Example: https://pypi.org/project/tensorflow + case "pypi.org": + if len(parts) < 3 || (parts[1] != "project" && parts[1] != "simple") { + return "" + } + + return NormalizePackageName(parts[2]) + // Example: https://pypi.python.org/pypi/tensorflow + case "pypi.python.org": + if len(parts) < 3 || parts[1] != "pypi" { + return "" + } + + return NormalizePackageName(parts[2]) + case "upload.pypi.org": + if len(parts) < 3 || parts[1] != "legacy" { + return "" + } + + return NormalizePackageName(parts[2]) + } + + return "" +} diff --git a/external/pypi/pypi_links.sql b/external/pypi/pypi_links.sql new file mode 100644 index 00000000000..d8394cd6b92 --- /dev/null +++ b/external/pypi/pypi_links.sql @@ -0,0 +1,54 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remove duplicates and "Type, " prefixes. +CREATE TEMP FUNCTION PROCESS_LINKS(val ARRAY) AS (( + SELECT ARRAY_AGG(REGEXP_REPLACE(t.v, "^.*,\\s*", "") IGNORE NULLS) + FROM (SELECT DISTINCT * FROM UNNEST(val) v) t +)); + +# Extract https links from package description. +CREATE TEMP FUNCTION EXTRACT_LINKS(name STRING, description STRING) +RETURNS ARRAY +LANGUAGE js +AS r""" +if (!description) { + return []; +} + +let results = []; +for (let link of description.matchAll(/https:\/\/[A-Za-z0-9.\/#?&@=_\-]+/g)) { + link = link[0]; + if (link.toLowerCase().includes(name.toLowerCase())) { + // Remove trailing periods (in cases where the link is at the end of a sentence). + results.push(link.replace(/\.$/, '')); + } + if (results.length >= 32) { + break; + } +} +return results; + +"""; + +SELECT name, +PROCESS_LINKS(ARRAY_CONCAT( + ARRAY_AGG(DISTINCT home_page), + ARRAY_AGG(DISTINCT download_url), + ARRAY_CONCAT_AGG(project_urls), + ARRAY_CONCAT_AGG(EXTRACT_LINKS(name, description)))) as links +FROM `bigquery-public-data.pypi.distribution_metadata` +WHERE home_page is not NULL +GROUP BY name +ORDER BY name diff --git a/external/pypi/pypi_versions.sql b/external/pypi/pypi_versions.sql new file mode 100644 index 00000000000..9c14fc24f12 --- /dev/null +++ b/external/pypi/pypi_versions.sql @@ -0,0 +1,18 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SELECT name, ARRAY_AGG(DISTINCT version) as versions +FROM `bigquery-public-data.pypi.distribution_metadata` +GROUP BY name +