Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ import (
"time"
)

const (
defaultTimeout = 30 * time.Second
defaultMaxRetries = 5
defaultBaseDelay = 50 * time.Millisecond
backoffBase = 2
jitterFactor = 0.1
)

// RateLimiter controls request pacing.
type RateLimiter interface {
Wait(ctx context.Context) error
Expand All @@ -30,11 +38,11 @@ type Client struct {
func DefaultClient() *Client {
return &Client{
HTTPClient: &http.Client{
Timeout: 30 * time.Second,
Timeout: defaultTimeout,
},
UserAgent: "registries",
MaxRetries: 5,
BaseDelay: 50 * time.Millisecond,
MaxRetries: defaultMaxRetries,
BaseDelay: defaultBaseDelay,
}
}

Expand All @@ -53,8 +61,8 @@ func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) {

for attempt := 0; attempt <= c.MaxRetries; attempt++ {
if attempt > 0 {
delay := c.BaseDelay * time.Duration(math.Pow(2, float64(attempt-1)))
jitter := time.Duration(float64(delay) * (rand.Float64() * 0.1))
delay := c.BaseDelay * time.Duration(math.Pow(backoffBase, float64(attempt-1)))
jitter := time.Duration(float64(delay) * (rand.Float64() * jitterFactor))
delay += jitter

select {
Expand All @@ -79,10 +87,10 @@ func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) {

var httpErr *HTTPError
if ok := isHTTPError(err, &httpErr); ok {
if httpErr.StatusCode == 404 {
if httpErr.StatusCode == http.StatusNotFound {
return nil, err
}
if httpErr.StatusCode == 429 || httpErr.StatusCode >= 500 {
if httpErr.StatusCode == http.StatusTooManyRequests || httpErr.StatusCode >= http.StatusInternalServerError {
continue
}
return nil, err
Expand Down Expand Up @@ -112,13 +120,13 @@ func (c *Client) doRequest(ctx context.Context, url string) ([]byte, error) {
return nil, err
}

if resp.StatusCode >= 400 {
if resp.StatusCode >= http.StatusBadRequest {
httpErr := &HTTPError{
StatusCode: resp.StatusCode,
URL: url,
Body: string(body),
}
if resp.StatusCode == 429 {
if resp.StatusCode == http.StatusTooManyRequests {
if retryAfter := resp.Header.Get("Retry-After"); retryAfter != "" {
if seconds, err := strconv.Atoi(retryAfter); err == nil {
return nil, &RateLimitError{RetryAfter: seconds}
Expand Down
3 changes: 2 additions & 1 deletion client/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package client
import (
"errors"
"fmt"
"net/http"
)

// ErrNotFound is returned when a package or version is not found.
Expand All @@ -21,7 +22,7 @@ func (e *HTTPError) Error() string {

// IsNotFound returns true if the error represents a 404 response.
func (e *HTTPError) IsNotFound() bool {
return e.StatusCode == 404
return e.StatusCode == http.StatusNotFound
}

// NotFoundError wraps ErrNotFound with additional context.
Expand Down
15 changes: 11 additions & 4 deletions fetch/circuit_breaker.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ import (
circuit "github.com/rubyist/circuitbreaker"
)

const (
cbInitialInterval = 30 * time.Second
cbMaxInterval = 5 * time.Minute
cbThreshold = 5
maxURLTruncate = 50
)

// CircuitBreakerFetcher wraps a Fetcher with per-registry circuit breakers.
type CircuitBreakerFetcher struct {
fetcher *Fetcher
Expand Down Expand Up @@ -47,14 +54,14 @@ func (cbf *CircuitBreakerFetcher) getBreaker(registry string) *circuit.Breaker {
// Create new circuit breaker with exponential backoff
// Trips after 5 consecutive failures
expBackoff := backoff.NewExponentialBackOff()
expBackoff.InitialInterval = 30 * time.Second
expBackoff.MaxInterval = 5 * time.Minute
expBackoff.InitialInterval = cbInitialInterval
expBackoff.MaxInterval = cbMaxInterval
expBackoff.Multiplier = 2.0
expBackoff.Reset()

opts := &circuit.Options{
BackOff: expBackoff,
ShouldTrip: circuit.ThresholdTripFunc(5),
ShouldTrip: circuit.ThresholdTripFunc(cbThreshold),
}
breaker = circuit.NewBreakerWithOptions(opts)

Expand Down Expand Up @@ -112,7 +119,7 @@ func extractRegistry(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil || parsed.Host == "" {
// Fallback to simple truncation
if len(rawURL) > 50 {
if len(rawURL) > maxURLTruncate {
return rawURL[:50]
}
return rawURL
Expand Down
45 changes: 31 additions & 14 deletions fetch/fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,23 @@ import (
"github.com/rs/dnscache"
)

const (
dnsRefreshInterval = 5 * time.Minute
dialTimeout = 30 * time.Second
dialKeepAlive = 30 * time.Second
httpClientTimeout = 5 * time.Minute
maxIdleConns = 100
maxIdleConnsPerHost = 10
idleConnTimeout = 90 * time.Second
tlsHandshakeTimeout = 10 * time.Second
defaultMaxRetries = 3
defaultBaseDelay = 500 * time.Millisecond
backoffBase = 2
jitterFactor = 0.1
serverErrThreshold = 500
maxErrBodySize = 1024
)

var (
ErrNotFound = errors.New("artifact not found")
ErrRateLimited = errors.New("rate limited by upstream")
Expand Down Expand Up @@ -91,7 +108,7 @@ func NewFetcher(opts ...Option) *Fetcher {
// Create DNS cache with 5 minute refresh interval
resolver := &dnscache.Resolver{}
go func() {
ticker := time.NewTicker(5 * time.Minute)
ticker := time.NewTicker(dnsRefreshInterval)
defer ticker.Stop()
for range ticker.C {
resolver.Refresh(true)
Expand All @@ -100,13 +117,13 @@ func NewFetcher(opts ...Option) *Fetcher {

// Create custom dialer with DNS caching
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
Timeout: dialTimeout,
KeepAlive: dialKeepAlive,
}

f := &Fetcher{
client: &http.Client{
Timeout: 5 * time.Minute, // Artifacts can be large
Timeout: httpClientTimeout,
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
Expand All @@ -125,16 +142,16 @@ func NewFetcher(opts ...Option) *Fetcher {
}
return nil, fmt.Errorf("failed to dial any resolved IP")
},
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
MaxIdleConns: maxIdleConns,
MaxIdleConnsPerHost: maxIdleConnsPerHost,
IdleConnTimeout: idleConnTimeout,
TLSHandshakeTimeout: tlsHandshakeTimeout,
ExpectContinueTimeout: 1 * time.Second,
},
},
userAgent: "git-pkgs-proxy/1.0",
maxRetries: 3,
baseDelay: 500 * time.Millisecond,
maxRetries: defaultMaxRetries,
baseDelay: defaultBaseDelay,
}
for _, opt := range opts {
opt(f)
Expand All @@ -150,8 +167,8 @@ func (f *Fetcher) Fetch(ctx context.Context, url string) (*Artifact, error) {
for attempt := 0; attempt <= f.maxRetries; attempt++ {
if attempt > 0 {
// Exponential backoff with 10% jitter to prevent thundering herd
delay := f.baseDelay * time.Duration(math.Pow(2, float64(attempt-1)))
jitter := time.Duration(float64(delay) * (rand.Float64() * 0.1))
delay := f.baseDelay * time.Duration(math.Pow(backoffBase, float64(attempt-1)))
jitter := time.Duration(float64(delay) * (rand.Float64() * jitterFactor))
delay += jitter

select {
Expand Down Expand Up @@ -230,12 +247,12 @@ func (f *Fetcher) doFetch(ctx context.Context, url string) (*Artifact, error) {
_ = resp.Body.Close()
return nil, ErrRateLimited

case resp.StatusCode >= 500:
case resp.StatusCode >= serverErrThreshold:
_ = resp.Body.Close()
return nil, ErrUpstreamDown

default:
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrBodySize))
_ = resp.Body.Close()
return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body))
}
Expand Down
10 changes: 5 additions & 5 deletions fetch/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"strings"
"unicode"

"github.com/git-pkgs/registries"
"github.com/git-pkgs/registries/client"
Expand Down Expand Up @@ -103,12 +104,11 @@ func (r *Resolver) resolveWithoutRegistry(ecosystem, name, version string) (*Art

case "maven":
// Maven name format is "group:artifact", e.g., "com.google.guava:guava"
parts := strings.SplitN(name, ":", 2)
if len(parts) != 2 {
group, artifact, found := strings.Cut(name, ":")
if !found {
return nil, fmt.Errorf("invalid maven name format, expected group:artifact")
}
group := strings.ReplaceAll(parts[0], ".", "/")
artifact := parts[1]
group = strings.ReplaceAll(group, ".", "/")
url = fmt.Sprintf("https://repo1.maven.org/maven2/%s/%s/%s/%s-%s.jar", group, artifact, version, artifact, version)
filename = fmt.Sprintf("%s-%s.jar", artifact, version)

Expand Down Expand Up @@ -185,7 +185,7 @@ func encodeGoModule(path string) string {
for _, r := range path {
if r >= 'A' && r <= 'Z' {
b.WriteRune('!')
b.WriteRune(r + 32)
b.WriteRune(unicode.ToLower(r))
} else {
b.WriteRune(r)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/cargo/cargo.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (r *Registry) Ecosystem() string {
return ecosystem
}

func (r *Registry) URLs() core.URLBuilder {
func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn
return r.urls
}

Expand Down
14 changes: 7 additions & 7 deletions internal/clojars/clojars.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ import (
)

const (
DefaultURL = "https://clojars.org"
ecosystem = "clojars"
DefaultURL = "https://clojars.org"
ecosystem = "clojars"
msPerSecond = 1000
)

func init() {
Expand Down Expand Up @@ -43,7 +44,7 @@ func (r *Registry) Ecosystem() string {
return ecosystem
}

func (r *Registry) URLs() core.URLBuilder {
func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn
return r.urls
}

Expand Down Expand Up @@ -86,9 +87,8 @@ type depInfo struct {
// ParseCoordinates parses a Clojars coordinate string (group/artifact or just artifact)
// If no group is specified, the artifact name is used as both group and artifact
func ParseCoordinates(name string) (group, artifact string) {
parts := strings.SplitN(name, "/", 2)
if len(parts) == 2 {
return parts[0], parts[1]
if before, after, found := strings.Cut(name, "/"); found {
return before, after
}
// Single name means group == artifact
return name, name
Expand Down Expand Up @@ -168,7 +168,7 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi
var versionResp versionDetailResponse
if err := r.client.GetJSON(ctx, versionURL, &versionResp); err == nil {
if versionResp.CreatedEpoch > 0 {
versions[i].PublishedAt = time.Unix(versionResp.CreatedEpoch/1000, 0)
versions[i].PublishedAt = time.Unix(versionResp.CreatedEpoch/msPerSecond, 0)
}
if len(versionResp.Licenses) > 0 {
versions[i].Licenses = strings.Join(versionResp.Licenses, ",")
Expand Down
2 changes: 1 addition & 1 deletion internal/cocoapods/cocoapods.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func (r *Registry) Ecosystem() string {
return ecosystem
}

func (r *Registry) URLs() core.URLBuilder {
func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn
return r.urls
}

Expand Down
18 changes: 6 additions & 12 deletions internal/conda/conda.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func (r *Registry) Ecosystem() string {
return ecosystem
}

func (r *Registry) URLs() core.URLBuilder {
func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn
return r.urls
}

Expand Down Expand Up @@ -99,9 +99,8 @@ type fileAttrs struct {
// parsePackageName parses a package name that may include a channel prefix
// Format: "channel/name" or just "name" (uses default channel)
func parsePackageName(name string) (channel, pkgName string) {
parts := strings.SplitN(name, "/", 2)
if len(parts) == 2 {
return parts[0], parts[1]
if before, after, found := strings.Cut(name, "/"); found {
return before, after
}
return "", name
}
Expand Down Expand Up @@ -249,15 +248,10 @@ func (r *Registry) FetchDependencies(ctx context.Context, name, version string)
}

func parseDependency(dep string) (name, requirements string) {
// Conda dependency format: "name version_constraint" or just "name"
// Examples: "python >=3.8", "numpy", "pandas >=1.0,<2.0"
dep = strings.TrimSpace(dep)
parts := strings.SplitN(dep, " ", 2)
name = parts[0]
if len(parts) > 1 {
requirements = strings.TrimSpace(parts[1])
}
return
name, requirements, _ = strings.Cut(dep, " ")
requirements = strings.TrimSpace(requirements)
return name, requirements
}

func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) {
Expand Down
2 changes: 1 addition & 1 deletion internal/core/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ const defaultConcurrency = 15
// NewFromPURL creates a registry client from a PURL and returns the parsed components.
// Returns the registry, full package name, and version (empty if not in PURL).
// If the PURL has a repository_url qualifier, it's used as the base URL for private registries.
func NewFromPURL(purlStr string, client *Client) (Registry, string, string, error) {
func NewFromPURL(purlStr string, client *Client) (Registry, string, string, error) { //nolint:ireturn
p, err := purl.Parse(purlStr)
if err != nil {
return nil, "", "", err
Expand Down
2 changes: 1 addition & 1 deletion internal/core/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func Register(ecosystem string, defaultURL string, factory Factory) {

// New creates a new registry for the given ecosystem.
// If baseURL is empty, the default registry URL is used.
func New(ecosystem string, baseURL string, client *Client) (Registry, error) {
func New(ecosystem string, baseURL string, client *Client) (Registry, error) { //nolint:ireturn
mu.RLock()
factory, ok := factories[ecosystem]
defaultURL := defaults[ecosystem]
Expand Down
2 changes: 1 addition & 1 deletion internal/cpan/cpan.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func (r *Registry) Ecosystem() string {
return ecosystem
}

func (r *Registry) URLs() core.URLBuilder {
func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn
return r.urls
}

Expand Down
Loading