-
-
Notifications
You must be signed in to change notification settings - Fork 125
/
Copy pathhttp.go
156 lines (140 loc) · 3.85 KB
/
http.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
package goproxy
import (
"context"
"crypto/x509"
"errors"
"fmt"
"io"
"io/fs"
"math"
"math/rand/v2"
"net/http"
"net/url"
"os"
"time"
)
var (
// errBadUpstream indicates an upstream is in a bad state.
errBadUpstream = errors.New("bad upstream")
// errFetchTimedOut indicates a fetch operation has timed out.
errFetchTimedOut = errors.New("fetch timed out")
)
// notExistError is like [fs.ErrNotExist] but with a custom underlying error.
//
// NOTE: Do not use [notExistError] directly, use [notExistErrorf] instead.
type notExistError struct{ err error }
// Error implements [error].
func (e *notExistError) Error() string { return e.err.Error() }
// Unwrap returns the underlying error.
func (e *notExistError) Unwrap() error { return e.err }
// Is reports whether the target is [fs.ErrNotExist].
func (notExistError) Is(target error) bool { return target == fs.ErrNotExist }
// notExistErrorf formats according to a format specifier and returns the string
// as a value that satisfies error that is equivalent to [fs.ErrNotExist].
func notExistErrorf(format string, v ...interface{}) error {
return ¬ExistError{err: fmt.Errorf(format, v...)}
}
// httpGet gets the content from the given url and writes it to the dst.
func httpGet(ctx context.Context, client *http.Client, url string, dst io.Writer) error {
var lastErr error
for attempt := 0; attempt < 10; attempt++ {
if attempt > 0 {
select {
case <
8000
;-time.After(backoffSleep(100*time.Millisecond, time.Second, attempt)):
case <-ctx.Done():
return lastErr
}
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := client.Do(req)
if err != nil {
if isRetryableHTTPClientDoError(err) {
lastErr = err
continue
}
return err
}
if resp.StatusCode == http.StatusOK {
if dst != nil {
_, err = io.Copy(dst, resp.Body)
}
resp.Body.Close()
return err
}
respBody, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return err
}
switch resp.StatusCode {
case http.StatusBadRequest,
http.StatusNotFound,
http.StatusGone:
return notExistErrorf("%s", respBody)
case http.StatusTooManyRequests,
http.StatusInternalServerError,
http.StatusBadGateway,
http.StatusServiceUnavailable:
lastErr = errBadUpstream
case http.StatusGatewayTimeout:
lastErr = errFetchTimedOut
default:
return fmt.Errorf("GET %s: %s: %s", resp.Request.URL.Redacted(), resp.Status, respBody)
}
}
return lastErr
}
// httpGetTemp is like [httpGet] but writes the content to a new temporary file
// in tempDir.
func httpGetTemp(ctx context.Context, client *http.Client, url, tempDir string) (tempFile string, err error) {
f, err := os.CreateTemp(tempDir, "")
if err != nil {
return "", err
}
defer func() {
if err != nil {
os.Remove(f.Name())
}
}()
if err := httpGet(ctx, client, url, f); err != nil {
return "", err
}
return f.Name(), f.Close()
}
// isRetryableHTTPClientDoError reports whether the err is a retryable error
// returned by [http.Client.Do].
func isRetryableHTTPClientDoError(err error) bool {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
if ue, ok := err.(*url.Error); ok {
e := ue.Unwrap()
switch e.(type) {
case x509.UnknownAuthorityError:
return false
}
if errors.Is(e, http.ErrSchemeMismatch) {
return false
}
}
return true
}
// backoffSleep computes the exponential backoff sleep duration based on the
// algorithm described in https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/.
func backoffSleep(base, cap time.Duration, attempt int) time.Duration {
var pow time.Duration
if attempt < 63 {
pow = 1 << attempt
} else {
pow = math.MaxInt64
}
sleep := base * pow
if sleep > cap || sleep/pow != base {
sleep = cap
}
sleep = rand.N(sleep)
return sleep
}