Updates Godeps to a compiling/stable state

Includes newer docker/libtrust and crowdmob/goamz
This commit is contained in:
Brian Bland 2015-01-16 13:56:37 -08:00
parent 81ea264dea
commit e1ab3443db
6 changed files with 239 additions and 80 deletions

8
Godeps/Godeps.json generated
View File

@ -30,15 +30,15 @@
},
{
"ImportPath": "github.com/crowdmob/goamz/aws",
"Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67"
"Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941"
},
{
"ImportPath": "github.com/crowdmob/goamz/cloudfront",
"Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67"
"Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941"
},
{
"ImportPath": "github.com/crowdmob/goamz/s3",
"Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67"
"Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941"
},
{
"ImportPath": "github.com/docker/docker/pkg/tarsum",
@ -52,7 +52,7 @@
},
{
"ImportPath": "github.com/docker/libtrust",
"Rev": "a9625ce37e2dc5fed2e51eec2d39c39e4ac4c1df"
"Rev": "c54fbb67c1f1e68d7d6f8d2ad7c9360404616a41"
},
{
"ImportPath": "github.com/gorilla/context",

View File

@ -518,7 +518,7 @@ func dialTimeout(network, addr string) (net.Conn, error) {
return net.DialTimeout(network, addr, time.Duration(2*time.Second))
}
func InstanceRegion() string {
func AvailabilityZone() string {
transport := http.Transport{Dial: dialTimeout}
client := http.Client{
Transport: &transport,
@ -532,13 +532,21 @@ func InstanceRegion() string {
if err != nil {
return "unknown"
} else {
b := string(body)
region := b[:len(b)-1]
return region
return string(body)
}
}
}
func InstanceRegion() string {
az := AvailabilityZone()
if az == "unknown" {
return az
} else {
region := az[:len(az)-1]
return region
}
}
func InstanceId() string {
transport := http.Transport{Dial: dialTimeout}
client := http.Client{

View File

@ -151,16 +151,37 @@ Any changes to the request after signing the request will invalidate the signatu
*/
func (s *V4Signer) Sign(req *http.Request) {
req.Header.Set("host", req.Host) // host header must be included as a signed header
payloadHash := s.payloadHash(req)
if s.IncludeXAmzContentSha256 {
req.Header.Set("x-amz-content-sha256", payloadHash) // x-amz-content-sha256 contains the payload hash
t := s.requestTime(req) // Get request time
payloadHash := ""
if _, ok := req.Form["X-Amz-Expires"]; ok {
// We are authenticating the the request by using query params
// (also known as pre-signing a url, http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
payloadHash = "UNSIGNED-PAYLOAD"
req.Header.Del("x-amz-date")
req.Form["X-Amz-SignedHeaders"] = []string{s.signedHeaders(req.Header)}
req.Form["X-Amz-Algorithm"] = []string{"AWS4-HMAC-SHA256"}
req.Form["X-Amz-Credential"] = []string{s.auth.AccessKey + "/" + s.credentialScope(t)}
req.Form["X-Amz-Date"] = []string{t.Format(ISO8601BasicFormat)}
req.URL.RawQuery = req.Form.Encode()
} else {
payloadHash = s.payloadHash(req)
if s.IncludeXAmzContentSha256 {
req.Header.Set("x-amz-content-sha256", payloadHash) // x-amz-content-sha256 contains the payload hash
}
}
t := s.requestTime(req) // Get request time
creq := s.canonicalRequest(req, payloadHash) // Build canonical request
sts := s.stringToSign(t, creq) // Build string to sign
signature := s.signature(t, sts) // Calculate the AWS Signature Version 4
auth := s.authorization(req.Header, t, signature) // Create Authorization header value
req.Header.Set("Authorization", auth) // Add Authorization header to request
if _, ok := req.Form["X-Amz-Expires"]; ok {
req.Form["X-Amz-Signature"] = []string{signature}
} else {
req.Header.Set("Authorization", auth) // Add Authorization header to request
}
return
}
@ -266,8 +287,20 @@ func (s *V4Signer) canonicalQueryString(u *url.URL) string {
}
func (s *V4Signer) canonicalHeaders(h http.Header) string {
i, a := 0, make([]string, len(h))
i, a, lowerCase := 0, make([]string, len(h)), make(map[string][]string)
for k, v := range h {
lowerCase[strings.ToLower(k)] = v
}
var keys []string
for k := range lowerCase {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
v := lowerCase[k]
for j, w := range v {
v[j] = strings.Trim(w, " ")
}
@ -275,7 +308,6 @@ func (s *V4Signer) canonicalHeaders(h http.Header) string {
a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",")
i++
}
sort.Strings(a)
return strings.Join(a, "\n")
}

View File

@ -165,7 +165,7 @@ func (m *Multi) PutPartCopy(n int, options CopyOptions, source string) (*CopyObj
params: params,
}
resp := &CopyObjectResult{}
err := m.Bucket.S3.query(req, resp)
err = m.Bucket.S3.query(req, resp)
if shouldRetry(err) && attempt.HasNext() {
continue
}

View File

@ -40,6 +40,7 @@ type S3 struct {
aws.Region
ConnectTimeout time.Duration
ReadTimeout time.Duration
Signature int
private byte // Reserve the right of using private data.
}
@ -95,7 +96,7 @@ var attempts = aws.AttemptStrategy{
// New creates a new S3.
func New(auth aws.Auth, region aws.Region) *S3 {
return &S3{auth, region, 0, 0, 0}
return &S3{auth, region, 0, 0, 0, aws.V2Signature}
}
// Bucket returns a Bucket with the given name.
@ -772,15 +773,26 @@ func (b *Bucket) SignedURL(path string, expires time.Time) string {
// SignedURLWithArgs returns a signed URL that allows anyone holding the URL
// to retrieve the object at path. The signature is valid until expires.
func (b *Bucket) SignedURLWithArgs(path string, expires time.Time, params url.Values, headers http.Header) string {
return b.SignedURLWithMethod("GET", path, expires, params, headers)
}
// SignedURLWithMethod returns a signed URL that allows anyone holding the URL
// to either retrieve the object at path or make a HEAD request against it. The signature is valid until expires.
func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, params url.Values, headers http.Header) string {
var uv = url.Values{}
if params != nil {
uv = params
}
uv.Set("Expires", strconv.FormatInt(expires.Unix(), 10))
if b.S3.Signature == aws.V2Signature {
uv.Set("Expires", strconv.FormatInt(expires.Unix(), 10))
} else {
uv.Set("X-Amz-Expires", strconv.FormatInt(expires.Unix()-time.Now().Unix(), 10))
}
req := &request{
method: method,
bucket: b.Name,
path: path,
params: uv,
@ -810,9 +822,15 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
if method != "POST" {
method = "PUT"
}
stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n/" + b.Name + "/" + path
fmt.Println("String to sign:\n", stringToSign)
a := b.S3.Auth
tokenData := ""
if a.Token() != "" {
tokenData = "x-amz-security-token:" + a.Token() + "\n"
}
stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path
secretKey := a.SecretKey
accessId := a.AccessKey
mac := hmac.New(sha1.New, []byte(secretKey))
@ -832,7 +850,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
params.Add("Expires", strconv.FormatInt(expire_date, 10))
params.Add("Signature", signature)
if a.Token() != "" {
params.Add("token", a.Token())
params.Add("x-amz-security-token", a.Token())
}
signedurl.RawQuery = params.Encode()
@ -924,7 +942,10 @@ func (s3 *S3) queryV4Sign(req *request, resp interface{}) error {
req.headers = map[string][]string{}
}
s3.setBaseURL(req)
err := s3.setBaseURL(req)
if err != nil {
return err
}
hreq, err := s3.setupHttpRequest(req)
if err != nil {
@ -992,57 +1013,79 @@ func partiallyEscapedPath(path string) string {
// prepare sets up req to be delivered to S3.
func (s3 *S3) prepare(req *request) error {
var signpath = req.path
// Copy so they can be mutated without affecting on retries.
params := make(url.Values)
headers := make(http.Header)
for k, v := range req.params {
params[k] = v
}
for k, v := range req.headers {
headers[k] = v
}
req.params = params
req.headers = headers
if !req.prepared {
req.prepared = true
if req.method == "" {
req.method = "GET"
}
// Copy so they can be mutated without affecting on retries.
params := make(url.Values)
headers := make(http.Header)
for k, v := range req.params {
params[k] = v
}
for k, v := range req.headers {
headers[k] = v
}
req.params = params
req.headers = headers
if !strings.HasPrefix(req.path, "/") {
req.path = "/" + req.path
}
signpath = req.path
err := s3.setBaseURL(req)
if err != nil {
return err
}
if req.bucket != "" {
signpath = "/" + req.bucket + signpath
}
}
// Always sign again as it's not clear how far the
// server has handled a previous attempt.
u, err := url.Parse(req.baseurl)
if err != nil {
return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err)
}
signpathPatiallyEscaped := partiallyEscapedPath(signpath)
req.headers["Host"] = []string{u.Host}
req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
if s3.Auth.Token() != "" {
req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()}
}
sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers)
if s3.Signature == aws.V2Signature {
// Always sign again as it's not clear how far the
// server has handled a previous attempt.
u, err := url.Parse(req.baseurl)
if err != nil {
return err
}
signpathPatiallyEscaped := partiallyEscapedPath(req.path)
req.headers["Host"] = []string{u.Host}
req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers)
} else {
hreq, err := s3.setupHttpRequest(req)
if err != nil {
return err
}
hreq.Host = hreq.URL.Host
signer := aws.NewV4Signer(s3.Auth, "s3", s3.Region)
signer.IncludeXAmzContentSha256 = true
signer.Sign(hreq)
req.payload = hreq.Body
if _, ok := headers["Content-Length"]; ok {
req.headers["Content-Length"] = headers["Content-Length"]
}
}
return nil
}
// Prepares an *http.Request for doHttpRequest
func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
// Copy so that signing the http request will not mutate it
headers := make(http.Header)
for k, v := range req.headers {
headers[k] = v
}
req.headers = headers
u, err := req.url()
if err != nil {
return nil, err
@ -1056,6 +1099,7 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
ProtoMinor: 1,
Close: true,
Header: req.headers,
Form: req.params,
}
if v, ok := req.headers["Content-Length"]; ok {

View File

@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"fmt"
"sort"
"time"
"unicode"
)
@ -31,9 +32,25 @@ type jsHeader struct {
}
type jsSignature struct {
Header *jsHeader `json:"header"`
Signature string `json:"signature"`
Protected string `json:"protected,omitempty"`
Header jsHeader `json:"header"`
Signature string `json:"signature"`
Protected string `json:"protected,omitempty"`
}
type jsSignaturesSorted []jsSignature
func (jsbkid jsSignaturesSorted) Swap(i, j int) { jsbkid[i], jsbkid[j] = jsbkid[j], jsbkid[i] }
func (jsbkid jsSignaturesSorted) Len() int { return len(jsbkid) }
func (jsbkid jsSignaturesSorted) Less(i, j int) bool {
ki, kj := jsbkid[i].Header.JWK.KeyID(), jsbkid[j].Header.JWK.KeyID()
si, sj := jsbkid[i].Signature, jsbkid[j].Signature
if ki == kj {
return si < sj
}
return ki < kj
}
type signKey struct {
@ -44,7 +61,7 @@ type signKey struct {
// JSONSignature represents a signature of a json object.
type JSONSignature struct {
payload string
signatures []*jsSignature
signatures []jsSignature
indent string
formatLength int
formatTail []byte
@ -52,7 +69,7 @@ type JSONSignature struct {
func newJSONSignature() *JSONSignature {
return &JSONSignature{
signatures: make([]*jsSignature, 0, 1),
signatures: make([]jsSignature, 0, 1),
}
}
@ -99,17 +116,14 @@ func (js *JSONSignature) Sign(key PrivateKey) error {
return err
}
header := &jsHeader{
JWK: key.PublicKey(),
Algorithm: algorithm,
}
sig := &jsSignature{
Header: header,
js.signatures = append(js.signatures, jsSignature{
Header: jsHeader{
JWK: key.PublicKey(),
Algorithm: algorithm,
},
Signature: joseBase64UrlEncode(sigBytes),
Protected: protected,
}
js.signatures = append(js.signatures, sig)
})
return nil
}
@ -136,7 +150,7 @@ func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate
return err
}
header := &jsHeader{
header := jsHeader{
Chain: make([]string, len(chain)),
Algorithm: algorithm,
}
@ -145,13 +159,11 @@ func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate
header.Chain[i] = base64.StdEncoding.EncodeToString(cert.Raw)
}
sig := &jsSignature{
js.signatures = append(js.signatures, jsSignature{
Header: header,
Signature: joseBase64UrlEncode(sigBytes),
Protected: protected,
}
js.signatures = append(js.signatures, sig)
})
return nil
}
@ -272,6 +284,9 @@ func (js *JSONSignature) JWS() ([]byte, error) {
if len(js.signatures) == 0 {
return nil, errors.New("missing signature")
}
sort.Sort(jsSignaturesSorted(js.signatures))
jsonMap := map[string]interface{}{
"payload": js.payload,
"signatures": js.signatures,
@ -301,16 +316,16 @@ type jsParsedHeader struct {
}
type jsParsedSignature struct {
Header *jsParsedHeader `json:"header"`
Signature string `json:"signature"`
Protected string `json:"protected"`
Header jsParsedHeader `json:"header"`
Signature string `json:"signature"`
Protected string `json:"protected"`
}
// ParseJWS parses a JWS serialized JSON object into a Json Signature.
func ParseJWS(content []byte) (*JSONSignature, error) {
type jsParsed struct {
Payload string `json:"payload"`
Signatures []*jsParsedSignature `json:"signatures"`
Payload string `json:"payload"`
Signatures []jsParsedSignature `json:"signatures"`
}
parsed := &jsParsed{}
err := json.Unmarshal(content, parsed)
@ -329,9 +344,9 @@ func ParseJWS(content []byte) (*JSONSignature, error) {
if err != nil {
return nil, err
}
js.signatures = make([]*jsSignature, len(parsed.Signatures))
js.signatures = make([]jsSignature, len(parsed.Signatures))
for i, signature := range parsed.Signatures {
header := &jsHeader{
header := jsHeader{
Algorithm: signature.Header.Algorithm,
}
if signature.Header.Chain != nil {
@ -344,7 +359,7 @@ func ParseJWS(content []byte) (*JSONSignature, error) {
}
header.JWK = publicKey
}
js.signatures[i] = &jsSignature{
js.signatures[i] = jsSignature{
Header: header,
Signature: signature.Signature,
Protected: signature.Protected,
@ -356,7 +371,11 @@ func ParseJWS(content []byte) (*JSONSignature, error) {
// NewJSONSignature returns a new unsigned JWS from a json byte array.
// JSONSignature will need to be signed before serializing or storing.
func NewJSONSignature(content []byte) (*JSONSignature, error) {
// Optionally, one or more signatures can be provided as byte buffers,
// containing serialized JWS signatures, to assemble a fully signed JWS
// package. It is the callers responsibility to ensure uniqueness of the
// provided signatures.
func NewJSONSignature(content []byte, signatures ...[]byte) (*JSONSignature, error) {
var dataMap map[string]interface{}
err := json.Unmarshal(content, &dataMap)
if err != nil {
@ -380,6 +399,40 @@ func NewJSONSignature(content []byte) (*JSONSignature, error) {
js.formatLength = lastRuneIndex + 1
js.formatTail = content[js.formatLength:]
if len(signatures) > 0 {
for _, signature := range signatures {
var parsedJSig jsParsedSignature
if err := json.Unmarshal(signature, &parsedJSig); err != nil {
return nil, err
}
// TODO(stevvooe): A lot of the code below is repeated in
// ParseJWS. It will require more refactoring to fix that.
jsig := jsSignature{
Header: jsHeader{
Algorithm: parsedJSig.Header.Algorithm,
},
Signature: parsedJSig.Signature,
Protected: parsedJSig.Protected,
}
if parsedJSig.Header.Chain != nil {
jsig.Header.Chain = parsedJSig.Header.Chain
}
if parsedJSig.Header.JWK != nil {
publicKey, err := UnmarshalPublicKeyJWK([]byte(parsedJSig.Header.JWK))
if err != nil {
return nil, err
}
jsig.Header.JWK = publicKey
}
js.signatures = append(js.signatures, jsig)
}
}
return js, nil
}
@ -455,7 +508,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature,
}
js := newJSONSignature()
js.signatures = make([]*jsSignature, len(signatureBlocks))
js.signatures = make([]jsSignature, len(signatureBlocks))
for i, signatureBlock := range signatureBlocks {
protectedBytes, err := joseBase64UrlDecode(signatureBlock.Protected)
@ -491,7 +544,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature,
return nil, errors.New("conflicting format tail")
}
header := &jsHeader{
header := jsHeader{
Algorithm: signatureBlock.Header.Algorithm,
Chain: signatureBlock.Header.Chain,
}
@ -502,7 +555,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature,
}
header.JWK = publicKey
}
js.signatures[i] = &jsSignature{
js.signatures[i] = jsSignature{
Header: header,
Signature: signatureBlock.Signature,
Protected: signatureBlock.Protected,
@ -532,6 +585,8 @@ func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) {
}
payload = payload[:js.formatLength]
sort.Sort(jsSignaturesSorted(js.signatures))
var marshalled []byte
var marshallErr error
if js.indent != "" {
@ -565,6 +620,26 @@ func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) {
return buf.Bytes(), nil
}
// Signatures provides the signatures on this JWS as opaque blobs, sorted by
// keyID. These blobs can be stored and reassembled with payloads. Internally,
// they are simply marshaled json web signatures but implementations should
// not rely on this.
func (js *JSONSignature) Signatures() ([][]byte, error) {
sort.Sort(jsSignaturesSorted(js.signatures))
var sb [][]byte
for _, jsig := range js.signatures {
p, err := json.Marshal(jsig)
if err != nil {
return nil, err
}
sb = append(sb, p)
}
return sb, nil
}
// Merge combines the signatures from one or more other signatures into the
// method receiver. If the payloads differ for any argument, an error will be
// returned and the receiver will not be modified.