github.com/distribution
1
0
Fork 0
mirror of https://github.com/distribution/distribution synced 2024-11-06 19:35:52 +01:00
Code Releases Wiki Activity

Merge pull request #1739 from cezarsa/master

Browse Source 
[Swift] Expose EndpointType parameter in driver
This commit is contained in:
Richard Scothern 2016-06-15 10:33:48 -07:00 committed by GitHub
commit edd7cb5249
9 changed files with 140 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Godeps/Godeps.json generated
View File

@ -1,7 +1,7 @@
{
"ImportPath": "github.com/docker/distribution",
"GoVersion": "go1.6",
"GodepVersion": "v60",
"GodepVersion": "v70",
"Packages": [
"./..."
],
@ -223,11 +223,11 @@
},
{
"ImportPath": "github.com/ncw/swift",
"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3"
"Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86"
},
{
"ImportPath": "github.com/ncw/swift/swifttest",
"Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3"
"Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86"
},
{
"ImportPath": "github.com/spf13/cobra",

161
docs/storage-drivers/swift.md
View File

@ -187,169 +187,28 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
The access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.
</td>
</tr>
</table>
<table>
<tr>
<td>
<code>authurl</code>
</td>
<td>
<p>URL for obtaining an auth token.</p>
</td>
</tr>
<tr>
<td>
<code>username</code>
</td>
<td>
<p>
Your OpenStack user name.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>password</code>
<p>
</td>
<td>
<p>
Your OpenStack password.
</p>
</td>
</tr>
<tr>
<td>
<code>container</code>
</td>
<td>
<p>
The name of your Swift container where you wish to store the registry's data. The driver creates the named container during its initialization.
</p>
</td>
</tr>
<tr>
<td>
<code>tenant</code>
</td>
<td>
<p>
Optionally, your OpenStack tenant name. You can either use <code>tenant</code> or <code>tenantid</code>.
</p>
</td>
</tr>
<tr>
<tr>
<td>
<code>tenantid</code>
<code>authversion</code>
</td>
<td>
<p>
Optionally, your OpenStack tenant id. You can either use <code>tenant</code> or <code>tenantid</code>.
</p>
</td>
</tr>
<tr>
<td>
<code>domain</code>
no
</td>
<td>
<p>
Optionally, your OpenStack domain name for Identity v3 API. You can either use <code>domain</code> or <code>domainid</code>.
</p>
Specify the OpenStack Auth's version,for example <code>3</code>. By default the driver will autodetect the auth's version from the AuthURL.
</td>
</tr>
<tr>
</tr>
<tr>
<td>
<code>domainid</code>
<code>endpointtype</code>
</td>
<td>
<p>
Optionally, your OpenStack domain id for Identity v3 API. You can either use <code>domain</code> or <code>domainid</code>.
</p>
</td>
</tr>
<tr>
<td>
<code>trustid</code>
no
</td>
<td>
<p>
Optionally, your OpenStack trust id for Identity v3 API.
</p>
The endpoint type used when connecting to swift. Possible values are `public`, `internal` and `admin`. Default is `public`.
</td>
</tr>
<tr>
<td>
<code>insecureskipverify</code>
</td>
<td>
<p>
Optionally, set <code>insecureskipverify</code> to true to skip TLS verification for your OpenStack provider. The driver uses false by default.
</p>
</td>
</tr>
<tr>
<td>
<code>region</code>
</td>
<td>
<p>
Optionally, specify the OpenStack region name in which you would like to store objects (for example <code>fr</code>).
</p>
</td>
</tr>
<tr>
<td>
<code>authversion</code>
</td>
<td>
<p>
Optionally, specify the OpenStack Auth's version,for example <code>3</code>. By default the driver will autodetect the auth's version from the AuthURL.
</p>
</td>
</tr>
<tr>
<td>
<code>chunksize</code>
</td>
<td>
<p>
Optionally, specify the segment size for Dynamic Large Objects uploads (performed by WriteStream) to Swift. The default is 5 MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to Swift.
</p>
</td>
</tr>
<tr>
<td>
<code>prefix</code>
</td>
<td>
<p>
Optionally, supply a prefix that will be applied to all Swift keys to allow you to segment data in your container if necessary. Defaults to the empty string which is the container's root.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>secretkey</code>
</td>
<td>
<p>
Optionally, the secret key used to generate temporary URLs.</p>
</p>
</td>
</tr>
<tr>
<td>
<code>accesskey</code>
</td>
<td>
<p>
Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
</p>
</td>
</tr>
</tr>
</table>
The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator

2
registry/storage/driver/swift/swift.go
View File

@ -72,6 +72,7 @@ type Parameters struct {
AuthVersion int
Container string
Prefix string
EndpointType string
InsecureSkipVerify bool
ChunkSize int
SecretKey string
@ -182,6 +183,7 @@ func New(params Parameters) (*Driver, error) {
Domain: params.Domain,
DomainId: params.DomainID,
TrustId: params.TrustID,
EndpointType: swift.EndpointType(params.EndpointType),
Transport: transport,
ConnectTimeout: 60 * time.Second,
Timeout: 15 * 60 * time.Second,

3
registry/storage/driver/swift/swift_test.go
View File

@ -34,6 +34,7 @@ func init() {
container string
region string
AuthVersion int
endpointType string
insecureSkipVerify bool
secretKey string
accessKey string
@ -54,6 +55,7 @@ func init() {
container = os.Getenv("SWIFT_CONTAINER_NAME")
region = os.Getenv("SWIFT_REGION_NAME")
AuthVersion, _ = strconv.Atoi(os.Getenv("SWIFT_AUTH_VERSION"))
endpointType = os.Getenv("SWIFT_ENDPOINT_TYPE")
insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
secretKey = os.Getenv("SWIFT_SECRET_KEY")
accessKey = os.Getenv("SWIFT_ACCESS_KEY")
@ -90,6 +92,7 @@ func init() {
AuthVersion,
container,
root,
endpointType,
insecureSkipVerify,
defaultChunkSize,
secretKey,

5
vendor/github.com/ncw/swift/.travis.yml generated vendored
View File

@ -4,8 +4,11 @@ sudo: false
go:
- 1.1.2
- 1.2.2
- 1.3
- 1.3.3
- 1.4.2
- 1.5.1
- tip
script:
- test -z "$(go fmt ./...)"
- go test

5
vendor/github.com/ncw/swift/README.md generated vendored
View File

@ -9,7 +9,7 @@ See here for package docs
http://godoc.org/github.com/ncw/swift
[![Build Status](https://travis-ci.org/ncw/swift.png)](https://travis-ci.org/ncw/swift)
[![Build Status](https://api.travis-ci.org/ncw/swift.svg?branch=master)](https://travis-ci.org/ncw/swift) [![GoDoc](https://godoc.org/github.com/ncw/swift?status.svg)](https://godoc.org/github.com/ncw/swift)
Install
-------
@ -134,3 +134,6 @@ Contributors
- Fabian Ruff <fabian@progra.de>
- Arturo Reuschenbach Puncernau <reuschenbach@gmail.com>
- Petr Kotek <petr.kotek@bigcommerce.com>
- Stefan Majewsky <stefan.majewsky@sap.com>
- Cezar Sa Espinola <cezarsa@gmail.com>
- Sam Gunaratne <samgzeit@gmail.com>

46
vendor/github.com/ncw/swift/auth.go generated vendored
View File

@ -12,7 +12,9 @@ import (
//
// This encapsulates the different authentication schemes in use
type Authenticator interface {
// Request creates an http.Request for the auth - return nil if not needed
Request(*Connection) (*http.Request, error)
// Response parses the http.Response
Response(resp *http.Response) error
// The public storage URL - set Internal to true to read
// internal/service net URL
@ -23,6 +25,23 @@ type Authenticator interface {
CdnUrl() string
}
type CustomEndpointAuthenticator interface {
StorageUrlForEndpoint(endpointType EndpointType) string
}
type EndpointType string
const (
// Use public URL as storage URL
EndpointTypePublic = EndpointType("public")
// Use internal URL as storage URL
EndpointTypeInternal = EndpointType("internal")
// Use admin URL as storage URL
EndpointTypeAdmin = EndpointType("admin")
)
// newAuth - create a new Authenticator from the AuthUrl
//
// A hint for AuthVersion can be provided
@ -175,15 +194,20 @@ func (auth *v2Auth) Response(resp *http.Response) error {
// Region if set or defaulting to the first one if not
//
// Returns "" if not found
func (auth *v2Auth) endpointUrl(Type string, Internal bool) string {
func (auth *v2Auth) endpointUrl(Type string, endpointType EndpointType) string {
for _, catalog := range auth.Auth.Access.ServiceCatalog {
if catalog.Type == Type {
for _, endpoint := range catalog.Endpoints {
if auth.Region == "" || (auth.Region == endpoint.Region) {
if Internal {
switch endpointType {
case EndpointTypeInternal:
return endpoint.InternalUrl
} else {
case EndpointTypePublic:
return endpoint.PublicUrl
case EndpointTypeAdmin:
return endpoint.AdminUrl
default:
return ""
}
}
}
@ -197,7 +221,18 @@ func (auth *v2Auth) endpointUrl(Type string, Internal bool) string {
// If Internal is true then it reads the private (internal / service
// net) URL.
func (auth *v2Auth) StorageUrl(Internal bool) string {
return auth.endpointUrl("object-store", Internal)
endpointType := EndpointTypePublic
if Internal {
endpointType = EndpointTypeInternal
}
return auth.StorageUrlForEndpoint(endpointType)
}
// v2 Authentication - read storage url
//
// Use the indicated endpointType to choose a URL.
func (auth *v2Auth) StorageUrlForEndpoint(endpointType EndpointType) string {
return auth.endpointUrl("object-store", endpointType)
}
// v2 Authentication - read auth token
@ -207,7 +242,7 @@ func (auth *v2Auth) Token() string {
// v2 Authentication - read cdn url
func (auth *v2Auth) CdnUrl() string {
return auth.endpointUrl("rax:object-cdn", false)
return auth.endpointUrl("rax:object-cdn", EndpointTypePublic)
}
// ------------------------------------------------------------
@ -255,6 +290,7 @@ type v2AuthResponse struct {
Endpoints []struct {
InternalUrl string
PublicUrl string
AdminUrl string
Region string
TenantId string
}

47
vendor/github.com/ncw/swift/auth_v3.go generated vendored
View File

@ -10,9 +10,6 @@ import (
const (
v3AuthMethodToken = "token"
v3AuthMethodPassword = "password"
v3InterfacePublic = "public"
v3InterfaceInternal = "internal"
v3InterfaceAdmin = "admin"
v3CatalogTypeObjectStore = "object-store"
)
@ -88,7 +85,8 @@ type v3AuthResponse struct {
Catalog []struct {
Id, Namem, Type string
Endpoints []struct {
Id, Region_Id, Url, Region, Interface string
Id, Region_Id, Url, Region string
Interface EndpointType
}
}
@ -107,11 +105,13 @@ type v3AuthResponse struct {
}
type v3Auth struct {
Region string
Auth *v3AuthResponse
Headers http.Header
}
func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
auth.Region = c.Region
var v3i interface{}
@ -149,13 +149,18 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
v3.Auth.Scope.Project.Id = c.TenantId
} else if c.Tenant != "" {
v3.Auth.Scope.Project.Name = c.Tenant
var defaultDomain v3Domain
if c.Domain != "" {
defaultDomain = v3Domain{Name: "Default"}
} else if c.DomainId != "" {
defaultDomain = v3Domain{Id: "Default"}
switch {
case c.TenantDomain != "":
v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain}
case c.TenantDomainId != "":
v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId}
case c.Domain != "":
v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain}
case c.DomainId != "":
v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId}
default:
v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"}
}
v3.Auth.Scope.Project.Domain = &defaultDomain
}
}
@ -188,18 +193,12 @@ func (auth *v3Auth) Response(resp *http.Response) error {
return err
}
func (auth *v3Auth) endpointUrl(Type string, Internal bool) string {
func (auth *v3Auth) endpointUrl(Type string, endpointType EndpointType) string {
for _, catalog := range auth.Auth.Token.Catalog {
if catalog.Type == Type {
for _, endpoint := range catalog.Endpoints {
if Internal {
if endpoint.Interface == v3InterfaceInternal {
return endpoint.Url
}
} else {
if endpoint.Interface == v3InterfacePublic {
return endpoint.Url
}
if endpoint.Interface == endpointType && (auth.Region == "" || (auth.Region == endpoint.Region)) {
return endpoint.Url
}
}
}
@ -208,7 +207,15 @@ func (auth *v3Auth) endpointUrl(Type string, Internal bool) string {
}
func (auth *v3Auth) StorageUrl(Internal bool) string {
return auth.endpointUrl(v3CatalogTypeObjectStore, Internal)
endpointType := EndpointTypePublic
if Internal {
endpointType = EndpointTypeInternal
}
return auth.StorageUrlForEndpoint(endpointType)
}
func (auth *v3Auth) StorageUrlForEndpoint(endpointType EndpointType) string {
return auth.endpointUrl("object-store", endpointType)
}
func (auth *v3Auth) Token() string {

83
vendor/github.com/ncw/swift/swift.go generated vendored
View File

@ -92,11 +92,14 @@ type Connection struct {
UserAgent string // Http User agent (default goswift/1.0)
ConnectTimeout time.Duration // Connect channel timeout (default 10s)
Timeout time.Duration // Data channel timeout (default 60s)
Region string // Region to use eg "LON", "ORD" - default is use first region (V2 auth only)
AuthVersion int // Set to 1 or 2 or leave at 0 for autodetect
Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only)
AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect
Internal bool // Set this to true to use the the internal / service network
Tenant string // Name of the tenant (v2 auth only)
TenantId string // Id of the tenant (v2 auth only)
Tenant string // Name of the tenant (v2,v3 auth only)
TenantId string // Id of the tenant (v2,v3 auth only)
EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set)
TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain
TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain
TrustId string // Id of the trust (v3 auth only)
Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
// These are filled in after Authenticate is called as are the defaults for above
@ -300,33 +303,39 @@ again:
if err != nil {
return
}
timer := time.NewTimer(c.ConnectTimeout)
var resp *http.Response
resp, err = c.doTimeoutRequest(timer, req)
if err != nil {
return
}
defer func() {
checkClose(resp.Body, &err)
// Flush the auth connection - we don't want to keep
// it open if keepalives were enabled
flushKeepaliveConnections(c.Transport)
}()
if err = c.parseHeaders(resp, authErrorMap); err != nil {
// Try again for a limited number of times on
// AuthorizationFailed or BadRequest. This allows us
// to try some alternate forms of the request
if (err == AuthorizationFailed || err == BadRequest) && retries > 0 {
retries--
goto again
if req != nil {
timer := time.NewTimer(c.ConnectTimeout)
var resp *http.Response
resp, err = c.doTimeoutRequest(timer, req)
if err != nil {
return
}
defer func() {
checkClose(resp.Body, &err)
// Flush the auth connection - we don't want to keep
// it open if keepalives were enabled
flushKeepaliveConnections(c.Transport)
}()
if err = c.parseHeaders(resp, authErrorMap); err != nil {
// Try again for a limited number of times on
// AuthorizationFailed or BadRequest. This allows us
// to try some alternate forms of the request
if (err == AuthorizationFailed || err == BadRequest) && retries > 0 {
retries--
goto again
}
return
}
err = c.Auth.Response(resp)
if err != nil {
return
}
return
}
err = c.Auth.Response(resp)
if err != nil {
return
if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" {
c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType)
} else {
c.StorageUrl = c.Auth.StorageUrl(c.Internal)
}
c.StorageUrl = c.Auth.StorageUrl(c.Internal)
c.AuthToken = c.Auth.Token()
if !c.authenticated() {
err = newError(0, "Response didn't have storage url and auth token")
@ -1424,13 +1433,13 @@ var _ io.Seeker = &ObjectOpenFile{}
// will also check the length returned. No checking will be done if
// you don't read all the contents.
//
// Note that objects with X-Object-Manifest set won't ever have their
// md5sum's checked as the md5sum reported on the object is actually
// the md5sum of the md5sums of the parts. This isn't very helpful to
// detect a corrupted download as the size of the parts aren't known
// without doing more operations. If you want to ensure integrity of
// an object with a manifest then you will need to download everything
// in the manifest separately.
// Note that objects with X-Object-Manifest or X-Static-Large-Object
// set won't ever have their md5sum's checked as the md5sum reported
// on the object is actually the md5sum of the md5sums of the
// parts. This isn't very helpful to detect a corrupted download as
// the size of the parts aren't known without doing more operations.
// If you want to ensure integrity of an object with a manifest then
// you will need to download everything in the manifest separately.
//
// headers["Content-Type"] will give the content type if desired.
func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) {
@ -1445,8 +1454,8 @@ func (c *Connection) ObjectOpen(container string, objectName string, checkHash b
if err != nil {
return
}
// Can't check MD5 on an object with X-Object-Manifest set
if checkHash && headers["X-Object-Manifest"] != "" {
// Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set
if checkHash && (headers["X-Object-Manifest"] != "" || headers["X-Static-Large-Object"] != "") {
// log.Printf("swift: turning off md5 checking on object with manifest %v", objectName)
checkHash = false
}