Merge pull request #381 from RichardScothern/purge-config

Add configuration for upload purging

cmd/registry/config.yml

@@ -9,6 +9,9 @@ storage:
         layerinfo: inmemory
     filesystem:
         rootdirectory: /tmp/registry-dev
+    maintenance:
+        uploadpurging:
+            enabled: false
 http:
     addr: :5000
     secret: asecretforlocaldevelopment
@@ -39,3 +42,4 @@ notifications:
           threshold: 10
           backoff: 1s
           disabled: true
+   

configuration/configuration.go

@@ -188,6 +188,8 @@ func (storage Storage) Type() string {
 	// Return only key in this map
 	for k := range storage {
 		switch k {
+		case "maintenance":
+			// allow configuration of maintenance
 		case "cache":
 			// allow configuration of caching
 		default:
@@ -217,6 +219,8 @@ func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			types := make([]string, 0, len(storageMap))
 			for k := range storageMap {
 				switch k {
+				case "maintenance":
+					// allow for configuration of maintenance
 				case "cache":
 					// allow configuration of caching
 				default:

docs/configuration.md

@@ -55,6 +55,12 @@ storage:
 		rootdirectory: /s3/object/name/prefix
 	cache:
 		layerinfo: inmemory
+	maintenance:
+		uploadpurging:
+			enabled: true
+			age: 168h
+			interval: 24h
+			dryrun: false
 auth:
 	silly:
 		realm: silly-realm
@@ -233,6 +239,12 @@ storage:
 		rootdirectory: /s3/object/name/prefix
 	cache:
 		layerinfo: inmemory
+	maintenance:
+		uploadpurging:
+			enabled: true
+			age: 168h
+			interval: 24h
+			dryrun: false
 ```
 
 The storage option is **required** and defines which storage backend is in use.
@@ -422,6 +434,27 @@ This storage backend uses Amazon's Simple Storage Service (S3).
   </tr> 
 </table>
 
+### Maintenance
+
+Currently the registry can perform one maintenance function: upload purging.  This and future
+maintenance functions which are related to storage can be configured under the maintenance section.
+
+### Upload Purging
+
+Upload purging is a background process that periodically removes orphaned files from the upload
+directories of the registry.  Upload purging is enabled by default.  To 
+ configure upload directory purging, the following parameters
+must be set.
+
+
+| Parameter | Required | Description
+  --------- | -------- | -----------
+`enabled` | yes | Set to true to enable upload purging.  Default=true. |
+`age` | yes | Upload directories which are older than this age will be deleted.  Default=168h (1 week)
+`interval` | yes | The interval between upload directory purging.  Default=24h.  
+`dryrun` | yes |  dryrun can be set to true to obtain a summary of what directories will be deleted.  Default=false.
+
+Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).  
 
 ## auth
 
@@ -1152,6 +1185,7 @@ Configure the behavior of the Redis connection pool.
   </tr>  
 </table>
 
+
 ## Example: Development configuration
 
 The following is a simple example you can use for local development:

docs/deploying.md

@@ -205,6 +205,9 @@ storage:
       layerinfo: inmemory
   filesystem:
       rootdirectory: /tmp/registry-dev
+  maintenance:
+		uploadpurging:
+			enabled: false
 http:
   addr: :5000
   secret: asecretforlocaldevelopment

registry/handlers/app.go

@@ -81,7 +81,18 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App
 		panic(err)
 	}
 
-	startUploadPurger(app.driver, ctxu.GetLogger(app))
+	purgeConfig := uploadPurgeDefaultConfig()
+	if mc, ok := configuration.Storage["maintenance"]; ok {
+		for k, v := range mc {
+			switch k {
+			case "uploadpurging":
+				purgeConfig = v.(map[interface{}]interface{})
+			}
+		}
+
+	}
+
+	startUploadPurger(app.driver, ctxu.GetLogger(app), purgeConfig)
 
 	app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
 	if err != nil {
@@ -568,26 +579,82 @@ func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []co
 	return driver, nil
 }
 
+// uploadPurgeDefaultConfig provides a default configuration for upload
+// purging to be used in the absence of configuration in the
+// confifuration file
+func uploadPurgeDefaultConfig() map[interface{}]interface{} {
+	config := map[interface{}]interface{}{}
+	config["enabled"] = true
+	config["age"] = "168h"
+	config["interval"] = "24h"
+	config["dryrun"] = false
+	return config
+}
+
+func badPurgeUploadConfig(reason string) {
+	panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason))
+}
+
 // startUploadPurger schedules a goroutine which will periodically
 // check upload directories for old files and delete them
-func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger) {
-	rand.Seed(time.Now().Unix())
-	jitter := time.Duration(rand.Int()%60) * time.Minute
+func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger, config map[interface{}]interface{}) {
+	if config["enabled"] == false {
+		return
+	}
 
-	// Start with reasonable defaults
-	// TODO:(richardscothern) make configurable
-	purgeAge := time.Duration(7 * 24 * time.Hour)
-	timeBetweenPurges := time.Duration(1 * 24 * time.Hour)
+	var purgeAgeDuration time.Duration
+	var err error
+	purgeAge, ok := config["age"]
+	if ok {
+		ageStr, ok := purgeAge.(string)
+		if !ok {
+			badPurgeUploadConfig("age is not a string")
+		}
+		purgeAgeDuration, err = time.ParseDuration(ageStr)
+		if err != nil {
+			badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error()))
+		}
+	} else {
+		badPurgeUploadConfig("age missing")
+	}
+
+	var intervalDuration time.Duration
+	interval, ok := config["interval"]
+	if ok {
+		intervalStr, ok := interval.(string)
+		if !ok {
+			badPurgeUploadConfig("interval is not a string")
+		}
+
+		intervalDuration, err = time.ParseDuration(intervalStr)
+		if err != nil {
+			badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error()))
+		}
+	} else {
+		badPurgeUploadConfig("interval missing")
+	}
+
+	var dryRunBool bool
+	dryRun, ok := config["dryrun"]
+	if ok {
+		dryRunBool, ok = dryRun.(bool)
+		if !ok {
+			badPurgeUploadConfig("cannot parse dryrun")
+		}
+	} else {
+		badPurgeUploadConfig("dryrun missing")
+	}
 
 	go func() {
+		rand.Seed(time.Now().Unix())
+		jitter := time.Duration(rand.Int()%60) * time.Minute
 		log.Infof("Starting upload purge in %s", jitter)
 		time.Sleep(jitter)
 
 		for {
-			storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAge), true)
-			log.Infof("Starting upload purge in %s", timeBetweenPurges)
-			time.Sleep(timeBetweenPurges)
+			storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool)
+			log.Infof("Starting upload purge in %s", intervalDuration)
+			time.Sleep(intervalDuration)
 		}
 	}()
-
 }