From b523eef294f90376450aff9b80e0687c3e66237e Mon Sep 17 00:00:00 2001
From: trbs <trbs@trbs.net>
Date: Tue, 14 Mar 2017 13:51:34 +0100
Subject: [PATCH 1/2] Cache size of last ReadAt on S3 for performance

Each obj.Stat() call adds another request to the S3 endpoint
for some commands a lot of ReadAt calls are made for the same
object in S3. This patch essentially cuts the number of calls
to S3 in this case in half. Speeding up the progress and lowering
costs to S3.
---
 src/restic/backend/s3/s3.go | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go
index c7b92c80a..d5e537e5a 100644
--- a/src/restic/backend/s3/s3.go
+++ b/src/restic/backend/s3/s3.go
@@ -20,10 +20,12 @@ const connLimit = 40
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
-	client     *minio.Client
-	connChan   chan struct{}
-	bucketname string
-	prefix     string
+	client       *minio.Client
+	connChan     chan struct{}
+	bucketname   string
+	prefix       string
+	cacheObjName string
+	cacheSize    int64
 }
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -139,6 +141,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
 	}
 
 	var obj *minio.Object
+	var size int64
 
 	objName := be.s3path(h)
 
@@ -186,20 +189,27 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
 	}()
 
 	// otherwise use a buffer with ReadAt
-	info, err := obj.Stat()
-	if err != nil {
-		_ = obj.Close()
-		return nil, errors.Wrap(err, "obj.Stat")
+	if be.cacheObjName == objName {
+		size = be.cacheSize
+	} else {
+		info, err := obj.Stat()
+		if err != nil {
+			_ = obj.Close()
+			return nil, errors.Wrap(err, "obj.Stat")
+		}
+		size = info.Size
+		be.cacheObjName = objName
+		be.cacheSize = size
 	}
 
-	if offset > info.Size {
+	if offset > size {
 		_ = obj.Close()
 		return nil, errors.New("offset larger than file size")
 	}
 
 	l := int64(length)
-	if offset+l > info.Size {
-		l = info.Size - offset
+	if offset+l > size {
+		l = size - offset
 	}
 
 	buf := make([]byte, l)

From f1ba45723c131228b10404db9fdab8a782248dd7 Mon Sep 17 00:00:00 2001
From: trbs <trbs@trbs.net>
Date: Tue, 14 Mar 2017 23:05:51 +0100
Subject: [PATCH 2/2] introduce RWLock for caching obj.Stat()

---
 src/restic/backend/s3/s3.go | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go
index d5e537e5a..684588164 100644
--- a/src/restic/backend/s3/s3.go
+++ b/src/restic/backend/s3/s3.go
@@ -7,6 +7,7 @@ import (
 	"path"
 	"restic"
 	"strings"
+	"sync"
 
 	"restic/backend"
 	"restic/errors"
@@ -24,8 +25,8 @@ type s3 struct {
 	connChan     chan struct{}
 	bucketname   string
 	prefix       string
-	cacheObjName string
-	cacheSize    int64
+	cacheMutex   sync.RWMutex
+	cacheObjSize map[string]int64
 }
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -38,7 +39,12 @@ func Open(cfg Config) (restic.Backend, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
-	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+	be := &s3{
+		client:       client,
+		bucketname:   cfg.Bucket,
+		prefix:       cfg.Prefix,
+		cacheObjSize: make(map[string]int64),
+	}
 
 	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
 	client.SetCustomTransport(tr)
@@ -189,17 +195,20 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
 	}()
 
 	// otherwise use a buffer with ReadAt
-	if be.cacheObjName == objName {
-		size = be.cacheSize
-	} else {
+	be.cacheMutex.RLock()
+	size, cacheHit := be.cacheObjSize[objName]
+	be.cacheMutex.RUnlock()
+
+	if !cacheHit {
 		info, err := obj.Stat()
 		if err != nil {
 			_ = obj.Close()
 			return nil, errors.Wrap(err, "obj.Stat")
 		}
 		size = info.Size
-		be.cacheObjName = objName
-		be.cacheSize = size
+		be.cacheMutex.Lock()
+		be.cacheObjSize[objName] = size
+		be.cacheMutex.Unlock()
 	}
 
 	if offset > size {