mirrors
/
restic
mirror of https://github.com/restic/restic.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #443 from ckemper67/fix-387 

Introduced a configurable object path prefix for s3 repositories to address #387
This commit is contained in:
Alexander Neumann 2016-02-14 22:43:40 +01:00
parent 517eff7e48 f3f1404849
commit d918d0f0c3
4 changed files with 151 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
backend/s3/config.go
View File

@ -3,6 +3,7 @@ package s3
import (
"errors"
"net/url"
"path"
"strings"
)
@ -13,54 +14,22 @@ type Config struct {
UseHTTP bool
KeyID, Secret string
Bucket string
Prefix string
}
const defaultPrefix = "restic"
// ParseConfig parses the string s and extracts the s3 config. The two
// supported configuration formats are s3://host/bucketname and
// s3:host:bucketname. The host can also be a valid s3 region name.
// supported configuration formats are s3://host/bucketname/prefix and
// s3:host:bucketname/prefix. The host can also be a valid s3 region
// name. If no prefix is given the prefix "restic" will be used.
func ParseConfig(s string) (interface{}, error) {
if strings.HasPrefix(s, "s3://") {
s = s[5:]
data := strings.SplitN(s, "/", 2)
if len(data) != 2 {
return nil, errors.New("s3: invalid format, host/region or bucket name not found")
}
cfg := Config{
Endpoint: data[0],
Bucket: data[1],
}
return cfg, nil
}
data := strings.SplitN(s, ":", 2)
if len(data) != 2 {
return nil, errors.New("s3: invalid format")
}
if data[0] != "s3" {
return nil, errors.New(`s3: config does not start with "s3"`)
}
s = data[1]
cfg := Config{}
rest := strings.Split(s, "/")
if len(rest) < 2 {
return nil, errors.New("s3: region or bucket not found")
}
if len(rest) == 2 {
// assume that just a region name and a bucket has been specified, in
// the format region/bucket
cfg.Endpoint = rest[0]
cfg.Bucket = rest[1]
} else {
// assume that a URL has been specified, parse it and use the path as
// the bucket name.
url, err := url.Parse(s)
switch {
case strings.HasPrefix(s, "s3:http"):
// assume that a URL has been specified, parse it and
// use the host as the endpoint and the path as the
// bucket name and prefix
url, err := url.Parse(s[3:])
if err != nil {
return nil, err
}
@ -69,13 +38,35 @@ func ParseConfig(s string) (interface{}, error) {
return nil, errors.New("s3: bucket name not found")
}
cfg.Endpoint = url.Host
if url.Scheme == "http" {
cfg.UseHTTP = true
}
cfg.Bucket = url.Path[1:]
path := strings.SplitN(url.Path[1:], "/", 2)
return createConfig(url.Host, path, url.Scheme == "http")
case strings.HasPrefix(s, "s3://"):
s = s[5:]
case strings.HasPrefix(s, "s3:"):
s = s[3:]
default:
return nil, errors.New("s3: invalid format")
}
return cfg, nil
// use the first entry of the path as the endpoint and the
// remainder as bucket name and prefix
path := strings.SplitN(s, "/", 3)
return createConfig(path[0], path[1:], false)
}
func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
var prefix string
switch {
case len(p) < 1:
return nil, errors.New("s3: invalid format, host/region or bucket name not found")
case len(p) == 1 || p[1] == "":
prefix = defaultPrefix
default:
prefix = path.Clean(p[1])
}
return Config{
Endpoint: endpoint,
UseHTTP: useHTTP,
Bucket: p[0],
Prefix: prefix,
}, nil
}

63
backend/s3/config_test.go
View File

@ -9,18 +9,75 @@ var configTests = []struct {
{"s3://eu-central-1/bucketname", Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "restic",
}},
{"s3://eu-central-1/bucketname/", Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "restic",
}},
{"s3://eu-central-1/bucketname/prefix/directory", Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "prefix/directory",
}},
{"s3://eu-central-1/bucketname/prefix/directory/", Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "prefix/directory",
}},
{"s3:eu-central-1/foobar", Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "restic",
}},
{"s3:eu-central-1/foobar/", Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "restic",
}},
{"s3:eu-central-1/foobar/prefix/directory", Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "prefix/directory",
}},
{"s3:eu-central-1/foobar/prefix/directory/", Config{
Endpoint: "eu-central-1",
Bucket: "foobar",
Prefix: "prefix/directory",
}},
{"s3:https://hostname:9999/foobar", Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
}},
{"s3:https://hostname:9999/foobar/", Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
}},
{"s3:http://hostname:9999/foobar", Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
UseHTTP: true,
}},
{"s3:http://hostname:9999/foobar/", Config{
Endpoint: "hostname:9999",
Bucket: "foobar",
Prefix: "restic",
UseHTTP: true,
}},
{"s3:http://hostname:9999/bucket/prefix/directory", Config{
Endpoint: "hostname:9999",
Bucket: "bucket",
Prefix: "prefix/directory",
UseHTTP: true,
}},
{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
Endpoint: "hostname:9999",
Bucket: "bucket",
Prefix: "prefix/directory",
UseHTTP: true,
}},
}
@ -29,13 +86,13 @@ func TestParseConfig(t *testing.T) {
for i, test := range configTests {
cfg, err := ParseConfig(test.s)
if err != nil {
t.Errorf("test %d failed: %v", i, err)
t.Errorf("test %d:%s failed: %v", i, test.s, err)
continue
}
if cfg != test.cfg {
t.Errorf("test %d: wrong config, want:\n %v\ngot:\n %v",
i, test.cfg, cfg)
t.Errorf("test %d:\ninput:\n %s\n wrong config, want:\n %v\ngot:\n %v",
i, test.s, test.cfg, cfg)
continue
}
}

37
backend/s3/s3.go
View File

@ -13,20 +13,13 @@ import (
)
const connLimit = 10
const backendPrefix = "restic"
func s3path(t backend.Type, name string) string {
if t == backend.Config {
return backendPrefix + "/" + string(t)
}
return backendPrefix + "/" + string(t) + "/" + name
}
// s3 is a backend which stores the data on an S3 endpoint.
type s3 struct {
client minio.CloudStorageClient
connChan chan struct{}
bucketname string
prefix string
}
// Open opens the S3 backend at bucket and region. The bucket is created if it
@ -39,7 +32,7 @@ func Open(cfg Config) (backend.Backend, error) {
return nil, err
}
be := &s3{client: client, bucketname: cfg.Bucket}
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
be.createConnections()
if err := client.BucketExists(cfg.Bucket); err != nil {
@ -56,6 +49,20 @@ func Open(cfg Config) (backend.Backend, error) {
return be, nil
}
func (be *s3) s3path(t backend.Type, name string) string {
var path string
if be.prefix != "" {
path = be.prefix + "/"
}
path += string(t)
if t == backend.Config {
return path
}
return path + "/" + name
}
func (be *s3) createConnections() {
be.connChan = make(chan struct{}, connLimit)
for i := 0; i < connLimit; i++ {
@ -72,7 +79,7 @@ func (be *s3) Location() string {
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) {
debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p))
path := s3path(h.Type, h.Name)
path := be.s3path(h.Type, h.Name)
obj, err := be.client.GetObject(be.bucketname, path)
if err != nil {
debug.Log("s3.GetReader", " err %v", err)
@ -101,7 +108,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) {
debug.Log("s3.Save", "%v bytes at %d", len(p), h)
path := s3path(h.Type, h.Name)
path := be.s3path(h.Type, h.Name)
// Check key does not already exist
_, err = be.client.StatObject(be.bucketname, path)
@ -126,7 +133,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) {
// Stat returns information about a blob.
func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
debug.Log("s3.Stat", "%v")
path := s3path(h.Type, h.Name)
path := be.s3path(h.Type, h.Name)
obj, err := be.client.GetObject(be.bucketname, path)
if err != nil {
debug.Log("s3.Stat", "GetObject() err %v", err)
@ -145,7 +152,7 @@ func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
// Test returns true if a blob of the given type and name exists in the backend.
func (be *s3) Test(t backend.Type, name string) (bool, error) {
found := false
path := s3path(t, name)
path := be.s3path(t, name)
_, err := be.client.StatObject(be.bucketname, path)
if err == nil {
found = true
@ -157,7 +164,7 @@ func (be *s3) Test(t backend.Type, name string) (bool, error) {
// Remove removes the blob with the given name and type.
func (be *s3) Remove(t backend.Type, name string) error {
path := s3path(t, name)
path := be.s3path(t, name)
err := be.client.RemoveObject(be.bucketname, path)
debug.Log("s3.Remove", "%v %v -> err %v", t, name, err)
return err
@ -170,7 +177,7 @@ func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string {
debug.Log("s3.List", "listing %v", t)
ch := make(chan string)
prefix := s3path(t, "")
prefix := be.s3path(t, "")
listresp := be.client.ListObjects(be.bucketname, prefix, true, done)

26
location/location_test.go
View File

@ -48,30 +48,56 @@ var parseTests = []struct {
Config: s3.Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "restic",
}},
},
{"s3://hostname.foo/bucketname", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "hostname.foo",
Bucket: "bucketname",
Prefix: "restic",
}},
},
{"s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "hostname.foo",
Bucket: "bucketname",
Prefix: "prefix/directory",
}},
},
{"s3:eu-central-1/repo", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "eu-central-1",
Bucket: "repo",
Prefix: "restic",
}},
},
{"s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "eu-central-1",
Bucket: "repo",
Prefix: "prefix/directory",
}},
},
{"s3:https://hostname.foo/repo", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "restic",
}},
},
{"s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "prefix/directory",
}},
},
{"s3:http://hostname.foo/repo", Location{Scheme: "s3",
Config: s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "restic",
UseHTTP: true,
}},
},