diff --git a/doc/Manual.md b/doc/Manual.md index 6d4d1f513..684bc454a 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -380,6 +380,98 @@ Now you can easily initialize restic to use Minio server as backend with this co Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. +# Removing old snapshots + +All backup space is finite, so restic allows removing old snapshots. This can +be done either manually (by specifying a snapshot ID to remove) or by using a +policy that describes which snapshots to forget. For all remove operations, two +commands need to be called in sequence: `forget` to remove a snapshot and +`prune` to actually remove the data that was referenced by the snapshot from +the repository. + +## Remove a single snapshot + +The command `snapshots` can be used to list all snapshots in a repository like this: + + $ restic -r /tmp/backup snapshots + enter password for repository: + ID Date Host Directory + ---------------------------------------------------------------------- + 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work + 79766175 2015-05-08 21:40:19 kasimir /home/user/work + bdbd3439 2015-05-08 21:45:17 luigi /home/art + 590c8fc8 2015-05-08 21:47:38 kazik /srv + 9f0bc19e 2015-05-08 21:46:11 luigi /srv + +In order to remove the snapshot of `/home/art`, use the `forget` command and +specify the snapshot ID on the command line: + + $ restic -r /tmp/backup forget bdbd3439 + enter password for repository: + removed snapshot d3f01f63 + +Afterwards this snapshot is removed: + + $ restic -r /tmp/backup snapshots + enter password for repository: + ID Date Host Directory + ---------------------------------------------------------------------- + 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work + 79766175 2015-05-08 21:40:19 kasimir /home/user/work + 590c8fc8 2015-05-08 21:47:38 kazik /srv + 9f0bc19e 2015-05-08 21:46:11 luigi /srv + +But the data that was referenced by files in this snapshot is still stored in +the repository. To cleanup unreferenced data, the `prune` command must be run: + + $ restic -r /tmp/backup prune + enter password for repository: + + counting files in repo + building new index for repo + [0:00] 100.00% 22 / 22 files + repository contains 22 packs (8512 blobs) with 100.092 MiB bytes + processed 8512 blobs: 0 duplicate blobs, 0B duplicate + load all snapshots + find data that is still in use for 1 snapshots + [0:00] 100.00% 1 / 1 snapshots + found 8433 of 8512 data blobs still in use + will rewrite 3 packs + creating new index + [0:00] 86.36% 19 / 22 files + saved new index as 544a5084 + done + +Afterwards the repository is smaller. + +## Removing snapshots according to a policy + +Removing snapshots manually is tedious and error-prone, therefore restic allows +specifying which snapshots should be removed automatically according to a +policy. You can specify how many hourly, daily, weekly, monthly and yearly +snapshots to keep, any other snapshots are removed. The most important +command-line parameter here is `--dry-run` which instructs restic to not remove +anything but print which snapshots would be removed. + +When `forget` is run with a policy, restic loads the list of all snapshots, +then groups these by host name and list of directories. The policy is then +applied to each group of snapshots separately. This is a safety feature. + +The `forget` command accepts the following parameters: + + * `--keep-last n` never delete the `n` last (most recent) snapshots + * `--keep-hourly n` for the last `n` hours in which a snapshot was made, keep + only the last snapshot for each hour. + * `--keep-daily n` for the last `n` days which have one or more snapshots, only + keep the last one for that day. + * `--keep-monthly n` for the last `n` months which have one or more snapshots, only + keep the last one for that month. + * `--keep-yearly n` for the last `n` years which have one or more snapshots, only + keep the last one for that year. + +Additionally, you can restrict removing snapshots to those which have a +particular hostname with the `--hostname` parameter. + # Debugging restic The program can be built with debug support like this: diff --git a/src/cmds/restic/cmd_cat.go b/src/cmds/restic/cmd_cat.go index e3fdebe4c..58e5dc452 100644 --- a/src/cmds/restic/cmd_cat.go +++ b/src/cmds/restic/cmd_cat.go @@ -161,13 +161,14 @@ func (cmd CmdCat) Execute(args []string) error { return err case "blob": - blob, err := repo.Index().Lookup(id) + list, err := repo.Index().Lookup(id, pack.Data) if err != nil { return err } + blob := list[0] buf := make([]byte, blob.Length) - data, err := repo.LoadBlob(blob.Type, id, buf) + data, err := repo.LoadBlob(id, pack.Data, buf) if err != nil { return err } diff --git a/src/cmds/restic/cmd_dump.go b/src/cmds/restic/cmd_dump.go index 72a9d85b8..95efa2574 100644 --- a/src/cmds/restic/cmd_dump.go +++ b/src/cmds/restic/cmd_dump.go @@ -48,7 +48,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error { return err } -func printSnapshots(repo *repository.Repository, wr io.Writer) error { +func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error { done := make(chan struct{}) defer close(done) @@ -126,9 +126,9 @@ func printPacks(repo *repository.Repository, wr io.Writer) error { name := job.Data.(string) h := backend.Handle{Type: backend.Data, Name: name} - rd := backend.NewReadSeeker(repo.Backend(), h) + ldr := pack.BackendLoader{Backend: repo.Backend(), Handle: h} - unpacker, err := pack.NewUnpacker(repo.Key(), rd) + unpacker, err := pack.NewUnpacker(repo.Key(), ldr) if err != nil { return nil, err } @@ -226,14 +226,14 @@ func (cmd CmdDump) Execute(args []string) error { case "indexes": return cmd.DumpIndexes() case "snapshots": - return printSnapshots(repo, os.Stdout) + return debugPrintSnapshots(repo, os.Stdout) case "trees": return printTrees(repo, os.Stdout) case "packs": return printPacks(repo, os.Stdout) case "all": fmt.Printf("snapshots:\n") - err := printSnapshots(repo, os.Stdout) + err := debugPrintSnapshots(repo, os.Stdout) if err != nil { return err } diff --git a/src/cmds/restic/cmd_forget.go b/src/cmds/restic/cmd_forget.go new file mode 100644 index 000000000..16da4b556 --- /dev/null +++ b/src/cmds/restic/cmd_forget.go @@ -0,0 +1,168 @@ +package main + +import ( + "fmt" + "io" + "restic" + "restic/backend" + "strings" +) + +// CmdForget implements the 'forget' command. +type CmdForget struct { + Last int `short:"l" long:"keep-last" description:"keep the last n snapshots"` + Hourly int `short:"H" long:"keep-hourly" description:"keep the last n hourly snapshots"` + Daily int `short:"d" long:"keep-daily" description:"keep the last n daily snapshots"` + Weekly int `short:"w" long:"keep-weekly" description:"keep the last n weekly snapshots"` + Monthly int `short:"m" long:"keep-monthly" description:"keep the last n monthly snapshots"` + Yearly int `short:"y" long:"keep-yearly" description:"keep the last n yearly snapshots"` + + Hostname string `long:"hostname" description:"only forget snapshots for the given hostname"` + + DryRun bool `short:"n" long:"dry-run" description:"do not delete anything, just print what would be done"` + + global *GlobalOptions +} + +func init() { + _, err := parser.AddCommand("forget", + "removes snapshots from a repository", + ` +The forget command removes snapshots according to a policy. Please note +that this command really only deletes the snapshot object in the repo, which +is a reference to data stored there. In order to remove this (now +unreferenced) data after 'forget' was run successfully, see the 'prune' +command. +`, + &CmdForget{global: &globalOpts}) + if err != nil { + panic(err) + } +} + +// Usage returns usage information for 'forget'. +func (cmd CmdForget) Usage() string { + return "[snapshot ID] ..." +} + +func printSnapshots(w io.Writer, snapshots restic.Snapshots) { + tab := NewTable() + tab.Header = fmt.Sprintf("%-8s %-19s %-10s %s", "ID", "Date", "Host", "Directory") + tab.RowFormat = "%-8s %-19s %-10s %s" + + for _, sn := range snapshots { + if len(sn.Paths) == 0 { + continue + } + id := sn.ID() + tab.Rows = append(tab.Rows, []interface{}{id.Str(), sn.Time.Format(TimeFormat), sn.Hostname, sn.Paths[0]}) + + if len(sn.Paths) > 1 { + for _, path := range sn.Paths[1:] { + tab.Rows = append(tab.Rows, []interface{}{"", "", "", path}) + } + } + } + + tab.Write(w) +} + +// Execute runs the 'forget' command. +func (cmd CmdForget) Execute(args []string) error { + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + err = repo.LoadIndex() + if err != nil { + return err + } + + // first, process all snapshot IDs given as arguments + for _, s := range args { + id, err := restic.FindSnapshot(repo, s) + if err != nil { + return err + } + + if !cmd.DryRun { + err = repo.Backend().Remove(backend.Snapshot, id.String()) + if err != nil { + return err + } + + cmd.global.Verbosef("removed snapshot %v\n", id.Str()) + } else { + cmd.global.Verbosef("would removed snapshot %v\n", id.Str()) + } + } + + policy := restic.ExpirePolicy{ + Last: cmd.Last, + Hourly: cmd.Hourly, + Daily: cmd.Daily, + Weekly: cmd.Weekly, + Monthly: cmd.Monthly, + Yearly: cmd.Yearly, + } + + if policy.Empty() { + return nil + } + + // then, load all remaining snapshots + snapshots, err := restic.LoadAllSnapshots(repo) + if err != nil { + return err + } + + // group by hostname and dirs + type key struct { + Hostname string + Dirs string + } + + snapshotGroups := make(map[key]restic.Snapshots) + + for _, sn := range snapshots { + if cmd.Hostname != "" && sn.Hostname != cmd.Hostname { + continue + } + + k := key{Hostname: sn.Hostname, Dirs: strings.Join(sn.Paths, ":")} + list := snapshotGroups[k] + list = append(list, sn) + snapshotGroups[k] = list + } + + for key, snapshotGroup := range snapshotGroups { + cmd.global.Printf("snapshots for host %v, directories %v:\n\n", key.Hostname, key.Dirs) + keep, remove := restic.ApplyPolicy(snapshotGroup, policy) + + cmd.global.Printf("keep %d snapshots:\n", len(keep)) + printSnapshots(cmd.global.stdout, keep) + cmd.global.Printf("\n") + + cmd.global.Printf("remove %d snapshots:\n", len(remove)) + printSnapshots(cmd.global.stdout, remove) + cmd.global.Printf("\n") + + if !cmd.DryRun { + for _, sn := range remove { + err = repo.Backend().Remove(backend.Snapshot, sn.ID().String()) + if err != nil { + return err + } + } + } + } + + return nil +} diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go new file mode 100644 index 000000000..a2aa6451d --- /dev/null +++ b/src/cmds/restic/cmd_prune.go @@ -0,0 +1,215 @@ +package main + +import ( + "fmt" + "os" + "restic" + "restic/backend" + "restic/debug" + "restic/index" + "restic/pack" + "restic/repository" + "time" + + "golang.org/x/crypto/ssh/terminal" +) + +// CmdPrune implements the 'prune' command. +type CmdPrune struct { + global *GlobalOptions +} + +func init() { + _, err := parser.AddCommand("prune", + "removes content from a repository", + ` +The prune command removes rendundant and unneeded data from the repository. +For removing snapshots, please see the 'forget' command, then afterwards run +'prune'. +`, + &CmdPrune{global: &globalOpts}) + if err != nil { + panic(err) + } +} + +// newProgressMax returns a progress that counts blobs. +func newProgressMax(show bool, max uint64, description string) *restic.Progress { + if !show { + return nil + } + + p := restic.NewProgress(time.Second) + + p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { + status := fmt.Sprintf("[%s] %s %d / %d %s", + formatDuration(d), + formatPercent(s.Blobs, max), + s.Blobs, max, description) + + w, _, err := terminal.GetSize(int(os.Stdout.Fd())) + if err == nil { + if len(status) > w { + max := w - len(status) - 4 + status = status[:max] + "... " + } + } + + fmt.Printf("\x1b[2K%s\r", status) + } + + p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { + fmt.Printf("\n") + } + + return p +} + +// Execute runs the 'prune' command. +func (cmd CmdPrune) Execute(args []string) error { + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + err = repo.LoadIndex() + if err != nil { + return err + } + + done := make(chan struct{}) + defer close(done) + + var stats struct { + blobs int + packs int + snapshots int + bytes int64 + } + + cmd.global.Verbosef("counting files in repo\n") + for _ = range repo.List(backend.Data, done) { + stats.packs++ + } + + cmd.global.Verbosef("building new index for repo\n") + + bar := newProgressMax(cmd.global.ShowProgress(), uint64(stats.packs), "files") + idx, err := index.New(repo, bar) + if err != nil { + return err + } + + for _, pack := range idx.Packs { + stats.bytes += pack.Size + } + cmd.global.Verbosef("repository contains %v packs (%v blobs) with %v bytes\n", + len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes))) + + blobCount := make(map[pack.Handle]int) + duplicateBlobs := 0 + duplicateBytes := 0 + + // find duplicate blobs + for _, p := range idx.Packs { + for _, entry := range p.Entries { + stats.blobs++ + h := pack.Handle{ID: entry.ID, Type: entry.Type} + blobCount[h]++ + + if blobCount[h] > 1 { + duplicateBlobs++ + duplicateBytes += int(entry.Length) + } + } + } + + cmd.global.Verbosef("processed %d blobs: %d duplicate blobs, %v duplicate\n", + stats.blobs, duplicateBlobs, formatBytes(uint64(duplicateBytes))) + cmd.global.Verbosef("load all snapshots\n") + + // find referenced blobs + snapshots, err := restic.LoadAllSnapshots(repo) + if err != nil { + return err + } + + stats.snapshots = len(snapshots) + + cmd.global.Verbosef("find data that is still in use for %d snapshots\n", stats.snapshots) + + usedBlobs := pack.NewBlobSet() + seenBlobs := pack.NewBlobSet() + + bar = newProgressMax(cmd.global.ShowProgress(), uint64(len(snapshots)), "snapshots") + bar.Start() + for _, sn := range snapshots { + debug.Log("CmdPrune.Execute", "process snapshot %v", sn.ID().Str()) + + err = restic.FindUsedBlobs(repo, *sn.Tree, usedBlobs, seenBlobs) + if err != nil { + return err + } + + debug.Log("CmdPrune.Execute", "found %v blobs for snapshot %v", sn.ID().Str()) + bar.Report(restic.Stat{Blobs: 1}) + } + bar.Done() + + cmd.global.Verbosef("found %d of %d data blobs still in use\n", len(usedBlobs), stats.blobs) + + // find packs that need a rewrite + rewritePacks := backend.NewIDSet() + for h, blob := range idx.Blobs { + if !usedBlobs.Has(h) { + rewritePacks.Merge(blob.Packs) + } + + if blobCount[h] > 1 { + rewritePacks.Merge(blob.Packs) + } + } + + cmd.global.Verbosef("will rewrite %d packs\n", len(rewritePacks)) + + err = repository.Repack(repo, rewritePacks, usedBlobs) + if err != nil { + return err + } + + cmd.global.Verbosef("creating new index\n") + + for _ = range repo.List(backend.Data, done) { + stats.packs++ + } + bar = newProgressMax(cmd.global.ShowProgress(), uint64(stats.packs), "files") + idx, err = index.New(repo, bar) + if err != nil { + return err + } + + var supersedes backend.IDs + for idxID := range repo.List(backend.Index, done) { + err := repo.Backend().Remove(backend.Index, idxID.String()) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err) + } + + supersedes = append(supersedes, idxID) + } + + id, err := idx.Save(repo, supersedes) + if err != nil { + return err + } + cmd.global.Verbosef("saved new index as %v\n", id.Str()) + + cmd.global.Verbosef("done\n") + return nil +} diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index 8ac54d80b..cb50c1054 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -1,14 +1,6 @@ package main -import ( - "fmt" - "os" - "restic/backend" - "restic/debug" - "restic/pack" - "restic/repository" - "restic/worker" -) +import "restic/repository" type CmdRebuildIndex struct { global *GlobalOptions @@ -26,94 +18,6 @@ func init() { } } -const rebuildIndexWorkers = 10 - -func loadBlobsFromPacks(repo *repository.Repository) (packs map[backend.ID][]pack.Blob) { - done := make(chan struct{}) - defer close(done) - - f := func(job worker.Job, done <-chan struct{}) (interface{}, error) { - return repo.ListPack(job.Data.(backend.ID)) - } - - jobCh := make(chan worker.Job) - resCh := make(chan worker.Job) - wp := worker.New(rebuildIndexWorkers, f, jobCh, resCh) - - go func() { - for id := range repo.List(backend.Data, done) { - jobCh <- worker.Job{Data: id} - } - close(jobCh) - }() - - packs = make(map[backend.ID][]pack.Blob) - for job := range resCh { - id := job.Data.(backend.ID) - - if job.Error != nil { - fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error) - continue - } - - entries := job.Result.([]pack.Blob) - packs[id] = entries - } - - wp.Wait() - - return packs -} - -func listIndexIDs(repo *repository.Repository) (list backend.IDs) { - done := make(chan struct{}) - for id := range repo.List(backend.Index, done) { - list = append(list, id) - } - - return list -} - -func (cmd CmdRebuildIndex) rebuildIndex() error { - debug.Log("RebuildIndex.RebuildIndex", "start rebuilding index") - - packs := loadBlobsFromPacks(cmd.repo) - cmd.global.Verbosef("loaded blobs from %d packs\n", len(packs)) - - idx := repository.NewIndex() - for packID, entries := range packs { - for _, entry := range entries { - pb := repository.PackedBlob{ - ID: entry.ID, - Type: entry.Type, - Length: entry.Length, - Offset: entry.Offset, - PackID: packID, - } - idx.Store(pb) - } - } - - oldIndexes := listIndexIDs(cmd.repo) - idx.AddToSupersedes(oldIndexes...) - cmd.global.Printf(" saving new index\n") - id, err := repository.SaveIndex(cmd.repo, idx) - if err != nil { - debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) - return err - } - debug.Log("RebuildIndex.RebuildIndex", "new index saved as %v", id.Str()) - - for _, indexID := range oldIndexes { - err := cmd.repo.Backend().Remove(backend.Index, indexID.String()) - if err != nil { - cmd.global.Warnf("unable to remove index %v: %v\n", indexID.Str(), err) - } - } - - return nil -} - func (cmd CmdRebuildIndex) Execute(args []string) error { repo, err := cmd.global.OpenRepository() if err != nil { @@ -127,5 +31,5 @@ func (cmd CmdRebuildIndex) Execute(args []string) error { return err } - return cmd.rebuildIndex() + return repository.RebuildIndex(repo) } diff --git a/src/restic/archive_reader.go b/src/restic/archive_reader.go index 5a485e0fd..ad6bded8c 100644 --- a/src/restic/archive_reader.go +++ b/src/restic/archive_reader.go @@ -22,7 +22,7 @@ func saveTreeJSON(repo *repository.Repository, item interface{}) (backend.ID, er // check if tree has been saved before id := backend.Hash(data) - if repo.Index().Has(id) { + if repo.Index().Has(id, pack.Tree) { return id, nil } @@ -58,7 +58,7 @@ func ArchiveReader(repo *repository.Repository, p *Progress, rd io.Reader, name id := backend.Hash(chunk.Data) - if !repo.Index().Has(id) { + if !repo.Index().Has(id, pack.Data) { _, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil) if err != nil { return nil, backend.ID{}, err diff --git a/src/restic/archive_reader_test.go b/src/restic/archive_reader_test.go index 4397bb90e..49fcbecab 100644 --- a/src/restic/archive_reader_test.go +++ b/src/restic/archive_reader_test.go @@ -13,7 +13,7 @@ import ( ) func loadBlob(t *testing.T, repo *repository.Repository, id backend.ID, buf []byte) []byte { - buf, err := repo.LoadBlob(pack.Data, id, buf) + buf, err := repo.LoadBlob(id, pack.Data, buf) if err != nil { t.Fatalf("LoadBlob(%v) returned error %v", id, err) } diff --git a/src/restic/archiver.go b/src/restic/archiver.go index 5aa27dcc4..e3ac4237c 100644 --- a/src/restic/archiver.go +++ b/src/restic/archiver.go @@ -72,7 +72,7 @@ func NewArchiver(repo *repository.Repository) *Archiver { // When the blob is not known, false is returned and the blob is added to the // list. This means that the caller false is returned to is responsible to save // the blob to the backend. -func (arch *Archiver) isKnownBlob(id backend.ID) bool { +func (arch *Archiver) isKnownBlob(id backend.ID, t pack.BlobType) bool { arch.knownBlobs.Lock() defer arch.knownBlobs.Unlock() @@ -82,7 +82,7 @@ func (arch *Archiver) isKnownBlob(id backend.ID) bool { arch.knownBlobs.Insert(id) - _, err := arch.repo.Index().Lookup(id) + _, err := arch.repo.Index().Lookup(id, t) if err == nil { return true } @@ -94,7 +94,7 @@ func (arch *Archiver) isKnownBlob(id backend.ID) bool { func (arch *Archiver) Save(t pack.BlobType, data []byte, id backend.ID) error { debug.Log("Archiver.Save", "Save(%v, %v)\n", t, id.Str()) - if arch.isKnownBlob(id) { + if arch.isKnownBlob(id, pack.Data) { debug.Log("Archiver.Save", "blob %v is known\n", id.Str()) return nil } @@ -119,7 +119,7 @@ func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) { // check if tree has been saved before id := backend.Hash(data) - if arch.isKnownBlob(id) { + if arch.isKnownBlob(id, pack.Tree) { return id, nil } diff --git a/src/restic/archiver_duplication_test.go b/src/restic/archiver_duplication_test.go index 1c0193eab..56341a997 100644 --- a/src/restic/archiver_duplication_test.go +++ b/src/restic/archiver_duplication_test.go @@ -101,7 +101,7 @@ func testArchiverDuplication(t *testing.T) { id := randomID() - if repo.Index().Has(id) { + if repo.Index().Has(id, pack.Data) { continue } diff --git a/src/restic/backend/idset.go b/src/restic/backend/idset.go index 12307219c..4bfe52ca2 100644 --- a/src/restic/backend/idset.go +++ b/src/restic/backend/idset.go @@ -60,6 +60,47 @@ func (s IDSet) Equals(other IDSet) bool { return true } +// Merge adds the blobs in other to the current set. +func (s IDSet) Merge(other IDSet) { + for id := range other { + s.Insert(id) + } +} + +// Intersect returns a new set containing the IDs that are present in both sets. +func (s IDSet) Intersect(other IDSet) (result IDSet) { + result = NewIDSet() + + set1 := s + set2 := other + + // iterate over the smaller set + if len(set2) < len(set1) { + set1, set2 = set2, set1 + } + + for id := range set1 { + if set2.Has(id) { + result.Insert(id) + } + } + + return result +} + +// Sub returns a new set containing all IDs that are present in s but not in +// other. +func (s IDSet) Sub(other IDSet) (result IDSet) { + result = NewIDSet() + for id := range s { + if !other.Has(id) { + result.Insert(id) + } + } + + return result +} + func (s IDSet) String() string { str := s.List().String() if len(str) < 2 { diff --git a/src/restic/backend/interface.go b/src/restic/backend/interface.go index fb0927c6e..24838ddfd 100644 --- a/src/restic/backend/interface.go +++ b/src/restic/backend/interface.go @@ -31,7 +31,9 @@ type Backend interface { Lister // Load returns the data stored in the backend for h at the given offset - // and saves it in p. Load has the same semantics as io.ReaderAt. + // and saves it in p. Load has the same semantics as io.ReaderAt, except + // that a negative offset is also allowed. In this case it references a + // position relative to the end of the file (similar to Seek()). Load(h Handle, p []byte, off int64) (int, error) // Save stores the data in the backend under the given handle. diff --git a/src/restic/backend/local/backend_test.go b/src/restic/backend/local/backend_test.go index 8607f01b7..8954dc83d 100644 --- a/src/restic/backend/local/backend_test.go +++ b/src/restic/backend/local/backend_test.go @@ -51,6 +51,13 @@ func TestLocalBackendLoad(t *testing.T) { test.TestLoad(t) } +func TestLocalBackendLoadNegativeOffset(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoadNegativeOffset(t) +} + func TestLocalBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/local/local.go b/src/restic/backend/local/local.go index 0821720c7..6fa1ac9f4 100644 --- a/src/restic/backend/local/local.go +++ b/src/restic/backend/local/local.go @@ -98,9 +98,12 @@ func dirname(base string, t backend.Type, name string) string { return filepath.Join(base, n) } -// Load returns the data stored in the backend for h at the given offset -// and saves it in p. Load has the same semantics as io.ReaderAt. +// Load returns the data stored in the backend for h at the given offset and +// saves it in p. Load has the same semantics as io.ReaderAt, with one +// exception: when off is lower than zero, it is treated as an offset relative +// to the end of the file. func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + debug.Log("backend.local.Load", "Load %v, length %v at %v", h, len(p), off) if err := h.Valid(); err != nil { return 0, err } @@ -117,11 +120,15 @@ func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { } }() - if off > 0 { + switch { + case off > 0: _, err = f.Seek(off, 0) - if err != nil { - return 0, err - } + case off < 0: + _, err = f.Seek(off, 2) + } + + if err != nil { + return 0, err } return io.ReadFull(f, p) @@ -162,6 +169,7 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) { // Save stores data in the backend at the handle. func (b *Local) Save(h backend.Handle, p []byte) (err error) { + debug.Log("backend.local.Save", "Save %v, length %v", h, len(p)) if err := h.Valid(); err != nil { return err } @@ -203,6 +211,7 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { // Stat returns information about a blob. func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) { + debug.Log("backend.local.Stat", "Stat %v", h) if err := h.Valid(); err != nil { return backend.BlobInfo{}, err } @@ -217,6 +226,7 @@ func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) { // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(t backend.Type, name string) (bool, error) { + debug.Log("backend.local.Test", "Test %v %v", t, name) _, err := fs.Stat(filename(b.p, t, name)) if err != nil { if os.IsNotExist(err) { @@ -230,6 +240,7 @@ func (b *Local) Test(t backend.Type, name string) (bool, error) { // Remove removes the blob with the given name and type. func (b *Local) Remove(t backend.Type, name string) error { + debug.Log("backend.local.Remove", "Remove %v %v", t, name) fn := filename(b.p, t, name) // reset read-only flag @@ -304,6 +315,7 @@ func listDirs(dir string) (filenames []string, err error) { // goroutine is started for this. If the channel done is closed, sending // stops. func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { + debug.Log("backend.local.List", "List %v", t) lister := listDir if t == backend.Data { lister = listDirs @@ -336,11 +348,13 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { // Delete removes the repository and all files. func (b *Local) Delete() error { + debug.Log("backend.local.Delete", "Delete()") return fs.RemoveAll(b.p) } // Close closes all open files. func (b *Local) Close() error { + debug.Log("backend.local.Close", "Close()") // this does not need to do anything, all open files are closed within the // same function. return nil diff --git a/src/restic/backend/mem/backend_test.go b/src/restic/backend/mem/backend_test.go index 13e95f115..6bf19580f 100644 --- a/src/restic/backend/mem/backend_test.go +++ b/src/restic/backend/mem/backend_test.go @@ -51,6 +51,13 @@ func TestMemBackendLoad(t *testing.T) { test.TestLoad(t) } +func TestMemBackendLoadNegativeOffset(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoadNegativeOffset(t) +} + func TestMemBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/mem/mem_backend.go b/src/restic/backend/mem/mem_backend.go index 2dde7e320..5682d4915 100644 --- a/src/restic/backend/mem/mem_backend.go +++ b/src/restic/backend/mem/mem_backend.go @@ -116,8 +116,13 @@ func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, err } buf := be.data[entry{h.Type, h.Name}] - if off > int64(len(buf)) { + switch { + case off > int64(len(buf)): return 0, errors.New("offset beyond end of file") + case off < -int64(len(buf)): + off = 0 + case off < 0: + off = int64(len(buf)) + off } buf = buf[off:] diff --git a/src/restic/backend/readseeker.go b/src/restic/backend/readseeker.go deleted file mode 100644 index ea063e3f3..000000000 --- a/src/restic/backend/readseeker.go +++ /dev/null @@ -1,63 +0,0 @@ -package backend - -import ( - "errors" - "io" -) - -type readSeeker struct { - be Backend - h Handle - t Type - name string - offset int64 - size int64 -} - -// NewReadSeeker returns an io.ReadSeeker for the given object in the backend. -func NewReadSeeker(be Backend, h Handle) io.ReadSeeker { - return &readSeeker{be: be, h: h} -} - -func (rd *readSeeker) Read(p []byte) (int, error) { - n, err := rd.be.Load(rd.h, p, rd.offset) - rd.offset += int64(n) - return n, err -} - -func (rd *readSeeker) Seek(offset int64, whence int) (n int64, err error) { - switch whence { - case 0: - rd.offset = offset - case 1: - rd.offset += offset - case 2: - if rd.size == 0 { - rd.size, err = rd.getSize() - if err != nil { - return 0, err - } - } - - pos := rd.size + offset - if pos < 0 { - return 0, errors.New("invalid offset, before start of blob") - } - - rd.offset = pos - return rd.offset, nil - default: - return 0, errors.New("invalid value for parameter whence") - } - - return rd.offset, nil -} - -func (rd *readSeeker) getSize() (int64, error) { - stat, err := rd.be.Stat(rd.h) - if err != nil { - return 0, err - } - - return stat.Size, nil -} diff --git a/src/restic/backend/readseeker_test.go b/src/restic/backend/readseeker_test.go deleted file mode 100644 index 013f2528e..000000000 --- a/src/restic/backend/readseeker_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package backend_test - -import ( - "bytes" - "io" - "math/rand" - "restic/backend" - "restic/backend/mem" - "testing" - - . "restic/test" -) - -func abs(a int) int { - if a < 0 { - return -a - } - - return a -} - -func loadAndCompare(t testing.TB, rd io.ReadSeeker, size int, offset int64, expected []byte) { - var ( - pos int64 - err error - ) - - if offset >= 0 { - pos, err = rd.Seek(offset, 0) - } else { - pos, err = rd.Seek(offset, 2) - } - if err != nil { - t.Errorf("Seek(%d, 0) returned error: %v", offset, err) - return - } - - if offset >= 0 && pos != offset { - t.Errorf("pos after seek is wrong, want %d, got %d", offset, pos) - } else if offset < 0 && pos != int64(size)+offset { - t.Errorf("pos after relative seek is wrong, want %d, got %d", int64(size)+offset, pos) - } - - buf := make([]byte, len(expected)) - n, err := rd.Read(buf) - - // if we requested data beyond the end of the file, ignore - // ErrUnexpectedEOF error - if offset > 0 && len(buf) > size && err == io.ErrUnexpectedEOF { - err = nil - buf = buf[:size] - } - - if offset < 0 && len(buf) > abs(int(offset)) && err == io.ErrUnexpectedEOF { - err = nil - buf = buf[:abs(int(offset))] - } - - if n != len(buf) { - t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d", - len(buf), offset, len(buf), n) - return - } - - if err != nil { - t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), offset, err) - return - } - - buf = buf[:n] - if !bytes.Equal(buf, expected) { - t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), offset) - return - } -} - -func TestReadSeeker(t *testing.T) { - b := mem.New() - - length := rand.Intn(1<<24) + 2000 - - data := Random(23, length) - id := backend.Hash(data) - - handle := backend.Handle{Type: backend.Data, Name: id.String()} - err := b.Save(handle, data) - if err != nil { - t.Fatalf("Save() error: %v", err) - } - - for i := 0; i < 50; i++ { - l := rand.Intn(length + 2000) - o := rand.Intn(length + 2000) - - if rand.Float32() > 0.5 { - o = -o - } - - d := data - if o > 0 && o < len(d) { - d = d[o:] - } else { - o = len(d) - d = d[:0] - } - - if l > 0 && l < len(d) { - d = d[:l] - } - - rd := backend.NewReadSeeker(b, handle) - loadAndCompare(t, rd, len(data), int64(o), d) - } -} diff --git a/src/restic/backend/rest/backend_test.go b/src/restic/backend/rest/backend_test.go index 4274bfcb1..9605396d6 100644 --- a/src/restic/backend/rest/backend_test.go +++ b/src/restic/backend/rest/backend_test.go @@ -51,6 +51,13 @@ func TestRestBackendLoad(t *testing.T) { test.TestLoad(t) } +func TestRestBackendLoadNegativeOffset(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoadNegativeOffset(t) +} + func TestRestBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index e9303358a..125331d78 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -75,6 +75,20 @@ func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err er return 0, err } + // invert offset + if off < 0 { + info, err := b.Stat(h) + if err != nil { + return 0, err + } + + if -off > info.Size { + off = 0 + } else { + off = info.Size + off + } + } + req, err := http.NewRequest("GET", restPath(b.url, h), nil) if err != nil { return 0, err diff --git a/src/restic/backend/s3/backend_test.go b/src/restic/backend/s3/backend_test.go index 82eca2631..9fb4dd3fa 100644 --- a/src/restic/backend/s3/backend_test.go +++ b/src/restic/backend/s3/backend_test.go @@ -51,6 +51,13 @@ func TestS3BackendLoad(t *testing.T) { test.TestLoad(t) } +func TestS3BackendLoadNegativeOffset(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoadNegativeOffset(t) +} + func TestS3BackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index c35719329..2fc6f9f31 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -77,37 +77,75 @@ func (be *s3) Location() string { // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. -func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) { +func (be s3) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + var obj *minio.Object + debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p)) path := be.s3path(h.Type, h.Name) - obj, err := be.client.GetObject(be.bucketname, path) - if err != nil { - debug.Log("s3.GetReader", " err %v", err) - return 0, err - } - - if off > 0 { - _, err = obj.Seek(off, 0) - if err != nil { - return 0, err - } - } <-be.connChan defer func() { be.connChan <- struct{}{} }() - // This may not read the whole object, so ensure object - // is closed to avoid duplicate connections. - n, err := io.ReadFull(obj, p) + obj, err = be.client.GetObject(be.bucketname, path) if err != nil { - obj.Close() - } else { - err = obj.Close() + debug.Log("s3.Load", " err %v", err) + return 0, err } - return n, err + // make sure that the object is closed properly. + defer func() { + e := obj.Close() + if err == nil { + err = e + } + }() + + info, err := obj.Stat() + if err != nil { + return 0, err + } + + // handle negative offsets + if off < 0 { + // if the negative offset is larger than the object itself, read from + // the beginning. + if -off > info.Size { + off = 0 + } else { + // otherwise compute the offset from the end of the file. + off = info.Size + off + } + } + + // return an error if the offset is beyond the end of the file + if off > info.Size { + return 0, io.EOF + } + + var nextError error + + // manually create an io.ErrUnexpectedEOF + if off+int64(len(p)) > info.Size { + newlen := info.Size - off + p = p[:newlen] + + nextError = io.ErrUnexpectedEOF + + debug.Log("s3.Load", " capped buffer to %v byte", len(p)) + } + + n, err = obj.ReadAt(p, off) + if int64(n) == info.Size-off && err == io.EOF { + err = nil + } + + if err == nil { + err = nextError + } + + return n, err } // Save stores data in the backend at the handle. @@ -116,7 +154,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) { return err } - debug.Log("s3.Save", "%v bytes at %d", len(p), h) + debug.Log("s3.Save", "%v with %d bytes", h, len(p)) path := be.s3path(h.Type, h.Name) diff --git a/src/restic/backend/sftp/backend_test.go b/src/restic/backend/sftp/backend_test.go index a812f8cd0..c28dd8c99 100644 --- a/src/restic/backend/sftp/backend_test.go +++ b/src/restic/backend/sftp/backend_test.go @@ -51,6 +51,13 @@ func TestSftpBackendLoad(t *testing.T) { test.TestLoad(t) } +func TestSftpBackendLoadNegativeOffset(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoadNegativeOffset(t) +} + func TestSftpBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index 801c3075a..e6e245b3c 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -308,11 +308,15 @@ func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { } }() - if off > 0 { + switch { + case off > 0: _, err = f.Seek(off, 0) - if err != nil { - return 0, err - } + case off < 0: + _, err = f.Seek(off, 2) + } + + if err != nil { + return 0, err } return io.ReadFull(f, p) diff --git a/src/restic/backend/test/backend_test.go b/src/restic/backend/test/backend_test.go index b495ce663..c577092fb 100644 --- a/src/restic/backend/test/backend_test.go +++ b/src/restic/backend/test/backend_test.go @@ -51,6 +51,13 @@ func TestTestBackendLoad(t *testing.T) { test.TestLoad(t) } +func TestTestBackendLoadNegativeOffset(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestLoadNegativeOffset(t) +} + func TestTestBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index ef0efd77a..77896499e 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -220,9 +220,60 @@ func TestLoad(t testing.TB) { buf := make([]byte, l) n, err := b.Load(handle, buf, int64(o)) - // if we requested data beyond the end of the file, ignore + // if we requested data beyond the end of the file, require // ErrUnexpectedEOF error - if l > len(d) && err == io.ErrUnexpectedEOF { + if l > len(d) { + if err != io.ErrUnexpectedEOF { + t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o)) + } + err = nil + buf = buf[:len(d)] + } + + if err != nil { + t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err) + continue + } + + if n != len(buf) { + t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d", + len(buf), int64(o), len(buf), n) + continue + } + + buf = buf[:n] + if !bytes.Equal(buf, d) { + t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o)) + continue + } + } + + // test with negative offset + for i := 0; i < 50; i++ { + l := rand.Intn(length + 2000) + o := rand.Intn(length + 2000) + + d := data + if o < len(d) { + d = d[len(d)-o:] + } else { + o = 0 + } + + if l > 0 && l < len(d) { + d = d[:l] + } + + buf := make([]byte, l) + n, err := b.Load(handle, buf, -int64(o)) + + // if we requested data beyond the end of the file, require + // ErrUnexpectedEOF error + if l > len(d) { + if err != io.ErrUnexpectedEOF { + t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o)) + continue + } err = nil buf = buf[:len(d)] } @@ -259,6 +310,62 @@ func TestLoad(t testing.TB) { OK(t, b.Remove(backend.Data, id.String())) } +// TestLoadNegativeOffset tests the backend's Load function with negative offsets. +func TestLoadNegativeOffset(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<24) + 2000 + + data := Random(23, length) + id := backend.Hash(data) + + handle := backend.Handle{Type: backend.Data, Name: id.String()} + err := b.Save(handle, data) + if err != nil { + t.Fatalf("Save() error: %v", err) + } + + // test normal reads + for i := 0; i < 50; i++ { + l := rand.Intn(length + 2000) + o := -rand.Intn(length + 2000) + + buf := make([]byte, l) + n, err := b.Load(handle, buf, int64(o)) + + // if we requested data beyond the end of the file, require + // ErrUnexpectedEOF error + if len(buf) > -o { + if err != io.ErrUnexpectedEOF { + t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), o) + continue + } + err = nil + buf = buf[:-o] + } + + if err != nil { + t.Errorf("Load(%d, %d) returned error: %v", len(buf), o, err) + continue + } + + if n != len(buf) { + t.Errorf("Load(%d, %d) returned short read, only got %d bytes", len(buf), o, n) + continue + } + + p := len(data) + o + if !bytes.Equal(buf, data[p:p+len(buf)]) { + t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), o) + continue + } + + } + + OK(t, b.Remove(backend.Data, id.String())) +} + // TestSave tests saving data in the backend. func TestSave(t testing.TB) { b := open(t) diff --git a/src/restic/backend/testing.go b/src/restic/backend/testing.go new file mode 100644 index 000000000..e0c3dd569 --- /dev/null +++ b/src/restic/backend/testing.go @@ -0,0 +1,17 @@ +package backend + +import ( + "crypto/rand" + "io" +) + +// RandomID retuns a randomly generated ID. This is mainly used for testing. +// When reading from rand fails, the function panics. +func RandomID() ID { + id := ID{} + _, err := io.ReadFull(rand.Reader, id[:]) + if err != nil { + panic(err) + } + return id +} diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go index 2f796de69..bcfa56a04 100644 --- a/src/restic/checker/checker.go +++ b/src/restic/checker/checker.go @@ -1,7 +1,6 @@ package checker import ( - "bytes" "errors" "fmt" "sync" @@ -677,7 +676,7 @@ func checkPack(r *repository.Repository, id backend.ID) error { return fmt.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) } - unpacker, err := pack.NewUnpacker(r.Key(), bytes.NewReader(buf)) + unpacker, err := pack.NewUnpacker(r.Key(), pack.BufferLoader(buf)) if err != nil { return err } diff --git a/src/restic/checker/repacker.go b/src/restic/checker/repacker.go deleted file mode 100644 index 8aedfc089..000000000 --- a/src/restic/checker/repacker.go +++ /dev/null @@ -1,163 +0,0 @@ -package checker - -import ( - "errors" - - "restic/backend" - "restic/debug" - "restic/repository" -) - -// Repacker extracts still used blobs from packs with unused blobs and creates -// new packs. -type Repacker struct { - unusedBlobs backend.IDSet - repo *repository.Repository -} - -// NewRepacker returns a new repacker that (when Repack() in run) cleans up the -// repository and creates new packs and indexs so that all blobs in unusedBlobs -// aren't used any more. -func NewRepacker(repo *repository.Repository, unusedBlobs backend.IDSet) *Repacker { - return &Repacker{ - repo: repo, - unusedBlobs: unusedBlobs, - } -} - -// Repack runs the process of finding still used blobs in packs with unused -// blobs, extracts them and creates new packs with just the still-in-use blobs. -func (r *Repacker) Repack() error { - debug.Log("Repacker.Repack", "searching packs for %v", r.unusedBlobs) - - unneededPacks, err := FindPacksForBlobs(r.repo, r.unusedBlobs) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "found packs: %v", unneededPacks) - - blobs, err := FindBlobsForPacks(r.repo, unneededPacks) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "found blobs: %v", blobs) - - for id := range r.unusedBlobs { - debug.Log("Repacker.Repack", "remove unused blob %v", id.Str()) - blobs.Delete(id) - } - - debug.Log("Repacker.Repack", "need to repack blobs: %v", blobs) - - err = RepackBlobs(r.repo, r.repo, blobs) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "remove unneeded packs: %v", unneededPacks) - for packID := range unneededPacks { - err = r.repo.Backend().Remove(backend.Data, packID.String()) - if err != nil { - return err - } - } - - debug.Log("Repacker.Repack", "rebuild index, unneeded packs: %v", unneededPacks) - idx, err := r.repo.Index().RebuildIndex(unneededPacks) - - newIndexID, err := repository.SaveIndex(r.repo, idx) - debug.Log("Repacker.Repack", "saved new index at %v, err %v", newIndexID.Str(), err) - if err != nil { - return err - } - - debug.Log("Repacker.Repack", "remove old indexes: %v", idx.Supersedes()) - for _, id := range idx.Supersedes() { - err = r.repo.Backend().Remove(backend.Index, id.String()) - if err != nil { - debug.Log("Repacker.Repack", "error removing index %v: %v", id.Str(), err) - return err - } - - debug.Log("Repacker.Repack", "removed index %v", id.Str()) - } - - return nil -} - -// FindPacksForBlobs returns the set of packs that contain the blobs. -func FindPacksForBlobs(repo *repository.Repository, blobs backend.IDSet) (backend.IDSet, error) { - packs := backend.NewIDSet() - idx := repo.Index() - for id := range blobs { - blob, err := idx.Lookup(id) - if err != nil { - return nil, err - } - - packs.Insert(blob.PackID) - } - - return packs, nil -} - -// FindBlobsForPacks returns the set of blobs contained in a pack of packs. -func FindBlobsForPacks(repo *repository.Repository, packs backend.IDSet) (backend.IDSet, error) { - blobs := backend.NewIDSet() - - for packID := range packs { - for _, packedBlob := range repo.Index().ListPack(packID) { - blobs.Insert(packedBlob.ID) - } - } - - return blobs, nil -} - -// repackBlob loads a single blob from src and saves it in dst. -func repackBlob(src, dst *repository.Repository, id backend.ID) error { - blob, err := src.Index().Lookup(id) - if err != nil { - return err - } - - debug.Log("RepackBlobs", "repacking blob %v, len %v", id.Str(), blob.PlaintextLength()) - - buf := make([]byte, 0, blob.PlaintextLength()) - buf, err = src.LoadBlob(blob.Type, id, buf) - if err != nil { - return err - } - - if uint(len(buf)) != blob.PlaintextLength() { - debug.Log("RepackBlobs", "repack blob %v: len(buf) isn't equal to length: %v = %v", id.Str(), len(buf), blob.PlaintextLength()) - return errors.New("LoadBlob returned wrong data, len() doesn't match") - } - - _, err = dst.SaveAndEncrypt(blob.Type, buf, &id) - if err != nil { - return err - } - - return nil -} - -// RepackBlobs reads all blobs in blobIDs from src and saves them into new pack -// files in dst. Source and destination repo may be the same. -func RepackBlobs(src, dst *repository.Repository, blobIDs backend.IDSet) (err error) { - for id := range blobIDs { - err = repackBlob(src, dst, id) - if err != nil { - return err - } - } - - err = dst.Flush() - if err != nil { - return err - } - - return nil -} diff --git a/src/restic/checker/repacker_test.go b/src/restic/checker/repacker_test.go deleted file mode 100644 index 821828c8b..000000000 --- a/src/restic/checker/repacker_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package checker_test - -import ( - "testing" - - "restic/backend" - "restic/checker" - - . "restic/test" -) - -var findPackTests = []struct { - blobIDs backend.IDSet - packIDs backend.IDSet -}{ - { - backend.IDSet{ - ParseID("534f211b4fc2cf5b362a24e8eba22db5372a75b7e974603ff9263f5a471760f4"): struct{}{}, - ParseID("51aa04744b518c6a85b4e7643cfa99d58789c2a6ca2a3fda831fa3032f28535c"): struct{}{}, - ParseID("454515bca5f4f60349a527bd814cc2681bc3625716460cc6310771c966d8a3bf"): struct{}{}, - ParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"): struct{}{}, - }, - backend.IDSet{ - ParseID("19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068"): struct{}{}, - ParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"): struct{}{}, - }, - }, -} - -var findBlobTests = []struct { - packIDs backend.IDSet - blobIDs backend.IDSet -}{ - { - backend.IDSet{ - ParseID("60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"): struct{}{}, - }, - backend.IDSet{ - ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{}, - ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{}, - ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{}, - ParseID("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"): struct{}{}, - ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{}, - }, - }, - { - backend.IDSet{ - ParseID("60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"): struct{}{}, - ParseID("ff7e12cd66d896b08490e787d1915c641e678d7e6b4a00e60db5d13054f4def4"): struct{}{}, - }, - backend.IDSet{ - ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{}, - ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{}, - ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{}, - ParseID("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"): struct{}{}, - ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{}, - ParseID("aa79d596dbd4c863e5400deaca869830888fe1ce9f51b4a983f532c77f16a596"): struct{}{}, - ParseID("b2396c92781307111accf2ebb1cd62b58134b744d90cb6f153ca456a98dc3e76"): struct{}{}, - ParseID("5249af22d3b2acd6da8048ac37b2a87fa346fabde55ed23bb866f7618843c9fe"): struct{}{}, - ParseID("f41c2089a9d58a4b0bf39369fa37588e6578c928aea8e90a4490a6315b9905c1"): struct{}{}, - }, - }, -} - -func TestRepackerFindPacks(t *testing.T) { - WithTestEnvironment(t, checkerTestData, func(repodir string) { - repo := OpenLocalRepo(t, repodir) - - OK(t, repo.LoadIndex()) - - for _, test := range findPackTests { - packIDs, err := checker.FindPacksForBlobs(repo, test.blobIDs) - OK(t, err) - Equals(t, test.packIDs, packIDs) - } - - for _, test := range findBlobTests { - blobs, err := checker.FindBlobsForPacks(repo, test.packIDs) - OK(t, err) - - Assert(t, test.blobIDs.Equals(blobs), - "list of blobs for packs %v does not match, expected:\n %v\ngot:\n %v", - test.packIDs, test.blobIDs, blobs) - } - }) -} - -func TestRepacker(t *testing.T) { - WithTestEnvironment(t, checkerTestData, func(repodir string) { - repo := OpenLocalRepo(t, repodir) - OK(t, repo.LoadIndex()) - - repo.Backend().Remove(backend.Snapshot, "c2b53c5e6a16db92fbb9aa08bd2794c58b379d8724d661ee30d20898bdfdff22") - - unusedBlobs := backend.IDSet{ - ParseID("5714f7274a8aa69b1692916739dc3835d09aac5395946b8ec4f58e563947199a"): struct{}{}, - ParseID("08d0444e9987fa6e35ce4232b2b71473e1a8f66b2f9664cc44dc57aad3c5a63a"): struct{}{}, - ParseID("356493f0b00a614d36c698591bbb2b1d801932d85328c1f508019550034549fc"): struct{}{}, - ParseID("b8a6bcdddef5c0f542b4648b2ef79bc0ed4377d4109755d2fb78aff11e042663"): struct{}{}, - } - - chkr := checker.New(repo) - _, errs := chkr.LoadIndex() - OKs(t, errs) - - errs = checkStruct(chkr) - OKs(t, errs) - - list := backend.NewIDSet(chkr.UnusedBlobs()...) - if !unusedBlobs.Equals(list) { - t.Fatalf("expected unused blobs:\n %v\ngot:\n %v", unusedBlobs, list) - } - - repacker := checker.NewRepacker(repo, unusedBlobs) - OK(t, repacker.Repack()) - - chkr = checker.New(repo) - _, errs = chkr.LoadIndex() - OKs(t, errs) - OKs(t, checkPacks(chkr)) - OKs(t, checkStruct(chkr)) - - blobs := chkr.UnusedBlobs() - Assert(t, len(blobs) == 0, - "expected zero unused blobs, got %v", blobs) - }) -} diff --git a/src/restic/checker/testing.go b/src/restic/checker/testing.go new file mode 100644 index 000000000..3bf9aa2ec --- /dev/null +++ b/src/restic/checker/testing.go @@ -0,0 +1,53 @@ +package checker + +import ( + "restic/repository" + "testing" +) + +// TestCheckRepo runs the checker on repo. +func TestCheckRepo(t testing.TB, repo *repository.Repository) { + chkr := New(repo) + + hints, errs := chkr.LoadIndex() + if len(errs) != 0 { + t.Fatalf("errors loading index: %v", errs) + } + + if len(hints) != 0 { + t.Fatalf("errors loading index: %v", hints) + } + + done := make(chan struct{}) + defer close(done) + + // packs + errChan := make(chan error) + go chkr.Packs(errChan, done) + + for err := range errChan { + t.Error(err) + } + + // structure + errChan = make(chan error) + go chkr.Structure(errChan, done) + + for err := range errChan { + t.Error(err) + } + + // unused blobs + blobs := chkr.UnusedBlobs() + if len(blobs) > 0 { + t.Errorf("unused blobs found: %v", blobs) + } + + // read data + errChan = make(chan error) + go chkr.ReadData(nil, errChan, done) + + for err := range errChan { + t.Error(err) + } +} diff --git a/src/restic/find.go b/src/restic/find.go new file mode 100644 index 000000000..63c8bd813 --- /dev/null +++ b/src/restic/find.go @@ -0,0 +1,43 @@ +package restic + +import ( + "restic/backend" + "restic/pack" + "restic/repository" +) + +// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data +// blobs) to the set blobs. The tree blobs in the `seen` BlobSet will not be visited +// again. +func FindUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs pack.BlobSet, seen pack.BlobSet) error { + blobs.Insert(pack.Handle{ID: treeID, Type: pack.Tree}) + + tree, err := LoadTree(repo, treeID) + if err != nil { + return err + } + + for _, node := range tree.Nodes { + switch node.Type { + case "file": + for _, blob := range node.Content { + blobs.Insert(pack.Handle{ID: blob, Type: pack.Data}) + } + case "dir": + subtreeID := *node.Subtree + h := pack.Handle{ID: subtreeID, Type: pack.Tree} + if seen.Has(h) { + continue + } + + seen.Insert(h) + + err := FindUsedBlobs(repo, subtreeID, blobs, seen) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/src/restic/find_test.go b/src/restic/find_test.go new file mode 100644 index 000000000..f7e47bde4 --- /dev/null +++ b/src/restic/find_test.go @@ -0,0 +1,138 @@ +package restic + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "restic/pack" + "restic/repository" +) + +func loadIDSet(t testing.TB, filename string) pack.BlobSet { + f, err := os.Open(filename) + if err != nil { + t.Logf("unable to open golden file %v: %v", filename, err) + return pack.NewBlobSet() + } + + sc := bufio.NewScanner(f) + + blobs := pack.NewBlobSet() + for sc.Scan() { + var h pack.Handle + err := json.Unmarshal([]byte(sc.Text()), &h) + if err != nil { + t.Errorf("file %v contained invalid blob: %#v", filename, err) + continue + } + + blobs.Insert(h) + } + + if err = f.Close(); err != nil { + t.Errorf("closing file %v failed with error %v", filename, err) + } + + return blobs +} + +func saveIDSet(t testing.TB, filename string, s pack.BlobSet) { + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + t.Fatalf("unable to update golden file %v: %v", filename, err) + return + } + + var hs pack.Handles + for h := range s { + hs = append(hs, h) + } + + sort.Sort(hs) + + enc := json.NewEncoder(f) + for _, h := range hs { + err = enc.Encode(h) + if err != nil { + t.Fatalf("Encode() returned error: %v", err) + } + } + + if err = f.Close(); err != nil { + t.Fatalf("close file %v returned error: %v", filename, err) + } +} + +var updateGoldenFiles = flag.Bool("update", false, "update golden files in testdata/") + +const ( + findTestSnapshots = 3 + findTestDepth = 2 +) + +var findTestTime = time.Unix(1469960361, 23) + +func TestFindUsedBlobs(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + var snapshots []*Snapshot + for i := 0; i < findTestSnapshots; i++ { + sn := TestCreateSnapshot(t, repo, findTestTime.Add(time.Duration(i)*time.Second), findTestDepth, 0) + t.Logf("snapshot %v saved, tree %v", sn.ID().Str(), sn.Tree.Str()) + snapshots = append(snapshots, sn) + } + + for i, sn := range snapshots { + usedBlobs := pack.NewBlobSet() + err := FindUsedBlobs(repo, *sn.Tree, usedBlobs, pack.NewBlobSet()) + if err != nil { + t.Errorf("FindUsedBlobs returned error: %v", err) + continue + } + + if len(usedBlobs) == 0 { + t.Errorf("FindUsedBlobs returned an empty set") + continue + } + + goldenFilename := filepath.Join("testdata", fmt.Sprintf("used_blobs_snapshot%d", i)) + want := loadIDSet(t, goldenFilename) + + if !want.Equals(usedBlobs) { + t.Errorf("snapshot %d: wrong list of blobs returned:\n missing blobs: %v\n extra blobs: %v", + i, want.Sub(usedBlobs), usedBlobs.Sub(want)) + } + + if *updateGoldenFiles { + saveIDSet(t, goldenFilename, usedBlobs) + } + } +} + +func BenchmarkFindUsedBlobs(b *testing.B) { + repo, cleanup := repository.TestRepository(b) + defer cleanup() + + sn := TestCreateSnapshot(b, repo, findTestTime, findTestDepth, 0) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + seen := pack.NewBlobSet() + blobs := pack.NewBlobSet() + err := FindUsedBlobs(repo, *sn.Tree, blobs, seen) + if err != nil { + b.Error(err) + } + + b.Logf("found %v blobs", len(blobs)) + } +} diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go index 8a4095524..ef6766d9e 100644 --- a/src/restic/fuse/file.go +++ b/src/restic/fuse/file.go @@ -27,8 +27,8 @@ var _ = fs.HandleReleaser(&file{}) // BlobLoader is an abstracted repository with a reduced set of methods used // for fuse operations. type BlobLoader interface { - LookupBlobSize(backend.ID) (uint, error) - LoadBlob(pack.BlobType, backend.ID, []byte) ([]byte, error) + LookupBlobSize(backend.ID, pack.BlobType) (uint, error) + LoadBlob(backend.ID, pack.BlobType, []byte) ([]byte, error) } type file struct { @@ -53,7 +53,7 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error var bytes uint64 sizes := make([]uint, len(node.Content)) for i, id := range node.Content { - size, err := repo.LookupBlobSize(id) + size, err := repo.LookupBlobSize(id, pack.Data) if err != nil { return nil, err } @@ -110,7 +110,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) { buf = make([]byte, f.sizes[i]) } - blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf) + blob, err = f.repo.LoadBlob(f.node.Content[i], pack.Data, buf) if err != nil { debug.Log("file.getBlobAt", "LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err) return nil, err diff --git a/src/restic/fuse/file_test.go b/src/restic/fuse/file_test.go index b334f0bc6..12bcb8598 100644 --- a/src/restic/fuse/file_test.go +++ b/src/restic/fuse/file_test.go @@ -26,7 +26,7 @@ func NewMockRepo(content map[backend.ID][]byte) *MockRepo { return &MockRepo{blobs: content} } -func (m *MockRepo) LookupBlobSize(id backend.ID) (uint, error) { +func (m *MockRepo) LookupBlobSize(id backend.ID, t pack.BlobType) (uint, error) { buf, ok := m.blobs[id] if !ok { return 0, errors.New("blob not found") @@ -35,8 +35,8 @@ func (m *MockRepo) LookupBlobSize(id backend.ID) (uint, error) { return uint(len(buf)), nil } -func (m *MockRepo) LoadBlob(t pack.BlobType, id backend.ID, buf []byte) ([]byte, error) { - size, err := m.LookupBlobSize(id) +func (m *MockRepo) LoadBlob(id backend.ID, t pack.BlobType, buf []byte) ([]byte, error) { + size, err := m.LookupBlobSize(id, t) if err != nil { return nil, err } diff --git a/src/restic/index/index.go b/src/restic/index/index.go new file mode 100644 index 000000000..0d2d70397 --- /dev/null +++ b/src/restic/index/index.go @@ -0,0 +1,338 @@ +// Package index contains various data structures for indexing content in a repository or backend. +package index + +import ( + "errors" + "fmt" + "os" + "restic" + "restic/backend" + "restic/debug" + "restic/list" + "restic/pack" + "restic/types" + "restic/worker" +) + +// Pack contains information about the contents of a pack. +type Pack struct { + Size int64 + Entries []pack.Blob +} + +// Blob contains information about a blob. +type Blob struct { + Size int64 + Packs backend.IDSet +} + +// Index contains information about blobs and packs stored in a repo. +type Index struct { + Packs map[backend.ID]Pack + Blobs map[pack.Handle]Blob + IndexIDs backend.IDSet +} + +func newIndex() *Index { + return &Index{ + Packs: make(map[backend.ID]Pack), + Blobs: make(map[pack.Handle]Blob), + IndexIDs: backend.NewIDSet(), + } +} + +// New creates a new index for repo from scratch. +func New(repo types.Repository, p *restic.Progress) (*Index, error) { + done := make(chan struct{}) + defer close(done) + + p.Start() + defer p.Done() + + ch := make(chan worker.Job) + go list.AllPacks(repo, ch, done) + + idx := newIndex() + + for job := range ch { + p.Report(restic.Stat{Blobs: 1}) + + packID := job.Data.(backend.ID) + if job.Error != nil { + fmt.Fprintf(os.Stderr, "unable to list pack %v: %v\n", packID.Str(), job.Error) + continue + } + + j := job.Result.(list.Result) + + debug.Log("Index.New", "pack %v contains %d blobs", packID.Str(), len(j.Entries())) + + err := idx.AddPack(packID, j.Size(), j.Entries()) + if err != nil { + return nil, err + } + + p := Pack{Entries: j.Entries(), Size: j.Size()} + idx.Packs[packID] = p + } + + return idx, nil +} + +const loadIndexParallelism = 20 + +type packJSON struct { + ID backend.ID `json:"id"` + Blobs []blobJSON `json:"blobs"` +} + +type blobJSON struct { + ID backend.ID `json:"id"` + Type pack.BlobType `json:"type"` + Offset uint `json:"offset"` + Length uint `json:"length"` +} + +type indexJSON struct { + Supersedes backend.IDs `json:"supersedes,omitempty"` + Packs []*packJSON `json:"packs"` +} + +func loadIndexJSON(repo types.Repository, id backend.ID) (*indexJSON, error) { + debug.Log("index.loadIndexJSON", "process index %v\n", id.Str()) + + var idx indexJSON + err := repo.LoadJSONUnpacked(backend.Index, id, &idx) + if err != nil { + return nil, err + } + + return &idx, nil +} + +// Load creates an index by loading all index files from the repo. +func Load(repo types.Repository, p *restic.Progress) (*Index, error) { + debug.Log("index.Load", "loading indexes") + + p.Start() + defer p.Done() + + done := make(chan struct{}) + defer close(done) + + supersedes := make(map[backend.ID]backend.IDSet) + results := make(map[backend.ID]map[backend.ID]Pack) + + index := newIndex() + + for id := range repo.List(backend.Index, done) { + p.Report(restic.Stat{Blobs: 1}) + + debug.Log("index.Load", "Load index %v", id.Str()) + idx, err := loadIndexJSON(repo, id) + if err != nil { + return nil, err + } + + res := make(map[backend.ID]Pack) + supersedes[id] = backend.NewIDSet() + for _, sid := range idx.Supersedes { + debug.Log("index.Load", " index %v supersedes %v", id.Str(), sid) + supersedes[id].Insert(sid) + } + + for _, jpack := range idx.Packs { + entries := make([]pack.Blob, 0, len(jpack.Blobs)) + for _, blob := range jpack.Blobs { + entry := pack.Blob{ + ID: blob.ID, + Type: blob.Type, + Offset: blob.Offset, + Length: blob.Length, + } + entries = append(entries, entry) + } + + if err = index.AddPack(jpack.ID, 0, entries); err != nil { + return nil, err + } + } + + results[id] = res + index.IndexIDs.Insert(id) + } + + for superID, list := range supersedes { + for indexID := range list { + if _, ok := results[indexID]; !ok { + continue + } + debug.Log("index.Load", " removing index %v, superseded by %v", indexID.Str(), superID.Str()) + fmt.Fprintf(os.Stderr, "index %v can be removed, superseded by index %v\n", indexID.Str(), superID.Str()) + delete(results, indexID) + } + } + + return index, nil +} + +// AddPack adds a pack to the index. If this pack is already in the index, an +// error is returned. +func (idx *Index) AddPack(id backend.ID, size int64, entries []pack.Blob) error { + if _, ok := idx.Packs[id]; ok { + return fmt.Errorf("pack %v already present in the index", id.Str()) + } + + idx.Packs[id] = Pack{Size: size, Entries: entries} + + for _, entry := range entries { + h := pack.Handle{ID: entry.ID, Type: entry.Type} + if _, ok := idx.Blobs[h]; !ok { + idx.Blobs[h] = Blob{ + Size: int64(entry.Length), + Packs: backend.NewIDSet(), + } + } + + idx.Blobs[h].Packs.Insert(id) + } + + return nil +} + +// RemovePack deletes a pack from the index. +func (idx *Index) RemovePack(id backend.ID) error { + if _, ok := idx.Packs[id]; !ok { + return fmt.Errorf("pack %v not found in the index", id.Str()) + } + + for _, blob := range idx.Packs[id].Entries { + h := pack.Handle{ID: blob.ID, Type: blob.Type} + idx.Blobs[h].Packs.Delete(id) + + if len(idx.Blobs[h].Packs) == 0 { + delete(idx.Blobs, h) + } + } + + delete(idx.Packs, id) + + return nil +} + +// DuplicateBlobs returns a list of blobs that are stored more than once in the +// repo. +func (idx *Index) DuplicateBlobs() (dups pack.BlobSet) { + dups = pack.NewBlobSet() + seen := pack.NewBlobSet() + + for _, p := range idx.Packs { + for _, entry := range p.Entries { + h := pack.Handle{ID: entry.ID, Type: entry.Type} + if seen.Has(h) { + dups.Insert(h) + } + seen.Insert(h) + } + } + + return dups +} + +// PacksForBlobs returns the set of packs in which the blobs are contained. +func (idx *Index) PacksForBlobs(blobs pack.BlobSet) (packs backend.IDSet) { + packs = backend.NewIDSet() + + for h := range blobs { + blob, ok := idx.Blobs[h] + if !ok { + continue + } + + for id := range blob.Packs { + packs.Insert(id) + } + } + + return packs +} + +// Location describes the location of a blob in a pack. +type Location struct { + PackID backend.ID + pack.Blob +} + +// ErrBlobNotFound is return by FindBlob when the blob could not be found in +// the index. +var ErrBlobNotFound = errors.New("blob not found in index") + +// FindBlob returns a list of packs and positions the blob can be found in. +func (idx *Index) FindBlob(h pack.Handle) ([]Location, error) { + blob, ok := idx.Blobs[h] + if !ok { + return nil, ErrBlobNotFound + } + + result := make([]Location, 0, len(blob.Packs)) + for packID := range blob.Packs { + pack, ok := idx.Packs[packID] + if !ok { + return nil, fmt.Errorf("pack %v not found in index", packID.Str()) + } + + for _, entry := range pack.Entries { + if entry.Type != h.Type { + continue + } + + if !entry.ID.Equal(h.ID) { + continue + } + + loc := Location{PackID: packID, Blob: entry} + result = append(result, loc) + } + } + + return result, nil +} + +// Save writes the complete index to the repo. +func (idx *Index) Save(repo types.Repository, supersedes backend.IDs) (backend.ID, error) { + packs := make(map[backend.ID][]pack.Blob, len(idx.Packs)) + for id, p := range idx.Packs { + packs[id] = p.Entries + } + + return Save(repo, packs, supersedes) +} + +// Save writes a new index containing the given packs. +func Save(repo types.Repository, packs map[backend.ID][]pack.Blob, supersedes backend.IDs) (backend.ID, error) { + idx := &indexJSON{ + Supersedes: supersedes, + Packs: make([]*packJSON, 0, len(packs)), + } + + for packID, blobs := range packs { + b := make([]blobJSON, 0, len(blobs)) + for _, blob := range blobs { + b = append(b, blobJSON{ + ID: blob.ID, + Type: blob.Type, + Offset: blob.Offset, + Length: blob.Length, + }) + } + + p := &packJSON{ + ID: packID, + Blobs: b, + } + + idx.Packs = append(idx.Packs, p) + } + + return repo.SaveJSONUnpacked(backend.Index, idx) +} diff --git a/src/restic/index/index_test.go b/src/restic/index/index_test.go new file mode 100644 index 000000000..c950608ab --- /dev/null +++ b/src/restic/index/index_test.go @@ -0,0 +1,352 @@ +package index + +import ( + "math/rand" + "restic" + "restic/backend" + "restic/backend/local" + "restic/pack" + "restic/repository" + . "restic/test" + "testing" + "time" +) + +var ( + snapshotTime = time.Unix(1470492820, 207401672) + snapshots = 3 + depth = 3 +) + +func createFilledRepo(t testing.TB, snapshots int, dup float32) (*repository.Repository, func()) { + repo, cleanup := repository.TestRepository(t) + + for i := 0; i < 3; i++ { + restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup) + } + + return repo, cleanup +} + +func validateIndex(t testing.TB, repo *repository.Repository, idx *Index) { + for id := range repo.List(backend.Data, nil) { + if _, ok := idx.Packs[id]; !ok { + t.Errorf("pack %v missing from index", id.Str()) + } + } +} + +func TestIndexNew(t *testing.T) { + repo, cleanup := createFilledRepo(t, 3, 0) + defer cleanup() + + idx, err := New(repo, nil) + if err != nil { + t.Fatalf("New() returned error %v", err) + } + + if idx == nil { + t.Fatalf("New() returned nil index") + } + + validateIndex(t, repo, idx) +} + +func TestIndexLoad(t *testing.T) { + repo, cleanup := createFilledRepo(t, 3, 0) + defer cleanup() + + loadIdx, err := Load(repo, nil) + if err != nil { + t.Fatalf("Load() returned error %v", err) + } + + if loadIdx == nil { + t.Fatalf("Load() returned nil index") + } + + validateIndex(t, repo, loadIdx) + + newIdx, err := New(repo, nil) + if err != nil { + t.Fatalf("New() returned error %v", err) + } + + if len(loadIdx.Packs) != len(newIdx.Packs) { + t.Errorf("number of packs does not match: want %v, got %v", + len(loadIdx.Packs), len(newIdx.Packs)) + } + + validateIndex(t, repo, newIdx) + + for packID, packNew := range newIdx.Packs { + packLoad, ok := loadIdx.Packs[packID] + + if !ok { + t.Errorf("loaded index does not list pack %v", packID.Str()) + continue + } + + if len(packNew.Entries) != len(packLoad.Entries) { + t.Errorf(" number of entries in pack %v does not match: %d != %d\n %v\n %v", + packID.Str(), len(packNew.Entries), len(packLoad.Entries), + packNew.Entries, packLoad.Entries) + continue + } + + for _, entryNew := range packNew.Entries { + found := false + for _, entryLoad := range packLoad.Entries { + if !entryLoad.ID.Equal(entryNew.ID) { + continue + } + + if entryLoad.Type != entryNew.Type { + continue + } + + if entryLoad.Offset != entryNew.Offset { + continue + } + + if entryLoad.Length != entryNew.Length { + continue + } + + found = true + break + } + + if !found { + t.Errorf("blob not found in loaded index: %v", entryNew) + } + } + } +} + +func openRepo(t testing.TB, dir, password string) *repository.Repository { + b, err := local.Open(dir) + if err != nil { + t.Fatalf("open backend %v failed: %v", dir, err) + } + + r := repository.New(b) + err = r.SearchKey(password) + if err != nil { + t.Fatalf("unable to open repo with password: %v", err) + } + + return r +} + +func BenchmarkIndexNew(b *testing.B) { + repo, cleanup := createFilledRepo(b, 3, 0) + defer cleanup() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + idx, err := New(repo, nil) + + if err != nil { + b.Fatalf("New() returned error %v", err) + } + + if idx == nil { + b.Fatalf("New() returned nil index") + } + } +} + +func TestIndexDuplicateBlobs(t *testing.T) { + repo, cleanup := createFilledRepo(t, 3, 0.01) + defer cleanup() + + idx, err := New(repo, nil) + if err != nil { + t.Fatal(err) + } + + dups := idx.DuplicateBlobs() + if len(dups) == 0 { + t.Errorf("no duplicate blobs found") + } + t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs)) + + packs := idx.PacksForBlobs(dups) + if len(packs) == 0 { + t.Errorf("no packs with duplicate blobs found") + } + t.Logf("%d packs with duplicate blobs", len(packs)) +} + +func loadIndex(t testing.TB, repo *repository.Repository) *Index { + idx, err := Load(repo, nil) + if err != nil { + t.Fatalf("Load() returned error %v", err) + } + + return idx +} + +func TestIndexSave(t *testing.T) { + repo, cleanup := createFilledRepo(t, 3, 0) + defer cleanup() + + idx := loadIndex(t, repo) + + packs := make(map[backend.ID][]pack.Blob) + for id := range idx.Packs { + if rand.Float32() < 0.5 { + packs[id] = idx.Packs[id].Entries + } + } + + t.Logf("save %d/%d packs in a new index\n", len(packs), len(idx.Packs)) + + id, err := Save(repo, packs, idx.IndexIDs.List()) + if err != nil { + t.Fatalf("unable to save new index: %v", err) + } + + t.Logf("new index saved as %v", id.Str()) + + for id := range idx.IndexIDs { + t.Logf("remove index %v", id.Str()) + err = repo.Backend().Remove(backend.Index, id.String()) + if err != nil { + t.Errorf("error removing index %v: %v", id, err) + } + } + + idx2 := loadIndex(t, repo) + t.Logf("load new index with %d packs", len(idx2.Packs)) + + if len(idx2.Packs) != len(packs) { + t.Errorf("wrong number of packs in new index, want %d, got %d", len(packs), len(idx2.Packs)) + } + + for id := range packs { + if _, ok := idx2.Packs[id]; !ok { + t.Errorf("pack %v is not contained in new index", id.Str()) + } + } + + for id := range idx2.Packs { + if _, ok := packs[id]; !ok { + t.Errorf("pack %v is not contained in new index", id.Str()) + } + } +} + +func TestIndexAddRemovePack(t *testing.T) { + repo, cleanup := createFilledRepo(t, 3, 0) + defer cleanup() + + idx, err := Load(repo, nil) + if err != nil { + t.Fatalf("Load() returned error %v", err) + } + + done := make(chan struct{}) + defer close(done) + + packID := <-repo.List(backend.Data, done) + + t.Logf("selected pack %v", packID.Str()) + + blobs := idx.Packs[packID].Entries + + idx.RemovePack(packID) + + if _, ok := idx.Packs[packID]; ok { + t.Errorf("removed pack %v found in index.Packs", packID.Str()) + } + + for _, blob := range blobs { + h := pack.Handle{ID: blob.ID, Type: blob.Type} + _, err := idx.FindBlob(h) + if err == nil { + t.Errorf("removed blob %v found in index", h) + } + + if _, ok := idx.Blobs[h]; ok { + t.Errorf("removed blob %v found in index.Blobs", h) + } + } + +} + +// example index serialization from doc/Design.md +var docExample = []byte(` +{ + "supersedes": [ + "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" + ], + "packs": [ + { + "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", + "blobs": [ + { + "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", + "type": "data", + "offset": 0, + "length": 25 + },{ + "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", + "type": "tree", + "offset": 38, + "length": 100 + }, + { + "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", + "type": "data", + "offset": 150, + "length": 123 + } + ] + } + ] +} +`) + +func TestIndexLoadDocReference(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + id, err := repo.SaveUnpacked(backend.Index, docExample) + if err != nil { + t.Fatalf("SaveUnpacked() returned error %v", err) + } + + t.Logf("index saved as %v", id.Str()) + + idx := loadIndex(t, repo) + + blobID := ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66") + locs, err := idx.FindBlob(pack.Handle{ID: blobID, Type: pack.Data}) + if err != nil { + t.Errorf("FindBlob() returned error %v", err) + } + + if len(locs) != 1 { + t.Errorf("blob found %d times, expected just one", len(locs)) + } + + l := locs[0] + if !l.ID.Equal(blobID) { + t.Errorf("blob IDs are not equal: %v != %v", l.ID, blobID) + } + + if l.Type != pack.Data { + t.Errorf("want type %v, got %v", pack.Data, l.Type) + } + + if l.Offset != 150 { + t.Errorf("wrong offset, want %d, got %v", 150, l.Offset) + } + + if l.Length != 123 { + t.Errorf("wrong length, want %d, got %v", 123, l.Length) + } +} diff --git a/src/restic/list/list.go b/src/restic/list/list.go new file mode 100644 index 000000000..e3a14798f --- /dev/null +++ b/src/restic/list/list.go @@ -0,0 +1,67 @@ +package list + +import ( + "restic/backend" + "restic/pack" + "restic/worker" +) + +const listPackWorkers = 10 + +// Lister combines lists packs in a repo and blobs in a pack. +type Lister interface { + List(backend.Type, <-chan struct{}) <-chan backend.ID + ListPack(backend.ID) ([]pack.Blob, int64, error) +} + +// Result is returned in the channel from LoadBlobsFromAllPacks. +type Result struct { + packID backend.ID + size int64 + entries []pack.Blob +} + +// PackID returns the pack ID of this result. +func (l Result) PackID() backend.ID { + return l.packID +} + +// Size ruturns the size of the pack. +func (l Result) Size() int64 { + return l.size +} + +// Entries returns a list of all blobs saved in the pack. +func (l Result) Entries() []pack.Blob { + return l.entries +} + +// AllPacks sends the contents of all packs to ch. +func AllPacks(repo Lister, ch chan<- worker.Job, done <-chan struct{}) { + f := func(job worker.Job, done <-chan struct{}) (interface{}, error) { + packID := job.Data.(backend.ID) + entries, size, err := repo.ListPack(packID) + + return Result{ + packID: packID, + size: size, + entries: entries, + }, err + } + + jobCh := make(chan worker.Job) + wp := worker.New(listPackWorkers, f, jobCh, ch) + + go func() { + defer close(jobCh) + for id := range repo.List(backend.Data, done) { + select { + case jobCh <- worker.Job{Data: id}: + case <-done: + return + } + } + }() + + wp.Wait() +} diff --git a/src/restic/lock.go b/src/restic/lock.go index ed1593444..5628e7577 100644 --- a/src/restic/lock.go +++ b/src/restic/lock.go @@ -188,7 +188,7 @@ var staleTimeout = 30 * time.Minute // older than 30 minutes or if it was created on the current machine and the // process isn't alive any more. func (l *Lock) Stale() bool { - debug.Log("Lock.Stale", "testing if lock %v for process %d is stale", l.lockID.Str(), l.PID) + debug.Log("Lock.Stale", "testing if lock %v for process %d is stale", l, l.PID) if time.Now().Sub(l.Time) > staleTimeout { debug.Log("Lock.Stale", "lock is stale, timestamp is too old: %v\n", l.Time) return true diff --git a/src/restic/node.go b/src/restic/node.go index 21c447d1a..a2f64dacd 100644 --- a/src/restic/node.go +++ b/src/restic/node.go @@ -12,12 +12,13 @@ import ( "runtime" - "github.com/juju/errors" "restic/backend" "restic/debug" "restic/fs" "restic/pack" "restic/repository" + + "github.com/juju/errors" ) // Node is a file, directory or other item in a backup. @@ -215,17 +216,17 @@ func (node Node) createFileAt(path string, repo *repository.Repository) error { var buf []byte for _, id := range node.Content { - blob, err := repo.Index().Lookup(id) + size, err := repo.LookupBlobSize(id, pack.Data) if err != nil { return err } buf = buf[:cap(buf)] - if uint(len(buf)) < blob.Length { - buf = make([]byte, blob.Length) + if uint(len(buf)) < size { + buf = make([]byte, size) } - buf, err := repo.LoadBlob(pack.Data, id, buf) + buf, err := repo.LoadBlob(id, pack.Data, buf) if err != nil { return errors.Annotate(err, "Load") } diff --git a/src/restic/pack/blob_set.go b/src/restic/pack/blob_set.go new file mode 100644 index 000000000..686ea9315 --- /dev/null +++ b/src/restic/pack/blob_set.go @@ -0,0 +1,109 @@ +package pack + +import "sort" + +// BlobSet is a set of blobs. +type BlobSet map[Handle]struct{} + +// NewBlobSet returns a new BlobSet, populated with ids. +func NewBlobSet(handles ...Handle) BlobSet { + m := make(BlobSet) + for _, h := range handles { + m[h] = struct{}{} + } + + return m +} + +// Has returns true iff id is contained in the set. +func (s BlobSet) Has(h Handle) bool { + _, ok := s[h] + return ok +} + +// Insert adds id to the set. +func (s BlobSet) Insert(h Handle) { + s[h] = struct{}{} +} + +// Delete removes id from the set. +func (s BlobSet) Delete(h Handle) { + delete(s, h) +} + +// Equals returns true iff s equals other. +func (s BlobSet) Equals(other BlobSet) bool { + if len(s) != len(other) { + return false + } + + for h := range s { + if _, ok := other[h]; !ok { + return false + } + } + + return true +} + +// Merge adds the blobs in other to the current set. +func (s BlobSet) Merge(other BlobSet) { + for h := range other { + s.Insert(h) + } +} + +// Intersect returns a new set containing the handles that are present in both sets. +func (s BlobSet) Intersect(other BlobSet) (result BlobSet) { + result = NewBlobSet() + + set1 := s + set2 := other + + // iterate over the smaller set + if len(set2) < len(set1) { + set1, set2 = set2, set1 + } + + for h := range set1 { + if set2.Has(h) { + result.Insert(h) + } + } + + return result +} + +// Sub returns a new set containing all handles that are present in s but not in +// other. +func (s BlobSet) Sub(other BlobSet) (result BlobSet) { + result = NewBlobSet() + for h := range s { + if !other.Has(h) { + result.Insert(h) + } + } + + return result +} + +// List returns a slice of all Handles in the set. +func (s BlobSet) List() Handles { + list := make(Handles, 0, len(s)) + for h := range s { + list = append(list, h) + } + + sort.Sort(list) + + return list +} + +func (s BlobSet) String() string { + str := s.List().String() + if len(str) < 2 { + return "{}" + } + + return "{" + str[1:len(str)-1] + "}" +} diff --git a/src/restic/pack/handle.go b/src/restic/pack/handle.go new file mode 100644 index 000000000..9a0ce58f6 --- /dev/null +++ b/src/restic/pack/handle.go @@ -0,0 +1,51 @@ +package pack + +import ( + "fmt" + "restic/backend" +) + +// Handle identifies a blob of a given type. +type Handle struct { + ID backend.ID + Type BlobType +} + +func (h Handle) String() string { + return fmt.Sprintf("<%s/%s>", h.Type, h.ID.Str()) +} + +// Handles is an ordered list of Handles that implements sort.Interface. +type Handles []Handle + +func (h Handles) Len() int { + return len(h) +} + +func (h Handles) Less(i, j int) bool { + for k, b := range h[i].ID { + if b == h[j].ID[k] { + continue + } + + if b < h[j].ID[k] { + return true + } + + return false + } + + return h[i].Type < h[j].Type +} + +func (h Handles) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h Handles) String() string { + elements := make([]string, 0, len(h)) + for _, e := range h { + elements = append(elements, e.String()) + } + return fmt.Sprintf("%v", elements) +} diff --git a/src/restic/pack/loader.go b/src/restic/pack/loader.go new file mode 100644 index 000000000..2b4ada8a7 --- /dev/null +++ b/src/restic/pack/loader.go @@ -0,0 +1,43 @@ +package pack + +import ( + "errors" + "restic/backend" +) + +// Loader loads data from somewhere at a given offset. In contrast to +// io.ReaderAt, off may be negative, in which case it references a position +// relative to the end of the file (similar to Seek()). +type Loader interface { + Load(p []byte, off int64) (int, error) +} + +// BackendLoader creates a Loader from a Backend and a Handle. +type BackendLoader struct { + Backend backend.Backend + Handle backend.Handle +} + +// Load returns data at the given offset. +func (l BackendLoader) Load(p []byte, off int64) (int, error) { + return l.Backend.Load(l.Handle, p, off) +} + +// BufferLoader allows using a buffer as a Loader. +type BufferLoader []byte + +// Load returns data at the given offset. +func (b BufferLoader) Load(p []byte, off int64) (int, error) { + switch { + case off > int64(len(b)): + return 0, errors.New("offset is larger than data") + case off < -int64(len(b)): + off = 0 + case off < 0: + off = int64(len(b)) + off + } + + b = b[off:] + + return copy(p, b), nil +} diff --git a/src/restic/pack/pack.go b/src/restic/pack/pack.go index 931bda869..78159dfa3 100644 --- a/src/restic/pack/pack.go +++ b/src/restic/pack/pack.go @@ -17,8 +17,9 @@ type BlobType uint8 // These are the blob types that can be stored in a pack. const ( - Data BlobType = 0 - Tree = 1 + Invalid BlobType = iota + Data + Tree ) func (t BlobType) String() string { @@ -66,15 +67,9 @@ type Blob struct { Offset uint } -// GetReader returns an io.Reader for the blob entry e. -func (e Blob) GetReader(rd io.ReadSeeker) (io.Reader, error) { - // seek to the correct location - _, err := rd.Seek(int64(e.Offset), 0) - if err != nil { - return nil, err - } - - return io.LimitReader(rd, int64(e.Length)), nil +func (b Blob) String() string { + return fmt.Sprintf("", + b.ID.Str(), b.Type, b.Length, b.Offset) } // Packer is used to create a new Pack. @@ -118,7 +113,7 @@ var entrySize = uint(binary.Size(BlobType(0)) + binary.Size(uint32(0)) + backend // headerEntry is used with encoding/binary to read and write header entries type headerEntry struct { - Type BlobType + Type uint8 Length uint32 ID [backend.IDSize]byte } @@ -176,11 +171,19 @@ func (p *Packer) Finalize() (uint, error) { func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) { for _, b := range p.blobs { entry := headerEntry{ - Type: b.Type, Length: uint32(b.Length), ID: b.ID, } + switch b.Type { + case Data: + entry.Type = 0 + case Tree: + entry.Type = 1 + default: + return 0, fmt.Errorf("invalid blob type %v", b.Type) + } + err := binary.Write(wr, binary.LittleEndian, entry) if err != nil { return bytesWritten, err @@ -232,42 +235,61 @@ type Unpacker struct { k *crypto.Key } +const preloadHeaderSize = 2048 + // NewUnpacker returns a pointer to Unpacker which can be used to read // individual Blobs from a pack. -func NewUnpacker(k *crypto.Key, rd io.ReadSeeker) (*Unpacker, error) { +func NewUnpacker(k *crypto.Key, ldr Loader) (*Unpacker, error) { var err error - ls := binary.Size(uint32(0)) - // reset to the end to read header length - _, err = rd.Seek(-int64(ls), 2) - if err != nil { - return nil, fmt.Errorf("seeking to read header length failed: %v", err) + // read the last 2048 byte, this will mostly be enough for the header, so + // we do not need another round trip. + buf := make([]byte, preloadHeaderSize) + n, err := ldr.Load(buf, -int64(len(buf))) + + if err == io.ErrUnexpectedEOF { + err = nil + buf = buf[:n] } - var length uint32 - err = binary.Read(rd, binary.LittleEndian, &length) if err != nil { - return nil, fmt.Errorf("reading header length failed: %v", err) + return nil, fmt.Errorf("Load at -%d failed: %v", len(buf), err) + } + buf = buf[:n] + + bs := binary.Size(uint32(0)) + p := len(buf) - bs + + // read the length from the end of the buffer + length := int(binary.LittleEndian.Uint32(buf[p : p+bs])) + buf = buf[:p] + + // if the header is longer than the preloaded buffer, call the loader again. + if length > len(buf) { + buf = make([]byte, length) + n, err := ldr.Load(buf, -int64(len(buf)+bs)) + if err != nil { + return nil, fmt.Errorf("Load at -%d failed: %v", len(buf), err) + } + buf = buf[:n] } - // reset to the beginning of the header - _, err = rd.Seek(-int64(ls)-int64(length), 2) - if err != nil { - return nil, fmt.Errorf("seeking to read header length failed: %v", err) - } + buf = buf[len(buf)-length:] // read header - hrd, err := crypto.DecryptFrom(k, io.LimitReader(rd, int64(length))) + hdr, err := crypto.Decrypt(k, buf, buf) if err != nil { return nil, err } + rd := bytes.NewReader(hdr) + var entries []Blob pos := uint(0) for { e := headerEntry{} - err = binary.Read(hrd, binary.LittleEndian, &e) + err = binary.Read(rd, binary.LittleEndian, &e) if err == io.EOF { break } @@ -276,21 +298,31 @@ func NewUnpacker(k *crypto.Key, rd io.ReadSeeker) (*Unpacker, error) { return nil, err } - entries = append(entries, Blob{ - Type: e.Type, + entry := Blob{ Length: uint(e.Length), ID: e.ID, Offset: pos, - }) + } + + switch e.Type { + case 0: + entry.Type = Data + case 1: + entry.Type = Tree + default: + return nil, fmt.Errorf("invalid type %d", e.Type) + } + + entries = append(entries, entry) pos += uint(e.Length) } - p := &Unpacker{ + up := &Unpacker{ rd: rd, k: k, Entries: entries, } - return p, nil + return up, nil } diff --git a/src/restic/pack/pack_test.go b/src/restic/pack/pack_test.go index e987ced7c..e13c9884a 100644 --- a/src/restic/pack/pack_test.go +++ b/src/restic/pack/pack_test.go @@ -7,7 +7,6 @@ import ( "encoding/binary" "encoding/json" "io" - "io/ioutil" "testing" "restic/backend" @@ -17,14 +16,14 @@ import ( . "restic/test" ) -var lengths = []int{23, 31650, 25860, 10928, 13769, 19862, 5211, 127, 13690, 30231} +var testLens = []int{23, 31650, 25860, 10928, 13769, 19862, 5211, 127, 13690, 30231} type Buf struct { data []byte id backend.ID } -func newPack(t testing.TB, k *crypto.Key) ([]Buf, []byte, uint) { +func newPack(t testing.TB, k *crypto.Key, lengths []int) ([]Buf, []byte, uint) { bufs := []Buf{} for _, l := range lengths { @@ -48,15 +47,15 @@ func newPack(t testing.TB, k *crypto.Key) ([]Buf, []byte, uint) { return bufs, packData, p.Size() } -func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReadSeeker, packSize uint) { +func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, ldr pack.Loader, packSize uint) { written := 0 - for _, l := range lengths { - written += l + for _, buf := range bufs { + written += len(buf.data) } // header length written += binary.Size(uint32(0)) // header - written += len(lengths) * (binary.Size(pack.BlobType(0)) + binary.Size(uint32(0)) + backend.IDSize) + written += len(bufs) * (binary.Size(pack.BlobType(0)) + binary.Size(uint32(0)) + backend.IDSize) // header crypto written += crypto.Extension @@ -64,20 +63,24 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReadSeeker, pack Equals(t, uint(written), packSize) // read and parse it again - np, err := pack.NewUnpacker(k, rd) + np, err := pack.NewUnpacker(k, ldr) OK(t, err) Equals(t, len(np.Entries), len(bufs)) + var buf []byte for i, b := range bufs { e := np.Entries[i] Equals(t, b.id, e.ID) - brd, err := e.GetReader(rd) - OK(t, err) - data, err := ioutil.ReadAll(brd) + if len(buf) < int(e.Length) { + buf = make([]byte, int(e.Length)) + } + buf = buf[:int(e.Length)] + n, err := ldr.Load(buf, int64(e.Offset)) OK(t, err) + buf = buf[:n] - Assert(t, bytes.Equal(b.data, data), + Assert(t, bytes.Equal(b.data, buf), "data for blob %v doesn't match", i) } } @@ -86,9 +89,9 @@ func TestCreatePack(t *testing.T) { // create random keys k := crypto.NewRandomKey() - bufs, packData, packSize := newPack(t, k) + bufs, packData, packSize := newPack(t, k, testLens) Equals(t, uint(len(packData)), packSize) - verifyBlobs(t, bufs, k, bytes.NewReader(packData), packSize) + verifyBlobs(t, bufs, k, pack.BufferLoader(packData), packSize) } var blobTypeJSON = []struct { @@ -118,13 +121,27 @@ func TestUnpackReadSeeker(t *testing.T) { // create random keys k := crypto.NewRandomKey() - bufs, packData, packSize := newPack(t, k) + bufs, packData, packSize := newPack(t, k, testLens) b := mem.New() id := backend.Hash(packData) handle := backend.Handle{Type: backend.Data, Name: id.String()} OK(t, b.Save(handle, packData)) - rd := backend.NewReadSeeker(b, handle) - verifyBlobs(t, bufs, k, rd, packSize) + ldr := pack.BackendLoader{Backend: b, Handle: handle} + verifyBlobs(t, bufs, k, ldr, packSize) +} + +func TestShortPack(t *testing.T) { + k := crypto.NewRandomKey() + + bufs, packData, packSize := newPack(t, k, []int{23}) + + b := mem.New() + id := backend.Hash(packData) + + handle := backend.Handle{Type: backend.Data, Name: id.String()} + OK(t, b.Save(handle, packData)) + ldr := pack.BackendLoader{Backend: b, Handle: handle} + verifyBlobs(t, bufs, k, ldr, packSize) } diff --git a/src/restic/progress.go b/src/restic/progress.go index cca1a4e47..4721fac3c 100644 --- a/src/restic/progress.go +++ b/src/restic/progress.go @@ -152,6 +152,7 @@ func (p *Progress) Done() { if p.OnDone != nil { p.fnM.Lock() + p.OnUpdate(cur, time.Since(p.start), false) p.OnDone(cur, time.Since(p.start), false) p.fnM.Unlock() } diff --git a/src/restic/repository/config.go b/src/restic/repository/config.go index a41517f3c..46f739d35 100644 --- a/src/restic/repository/config.go +++ b/src/restic/repository/config.go @@ -6,10 +6,12 @@ import ( "encoding/hex" "errors" "io" + "testing" - "github.com/restic/chunker" "restic/backend" "restic/debug" + + "github.com/restic/chunker" ) // Config contains the configuration for a repository. @@ -37,8 +39,8 @@ type JSONUnpackedLoader interface { } // CreateConfig creates a config file with a randomly selected polynomial and -// ID and saves the config in the repository. -func CreateConfig(r JSONUnpackedSaver) (Config, error) { +// ID. +func CreateConfig() (Config, error) { var ( err error cfg Config @@ -59,9 +61,23 @@ func CreateConfig(r JSONUnpackedSaver) (Config, error) { cfg.Version = RepoVersion debug.Log("Repo.CreateConfig", "New config: %#v", cfg) + return cfg, nil +} - _, err = r.SaveJSONUnpacked(backend.Config, cfg) - return cfg, err +// TestCreateConfig creates a config for use within tests. +func TestCreateConfig(t testing.TB, pol chunker.Pol) (cfg Config) { + cfg.ChunkerPolynomial = pol + + newID := make([]byte, repositoryIDSize) + _, err := io.ReadFull(rand.Reader, newID) + if err != nil { + t.Fatalf("unable to create random ID: %v", err) + } + + cfg.ID = hex.EncodeToString(newID) + cfg.Version = RepoVersion + + return cfg } // LoadConfig returns loads, checks and returns the config for a repository. diff --git a/src/restic/repository/config_test.go b/src/restic/repository/config_test.go index 10fc61d94..71f2fd810 100644 --- a/src/restic/repository/config_test.go +++ b/src/restic/repository/config_test.go @@ -32,9 +32,11 @@ func TestConfig(t *testing.T) { return backend.ID{}, nil } - cfg1, err := repository.CreateConfig(saver(save)) + cfg1, err := repository.CreateConfig() OK(t, err) + _, err = saver(save).SaveJSONUnpacked(backend.Config, cfg1) + load := func(tpe backend.Type, id backend.ID, arg interface{}) error { Assert(t, tpe == backend.Config, "wrong backend type: got %v, wanted %v", diff --git a/src/restic/repository/index.go b/src/restic/repository/index.go index 56534bd9d..86968ba1d 100644 --- a/src/restic/repository/index.go +++ b/src/restic/repository/index.go @@ -18,7 +18,7 @@ import ( // Index holds a lookup table for id -> pack. type Index struct { m sync.Mutex - pack map[backend.ID]indexEntry + pack map[pack.Handle][]indexEntry final bool // set to true for all indexes read from the backend ("finalized") id backend.ID // set to the ID of the index when it's finalized @@ -27,7 +27,6 @@ type Index struct { } type indexEntry struct { - tpe pack.BlobType packID backend.ID offset uint length uint @@ -36,18 +35,19 @@ type indexEntry struct { // NewIndex returns a new index. func NewIndex() *Index { return &Index{ - pack: make(map[backend.ID]indexEntry), + pack: make(map[pack.Handle][]indexEntry), created: time.Now(), } } func (idx *Index) store(blob PackedBlob) { - idx.pack[blob.ID] = indexEntry{ - tpe: blob.Type, + newEntry := indexEntry{ packID: blob.PackID, offset: blob.Offset, length: blob.Length, } + h := pack.Handle{ID: blob.ID, Type: blob.Type} + idx.pack[h] = append(idx.pack[h], newEntry) } // Final returns true iff the index is already written to the repository, it is @@ -110,43 +110,36 @@ func (idx *Index) Store(blob PackedBlob) { idx.store(blob) } -// StoreBlobs saves information about the blobs to the index in one atomic transaction. -func (idx *Index) StoreBlobs(blobs []PackedBlob) { - idx.m.Lock() - defer idx.m.Unlock() - - if idx.final { - panic("store new item in finalized index") - } - - debug.Log("Index.StoreBlobs", "stored %d blobs", len(blobs)) - - for _, blob := range blobs { - idx.store(blob) - } -} - // Lookup queries the index for the blob ID and returns a PackedBlob. -func (idx *Index) Lookup(id backend.ID) (pb PackedBlob, err error) { +func (idx *Index) Lookup(id backend.ID, tpe pack.BlobType) (blobs []PackedBlob, err error) { idx.m.Lock() defer idx.m.Unlock() - if p, ok := idx.pack[id]; ok { - debug.Log("Index.Lookup", "id %v found in pack %v at %d, length %d", - id.Str(), p.packID.Str(), p.offset, p.length) + h := pack.Handle{ID: id, Type: tpe} - pb := PackedBlob{ - Type: p.tpe, - Length: p.length, - ID: id, - Offset: p.offset, - PackID: p.packID, + if packs, ok := idx.pack[h]; ok { + blobs = make([]PackedBlob, 0, len(packs)) + + for _, p := range packs { + debug.Log("Index.Lookup", "id %v found in pack %v at %d, length %d", + id.Str(), p.packID.Str(), p.offset, p.length) + + blob := PackedBlob{ + Type: tpe, + Length: p.length, + ID: id, + Offset: p.offset, + PackID: p.packID, + } + + blobs = append(blobs, blob) } - return pb, nil + + return blobs, nil } debug.Log("Index.Lookup", "id %v not found", id.Str()) - return PackedBlob{}, fmt.Errorf("id %v not found in index", id) + return nil, fmt.Errorf("id %v not found in index", id) } // ListPack returns a list of blobs contained in a pack. @@ -154,15 +147,17 @@ func (idx *Index) ListPack(id backend.ID) (list []PackedBlob) { idx.m.Lock() defer idx.m.Unlock() - for blobID, entry := range idx.pack { - if entry.packID == id { - list = append(list, PackedBlob{ - ID: blobID, - Type: entry.tpe, - Length: entry.length, - Offset: entry.offset, - PackID: entry.packID, - }) + for h, packList := range idx.pack { + for _, entry := range packList { + if entry.packID == id { + list = append(list, PackedBlob{ + ID: h.ID, + Type: h.Type, + Length: entry.length, + Offset: entry.offset, + PackID: entry.packID, + }) + } } } @@ -170,8 +165,8 @@ func (idx *Index) ListPack(id backend.ID) (list []PackedBlob) { } // Has returns true iff the id is listed in the index. -func (idx *Index) Has(id backend.ID) bool { - _, err := idx.Lookup(id) +func (idx *Index) Has(id backend.ID, tpe pack.BlobType) bool { + _, err := idx.Lookup(id, tpe) if err == nil { return true } @@ -181,28 +176,13 @@ func (idx *Index) Has(id backend.ID) bool { // LookupSize returns the length of the cleartext content behind the // given id -func (idx *Index) LookupSize(id backend.ID) (cleartextLength uint, err error) { - blob, err := idx.Lookup(id) +func (idx *Index) LookupSize(id backend.ID, tpe pack.BlobType) (cleartextLength uint, err error) { + blobs, err := idx.Lookup(id, tpe) if err != nil { return 0, err } - return blob.PlaintextLength(), nil -} -// Merge loads all items from other into idx. -func (idx *Index) Merge(other *Index) { - debug.Log("Index.Merge", "Merge index with %p", other) - idx.m.Lock() - defer idx.m.Unlock() - - for k, v := range other.pack { - if _, ok := idx.pack[k]; ok { - debug.Log("Index.Merge", "index already has key %v, updating", k.Str()) - } - - idx.pack[k] = v - } - debug.Log("Index.Merge", "done merging index") + return blobs[0].PlaintextLength(), nil } // Supersedes returns the list of indexes this index supersedes, if any. @@ -257,17 +237,19 @@ func (idx *Index) Each(done chan struct{}) <-chan PackedBlob { close(ch) }() - for id, blob := range idx.pack { - select { - case <-done: - return - case ch <- PackedBlob{ - ID: id, - Offset: blob.offset, - Type: blob.tpe, - Length: blob.length, - PackID: blob.packID, - }: + for h, packs := range idx.pack { + for _, blob := range packs { + select { + case <-done: + return + case ch <- PackedBlob{ + ID: h.ID, + Type: h.Type, + Offset: blob.offset, + Length: blob.length, + PackID: blob.packID, + }: + } } } }() @@ -281,8 +263,10 @@ func (idx *Index) Packs() backend.IDSet { defer idx.m.Unlock() packs := backend.NewIDSet() - for _, entry := range idx.pack { - packs.Insert(entry.packID) + for _, list := range idx.pack { + for _, entry := range list { + packs.Insert(entry.packID) + } } return packs @@ -294,11 +278,12 @@ func (idx *Index) Count(t pack.BlobType) (n uint) { idx.m.Lock() defer idx.m.Unlock() - for id, blob := range idx.pack { - if blob.tpe == t { - n++ - debug.Log("Index.Count", " blob %v counted: %v", id.Str(), blob) + for h, list := range idx.pack { + if h.Type != t { + continue } + + n += uint(len(list)) } return @@ -330,37 +315,39 @@ func (idx *Index) generatePackList() ([]*packJSON, error) { list := []*packJSON{} packs := make(map[backend.ID]*packJSON) - for id, blob := range idx.pack { - if blob.packID.IsNull() { - panic("null pack id") + for h, packedBlobs := range idx.pack { + for _, blob := range packedBlobs { + if blob.packID.IsNull() { + panic("null pack id") + } + + debug.Log("Index.generatePackList", "handle blob %v", h) + + if blob.packID.IsNull() { + debug.Log("Index.generatePackList", "blob %v has no packID! (offset %v, length %v)", + h, blob.offset, blob.length) + return nil, fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", h) + } + + // see if pack is already in map + p, ok := packs[blob.packID] + if !ok { + // else create new pack + p = &packJSON{ID: blob.packID} + + // and append it to the list and map + list = append(list, p) + packs[p.ID] = p + } + + // add blob + p.Blobs = append(p.Blobs, blobJSON{ + ID: h.ID, + Type: h.Type, + Offset: blob.offset, + Length: blob.length, + }) } - - debug.Log("Index.generatePackList", "handle blob %v", id.Str()) - - if blob.packID.IsNull() { - debug.Log("Index.generatePackList", "blob %q has no packID! (type %v, offset %v, length %v)", - id.Str(), blob.tpe, blob.offset, blob.length) - return nil, fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id) - } - - // see if pack is already in map - p, ok := packs[blob.packID] - if !ok { - // else create new pack - p = &packJSON{ID: blob.packID} - - // and append it to the list and map - list = append(list, p) - packs[p.ID] = p - } - - // add blob - p.Blobs = append(p.Blobs, blobJSON{ - ID: id, - Type: blob.tpe, - Offset: blob.offset, - Length: blob.length, - }) } debug.Log("Index.generatePackList", "done") diff --git a/src/restic/repository/index_rebuild.go b/src/restic/repository/index_rebuild.go new file mode 100644 index 000000000..99c281484 --- /dev/null +++ b/src/restic/repository/index_rebuild.go @@ -0,0 +1,68 @@ +package repository + +import ( + "fmt" + "os" + "restic/backend" + "restic/debug" + "restic/list" + "restic/worker" +) + +// RebuildIndex lists all packs in the repo, writes a new index and removes all +// old indexes. This operation should only be done with an exclusive lock in +// place. +func RebuildIndex(repo *Repository) error { + debug.Log("RebuildIndex", "start rebuilding index") + + done := make(chan struct{}) + defer close(done) + + ch := make(chan worker.Job) + go list.AllPacks(repo, ch, done) + + idx := NewIndex() + for job := range ch { + id := job.Data.(backend.ID) + + if job.Error != nil { + fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error) + continue + } + + res := job.Result.(list.Result) + + for _, entry := range res.Entries() { + pb := PackedBlob{ + ID: entry.ID, + Type: entry.Type, + Length: entry.Length, + Offset: entry.Offset, + PackID: res.PackID(), + } + idx.Store(pb) + } + } + + oldIndexes := backend.NewIDSet() + for id := range repo.List(backend.Index, done) { + idx.AddToSupersedes(id) + oldIndexes.Insert(id) + } + + id, err := SaveIndex(repo, idx) + if err != nil { + debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err) + return err + } + debug.Log("RebuildIndex.RebuildIndex", "new index saved as %v", id.Str()) + + for indexID := range oldIndexes { + err := repo.Backend().Remove(backend.Index, indexID.String()) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err) + } + } + + return nil +} diff --git a/src/restic/repository/index_test.go b/src/restic/repository/index_test.go index 0fafc409c..a16c6f2e1 100644 --- a/src/restic/repository/index_test.go +++ b/src/restic/repository/index_test.go @@ -2,8 +2,6 @@ package repository_test import ( "bytes" - "crypto/rand" - "io" "testing" "restic/backend" @@ -12,15 +10,6 @@ import ( . "restic/test" ) -func randomID() backend.ID { - id := backend.ID{} - _, err := io.ReadFull(rand.Reader, id[:]) - if err != nil { - panic(err) - } - return id -} - func TestIndexSerialize(t *testing.T) { type testEntry struct { id backend.ID @@ -34,11 +23,11 @@ func TestIndexSerialize(t *testing.T) { // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { - packID := randomID() + packID := backend.RandomID() pos := uint(0) for j := 0; j < 20; j++ { - id := randomID() + id := backend.RandomID() length := uint(i*100 + j) idx.Store(repository.PackedBlob{ Type: pack.Data, @@ -74,17 +63,27 @@ func TestIndexSerialize(t *testing.T) { OK(t, err) for _, testBlob := range tests { - result, err := idx.Lookup(testBlob.id) + list, err := idx.Lookup(testBlob.id, testBlob.tpe) OK(t, err) + if len(list) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list) + } + result := list[0] + Equals(t, testBlob.pack, result.PackID) Equals(t, testBlob.tpe, result.Type) Equals(t, testBlob.offset, result.Offset) Equals(t, testBlob.length, result.Length) - result2, err := idx2.Lookup(testBlob.id) + list2, err := idx2.Lookup(testBlob.id, testBlob.tpe) OK(t, err) + if len(list2) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list2), list2) + } + result2 := list2[0] + Equals(t, testBlob.pack, result2.PackID) Equals(t, testBlob.tpe, result2.Type) Equals(t, testBlob.offset, result2.Offset) @@ -94,11 +93,11 @@ func TestIndexSerialize(t *testing.T) { // add more blobs to idx newtests := []testEntry{} for i := 0; i < 10; i++ { - packID := randomID() + packID := backend.RandomID() pos := uint(0) for j := 0; j < 10; j++ { - id := randomID() + id := backend.RandomID() length := uint(i*100 + j) idx.Store(repository.PackedBlob{ Type: pack.Data, @@ -128,7 +127,7 @@ func TestIndexSerialize(t *testing.T) { Assert(t, idx.Final(), "index not final after encoding") - id := randomID() + id := backend.RandomID() OK(t, idx.SetID(id)) id2, err := idx.ID() Assert(t, id2.Equal(id), @@ -143,9 +142,15 @@ func TestIndexSerialize(t *testing.T) { // all new blobs must be in the index for _, testBlob := range newtests { - blob, err := idx3.Lookup(testBlob.id) + list, err := idx3.Lookup(testBlob.id, testBlob.tpe) OK(t, err) + if len(list) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list) + } + + blob := list[0] + Equals(t, testBlob.pack, blob.PackID) Equals(t, testBlob.tpe, blob.Type) Equals(t, testBlob.offset, blob.Offset) @@ -159,11 +164,11 @@ func TestIndexSize(t *testing.T) { packs := 200 blobs := 100 for i := 0; i < packs; i++ { - packID := randomID() + packID := backend.RandomID() pos := uint(0) for j := 0; j < blobs; j++ { - id := randomID() + id := backend.RandomID() length := uint(i*100 + j) idx.Store(repository.PackedBlob{ Type: pack.Data, @@ -265,13 +270,13 @@ var exampleTests = []struct { var exampleLookupTest = struct { packID backend.ID - blobs backend.IDSet + blobs map[backend.ID]pack.BlobType }{ ParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), - backend.IDSet{ - ParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"): struct{}{}, - ParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"): struct{}{}, - ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"): struct{}{}, + map[backend.ID]pack.BlobType{ + ParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"): pack.Data, + ParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"): pack.Tree, + ParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"): pack.Data, }, } @@ -282,9 +287,16 @@ func TestIndexUnserialize(t *testing.T) { OK(t, err) for _, test := range exampleTests { - blob, err := idx.Lookup(test.id) + list, err := idx.Lookup(test.id, test.tpe) OK(t, err) + if len(list) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) + } + blob := list[0] + + t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) + Equals(t, test.packID, blob.PackID) Equals(t, test.tpe, blob.Type) Equals(t, test.offset, blob.Offset) @@ -299,9 +311,13 @@ func TestIndexUnserialize(t *testing.T) { } for _, blob := range blobs { - if !exampleLookupTest.blobs.Has(blob.ID) { + b, ok := exampleLookupTest.blobs[blob.ID] + if !ok { t.Errorf("unexpected blob %v found", blob.ID.Str()) } + if blob.Type != b { + t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + } } } @@ -310,9 +326,14 @@ func TestIndexUnserializeOld(t *testing.T) { OK(t, err) for _, test := range exampleTests { - blob, err := idx.Lookup(test.id) + list, err := idx.Lookup(test.id, test.tpe) OK(t, err) + if len(list) != 1 { + t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) + } + blob := list[0] + Equals(t, test.packID, blob.PackID) Equals(t, test.tpe, blob.Type) Equals(t, test.offset, blob.Offset) @@ -327,10 +348,10 @@ func TestIndexPacks(t *testing.T) { packs := backend.NewIDSet() for i := 0; i < 20; i++ { - packID := randomID() + packID := backend.RandomID() idx.Store(repository.PackedBlob{ Type: pack.Data, - ID: randomID(), + ID: backend.RandomID(), PackID: packID, Offset: 0, Length: 23, diff --git a/src/restic/repository/master_index.go b/src/restic/repository/master_index.go index 5fcf63378..c6114d055 100644 --- a/src/restic/repository/master_index.go +++ b/src/restic/repository/master_index.go @@ -21,32 +21,32 @@ func NewMasterIndex() *MasterIndex { } // Lookup queries all known Indexes for the ID and returns the first match. -func (mi *MasterIndex) Lookup(id backend.ID) (blob PackedBlob, err error) { +func (mi *MasterIndex) Lookup(id backend.ID, tpe pack.BlobType) (blobs []PackedBlob, err error) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() - debug.Log("MasterIndex.Lookup", "looking up id %v", id.Str()) + debug.Log("MasterIndex.Lookup", "looking up id %v, tpe %v", id.Str(), tpe) for _, idx := range mi.idx { - blob, err = idx.Lookup(id) + blobs, err = idx.Lookup(id, tpe) if err == nil { debug.Log("MasterIndex.Lookup", - "found id %v: %v", id.Str(), blob) + "found id %v: %v", id.Str(), blobs) return } } debug.Log("MasterIndex.Lookup", "id %v not found in any index", id.Str()) - return PackedBlob{}, fmt.Errorf("id %v not found in any index", id) + return nil, fmt.Errorf("id %v not found in any index", id) } // LookupSize queries all known Indexes for the ID and returns the first match. -func (mi *MasterIndex) LookupSize(id backend.ID) (uint, error) { +func (mi *MasterIndex) LookupSize(id backend.ID, tpe pack.BlobType) (uint, error) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() for _, idx := range mi.idx { - length, err := idx.LookupSize(id) + length, err := idx.LookupSize(id, tpe) if err == nil { return length, nil } @@ -72,12 +72,12 @@ func (mi *MasterIndex) ListPack(id backend.ID) (list []PackedBlob) { } // Has queries all known Indexes for the ID and returns the first match. -func (mi *MasterIndex) Has(id backend.ID) bool { +func (mi *MasterIndex) Has(id backend.ID, tpe pack.BlobType) bool { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() for _, idx := range mi.idx { - if idx.Has(id) { + if idx.Has(id, tpe) { return true } } diff --git a/src/restic/repository/parallel.go b/src/restic/repository/parallel.go index d1e66c58a..7094ae299 100644 --- a/src/restic/repository/parallel.go +++ b/src/restic/repository/parallel.go @@ -83,7 +83,7 @@ func ParallelWorkFuncParseID(f ParallelIDWorkFunc) ParallelWorkFunc { id, err := backend.ParseID(s) if err != nil { debug.Log("repository.ParallelWorkFuncParseID", "invalid ID %q: %v", id, err) - return nil + return err } return f(id, done) diff --git a/src/restic/repository/rand_reader.go b/src/restic/repository/rand_reader.go new file mode 100644 index 000000000..201826a94 --- /dev/null +++ b/src/restic/repository/rand_reader.go @@ -0,0 +1,79 @@ +package repository + +import ( + "io" + "math/rand" +) + +// RandReader allows reading from a rand.Rand. +type RandReader struct { + rnd *rand.Rand + buf []byte +} + +// NewRandReader creates a new Reader from a random source. +func NewRandReader(rnd *rand.Rand) io.Reader { + return &RandReader{rnd: rnd, buf: make([]byte, 0, 7)} +} + +func (rd *RandReader) read(p []byte) (n int, err error) { + if len(p)%7 != 0 { + panic("invalid buffer length, not multiple of 7") + } + + rnd := rd.rnd + for i := 0; i < len(p); i += 7 { + val := rnd.Int63() + + p[i+0] = byte(val >> 0) + p[i+1] = byte(val >> 8) + p[i+2] = byte(val >> 16) + p[i+3] = byte(val >> 24) + p[i+4] = byte(val >> 32) + p[i+5] = byte(val >> 40) + p[i+6] = byte(val >> 48) + } + + return len(p), nil +} + +func (rd *RandReader) Read(p []byte) (int, error) { + // first, copy buffer to p + pos := copy(p, rd.buf) + copy(rd.buf, rd.buf[pos:]) + + // shorten buf and p accordingly + rd.buf = rd.buf[:len(rd.buf)-pos] + p = p[pos:] + + // if this is enough to fill p, return + if len(p) == 0 { + return pos, nil + } + + // load multiple of 7 byte + l := (len(p) / 7) * 7 + n, err := rd.read(p[:l]) + pos += n + if err != nil { + return pos, err + } + p = p[n:] + + // load 7 byte to temp buffer + rd.buf = rd.buf[:7] + n, err = rd.read(rd.buf) + if err != nil { + return pos, err + } + + // copy the remaining bytes from the buffer to p + n = copy(p, rd.buf) + pos += n + + // save the remaining bytes in rd.buf + n = copy(rd.buf, rd.buf[n:]) + rd.buf = rd.buf[:n] + + return pos, nil +} diff --git a/src/restic/repository/repack.go b/src/restic/repository/repack.go new file mode 100644 index 000000000..0498164a4 --- /dev/null +++ b/src/restic/repository/repack.go @@ -0,0 +1,84 @@ +package repository + +import ( + "io" + "restic/backend" + "restic/crypto" + "restic/debug" + "restic/pack" +) + +// Repack takes a list of packs together with a list of blobs contained in +// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved +// into a new pack. Afterwards, the packs are removed. This operation requires +// an exclusive lock on the repo. +func Repack(repo *Repository, packs backend.IDSet, keepBlobs pack.BlobSet) (err error) { + debug.Log("Repack", "repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs)) + + buf := make([]byte, 0, maxPackSize) + for packID := range packs { + // load the complete pack + h := backend.Handle{Type: backend.Data, Name: packID.String()} + + l, err := repo.Backend().Load(h, buf[:cap(buf)], 0) + if err == io.ErrUnexpectedEOF { + err = nil + buf = buf[:l] + } + + if err != nil { + return err + } + + debug.Log("Repack", "pack %v loaded (%d bytes)", packID.Str(), len(buf)) + + unpck, err := pack.NewUnpacker(repo.Key(), pack.BufferLoader(buf)) + if err != nil { + return err + } + + debug.Log("Repack", "processing pack %v, blobs: %v", packID.Str(), len(unpck.Entries)) + var plaintext []byte + for _, entry := range unpck.Entries { + h := pack.Handle{ID: entry.ID, Type: entry.Type} + if !keepBlobs.Has(h) { + continue + } + + ciphertext := buf[entry.Offset : entry.Offset+entry.Length] + + if cap(plaintext) < len(ciphertext) { + plaintext = make([]byte, len(ciphertext)) + } + + plaintext, err = crypto.Decrypt(repo.Key(), plaintext, ciphertext) + if err != nil { + return err + } + + _, err = repo.SaveAndEncrypt(entry.Type, plaintext, &entry.ID) + if err != nil { + return err + } + + debug.Log("Repack", " saved blob %v", entry.ID.Str()) + + keepBlobs.Delete(h) + } + } + + if err := repo.Flush(); err != nil { + return err + } + + for packID := range packs { + err := repo.Backend().Remove(backend.Data, packID.String()) + if err != nil { + debug.Log("Repack", "error removing pack %v: %v", packID.Str(), err) + return err + } + debug.Log("Repack", "removed pack %v", packID.Str()) + } + + return nil +} diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go new file mode 100644 index 000000000..b29c7e622 --- /dev/null +++ b/src/restic/repository/repack_test.go @@ -0,0 +1,221 @@ +package repository_test + +import ( + "io" + "math/rand" + "restic/backend" + "restic/pack" + "restic/repository" + "testing" +) + +func randomSize(min, max int) int { + return rand.Intn(max-min) + min +} + +func random(t testing.TB, length int) []byte { + rd := repository.NewRandReader(rand.New(rand.NewSource(int64(length)))) + buf := make([]byte, length) + _, err := io.ReadFull(rd, buf) + if err != nil { + t.Fatalf("unable to read %d random bytes: %v", length, err) + } + + return buf +} + +func createRandomBlobs(t testing.TB, repo *repository.Repository, blobs int, pData float32) { + for i := 0; i < blobs; i++ { + var ( + tpe pack.BlobType + length int + ) + + if rand.Float32() < pData { + tpe = pack.Data + length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data + } else { + tpe = pack.Tree + length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB + } + + buf := random(t, length) + id := backend.Hash(buf) + + if repo.Index().Has(id, pack.Data) { + t.Errorf("duplicate blob %v/%v ignored", id, pack.Data) + continue + } + + _, err := repo.SaveAndEncrypt(tpe, buf, &id) + if err != nil { + t.Fatalf("SaveFrom() error %v", err) + } + + if rand.Float32() < 0.2 { + if err = repo.Flush(); err != nil { + t.Fatalf("repo.Flush() returned error %v", err) + } + } + } + + if err := repo.Flush(); err != nil { + t.Fatalf("repo.Flush() returned error %v", err) + } +} + +// selectBlobs splits the list of all blobs randomly into two lists. A blob +// will be contained in the firstone ith probability p. +func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, list2 pack.BlobSet) { + done := make(chan struct{}) + defer close(done) + + list1 = pack.NewBlobSet() + list2 = pack.NewBlobSet() + + blobs := pack.NewBlobSet() + + for id := range repo.List(backend.Data, done) { + entries, _, err := repo.ListPack(id) + if err != nil { + t.Fatalf("error listing pack %v: %v", id, err) + } + + for _, entry := range entries { + h := pack.Handle{ID: entry.ID, Type: entry.Type} + if blobs.Has(h) { + t.Errorf("ignoring duplicate blob %v", h) + continue + } + blobs.Insert(h) + + if rand.Float32() <= p { + list1.Insert(pack.Handle{ID: entry.ID, Type: entry.Type}) + } else { + list2.Insert(pack.Handle{ID: entry.ID, Type: entry.Type}) + } + + } + } + + return list1, list2 +} + +func listPacks(t *testing.T, repo *repository.Repository) backend.IDSet { + done := make(chan struct{}) + defer close(done) + + list := backend.NewIDSet() + for id := range repo.List(backend.Data, done) { + list.Insert(id) + } + + return list +} + +func findPacksForBlobs(t *testing.T, repo *repository.Repository, blobs pack.BlobSet) backend.IDSet { + packs := backend.NewIDSet() + + idx := repo.Index() + for h := range blobs { + list, err := idx.Lookup(h.ID, h.Type) + if err != nil { + t.Fatal(err) + } + + for _, pb := range list { + packs.Insert(pb.PackID) + } + } + + return packs +} + +func repack(t *testing.T, repo *repository.Repository, packs backend.IDSet, blobs pack.BlobSet) { + err := repository.Repack(repo, packs, blobs) + if err != nil { + t.Fatal(err) + } +} + +func saveIndex(t *testing.T, repo *repository.Repository) { + if err := repo.SaveIndex(); err != nil { + t.Fatalf("repo.SaveIndex() %v", err) + } +} + +func rebuildIndex(t *testing.T, repo *repository.Repository) { + if err := repository.RebuildIndex(repo); err != nil { + t.Fatalf("error rebuilding index: %v", err) + } +} + +func reloadIndex(t *testing.T, repo *repository.Repository) { + repo.SetIndex(repository.NewMasterIndex()) + if err := repo.LoadIndex(); err != nil { + t.Fatalf("error loading new index: %v", err) + } +} + +func TestRepack(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + createRandomBlobs(t, repo, 100, 0.7) + + packsBefore := listPacks(t, repo) + + // Running repack on empty ID sets should not do anything at all. + repack(t, repo, nil, nil) + + packsAfter := listPacks(t, repo) + + if !packsAfter.Equals(packsBefore) { + t.Fatalf("packs are not equal, Repack modified something. Before:\n %v\nAfter:\n %v", + packsBefore, packsAfter) + } + + saveIndex(t, repo) + + removeBlobs, keepBlobs := selectBlobs(t, repo, 0.2) + + removePacks := findPacksForBlobs(t, repo, removeBlobs) + + repack(t, repo, removePacks, keepBlobs) + rebuildIndex(t, repo) + reloadIndex(t, repo) + + packsAfter = listPacks(t, repo) + for id := range removePacks { + if packsAfter.Has(id) { + t.Errorf("pack %v still present although it should have been repacked and removed", id.Str()) + } + } + + idx := repo.Index() + + for h := range keepBlobs { + list, err := idx.Lookup(h.ID, h.Type) + if err != nil { + t.Errorf("unable to find blob %v in repo", h.ID.Str()) + continue + } + + if len(list) != 1 { + t.Errorf("expected one pack in the list, got: %v", list) + continue + } + + pb := list[0] + + if removePacks.Has(pb.PackID) { + t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID) + } + } + + for h := range removeBlobs { + if _, err := idx.Lookup(h.ID, h.Type); err == nil { + t.Errorf("blob %v still contained in the repo", h) + } + } +} diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index aa75d322e..ca53fb39d 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -77,56 +77,70 @@ func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, erro // LoadBlob tries to load and decrypt content identified by t and id from a // pack from the backend, the result is stored in plaintextBuf, which must be // large enough to hold the complete blob. -func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, plaintextBuf []byte) ([]byte, error) { +func (r *Repository) LoadBlob(id backend.ID, t pack.BlobType, plaintextBuf []byte) ([]byte, error) { debug.Log("Repo.LoadBlob", "load %v with id %v", t, id.Str()) - // lookup pack - blob, err := r.idx.Lookup(id) + + // lookup plaintext size of blob + size, err := r.idx.LookupSize(id, t) + if err != nil { + return nil, err + } + + // make sure the plaintext buffer is large enough, extend otherwise + plaintextBufSize := uint(cap(plaintextBuf)) + if size > plaintextBufSize { + debug.Log("Repo.LoadBlob", "need to expand buffer: want %d bytes, got %d", + size, plaintextBufSize) + plaintextBuf = make([]byte, size) + } + + // lookup packs + blobs, err := r.idx.Lookup(id, t) if err != nil { debug.Log("Repo.LoadBlob", "id %v not found in index: %v", id.Str(), err) return nil, err } - plaintextBufSize := uint(cap(plaintextBuf)) - if blob.PlaintextLength() > plaintextBufSize { - debug.Log("Repo.LoadBlob", "need to expand buffer: want %d bytes, got %d", - blob.PlaintextLength(), plaintextBufSize) - plaintextBuf = make([]byte, blob.PlaintextLength()) + for _, blob := range blobs { + debug.Log("Repo.LoadBlob", "id %v found: %v", id.Str(), blob) + + if blob.Type != t { + debug.Log("Repo.LoadBlob", "blob %v has wrong block type, want %v", blob, t) + } + + // load blob from pack + h := backend.Handle{Type: backend.Data, Name: blob.PackID.String()} + ciphertextBuf := make([]byte, blob.Length) + n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset)) + if err != nil { + debug.Log("Repo.LoadBlob", "error loading blob %v: %v", blob, err) + fmt.Fprintf(os.Stderr, "error loading blob %v: %v", id, err) + continue + } + + if uint(n) != blob.Length { + debug.Log("Repo.LoadBlob", "error loading blob %v: wrong length returned, want %d, got %d", + blob.Length, uint(n)) + continue + } + + // decrypt + plaintextBuf, err = r.decryptTo(plaintextBuf, ciphertextBuf) + if err != nil { + fmt.Fprintf(os.Stderr, "decrypting blob %v failed: %v", id, err) + continue + } + + // check hash + if !backend.Hash(plaintextBuf).Equal(id) { + fmt.Fprintf(os.Stderr, "blob %v returned invalid hash", id) + continue + } + + return plaintextBuf, nil } - if blob.Type != t { - debug.Log("Repo.LoadBlob", "wrong type returned for %v: wanted %v, got %v", id.Str(), t, blob.Type) - return nil, fmt.Errorf("blob has wrong type %v (wanted: %v)", blob.Type, t) - } - - debug.Log("Repo.LoadBlob", "id %v found: %v", id.Str(), blob) - - // load blob from pack - h := backend.Handle{Type: backend.Data, Name: blob.PackID.String()} - ciphertextBuf := make([]byte, blob.Length) - n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset)) - if err != nil { - debug.Log("Repo.LoadBlob", "error loading blob %v: %v", blob, err) - return nil, err - } - - if uint(n) != blob.Length { - debug.Log("Repo.LoadBlob", "error loading blob %v: wrong length returned, want %d, got %d", - blob.Length, uint(n)) - return nil, errors.New("wrong length returned") - } - - // decrypt - plaintextBuf, err = r.decryptTo(plaintextBuf, ciphertextBuf) - if err != nil { - return nil, err - } - - // check hash - if !backend.Hash(plaintextBuf).Equal(id) { - return nil, errors.New("invalid data returned") - } - - return plaintextBuf, nil + return nil, fmt.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs)) } // closeOrErr calls cl.Close() and sets err to the returned error value if @@ -153,7 +167,7 @@ func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interf // LoadJSONPack calls LoadBlob() to load a blob from the backend, decrypt the // data and afterwards call json.Unmarshal on the item. func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) (err error) { - buf, err := r.LoadBlob(t, id, nil) + buf, err := r.LoadBlob(id, t, nil) if err != nil { return err } @@ -162,8 +176,8 @@ func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface } // LookupBlobSize returns the size of blob id. -func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) { - return r.idx.LookupSize(id) +func (r *Repository) LookupBlobSize(id backend.ID, tpe pack.BlobType) (uint, error) { + return r.idx.LookupSize(id, tpe) } // SaveAndEncrypt encrypts data and stores it to the backend as type t. If data @@ -416,6 +430,17 @@ func (r *Repository) Init(password string) error { return errors.New("repository master key and config already initialized") } + cfg, err := CreateConfig() + if err != nil { + return err + } + + return r.init(password, cfg) +} + +// init creates a new master key with the supplied password and uses it to save +// the config into the repo. +func (r *Repository) init(password string, cfg Config) error { key, err := createMasterKey(r, password) if err != nil { return err @@ -424,7 +449,8 @@ func (r *Repository) Init(password string) error { r.key = key.master r.packerManager.key = key.master r.keyName = key.Name() - r.Config, err = CreateConfig(r) + r.Config = cfg + _, err = r.SaveJSONUnpacked(backend.Config, cfg) return err } @@ -518,17 +544,24 @@ func (r *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I return outCh } -// ListPack returns the list of blobs saved in the pack id. -func (r *Repository) ListPack(id backend.ID) ([]pack.Blob, error) { +// ListPack returns the list of blobs saved in the pack id and the length of +// the file as stored in the backend. +func (r *Repository) ListPack(id backend.ID) ([]pack.Blob, int64, error) { h := backend.Handle{Type: backend.Data, Name: id.String()} - rd := backend.NewReadSeeker(r.Backend(), h) - unpacker, err := pack.NewUnpacker(r.Key(), rd) + blobInfo, err := r.Backend().Stat(h) if err != nil { - return nil, err + return nil, 0, err } - return unpacker.Entries, nil + ldr := pack.BackendLoader{Backend: r.Backend(), Handle: h} + + unpacker, err := pack.NewUnpacker(r.Key(), ldr) + if err != nil { + return nil, 0, err + } + + return unpacker.Entries, blobInfo.Size, nil } // Delete calls backend.Delete() if implemented, and returns an error diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 3df824771..db70765a8 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -92,7 +92,7 @@ func TestSave(t *testing.T) { // OK(t, repo.SaveIndex()) // read back - buf, err := repo.LoadBlob(pack.Data, id, make([]byte, size)) + buf, err := repo.LoadBlob(id, pack.Data, make([]byte, size)) OK(t, err) Assert(t, len(buf) == len(data), @@ -124,7 +124,7 @@ func TestSaveFrom(t *testing.T) { OK(t, repo.Flush()) // read back - buf, err := repo.LoadBlob(pack.Data, id, make([]byte, size)) + buf, err := repo.LoadBlob(id, pack.Data, make([]byte, size)) OK(t, err) Assert(t, len(buf) == len(data), diff --git a/src/restic/repository/testing.go b/src/restic/repository/testing.go index f45714d80..f0a9913f1 100644 --- a/src/restic/repository/testing.go +++ b/src/restic/repository/testing.go @@ -6,6 +6,8 @@ import ( "restic/backend/local" "restic/backend/mem" "testing" + + "github.com/restic/chunker" ) // TestBackend returns a fully configured in-memory backend. @@ -16,8 +18,11 @@ func TestBackend(t testing.TB) (be backend.Backend, cleanup func()) { // TestPassword is used for all repositories created by the Test* functions. const TestPassword = "geheim" +const testChunkerPol = chunker.Pol(0x3DA3358B4DC173) + // TestRepositoryWithBackend returns a repository initialized with a test -// password. If be is nil, an in-memory backend is used. +// password. If be is nil, an in-memory backend is used. A constant polynomial +// is used for the chunker. func TestRepositoryWithBackend(t testing.TB, be backend.Backend) (r *Repository, cleanup func()) { var beCleanup func() if be == nil { @@ -26,9 +31,10 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend) (r *Repository, r = New(be) - err := r.Init(TestPassword) + cfg := TestCreateConfig(t, testChunkerPol) + err := r.init(TestPassword, cfg) if err != nil { - t.Fatalf("TestRepopository(): initialize repo failed: %v", err) + t.Fatalf("TestRepository(): initialize repo failed: %v", err) } return r, func() { @@ -41,7 +47,7 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend) (r *Repository, // TestRepository returns a repository initialized with a test password on an // in-memory backend. When the environment variable RESTIC_TEST_REPO is set to // a non-existing directory, a local backend is created there and this is used -// instead. The directory is not removed. +// instead. The directory is not removed, but left there for inspection. func TestRepository(t testing.TB) (r *Repository, cleanup func()) { dir := os.Getenv("RESTIC_TEST_REPO") if dir != "" { diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go index 9c95e84cf..3eaa0b61f 100644 --- a/src/restic/snapshot.go +++ b/src/restic/snapshot.go @@ -83,7 +83,8 @@ func LoadAllSnapshots(repo *repository.Repository) (snapshots []*Snapshot, err e } func (sn Snapshot) String() string { - return fmt.Sprintf("", sn.id.Str(), sn.Paths, sn.Time) + return fmt.Sprintf("", + sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) } // ID retuns the snapshot's ID. @@ -125,7 +126,7 @@ func SamePaths(expected, actual []string) bool { return true } -// Error when no snapshot is found for the given criteria +// ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. var ErrNoSnapshotFound = errors.New("no snapshot found") // FindLatestSnapshot finds latest snapshot with optional target/directory and source filters diff --git a/src/restic/snapshot_filter.go b/src/restic/snapshot_filter.go new file mode 100644 index 000000000..595987d95 --- /dev/null +++ b/src/restic/snapshot_filter.go @@ -0,0 +1,207 @@ +package restic + +import ( + "fmt" + "reflect" + "sort" + "time" +) + +// Snapshots is a list of snapshots. +type Snapshots []*Snapshot + +// Len returns the number of snapshots in sn. +func (sn Snapshots) Len() int { + return len(sn) +} + +// Less returns true iff the ith snapshot has been made after the jth. +func (sn Snapshots) Less(i, j int) bool { + return sn[i].Time.After(sn[j].Time) +} + +// Swap exchanges the two snapshots. +func (sn Snapshots) Swap(i, j int) { + sn[i], sn[j] = sn[j], sn[i] +} + +// SnapshotFilter configures criteria for filtering snapshots before an +// ExpirePolicy can be applied. +type SnapshotFilter struct { + Hostname string + Username string + Paths []string +} + +// FilterSnapshots returns the snapshots from s which match the filter f. +func FilterSnapshots(s Snapshots, f SnapshotFilter) (result Snapshots) { + for _, snap := range s { + if f.Hostname != "" && f.Hostname != snap.Hostname { + continue + } + + if f.Username != "" && f.Username != snap.Username { + continue + } + + if f.Paths != nil && !reflect.DeepEqual(f.Paths, snap.Paths) { + continue + } + + result = append(result, snap) + } + + return result +} + +// ExpirePolicy configures which snapshots should be automatically removed. +type ExpirePolicy struct { + Last int // keep the last n snapshots + Hourly int // keep the last n hourly snapshots + Daily int // keep the last n daily snapshots + Weekly int // keep the last n weekly snapshots + Monthly int // keep the last n monthly snapshots + Yearly int // keep the last n yearly snapshots +} + +// Sum returns the maximum number of snapshots to be kept according to this +// policy. +func (e ExpirePolicy) Sum() int { + return e.Last + e.Hourly + e.Daily + e.Weekly + e.Monthly + e.Yearly +} + +// Empty returns true iff no policy has been configured (all values zero). +func (e ExpirePolicy) Empty() bool { + empty := ExpirePolicy{} + return e == empty +} + +// filter is used to split a list of snapshots into those to keep and those to +// remove according to a policy. +type filter struct { + Unprocessed Snapshots + Remove Snapshots + Keep Snapshots +} + +func (f filter) String() string { + return fmt.Sprintf("", len(f.Unprocessed), len(f.Keep), len(f.Remove)) +} + +// ymdh returns an integer in the form YYYYMMDDHH. +func ymdh(d time.Time) int { + return d.Year()*1000000 + int(d.Month())*10000 + d.Day()*100 + d.Hour() +} + +// ymd returns an integer in the form YYYYMMDD. +func ymd(d time.Time) int { + return d.Year()*10000 + int(d.Month())*100 + d.Day() +} + +// yw returns an integer in the form YYYYWW, where WW is the week number. +func yw(d time.Time) int { + year, week := d.ISOWeek() + return year*100 + week +} + +// ym returns an integer in the form YYYYMM. +func ym(d time.Time) int { + return d.Year()*100 + int(d.Month()) +} + +// y returns the year of d. +func y(d time.Time) int { + return d.Year() +} + +// apply moves snapshots from Unprocess to either Keep or Remove. It sorts the +// snapshots into buckets according to the return of fn, and then moves the +// newest snapshot in each bucket to Keep and all others to Remove. When max +// snapshots were found, processing stops. +func (f *filter) apply(fn func(time.Time) int, max int) { + if max == 0 || len(f.Unprocessed) == 0 { + return + } + + sameDay := Snapshots{} + lastDay := fn(f.Unprocessed[0].Time) + + for len(f.Unprocessed) > 0 { + cur := f.Unprocessed[0] + + day := fn(cur.Time) + + // if the snapshots are from a new day, forget all but the first (=last + // in time) snapshot from the previous day. + if day != lastDay { + f.Keep = append(f.Keep, sameDay[0]) + for _, snapshot := range sameDay[1:] { + f.Remove = append(f.Remove, snapshot) + } + + sameDay = Snapshots{} + lastDay = day + max-- + + if max == 0 { + break + } + } + + // collect all snapshots for the current day + sameDay = append(sameDay, cur) + f.Unprocessed = f.Unprocessed[1:] + } + + if len(sameDay) > 0 { + f.Keep = append(f.Keep, sameDay[0]) + for _, snapshot := range sameDay[1:] { + f.Remove = append(f.Remove, snapshot) + } + } +} + +// keepLast marks the last n snapshots as to be kept. +func (f *filter) keepLast(n int) { + if n > len(f.Unprocessed) { + n = len(f.Unprocessed) + } + + f.Keep = append(f.Keep, f.Unprocessed[:n]...) + f.Unprocessed = f.Unprocessed[n:] +} + +// finish moves all remaining snapshots to remove. +func (f *filter) finish() { + f.Remove = append(f.Remove, f.Unprocessed...) +} + +// ApplyPolicy runs returns the snapshots from s that are to be deleted according +// to the policy p. s is sorted in the process. +func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { + sort.Sort(list) + + if p.Empty() { + return list, remove + } + + if len(list) == 0 { + return list, remove + } + + f := filter{ + Unprocessed: list, + Remove: Snapshots{}, + Keep: Snapshots{}, + } + + f.keepLast(p.Last) + f.apply(ymdh, p.Hourly) + f.apply(ymd, p.Daily) + f.apply(yw, p.Weekly) + f.apply(ym, p.Monthly) + f.apply(y, p.Yearly) + f.finish() + + return f.Keep, f.Remove +} diff --git a/src/restic/snapshot_filter_test.go b/src/restic/snapshot_filter_test.go new file mode 100644 index 000000000..07d2e106d --- /dev/null +++ b/src/restic/snapshot_filter_test.go @@ -0,0 +1,263 @@ +package restic + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "reflect" + "sort" + "testing" + "time" +) + +func parseTime(s string) time.Time { + t, err := time.Parse("2006-01-02 15:04:05", s) + if err != nil { + panic(err) + } + + return t.UTC() +} + +var testFilterSnapshots = Snapshots{ + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-01 01:02:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "bar", Username: "testuser", Time: parseTime("2016-01-01 01:03:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-03 07:02:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "bar", Username: "testuser", Time: parseTime("2016-01-01 07:08:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 10:23:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 11:23:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 12:23:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 12:24:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 12:28:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 12:30:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-04 16:23:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-05 09:02:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-06 08:02:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-07 10:02:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "root", Time: parseTime("2016-01-08 20:02:03"), Paths: []string{"/usr", "/sbin"}}, + {Hostname: "foo", Username: "root", Time: parseTime("2016-01-09 21:02:03"), Paths: []string{"/usr", "/sbin"}}, + {Hostname: "bar", Username: "root", Time: parseTime("2016-01-12 21:02:03"), Paths: []string{"/usr", "/sbin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-12 21:08:03"), Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "testuser", Time: parseTime("2016-01-18 12:02:03"), Paths: []string{"/usr", "/bin"}}, +} + +var filterTests = []SnapshotFilter{ + {Hostname: "foo"}, + {Username: "root"}, + {Hostname: "foo", Username: "root"}, + {Paths: []string{"/usr", "/bin"}}, + {Hostname: "bar", Paths: []string{"/usr", "/bin"}}, + {Hostname: "foo", Username: "root", Paths: []string{"/usr", "/sbin"}}, +} + +func TestFilterSnapshots(t *testing.T) { + sort.Sort(testFilterSnapshots) + + for i, f := range filterTests { + res := FilterSnapshots(testFilterSnapshots, f) + + goldenFilename := filepath.Join("testdata", fmt.Sprintf("filter_snapshots_%d", i)) + + if *updateGoldenFiles { + buf, err := json.MarshalIndent(res, "", " ") + if err != nil { + t.Fatalf("error marshaling result: %v", err) + } + + if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { + t.Fatalf("unable to update golden file: %v", err) + } + } + + buf, err := ioutil.ReadFile(goldenFilename) + if err != nil { + t.Errorf("error loading golden file %v: %v", goldenFilename, err) + continue + } + + var want Snapshots + err = json.Unmarshal(buf, &want) + + if !reflect.DeepEqual(res, want) { + t.Errorf("test %v: wrong result, want:\n %#v\ngot:\n %#v", i, want, res) + continue + } + } +} + +var testExpireSnapshots = Snapshots{ + {Time: parseTime("2014-09-01 10:20:30")}, + {Time: parseTime("2014-09-02 10:20:30")}, + {Time: parseTime("2014-09-05 10:20:30")}, + {Time: parseTime("2014-09-06 10:20:30")}, + {Time: parseTime("2014-09-08 10:20:30")}, + {Time: parseTime("2014-09-09 10:20:30")}, + {Time: parseTime("2014-09-10 10:20:30")}, + {Time: parseTime("2014-09-11 10:20:30")}, + {Time: parseTime("2014-09-20 10:20:30")}, + {Time: parseTime("2014-09-22 10:20:30")}, + {Time: parseTime("2014-08-08 10:20:30")}, + {Time: parseTime("2014-08-10 10:20:30")}, + {Time: parseTime("2014-08-12 10:20:30")}, + {Time: parseTime("2014-08-13 10:20:30")}, + {Time: parseTime("2014-08-13 10:20:30")}, + {Time: parseTime("2014-08-15 10:20:30")}, + {Time: parseTime("2014-08-18 10:20:30")}, + {Time: parseTime("2014-08-20 10:20:30")}, + {Time: parseTime("2014-08-21 10:20:30")}, + {Time: parseTime("2014-08-22 10:20:30")}, + {Time: parseTime("2014-10-01 10:20:30")}, + {Time: parseTime("2014-10-02 10:20:30")}, + {Time: parseTime("2014-10-05 10:20:30")}, + {Time: parseTime("2014-10-06 10:20:30")}, + {Time: parseTime("2014-10-08 10:20:30")}, + {Time: parseTime("2014-10-09 10:20:30")}, + {Time: parseTime("2014-10-10 10:20:30")}, + {Time: parseTime("2014-10-11 10:20:30")}, + {Time: parseTime("2014-10-20 10:20:30")}, + {Time: parseTime("2014-10-22 10:20:30")}, + {Time: parseTime("2014-11-08 10:20:30")}, + {Time: parseTime("2014-11-10 10:20:30")}, + {Time: parseTime("2014-11-12 10:20:30")}, + {Time: parseTime("2014-11-13 10:20:30")}, + {Time: parseTime("2014-11-13 10:20:30")}, + {Time: parseTime("2014-11-15 10:20:30")}, + {Time: parseTime("2014-11-18 10:20:30")}, + {Time: parseTime("2014-11-20 10:20:30")}, + {Time: parseTime("2014-11-21 10:20:30")}, + {Time: parseTime("2014-11-22 10:20:30")}, + {Time: parseTime("2015-09-01 10:20:30")}, + {Time: parseTime("2015-09-02 10:20:30")}, + {Time: parseTime("2015-09-05 10:20:30")}, + {Time: parseTime("2015-09-06 10:20:30")}, + {Time: parseTime("2015-09-08 10:20:30")}, + {Time: parseTime("2015-09-09 10:20:30")}, + {Time: parseTime("2015-09-10 10:20:30")}, + {Time: parseTime("2015-09-11 10:20:30")}, + {Time: parseTime("2015-09-20 10:20:30")}, + {Time: parseTime("2015-09-22 10:20:30")}, + {Time: parseTime("2015-08-08 10:20:30")}, + {Time: parseTime("2015-08-10 10:20:30")}, + {Time: parseTime("2015-08-12 10:20:30")}, + {Time: parseTime("2015-08-13 10:20:30")}, + {Time: parseTime("2015-08-13 10:20:30")}, + {Time: parseTime("2015-08-15 10:20:30")}, + {Time: parseTime("2015-08-18 10:20:30")}, + {Time: parseTime("2015-08-20 10:20:30")}, + {Time: parseTime("2015-08-21 10:20:30")}, + {Time: parseTime("2015-08-22 10:20:30")}, + {Time: parseTime("2015-10-01 10:20:30")}, + {Time: parseTime("2015-10-02 10:20:30")}, + {Time: parseTime("2015-10-05 10:20:30")}, + {Time: parseTime("2015-10-06 10:20:30")}, + {Time: parseTime("2015-10-08 10:20:30")}, + {Time: parseTime("2015-10-09 10:20:30")}, + {Time: parseTime("2015-10-10 10:20:30")}, + {Time: parseTime("2015-10-11 10:20:30")}, + {Time: parseTime("2015-10-20 10:20:30")}, + {Time: parseTime("2015-10-22 10:20:30")}, + {Time: parseTime("2015-11-08 10:20:30")}, + {Time: parseTime("2015-11-10 10:20:30")}, + {Time: parseTime("2015-11-12 10:20:30")}, + {Time: parseTime("2015-11-13 10:20:30")}, + {Time: parseTime("2015-11-13 10:20:30")}, + {Time: parseTime("2015-11-15 10:20:30")}, + {Time: parseTime("2015-11-18 10:20:30")}, + {Time: parseTime("2015-11-20 10:20:30")}, + {Time: parseTime("2015-11-21 10:20:30")}, + {Time: parseTime("2015-11-22 10:20:30")}, + {Time: parseTime("2016-01-01 01:02:03")}, + {Time: parseTime("2016-01-01 01:03:03")}, + {Time: parseTime("2016-01-01 07:08:03")}, + {Time: parseTime("2016-01-03 07:02:03")}, + {Time: parseTime("2016-01-04 10:23:03")}, + {Time: parseTime("2016-01-04 11:23:03")}, + {Time: parseTime("2016-01-04 12:23:03")}, + {Time: parseTime("2016-01-04 12:24:03")}, + {Time: parseTime("2016-01-04 12:28:03")}, + {Time: parseTime("2016-01-04 12:30:03")}, + {Time: parseTime("2016-01-04 16:23:03")}, + {Time: parseTime("2016-01-05 09:02:03")}, + {Time: parseTime("2016-01-06 08:02:03")}, + {Time: parseTime("2016-01-07 10:02:03")}, + {Time: parseTime("2016-01-08 20:02:03")}, + {Time: parseTime("2016-01-09 21:02:03")}, + {Time: parseTime("2016-01-12 21:02:03")}, + {Time: parseTime("2016-01-12 21:08:03")}, + {Time: parseTime("2016-01-18 12:02:03")}, +} + +var expireTests = []ExpirePolicy{ + {}, + {Last: 10}, + {Last: 15}, + {Last: 99}, + {Last: 200}, + {Hourly: 20}, + {Daily: 3}, + {Daily: 10}, + {Daily: 30}, + {Last: 5, Daily: 5}, + {Last: 2, Daily: 10}, + {Weekly: 2}, + {Weekly: 4}, + {Daily: 3, Weekly: 4}, + {Monthly: 6}, + {Daily: 2, Weekly: 2, Monthly: 6}, + {Yearly: 10}, + {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, +} + +func TestApplyPolicy(t *testing.T) { + for i, p := range expireTests { + keep, remove := ApplyPolicy(testExpireSnapshots, p) + + t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", + i, len(keep), len(remove), len(testExpireSnapshots), p) + + if len(keep)+len(remove) != len(testExpireSnapshots) { + t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", + i, len(keep)+len(remove), len(testExpireSnapshots)) + } + + if p.Sum() > 0 && len(keep) > p.Sum() { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + p.Sum(), len(keep)) + } + + for _, sn := range keep { + t.Logf("test %d: keep snapshot at %v\n", i, sn.Time) + } + for _, sn := range remove { + t.Logf("test %d: forget snapshot at %v\n", i, sn.Time) + } + + goldenFilename := filepath.Join("testdata", fmt.Sprintf("expired_snapshots_%d", i)) + + if *updateGoldenFiles { + buf, err := json.MarshalIndent(keep, "", " ") + if err != nil { + t.Fatalf("error marshaling result: %v", err) + } + + if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { + t.Fatalf("unable to update golden file: %v", err) + } + } + + buf, err := ioutil.ReadFile(goldenFilename) + if err != nil { + t.Errorf("error loading golden file %v: %v", goldenFilename, err) + continue + } + + var want Snapshots + err = json.Unmarshal(buf, &want) + + if !reflect.DeepEqual(keep, want) { + t.Errorf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) + continue + } + } +} diff --git a/src/restic/testdata/expired_snapshots_0 b/src/restic/testdata/expired_snapshots_0 new file mode 100644 index 000000000..d70bdbaa1 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_0 @@ -0,0 +1,497 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_1 b/src/restic/testdata/expired_snapshots_1 new file mode 100644 index 000000000..22e6c2141 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_1 @@ -0,0 +1,52 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_10 b/src/restic/testdata/expired_snapshots_10 new file mode 100644 index 000000000..72ae755ce --- /dev/null +++ b/src/restic/testdata/expired_snapshots_10 @@ -0,0 +1,62 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_11 b/src/restic/testdata/expired_snapshots_11 new file mode 100644 index 000000000..d36f97b3c --- /dev/null +++ b/src/restic/testdata/expired_snapshots_11 @@ -0,0 +1,12 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_12 b/src/restic/testdata/expired_snapshots_12 new file mode 100644 index 000000000..57b4ab846 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_12 @@ -0,0 +1,22 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_13 b/src/restic/testdata/expired_snapshots_13 new file mode 100644 index 000000000..93a52ad8c --- /dev/null +++ b/src/restic/testdata/expired_snapshots_13 @@ -0,0 +1,37 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_14 b/src/restic/testdata/expired_snapshots_14 new file mode 100644 index 000000000..5126b3e01 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_14 @@ -0,0 +1,32 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_15 b/src/restic/testdata/expired_snapshots_15 new file mode 100644 index 000000000..58d52ae4a --- /dev/null +++ b/src/restic/testdata/expired_snapshots_15 @@ -0,0 +1,52 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_16 b/src/restic/testdata/expired_snapshots_16 new file mode 100644 index 000000000..37a905da0 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_16 @@ -0,0 +1,17 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_17 b/src/restic/testdata/expired_snapshots_17 new file mode 100644 index 000000000..553c18d89 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_17 @@ -0,0 +1,72 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_2 b/src/restic/testdata/expired_snapshots_2 new file mode 100644 index 000000000..867e90ff0 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_2 @@ -0,0 +1,77 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_3 b/src/restic/testdata/expired_snapshots_3 new file mode 100644 index 000000000..d70bdbaa1 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_3 @@ -0,0 +1,497 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_4 b/src/restic/testdata/expired_snapshots_4 new file mode 100644 index 000000000..d70bdbaa1 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_4 @@ -0,0 +1,497 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2014-08-08T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_5 b/src/restic/testdata/expired_snapshots_5 new file mode 100644 index 000000000..42ed6051a --- /dev/null +++ b/src/restic/testdata/expired_snapshots_5 @@ -0,0 +1,102 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_6 b/src/restic/testdata/expired_snapshots_6 new file mode 100644 index 000000000..69b347d7e --- /dev/null +++ b/src/restic/testdata/expired_snapshots_6 @@ -0,0 +1,17 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_7 b/src/restic/testdata/expired_snapshots_7 new file mode 100644 index 000000000..ca08ff08a --- /dev/null +++ b/src/restic/testdata/expired_snapshots_7 @@ -0,0 +1,52 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_8 b/src/restic/testdata/expired_snapshots_8 new file mode 100644 index 000000000..b0a64d691 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_8 @@ -0,0 +1,152 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/expired_snapshots_9 b/src/restic/testdata/expired_snapshots_9 new file mode 100644 index 000000000..0b577ae74 --- /dev/null +++ b/src/restic/testdata/expired_snapshots_9 @@ -0,0 +1,52 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/src/restic/testdata/filter_snapshots_0 b/src/restic/testdata/filter_snapshots_0 new file mode 100644 index 000000000..4afbe3b55 --- /dev/null +++ b/src/restic/testdata/filter_snapshots_0 @@ -0,0 +1,162 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + } +] \ No newline at end of file diff --git a/src/restic/testdata/filter_snapshots_1 b/src/restic/testdata/filter_snapshots_1 new file mode 100644 index 000000000..c9bd44ddc --- /dev/null +++ b/src/restic/testdata/filter_snapshots_1 @@ -0,0 +1,32 @@ +[ + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "bar", + "username": "root" + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + } +] \ No newline at end of file diff --git a/src/restic/testdata/filter_snapshots_2 b/src/restic/testdata/filter_snapshots_2 new file mode 100644 index 000000000..cae5b39e7 --- /dev/null +++ b/src/restic/testdata/filter_snapshots_2 @@ -0,0 +1,22 @@ +[ + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + } +] \ No newline at end of file diff --git a/src/restic/testdata/filter_snapshots_3 b/src/restic/testdata/filter_snapshots_3 new file mode 100644 index 000000000..d30f4bf37 --- /dev/null +++ b/src/restic/testdata/filter_snapshots_3 @@ -0,0 +1,162 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "bar", + "username": "testuser" + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "bar", + "username": "testuser" + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "foo", + "username": "testuser" + } +] \ No newline at end of file diff --git a/src/restic/testdata/filter_snapshots_4 b/src/restic/testdata/filter_snapshots_4 new file mode 100644 index 000000000..a334bbcb6 --- /dev/null +++ b/src/restic/testdata/filter_snapshots_4 @@ -0,0 +1,22 @@ +[ + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "bar", + "username": "testuser" + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": [ + "/usr", + "/bin" + ], + "hostname": "bar", + "username": "testuser" + } +] \ No newline at end of file diff --git a/src/restic/testdata/filter_snapshots_5 b/src/restic/testdata/filter_snapshots_5 new file mode 100644 index 000000000..cae5b39e7 --- /dev/null +++ b/src/restic/testdata/filter_snapshots_5 @@ -0,0 +1,22 @@ +[ + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": [ + "/usr", + "/sbin" + ], + "hostname": "foo", + "username": "root" + } +] \ No newline at end of file diff --git a/src/restic/testdata/used_blobs_snapshot0 b/src/restic/testdata/used_blobs_snapshot0 new file mode 100644 index 000000000..9443e1e16 --- /dev/null +++ b/src/restic/testdata/used_blobs_snapshot0 @@ -0,0 +1,37 @@ +{"ID":"087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78","Type":"data"} +{"ID":"0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3","Type":"data"} +{"ID":"0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d","Type":"data"} +{"ID":"2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45","Type":"data"} +{"ID":"378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4","Type":"data"} +{"ID":"3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567","Type":"data"} +{"ID":"40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14","Type":"data"} +{"ID":"42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524","Type":"data"} +{"ID":"42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b","Type":"data"} +{"ID":"47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4","Type":"data"} +{"ID":"587045d0ec69e47a3cc91b13c959aa80add9118ecfac47232ea992650f25f0b9","Type":"data"} +{"ID":"615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f","Type":"data"} +{"ID":"63ec5e835e11203bbeef69095523344dd975f1ab52bdbf4a1db7a53914d967ca","Type":"tree"} +{"ID":"714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52","Type":"data"} +{"ID":"80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0","Type":"data"} +{"ID":"86af714d79d18be1c9c0ae23cca9dbd7cef44530e253e80af5bd5c34eab09714","Type":"data"} +{"ID":"8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058","Type":"data"} +{"ID":"8e171f7367d1b68012ed1ceec8f54b7b9b8654ebaf63a760017c34d761b17878","Type":"tree"} +{"ID":"8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc","Type":"data"} +{"ID":"9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c","Type":"data"} +{"ID":"9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa","Type":"data"} +{"ID":"9f2899688d2f23391cfd86e7b6d326a54f352bb294160878178639aab4aa378f","Type":"tree"} +{"ID":"a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784","Type":"data"} +{"ID":"b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df","Type":"data"} +{"ID":"b81870ebe27b98f6b8746349e8ea444c96bf2eaac5dbd6236175150ce579f46b","Type":"tree"} +{"ID":"bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf","Type":"data"} +{"ID":"c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b","Type":"data"} +{"ID":"c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d","Type":"data"} +{"ID":"c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e","Type":"data"} +{"ID":"ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa","Type":"data"} +{"ID":"ce8b656cead478c34060510962daf97cea52abde68bbef7934dd5c5513cf6f3b","Type":"data"} +{"ID":"dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279","Type":"data"} +{"ID":"e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d","Type":"data"} +{"ID":"e752efd93f9850ba0cafbbac01bb283c10095ac923cdb8ff027393001123d406","Type":"tree"} +{"ID":"f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb","Type":"data"} +{"ID":"f75b6460b68d254f2195b08c606672fb55c05fb7bed7e16699b3231104b673ea","Type":"tree"} +{"ID":"fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266","Type":"data"} diff --git a/src/restic/testdata/used_blobs_snapshot1 b/src/restic/testdata/used_blobs_snapshot1 new file mode 100644 index 000000000..3e6b6f395 --- /dev/null +++ b/src/restic/testdata/used_blobs_snapshot1 @@ -0,0 +1,34 @@ +{"ID":"011a951a9796979c2b515ef4209662013bd1f16a20a1b35d1d950d7408bdc8b4","Type":"tree"} +{"ID":"087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78","Type":"data"} +{"ID":"0bad18b7f2d82d7c9cf8e405262ad2f3dbe57928aa242c1070b917042a99072d","Type":"data"} +{"ID":"0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3","Type":"data"} +{"ID":"0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d","Type":"data"} +{"ID":"2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45","Type":"data"} +{"ID":"3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567","Type":"data"} +{"ID":"40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14","Type":"data"} +{"ID":"42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b","Type":"data"} +{"ID":"47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4","Type":"data"} +{"ID":"4b2e91022c34c756b7bd8ece046a2bab6f0dcad89f46c52d1f84cd48e8da55df","Type":"tree"} +{"ID":"6416bc2321cdeb8758188af2b3925f2c82ffde014bf53b7a69c0f113a5c460fe","Type":"data"} +{"ID":"714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52","Type":"data"} +{"ID":"80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0","Type":"data"} +{"ID":"83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06","Type":"data"} +{"ID":"8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058","Type":"data"} +{"ID":"8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc","Type":"data"} +{"ID":"907acef01e05c3e0140858423e9284ddd3d64145ba8b0c3293371c5c7ab3d6b7","Type":"data"} +{"ID":"9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c","Type":"data"} +{"ID":"9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa","Type":"data"} +{"ID":"a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784","Type":"data"} +{"ID":"b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df","Type":"data"} +{"ID":"b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c","Type":"data"} +{"ID":"bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf","Type":"data"} +{"ID":"c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d","Type":"data"} +{"ID":"c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e","Type":"data"} +{"ID":"ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa","Type":"data"} +{"ID":"cb8001715217b4f6960aa24c1abb4b60a20c10f23abc1e5f69e0f5436bd788c8","Type":"data"} +{"ID":"d39c4c264e01ec47b0386da3775c6b0cc337974627ff55792938cca4895ac6c4","Type":"data"} +{"ID":"dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279","Type":"data"} +{"ID":"e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d","Type":"data"} +{"ID":"e791912a7fad8954c764fae41d2958d2feeae2278e403429add9119ab43a36f5","Type":"tree"} +{"ID":"f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb","Type":"data"} +{"ID":"fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266","Type":"data"} diff --git a/src/restic/testdata/used_blobs_snapshot2 b/src/restic/testdata/used_blobs_snapshot2 new file mode 100644 index 000000000..b57f4a417 --- /dev/null +++ b/src/restic/testdata/used_blobs_snapshot2 @@ -0,0 +1,9 @@ +{"ID":"35e13e123748cd27d1634c4e07e5ff2fc86901b09b215f3125331d1226c782be","Type":"tree"} +{"ID":"378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4","Type":"data"} +{"ID":"42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524","Type":"data"} +{"ID":"47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4","Type":"data"} +{"ID":"615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f","Type":"data"} +{"ID":"83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06","Type":"data"} +{"ID":"9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c","Type":"data"} +{"ID":"b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c","Type":"data"} +{"ID":"c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b","Type":"data"} diff --git a/src/restic/testing.go b/src/restic/testing.go index fa2ab649d..e1d6bf610 100644 --- a/src/restic/testing.go +++ b/src/restic/testing.go @@ -1,6 +1,7 @@ package restic import ( + "encoding/json" "fmt" "io" "math/rand" @@ -13,86 +14,23 @@ import ( "github.com/restic/chunker" ) -type randReader struct { - rnd *rand.Rand - buf []byte -} - -func newRandReader(rnd *rand.Rand) io.Reader { - return &randReader{rnd: rnd, buf: make([]byte, 0, 7)} -} - -func (rd *randReader) read(p []byte) (n int, err error) { - if len(p)%7 != 0 { - panic("invalid buffer length, not multiple of 7") - } - - rnd := rd.rnd - for i := 0; i < len(p); i += 7 { - val := rnd.Int63() - - p[i+0] = byte(val >> 0) - p[i+1] = byte(val >> 8) - p[i+2] = byte(val >> 16) - p[i+3] = byte(val >> 24) - p[i+4] = byte(val >> 32) - p[i+5] = byte(val >> 40) - p[i+6] = byte(val >> 48) - } - - return len(p), nil -} - -func (rd *randReader) Read(p []byte) (int, error) { - // first, copy buffer to p - pos := copy(p, rd.buf) - copy(rd.buf, rd.buf[pos:]) - - // shorten buf and p accordingly - rd.buf = rd.buf[:len(rd.buf)-pos] - p = p[pos:] - - // if this is enough to fill p, return - if len(p) == 0 { - return pos, nil - } - - // load multiple of 7 byte - l := (len(p) / 7) * 7 - n, err := rd.read(p[:l]) - pos += n - if err != nil { - return pos, err - } - p = p[n:] - - // load 7 byte to temp buffer - rd.buf = rd.buf[:7] - n, err = rd.read(rd.buf) - if err != nil { - return pos, err - } - - // copy the remaining bytes from the buffer to p - n = copy(p, rd.buf) - pos += n - - // save the remaining bytes in rd.buf - n = copy(rd.buf, rd.buf[n:]) - rd.buf = rd.buf[:n] - - return pos, nil -} - // fakeFile returns a reader which yields deterministic pseudo-random data. func fakeFile(t testing.TB, seed, size int64) io.Reader { - return io.LimitReader(newRandReader(rand.New(rand.NewSource(seed))), size) + return io.LimitReader(repository.NewRandReader(rand.New(rand.NewSource(seed))), size) +} + +type fakeFileSystem struct { + t testing.TB + repo *repository.Repository + knownBlobs backend.IDSet + duplication float32 } // saveFile reads from rd and saves the blobs in the repository. The list of // IDs is returned. -func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs backend.IDs) { - ch := chunker.New(rd, repo.Config.ChunkerPolynomial) +func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs backend.IDs) { + blobs = backend.IDs{} + ch := chunker.New(rd, fs.repo.Config.ChunkerPolynomial) for { chunk, err := ch.Next(getBuf()) @@ -101,47 +39,107 @@ func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs ba } if err != nil { - t.Fatalf("unabel to save chunk in repo: %v", err) + fs.t.Fatalf("unable to save chunk in repo: %v", err) } - id, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil) - if err != nil { - t.Fatalf("error saving chunk: %v", err) + id := backend.Hash(chunk.Data) + if !fs.blobIsKnown(id, pack.Data) { + _, err := fs.repo.SaveAndEncrypt(pack.Data, chunk.Data, &id) + if err != nil { + fs.t.Fatalf("error saving chunk: %v", err) + } + + fs.knownBlobs.Insert(id) } + freeBuf(chunk.Data) + blobs = append(blobs, id) } return blobs } -const maxFileSize = 1500000 -const maxSeed = 100 +const ( + maxFileSize = 1500000 + maxSeed = 32 + maxNodes = 32 +) + +func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, backend.ID) { + data, err := json.Marshal(tree) + if err != nil { + fs.t.Fatalf("json.Marshal(tree) returned error: %v", err) + return false, backend.ID{} + } + data = append(data, '\n') + + id := backend.Hash(data) + return fs.blobIsKnown(id, pack.Tree), id + +} + +func (fs fakeFileSystem) blobIsKnown(id backend.ID, t pack.BlobType) bool { + if rand.Float32() < fs.duplication { + return false + } + + if fs.knownBlobs.Has(id) { + return true + } + + if fs.repo.Index().Has(id, t) { + return true + } + + fs.knownBlobs.Insert(id) + return false +} // saveTree saves a tree of fake files in the repo and returns the ID. -func saveTree(t testing.TB, repo *repository.Repository, seed int64) backend.ID { +func (fs fakeFileSystem) saveTree(seed int64, depth int) backend.ID { rnd := rand.NewSource(seed) - numNodes := int(rnd.Int63() % 64) - t.Logf("create %v nodes", numNodes) + numNodes := int(rnd.Int63() % maxNodes) var tree Tree for i := 0; i < numNodes; i++ { - seed := rnd.Int63() % maxSeed - size := rnd.Int63() % maxFileSize - node := &Node{ - Name: fmt.Sprintf("file-%v", seed), - Type: "file", - Mode: 0644, - Size: uint64(size), + // randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4). + if depth > 1 && rnd.Int63()%4 == 0 { + treeSeed := rnd.Int63() % maxSeed + id := fs.saveTree(treeSeed, depth-1) + + node := &Node{ + Name: fmt.Sprintf("dir-%v", treeSeed), + Type: "dir", + Mode: 0755, + Subtree: &id, + } + + tree.Nodes = append(tree.Nodes, node) + continue } - node.Content = saveFile(t, repo, fakeFile(t, seed, size)) + fileSeed := rnd.Int63() % maxSeed + fileSize := (maxFileSize / maxSeed) * fileSeed + + node := &Node{ + Name: fmt.Sprintf("file-%v", fileSeed), + Type: "file", + Mode: 0644, + Size: uint64(fileSize), + } + + node.Content = fs.saveFile(fakeFile(fs.t, fileSeed, fileSize)) tree.Nodes = append(tree.Nodes, node) } - id, err := repo.SaveJSON(pack.Tree, tree) + if known, id := fs.treeIsKnown(&tree); known { + return id + } + + id, err := fs.repo.SaveJSON(pack.Tree, tree) if err != nil { - t.Fatal(err) + fs.t.Fatal(err) } return id @@ -149,8 +147,13 @@ func saveTree(t testing.TB, repo *repository.Repository, seed int64) backend.ID // TestCreateSnapshot creates a snapshot filled with fake data. The // fake data is generated deterministically from the timestamp `at`, which is -// also used as the snapshot's timestamp. -func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) backend.ID { +// also used as the snapshot's timestamp. The tree's depth can be specified +// with the parameter depth. The parameter duplication is a probability that +// the same blob will saved again. +func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time, depth int, duplication float32) *Snapshot { + seed := at.Unix() + t.Logf("create fake snapshot at %s with seed %d", at, seed) + fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) snapshot, err := NewSnapshot([]string{fakedir}) if err != nil { @@ -158,7 +161,14 @@ func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) } snapshot.Time = at - treeID := saveTree(t, repo, at.UnixNano()) + fs := fakeFileSystem{ + t: t, + repo: repo, + knownBlobs: backend.NewIDSet(), + duplication: duplication, + } + + treeID := fs.saveTree(seed, depth) snapshot.Tree = &treeID id, err := repo.SaveJSONUnpacked(backend.Snapshot, snapshot) @@ -166,6 +176,8 @@ func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) t.Fatal(err) } + snapshot.id = &id + t.Logf("saved snapshot %v", id.Str()) err = repo.Flush() @@ -178,5 +190,22 @@ func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) t.Fatal(err) } - return id + return snapshot +} + +// TestResetRepository removes all packs and indexes from the repository. +func TestResetRepository(t testing.TB, repo *repository.Repository) { + done := make(chan struct{}) + defer close(done) + + for _, tpe := range []backend.Type{backend.Snapshot, backend.Index, backend.Data} { + for id := range repo.Backend().List(tpe, done) { + err := repo.Backend().Remove(tpe, id) + if err != nil { + t.Errorf("removing %v (%v) failed: %v", id[0:12], tpe, err) + } + } + } + + repo.SetIndex(repository.NewMasterIndex()) } diff --git a/src/restic/testing_test.go b/src/restic/testing_test.go index 8243a01a8..3c5ea5a6f 100644 --- a/src/restic/testing_test.go +++ b/src/restic/testing_test.go @@ -10,14 +10,17 @@ import ( var testSnapshotTime = time.Unix(1460289341, 207401672) -const testCreateSnapshots = 3 +const ( + testCreateSnapshots = 3 + testDepth = 2 +) func TestCreateSnapshot(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() for i := 0; i < testCreateSnapshots; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second)) + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0) } snapshots, err := restic.LoadAllSnapshots(repo) @@ -42,30 +45,17 @@ func TestCreateSnapshot(t *testing.T) { t.Fatalf("snapshot has zero tree ID") } - chkr := checker.New(repo) + checker.TestCheckRepo(t, repo) +} - hints, errs := chkr.LoadIndex() - if len(errs) != 0 { - t.Fatalf("errors loading index: %v", errs) - } +func BenchmarkCreateSnapshot(b *testing.B) { + repo, cleanup := repository.TestRepository(b) + defer cleanup() - if len(hints) != 0 { - t.Fatalf("errors loading index: %v", hints) - } + b.ResetTimer() - done := make(chan struct{}) - defer close(done) - errChan := make(chan error) - go chkr.Structure(errChan, done) - - for err := range errChan { - t.Error(err) - } - - errChan = make(chan error) - go chkr.ReadData(nil, errChan, done) - - for err := range errChan { - t.Error(err) + for i := 0; i < b.N; i++ { + restic.TestCreateSnapshot(b, repo, testSnapshotTime, testDepth, 0) + restic.TestResetRepository(b, repo) } } diff --git a/src/restic/types/repository.go b/src/restic/types/repository.go new file mode 100644 index 000000000..d13d93333 --- /dev/null +++ b/src/restic/types/repository.go @@ -0,0 +1,20 @@ +package types + +import ( + "restic/backend" + "restic/pack" +) + +// Repository manages encrypted and packed data stored in a backend. +type Repository interface { + LoadJSONUnpacked(backend.Type, backend.ID, interface{}) error + SaveJSONUnpacked(backend.Type, interface{}) (backend.ID, error) + + Lister +} + +// Lister combines lists packs in a repo and blobs in a pack. +type Lister interface { + List(backend.Type, <-chan struct{}) <-chan backend.ID + ListPack(backend.ID) ([]pack.Blob, int64, error) +}