From ce3acbd30a8638bb6b43e32029aae73a95a27869 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Thu, 11 May 2017 21:03:28 +0200 Subject: [PATCH 1/4] Remove unused SnapshotFilter and FilterSnapshots(). --- src/restic/snapshot_filter.go | 34 -------------- src/restic/snapshot_filter_test.go | 71 ------------------------------ 2 files changed, 105 deletions(-) diff --git a/src/restic/snapshot_filter.go b/src/restic/snapshot_filter.go index 69b9e44f8..559150a5e 100644 --- a/src/restic/snapshot_filter.go +++ b/src/restic/snapshot_filter.go @@ -25,40 +25,6 @@ func (sn Snapshots) Swap(i, j int) { sn[i], sn[j] = sn[j], sn[i] } -// SnapshotFilter configures criteria for filtering snapshots before an -// ExpirePolicy can be applied. -type SnapshotFilter struct { - Hostname string - Username string - Paths []string - Tags []string -} - -// FilterSnapshots returns the snapshots from s which match the filter f. -func FilterSnapshots(s Snapshots, f SnapshotFilter) (result Snapshots) { - for _, snap := range s { - if f.Hostname != "" && f.Hostname != snap.Hostname { - continue - } - - if f.Username != "" && f.Username != snap.Username { - continue - } - - if f.Paths != nil && !reflect.DeepEqual(f.Paths, snap.Paths) { - continue - } - - if !snap.HasTags(f.Tags) { - continue - } - - result = append(result, snap) - } - - return result -} - // ExpirePolicy configures which snapshots should be automatically removed. type ExpirePolicy struct { Last int // keep the last n snapshots diff --git a/src/restic/snapshot_filter_test.go b/src/restic/snapshot_filter_test.go index fbeef229d..c31eeab57 100644 --- a/src/restic/snapshot_filter_test.go +++ b/src/restic/snapshot_filter_test.go @@ -7,7 +7,6 @@ import ( "path/filepath" "reflect" "restic" - "sort" "testing" "time" ) @@ -21,76 +20,6 @@ func parseTimeUTC(s string) time.Time { return t.UTC() } -var testFilterSnapshots = restic.Snapshots{ - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-01 01:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "bar", Username: "testuser", Time: parseTimeUTC("2016-01-01 01:03:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-03 07:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "bar", Username: "testuser", Time: parseTimeUTC("2016-01-01 07:08:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 10:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 11:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:24:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:28:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 12:30:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test", "foo", "bar"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-04 16:23:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"test", "test2"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-05 09:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-06 08:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"fox"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-07 10:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"fox"}}, - {Hostname: "foo", Username: "root", Time: parseTimeUTC("2016-01-08 20:02:03"), Paths: []string{"/usr", "/sbin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "root", Time: parseTimeUTC("2016-01-09 21:02:03"), Paths: []string{"/usr", "/sbin"}, Tags: []string{"fox"}}, - {Hostname: "bar", Username: "root", Time: parseTimeUTC("2016-01-12 21:02:03"), Paths: []string{"/usr", "/sbin"}, Tags: []string{"foo"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-12 21:08:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"bar"}}, - {Hostname: "foo", Username: "testuser", Time: parseTimeUTC("2016-01-18 12:02:03"), Paths: []string{"/usr", "/bin"}, Tags: []string{"bar"}}, -} - -var filterTests = []restic.SnapshotFilter{ - {Hostname: "foo"}, - {Username: "root"}, - {Hostname: "foo", Username: "root"}, - {Paths: []string{"/usr", "/bin"}}, - {Hostname: "bar", Paths: []string{"/usr", "/bin"}}, - {Hostname: "foo", Username: "root", Paths: []string{"/usr", "/sbin"}}, - {Tags: []string{"foo"}}, - {Tags: []string{"fox"}, Username: "root"}, - {Tags: []string{"foo", "test"}}, - {Tags: []string{"foo", "test2"}}, -} - -func TestFilterSnapshots(t *testing.T) { - sort.Sort(testFilterSnapshots) - - for i, f := range filterTests { - res := restic.FilterSnapshots(testFilterSnapshots, f) - - goldenFilename := filepath.Join("testdata", fmt.Sprintf("filter_snapshots_%d", i)) - - if *updateGoldenFiles { - buf, err := json.MarshalIndent(res, "", " ") - if err != nil { - t.Fatalf("error marshaling result: %v", err) - } - - if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { - t.Fatalf("unable to update golden file: %v", err) - } - } - - buf, err := ioutil.ReadFile(goldenFilename) - if err != nil { - t.Errorf("error loading golden file %v: %v", goldenFilename, err) - continue - } - - var want restic.Snapshots - err = json.Unmarshal(buf, &want) - - if !reflect.DeepEqual(res, want) { - t.Errorf("test %v: wrong result, want:\n %#v\ngot:\n %#v", i, want, res) - continue - } - } -} - func TestExpireSnapshotOps(t *testing.T) { data := []struct { expectEmpty bool From 929f90344ef8c145694fe5860615dbd49dc43b68 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Thu, 11 May 2017 22:26:44 +0200 Subject: [PATCH 2/4] Change backup policy to be inclusive, meaning all given policies are evaluated for each snapshot, thereby making sure that each keep-* is able to retain its most recent snapshot. Thereby insuring that weeklies keep Sundays around and monthlies keep the last day of the month around. Added testcase to make sure when multiple --keep-tags are given, ALL of them need to match. --- doc/manual.rst | 11 ++ src/restic/snapshot_filter.go | 147 ++++++------------- src/restic/snapshot_filter_test.go | 81 +++++----- src/restic/testdata/policy_keep_snapshots_0 | 13 +- src/restic/testdata/policy_keep_snapshots_10 | 12 +- src/restic/testdata/policy_keep_snapshots_13 | 17 +-- src/restic/testdata/policy_keep_snapshots_15 | 20 +-- src/restic/testdata/policy_keep_snapshots_17 | 22 +-- src/restic/testdata/policy_keep_snapshots_18 | 11 +- src/restic/testdata/policy_keep_snapshots_19 | 11 ++ src/restic/testdata/policy_keep_snapshots_3 | 13 +- src/restic/testdata/policy_keep_snapshots_4 | 13 +- src/restic/testdata/policy_keep_snapshots_9 | 22 +-- 13 files changed, 154 insertions(+), 239 deletions(-) create mode 100644 src/restic/testdata/policy_keep_snapshots_19 diff --git a/doc/manual.rst b/doc/manual.rst index cdf166c7a..10cd1282b 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -788,6 +788,9 @@ All the ``--keep-*`` options above only count hours/days/weeks/months/years which have a snapshot, so those without a snapshot are ignored. +All snapshots are evaluated counted against all matching keep-* counts. A +single snapshot on 30-09-2017 (Sun) will count as a daily, weekly and monthly. + Let's explain this with an example: Suppose you have only made a backup on each Sunday for 12 weeks. Then ``forget --keep-daily 4`` will keep the last four snapshots for the last four Sundays, but remove the rest. @@ -796,6 +799,14 @@ is a safety feature: it prevents restic from removing many snapshots when no new ones are created. If it was implemented otherwise, running ``forget --keep-daily 4`` on a Friday would remove all snapshots! +Another example: Suppose you make daily backups for 100 years. Then +``forget --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 75`` +will keep the most recent 7 daily snapshots, then 4 (remember, 7 dailies +already include a week!) last-day-of-the-weeks and 11 or 12 +last-day-of-the-months. (11 or 12 depends if the 5 weeklies cross a month). +And ofcourse 75 last-day-of-the-year snapshots. All other snapshots are +removed. + Autocompletion -------------- diff --git a/src/restic/snapshot_filter.go b/src/restic/snapshot_filter.go index 559150a5e..f56334428 100644 --- a/src/restic/snapshot_filter.go +++ b/src/restic/snapshot_filter.go @@ -1,7 +1,6 @@ package restic import ( - "fmt" "reflect" "sort" "time" @@ -52,18 +51,6 @@ func (e ExpirePolicy) Empty() bool { return reflect.DeepEqual(e, empty) } -// filter is used to split a list of snapshots into those to keep and those to -// remove according to a policy. -type filter struct { - Unprocessed Snapshots - Remove Snapshots - Keep Snapshots -} - -func (f filter) String() string { - return fmt.Sprintf("", len(f.Unprocessed), len(f.Keep), len(f.Remove)) -} - // ymdh returns an integer in the form YYYYMMDDHH. func ymdh(d time.Time) int { return d.Year()*1000000 + int(d.Month())*10000 + d.Day()*100 + d.Hour() @@ -90,84 +77,16 @@ func y(d time.Time) int { return d.Year() } -// apply moves snapshots from Unprocess to either Keep or Remove. It sorts the -// snapshots into buckets according to the return value of fn, and then moves -// the newest snapshot in each bucket to Keep and all others to Remove. When -// max snapshots were found, processing stops. -func (f *filter) apply(fn func(time.Time) int, max int) { - if max == 0 || len(f.Unprocessed) == 0 { - return - } +var a int - sameBucket := Snapshots{} - lastBucket := fn(f.Unprocessed[0].Time) - - for len(f.Unprocessed) > 0 { - cur := f.Unprocessed[0] - - bucket := fn(cur.Time) - - // if the snapshots are from a new bucket, forget all but the first - // (=last in time) snapshot from the previous bucket. - if bucket != lastBucket { - f.Keep = append(f.Keep, sameBucket[0]) - f.Remove = append(f.Remove, sameBucket[1:]...) - - sameBucket = Snapshots{} - lastBucket = bucket - max-- - - if max == 0 { - return - } - } - - // collect all snapshots for the current bucket - sameBucket = append(sameBucket, cur) - f.Unprocessed = f.Unprocessed[1:] - } - - // if we have leftovers, process them too. - if len(sameBucket) > 0 { - f.Keep = append(f.Keep, sameBucket[0]) - f.Remove = append(f.Remove, sameBucket[1:]...) - } +// always retuns a unique number for d. +func always(d time.Time) int { + a++ + return a } -// keepTags marks the snapshots which have all tags as to be kept. -func (f *filter) keepTags(tags []string) { - if len(tags) == 0 { - return - } - - unprocessed := f.Unprocessed[:0] - for _, sn := range f.Unprocessed { - if sn.HasTags(tags) { - f.Keep = append(f.Keep, sn) - continue - } - unprocessed = append(unprocessed, sn) - } - f.Unprocessed = unprocessed -} - -// keepLast marks the last n snapshots as to be kept. -func (f *filter) keepLast(n int) { - if n > len(f.Unprocessed) { - n = len(f.Unprocessed) - } - - f.Keep = append(f.Keep, f.Unprocessed[:n]...) - f.Unprocessed = f.Unprocessed[n:] -} - -// finish moves all remaining snapshots to remove. -func (f *filter) finish() { - f.Remove = append(f.Remove, f.Unprocessed...) -} - -// ApplyPolicy runs returns the snapshots from s that are to be deleted according -// to the policy p. s is sorted in the process. +// ApplyPolicy returns the snapshots from list that are to be kept and removed +// according to the policy p. list is sorted in the process. func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { sort.Sort(list) @@ -179,20 +98,46 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { return list, remove } - f := filter{ - Unprocessed: list, - Remove: Snapshots{}, - Keep: Snapshots{}, + var buckets = [6]struct { + Count int + bucker func(d time.Time) int + Last int + }{ + {p.Last, always, -1}, + {p.Hourly, ymdh, -1}, + {p.Daily, ymd, -1}, + {p.Weekly, yw, -1}, + {p.Monthly, ym, -1}, + {p.Yearly, y, -1}, } - f.keepTags(p.Tags) - f.keepLast(p.Last) - f.apply(ymdh, p.Hourly) - f.apply(ymd, p.Daily) - f.apply(yw, p.Weekly) - f.apply(ym, p.Monthly) - f.apply(y, p.Yearly) - f.finish() + for _, cur := range list { + var keep_snap bool - return f.Keep, f.Remove + // Tags are handled specially as they are not counted. + if len(p.Tags) > 0 { + if cur.HasTags(p.Tags) { + keep_snap = true + } + } + // Now update the other buckets and see if they have some counts left. + for i, b := range buckets { + if b.Count > 0 { + val := b.bucker(cur.Time) + if val != b.Last { + keep_snap = true + buckets[i].Last = val + buckets[i].Count-- + } + } + } + + if keep_snap { + keep = append(keep, cur) + } else { + remove = append(remove, cur) + } + } + + return keep, remove } diff --git a/src/restic/snapshot_filter_test.go b/src/restic/snapshot_filter_test.go index c31eeab57..d8bbd4ec8 100644 --- a/src/restic/snapshot_filter_test.go +++ b/src/restic/snapshot_filter_test.go @@ -77,8 +77,8 @@ var testExpireSnapshots = restic.Snapshots{ {Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}}, {Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}}, {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-13 10:20:30")}, - {Time: parseTimeUTC("2014-11-15 10:20:30")}, + {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"bar"}}, + {Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}}, {Time: parseTimeUTC("2014-11-18 10:20:30")}, {Time: parseTimeUTC("2014-11-20 10:20:30")}, {Time: parseTimeUTC("2014-11-21 10:20:30")}, @@ -164,57 +164,58 @@ var expireTests = []restic.ExpirePolicy{ {Yearly: 10}, {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, {Tags: []string{"foo"}}, + {Tags: []string{"foo", "bar"}}, } func TestApplyPolicy(t *testing.T) { for i, p := range expireTests { - keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) - t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", - i, len(keep), len(remove), len(testExpireSnapshots), p) + t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", + i, len(keep), len(remove), len(testExpireSnapshots), p) - if len(keep)+len(remove) != len(testExpireSnapshots) { - t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", - i, len(keep)+len(remove), len(testExpireSnapshots)) - } + if len(keep)+len(remove) != len(testExpireSnapshots) { + t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", + i, len(keep)+len(remove), len(testExpireSnapshots)) + } - if p.Sum() > 0 && len(keep) > p.Sum() { - t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) - } + if p.Sum() > 0 && len(keep) > p.Sum() { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + p.Sum(), len(keep)) + } - for _, sn := range keep { - t.Logf("test %d: keep snapshot at %v %s\n", i, sn.Time, sn.Tags) - } - for _, sn := range remove { - t.Logf("test %d: forget snapshot at %v %s\n", i, sn.Time, sn.Tags) - } + for _, sn := range keep { + t.Logf("test %d: keep snapshot at %v %s\n", i, sn.Time, sn.Tags) + } + for _, sn := range remove { + t.Logf("test %d: forget snapshot at %v %s\n", i, sn.Time, sn.Tags) + } - goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) + goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) - if *updateGoldenFiles { - buf, err := json.MarshalIndent(keep, "", " ") + if *updateGoldenFiles { + buf, err := json.MarshalIndent(keep, "", " ") + if err != nil { + t.Fatalf("error marshaling result: %v", err) + } + + if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { + t.Fatalf("unable to update golden file: %v", err) + } + } + + buf, err := ioutil.ReadFile(goldenFilename) if err != nil { - t.Fatalf("error marshaling result: %v", err) + t.Fatalf("error loading golden file %v: %v", goldenFilename, err) } - if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { - t.Fatalf("unable to update golden file: %v", err) + var want restic.Snapshots + err = json.Unmarshal(buf, &want) + + if !reflect.DeepEqual(keep, want) { + t.Fatalf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) } - } - - buf, err := ioutil.ReadFile(goldenFilename) - if err != nil { - t.Errorf("error loading golden file %v: %v", goldenFilename, err) - continue - } - - var want restic.Snapshots - err = json.Unmarshal(buf, &want) - - if !reflect.DeepEqual(keep, want) { - t.Errorf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) - continue - } + }) } } diff --git a/src/restic/testdata/policy_keep_snapshots_0 b/src/restic/testdata/policy_keep_snapshots_0 index 261d70719..952f8c02e 100644 --- a/src/restic/testdata/policy_keep_snapshots_0 +++ b/src/restic/testdata/policy_keep_snapshots_0 @@ -317,12 +317,19 @@ { "time": "2014-11-15T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2014-11-13T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "bar" + ] }, { "time": "2014-11-13T10:20:30Z", @@ -536,4 +543,4 @@ "tree": null, "paths": null } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_10 b/src/restic/testdata/policy_keep_snapshots_10 index 72ae755ce..e2d61b68b 100644 --- a/src/restic/testdata/policy_keep_snapshots_10 +++ b/src/restic/testdata/policy_keep_snapshots_10 @@ -9,11 +9,6 @@ "tree": null, "paths": null }, - { - "time": "2016-01-12T21:02:03Z", - "tree": null, - "paths": null - }, { "time": "2016-01-09T21:02:03Z", "tree": null, @@ -53,10 +48,5 @@ "time": "2016-01-01T07:08:03Z", "tree": null, "paths": null - }, - { - "time": "2015-11-22T10:20:30Z", - "tree": null, - "paths": null } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_13 b/src/restic/testdata/policy_keep_snapshots_13 index 93a52ad8c..abf4ed809 100644 --- a/src/restic/testdata/policy_keep_snapshots_13 +++ b/src/restic/testdata/policy_keep_snapshots_13 @@ -14,24 +14,9 @@ "tree": null, "paths": null }, - { - "time": "2016-01-08T20:02:03Z", - "tree": null, - "paths": null - }, { "time": "2016-01-03T07:02:03Z", "tree": null, "paths": null - }, - { - "time": "2015-11-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-15T10:20:30Z", - "tree": null, - "paths": null } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_15 b/src/restic/testdata/policy_keep_snapshots_15 index 973e951ef..51bed0e6e 100644 --- a/src/restic/testdata/policy_keep_snapshots_15 +++ b/src/restic/testdata/policy_keep_snapshots_15 @@ -9,16 +9,6 @@ "tree": null, "paths": null }, - { - "time": "2016-01-09T21:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-03T07:02:03Z", - "tree": null, - "paths": null - }, { "time": "2015-11-22T10:20:30Z", "tree": null, @@ -43,13 +33,5 @@ "time": "2014-11-22T10:20:30Z", "tree": null, "paths": null - }, - { - "time": "2014-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_17 b/src/restic/testdata/policy_keep_snapshots_17 index 553c18d89..8eb4ce6f8 100644 --- a/src/restic/testdata/policy_keep_snapshots_17 +++ b/src/restic/testdata/policy_keep_snapshots_17 @@ -34,16 +34,6 @@ "tree": null, "paths": null }, - { - "time": "2016-01-04T16:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-03T07:02:03Z", - "tree": null, - "paths": null - }, { "time": "2015-11-22T10:20:30Z", "tree": null, @@ -54,19 +44,9 @@ "tree": null, "paths": null }, - { - "time": "2015-09-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-22T10:20:30Z", - "tree": null, - "paths": null - }, { "time": "2014-11-22T10:20:30Z", "tree": null, "paths": null } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_18 b/src/restic/testdata/policy_keep_snapshots_18 index 5898b0950..caf2b746e 100644 --- a/src/restic/testdata/policy_keep_snapshots_18 +++ b/src/restic/testdata/policy_keep_snapshots_18 @@ -1,4 +1,13 @@ [ + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, { "time": "2014-11-13T10:20:30Z", "tree": null, @@ -111,4 +120,4 @@ "foo" ] } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_19 b/src/restic/testdata/policy_keep_snapshots_19 new file mode 100644 index 000000000..177e8b400 --- /dev/null +++ b/src/restic/testdata/policy_keep_snapshots_19 @@ -0,0 +1,11 @@ +[ + { + "time": "2014-11-15T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + } +] diff --git a/src/restic/testdata/policy_keep_snapshots_3 b/src/restic/testdata/policy_keep_snapshots_3 index 261d70719..952f8c02e 100644 --- a/src/restic/testdata/policy_keep_snapshots_3 +++ b/src/restic/testdata/policy_keep_snapshots_3 @@ -317,12 +317,19 @@ { "time": "2014-11-15T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2014-11-13T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "bar" + ] }, { "time": "2014-11-13T10:20:30Z", @@ -536,4 +543,4 @@ "tree": null, "paths": null } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_4 b/src/restic/testdata/policy_keep_snapshots_4 index 261d70719..952f8c02e 100644 --- a/src/restic/testdata/policy_keep_snapshots_4 +++ b/src/restic/testdata/policy_keep_snapshots_4 @@ -317,12 +317,19 @@ { "time": "2014-11-15T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "foo", + "bar" + ] }, { "time": "2014-11-13T10:20:30Z", "tree": null, - "paths": null + "paths": null, + "tags": [ + "bar" + ] }, { "time": "2014-11-13T10:20:30Z", @@ -536,4 +543,4 @@ "tree": null, "paths": null } -] \ No newline at end of file +] diff --git a/src/restic/testdata/policy_keep_snapshots_9 b/src/restic/testdata/policy_keep_snapshots_9 index 0b577ae74..776816b08 100644 --- a/src/restic/testdata/policy_keep_snapshots_9 +++ b/src/restic/testdata/policy_keep_snapshots_9 @@ -28,25 +28,5 @@ "time": "2016-01-07T10:02:03Z", "tree": null, "paths": null - }, - { - "time": "2016-01-06T08:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-05T09:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T16:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-03T07:02:03Z", - "tree": null, - "paths": null } -] \ No newline at end of file +] From 103a491ac044b7a93d652a4b34fe14aefd6f5267 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Thu, 11 May 2017 22:38:12 +0200 Subject: [PATCH 3/4] Make houndci-bot happy. --- src/restic/snapshot_filter.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/restic/snapshot_filter.go b/src/restic/snapshot_filter.go index f56334428..5461f18ff 100644 --- a/src/restic/snapshot_filter.go +++ b/src/restic/snapshot_filter.go @@ -112,12 +112,12 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { } for _, cur := range list { - var keep_snap bool + var keepSnap bool // Tags are handled specially as they are not counted. if len(p.Tags) > 0 { if cur.HasTags(p.Tags) { - keep_snap = true + keepSnap = true } } // Now update the other buckets and see if they have some counts left. @@ -125,14 +125,14 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { if b.Count > 0 { val := b.bucker(cur.Time) if val != b.Last { - keep_snap = true + keepSnap = true buckets[i].Last = val buckets[i].Count-- } } } - if keep_snap { + if keepSnap { keep = append(keep, cur) } else { remove = append(remove, cur) From 7fffd408af218b1af50da612b4bc7828ce2d10a6 Mon Sep 17 00:00:00 2001 From: Pauline Middelink Date: Mon, 15 May 2017 08:53:59 +0200 Subject: [PATCH 4/4] Do not use singleton counter. Revert change for running the numbered tests as subtests. --- doc/manual.rst | 6 +- src/restic/snapshot_filter.go | 7 +- src/restic/snapshot_filter_test.go | 84 ++++++++++---------- src/restic/testdata/policy_keep_snapshots_0 | 10 +-- src/restic/testdata/policy_keep_snapshots_10 | 2 +- src/restic/testdata/policy_keep_snapshots_13 | 2 +- src/restic/testdata/policy_keep_snapshots_15 | 2 +- src/restic/testdata/policy_keep_snapshots_17 | 2 +- src/restic/testdata/policy_keep_snapshots_18 | 2 +- src/restic/testdata/policy_keep_snapshots_19 | 2 +- src/restic/testdata/policy_keep_snapshots_3 | 10 +-- src/restic/testdata/policy_keep_snapshots_4 | 10 +-- src/restic/testdata/policy_keep_snapshots_5 | 2 +- src/restic/testdata/policy_keep_snapshots_8 | 2 +- src/restic/testdata/policy_keep_snapshots_9 | 2 +- 15 files changed, 71 insertions(+), 74 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 10cd1282b..7a7221046 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -789,7 +789,7 @@ hours/days/weeks/months/years which have a snapshot, so those without a snapshot are ignored. All snapshots are evaluated counted against all matching keep-* counts. A -single snapshot on 30-09-2017 (Sun) will count as a daily, weekly and monthly. +single snapshot on 2017-09-30 (Sun) will count as a daily, weekly and monthly. Let's explain this with an example: Suppose you have only made a backup on each Sunday for 12 weeks. Then ``forget --keep-daily 4`` will keep @@ -803,8 +803,8 @@ Another example: Suppose you make daily backups for 100 years. Then ``forget --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 75`` will keep the most recent 7 daily snapshots, then 4 (remember, 7 dailies already include a week!) last-day-of-the-weeks and 11 or 12 -last-day-of-the-months. (11 or 12 depends if the 5 weeklies cross a month). -And ofcourse 75 last-day-of-the-year snapshots. All other snapshots are +last-day-of-the-months (11 or 12 depends if the 5 weeklies cross a month). +And finally 75 last-day-of-the-year snapshots. All other snapshots are removed. Autocompletion diff --git a/src/restic/snapshot_filter.go b/src/restic/snapshot_filter.go index 5461f18ff..450b2acc2 100644 --- a/src/restic/snapshot_filter.go +++ b/src/restic/snapshot_filter.go @@ -77,12 +77,9 @@ func y(d time.Time) int { return d.Year() } -var a int - -// always retuns a unique number for d. +// always returns a unique number for d. func always(d time.Time) int { - a++ - return a + return int(d.UnixNano()) } // ApplyPolicy returns the snapshots from list that are to be kept and removed diff --git a/src/restic/snapshot_filter_test.go b/src/restic/snapshot_filter_test.go index d8bbd4ec8..05e25fab6 100644 --- a/src/restic/snapshot_filter_test.go +++ b/src/restic/snapshot_filter_test.go @@ -57,7 +57,7 @@ var testExpireSnapshots = restic.Snapshots{ {Time: parseTimeUTC("2014-08-10 10:20:30")}, {Time: parseTimeUTC("2014-08-12 10:20:30")}, {Time: parseTimeUTC("2014-08-13 10:20:30")}, - {Time: parseTimeUTC("2014-08-13 10:20:30")}, + {Time: parseTimeUTC("2014-08-13 10:20:30.1")}, {Time: parseTimeUTC("2014-08-15 10:20:30")}, {Time: parseTimeUTC("2014-08-18 10:20:30")}, {Time: parseTimeUTC("2014-08-20 10:20:30")}, @@ -77,7 +77,7 @@ var testExpireSnapshots = restic.Snapshots{ {Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}}, {Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}}, {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"bar"}}, + {Time: parseTimeUTC("2014-11-13 10:20:30.1"), Tags: []string{"bar"}}, {Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}}, {Time: parseTimeUTC("2014-11-18 10:20:30")}, {Time: parseTimeUTC("2014-11-20 10:20:30")}, @@ -97,7 +97,7 @@ var testExpireSnapshots = restic.Snapshots{ {Time: parseTimeUTC("2015-08-10 10:20:30")}, {Time: parseTimeUTC("2015-08-12 10:20:30")}, {Time: parseTimeUTC("2015-08-13 10:20:30")}, - {Time: parseTimeUTC("2015-08-13 10:20:30")}, + {Time: parseTimeUTC("2015-08-13 10:20:30.1")}, {Time: parseTimeUTC("2015-08-15 10:20:30")}, {Time: parseTimeUTC("2015-08-18 10:20:30")}, {Time: parseTimeUTC("2015-08-20 10:20:30")}, @@ -117,7 +117,7 @@ var testExpireSnapshots = restic.Snapshots{ {Time: parseTimeUTC("2015-11-10 10:20:30")}, {Time: parseTimeUTC("2015-11-12 10:20:30")}, {Time: parseTimeUTC("2015-11-13 10:20:30")}, - {Time: parseTimeUTC("2015-11-13 10:20:30")}, + {Time: parseTimeUTC("2015-11-13 10:20:30.1")}, {Time: parseTimeUTC("2015-11-15 10:20:30")}, {Time: parseTimeUTC("2015-11-18 10:20:30")}, {Time: parseTimeUTC("2015-11-20 10:20:30")}, @@ -169,53 +169,53 @@ var expireTests = []restic.ExpirePolicy{ func TestApplyPolicy(t *testing.T) { for i, p := range expireTests { - t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) + keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) - t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", - i, len(keep), len(remove), len(testExpireSnapshots), p) + t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", + i, len(keep), len(remove), len(testExpireSnapshots), p) - if len(keep)+len(remove) != len(testExpireSnapshots) { - t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", - i, len(keep)+len(remove), len(testExpireSnapshots)) - } + if len(keep)+len(remove) != len(testExpireSnapshots) { + t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", + i, len(keep)+len(remove), len(testExpireSnapshots)) + } - if p.Sum() > 0 && len(keep) > p.Sum() { - t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) - } + if p.Sum() > 0 && len(keep) > p.Sum() { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + p.Sum(), len(keep)) + } - for _, sn := range keep { - t.Logf("test %d: keep snapshot at %v %s\n", i, sn.Time, sn.Tags) - } - for _, sn := range remove { - t.Logf("test %d: forget snapshot at %v %s\n", i, sn.Time, sn.Tags) - } + for _, sn := range keep { + t.Logf("test %d: keep snapshot at %v %s\n", i, sn.Time, sn.Tags) + } + for _, sn := range remove { + t.Logf("test %d: forget snapshot at %v %s\n", i, sn.Time, sn.Tags) + } - goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) + goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) - if *updateGoldenFiles { - buf, err := json.MarshalIndent(keep, "", " ") - if err != nil { - t.Fatalf("error marshaling result: %v", err) - } - - if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { - t.Fatalf("unable to update golden file: %v", err) - } - } - - buf, err := ioutil.ReadFile(goldenFilename) + if *updateGoldenFiles { + buf, err := json.MarshalIndent(keep, "", " ") if err != nil { - t.Fatalf("error loading golden file %v: %v", goldenFilename, err) + t.Fatalf("error marshaling result: %v", err) } - var want restic.Snapshots - err = json.Unmarshal(buf, &want) - - if !reflect.DeepEqual(keep, want) { - t.Fatalf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) + if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { + t.Fatalf("unable to update golden file: %v", err) } - }) + } + + buf, err := ioutil.ReadFile(goldenFilename) + if err != nil { + t.Errorf("error loading golden file %v: %v", goldenFilename, err) + continue + } + + var want restic.Snapshots + err = json.Unmarshal(buf, &want) + + if !reflect.DeepEqual(keep, want) { + t.Errorf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) + continue + } } } diff --git a/src/restic/testdata/policy_keep_snapshots_0 b/src/restic/testdata/policy_keep_snapshots_0 index 952f8c02e..159d55d0c 100644 --- a/src/restic/testdata/policy_keep_snapshots_0 +++ b/src/restic/testdata/policy_keep_snapshots_0 @@ -120,7 +120,7 @@ "paths": null }, { - "time": "2015-11-13T10:20:30Z", + "time": "2015-11-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -270,7 +270,7 @@ "paths": null }, { - "time": "2015-08-13T10:20:30Z", + "time": "2015-08-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -324,7 +324,7 @@ ] }, { - "time": "2014-11-13T10:20:30Z", + "time": "2014-11-13T10:20:30.1Z", "tree": null, "paths": null, "tags": [ @@ -519,7 +519,7 @@ "paths": null }, { - "time": "2014-08-13T10:20:30Z", + "time": "2014-08-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -543,4 +543,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_10 b/src/restic/testdata/policy_keep_snapshots_10 index e2d61b68b..ca08ff08a 100644 --- a/src/restic/testdata/policy_keep_snapshots_10 +++ b/src/restic/testdata/policy_keep_snapshots_10 @@ -49,4 +49,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_13 b/src/restic/testdata/policy_keep_snapshots_13 index abf4ed809..57b4ab846 100644 --- a/src/restic/testdata/policy_keep_snapshots_13 +++ b/src/restic/testdata/policy_keep_snapshots_13 @@ -19,4 +19,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_15 b/src/restic/testdata/policy_keep_snapshots_15 index 51bed0e6e..c8fcf1e20 100644 --- a/src/restic/testdata/policy_keep_snapshots_15 +++ b/src/restic/testdata/policy_keep_snapshots_15 @@ -34,4 +34,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_17 b/src/restic/testdata/policy_keep_snapshots_17 index 8eb4ce6f8..1398e524f 100644 --- a/src/restic/testdata/policy_keep_snapshots_17 +++ b/src/restic/testdata/policy_keep_snapshots_17 @@ -49,4 +49,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_18 b/src/restic/testdata/policy_keep_snapshots_18 index caf2b746e..3ca6727a0 100644 --- a/src/restic/testdata/policy_keep_snapshots_18 +++ b/src/restic/testdata/policy_keep_snapshots_18 @@ -120,4 +120,4 @@ "foo" ] } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_19 b/src/restic/testdata/policy_keep_snapshots_19 index 177e8b400..af86beb84 100644 --- a/src/restic/testdata/policy_keep_snapshots_19 +++ b/src/restic/testdata/policy_keep_snapshots_19 @@ -8,4 +8,4 @@ "bar" ] } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_3 b/src/restic/testdata/policy_keep_snapshots_3 index 952f8c02e..159d55d0c 100644 --- a/src/restic/testdata/policy_keep_snapshots_3 +++ b/src/restic/testdata/policy_keep_snapshots_3 @@ -120,7 +120,7 @@ "paths": null }, { - "time": "2015-11-13T10:20:30Z", + "time": "2015-11-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -270,7 +270,7 @@ "paths": null }, { - "time": "2015-08-13T10:20:30Z", + "time": "2015-08-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -324,7 +324,7 @@ ] }, { - "time": "2014-11-13T10:20:30Z", + "time": "2014-11-13T10:20:30.1Z", "tree": null, "paths": null, "tags": [ @@ -519,7 +519,7 @@ "paths": null }, { - "time": "2014-08-13T10:20:30Z", + "time": "2014-08-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -543,4 +543,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_4 b/src/restic/testdata/policy_keep_snapshots_4 index 952f8c02e..159d55d0c 100644 --- a/src/restic/testdata/policy_keep_snapshots_4 +++ b/src/restic/testdata/policy_keep_snapshots_4 @@ -120,7 +120,7 @@ "paths": null }, { - "time": "2015-11-13T10:20:30Z", + "time": "2015-11-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -270,7 +270,7 @@ "paths": null }, { - "time": "2015-08-13T10:20:30Z", + "time": "2015-08-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -324,7 +324,7 @@ ] }, { - "time": "2014-11-13T10:20:30Z", + "time": "2014-11-13T10:20:30.1Z", "tree": null, "paths": null, "tags": [ @@ -519,7 +519,7 @@ "paths": null }, { - "time": "2014-08-13T10:20:30Z", + "time": "2014-08-13T10:20:30.1Z", "tree": null, "paths": null }, @@ -543,4 +543,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file diff --git a/src/restic/testdata/policy_keep_snapshots_5 b/src/restic/testdata/policy_keep_snapshots_5 index 42ed6051a..dcc09e864 100644 --- a/src/restic/testdata/policy_keep_snapshots_5 +++ b/src/restic/testdata/policy_keep_snapshots_5 @@ -95,7 +95,7 @@ "paths": null }, { - "time": "2015-11-13T10:20:30Z", + "time": "2015-11-13T10:20:30.1Z", "tree": null, "paths": null } diff --git a/src/restic/testdata/policy_keep_snapshots_8 b/src/restic/testdata/policy_keep_snapshots_8 index b0a64d691..a9656bd3c 100644 --- a/src/restic/testdata/policy_keep_snapshots_8 +++ b/src/restic/testdata/policy_keep_snapshots_8 @@ -75,7 +75,7 @@ "paths": null }, { - "time": "2015-11-13T10:20:30Z", + "time": "2015-11-13T10:20:30.1Z", "tree": null, "paths": null }, diff --git a/src/restic/testdata/policy_keep_snapshots_9 b/src/restic/testdata/policy_keep_snapshots_9 index 776816b08..c491b089f 100644 --- a/src/restic/testdata/policy_keep_snapshots_9 +++ b/src/restic/testdata/policy_keep_snapshots_9 @@ -29,4 +29,4 @@ "tree": null, "paths": null } -] +] \ No newline at end of file