From db82e6b80c717e1bfa5db3f0bd96de080b3d51d6 Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Tue, 16 Oct 2018 22:39:14 +0200 Subject: [PATCH 1/7] Add iexclude option to backup command --- cmd/restic/cmd_backup.go | 34 ++++++++++++++++++++-------------- cmd/restic/exclude.go | 10 ++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index fc24868a5..1cc673d2f 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -68,20 +68,21 @@ given as the arguments. // BackupOptions bundles all options for the backup command. type BackupOptions struct { - Parent string - Force bool - Excludes []string - ExcludeFiles []string - ExcludeOtherFS bool - ExcludeIfPresent []string - ExcludeCaches bool - Stdin bool - StdinFilename string - Tags []string - Host string - FilesFrom []string - TimeStamp string - WithAtime bool + Parent string + Force bool + Excludes []string + InsensitiveExcludes []string + ExcludeFiles []string + ExcludeOtherFS bool + ExcludeIfPresent []string + ExcludeCaches bool + Stdin bool + StdinFilename string + Tags []string + Host string + FilesFrom []string + TimeStamp string + WithAtime bool } var backupOptions BackupOptions @@ -93,6 +94,7 @@ func init() { f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") + f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as `--exclude` but ignores the casing of filenames") f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") @@ -222,6 +224,10 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t opts.Excludes = append(opts.Excludes, excludes...) } + if len(opts.InsensitiveExcludes) > 0 { + fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes)) + } + if len(opts.Excludes) > 0 { fs = append(fs, rejectByPattern(opts.Excludes)) } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 479f8a308..35ee5f81e 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -88,6 +88,16 @@ func rejectByPattern(patterns []string) RejectByNameFunc { } } +// Same as `rejectByPattern` but case insensitive. +func rejectByInsensitivePattern(patterns []string) RejectByNameFunc { + for index, path := range patterns { + patterns[index] = strings.ToLower(path) + } + return func(item string) bool { + return rejectByPattern(patterns)(strings.ToLower(item)) + } +} + // rejectIfPresent returns a RejectByNameFunc which itself returns whether a path // should be excluded. The RejectByNameFunc considers a file to be excluded when // it resides in a directory with an exclusion file, that is specified by From c13f79da0231a3a2b63e37ed3d755accc6614ce1 Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Tue, 16 Oct 2018 22:39:55 +0200 Subject: [PATCH 2/7] Add iexclude and iinclude options to restore command --- cmd/restic/cmd_restore.go | 42 +++++++++++++++++++++++++++------------ internal/filter/filter.go | 9 +++++++++ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 477192eab..a711da4dd 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -28,13 +28,15 @@ repository. // RestoreOptions collects all options for the restore command. type RestoreOptions struct { - Exclude []string - Include []string - Target string - Host string - Paths []string - Tags restic.TagLists - Verify bool + Exclude []string + InsensitiveExclude []string + Include []string + InsensitiveInclude []string + Target string + Host string + Paths []string + Tags restic.TagLists + Verify bool } var restoreOptions RestoreOptions @@ -44,7 +46,9 @@ func init() { flags := cmdRestore.Flags() flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") + flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as `--exclude` but ignores the casing of filenames") flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)") + flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`) @@ -55,6 +59,8 @@ func init() { func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { ctx := gopts.ctx + hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 + hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 switch { case len(args) == 0: @@ -67,7 +73,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { return errors.Fatal("please specify a directory to restore to (--target)") } - if len(opts.Exclude) > 0 && len(opts.Include) > 0 { + if hasExcludes && hasIncludes { return errors.Fatal("exclude and include patterns are mutually exclusive") } @@ -125,11 +131,16 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { Warnf("error for exclude pattern: %v", err) } + matchedInsensitive, _, err := filter.InsensitiveList(opts.InsensitiveExclude, item) + if err != nil { + Warnf("error for iexclude pattern: %v", err) + } + // An exclude filter is basically a 'wildcard but foo', // so even if a childMayMatch, other children of a dir may not, // therefore childMayMatch does not matter, but we should not go down // unless the dir is selected for restore - selectedForRestore = !matched + selectedForRestore = !matched && !matchedInsensitive childMayBeSelected = selectedForRestore && node.Type == "dir" return selectedForRestore, childMayBeSelected @@ -141,15 +152,20 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { Warnf("error for include pattern: %v", err) } - selectedForRestore = matched - childMayBeSelected = childMayMatch && node.Type == "dir" + matchedInsensitive, childMayMatchInsensitive, err := filter.InsensitiveList(opts.InsensitiveInclude, item) + if err != nil { + Warnf("error for iexclude pattern: %v", err) + } + + selectedForRestore = matched || matchedInsensitive + childMayBeSelected = (childMayMatch || childMayMatchInsensitive) && node.Type == "dir" return selectedForRestore, childMayBeSelected } - if len(opts.Exclude) > 0 { + if hasExcludes { res.SelectFilter = selectExcludeFilter - } else if len(opts.Include) > 0 { + } else if hasIncludes { res.SelectFilter = selectIncludeFilter } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 74deddb03..71ae4817c 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -187,3 +187,12 @@ func List(patterns []string, str string) (matched bool, childMayMatch bool, err return matched, childMayMatch, nil } + +// InsensitiveList is the same as List but case insensitive. +func InsensitiveList(patterns []string, str string) (matched bool, childMayMatch bool, err error) { + str = strings.ToLower(str) + for index, path := range patterns { + patterns[index] = strings.ToLower(path) + } + return List(patterns, str) +} From 5fe6de219df1cd7f53718a3fe0e3e66a0f4a2f0a Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Tue, 16 Oct 2018 23:29:44 +0200 Subject: [PATCH 3/7] Add tests for insensitive variants of filter methods --- cmd/restic/exclude_test.go | 27 +++++++++++++++++++++++++++ internal/filter/filter_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go index 741dbdb64..6c8ce6e14 100644 --- a/cmd/restic/exclude_test.go +++ b/cmd/restic/exclude_test.go @@ -36,6 +36,33 @@ func TestRejectByPattern(t *testing.T) { } } +func TestRejectByInsensitivePattern(t *testing.T) { + var tests = []struct { + filename string + reject bool + }{ + {filename: "/home/user/foo.GO", reject: true}, + {filename: "/home/user/foo.c", reject: false}, + {filename: "/home/user/foobar", reject: false}, + {filename: "/home/user/FOObar/x", reject: true}, + {filename: "/home/user/README", reject: false}, + {filename: "/home/user/readme.md", reject: true}, + } + + patterns := []string{"*.go", "README.md", "/home/user/foobar/*"} + + for _, tc := range tests { + t.Run("", func(t *testing.T) { + reject := rejectByInsensitivePattern(patterns) + res := reject(tc.filename) + if res != tc.reject { + t.Fatalf("wrong result for filename %v: want %v, got %v", + tc.filename, tc.reject, res) + } + }) + } +} + func TestIsExcludedByFile(t *testing.T) { const ( tagFilename = "CACHEDIR.TAG" diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index 97df452fb..7be5dcb30 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -279,6 +279,33 @@ func ExampleList() { // match: true } +var filterInsensitiveListTests = []struct { + patterns []string + path string + match bool +}{ + {[]string{"*.go"}, "/foo/bar/test.go", true}, + {[]string{"test.go"}, "/foo/bar/test.go", true}, + {[]string{"test.go"}, "/foo/bar/TEST.go", true}, + {[]string{"BAR"}, "/foo/BAR/TEST.go", true}, +} + +func TestInsensitiveList(t *testing.T) { + for i, test := range filterInsensitiveListTests { + match, _, err := filter.InsensitiveList(test.patterns, test.path) + if err != nil { + t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v", + i, test.patterns, err) + continue + } + + if match != test.match { + t.Errorf("test %d: filter.InsensitiveList(%q, %q): expected %v, got %v", + i, test.patterns, test.path, test.match, match) + } + } +} + func extractTestLines(t testing.TB) (lines []string) { f, err := os.Open("testdata/libreoffice.txt.bz2") if err != nil { From 8a97bb8661814b1381d5cdd0c47ea95268edfebc Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Thu, 18 Oct 2018 21:39:41 +0200 Subject: [PATCH 4/7] Add documentation for insensitive includes & excludes --- doc/040_backup.rst | 1 + doc/050_restore.rst | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 951f65a4c..82b066764 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -139,6 +139,7 @@ You can exclude folders and files by specifying exclude patterns, currently the exclude options are: - ``--exclude`` Specified one or more times to exclude one or more items +- ``--iexclude`` Same as ``--exclude`` but ignores the case of paths - ``--exclude-caches`` Specified once to exclude folders containing a special file - ``--exclude-file`` Specified one or more times to exclude items listed in a given file - ``--exclude-if-present`` Specified one or more times to exclude a folders content diff --git a/doc/050_restore.rst b/doc/050_restore.rst index e602c7e8a..f5d584042 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -52,6 +52,10 @@ You can use the command ``restic ls latest`` or ``restic find foo`` to find the path to the file within the snapshot. This path you can then pass to `--include` in verbatim to only restore the single file or directory. +There are case insensitive variants of of ``--exclude`` and ``--include`` called +``--iexclude`` and ``--iinclude``. These options will behave the same way but +ignore the casing of paths. + Restore using mount =================== From 879f6e0c818f1b3b22a1a1d0fb8c5e4ecda92cdc Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Thu, 18 Oct 2018 21:45:47 +0200 Subject: [PATCH 5/7] Add changelog file for insensitive includes & excludes --- changelog/unreleased/issue-1895 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/issue-1895 diff --git a/changelog/unreleased/issue-1895 b/changelog/unreleased/issue-1895 new file mode 100644 index 000000000..70aa0653d --- /dev/null +++ b/changelog/unreleased/issue-1895 @@ -0,0 +1,7 @@ +Enhancement: Add case insensitive include & exclude options + +The backup and restore commands now have --iexclude and --iinclude flags +as case insensitive variants of --exclude and --include. + +https://github.com/restic/restic/issues/1895 +https://github.com/restic/restic/pull/2032 From deedc3812933e7a6670b5587833dfa42338009e6 Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Sat, 19 Jan 2019 10:59:48 +0000 Subject: [PATCH 6/7] Only convert iexclude & iinclude casing once --- cmd/restic/cmd_restore.go | 13 +++++++++++-- internal/filter/filter.go | 9 --------- internal/filter/filter_test.go | 27 --------------------------- 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index a711da4dd..7c056fdab 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -6,6 +6,7 @@ import ( "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" + "strings" "github.com/spf13/cobra" ) @@ -62,6 +63,14 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 + for i, str := range opts.InsensitiveExclude { + opts.InsensitiveExclude[i] = strings.ToLower(str) + } + + for i, str := range opts.InsensitiveInclude { + opts.InsensitiveInclude[i] = strings.ToLower(str) + } + switch { case len(args) == 0: return errors.Fatal("no snapshot ID specified") @@ -131,7 +140,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { Warnf("error for exclude pattern: %v", err) } - matchedInsensitive, _, err := filter.InsensitiveList(opts.InsensitiveExclude, item) + matchedInsensitive, _, err := filter.List(opts.InsensitiveExclude, strings.ToLower(item)) if err != nil { Warnf("error for iexclude pattern: %v", err) } @@ -152,7 +161,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { Warnf("error for include pattern: %v", err) } - matchedInsensitive, childMayMatchInsensitive, err := filter.InsensitiveList(opts.InsensitiveInclude, item) + matchedInsensitive, childMayMatchInsensitive, err := filter.List(opts.InsensitiveInclude, strings.ToLower(item)) if err != nil { Warnf("error for iexclude pattern: %v", err) } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 71ae4817c..74deddb03 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -187,12 +187,3 @@ func List(patterns []string, str string) (matched bool, childMayMatch bool, err return matched, childMayMatch, nil } - -// InsensitiveList is the same as List but case insensitive. -func InsensitiveList(patterns []string, str string) (matched bool, childMayMatch bool, err error) { - str = strings.ToLower(str) - for index, path := range patterns { - patterns[index] = strings.ToLower(path) - } - return List(patterns, str) -} diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index 7be5dcb30..97df452fb 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -279,33 +279,6 @@ func ExampleList() { // match: true } -var filterInsensitiveListTests = []struct { - patterns []string - path string - match bool -}{ - {[]string{"*.go"}, "/foo/bar/test.go", true}, - {[]string{"test.go"}, "/foo/bar/test.go", true}, - {[]string{"test.go"}, "/foo/bar/TEST.go", true}, - {[]string{"BAR"}, "/foo/BAR/TEST.go", true}, -} - -func TestInsensitiveList(t *testing.T) { - for i, test := range filterInsensitiveListTests { - match, _, err := filter.InsensitiveList(test.patterns, test.path) - if err != nil { - t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v", - i, test.patterns, err) - continue - } - - if match != test.match { - t.Errorf("test %d: filter.InsensitiveList(%q, %q): expected %v, got %v", - i, test.patterns, test.path, test.match, match) - } - } -} - func extractTestLines(t testing.TB) (lines []string) { f, err := os.Open("testdata/libreoffice.txt.bz2") if err != nil { From cdc287a7f6406aabb4121b4f61b888f1f70fa90d Mon Sep 17 00:00:00 2001 From: Johannes Hertenstein Date: Sat, 19 Jan 2019 11:08:13 +0000 Subject: [PATCH 7/7] Store reference to reject function for insensitive pattern rejection --- cmd/restic/exclude.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 35ee5f81e..09d35b226 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -93,8 +93,10 @@ func rejectByInsensitivePattern(patterns []string) RejectByNameFunc { for index, path := range patterns { patterns[index] = strings.ToLower(path) } + + rejFunc := rejectByPattern(patterns) return func(item string) bool { - return rejectByPattern(patterns)(strings.ToLower(item)) + return rejFunc(strings.ToLower(item)) } }