From 6a5b02293990fd5f61699710e84b75d47ce60dd8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 7 Feb 2016 22:18:00 +0100 Subject: [PATCH 1/4] archiver: Add error reporting for directories When an error occurred while walking a directory, this error wasn't reported to the user before. --- archiver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/archiver.go b/archiver.go index 2f8078733..faf6f56ee 100644 --- a/archiver.go +++ b/archiver.go @@ -350,6 +350,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str // ignore dir nodes with errors if dir.Error() != nil { + fmt.Fprintf(os.Stderr, "error walking dir %v: %v\n", dir.Path(), dir.Error()) dir.Result() <- nil p.Report(Stat{Errors: 1}) continue From 9048eb676b55e0a6227eed597b54e37e1f659a82 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 7 Feb 2016 22:18:37 +0100 Subject: [PATCH 2/4] pipe: join replaced paths with original path When saving `/`, it was replaced with the contents, but without the proper path. So `/` was replaced by [`boot`, `bin`, `home`, ...], but without prefixing the entry name with the proper path. --- pipe/pipe.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pipe/pipe.go b/pipe/pipe.go index 6aa4f3588..aeabcf8f4 100644 --- a/pipe/pipe.go +++ b/pipe/pipe.go @@ -174,7 +174,16 @@ func cleanupPath(path string) ([]string, error) { return []string{path}, nil } - return readDirNames(path) + paths, err := readDirNames(path) + if err != nil { + return nil, err + } + + for i, p := range paths { + paths[i] = filepath.Join(path, p) + } + + return paths, nil } // Walk sends a Job for each file and directory it finds below the paths. When From 68db75b4e3506f6cce02bfccd313bc4938c4b7dd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 7 Feb 2016 23:22:06 +0100 Subject: [PATCH 3/4] pipe/archiver: Add more debug messages --- archiver.go | 3 ++- pipe/pipe.go | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/archiver.go b/archiver.go index faf6f56ee..a8e94e516 100644 --- a/archiver.go +++ b/archiver.go @@ -366,6 +366,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str // if we get a nil pointer here, an error has happened while // processing this entry. Ignore it for now. if res == nil { + debug.Log("Archiver.dirWorker", "got nil result?") continue } @@ -374,7 +375,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str tree.Insert(node) if node.Type == "dir" { - debug.Log("Archiver.dirWorker", "got tree node for %s: %v", node.path, node.blobs) + debug.Log("Archiver.dirWorker", "got tree node for %s: %v", node.path, node.Subtree) if node.Subtree.IsNull() { panic("invalid null subtree ID") diff --git a/pipe/pipe.go b/pipe/pipe.go index aeabcf8f4..2499c2bf2 100644 --- a/pipe/pipe.go +++ b/pipe/pipe.go @@ -92,7 +92,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs info, err := os.Lstat(dir) if err != nil { - debug.Log("pipe.walk", "error for %v: %v", dir, err) + debug.Log("pipe.walk", "error for %v: %v, res %p", dir, err, res) select { case jobs <- Dir{basedir: basedir, path: relpath, info: info, error: err, result: res}: case <-done: @@ -101,11 +101,12 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs } if !selectFunc(dir, info) { - debug.Log("pipe.walk", "file %v excluded by filter", dir) + debug.Log("pipe.walk", "file %v excluded by filter, res %p", dir, res) return } if !info.IsDir() { + debug.Log("pipe.walk", "sending file job for %v, res %p", dir, res) select { case jobs <- Entry{info: info, basedir: basedir, path: relpath, result: res}: case <-done: @@ -116,7 +117,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs debug.RunHook("pipe.readdirnames", dir) names, err := readDirNames(dir) if err != nil { - debug.Log("pipe.walk", "Readdirnames(%v) returned error: %v", dir, err) + debug.Log("pipe.walk", "Readdirnames(%v) returned error: %v, res %p", dir, err, res) select { case <-done: case jobs <- Dir{basedir: basedir, path: relpath, info: info, error: err, result: res}: @@ -143,6 +144,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs entries = append(entries, ch) if statErr != nil { + debug.Log("pipe.walk", "sending file job for %v, err %v, res %p", subpath, err, res) select { case jobs <- Entry{info: fi, error: statErr, basedir: basedir, path: filepath.Join(relpath, name), result: ch}: case <-done: @@ -158,7 +160,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs walk(basedir, subpath, selectFunc, done, jobs, ch) } - debug.Log("pipe.walk", "sending dirjob for %q, basedir %q", dir, basedir) + debug.Log("pipe.walk", "sending dirjob for %q, basedir %q, res %p", dir, basedir, res) select { case jobs <- Dir{basedir: basedir, path: relpath, info: info, Entries: entries, result: res}: case <-done: @@ -217,7 +219,7 @@ func Walk(walkPaths []string, selectFunc SelectFunc, done chan struct{}, jobs ch debug.Log("pipe.Walk", "walker for %v done", path) } - debug.Log("pipe.Walk", "sending root node") + debug.Log("pipe.Walk", "sending root node, res %p", res) select { case <-done: return From 26484d0c7b783ab05fa38e84a0675658738dd3ad Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 7 Feb 2016 23:22:52 +0100 Subject: [PATCH 4/4] pipe: Ignore excluded directories When top-level directories are ignored, they were still added to the top-level Dir{} object. This commit ignores them. --- pipe/pipe.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pipe/pipe.go b/pipe/pipe.go index 2499c2bf2..45bb8a905 100644 --- a/pipe/pipe.go +++ b/pipe/pipe.go @@ -82,7 +82,7 @@ var errCancelled = errors.New("walk cancelled") // dirs). If false is returned, files are ignored and dirs are not even walked. type SelectFunc func(item string, fi os.FileInfo) bool -func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs chan<- Job, res chan<- Result) { +func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs chan<- Job, res chan<- Result) (excluded bool) { debug.Log("pipe.walk", "start on %q, basedir %q", dir, basedir) relpath, err := filepath.Rel(basedir, dir) @@ -102,6 +102,7 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs if !selectFunc(dir, info) { debug.Log("pipe.walk", "file %v excluded by filter, res %p", dir, res) + excluded = true return } @@ -165,6 +166,8 @@ func walk(basedir, dir string, selectFunc SelectFunc, done <-chan struct{}, jobs case jobs <- Dir{basedir: basedir, path: relpath, info: info, Entries: entries, result: res}: case <-done: } + + return } // cleanupPath is used to clean a path. For a normal path, a slice with just @@ -214,7 +217,13 @@ func Walk(walkPaths []string, selectFunc SelectFunc, done chan struct{}, jobs ch for _, path := range paths { debug.Log("pipe.Walk", "start walker for %v", path) ch := make(chan Result, 1) - walk(filepath.Dir(path), path, selectFunc, done, jobs, ch) + excluded := walk(filepath.Dir(path), path, selectFunc, done, jobs, ch) + + if excluded { + debug.Log("pipe.Walk", "walker for %v done, it was excluded by the filter", path) + continue + } + entries = append(entries, ch) debug.Log("pipe.Walk", "walker for %v done", path) }