From ed65a7dbca435e4a80b54a5b5701122135da417e Mon Sep 17 00:00:00 2001 From: arjunajesh <34989598+arjunajesh@users.noreply.github.com> Date: Sat, 15 Jul 2023 22:48:30 -0400 Subject: [PATCH 1/6] implement progress bar for index loading --- cmd/restic/cmd_backup.go | 5 ++++- cmd/restic/cmd_backup_integration_test.go | 2 +- cmd/restic/cmd_cat.go | 6 ++++-- cmd/restic/cmd_copy.go | 7 ++++--- cmd/restic/cmd_debug.go | 3 ++- cmd/restic/cmd_diff.go | 4 ++-- cmd/restic/cmd_dump.go | 3 ++- cmd/restic/cmd_find.go | 4 ++-- cmd/restic/cmd_ls.go | 3 ++- cmd/restic/cmd_mount.go | 3 ++- cmd/restic/cmd_prune.go | 3 ++- cmd/restic/cmd_recover.go | 5 +++-- cmd/restic/cmd_repair_snapshots.go | 3 ++- cmd/restic/cmd_restore.go | 3 ++- cmd/restic/cmd_rewrite.go | 3 ++- cmd/restic/cmd_stats.go | 4 ++-- cmd/restic/integration_helpers_test.go | 2 +- cmd/restic/progress.go | 23 ++++++++++++++++++----- internal/fuse/snapshots_dirstruct.go | 2 +- internal/index/master_index_test.go | 2 +- internal/repository/repack_test.go | 2 +- internal/repository/repository.go | 19 +++++++++++++++++-- internal/repository/repository_test.go | 4 ++-- internal/restic/repository.go | 2 +- 24 files changed, 80 insertions(+), 37 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 6b5706855..09d5191b5 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -546,7 +546,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("load index files") } - err = repo.LoadIndex(ctx) + + bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term) + + err = repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index fb7bef633..76c227e3d 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -252,7 +252,7 @@ func TestBackupTreeLoadError(t *testing.T) { r, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) - rtest.OK(t, r.LoadIndex(context.TODO())) + rtest.OK(t, r.LoadIndex(context.TODO(), nil)) treePacks := restic.NewIDSet() r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { if pb.Type == restic.TreeBlob { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 97789d271..fe4314179 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -169,7 +169,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return err case "blob": - err = repo.LoadIndex(ctx) + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + err = repo.LoadIndex(ctx, bar) if err != nil { return err } @@ -197,7 +198,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatalf("could not find snapshot: %v\n", err) } - err = repo.LoadIndex(ctx) + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + err = repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index eaa0ef81a..32d21e50b 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -99,12 +99,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } debug.Log("Loading source index") - if err := srcRepo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err := srcRepo.LoadIndex(ctx, bar); err != nil { return err } - + bar = newProgressMax(!gopts.Quiet, 0, "index files loaded") debug.Log("Loading destination index") - if err := dstRepo.LoadIndex(ctx); err != nil { + if err := dstRepo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index a54200c45..c7aa82449 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -469,7 +469,8 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } } - err = repo.LoadIndex(ctx) + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + err = repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index accd55db2..a82700358 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -363,8 +363,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if !gopts.JSON { Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) } - - if err = repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 325607017..970d323bb 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -152,7 +152,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("failed to find snapshot: %v", err) } - err = repo.LoadIndex(ctx) + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + err = repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 181d8595d..d1c6e6628 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -589,8 +589,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] if err != nil { return err } - - if err = repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 256c9e002..9f1b64d60 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -173,7 +173,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - if err = repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index ec3662d5c..39bcd6729 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -130,7 +130,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args } } - err = repo.LoadIndex(ctx) + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + err = repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index e4c2c7b29..3b9590595 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -187,7 +187,8 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption Verbosef("loading indexes...\n") // loading the index before the snapshots is ok, as we use an exclusive lock here - err := repo.LoadIndex(ctx) + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + err := repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 85dcc23d7..b8035d102 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -58,7 +58,8 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { } Verbosef("load index files\n") - if err = repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err = repo.LoadIndex(ctx, bar); err != nil { return err } @@ -73,7 +74,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { }) Verbosef("load %d trees\n", len(trees)) - bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") + bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") for id := range trees { tree, err := restic.LoadTree(ctx, repo, id) if err != nil { diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 03736795c..a7d3af4a8 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -89,7 +89,8 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt return err } - if err := repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err := repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 9bce90a26..11ce5e4a5 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -173,7 +173,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return errors.Fatalf("failed to find snapshot: %v", err) } - err = repo.LoadIndex(ctx) + bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term) + err = repo.LoadIndex(ctx, bar) if err != nil { return err } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index c08797c48..008c133d1 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -212,7 +212,8 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a return err } - if err = repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index a7ecd438f..900000439 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -98,8 +98,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args if err != nil { return err } - - if err = repo.LoadIndex(ctx); err != nil { + bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index b7cb5b333..2afdbb938 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -258,7 +258,7 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem rtest.OK(t, err) // Get all tree packs - rtest.OK(t, r.LoadIndex(context.TODO())) + rtest.OK(t, r.LoadIndex(context.TODO(), nil)) treePacks := restic.NewIDSet() r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 4b6025a54..56cca9ace 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -29,13 +29,12 @@ func calculateProgressInterval(show bool, json bool) time.Duration { return interval } -// newProgressMax returns a progress.Counter that prints to stdout. -func newProgressMax(show bool, max uint64, description string) *progress.Counter { +// newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided. +func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter { if !show { return nil } interval := calculateProgressInterval(show, false) - canUpdateStatus := stdoutCanUpdateStatus() return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) { var status string @@ -47,14 +46,28 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description) } - printProgress(status, canUpdateStatus) + print(status) if final { fmt.Print("\n") } }) } -func printProgress(status string, canUpdateStatus bool) { +func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter { + return newGenericProgressMax(show, max, description, func(status string) { + term.SetStatus([]string{status}) + }) +} + +// newProgressMax calls newTerminalProgress without a terminal (print to stdout) +func newProgressMax(show bool, max uint64, description string) *progress.Counter { + return newGenericProgressMax(show, max, description, printProgress) +} + +func printProgress(status string) { + + canUpdateStatus := stdoutCanUpdateStatus() + w := stdoutTerminalWidth() if w > 0 { if w < 3 { diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index 3080d4de8..a3cd11b14 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -328,7 +328,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { return nil } - err = d.root.repo.LoadIndex(ctx) + err = d.root.repo.LoadIndex(ctx, nil) if err != nil { return err } diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index 5260a11a0..493d11ace 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -357,7 +357,7 @@ func TestIndexSave(t *testing.T) { func testIndexSave(t *testing.T, version uint) { repo := createFilledRepo(t, 3, version) - err := repo.LoadIndex(context.TODO()) + err := repo.LoadIndex(context.TODO(), nil) if err != nil { t.Fatal(err) } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index bb31eba77..c8570a9d4 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -213,7 +213,7 @@ func reloadIndex(t *testing.T, repo restic.Repository) { t.Fatal(err) } - if err := repo.LoadIndex(context.TODO()); err != nil { + if err := repo.LoadIndex(context.TODO(), nil); err != nil { t.Fatalf("error loading new index: %v", err) } } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 653c1f774..7c3c86d2b 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -581,15 +581,30 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { } // LoadIndex loads all index files from the backend in parallel and stores them -// in the master index. The first error that occurred is returned. -func (r *Repository) LoadIndex(ctx context.Context) error { +func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { debug.Log("Loading index") + if p != nil { + var numIndexFiles uint64 + err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + numIndexFiles++ + return nil + }) + if err != nil { + return err + } + p.SetMax(numIndexFiles) + defer p.Done() + } + err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } r.idx.Insert(idx) + if p != nil { + p.Add(1) + } return nil }) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index f26bf46f2..d362b0b5e 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -255,7 +255,7 @@ func TestRepositoryLoadIndex(t *testing.T) { defer cleanup() repo := repository.TestOpenLocal(t, repodir) - rtest.OK(t, repo.LoadIndex(context.TODO())) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } // loadIndex loads the index id from backend and returns it. @@ -324,7 +324,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "") rtest.OK(t, err) - rtest.OK(t, repo.LoadIndex(context.TODO())) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } func BenchmarkLoadIndex(b *testing.B) { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 6990200e4..a651f9906 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -24,7 +24,7 @@ type Repository interface { Key() *crypto.Key Index() MasterIndex - LoadIndex(context.Context) error + LoadIndex(context.Context, *progress.Counter) error SetIndex(MasterIndex) error LookupBlobSize(ID, BlobType) (uint, bool) From b6593ad7df01559bbba2de6a4f1a8c2be6e87141 Mon Sep 17 00:00:00 2001 From: arjunajesh <34989598+arjunajesh@users.noreply.github.com> Date: Mon, 24 Jul 2023 00:28:15 -0400 Subject: [PATCH 2/6] changelog --- changelog/unreleased/pull-299 | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/pull-299 diff --git a/changelog/unreleased/pull-299 b/changelog/unreleased/pull-299 new file mode 100644 index 000000000..aae5c0e05 --- /dev/null +++ b/changelog/unreleased/pull-299 @@ -0,0 +1,6 @@ +Enhancement: Show progress bar while loading the index + +Restic did not provide any feedback while loading index files. Now there is a progress bar for the index loading process. + +https://github.com/restic/restic/issues/229 +https://github.com/restic/restic/pull/4419 From 3fd0ad744807a1b75c1f5caba69c05c044a84a45 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 13 Aug 2023 20:33:13 +0200 Subject: [PATCH 3/6] repository: list index files only once --- cmd/restic/cmd_debug.go | 2 +- cmd/restic/cmd_list.go | 2 +- cmd/restic/cmd_repair_index.go | 2 +- internal/checker/checker.go | 2 +- internal/index/index_parallel.go | 4 ++-- internal/index/index_parallel_test.go | 4 ++-- internal/repository/repository.go | 15 +++++++++++++-- 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index c7aa82449..5890b4a38 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) } func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { - return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { return err diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index bd02cedc7..5974da9ac 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index b1905836a..622c77801 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } else { Verbosef("loading indexes...\n") mi := index.NewMasterIndex() - err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { Warnf("removing invalid index %v: %v\n", id, err) obsoleteIndexes = append(obsoleteIndexes, id) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index b2512969e..85e078a08 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -117,7 +117,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { debug.Log("Start") packToIndex := make(map[restic.ID]restic.IDSet) - err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, c.repo.Backend(), c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { debug.Log("process index %v, err %v", id, err) if oldFormat { diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index e7e46e88a..d505d756e 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -11,7 +11,7 @@ import ( // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound @@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) var m sync.Mutex - return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { var err error var idx *Index oldFormat := false diff --git a/internal/index/index_parallel_test.go b/internal/index/index_parallel_test.go index 760374510..86be46473 100644 --- a/internal/index/index_parallel_test.go +++ b/internal/index/index_parallel_test.go @@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // check that all expected indexes are loaded without errors indexIDs := restic.NewIDSet() var indexErr error - rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { if err != nil { indexErr = err } @@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // must failed with the returned error iterErr := errors.New("error to pass upwards") - err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { return iterErr }) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 7c3c86d2b..8ec3c598e 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -584,9 +584,20 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { debug.Log("Loading index") + indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile) + if err != nil { + return err + } + if p != nil { var numIndexFiles uint64 - err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { + _, err := restic.ParseID(fi.Name) + if err != nil { + debug.Log("unable to parse %v as an ID", fi.Name) + return nil + } + numIndexFiles++ return nil }) @@ -597,7 +608,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { defer p.Done() } - err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } From 75f6bd89ed3756af43920ee5aa22d2d691b9899e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 19:33:02 +0200 Subject: [PATCH 4/6] hide index loading progress for JSON output --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_cat.go | 4 ++-- cmd/restic/cmd_copy.go | 4 ++-- cmd/restic/cmd_debug.go | 2 +- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_dump.go | 2 +- cmd/restic/cmd_find.go | 2 +- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_mount.go | 2 +- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_repair_snapshots.go | 2 +- cmd/restic/cmd_restore.go | 2 +- cmd/restic/cmd_rewrite.go | 2 +- cmd/restic/cmd_stats.go | 2 +- 15 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 09d5191b5..aabdeba54 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -547,7 +547,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter progressPrinter.V("load index files") } - bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term) + bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded", term) err = repo.LoadIndex(ctx, bar) if err != nil { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index fe4314179..f311eee88 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -169,7 +169,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return err case "blob": - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") err = repo.LoadIndex(ctx, bar) if err != nil { return err @@ -198,7 +198,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatalf("could not find snapshot: %v\n", err) } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 32d21e50b..3f410fec6 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -99,11 +99,11 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } debug.Log("Loading source index") - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err := srcRepo.LoadIndex(ctx, bar); err != nil { return err } - bar = newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar = newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") debug.Log("Loading destination index") if err := dstRepo.LoadIndex(ctx, bar); err != nil { return err diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 5890b4a38..6ced508ff 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -469,7 +469,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index a82700358..bedbaae5d 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -363,7 +363,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if !gopts.JSON { Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 970d323bb..9e811bf61 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -152,7 +152,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("failed to find snapshot: %v", err) } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index d1c6e6628..7a6c4196f 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -589,7 +589,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] if err != nil { return err } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 9f1b64d60..eaff272e0 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -173,7 +173,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 39bcd6729..34411fe37 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -130,7 +130,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args } } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 3b9590595..2f9d3069a 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -187,7 +187,7 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption Verbosef("loading indexes...\n") // loading the index before the snapshots is ok, as we use an exclusive lock here - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") err := repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index b8035d102..b7eaea087 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -58,7 +58,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { } Verbosef("load index files\n") - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index a7d3af4a8..4c15cd465 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -89,7 +89,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt return err } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err := repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 11ce5e4a5..6e3c3a757 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -173,7 +173,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return errors.Fatalf("failed to find snapshot: %v", err) } - bar := newTerminalProgressMax(!gopts.Quiet, 0, "index files loaded", term) + bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded", term) err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 008c133d1..816df71a9 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -212,7 +212,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a return err } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 900000439..63048b455 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -98,7 +98,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args if err != nil { return err } - bar := newProgressMax(!gopts.Quiet, 0, "index files loaded") + bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") if err = repo.LoadIndex(ctx, bar); err != nil { return err } From b0da0f152f3c217c48684932915cea617c973f6b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 19:38:09 +0200 Subject: [PATCH 5/6] deduplicate index progress bar setup --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_cat.go | 4 ++-- cmd/restic/cmd_copy.go | 4 ++-- cmd/restic/cmd_debug.go | 2 +- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_dump.go | 2 +- cmd/restic/cmd_find.go | 2 +- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_mount.go | 2 +- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_repair_snapshots.go | 2 +- cmd/restic/cmd_restore.go | 2 +- cmd/restic/cmd_rewrite.go | 2 +- cmd/restic/cmd_stats.go | 2 +- cmd/restic/progress.go | 8 ++++++++ 16 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index aabdeba54..e476ae7b8 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -547,7 +547,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter progressPrinter.V("load index files") } - bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded", term) + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) err = repo.LoadIndex(ctx, bar) if err != nil { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index f311eee88..238614cd0 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -169,7 +169,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return err case "blob": - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { return err @@ -198,7 +198,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatalf("could not find snapshot: %v\n", err) } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 3f410fec6..f31c17adb 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -99,11 +99,11 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } debug.Log("Loading source index") - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err := srcRepo.LoadIndex(ctx, bar); err != nil { return err } - bar = newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar = newIndexProgress(gopts.Quiet, gopts.JSON) debug.Log("Loading destination index") if err := dstRepo.LoadIndex(ctx, bar); err != nil { return err diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 6ced508ff..bf05c448d 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -469,7 +469,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index bedbaae5d..125904068 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -363,7 +363,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] if !gopts.JSON { Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 9e811bf61..8b9fa9624 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -152,7 +152,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("failed to find snapshot: %v", err) } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 7a6c4196f..9c21cc9bb 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -589,7 +589,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] if err != nil { return err } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index eaff272e0..a0c07a752 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -173,7 +173,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 34411fe37..04c072daf 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -130,7 +130,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args } } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 2f9d3069a..638a0de5e 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -187,7 +187,7 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption Verbosef("loading indexes...\n") // loading the index before the snapshots is ok, as we use an exclusive lock here - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) err := repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index b7eaea087..63084dd5f 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -58,7 +58,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { } Verbosef("load index files\n") - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 4c15cd465..720523762 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -89,7 +89,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt return err } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err := repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 6e3c3a757..494c6b86a 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -173,7 +173,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, return errors.Fatalf("failed to find snapshot: %v", err) } - bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded", term) + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) err = repo.LoadIndex(ctx, bar) if err != nil { return err diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 816df71a9..9d9ab5d5d 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -212,7 +212,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a return err } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 63048b455..f16b59d3c 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -98,7 +98,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args if err != nil { return err } - bar := newProgressMax(!gopts.Quiet && !gopts.JSON, 0, "index files loaded") + bar := newIndexProgress(gopts.Quiet, gopts.JSON) if err = repo.LoadIndex(ctx, bar); err != nil { return err } diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 56cca9ace..fe4f654d0 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -96,3 +96,11 @@ func printProgress(status string) { _, _ = os.Stdout.Write([]byte(clear + status + carriageControl)) } + +func newIndexProgress(quiet bool, json bool) *progress.Counter { + return newProgressMax(!quiet && !json, 0, "index files loaded") +} + +func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter { + return newTerminalProgressMax(!quiet && !json, 0, "index files loaded", term) +} From 91aef00df333513819ec8e7570531e6bb4c2e209 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 1 Oct 2023 19:48:56 +0200 Subject: [PATCH 6/6] check: add index loading progress bar --- cmd/restic/cmd_check.go | 3 ++- internal/checker/checker.go | 33 +++++++++++++++++++++++++++-- internal/checker/checker_test.go | 20 ++++++++--------- internal/checker/testing.go | 2 +- internal/index/master_index_test.go | 2 +- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 3c4c9daa9..5f03c446b 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -226,7 +226,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } Verbosef("load indexes\n") - hints, errs := chkr.LoadIndex(ctx) + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + hints, errs := chkr.LoadIndex(ctx, bar) errorsFound := false suggestIndexRebuild := false diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 85e078a08..a5bb43731 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -113,13 +113,42 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID } // LoadIndex loads all index files. -func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { +func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) { debug.Log("Start") + indexList, err := backend.MemorizeList(ctx, c.repo.Backend(), restic.IndexFile) + if err != nil { + // abort if an error occurs while listing the indexes + return hints, append(errs, err) + } + + if p != nil { + var numIndexFiles uint64 + err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { + _, err := restic.ParseID(fi.Name) + if err != nil { + debug.Log("unable to parse %v as an ID", fi.Name) + return nil + } + + numIndexFiles++ + return nil + }) + if err != nil { + return hints, append(errs, err) + } + p.SetMax(numIndexFiles) + defer p.Done() + } + packToIndex := make(map[restic.ID]restic.IDSet) - err := index.ForAllIndexes(ctx, c.repo.Backend(), c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err = index.ForAllIndexes(ctx, indexList, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { debug.Log("process index %v, err %v", id, err) + if p != nil { + p.Add(1) + } + if oldFormat { debug.Log("index %v has old format", id) hints = append(hints, &ErrOldIndexFormat{id}) diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 6405ecfbd..ee7e2867c 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -77,7 +77,7 @@ func TestCheckRepo(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -103,7 +103,7 @@ func TestMissingPack(t *testing.T) { test.OK(t, repo.Backend().Remove(context.TODO(), packHandle)) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -136,7 +136,7 @@ func TestUnreferencedPack(t *testing.T) { test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle)) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -178,7 +178,7 @@ func TestUnreferencedBlobs(t *testing.T) { sort.Sort(unusedBlobsBySnapshot) chkr := checker.New(repo, true) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -258,7 +258,7 @@ func TestModifiedIndex(t *testing.T) { } chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) == 0 { t.Fatalf("expected errors not found") } @@ -279,7 +279,7 @@ func TestDuplicatePacksInIndex(t *testing.T) { repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(hints) == 0 { t.Fatalf("did not get expected checker hints for duplicate packs in indexes") } @@ -347,7 +347,7 @@ func TestCheckerModifiedData(t *testing.T) { chkr := checker.New(checkRepo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -408,7 +408,7 @@ func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { } chkr := checker.New(checkRepo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -524,7 +524,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { delayRepo.Unblock() }() - hints, errs := chkr.LoadIndex(ctx) + hints, errs := chkr.LoadIndex(ctx, nil) if len(errs) > 0 { t.Fatalf("expected no errors, got %v: %v", len(errs), errs) } @@ -553,7 +553,7 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun repo := repository.TestOpenLocal(t, repodir) chkr := checker.New(repo, false) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { defer cleanup() t.Fatalf("expected no errors, got %v: %v", len(errs), errs) diff --git a/internal/checker/testing.go b/internal/checker/testing.go index 0668406d8..fe1679393 100644 --- a/internal/checker/testing.go +++ b/internal/checker/testing.go @@ -11,7 +11,7 @@ import ( func TestCheckRepo(t testing.TB, repo restic.Repository) { chkr := New(repo, true) - hints, errs := chkr.LoadIndex(context.TODO()) + hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) != 0 { t.Fatalf("errors loading index: %v", errs) } diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index 493d11ace..bf8ec3f41 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -382,7 +382,7 @@ func testIndexSave(t *testing.T, version uint) { t.Error(err) } - hints, errs := checker.LoadIndex(context.TODO()) + hints, errs := checker.LoadIndex(context.TODO(), nil) for _, h := range hints { t.Logf("hint: %v\n", h) }