From 6c4ceaf1e77b8cd8ad10c1f08957271c7faaa921 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 1 May 2022 14:41:36 +0200 Subject: [PATCH] Print number of bytes added to the repo This includes optional compression and crypto overhead. --- internal/archiver/archiver.go | 15 ++++--- internal/archiver/archiver_test.go | 11 ++--- internal/archiver/blob_saver.go | 8 +++- internal/archiver/file_saver.go | 1 + internal/ui/backup/json.go | 66 +++++++++++++++++------------- internal/ui/backup/text.go | 8 ++-- 6 files changed, 65 insertions(+), 44 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 2c7af3ae0..43153030a 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -31,18 +31,22 @@ type ErrorFunc func(file string, fi os.FileInfo, err error) error // ItemStats collects some statistics about a particular file or directory. type ItemStats struct { - DataBlobs int // number of new data blobs added for this item - DataSize uint64 // sum of the sizes of all new data blobs - TreeBlobs int // number of new tree blobs added for this item - TreeSize uint64 // sum of the sizes of all new tree blobs + DataBlobs int // number of new data blobs added for this item + DataSize uint64 // sum of the sizes of all new data blobs + DataSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) + TreeBlobs int // number of new tree blobs added for this item + TreeSize uint64 // sum of the sizes of all new tree blobs + TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) } // Add adds other to the current ItemStats. func (s *ItemStats) Add(other ItemStats) { s.DataBlobs += other.DataBlobs s.DataSize += other.DataSize + s.DataSizeInRepo += other.DataSizeInRepo s.TreeBlobs += other.TreeBlobs s.TreeSize += other.TreeSize + s.TreeSizeInRepo += other.TreeSizeInRepo } // Archiver saves a directory structure to the repo. @@ -183,7 +187,8 @@ func (arch *Archiver) saveTree(ctx context.Context, t *restic.Tree) (restic.ID, res.Wait(ctx) if !res.Known() { s.TreeBlobs++ - s.TreeSize += uint64(len(buf)) + s.TreeSize += uint64(res.Length()) + s.TreeSizeInRepo += uint64(res.SizeInRepo()) } // The context was canceled in the meantime, res.ID() might be invalid if ctx.Err() != nil { diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index e181bc02a..dfc24b27b 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1019,7 +1019,7 @@ func TestArchiverSaveTree(t *testing.T) { want: TestDir{ "targetfile": TestFile{Content: string("foobar")}, }, - stat: ItemStats{1, 6, 0, 0}, + stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, }, { src: TestDir{ @@ -1031,7 +1031,7 @@ func TestArchiverSaveTree(t *testing.T) { "targetfile": TestFile{Content: string("foobar")}, "filesymlink": TestSymlink{Target: "targetfile"}, }, - stat: ItemStats{1, 6, 0, 0}, + stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, }, { src: TestDir{ @@ -1051,7 +1051,7 @@ func TestArchiverSaveTree(t *testing.T) { "symlink": TestSymlink{Target: "subdir"}, }, }, - stat: ItemStats{0, 0, 1, 0x154}, + stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a}, }, { src: TestDir{ @@ -1075,7 +1075,7 @@ func TestArchiverSaveTree(t *testing.T) { }, }, }, - stat: ItemStats{1, 6, 3, 0x47f}, + stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1}, }, } @@ -1140,7 +1140,8 @@ func TestArchiverSaveTree(t *testing.T) { bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs)) bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs)) bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize) - bothZeroOrNeither(t, test.stat.TreeSize, stat.TreeSize) + bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo) + bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo) }) } } diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index ffd9cb7e4..eca158c20 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -86,11 +86,17 @@ func (s *FutureBlob) Known() bool { return s.res.known } -// Length returns the length of the blob. +// Length returns the raw length of the blob. func (s *FutureBlob) Length() int { return s.length } +// SizeInRepo returns the number of bytes added to the repo (including +// compression and crypto overhead). +func (s *FutureBlob) SizeInRepo() int { + return s.res.size +} + type saveBlobJob struct { restic.BlobType buf *Buffer diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 9da589f19..0ec871e8b 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -210,6 +210,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat if !res.Known() { stats.DataBlobs++ stats.DataSize += uint64(res.Length()) + stats.DataSizeInRepo += uint64(res.SizeInRepo()) } node.Content = append(node.Content, res.ID()) diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go index fcc200b2a..3e8ce5e94 100644 --- a/internal/ui/backup/json.go +++ b/internal/ui/backup/json.go @@ -110,12 +110,14 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current switch messageType { case "dir new": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "new", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, - MetadataSize: s.TreeSize, + MessageType: "verbose_status", + Action: "new", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, + MetadataSize: s.TreeSize, + MetadataSizeInRepo: s.TreeSizeInRepo, }) case "dir unchanged": b.print(verboseUpdate{ @@ -125,20 +127,23 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current }) case "dir modified": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "modified", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, - MetadataSize: s.TreeSize, + MessageType: "verbose_status", + Action: "modified", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, + MetadataSize: s.TreeSize, + MetadataSizeInRepo: s.TreeSizeInRepo, }) case "file new": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "new", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, + MessageType: "verbose_status", + Action: "new", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, }) case "file unchanged": b.print(verboseUpdate{ @@ -148,11 +153,12 @@ func (b *JSONProgress) CompleteItem(messageType, item string, previous, current }) case "file modified": b.print(verboseUpdate{ - MessageType: "verbose_status", - Action: "modified", - Item: item, - Duration: d.Seconds(), - DataSize: s.DataSize, + MessageType: "verbose_status", + Action: "modified", + Item: item, + Duration: d.Seconds(), + DataSize: s.DataSize, + DataSizeInRepo: s.DataSizeInRepo, }) } } @@ -216,13 +222,15 @@ type errorUpdate struct { } type verboseUpdate struct { - MessageType string `json:"message_type"` // "verbose_status" - Action string `json:"action"` - Item string `json:"item"` - Duration float64 `json:"duration"` // in seconds - DataSize uint64 `json:"data_size"` - MetadataSize uint64 `json:"metadata_size"` - TotalFiles uint `json:"total_files"` + MessageType string `json:"message_type"` // "verbose_status" + Action string `json:"action"` + Item string `json:"item"` + Duration float64 `json:"duration"` // in seconds + DataSize uint64 `json:"data_size"` + DataSizeInRepo uint64 `json:"data_size_in_repo"` + MetadataSize uint64 `json:"metadata_size"` + MetadataSizeInRepo uint64 `json:"metadata_size_in_repo"` + TotalFiles uint `json:"total_files"` } type summaryOutput struct { diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go index 998805865..85408e467 100644 --- a/internal/ui/backup/text.go +++ b/internal/ui/backup/text.go @@ -138,17 +138,17 @@ func formatBytes(c uint64) string { func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { switch messageType { case "dir new": - b.VV("new %v, saved in %.3fs (%v added, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.TreeSize)) + b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo)) case "dir unchanged": b.VV("unchanged %v", item) case "dir modified": - b.VV("modified %v, saved in %.3fs (%v added, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.TreeSize)) + b.VV("modified %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo)) case "file new": b.VV("new %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize)) case "file unchanged": b.VV("unchanged %v", item) case "file modified": - b.VV("modified %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize)) + b.VV("modified %v, saved in %.3fs (%v added, %v stored)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo)) } } @@ -178,7 +178,7 @@ func (b *TextProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su if dryRun { verb = "Would add" } - b.P("%s to the repo: %-5s\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize)) + b.P("%s to the repo: %-5s (%-5s stored)\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize), formatBytes(summary.ItemStats.DataSizeInRepo+summary.ItemStats.TreeSizeInRepo)) b.P("\n") b.P("processed %v files, %v in %s", summary.Files.New+summary.Files.Changed+summary.Files.Unchanged,