From 99634c09367ac658fa83a4c8959f003daabd3ce6 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 1 May 2022 14:26:57 +0200 Subject: [PATCH] Return real size from SaveBlob --- cmd/restic/cmd_debug.go | 2 +- internal/archiver/archiver_test.go | 12 ++++----- internal/archiver/blob_saver.go | 6 +++-- internal/archiver/blob_saver_test.go | 6 ++--- internal/checker/checker_test.go | 2 +- internal/repository/fuzz_test.go | 2 +- internal/repository/repack.go | 2 +- internal/repository/repack_test.go | 4 +-- internal/repository/repository.go | 34 ++++++++++++++++---------- internal/repository/repository_test.go | 12 ++++----- internal/restic/repository.go | 2 +- internal/restic/testing.go | 4 +-- internal/restorer/restorer_test.go | 2 +- 13 files changed, 50 insertions(+), 40 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 8efb6c8a3..2a9115b19 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -386,7 +386,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list } } if reuploadBlobs { - _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) + _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) if err != nil { return err } diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 13c82a294..e181bc02a 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -415,16 +415,16 @@ type blobCountingRepo struct { saved map[restic.BlobHandle]uint } -func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) { - id, exists, err := repo.Repository.SaveBlob(ctx, t, buf, id, false) +func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { + id, exists, size, err := repo.Repository.SaveBlob(ctx, t, buf, id, false) if exists { - return id, exists, err + return id, exists, size, err } h := restic.BlobHandle{ID: id, Type: t} repo.m.Lock() repo.saved[h]++ repo.m.Unlock() - return id, exists, err + return id, exists, size, err } func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { @@ -1944,10 +1944,10 @@ type failSaveRepo struct { err error } -func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) { +func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { val := atomic.AddInt32(&f.cnt, 1) if val >= f.failAfter { - return restic.ID{}, false, f.err + return restic.ID{}, false, 0, f.err } return f.Repository.SaveBlob(ctx, t, buf, id, storeDuplicate) diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index 1ee319091..ffd9cb7e4 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -10,7 +10,7 @@ import ( // Saver allows saving a blob. type Saver interface { - SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, error) + SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) Index() restic.MasterIndex } @@ -100,10 +100,11 @@ type saveBlobJob struct { type saveBlobResponse struct { id restic.ID known bool + size int } func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) { - id, known, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) + id, known, size, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false) if err != nil { return saveBlobResponse{}, err @@ -112,6 +113,7 @@ func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) return saveBlobResponse{ id: id, known: known, + size: size, }, nil } diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index 54aa374cf..69cd4c2e2 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -21,13 +21,13 @@ type saveFail struct { failAt int32 } -func (b *saveFail) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicates bool) (restic.ID, bool, error) { +func (b *saveFail) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicates bool) (restic.ID, bool, int, error) { val := atomic.AddInt32(&b.cnt, 1) if val == b.failAt { - return restic.ID{}, false, errTest + return restic.ID{}, false, 0, errTest } - return id, false, nil + return id, false, 0, nil } func (b *saveFail) Index() restic.MasterIndex { diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 1a9a3b8f3..f658613c3 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -483,7 +483,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { buf, err := repo.LoadBlob(ctx, restic.TreeBlob, id, nil) test.OK(t, err) - _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) + _, _, _, err = repo.SaveBlob(ctx, restic.DataBlob, buf, id, false) test.OK(t, err) malNode := &restic.Node{ diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index 5af134d84..3847f37f7 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -23,7 +23,7 @@ func FuzzSaveLoadBlob(f *testing.F) { id := restic.Hash(blob) repo, _ := TestRepositoryWithBackend(t, mem.New(), 2) - _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false) + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, blob, id, false) if err != nil { t.Fatal(err) } diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 4d0ca8236..80902c11c 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -75,7 +75,7 @@ func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito } // We do want to save already saved blobs! - _, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true) + _, _, _, err = dstRepo.SaveBlob(wgCtx, blob.Type, buf, blob.ID, true) if err != nil { return err } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index f8e375a52..248477292 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -32,7 +32,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl buf := make([]byte, length) rand.Read(buf) - id, exists, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false) + id, exists, _, err := repo.SaveBlob(context.TODO(), tpe, buf, restic.ID{}, false) if err != nil { t.Fatalf("SaveFrom() error %v", err) } @@ -62,7 +62,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { // invert first data byte buf[0] ^= 0xff - _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false) + _, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, id, false) if err != nil { t.Fatalf("SaveFrom() error %v", err) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 1312ea754..beaf7fecc 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -378,9 +378,10 @@ func (r *Repository) getZstdDecoder() *zstd.Decoder { } // saveAndEncrypt encrypts data and stores it to the backend as type t. If data -// is small enough, it will be packed together with other small blobs. -// The caller must ensure that the id matches the data. -func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) error { +// is small enough, it will be packed together with other small blobs. The +// caller must ensure that the id matches the data. Returned is the size data +// occupies in the repo (compressed or not, including the encryption overhead). +func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) (size int, err error) { debug.Log("save id %v (%v, %d bytes)", id, t, len(data)) uncompressedLength := 0 @@ -417,24 +418,29 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data packer, err := pm.findPacker() if err != nil { - return err + return 0, err } // save ciphertext - _, err = packer.Add(t, id, ciphertext, uncompressedLength) + size, err = packer.Add(t, id, ciphertext, uncompressedLength) if err != nil { - return err + return 0, err } // if the pack is not full enough, put back to the list if packer.Size() < minPackSize { debug.Log("pack is not full enough (%d bytes)", packer.Size()) pm.insertPacker(packer) - return nil + return size, nil } // else write the pack to the backend - return r.savePacker(ctx, t, packer) + err = r.savePacker(ctx, t, packer) + if err != nil { + return 0, err + } + + return size, nil } // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the @@ -815,8 +821,10 @@ func (r *Repository) Close() error { // It takes care that no duplicates are saved; this can be overwritten // by setting storeDuplicate to true. // If id is the null id, it will be computed and returned. -// Also returns if the blob was already known before -func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, err error) { +// Also returns if the blob was already known before. +// If the blob was not known before, it returns the number of bytes the blob +// occupies in the repo (compressed or not, including encryption overhead). +func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) { // compute plaintext hash if not already set if id.IsNull() { @@ -830,10 +838,10 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte // only save when needed or explicitly told if !known || storeDuplicate { - err = r.saveAndEncrypt(ctx, t, buf, newID) + size, err = r.saveAndEncrypt(ctx, t, buf, newID) } - return newID, known, err + return newID, known, size, err } // LoadTree loads a tree from the repository. @@ -867,7 +875,7 @@ func (r *Repository) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, e // adds a newline after each object) buf = append(buf, '\n') - id, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) + id, _, _, err := r.SaveBlob(ctx, restic.TreeBlob, buf, restic.ID{}, false) return id, err } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 2265ba453..f0e25f520 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -44,7 +44,7 @@ func testSave(t *testing.T, version uint) { id := restic.Hash(data) // save - sid, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) rtest.OK(t, err) rtest.Equals(t, id, sid) @@ -83,7 +83,7 @@ func testSaveFrom(t *testing.T, version uint) { id := restic.Hash(data) // save - id2, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) + id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) rtest.OK(t, err) rtest.Equals(t, id, id2) @@ -125,7 +125,7 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) { t.SetBytes(int64(size)) for i := 0; i < t.N; i++ { - _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, true) rtest.OK(t, err) } } @@ -187,7 +187,7 @@ func testLoadBlob(t *testing.T, version uint) { _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) - id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(t, err) rtest.OK(t, repo.Flush(context.Background())) @@ -220,7 +220,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) { _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) - id, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + id, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(b, err) rtest.OK(b, repo.Flush(context.Background())) @@ -396,7 +396,7 @@ func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) - _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) + _, _, _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}, false) rtest.OK(t, err) } } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index fea151164..35fdbabcb 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -46,7 +46,7 @@ type Repository interface { LoadUnpacked(ctx context.Context, buf []byte, t FileType, id ID) (data []byte, err error) LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) - SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, error) + SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) LoadTree(context.Context, ID) (*Tree, error) SaveTree(context.Context, *Tree) (ID, error) diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 392538506..54621c183 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -52,7 +52,7 @@ func (fs *fakeFileSystem) saveFile(ctx context.Context, rd io.Reader) (blobs IDs id := Hash(chunk.Data) if !fs.blobIsKnown(BlobHandle{ID: id, Type: DataBlob}) { - _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) + _, _, _, err := fs.repo.SaveBlob(ctx, DataBlob, chunk.Data, id, true) if err != nil { fs.t.Fatalf("error saving chunk: %v", err) } @@ -138,7 +138,7 @@ func (fs *fakeFileSystem) saveTree(ctx context.Context, seed int64, depth int) I return id } - _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) + _, _, _, err := fs.repo.SaveBlob(ctx, TreeBlob, buf, id, false) if err != nil { fs.t.Fatal(err) } diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index a5a3bb5ba..7e7e0c4c6 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -41,7 +41,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - id, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) + id, _, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) if err != nil { t.Fatal(err) }