diff --git a/chunker/chunker.go b/chunker/chunker.go index 6e051d8c4..d2e35a107 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -43,12 +43,17 @@ type Chunk struct { // A chunker takes a stream of bytes and emits average size chunks. type Chunker interface { + // Next returns the next chunk of data. If an error occurs while reading, + // the error is returned. The state of the current chunk is undefined. When + // the last chunk has been returned, all subsequent calls yield a nil chunk + // and an io.EOF error. Next() (*Chunk, error) } // A chunker internally holds everything needed to split content. type chunker struct { - rd io.Reader + rd io.Reader + closed bool window []byte wpos int @@ -141,13 +146,25 @@ func (c *chunker) Next() (*Chunk, error) { err = nil } - if err != nil { + // io.ReadFull only returns io.EOF when no bytes could be read. If + // this is the case and we're in this branch, there are no more + // bytes to buffer, so this was the last chunk. If a different + // error has occurred, return that error and abandon the current + // chunk. + if err == io.EOF && !c.closed { + c.closed = true + + // return current chunk return &Chunk{ Start: c.start, Length: c.count, Cut: c.digest, Data: c.data, - }, err + }, nil + } + + if err != nil { + return nil, err } c.bpos = 0 diff --git a/chunker/chunker_test.go b/chunker/chunker_test.go index b1e39711c..5ec4fbe1e 100644 --- a/chunker/chunker_test.go +++ b/chunker/chunker_test.go @@ -50,14 +50,8 @@ func test_with_data(t *testing.T, chunker chunker.Chunker, chunks []chunk) { for i, chunk := range chunks { c, err := chunker.Next() - if i < len(chunks)-1 { - if err != nil { - t.Fatalf("Error returned with chunk %d: %v", i, err) - } - } else { - if err != io.EOF { - t.Fatalf("EOF not returned with chunk %d", i) - } + if err != nil { + t.Fatalf("Error returned with chunk %d: %v", i, err) } if c == nil { @@ -76,6 +70,16 @@ func test_with_data(t *testing.T, chunker chunker.Chunker, chunks []chunk) { } } } + + c, err := chunker.Next() + + if c != nil { + t.Fatal("additional non-nil chunk returned") + } + + if err != io.EOF { + t.Fatal("wrong error returned after last chunk") + } } func get_random(seed, count int) []byte { @@ -117,15 +121,15 @@ func BenchmarkChunker(b *testing.B) { for { _, err := ch.Next() - chunks++ - - if err != nil && err != io.EOF { - b.Fatalf("Unexpected error occurred: %v", err) - } - if err == io.EOF { break } + + if err != nil { + b.Fatalf("Unexpected error occurred: %v", err) + } + + chunks++ } }