chunker: Add buffer to New()

chunker/chunker.go

@@ -82,11 +82,18 @@ type Chunker struct {
 	h      hash.Hash
 }
 
-// New returns a new Chunker based on polynomial p that reads from data from rd
-// with bufsize and pass all data to hash along the way.
-func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker {
+const minBufSize = 32
+
+// New returns a new Chunker based on polynomial p that reads from rd
+// with bufsize and pass all data to hash along the way, using buf for
+// buffering. Buf must at least hold 32 bytes.
+func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker {
+	if len(buf) < minBufSize {
+		buf = make([]byte, minBufSize)
+	}
+
 	c := &Chunker{
-		buf: make([]byte, bufSize),
+		buf: buf,
 		h:   h,
 		pol: pol,
 		rd:  rd,

chunker/chunker_test.go

@@ -147,10 +147,13 @@ func getRandom(seed, count int) []byte {
 	return buf
 }
 
+const chunkerBufSize = 512 * chunker.KiB
+
 func TestChunker(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New())
+	chunkBuf := make([]byte, chunkerBufSize)
+	ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 	chunks := testWithData(t, ch, chunks1)
 
 	// test reader
@@ -177,7 +180,7 @@ func TestChunker(t *testing.T) {
 
 	// setup nullbyte data source
 	buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
-	ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
+	ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 
 	testWithData(t, ch, chunks2)
 }
@@ -185,6 +188,7 @@ func TestChunker(t *testing.T) {
 func TestChunkerWithRandomPolynomial(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
+	chunkBuf := make([]byte, chunkerBufSize)
 
 	// generate a new random polynomial
 	start := time.Now()
@@ -193,7 +197,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
 	t.Logf("generating random polynomial took %v", time.Since(start))
 
 	start = time.Now()
-	ch := chunker.New(bytes.NewReader(buf), p, sha256.New())
+	ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New())
 	t.Logf("creating chunker took %v", time.Since(start))
 
 	// make sure that first chunk is different
@@ -210,7 +214,9 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
 func TestChunkerWithoutHash(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	ch := chunker.New(bytes.NewReader(buf), testPol, nil)
+	chunkBuf := make([]byte, chunkerBufSize)
+
+	ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil)
 	chunks := testWithData(t, ch, chunks1)
 
 	// test reader
@@ -240,7 +246,7 @@ func TestChunkerWithoutHash(t *testing.T) {
 
 	// setup nullbyte data source
 	buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
-	ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
+	ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 
 	testWithData(t, ch, chunks2)
 }
@@ -270,6 +276,8 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
 		rd = bytes.NewReader(getRandom(23, size))
 	}
 
+	chunkBuf := make([]byte, chunkerBufSize)
+
 	b.ResetTimer()
 	b.SetBytes(int64(size))
 
@@ -278,7 +286,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
 		chunks = 0
 
 		rd.Seek(0, 0)
-		ch := chunker.New(rd, testPol, hash)
+		ch := chunker.New(rd, testPol, chunkBuf, hash)
 
 		for {
 			_, err := ch.Next()
@@ -314,9 +322,11 @@ func BenchmarkNewChunker(b *testing.B) {
 	p, err := chunker.RandomPolynomial()
 	OK(b, err)
 
+	chunkBuf := make([]byte, chunkerBufSize)
+
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		chunker.New(bytes.NewBuffer(nil), p, nil)
+		chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil)
 	}
 }