Upadte vendored dependencies

vendor/github.com/restic/chunker/.travis.yml

@@ -2,8 +2,9 @@ language: go
 sudo: false
 
 go:
-  - 1.5.4
-  - 1.6.2
+  - 1.6.4
+  - 1.7.4
+  - tip
 
 os:
   - linux

vendor/github.com/restic/chunker/chunker.go

@@ -13,16 +13,11 @@ const (
 	// WindowSize is the size of the sliding window.
 	windowSize = 64
 
-	// aim to create chunks of 20 bits or about 1MiB on average.
-	averageBits = 20
-
 	// MinSize is the default minimal size of a chunk.
 	MinSize = 512 * kiB
 	// MaxSize is the default maximal size of a chunk.
 	MaxSize = 8 * miB
 
-	splitmask = (1 << averageBits) - 1
-
 	chunkerBufSize = 512 * kiB
 )
 
@@ -74,6 +69,7 @@ type chunkerConfig struct {
 	polShift          uint
 	tables            tables
 	tablesInitialized bool
+	splitmask         uint64
 
 	rd     io.Reader
 	closed bool
@@ -85,17 +81,31 @@ type Chunker struct {
 	chunkerState
 }
 
+// SetAverageBits allows to control the frequency of chunk discovery:
+// the lower averageBits, the higher amount of chunks will be identified.
+// The default value is 20 bits, so chunks will be of 1MiB size on average.
+func (c *Chunker) SetAverageBits(averageBits int) {
+	c.splitmask = (1 << uint64(averageBits)) - 1
+}
+
 // New returns a new Chunker based on polynomial p that reads from rd.
 func New(rd io.Reader, pol Pol) *Chunker {
+	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
+}
+
+// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
+// rd and custom min and max size boundaries.
+func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
 	c := &Chunker{
 		chunkerState: chunkerState{
 			buf: make([]byte, chunkerBufSize),
 		},
 		chunkerConfig: chunkerConfig{
-			pol:     pol,
-			rd:      rd,
-			MinSize: MinSize,
-			MaxSize: MaxSize,
+			pol:       pol,
+			rd:        rd,
+			MinSize:   min,
+			MaxSize:   max,
+			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
 		},
 	}
 
@@ -106,15 +116,22 @@ func New(rd io.Reader, pol Pol) *Chunker {
 
 // Reset reinitializes the chunker with a new reader and polynomial.
 func (c *Chunker) Reset(rd io.Reader, pol Pol) {
+	c.ResetWithBoundaries(rd, pol, MinSize, MaxSize)
+}
+
+// ResetWithBoundaries reinitializes the chunker with a new reader, polynomial
+// and custom min and max size boundaries.
+func (c *Chunker) ResetWithBoundaries(rd io.Reader, pol Pol, min, max uint) {
 	*c = Chunker{
 		chunkerState: chunkerState{
 			buf: c.buf,
 		},
 		chunkerConfig: chunkerConfig{
-			pol:     pol,
-			rd:      rd,
-			MinSize: MinSize,
-			MaxSize: MaxSize,
+			pol:       pol,
+			rd:        rd,
+			MinSize:   min,
+			MaxSize:   max,
+			splitmask: (1 << 20) - 1,
 		},
 	}
 
@@ -292,7 +309,7 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 				continue
 			}
 
-			if (digest&splitmask) == 0 || add >= maxSize {
+			if (digest&c.splitmask) == 0 || add >= maxSize {
 				i := add - c.count - 1
 				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
 				c.count = add

vendor/github.com/restic/chunker/chunker_test.go

@@ -68,6 +68,41 @@ var chunks2 = []chunk{
 	chunk{MinSize, 0, parseDigest("07854d2fef297a06ba81685e660c332de36d5d18d546927d30daad6d7fda1541")},
 }
 
+// the same as chunks1, but avg chunksize is 1<<19
+var chunks3 = []chunk{
+	chunk{1491586, 0x00023e586ea80000, parseDigest("4c008237df602048039287427171cef568a6cb965d1b5ca28dc80504a24bb061")},
+	chunk{671874, 0x000b98d4cdf00000, parseDigest("fa8a42321b90c3d4ce9dd850562b2fd0c0fe4bdd26cf01a24f22046a224225d3")},
+	chunk{643703, 0x000d4e8364d00000, parseDigest("5727a63c0964f365ab8ed2ccf604912f2ea7be29759a2b53ede4d6841e397407")},
+	chunk{1284146, 0x0012b527e4780000, parseDigest("16d04cafecbeae9eaedd49da14c7ad7cdc2b1cc8569e5c16c32c9fb045aa899a")},
+	chunk{823366, 0x000d1d6752180000, parseDigest("48662c118514817825ad4761e8e2e5f28f9bd8281b07e95dcafc6d02e0aa45c3")},
+	chunk{810134, 0x0016071b6e180000, parseDigest("f629581aa05562f97f2c359890734c8574c5575da32f9289c5ba70bfd05f3f46")},
+	chunk{567118, 0x00102a8242e00000, parseDigest("d4f0797c56c60d01bac33bfd49957a4816b6c067fc155b026de8a214cab4d70a")},
+	chunk{821315, 0x001b3e42c8180000, parseDigest("8ebd0fd5db0293bd19140da936eb8b1bbd3cd6ffbec487385b956790014751ca")},
+	chunk{1401057, 0x00045da878000000, parseDigest("001360af59adf4871ef138cfa2bb49007e86edaf5ac2d6f0b3d3014510991848")},
+	chunk{2311122, 0x0005cbd885380000, parseDigest("8276d489b566086d9da95dc5c5fe6fc7d72646dd3308ced6b5b6ddb8595f0aa1")},
+	chunk{608723, 0x001cfcd86f280000, parseDigest("518db33ba6a79d4f3720946f3785c05b9611082586d47ea58390fc2f6de9449e")},
+	chunk{980456, 0x0013edb7a7f80000, parseDigest("0121b1690738395e15fecba1410cd0bf13fde02225160cad148829f77e7b6c99")},
+	chunk{1140278, 0x0001f9f017e80000, parseDigest("28ca7c74804b5075d4f5eeb11f0845d99f62e8ea3a42b9a05c7bd5f2fca619dd")},
+	chunk{2015542, 0x00097bf5d8180000, parseDigest("6fe8291f427d48650a5f0f944305d3a2dbc649bd401d2655fc0bdd42e890ca5a")},
+	chunk{904752, 0x000e1863eff80000, parseDigest("62af1f1eb3f588d18aff28473303cc4731fc3cafcc52ce818fee3c4c2820854d")},
+	chunk{713072, 0x001f3bb1b9b80000, parseDigest("4bda9dc2e3031d004d87a5cc93fe5207c4b0843186481b8f31597dc6ffa1496c")},
+	chunk{675937, 0x001fec043c700000, parseDigest("5299c8c5acec1b90bb020cd75718aab5e12abb9bf66291465fd10e6a823a8b4a")},
+	chunk{1525894, 0x000b1574b1500000, parseDigest("2f238180e4ca1f7520a05f3d6059233926341090f9236ce677690c1823eccab3")},
+	chunk{1352720, 0x00018965f2e00000, parseDigest("afd12f13286a3901430de816e62b85cc62468c059295ce5888b76b3af9028d84")},
+	chunk{811884, 0x00155628aa100000, parseDigest("42d0cdb1ee7c48e552705d18e061abb70ae7957027db8ae8db37ec756472a70a")},
+	chunk{1282314, 0x001909a0a1400000, parseDigest("819721c2457426eb4f4c7565050c44c32076a56fa9b4515a1c7796441730eb58")},
+	chunk{1093738, 0x0017f5d048880000, parseDigest("5dddfa7a241b68f65d267744bdb082ee865f3c2f0d8b946ea0ee47868a01bbff")},
+	chunk{962003, 0x000b921f7ef80000, parseDigest("0cb5c9ebba196b441c715c8d805f6e7143a81cd5b0d2c65c6aacf59ca9124af9")},
+	chunk{856384, 0x00030ce2d9400000, parseDigest("7734b206d46f3f387e8661e81edf5b1a91ea681867beb5831c18aaa86632d7fb")},
+	chunk{533758, 0x0004435c53c00000, parseDigest("4da778a25b72a9a0d53529eccfe2e5865a789116cb1800f470d8df685a8ab05d")},
+	chunk{1128303, 0x0000c48517800000, parseDigest("08c6b0b38095b348d80300f0be4c5184d2744a17147c2cba5cc4315abf4c048f")},
+	chunk{800374, 0x000968473f900000, parseDigest("820284d2c8fd243429674c996d8eb8d3450cbc32421f43113e980f516282c7bf")},
+	chunk{2453512, 0x001e197c92600000, parseDigest("5fa870ed107c67704258e5e50abe67509fb73562caf77caa843b5f243425d853")},
+	chunk{665901, 0x00118c842cb80000, parseDigest("deceec26163842fdef6560311c69bf8a9871a56e16d719e2c4b7e4d668ceb61f")},
+	chunk{1986074, 0x000ae6c868000000, parseDigest("64cd64bf3c3bc389eb20df8310f0427d1c36ab2eaaf09e346bfa7f0453fc1a18")},
+	chunk{237392, 0x0000000000000001, parseDigest("fcd567f5d866357a8e299fd5b2359bb2c8157c30395229c4e9b0a353944a7978")},
+}
+
 func testWithData(t *testing.T, chnker *Chunker, testChunks []chunk, checkDigest bool) []Chunk {
 	chunks := []Chunk{}
 
@@ -108,16 +143,20 @@ func testWithData(t *testing.T, chnker *Chunker, testChunks []chunk, checkDigest
 
 	_, err := chnker.Next(nil)
 	if err != io.EOF {
-		t.Fatal("wrong error returned after last chunk")
+		t.Fatal("Wrong error returned after last chunk")
+	}
+
+	if len(chunks) != len(testChunks) {
+		t.Fatal("Amounts of test and resulting chunks do not match")
 	}
 
 	return chunks
 }
 
-func getRandom(seed, count int) []byte {
+func getRandom(seed int64, count int) []byte {
 	buf := make([]byte, count)
 
-	rnd := rand.New(rand.NewSource(23))
+	rnd := rand.New(rand.NewSource(seed))
 	for i := 0; i < count; i += 4 {
 		r := rnd.Uint32()
 		buf[i] = byte(r)
@@ -148,6 +187,15 @@ func TestChunker(t *testing.T) {
 	testWithData(t, ch, chunks2, true)
 }
 
+func TestChunkerWithCustomAverageBits(t *testing.T) {
+	buf := getRandom(23, 32*1024*1024)
+	ch := New(bytes.NewReader(buf), testPol)
+
+	// sligthly decrease averageBits to get more chunks
+	ch.SetAverageBits(19)
+	testWithData(t, ch, chunks3, true)
+}
+
 func TestChunkerReset(t *testing.T) {
 	buf := getRandom(23, 32*1024*1024)
 	ch := New(bytes.NewReader(buf), testPol)