Update the chunker

This commit is contained in:
Alexander Neumann
2020-02-12 21:29:26 +01:00
parent 81c0b031f9
commit b229aa5cf1
8 changed files with 28 additions and 18 deletions
+11 -9
View File
@@ -1,18 +1,20 @@
language: go
sudo: false
go:
- 1.6.4
- 1.7.4
- tip
os:
- linux
- osx
matrix:
include:
- os: linux
go: "1.9.x"
- os: linux
go: "1.10.x"
- os: linux
go: "tip"
- os: osx
go: "1.10.x"
install:
- go get -t ./...
- go get -u github.com/golang/lint/golint
- go get -u golang.org/x/lint/golint
- go get -u golang.org/x/tools/cmd/goimports
script:
+1 -1
View File
@@ -1,5 +1,5 @@
[![GoDoc](https://godoc.org/github.com/restic/chunker?status.svg)](http://godoc.org/github.com/restic/chunker)
[![Build Status](https://travis-ci.org/restic/chunker.svg?branch=master)](https://travis-ci.org/restic/chunker)
[![Build Status](https://travis-ci.com/restic/chunker.svg?branch=master)](https://travis-ci.com/restic/chunker)
The package `chunker` implements content-defined-chunking (CDC) based on a
rolling Rabin Hash. The library is part of the [restic backup
+11 -3
View File
@@ -47,7 +47,7 @@ type Chunk struct {
type chunkerState struct {
window [windowSize]byte
wpos int
wpos uint
buf []byte
bpos uint
@@ -225,6 +225,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
tabout := c.tables.out
tabmod := c.tables.mod
polShift := c.polShift
// go guarantees the expected behavior for bit shifts even for shift counts
// larger than the value width. Bounding the value of polShift allows the compiler
// to optimize the code for 'digest >> polShift'
if polShift > 53-8 {
return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
}
minSize := c.MinSize
maxSize := c.MaxSize
buf := c.buf
@@ -291,10 +297,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
wpos := c.wpos
for _, b := range buf[c.bpos:c.bmax] {
// slide(b)
// limit wpos before to elide array bound checks
wpos = wpos % windowSize
out := win[wpos]
win[wpos] = b
digest ^= uint64(tabout[out])
wpos = (wpos + 1) % windowSize
wpos++
// updateDigest
index := byte(digest >> polShift)
@@ -331,7 +339,7 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
}
c.digest = digest
c.window = win
c.wpos = wpos
c.wpos = wpos % windowSize
steps := c.bmax - c.bpos
if steps > 0 {
+1
View File
@@ -0,0 +1 @@
module github.com/restic/chunker
-1
View File
@@ -94,7 +94,6 @@ func (x Pol) Deg() int {
}
if uint64(x)&0x2 > 0 {
x >>= 1
r |= 1
}