mirror of
https://github.com/restic/restic.git
synced 2026-10-05 02:10:29 +00:00
repository: restrict SaveUnpacked and RemoveUnpacked
Those methods now only allow modifying snapshots. Internal data types used by the repository are now read-only. The repository-internal code can bypass the restrictions by wrapping the repository in an `internalRepository` type. The restriction itself is implemented by using a new datatype WriteableFileType in the SaveUnpacked and RemoveUnpacked methods. This statically ensures that code cannot bypass the access restrictions. The test changes are somewhat noisy as some of them modify repository internals and therefore require some way to bypass the access restrictions. This works by capturing an `internalRepository` or `Backend` when creating the Repository using a test helper function.
This commit is contained in:
Michael Eischer
@@ -87,7 +87,7 @@ func LoadConfig(ctx context.Context, r LoaderUnpacked) (Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func SaveConfig(ctx context.Context, r SaverUnpacked, cfg Config) error {
|
||||
func SaveConfig(ctx context.Context, r SaverUnpacked[FileType], cfg Config) error {
|
||||
_, err := SaveJSONUnpacked(ctx, r, ConfigFile, cfg)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ func LoadJSONUnpacked(ctx context.Context, repo LoaderUnpacked, t FileType, id I
|
||||
|
||||
// SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the
|
||||
// backend as type t, without a pack. It returns the storage hash.
|
||||
func SaveJSONUnpacked(ctx context.Context, repo SaverUnpacked, t FileType, item interface{}) (ID, error) {
|
||||
func SaveJSONUnpacked[FT FileTypes](ctx context.Context, repo SaverUnpacked[FT], t FT, item interface{}) (ID, error) {
|
||||
debug.Log("save new blob %v", t)
|
||||
plaintext, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
|
||||
+7
-54
@@ -7,7 +7,6 @@ import (
|
||||
"os/signal"
|
||||
"os/user"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -39,7 +38,7 @@ type Lock struct {
|
||||
UID uint32 `json:"uid,omitempty"`
|
||||
GID uint32 `json:"gid,omitempty"`
|
||||
|
||||
repo Unpacked
|
||||
repo Unpacked[FileType]
|
||||
lockID *ID
|
||||
}
|
||||
|
||||
@@ -87,20 +86,6 @@ func IsInvalidLock(err error) bool {
|
||||
|
||||
var ErrRemovedLock = errors.New("lock file was removed in the meantime")
|
||||
|
||||
// NewLock returns a new, non-exclusive lock for the repository. If an
|
||||
// exclusive lock is already held by another process, it returns an error
|
||||
// that satisfies IsAlreadyLocked.
|
||||
func NewLock(ctx context.Context, repo Unpacked) (*Lock, error) {
|
||||
return newLock(ctx, repo, false)
|
||||
}
|
||||
|
||||
// NewExclusiveLock returns a new, exclusive lock for the repository. If
|
||||
// another lock (normal and exclusive) is already held by another process,
|
||||
// it returns an error that satisfies IsAlreadyLocked.
|
||||
func NewExclusiveLock(ctx context.Context, repo Unpacked) (*Lock, error) {
|
||||
return newLock(ctx, repo, true)
|
||||
}
|
||||
|
||||
var waitBeforeLockCheck = 200 * time.Millisecond
|
||||
|
||||
// delay increases by factor 2 on each retry
|
||||
@@ -113,11 +98,15 @@ func TestSetLockTimeout(t testing.TB, d time.Duration) {
|
||||
initialWaitBetweenLockRetries = d
|
||||
}
|
||||
|
||||
func newLock(ctx context.Context, repo Unpacked, excl bool) (*Lock, error) {
|
||||
// NewLock returns a new lock for the repository. If an
|
||||
// exclusive lock is already held by another process, it returns an error
|
||||
// that satisfies IsAlreadyLocked. If the new lock is exclude, then other
|
||||
// non-exclusive locks also result in an IsAlreadyLocked error.
|
||||
func NewLock(ctx context.Context, repo Unpacked[FileType], exclusive bool) (*Lock, error) {
|
||||
lock := &Lock{
|
||||
Time: time.Now(),
|
||||
PID: os.Getpid(),
|
||||
Exclusive: excl,
|
||||
Exclusive: exclusive,
|
||||
repo: repo,
|
||||
}
|
||||
|
||||
@@ -444,42 +433,6 @@ func LoadLock(ctx context.Context, repo LoaderUnpacked, id ID) (*Lock, error) {
|
||||
return lock, nil
|
||||
}
|
||||
|
||||
// RemoveStaleLocks deletes all locks detected as stale from the repository.
|
||||
func RemoveStaleLocks(ctx context.Context, repo Unpacked) (uint, error) {
|
||||
var processed uint
|
||||
err := ForAllLocks(ctx, repo, nil, func(id ID, lock *Lock, err error) error {
|
||||
if err != nil {
|
||||
// ignore locks that cannot be loaded
|
||||
debug.Log("ignore lock %v: %v", id, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
if lock.Stale() {
|
||||
err = repo.RemoveUnpacked(ctx, LockFile, id)
|
||||
if err == nil {
|
||||
processed++
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
return processed, err
|
||||
}
|
||||
|
||||
// RemoveAllLocks removes all locks forcefully.
|
||||
func RemoveAllLocks(ctx context.Context, repo Unpacked) (uint, error) {
|
||||
var processed uint32
|
||||
err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, _ int64) error {
|
||||
err := repo.RemoveUnpacked(ctx, LockFile, id)
|
||||
if err == nil {
|
||||
atomic.AddUint32(&processed, 1)
|
||||
}
|
||||
return err
|
||||
})
|
||||
return uint(processed), err
|
||||
}
|
||||
|
||||
// ForAllLocks reads all locks in parallel and calls the given callback.
|
||||
// It is guaranteed that the function is not run concurrently. If the
|
||||
// callback returns an error, this function is cancelled and also returns that error.
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestLock(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
lock, err := restic.NewLock(context.TODO(), repo)
|
||||
lock, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.OK(t, lock.Unlock(context.TODO()))
|
||||
@@ -29,7 +29,7 @@ func TestDoubleUnlock(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
lock, err := restic.NewLock(context.TODO(), repo)
|
||||
lock, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.OK(t, lock.Unlock(context.TODO()))
|
||||
@@ -43,10 +43,10 @@ func TestMultipleLock(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
lock1, err := restic.NewLock(context.TODO(), repo)
|
||||
lock1, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
|
||||
lock2, err := restic.NewLock(context.TODO(), repo)
|
||||
lock2, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.OK(t, lock1.Unlock(context.TODO()))
|
||||
@@ -69,10 +69,10 @@ func TestMultipleLockFailure(t *testing.T) {
|
||||
repo, _ := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{})
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
lock1, err := restic.NewLock(context.TODO(), repo)
|
||||
lock1, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
|
||||
_, err = restic.NewLock(context.TODO(), repo)
|
||||
_, err = repository.TestNewLock(t, repo, false)
|
||||
rtest.Assert(t, err != nil, "unreadable lock file did not result in an error")
|
||||
|
||||
rtest.OK(t, lock1.Unlock(context.TODO()))
|
||||
@@ -81,7 +81,7 @@ func TestMultipleLockFailure(t *testing.T) {
|
||||
func TestLockExclusive(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
|
||||
elock, err := restic.NewExclusiveLock(context.TODO(), repo)
|
||||
elock, err := repository.TestNewLock(t, repo, true)
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, elock.Unlock(context.TODO()))
|
||||
}
|
||||
@@ -90,10 +90,10 @@ func TestLockOnExclusiveLockedRepo(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
elock, err := restic.NewExclusiveLock(context.TODO(), repo)
|
||||
elock, err := repository.TestNewLock(t, repo, true)
|
||||
rtest.OK(t, err)
|
||||
|
||||
lock, err := restic.NewLock(context.TODO(), repo)
|
||||
lock, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.Assert(t, err != nil,
|
||||
"create normal lock with exclusively locked repo didn't return an error")
|
||||
rtest.Assert(t, restic.IsAlreadyLocked(err),
|
||||
@@ -107,10 +107,10 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
elock, err := restic.NewLock(context.TODO(), repo)
|
||||
elock, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
|
||||
lock, err := restic.NewExclusiveLock(context.TODO(), repo)
|
||||
lock, err := repository.TestNewLock(t, repo, true)
|
||||
rtest.Assert(t, err != nil,
|
||||
"create normal lock with exclusively locked repo didn't return an error")
|
||||
rtest.Assert(t, restic.IsAlreadyLocked(err),
|
||||
@@ -120,20 +120,6 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) {
|
||||
rtest.OK(t, elock.Unlock(context.TODO()))
|
||||
}
|
||||
|
||||
func createFakeLock(repo restic.SaverUnpacked, t time.Time, pid int) (restic.ID, error) {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return restic.ID{}, err
|
||||
}
|
||||
|
||||
newLock := &restic.Lock{Time: t, PID: pid, Hostname: hostname}
|
||||
return restic.SaveJSONUnpacked(context.TODO(), repo, restic.LockFile, &newLock)
|
||||
}
|
||||
|
||||
func removeLock(repo restic.RemoverUnpacked, id restic.ID) error {
|
||||
return repo.RemoveUnpacked(context.TODO(), restic.LockFile, id)
|
||||
}
|
||||
|
||||
var staleLockTests = []struct {
|
||||
timestamp time.Time
|
||||
stale bool
|
||||
@@ -190,72 +176,6 @@ func TestLockStale(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func lockExists(repo restic.Lister, t testing.TB, lockID restic.ID) bool {
|
||||
var exists bool
|
||||
rtest.OK(t, repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error {
|
||||
if id == lockID {
|
||||
exists = true
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
return exists
|
||||
}
|
||||
|
||||
func TestLockWithStaleLock(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
|
||||
id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
|
||||
rtest.OK(t, err)
|
||||
|
||||
id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
|
||||
rtest.OK(t, err)
|
||||
|
||||
id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
|
||||
rtest.OK(t, err)
|
||||
|
||||
processed, err := restic.RemoveStaleLocks(context.TODO(), repo)
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.Assert(t, lockExists(repo, t, id1) == false,
|
||||
"stale lock still exists after RemoveStaleLocks was called")
|
||||
rtest.Assert(t, lockExists(repo, t, id2) == true,
|
||||
"non-stale lock was removed by RemoveStaleLocks")
|
||||
rtest.Assert(t, lockExists(repo, t, id3) == false,
|
||||
"stale lock still exists after RemoveStaleLocks was called")
|
||||
rtest.Assert(t, processed == 2,
|
||||
"number of locks removed does not match: expected %d, got %d",
|
||||
2, processed)
|
||||
|
||||
rtest.OK(t, removeLock(repo, id2))
|
||||
}
|
||||
|
||||
func TestRemoveAllLocks(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
|
||||
id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
|
||||
rtest.OK(t, err)
|
||||
|
||||
id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
|
||||
rtest.OK(t, err)
|
||||
|
||||
id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
|
||||
rtest.OK(t, err)
|
||||
|
||||
processed, err := restic.RemoveAllLocks(context.TODO(), repo)
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.Assert(t, lockExists(repo, t, id1) == false,
|
||||
"lock still exists after RemoveAllLocks was called")
|
||||
rtest.Assert(t, lockExists(repo, t, id2) == false,
|
||||
"lock still exists after RemoveAllLocks was called")
|
||||
rtest.Assert(t, lockExists(repo, t, id3) == false,
|
||||
"lock still exists after RemoveAllLocks was called")
|
||||
rtest.Assert(t, processed == 3,
|
||||
"number of locks removed does not match: expected %d, got %d",
|
||||
3, processed)
|
||||
}
|
||||
|
||||
func checkSingleLock(t *testing.T, repo restic.Lister) restic.ID {
|
||||
t.Helper()
|
||||
var lockID *restic.ID
|
||||
@@ -279,7 +199,7 @@ func testLockRefresh(t *testing.T, refresh func(lock *restic.Lock) error) {
|
||||
repo := repository.TestRepository(t)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
lock, err := restic.NewLock(context.TODO(), repo)
|
||||
lock, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
time0 := lock.Time
|
||||
|
||||
@@ -312,10 +232,10 @@ func TestLockRefreshStale(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLockRefreshStaleMissing(t *testing.T) {
|
||||
repo, be := repository.TestRepositoryWithVersion(t, 0)
|
||||
repo, _, be := repository.TestRepositoryWithVersion(t, 0)
|
||||
restic.TestSetLockTimeout(t, 5*time.Millisecond)
|
||||
|
||||
lock, err := restic.NewLock(context.TODO(), repo)
|
||||
lock, err := repository.TestNewLock(t, repo, false)
|
||||
rtest.OK(t, err)
|
||||
lockID := checkSingleLock(t, repo)
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f
|
||||
|
||||
// ParallelRemove deletes the given fileList of fileType in parallel
|
||||
// if callback returns an error, then it will abort.
|
||||
func ParallelRemove(ctx context.Context, repo RemoverUnpacked, fileList IDSet, fileType FileType, report func(id ID, err error) error, bar *progress.Counter) error {
|
||||
func ParallelRemove[FT FileTypes](ctx context.Context, repo RemoverUnpacked[FT], fileList IDSet, fileType FT, report func(id ID, err error) error, bar *progress.Counter) error {
|
||||
fileChan := make(chan ID)
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
wg.Go(func() error {
|
||||
|
||||
@@ -57,14 +57,16 @@ type Repository interface {
|
||||
LoadRaw(ctx context.Context, t FileType, id ID) (data []byte, err error)
|
||||
// LoadUnpacked loads and decrypts the file with the given type and ID.
|
||||
LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error)
|
||||
SaveUnpacked(ctx context.Context, t FileType, buf []byte) (ID, error)
|
||||
SaveUnpacked(ctx context.Context, t WriteableFileType, buf []byte) (ID, error)
|
||||
// RemoveUnpacked removes a file from the repository. This will eventually be restricted to deleting only snapshots.
|
||||
RemoveUnpacked(ctx context.Context, t FileType, id ID) error
|
||||
RemoveUnpacked(ctx context.Context, t WriteableFileType, id ID) error
|
||||
}
|
||||
|
||||
type FileType = backend.FileType
|
||||
|
||||
// These are the different data types a backend can store.
|
||||
// These are the different data types a backend can store. Only filetypes contained
|
||||
// in the `WriteableFileType` subset can be modified via the Repository interface.
|
||||
// All other filetypes are considered internal datastructures of the Repository.
|
||||
const (
|
||||
PackFile FileType = backend.PackFile
|
||||
KeyFile FileType = backend.KeyFile
|
||||
@@ -74,6 +76,26 @@ const (
|
||||
ConfigFile FileType = backend.ConfigFile
|
||||
)
|
||||
|
||||
type WriteableFileType backend.FileType
|
||||
|
||||
// These are the different data types that can be modified via SaveUnpacked or RemoveUnpacked.
|
||||
const (
|
||||
WriteableSnapshotFile WriteableFileType = WriteableFileType(SnapshotFile)
|
||||
)
|
||||
|
||||
func (w *WriteableFileType) ToFileType() FileType {
|
||||
switch *w {
|
||||
case WriteableSnapshotFile:
|
||||
return SnapshotFile
|
||||
default:
|
||||
panic("invalid WriteableFileType")
|
||||
}
|
||||
}
|
||||
|
||||
type FileTypes interface {
|
||||
FileType | WriteableFileType
|
||||
}
|
||||
|
||||
// LoaderUnpacked allows loading a blob not stored in a pack file
|
||||
type LoaderUnpacked interface {
|
||||
// Connections returns the maximum number of concurrent backend operations
|
||||
@@ -82,22 +104,22 @@ type LoaderUnpacked interface {
|
||||
}
|
||||
|
||||
// SaverUnpacked allows saving a blob not stored in a pack file
|
||||
type SaverUnpacked interface {
|
||||
type SaverUnpacked[FT FileTypes] interface {
|
||||
// Connections returns the maximum number of concurrent backend operations
|
||||
Connections() uint
|
||||
SaveUnpacked(ctx context.Context, t FileType, buf []byte) (ID, error)
|
||||
SaveUnpacked(ctx context.Context, t FT, buf []byte) (ID, error)
|
||||
}
|
||||
|
||||
// RemoverUnpacked allows removing an unpacked blob
|
||||
type RemoverUnpacked interface {
|
||||
type RemoverUnpacked[FT FileTypes] interface {
|
||||
// Connections returns the maximum number of concurrent backend operations
|
||||
Connections() uint
|
||||
RemoveUnpacked(ctx context.Context, t FileType, id ID) error
|
||||
RemoveUnpacked(ctx context.Context, t FT, id ID) error
|
||||
}
|
||||
|
||||
type SaverRemoverUnpacked interface {
|
||||
SaverUnpacked
|
||||
RemoverUnpacked
|
||||
type SaverRemoverUnpacked[FT FileTypes] interface {
|
||||
SaverUnpacked[FT]
|
||||
RemoverUnpacked[FT]
|
||||
}
|
||||
|
||||
type PackBlobs struct {
|
||||
@@ -126,10 +148,10 @@ type ListerLoaderUnpacked interface {
|
||||
LoaderUnpacked
|
||||
}
|
||||
|
||||
type Unpacked interface {
|
||||
type Unpacked[FT FileTypes] interface {
|
||||
ListerLoaderUnpacked
|
||||
SaverUnpacked
|
||||
RemoverUnpacked
|
||||
SaverUnpacked[FT]
|
||||
RemoverUnpacked[FT]
|
||||
}
|
||||
|
||||
type ListBlobser interface {
|
||||
|
||||
@@ -90,8 +90,8 @@ func LoadSnapshot(ctx context.Context, loader LoaderUnpacked, id ID) (*Snapshot,
|
||||
}
|
||||
|
||||
// SaveSnapshot saves the snapshot sn and returns its ID.
|
||||
func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, error) {
|
||||
return SaveJSONUnpacked(ctx, repo, SnapshotFile, sn)
|
||||
func SaveSnapshot(ctx context.Context, repo SaverUnpacked[WriteableFileType], sn *Snapshot) (ID, error) {
|
||||
return SaveJSONUnpacked(ctx, repo, WriteableSnapshotFile, sn)
|
||||
}
|
||||
|
||||
// ForAllSnapshots reads all snapshots in parallel and calls the
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestLoadJSONUnpacked(t *testing.T) {
|
||||
}
|
||||
|
||||
func testLoadJSONUnpacked(t *testing.T, version uint) {
|
||||
repo, _ := repository.TestRepositoryWithVersion(t, version)
|
||||
repo, _, _ := repository.TestRepositoryWithVersion(t, version)
|
||||
|
||||
// archive a snapshot
|
||||
sn := restic.Snapshot{}
|
||||
|
||||
@@ -184,7 +184,7 @@ func testLoadTree(t *testing.T, version uint) {
|
||||
}
|
||||
|
||||
// archive a few files
|
||||
repo, _ := repository.TestRepositoryWithVersion(t, version)
|
||||
repo, _, _ := repository.TestRepositoryWithVersion(t, version)
|
||||
sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
|
||||
rtest.OK(t, repo.Flush(context.Background()))
|
||||
|
||||
@@ -202,7 +202,7 @@ func benchmarkLoadTree(t *testing.B, version uint) {
|
||||
}
|
||||
|
||||
// archive a few files
|
||||
repo, _ := repository.TestRepositoryWithVersion(t, version)
|
||||
repo, _, _ := repository.TestRepositoryWithVersion(t, version)
|
||||
sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
|
||||
rtest.OK(t, repo.Flush(context.Background()))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user