diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 25ea4f47c..3a1237ee8 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -16,7 +16,7 @@ import ( "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/fileio" "github.com/cenkalti/backoff/v4" ) @@ -151,7 +151,7 @@ func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReade // preallocate disk space if size := rd.Length(); size > 0 { - if err := fs.PreallocateFile(f, size); err != nil { + if err := fileio.PreallocateFile(f, size); err != nil { debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err) } } diff --git a/internal/fileio/file_unix.go b/internal/fileio/file_unix.go new file mode 100644 index 000000000..c32952dca --- /dev/null +++ b/internal/fileio/file_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package fileio + +import ( + "os" +) + +// TempFile creates a temporary file which has already been deleted (on +// supported platforms) +func TempFile(dir, prefix string) (f *os.File, err error) { + f, err = os.CreateTemp(dir, prefix) + if err != nil { + return nil, err + } + + if err = os.Remove(f.Name()); err != nil { + return nil, err + } + + return f, nil +} diff --git a/internal/fileio/file_windows.go b/internal/fileio/file_windows.go new file mode 100644 index 000000000..0dba2d2af --- /dev/null +++ b/internal/fileio/file_windows.go @@ -0,0 +1,60 @@ +package fileio + +import ( + "math/rand" + "os" + "path/filepath" + "strconv" + "syscall" + + "golang.org/x/sys/windows" +) + +// TempFile creates a temporary file which is marked as delete-on-close +func TempFile(dir, prefix string) (f *os.File, err error) { + // slightly modified implementation of os.CreateTemp(dir, prefix) to allow us to add + // the FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE flags. + // These provide two large benefits: + // FILE_ATTRIBUTE_TEMPORARY tells Windows to keep the file in memory only if possible + // which reduces the amount of unnecessary disk writes. + // FILE_FLAG_DELETE_ON_CLOSE instructs Windows to automatically delete the file once + // all file descriptors are closed. + + if dir == "" { + dir = os.TempDir() + } + + access := uint32(windows.GENERIC_READ | windows.GENERIC_WRITE) + creation := uint32(windows.CREATE_NEW) + share := uint32(0) // prevent other processes from accessing the file + flags := uint32(windows.FILE_ATTRIBUTE_TEMPORARY | windows.FILE_FLAG_DELETE_ON_CLOSE) + + for i := 0; i < 10000; i++ { + randSuffix := strconv.Itoa(int(1e9 + rand.Intn(1e9)%1e9))[1:] + path := filepath.Join(dir, prefix+randSuffix) + + ptr, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + h, err := windows.CreateFile(ptr, access, share, nil, creation, flags, 0) + if os.IsExist(err) { + continue + } + // Access denied error can occur if the tmp files conflict with each other. + if isAccessDeniedError(err) { + continue + } + return os.NewFile(uintptr(h), path), err + } + + // Proper error handling is still to do + return nil, os.ErrExist +} + +func isAccessDeniedError(err error) bool { + if errno, ok := err.(syscall.Errno); ok { + return errno == windows.ERROR_ACCESS_DENIED + } + return false +} diff --git a/internal/fs/file_windows_test.go b/internal/fileio/file_windows_test.go similarity index 82% rename from internal/fs/file_windows_test.go rename to internal/fileio/file_windows_test.go index 71077709b..4351f44b4 100644 --- a/internal/fs/file_windows_test.go +++ b/internal/fileio/file_windows_test.go @@ -1,21 +1,21 @@ -package fs_test +package fileio_test import ( "errors" "os" "testing" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/fileio" rtest "github.com/restic/restic/internal/test" ) func TestTempFile(t *testing.T) { // create two temp files at the same time to check that the // collision avoidance works - f, err := fs.TempFile("", "test") + f, err := fileio.TempFile("", "test") fn := f.Name() rtest.OK(t, err) - f2, err := fs.TempFile("", "test") + f2, err := fileio.TempFile("", "test") fn2 := f2.Name() rtest.OK(t, err) rtest.Assert(t, fn != fn2, "filenames don't differ %s", fn) diff --git a/internal/fs/preallocate_darwin.go b/internal/fileio/preallocate_darwin.go similarity index 97% rename from internal/fs/preallocate_darwin.go rename to internal/fileio/preallocate_darwin.go index af46e971b..f0e62a6d0 100644 --- a/internal/fs/preallocate_darwin.go +++ b/internal/fileio/preallocate_darwin.go @@ -1,4 +1,4 @@ -package fs +package fileio import ( "os" diff --git a/internal/fs/preallocate_linux.go b/internal/fileio/preallocate_linux.go similarity index 97% rename from internal/fs/preallocate_linux.go rename to internal/fileio/preallocate_linux.go index 7b0449507..07fa0940e 100644 --- a/internal/fs/preallocate_linux.go +++ b/internal/fileio/preallocate_linux.go @@ -1,4 +1,4 @@ -package fs +package fileio import ( "os" diff --git a/internal/fs/preallocate_other.go b/internal/fileio/preallocate_other.go similarity index 93% rename from internal/fs/preallocate_other.go rename to internal/fileio/preallocate_other.go index b04d68738..e471e699e 100644 --- a/internal/fs/preallocate_other.go +++ b/internal/fileio/preallocate_other.go @@ -1,6 +1,6 @@ //go:build !linux && !darwin -package fs +package fileio import "os" diff --git a/internal/fs/preallocate_test.go b/internal/fileio/preallocate_test.go similarity index 89% rename from internal/fs/preallocate_test.go rename to internal/fileio/preallocate_test.go index 9dabd2f36..a87872877 100644 --- a/internal/fs/preallocate_test.go +++ b/internal/fileio/preallocate_test.go @@ -1,4 +1,4 @@ -package fs +package fileio import ( "os" @@ -7,6 +7,7 @@ import ( "syscall" "testing" + "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/test" ) @@ -31,7 +32,7 @@ func TestPreallocate(t *testing.T) { fi, err := wr.Stat() test.OK(t, err) - efi := ExtendedStat(fi) + efi := fs.ExtendedStat(fi) test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks) }) } diff --git a/internal/fs/file_unix.go b/internal/fs/file_unix.go index 9b33f1ab7..fe3de064d 100644 --- a/internal/fs/file_unix.go +++ b/internal/fs/file_unix.go @@ -13,21 +13,6 @@ func fixpath(name string) string { return name } -// TempFile creates a temporary file which has already been deleted (on -// supported platforms) -func TempFile(dir, prefix string) (f *os.File, err error) { - f, err = os.CreateTemp(dir, prefix) - if err != nil { - return nil, err - } - - if err = os.Remove(f.Name()); err != nil { - return nil, err - } - - return f, nil -} - // isNotSupported returns true if the error is caused by an unsupported file system feature. func isNotSupported(err error) bool { if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ENOTSUP { diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index 30cb75da5..f655e731d 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -1,10 +1,8 @@ package fs import ( - "math/rand" "os" "path/filepath" - "strconv" "strings" "github.com/restic/restic/internal/data" @@ -43,48 +41,6 @@ func fixpath(name string) string { return name } -// TempFile creates a temporary file which is marked as delete-on-close -func TempFile(dir, prefix string) (f *os.File, err error) { - // slightly modified implementation of os.CreateTemp(dir, prefix) to allow us to add - // the FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE flags. - // These provide two large benefits: - // FILE_ATTRIBUTE_TEMPORARY tells Windows to keep the file in memory only if possible - // which reduces the amount of unnecessary disk writes. - // FILE_FLAG_DELETE_ON_CLOSE instructs Windows to automatically delete the file once - // all file descriptors are closed. - - if dir == "" { - dir = os.TempDir() - } - - access := uint32(windows.GENERIC_READ | windows.GENERIC_WRITE) - creation := uint32(windows.CREATE_NEW) - share := uint32(0) // prevent other processes from accessing the file - flags := uint32(windows.FILE_ATTRIBUTE_TEMPORARY | windows.FILE_FLAG_DELETE_ON_CLOSE) - - for i := 0; i < 10000; i++ { - randSuffix := strconv.Itoa(int(1e9 + rand.Intn(1e9)%1e9))[1:] - path := filepath.Join(dir, prefix+randSuffix) - - ptr, err := windows.UTF16PtrFromString(path) - if err != nil { - return nil, err - } - h, err := windows.CreateFile(ptr, access, share, nil, creation, flags, 0) - if os.IsExist(err) { - continue - } - // Access denied error can occur if the tmp files conflict with each other. - if isAccessDeniedError(err) { - continue - } - return os.NewFile(uintptr(h), path), err - } - - // Proper error handling is still to do - return nil, os.ErrExist -} - // Chmod changes the mode of the named file to mode. func chmod(name string, mode os.FileMode) error { return os.Chmod(fixpath(name), mode) diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index ea69b6b65..54455d33e 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -16,7 +16,7 @@ import ( "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/fileio" "github.com/restic/restic/internal/repository/crypto" "github.com/restic/restic/internal/repository/pack" ) @@ -201,7 +201,7 @@ func (r *packerManager) forgetPacker(packer *packer) { // created or one is returned that already has some blobs. func (r *packerManager) newPacker() (pck *packer, err error) { debug.Log("create new pack") - tmpfile, err := fs.TempFile("", "restic-temp-pack-") + tmpfile, err := fileio.TempFile("", "restic-temp-pack-") if err != nil { return nil, errors.WithStack(err) } diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 59871324c..20d458343 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/golang-lru/v2/simplelru" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fileio" "github.com/restic/restic/internal/fs" ) @@ -166,7 +167,7 @@ func ensureSize(f *os.File, fi os.FileInfo, createSize int64, sparse bool) (*os. return nil, err } } else if createSize > 0 { - err := fs.PreallocateFile(f, createSize) + err := fileio.PreallocateFile(f, createSize) if err != nil { // Just log the preallocate error but don't let it cause the restore process to fail. // Preallocate might return an error if the filesystem (implementation) does not