docs: keep S3 credentials out of Docker arguments (#22118)

prune: report failure to list pack files as a fatal error

.github/workflows/docker.yml

@@ -29,7 +29,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Log in to the Container registry
-        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
+        uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}

.github/workflows/tests.yml

@@ -183,12 +183,14 @@ jobs:
           RESTIC_TEST_OS_USERNAME: ${{ secrets.RESTIC_TEST_OS_USERNAME }}
           RESTIC_TEST_OS_PASSWORD: ${{ secrets.RESTIC_TEST_OS_PASSWORD }}
           RESTIC_TEST_OS_REGION_NAME: ${{ secrets.RESTIC_TEST_OS_REGION_NAME }}
+          RESTIC_TEST_SWIFT: ${{ secrets.RESTIC_TEST_SWIFT }}
           # fail if any of the following tests cannot be run
           RESTIC_TEST_DISALLOW_SKIP: "restic/backend/rest.TestBackendREST,\
             restic/backend/sftp.TestBackendSFTP,\
             restic/backend/s3.TestBackendSeaweedFS,\
             restic/backend/rclone.TestBackendRclone,\
             restic/backend/s3.TestBackendS3,\
+            restic/backend/swift.TestBackendSwift,\
             restic/backend/b2.TestBackendB2,\
             restic/backend/gs.TestBackendGS,\
             restic/backend/azure.TestBackendAzure"

build.go

@@ -59,7 +59,7 @@ var config = Config{
 	// see https://github.com/googleapis/google-cloud-go/issues/11448
 	DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used
 	Tests:            []string{"./..."},                              // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 26, Patch: 0},       // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 26, Patch: 10},      // minimum Go version supported
 }
 
 // Config configures the build.

changelog/unreleased/issue-22120

@@ -0,0 +1,8 @@
+Enhancement: Show a concise error when listing pack files fails during `prune`
+
+When listing the pack files of a repository failed during `prune`, for
+example due to a request timeout, restic printed the error together with a
+stack trace. Restic now reports a short error message that states that
+listing the pack files failed.
+
+https://github.com/restic/restic/issues/22120

changelog/unreleased/issue-5189

@@ -0,0 +1,11 @@
+Enhancement: Add option to disable AWS S3 dual-stack endpoints
+
+Restic used dual-stack endpoints for AWS regions that have them, with no way to
+opt out. VPC interface and gateway endpoints do not support dual-stack, so a
+repository reachable over a VPC endpoint could not be used.
+
+Passing `-o s3.disable-dualstack=true` now keeps restic on the single-stack
+endpoint.
+
+https://github.com/restic/restic/issues/5189
+https://github.com/restic/restic/pull/22060

changelog/unreleased/issue-5741

@@ -0,0 +1,8 @@
+Enhancement: Support Multi-Bucket Application Keys in the `b2` backend
+
+The `b2` backend now supports Backblaze B2 Multi-Bucket Application Keys,
+including single-bucket keys created through the multi-bucket API (for
+example by the Backblaze Terraform provider).
+
+https://github.com/restic/restic/issues/5741
+https://github.com/restic/restic/pull/22104

changelog/unreleased/pull-22104

@@ -0,0 +1,7 @@
+Change: Require Go 1.26.10 or newer
+
+Restic now requires Go 1.26.10 or newer to build, as the library used to
+access the B2 backend requires it.
+
+https://github.com/restic/restic/pull/22104
+https://github.com/restic/restic/pull/22121

changelog/unreleased/pull-22121

@@ -1,5 +0,0 @@
-Change: Require Go 1.26 or newer
-
-Building restic now requires Go 1.26 or newer.
-
-https://github.com/restic/restic/pull/22121

changelog/unreleased/pull-22163

@@ -0,0 +1,7 @@
+Bugfix: Report an error for too large numbers in `--keep-within*` durations
+
+Restic crashed with a stack trace when a number in a duration such as the one
+passed to `restic forget --keep-within` did not fit into an integer. It now
+reports an invalid argument instead.
+
+https://github.com/restic/restic/pull/22163

cmd/restic/cmd_rewrite_integration_test.go

@@ -352,3 +352,41 @@ func TestRewriteIncludeNothing(t *testing.T) {
 	rtest.Assert(t, snapsBefore[0] == snapsAfter[0], "snapshots should be identical but are %s and %s",
 		snapsBefore[0].Str(), snapsAfter[0].Str())
 }
+
+func testRunRewriteOutput(t testing.TB, opts RewriteOptions, gopts global.Options, args []string) error {
+	t.Helper()
+	_, _, err := withCaptureStdoutStderr(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		return runRewrite(ctx, opts, gopts, args, gopts.Term)
+	})
+	return err
+}
+
+func TestRewriteWrongMetadata(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+
+	opts := RewriteOptions{
+		Metadata: snapshotMetadataArgs{
+			Time: "2026-10-10 1x:2y:3z",
+		},
+	}
+	err := testRunRewriteOutput(t, opts, env.gopts, []string{})
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "error in time option: parsing time"),
+		"expected time string related errors, got %v", err)
+}
+
+func TestRewriteWrongExcludePattern(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+
+	opts := RewriteOptions{
+		ExcludePatternOptions: filter.ExcludePatternOptions{
+			Excludes: []string{"a["},
+		},
+	}
+	err := testRunRewriteOutput(t, opts, env.gopts, []string{})
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "--exclude: invalid pattern"),
+		"expected '--exclude: invalid pattern', got %v", err)
+}

doc/020_installation.rst

@@ -305,7 +305,7 @@ From source
 ***********
 
 restic is written in the Go programming language and you need at least
-Go version 1.26. Building restic may also work with older versions of Go,
+Go version 1.26.10. Building restic may also work with older versions of Go,
 but that's not supported. See the `Getting
 started <https://go.dev/doc/install>`__ guide of the Go project for
 instructions how to install Go.

doc/030_preparing_a_new_repo.rst

@@ -249,6 +249,14 @@ while creating the bucket.
     $ export AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
     $ export AWS_SECRET_ACCESS_KEY=<MY_SECRET_ACCESS_KEY>
 
+Restic can also read credentials from an AWS shared credentials file. Set
+``AWS_SHARED_CREDENTIALS_FILE`` to its path, or use the default location at
+``~/.aws/credentials``. When running restic in a container, mount the file
+read-only and set ``AWS_SHARED_CREDENTIALS_FILE`` to the path inside the
+container. Avoid passing secret values in ``docker run -e NAME=value``
+arguments, where they can be exposed in process listings. Docker's
+``--env-file`` option also keeps the values out of command-line arguments.
+
 When using temporary credentials make sure to include the session token via
 the environment variable ``AWS_SESSION_TOKEN``.
 
@@ -308,6 +316,11 @@ To select between path-style and virtual-hosted access, the extended option
 - ``dns``: Use virtual-hosted-style bucket access
 - ``path``: Use path-style bucket access
 
+Amazon's dual-stack endpoints are used by default for AWS regions that have
+them. VPC interface and gateway endpoints do not support dual-stack, so if
+restic cannot reach your bucket from inside a VPC, pass
+``-o s3.disable-dualstack=true``.
+
 Certain S3-compatible servers do not properly implement the ``ListObjectsV2`` API,
 most notably Ceph versions before v14.2.5. On these backends, as a temporary
 workaround, you can provide the ``-o s3.list-objects-v1=true`` option to use the

doc/075_scripting.rst

@@ -37,6 +37,7 @@ override the corresponding environment variables. For example, ``-r`` (or
     RESTIC_KEY_HINT                     ID of key to try decrypting first, before other keys
     RESTIC_CACERT                       Location(s) of certificate file(s), comma separated if multiple (overridden by --cacert)
     RESTIC_TLS_CLIENT_CERT              Location of TLS client certificate and private key (overridden by --tls-client-cert)
+    RESTIC_HTTP_USER_AGENT              HTTP user agent for outgoing HTTP requests (overridden by --http-user-agent)
     RESTIC_CACHE_DIR                    Location of the cache directory
     RESTIC_COMPRESSION                  Compression mode (only available for repository format version 2)
     RESTIC_HOST                         Only consider snapshots for this host / Set the hostname for the snapshot manually (overridden by --host)

go.mod

@@ -1,8 +1,6 @@
 module github.com/restic/restic
 
-go 1.26.0
+go 1.26.10
-
-toolchain go1.26.9
 
 // keep the old behavior for reparse points on windows until handling reparse points has been improved in restic
 // https://forum.restic.net/t/windows-junction-backup-with-go1-23-or-later/8940
@@ -13,7 +11,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1
-	github.com/Backblaze/blazer v0.7.2
+	github.com/Backblaze/blazer v0.9.0
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/ProtonMail/go-crypto v1.5.2
 	github.com/anacrolix/fuse v0.3.1
@@ -23,7 +21,7 @@ require (
 	github.com/go-ole/go-ole v1.3.0
 	github.com/google/go-cmp v0.7.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/klauspost/compress v1.20.0
+	github.com/klauspost/compress v1.20.3
 	github.com/minio/minio-go/v7 v7.3.0
 	github.com/ncw/swift/v2 v2.0.5
 	github.com/peterbourgon/unixtransport v0.0.7

go.sum

@@ -36,8 +36,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ
 github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
-github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM=
+github.com/Backblaze/blazer v0.9.0 h1:413rZHO8Vnpfjygrfxt8AO56LD6lE4w334MZIpnxdAs=
-github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10=
+github.com/Backblaze/blazer v0.9.0/go.mod h1:/JlJELHVlJWBwHknmVqQ92zYyVm0s4EI+Cn3XU6IgCc=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w=
@@ -141,8 +141,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
-github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
+github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
-github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
+github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
 github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=

internal/backend/s3/config.go

@@ -32,6 +32,7 @@ type Config struct {
 	Region              string `option:"region" help:"set region"`
 	BucketLookup        string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
 	ListObjectsV1       bool   `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
+	DisableDualstack    bool   `option:"disable-dualstack" help:"do not use AWS S3 dual-stack endpoints, which VPC endpoints do not support"`
 	UnsafeAnonymousAuth bool   `option:"unsafe-anonymous-auth" help:"use anonymous authentication"`
 
 	// For testing only

internal/backend/s3/s3.go

@@ -89,6 +89,12 @@ func open(cfg Config, rt http.RoundTripper) (*s3, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
+	if cfg.DisableDualstack {
+		// VPC interface and gateway endpoints have no dual-stack support, so
+		// resolving to one makes the bucket unreachable from within the VPC
+		client.SetS3EnableDualstack(false)
+	}
+
 	be := &s3{
 		client: client,
 		cfg:    cfg,

internal/data/duration.go

@@ -65,7 +65,7 @@ func nextNumber(input string) (num int, rest string, err error) {
 
 	num, err = strconv.Atoi(n)
 	if err != nil {
-		panic(err)
+		return 0, input, err
 	}
 
 	if negative {

internal/data/duration_test.go

@@ -40,6 +40,9 @@ func TestNextNumber(t *testing.T) {
 		{
 			input: "5", num: 5, rest: "",
 		},
+		{
+			input: "99999999999999999999999999y", num: 0, rest: "99999999999999999999999999y", err: true,
+		},
 	}
 
 	for _, test := range tests {
@@ -82,6 +85,8 @@ func TestParseDuration(t *testing.T) {
 		{input: "1y4m3w1d", err: true},
 		{input: "s", err: true},
 		{input: "\xdf\x80", err: true}, // NKO DIGIT ZERO; we want ASCII digits
+		{input: "99999999999999999999999999y", err: true},
+		{input: "1y-99999999999999999999999999d", err: true},
 	}
 
 	for _, test := range tests {

internal/global/global.go

@@ -99,7 +99,7 @@ func (opts *Options) AddFlags(f *pflag.FlagSet) {
 	f.BoolVar(&opts.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories")
 	f.DurationVar(&opts.RetryLock, "retry-lock", 0, "retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)")
 	f.BoolVarP(&opts.JSON, "json", "", false, "set output mode to JSON for commands that support it")
-	f.StringVar(&opts.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)")
+	f.StringVar(&opts.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory or $RESTIC_CACHE_DIR)")
 	f.BoolVar(&opts.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&opts.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates or $RESTIC_CACERT)")
 	f.StringVar(&opts.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)")
@@ -114,7 +114,7 @@ func (opts *Options) AddFlags(f *pflag.FlagSet) {
 	const packSizeFlag = "pack-size"
 	f.UintVar(&opts.PackSize, packSizeFlag, 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
 	f.StringSliceVarP(&opts.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
-	f.StringVar(&opts.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests")
+	f.StringVar(&opts.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests (default: $RESTIC_HTTP_USER_AGENT)")
 	f.DurationVar(&opts.StuckRequestTimeout, "stuck-request-timeout", 5*time.Minute, "`duration` after which to retry stuck requests")
 
 	opts.Repo = os.Getenv("RESTIC_REPOSITORY")

internal/repository/prune.go

@@ -467,8 +467,11 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo *Repository,
 
 	bar.Done()
 	if err != nil {
+		if errors.IsFatal(err) || ctx.Err() != nil {
 			return PrunePlan{}, err
 		}
+		return PrunePlan{}, errors.Fatalf("failed to list pack files: %v", err)
+	}
 
 	// At this point indexPacks contains only missing packs!
 

internal/repository/prune_test.go

@@ -2,12 +2,16 @@ package repository_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math"
 	"math/rand"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
+	resticerrors "github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
@@ -191,3 +195,34 @@ func TestPruneSmall(t *testing.T) {
 	rtest.Equals(t, lenPackfilesBefore > lenPackfilesAfter, true,
 		fmt.Sprintf("the number packfiles before %d and after repack %d", lenPackfilesBefore, lenPackfilesAfter))
 }
+
+type failingListBackend struct {
+	backend.Backend
+	err error
+}
+
+func (b *failingListBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
+	if t == backend.PackFile {
+		return b.err
+	}
+	return b.Backend.List(ctx, t, fn)
+}
+
+func TestPrunePackListError(t *testing.T) {
+	_, _, be := repository.TestRepositoryWithVersion(t, 0)
+
+	listErr := errors.New("request timeout")
+	repo := repository.TestOpenBackend(t, &failingListBackend{Backend: be, err: listErr})
+	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
+
+	_, err := repository.PlanPrune(context.TODO(), repository.PruneOptions{
+		MaxRepackBytes: math.MaxUint64,
+		MaxUnusedBytes: func(used uint64) uint64 { return used },
+	}, repo, func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error {
+		return nil
+	}, restic.NewNoopPrinter())
+	rtest.Assert(t, err != nil, "expected an error")
+	rtest.Assert(t, resticerrors.IsFatal(err), "expected a fatal error, got %v", err)
+	rtest.Assert(t, errors.Is(err, listErr), "expected error to wrap the list error, got %v", err)
+	rtest.Assert(t, strings.Contains(err.Error(), "failed to list pack files"), "unexpected message %q", err)
+}