From e805b968b16b60a70c82b6218d793e22c4b86208 Mon Sep 17 00:00:00 2001 From: Bryce Chidester Date: Fri, 29 Dec 2017 19:51:13 -0800 Subject: [PATCH 1/2] Support for TLS client certificate authentication This adds --tls-client-cert and --tls-client-key parameters and enables use of that certificate/key pair when connecting to https servers. --- changelog/0.8.2/issue-1522 | 9 +++++ cmd/restic/global.go | 26 +++++++++------ doc/manual_rest.rst | 50 +++++++++++++++------------- internal/backend/azure/azure_test.go | 2 +- internal/backend/b2/b2_test.go | 2 +- internal/backend/http_transport.go | 15 ++++++--- internal/backend/rest/rest_test.go | 2 +- internal/backend/s3/s3_test.go | 4 +-- internal/backend/swift/swift_test.go | 2 +- 9 files changed, 68 insertions(+), 44 deletions(-) create mode 100644 changelog/0.8.2/issue-1522 diff --git a/changelog/0.8.2/issue-1522 b/changelog/0.8.2/issue-1522 new file mode 100644 index 000000000..813b0b108 --- /dev/null +++ b/changelog/0.8.2/issue-1522 @@ -0,0 +1,9 @@ +Enhancement: Add support for TLS client certificate authentication. + +Support has been added to the http backend for providing a TLS client +certificate/key pair using `--tls-client-cert` and `--tls-client-key` when +connecting to https resources that perform TLS client certificate +authentication. + +https://github.com/restic/restic/issues/1522 +https://github.com/restic/restic/pull/1524 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 81e05d2e3..adfae4b94 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -39,15 +39,17 @@ var version = "compiled manually" // GlobalOptions hold all global options for restic. type GlobalOptions struct { - Repo string - PasswordFile string - Quiet bool - NoLock bool - JSON bool - CacheDir string - NoCache bool - CACerts []string - CleanupCache bool + Repo string + PasswordFile string + Quiet bool + NoLock bool + JSON bool + CacheDir string + NoCache bool + CACerts []string + TLSClientCert string + TLSClientKey string + CleanupCache bool LimitUploadKb int LimitDownloadKb int @@ -84,6 +86,8 @@ func init() { f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)") + f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a TLS client certificate") + f.StringVar(&globalOptions.TLSClientKey, "tls-client-key", "", "path to a TLS client certificate key") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") @@ -541,7 +545,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, return nil, err } - rt, err := backend.Transport(globalOptions.CACerts) + rt, err := backend.Transport(globalOptions.CACerts, globalOptions.TLSClientCert, globalOptions.TLSClientKey) if err != nil { return nil, err } @@ -605,7 +609,7 @@ func create(s string, opts options.Options) (restic.Backend, error) { return nil, err } - rt, err := backend.Transport(globalOptions.CACerts) + rt, err := backend.Transport(globalOptions.CACerts, globalOptions.TLSClientCert, globalOptions.TLSClientKey) if err != nil { return nil, err } diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 3974371b2..e203b6e30 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -39,18 +39,20 @@ Usage help is available: version Print version information Flags: - --cacert stringSlice path to load root certificates from (default: use system certificates) - --cache-dir string set the cache directory - -h, --help help for restic - --json set output mode to JSON for commands that support it - --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) - --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) - --no-cache do not use a local cache - --no-lock do not lock the repo, this allows some operations on read-only repos - -o, --option key=value set extended option (key=value, can be specified multiple times) - -p, --password-file string read the repository password from a file (default: $RESTIC_PASSWORD_FILE) - -q, --quiet do not output comprehensive progress report - -r, --repo string repository to backup to or restore from (default: $RESTIC_REPOSITORY) + --cacert stringSlice path to load root certificates from (default: use system certificates) + --cache-dir string set the cache directory + -h, --help help for restic + --json set output mode to JSON for commands that support it + --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) + --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) + --no-cache do not use a local cache + --no-lock do not lock the repo, this allows some operations on read-only repos + -o, --option key=value set extended option (key=value, can be specified multiple times) + -p, --password-file string read the repository password from a file (default: $RESTIC_PASSWORD_FILE) + -q, --quiet do not output comprehensive progress report + -r, --repo string repository to backup to or restore from (default: $RESTIC_REPOSITORY) + --tls-client-cert string path to a TLS client certificate + --tls-client-key string path to a TLS client certificate key Use "restic [command] --help" for more information about a command. @@ -87,17 +89,19 @@ command: --time string time of the backup (ex. '2012-11-01 22:08:41') (default: now) Global Flags: - --cacert stringSlice path to load root certificates from (default: use system certificates) - --cache-dir string set the cache directory - --json set output mode to JSON for commands that support it - --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) - --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) - --no-cache do not use a local cache - --no-lock do not lock the repo, this allows some operations on read-only repos - -o, --option key=value set extended option (key=value, can be specified multiple times) - -p, --password-file string read the repository password from a file (default: $RESTIC_PASSWORD_FILE) - -q, --quiet do not output comprehensive progress report - -r, --repo string repository to backup to or restore from (default: $RESTIC_REPOSITORY) + --cacert stringSlice path to load root certificates from (default: use system certificates) + --cache-dir string set the cache directory + --json set output mode to JSON for commands that support it + --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited) + --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited) + --no-cache do not use a local cache + --no-lock do not lock the repo, this allows some operations on read-only repos + -o, --option key=value set extended option (key=value, can be specified multiple times) + -p, --password-file string read the repository password from a file (default: $RESTIC_PASSWORD_FILE) + -q, --quiet do not output comprehensive progress report + -r, --repo string repository to backup to or restore from (default: $RESTIC_REPOSITORY) + --tls-client-cert string path to a TLS client certificate + --tls-client-key string path to a TLS client certificate key Subcommand that support showing progress information such as ``backup``, ``check`` and ``prune`` will do so unless the quiet flag ``-q`` or diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index fe3b3e83c..c6c61dfc1 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -16,7 +16,7 @@ import ( ) func newAzureTestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil) + tr, err := backend.Transport(nil, "", "") if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 7f22a7986..e95f7dd42 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -16,7 +16,7 @@ import ( ) func newB2TestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil) + tr, err := backend.Transport(nil, "", "") if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/http_transport.go b/internal/backend/http_transport.go index 040c673d0..71c4e0501 100644 --- a/internal/backend/http_transport.go +++ b/internal/backend/http_transport.go @@ -15,7 +15,7 @@ import ( // Transport returns a new http.RoundTripper with default settings applied. If // a custom rootCertFilename is non-empty, it must point to a valid PEM file, // otherwise the function will return an error. -func Transport(rootCertFilenames []string) (http.RoundTripper, error) { +func Transport(rootCertFilenames []string, tlsClientCert string, tlsClientKey string) (http.RoundTripper, error) { // copied from net/http tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -29,6 +29,15 @@ func Transport(rootCertFilenames []string) (http.RoundTripper, error) { IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{}, + } + + if tlsClientCert != "" && tlsClientKey != "" { + c, err := tls.LoadX509KeyPair(tlsClientCert, tlsClientKey) + if err != nil { + return nil, fmt.Errorf("unable to read client certificate/key pair: %v", err) + } + tr.TLSClientConfig.Certificates = []tls.Certificate{c} } if rootCertFilenames == nil { @@ -49,9 +58,7 @@ func Transport(rootCertFilenames []string) (http.RoundTripper, error) { } } - tr.TLSClientConfig = &tls.Config{ - RootCAs: p, - } + tr.TLSClientConfig.RootCAs = p // wrap in the debug round tripper return debug.RoundTripper(tr), nil diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index 074836c1b..487d1b413 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -68,7 +68,7 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun } func newTestSuite(ctx context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite { - tr, err := backend.Transport(nil) + tr, err := backend.Transport(nil, "", "") if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index fe5e92299..0a5ae48e4 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -121,7 +121,7 @@ func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be resti } func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { - tr, err := backend.Transport(nil) + tr, err := backend.Transport(nil, "", "") if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } @@ -221,7 +221,7 @@ func BenchmarkBackendMinio(t *testing.B) { } func newS3TestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil) + tr, err := backend.Transport(nil, "", "") if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index 30a61ea7b..8e81064f9 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -16,7 +16,7 @@ import ( ) func newSwiftTestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil) + tr, err := backend.Transport(nil, "", "") if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } From c34db983d8caac75b66eeae86835d64752333cfd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jan 2018 13:57:43 +0100 Subject: [PATCH 2/2] Read TLS client cert and key from the same file --- changelog/0.8.2/issue-1522 | 9 ++- cmd/restic/global.go | 16 +++-- doc/manual_rest.rst | 4 +- internal/backend/azure/azure_test.go | 2 +- internal/backend/b2/b2_test.go | 2 +- internal/backend/http_transport.go | 95 +++++++++++++++++++++------- internal/backend/rest/rest_test.go | 2 +- internal/backend/s3/s3_test.go | 4 +- internal/backend/swift/swift_test.go | 2 +- 9 files changed, 95 insertions(+), 41 deletions(-) diff --git a/changelog/0.8.2/issue-1522 b/changelog/0.8.2/issue-1522 index 813b0b108..57e0501c3 100644 --- a/changelog/0.8.2/issue-1522 +++ b/changelog/0.8.2/issue-1522 @@ -1,9 +1,8 @@ -Enhancement: Add support for TLS client certificate authentication. +Enhancement: Add support for TLS client certificate authentication -Support has been added to the http backend for providing a TLS client -certificate/key pair using `--tls-client-cert` and `--tls-client-key` when -connecting to https resources that perform TLS client certificate -authentication. +Support has been added for using a TLS client certificate for authentication to +HTTP based backend. A file containing the PEM encoded private key and +certificate can be set using the `--tls-client-cert` option. https://github.com/restic/restic/issues/1522 https://github.com/restic/restic/pull/1524 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index adfae4b94..fc916cbef 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -48,7 +48,6 @@ type GlobalOptions struct { NoCache bool CACerts []string TLSClientCert string - TLSClientKey string CleanupCache bool LimitUploadKb int @@ -86,8 +85,7 @@ func init() { f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)") - f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a TLS client certificate") - f.StringVar(&globalOptions.TLSClientKey, "tls-client-key", "", "path to a TLS client certificate key") + f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a file containing PEM encoded TLS client certificate and private key") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") @@ -545,7 +543,11 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, return nil, err } - rt, err := backend.Transport(globalOptions.CACerts, globalOptions.TLSClientCert, globalOptions.TLSClientKey) + tropts := backend.TransportOptions{ + RootCertFilenames: globalOptions.CACerts, + TLSClientCertKeyFilename: globalOptions.TLSClientCert, + } + rt, err := backend.Transport(tropts) if err != nil { return nil, err } @@ -609,7 +611,11 @@ func create(s string, opts options.Options) (restic.Backend, error) { return nil, err } - rt, err := backend.Transport(globalOptions.CACerts, globalOptions.TLSClientCert, globalOptions.TLSClientKey) + tropts := backend.TransportOptions{ + RootCertFilenames: globalOptions.CACerts, + TLSClientCertKeyFilename: globalOptions.TLSClientCert, + } + rt, err := backend.Transport(tropts) if err != nil { return nil, err } diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index e203b6e30..821f220fd 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -51,8 +51,8 @@ Usage help is available: -p, --password-file string read the repository password from a file (default: $RESTIC_PASSWORD_FILE) -q, --quiet do not output comprehensive progress report -r, --repo string repository to backup to or restore from (default: $RESTIC_REPOSITORY) - --tls-client-cert string path to a TLS client certificate - --tls-client-key string path to a TLS client certificate key + --tls-client-cert string path to a file containing PEM encoded TLS client certificate and private key + Use "restic [command] --help" for more information about a command. diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index c6c61dfc1..d738de857 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -16,7 +16,7 @@ import ( ) func newAzureTestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil, "", "") + tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index e95f7dd42..9f97de4f9 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -16,7 +16,7 @@ import ( ) func newB2TestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil, "", "") + tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/http_transport.go b/internal/backend/http_transport.go index 71c4e0501..324874c63 100644 --- a/internal/backend/http_transport.go +++ b/internal/backend/http_transport.go @@ -3,19 +3,66 @@ package backend import ( "crypto/tls" "crypto/x509" - "fmt" + "encoding/pem" "io/ioutil" "net" "net/http" + "os" + "strings" "time" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" ) +// TransportOptions collects various options which can be set for an HTTP based +// transport. +type TransportOptions struct { + // contains filenames of PEM encoded root certificates to trust + RootCertFilenames []string + + // contains the name of a file containing the TLS client certificate and private key in PEM format + TLSClientCertKeyFilename string +} + +// readPEMCertKey reads a file and returns the PEM encoded certificate and key +// blocks. +func readPEMCertKey(filename string) (certs []byte, key []byte, err error) { + data, err := ioutil.ReadFile(os.Args[1]) + if err != nil { + return nil, nil, errors.Wrap(err, "ReadFile") + } + + var block *pem.Block + for { + if len(data) == 0 { + break + } + block, data = pem.Decode(data) + if block == nil { + break + } + + switch { + case strings.HasSuffix(block.Type, "CERTIFICATE"): + certs = append(certs, pem.EncodeToMemory(block)...) + case strings.HasSuffix(block.Type, "PRIVATE KEY"): + if key != nil { + return nil, nil, errors.Errorf("error loading TLS cert and key from %v: more than one private key found", filename) + } + key = pem.EncodeToMemory(block) + default: + return nil, nil, errors.Errorf("error loading TLS cert and key from %v: unknown block type %v found", filename, block.Type) + } + } + + return certs, key, nil +} + // Transport returns a new http.RoundTripper with default settings applied. If // a custom rootCertFilename is non-empty, it must point to a valid PEM file, // otherwise the function will return an error. -func Transport(rootCertFilenames []string, tlsClientCert string, tlsClientKey string) (http.RoundTripper, error) { +func Transport(opts TransportOptions) (http.RoundTripper, error) { // copied from net/http tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -32,34 +79,36 @@ func Transport(rootCertFilenames []string, tlsClientCert string, tlsClientKey st TLSClientConfig: &tls.Config{}, } - if tlsClientCert != "" && tlsClientKey != "" { - c, err := tls.LoadX509KeyPair(tlsClientCert, tlsClientKey) + if opts.TLSClientCertKeyFilename != "" { + certs, key, err := readPEMCertKey(opts.TLSClientCertKeyFilename) if err != nil { - return nil, fmt.Errorf("unable to read client certificate/key pair: %v", err) + return nil, err } - tr.TLSClientConfig.Certificates = []tls.Certificate{c} - } - if rootCertFilenames == nil { - return debug.RoundTripper(tr), nil - } - - p := x509.NewCertPool() - for _, filename := range rootCertFilenames { - if filename == "" { - return nil, fmt.Errorf("empty filename for root certificate supplied") - } - b, err := ioutil.ReadFile(filename) + crt, err := tls.X509KeyPair(certs, key) if err != nil { - return nil, fmt.Errorf("unable to read root certificate: %v", err) - } - if ok := p.AppendCertsFromPEM(b); !ok { - return nil, fmt.Errorf("cannot parse root certificate from %q", filename) + return nil, errors.Errorf("parse TLS client cert or key: %v", err) } + tr.TLSClientConfig.Certificates = []tls.Certificate{crt} } - tr.TLSClientConfig.RootCAs = p + if opts.RootCertFilenames != nil { + pool := x509.NewCertPool() + for _, filename := range opts.RootCertFilenames { + if filename == "" { + return nil, errors.Errorf("empty filename for root certificate supplied") + } + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, errors.Errorf("unable to read root certificate: %v", err) + } + if ok := pool.AppendCertsFromPEM(b); !ok { + return nil, errors.Errorf("cannot parse root certificate from %q", filename) + } + } + tr.TLSClientConfig.RootCAs = pool + } - // wrap in the debug round tripper + // wrap in the debug round tripper (if active) return debug.RoundTripper(tr), nil } diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index 487d1b413..59dde38ee 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -68,7 +68,7 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun } func newTestSuite(ctx context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite { - tr, err := backend.Transport(nil, "", "") + tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index 0a5ae48e4..35a80bf7e 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -121,7 +121,7 @@ func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be resti } func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { - tr, err := backend.Transport(nil, "", "") + tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } @@ -221,7 +221,7 @@ func BenchmarkBackendMinio(t *testing.B) { } func newS3TestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil, "", "") + tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) } diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index 8e81064f9..2c4781554 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -16,7 +16,7 @@ import ( ) func newSwiftTestSuite(t testing.TB) *test.Suite { - tr, err := backend.Transport(nil, "", "") + tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) }