From 662e07d17abc214a2390ea6c52dfa27cf7f3fc32 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Mar 2015 19:53:51 +0100 Subject: [PATCH] Refactor crypto layer, switch HMAC for Poyl1305-AES HMAC-SHA256 calls SHA256() twice which is very expensive. Therefore, this commit uses Poly1305-AES instead of HMAC-SHA256. benchcmp: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 261033772 195114818 -25.25% BenchmarkChunkEncryptParallel 260973195 195787368 -24.98% BenchmarkArchiveDirectory 1050500651 1002615884 -4.56% BenchmarkPreload 23544286 24994508 +6.16% BenchmarkLoadTree 350065 427665 +22.17% BenchmarkEncryptWriter 87789753 31069126 -64.61% BenchmarkEncrypt 88283197 38259043 -56.66% BenchmarkDecryptReader 90478843 40714818 -55.00% BenchmarkEncryptDecryptReader 179917626 81231730 -54.85% BenchmarkDecrypt 87871591 37784207 -57.00% BenchmarkSaveJSON 52481 56861 +8.35% BenchmarkSaveFrom 75404085 51108596 -32.22% BenchmarkLoadJSONID 90545437 82696805 -8.67% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 40.17 53.74 1.34x BenchmarkChunkEncryptParallel 40.18 53.56 1.33x BenchmarkEncryptWriter 95.55 270.00 2.83x BenchmarkEncrypt 95.02 219.26 2.31x BenchmarkDecryptReader 92.71 206.03 2.22x BenchmarkEncryptDecryptReader 46.62 103.27 2.22x BenchmarkDecrypt 95.46 222.01 2.33x BenchmarkSaveFrom 55.62 82.07 1.48x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 112 110 -1.79% BenchmarkChunkEncryptParallel 103 100 -2.91% BenchmarkArchiveDirectory 383704 392083 +2.18% BenchmarkPreload 21765 21874 +0.50% BenchmarkLoadTree 341 436 +27.86% BenchmarkEncryptWriter 20 17 -15.00% BenchmarkEncrypt 14 13 -7.14% BenchmarkDecryptReader 18 15 -16.67% BenchmarkEncryptDecryptReader 46 39 -15.22% BenchmarkDecrypt 16 12 -25.00% BenchmarkSaveJSON 81 86 +6.17% BenchmarkSaveFrom 117 121 +3.42% BenchmarkLoadJSONID 80525 80264 -0.32% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 118956 64697 -45.61% BenchmarkChunkEncryptParallel 118972 64681 -45.63% BenchmarkArchiveDirectory 160236600 177498232 +10.77% BenchmarkPreload 2772488 3302992 +19.13% BenchmarkLoadTree 49102 46484 -5.33% BenchmarkEncryptWriter 28927 8388146 +28897.64% BenchmarkEncrypt 2473 1950 -21.15% BenchmarkDecryptReader 527827 2774 -99.47% BenchmarkEncryptDecryptReader 4100875 1528036 -62.74% BenchmarkDecrypt 2509 2154 -14.15% BenchmarkSaveJSON 4971 5892 +18.53% BenchmarkSaveFrom 40117 31742 -20.88% BenchmarkLoadJSONID 9444217 9442106 -0.02% This closes #102. --- crypto.go | 421 +++++++++++++++++++++++++++++++++++++++++++++ crypto_int_test.go | 138 +++++++++++++++ crypto_test.go | 318 ++++++++++++++++++++++++++++++++++ key.go | 407 ++++--------------------------------------- key_int_test.go | 85 --------- key_test.go | 309 --------------------------------- pools.go | 2 +- server.go | 4 +- 8 files changed, 910 insertions(+), 774 deletions(-) create mode 100644 crypto.go create mode 100644 crypto_int_test.go create mode 100644 crypto_test.go delete mode 100644 key_int_test.go diff --git a/crypto.go b/crypto.go new file mode 100644 index 000000000..6cd50557b --- /dev/null +++ b/crypto.go @@ -0,0 +1,421 @@ +package restic + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "fmt" + "io" + "io/ioutil" + "sync" + + "golang.org/x/crypto/poly1305" + "golang.org/x/crypto/scrypt" +) + +const ( + AESKeySize = 32 // for AES256 + MACKeySize = 16 + 16 // for Poly1305-AES128 + ivSize = aes.BlockSize +) + +type AESKey [32]byte +type MACKey [32]byte +type IV [ivSize]byte + +// mask for key, (cf. http://cr.yp.to/mac/poly1305-20050329.pdf) +var poly1305KeyMask = [16]byte{ + 0xff, + 0xff, + 0xff, + 0x0f, // 3: top four bits zero + 0xfc, // 4: bottom two bits zero + 0xff, + 0xff, + 0x0f, // 7: top four bits zero + 0xfc, // 8: bottom two bits zero + 0xff, + 0xff, + 0x0f, // 11: top four bits zero + 0xfc, // 12: bottom two bits zero + 0xff, + 0xff, + 0x0f, // 15: top four bits zero +} + +// key is a [32]byte, in the form k||r +func poly1305_sign(msg []byte, nonce []byte, key *MACKey) []byte { + // prepare key for low-level poly1305.Sum(): r||n + var k [32]byte + + // make sure key is masked + maskKey(key) + + // fill in nonce, encrypted with AES and key[:16] + cipher, err := aes.NewCipher(key[:16]) + if err != nil { + panic(err) + } + cipher.Encrypt(k[16:], nonce[:]) + + // copy r + copy(k[:16], key[16:]) + + // save mac in out + var out [16]byte + poly1305.Sum(&out, msg, &k) + + return out[:] +} + +// mask poly1305 key +func maskKey(k *MACKey) { + if k == nil { + return + } + for i := 0; i < poly1305.TagSize; i++ { + k[i+16] = k[i+16] & poly1305KeyMask[i] + } +} + +// key: k||r +func poly1305_verify(msg []byte, nonce []byte, key *MACKey, mac []byte) bool { + // prepare key for low-level poly1305.Sum(): r||n + var k [32]byte + + // make sure key is masked + maskKey(key) + + // fill in nonce, encrypted with AES and key[:16] + cipher, err := aes.NewCipher(key[:16]) + if err != nil { + panic(err) + } + cipher.Encrypt(k[16:], nonce[:]) + + // copy r + copy(k[:16], key[16:]) + + // copy mac to array + var m [16]byte + copy(m[:], mac) + + return poly1305.Verify(&m, msg, &k) +} + +func generateRandomAESKey() (k *AESKey) { + k = &AESKey{} + n, err := rand.Read(k[:]) + if n != AESKeySize || err != nil { + panic("unable to read enough random bytes for encryption key") + } + return +} + +// returns [32]byte == k||r +func generateRandomMACKey() (k *MACKey) { + k = &MACKey{} + n, err := rand.Read(k[:]) + if n != MACKeySize || err != nil { + panic("unable to read enough random bytes for mac key") + } + + // mask r in second half + maskKey(k) + + return +} + +func generateRandomIV() (iv IV) { + n, err := rand.Read(iv[:]) + if n != ivSize || err != nil { + panic("unable to read enough random bytes for iv") + } + return +} + +// Encrypt encrypts and signs data. Stored in ciphertext is IV || Ciphertext || +// MAC. Encrypt returns the ciphertext's length. +func Encrypt(ks *keys, ciphertext, plaintext []byte) (int, error) { + if cap(ciphertext) < len(plaintext)+ivSize+macSize { + return 0, ErrBufferTooSmall + } + + iv := generateRandomIV() + copy(ciphertext, iv[:]) + + c, err := aes.NewCipher(ks.Encrypt[:]) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + e := cipher.NewCTR(c, ciphertext[:ivSize]) + + e.XORKeyStream(ciphertext[ivSize:cap(ciphertext)], plaintext) + ciphertext = ciphertext[:ivSize+len(plaintext)] + + mac := poly1305_sign(ciphertext[ivSize:], ciphertext[:ivSize], ks.Sign) + ciphertext = append(ciphertext, mac...) + + return len(ciphertext), nil +} + +// Decrypt verifes and decrypts the ciphertext. Ciphertext must be in the form +// IV || Ciphertext || MAC. +func Decrypt(ks *keys, plaintext, ciphertext []byte) ([]byte, error) { + // check for plausible length + if len(ciphertext) < ivSize+macSize { + panic("trying to decrypt invalid data: ciphertext too small") + } + + if cap(plaintext) < len(ciphertext) { + // extend plaintext + plaintext = append(plaintext, make([]byte, len(ciphertext)-cap(plaintext))...) + } + + // extract mac + l := len(ciphertext) - macSize + ciphertext, mac := ciphertext[:l], ciphertext[l:] + + // verify mac + if !poly1305_verify(ciphertext[ivSize:], ciphertext[:ivSize], ks.Sign, mac) { + return nil, ErrUnauthenticated + } + + // extract iv + iv, ciphertext := ciphertext[:ivSize], ciphertext[ivSize:] + + // decrypt data + c, err := aes.NewCipher(ks.Encrypt[:]) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + // decrypt + e := cipher.NewCTR(c, iv) + plaintext = plaintext[:len(ciphertext)] + e.XORKeyStream(plaintext, ciphertext) + + return plaintext, nil +} + +// runs scrypt(password) +func kdf(k *Key, password string) (*keys, error) { + if len(k.Salt) == 0 { + return nil, fmt.Errorf("scrypt() called with empty salt") + } + + keybytes := MACKeySize + AESKeySize + scryptKeys, err := scrypt.Key([]byte(password), k.Salt, k.N, k.R, k.P, keybytes) + if err != nil { + return nil, fmt.Errorf("error deriving keys from password: %v", err) + } + + if len(scryptKeys) != keybytes { + return nil, fmt.Errorf("invalid numbers of bytes expanded from scrypt(): %d", len(scryptKeys)) + } + + ek := &AESKey{} + copy(ek[:], scryptKeys[:AESKeySize]) + + mk := &MACKey{} + copy(mk[:], scryptKeys[AESKeySize:]) + + ks := &keys{ + Encrypt: ek, + Sign: mk, + } + return ks, nil +} + +type encryptWriter struct { + iv IV + wroteIV bool + data *bytes.Buffer + key *keys + s cipher.Stream + w io.Writer + origWr io.Writer + err error // remember error writing iv +} + +func (e *encryptWriter) Close() error { + // write mac + mac := poly1305_sign(e.data.Bytes()[ivSize:], e.data.Bytes()[:ivSize], e.key.Sign) + _, err := e.origWr.Write(mac) + if err != nil { + return err + } + + // return buffer + FreeChunkBuf("EncryptWriter", e.data.Bytes()) + + return nil +} + +const encryptWriterChunkSize = 512 * 1024 // 512 KiB +var encryptWriterBufPool = sync.Pool{ + New: func() interface{} { + return make([]byte, encryptWriterChunkSize) + }, +} + +func (e *encryptWriter) Write(p []byte) (int, error) { + // write iv first + if !e.wroteIV { + _, e.err = e.origWr.Write(e.iv[:]) + e.wroteIV = true + } + + if e.err != nil { + return 0, e.err + } + + buf := encryptWriterBufPool.Get().([]byte) + defer encryptWriterBufPool.Put(buf) + + written := 0 + for len(p) > 0 { + max := len(p) + if max > encryptWriterChunkSize { + max = encryptWriterChunkSize + } + + e.s.XORKeyStream(buf, p[:max]) + n, err := e.w.Write(buf[:max]) + if n != max { + if err == nil { // should never happen + err = io.ErrShortWrite + } + } + + written += n + p = p[n:] + + if err != nil { + e.err = err + return written, err + } + } + + return written, nil +} + +// EncryptTo buffers data written to the returned io.WriteCloser. When Close() +// is called, the data is encrypted an written to the underlying writer. +func EncryptTo(ks *keys, wr io.Writer) io.WriteCloser { + ew := &encryptWriter{ + iv: generateRandomIV(), + data: bytes.NewBuffer(GetChunkBuf("EncryptWriter")[:0]), + key: ks, + origWr: wr, + } + + // buffer iv for mac + _, err := ew.data.Write(ew.iv[:]) + if err != nil { + panic(err) + } + + c, err := aes.NewCipher(ks.Encrypt[:]) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + ew.s = cipher.NewCTR(c, ew.iv[:]) + ew.w = io.MultiWriter(ew.data, wr) + + return ew +} + +type decryptReader struct { + buf []byte + pos int +} + +func (d *decryptReader) Read(dst []byte) (int, error) { + if d.buf == nil { + return 0, io.EOF + } + + if len(dst) == 0 { + return 0, nil + } + + remaining := len(d.buf) - d.pos + if len(dst) >= remaining { + n := copy(dst, d.buf[d.pos:]) + d.Close() + return n, io.EOF + } + + n := copy(dst, d.buf[d.pos:d.pos+len(dst)]) + d.pos += n + + return n, nil +} + +func (d *decryptReader) ReadByte() (c byte, err error) { + if d.buf == nil { + return 0, io.EOF + } + + remaining := len(d.buf) - d.pos + if remaining == 1 { + c = d.buf[d.pos] + d.Close() + return c, io.EOF + } + + c = d.buf[d.pos] + d.pos++ + + return +} + +func (d *decryptReader) Close() error { + if d.buf == nil { + return nil + } + + FreeChunkBuf("decryptReader", d.buf) + d.buf = nil + return nil +} + +// DecryptFrom verifies and decrypts the ciphertext read from rd with ks and +// makes it available on the returned Reader. Ciphertext must be in the form IV +// || Ciphertext || MAC. In order to correctly verify the ciphertext, rd is +// drained, locally buffered and made available on the returned Reader +// afterwards. If a MAC verification failure is observed, it is returned +// immediately. +func DecryptFrom(ks *keys, rd io.Reader) (io.ReadCloser, error) { + ciphertext := GetChunkBuf("decryptReader") + + ciphertext = ciphertext[0:cap(ciphertext)] + n, err := io.ReadFull(rd, ciphertext) + if err != io.ErrUnexpectedEOF { + // read remaining data + buf, e := ioutil.ReadAll(rd) + ciphertext = append(ciphertext, buf...) + n += len(buf) + err = e + } else { + err = nil + } + + if err != nil { + return nil, err + } + + ciphertext = ciphertext[:n] + + // decrypt + ciphertext, err = Decrypt(ks, ciphertext, ciphertext) + if err != nil { + return nil, err + } + + return &decryptReader{buf: ciphertext}, nil +} diff --git a/crypto_int_test.go b/crypto_int_test.go new file mode 100644 index 000000000..2da59ee19 --- /dev/null +++ b/crypto_int_test.go @@ -0,0 +1,138 @@ +package restic + +import ( + "bytes" + "encoding/hex" + "testing" +) + +// test vectors from http://cr.yp.to/mac/poly1305-20050329.pdf +var poly1305_tests = []struct { + msg []byte + r []byte + k []byte + nonce []byte + mac []byte +}{ + { + []byte("\xf3\xf6"), + []byte("\x85\x1f\xc4\x0c\x34\x67\xac\x0b\xe0\x5c\xc2\x04\x04\xf3\xf7\x00"), + []byte("\xec\x07\x4c\x83\x55\x80\x74\x17\x01\x42\x5b\x62\x32\x35\xad\xd6"), + []byte("\xfb\x44\x73\x50\xc4\xe8\x68\xc5\x2a\xc3\x27\x5c\xf9\xd4\x32\x7e"), + []byte("\xf4\xc6\x33\xc3\x04\x4f\xc1\x45\xf8\x4f\x33\x5c\xb8\x19\x53\xde"), + }, + { + []byte(""), + []byte("\xa0\xf3\x08\x00\x00\xf4\x64\x00\xd0\xc7\xe9\x07\x6c\x83\x44\x03"), + []byte("\x75\xde\xaa\x25\xc0\x9f\x20\x8e\x1d\xc4\xce\x6b\x5c\xad\x3f\xbf"), + []byte("\x61\xee\x09\x21\x8d\x29\xb0\xaa\xed\x7e\x15\x4a\x2c\x55\x09\xcc"), + []byte("\xdd\x3f\xab\x22\x51\xf1\x1a\xc7\x59\xf0\x88\x71\x29\xcc\x2e\xe7"), + }, + { + []byte("\x66\x3c\xea\x19\x0f\xfb\x83\xd8\x95\x93\xf3\xf4\x76\xb6\xbc\x24\xd7\xe6\x79\x10\x7e\xa2\x6a\xdb\x8c\xaf\x66\x52\xd0\x65\x61\x36"), + []byte("\x48\x44\x3d\x0b\xb0\xd2\x11\x09\xc8\x9a\x10\x0b\x5c\xe2\xc2\x08"), + []byte("\x6a\xcb\x5f\x61\xa7\x17\x6d\xd3\x20\xc5\xc1\xeb\x2e\xdc\xdc\x74"), + []byte("\xae\x21\x2a\x55\x39\x97\x29\x59\x5d\xea\x45\x8b\xc6\x21\xff\x0e"), + []byte("\x0e\xe1\xc1\x6b\xb7\x3f\x0f\x4f\xd1\x98\x81\x75\x3c\x01\xcd\xbe"), + }, { + []byte("\xab\x08\x12\x72\x4a\x7f\x1e\x34\x27\x42\xcb\xed\x37\x4d\x94\xd1\x36\xc6\xb8\x79\x5d\x45\xb3\x81\x98\x30\xf2\xc0\x44\x91\xfa\xf0\x99\x0c\x62\xe4\x8b\x80\x18\xb2\xc3\xe4\xa0\xfa\x31\x34\xcb\x67\xfa\x83\xe1\x58\xc9\x94\xd9\x61\xc4\xcb\x21\x09\x5c\x1b\xf9"), + []byte("\x12\x97\x6a\x08\xc4\x42\x6d\x0c\xe8\xa8\x24\x07\xc4\xf4\x82\x07"), + []byte("\xe1\xa5\x66\x8a\x4d\x5b\x66\xa5\xf6\x8c\xc5\x42\x4e\xd5\x98\x2d"), + []byte("\x9a\xe8\x31\xe7\x43\x97\x8d\x3a\x23\x52\x7c\x71\x28\x14\x9e\x3a"), + []byte("\x51\x54\xad\x0d\x2c\xb2\x6e\x01\x27\x4f\xc5\x11\x48\x49\x1f\x1b"), + }, +} + +func TestPoly1305(t *testing.T) { + for _, test := range poly1305_tests { + key := &MACKey{} + copy(key[:16], test.k) + copy(key[16:], test.r) + mac := poly1305_sign(test.msg, test.nonce, key) + + if !bytes.Equal(mac, test.mac) { + t.Fatalf("wrong mac calculated, want: %02x, got: %02x", test.mac, mac) + } + + if !poly1305_verify(test.msg, test.nonce, key, test.mac) { + t.Fatalf("mac does not verify: mac: %02x", test.mac) + } + } +} + +var test_values = []struct { + ekey AESKey + skey MACKey + ciphertext []byte + plaintext []byte + should_panic bool +}{ + { + ekey: AESKey([...]byte{0x30, 0x3e, 0x86, 0x87, 0xb1, 0xd7, 0xdb, 0x18, 0x42, 0x1b, 0xdc, 0x6b, 0xb8, 0x58, 0x8c, 0xca, + 0xda, 0xc4, 0xd5, 0x9e, 0xe8, 0x7b, 0x8f, 0xf7, 0x0c, 0x44, 0xe6, 0x35, 0x79, 0x0c, 0xaf, 0xef}), + skey: MACKey([...]byte{0xef, 0x4d, 0x88, 0x24, 0xcb, 0x80, 0xb2, 0xbc, 0xc5, 0xfb, 0xff, 0x8a, 0x9b, 0x12, 0xa4, 0x2c, + 0xcc, 0x8d, 0x4b, 0x94, 0x8e, 0xe0, 0xeb, 0xfe, 0x1d, 0x41, 0x5d, 0xe9, 0x21, 0xd1, 0x03, 0x53}), + ciphertext: decode_hex("69fb41c62d12def4593bd71757138606338f621aeaeb39da0fe4f99233f8037a54ea63338a813bcf3f75d8c3cc75dddf8750"), + plaintext: []byte("Dies ist ein Test!"), + }, +} + +func decode_hex(s string) []byte { + d, _ := hex.DecodeString(s) + return d +} + +// returns true if function called panic +func should_panic(f func()) (did_panic bool) { + defer func() { + if r := recover(); r != nil { + did_panic = true + } + }() + + f() + + return false +} + +func TestCrypto(t *testing.T) { + r := &Key{} + + for _, tv := range test_values { + // test encryption + r.master = &keys{ + Encrypt: &tv.ekey, + Sign: &tv.skey, + } + + msg := make([]byte, maxCiphertextSize) + n, err := Encrypt(r.master, msg, tv.plaintext) + if err != nil { + t.Fatal(err) + } + msg = msg[:n] + + // decrypt message + _, err = Decrypt(r.master, []byte{}, msg) + if err != nil { + t.Fatal(err) + } + + // change mac, this must fail + msg[len(msg)-8] ^= 0x23 + + if _, err = Decrypt(r.master, []byte{}, msg); err != ErrUnauthenticated { + t.Fatal("wrong HMAC value not detected") + } + + // test decryption + p, err := Decrypt(r.master, []byte{}, tv.ciphertext) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(p, tv.plaintext) { + t.Fatalf("wrong plaintext: expected %q but got %q\n", tv.plaintext, p) + } + } +} diff --git a/crypto_test.go b/crypto_test.go new file mode 100644 index 000000000..4c787199b --- /dev/null +++ b/crypto_test.go @@ -0,0 +1,318 @@ +package restic_test + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "testing" + + "github.com/restic/restic" + "github.com/restic/restic/chunker" +) + +func TestEncryptDecrypt(t *testing.T) { + s := setupBackend(t) + defer teardownBackend(t, s) + k := setupKey(t, s, testPassword) + + tests := []int{5, 23, 2<<18 + 23, 1 << 20} + if *testLargeCrypto { + tests = append(tests, 7<<20+123) + } + + for _, size := range tests { + data := make([]byte, size) + _, err := io.ReadFull(randomReader(42, size), data) + ok(t, err) + + ciphertext := restic.GetChunkBuf("TestEncryptDecrypt") + n, err := k.Encrypt(ciphertext, data) + ok(t, err) + + plaintext, err := k.Decrypt(nil, ciphertext[:n]) + ok(t, err) + + restic.FreeChunkBuf("TestEncryptDecrypt", ciphertext) + + equals(t, plaintext, data) + } +} + +func TestSmallBuffer(t *testing.T) { + s := setupBackend(t) + defer teardownBackend(t, s) + k := setupKey(t, s, testPassword) + + size := 600 + data := make([]byte, size) + f, err := os.Open("/dev/urandom") + ok(t, err) + + _, err = io.ReadFull(f, data) + ok(t, err) + + ciphertext := make([]byte, size/2) + _, err = k.Encrypt(ciphertext, data) + // this must throw an error, since the target slice is too small + assert(t, err != nil && err == restic.ErrBufferTooSmall, + "expected restic.ErrBufferTooSmall, got %#v", err) +} + +func TestLargeEncrypt(t *testing.T) { + if !*testLargeCrypto { + t.SkipNow() + } + + s := setupBackend(t) + defer teardownBackend(t, s) + k := setupKey(t, s, testPassword) + + for _, size := range []int{chunker.MaxSize, chunker.MaxSize + 1, chunker.MaxSize + 1<<20} { + data := make([]byte, size) + f, err := os.Open("/dev/urandom") + ok(t, err) + + _, err = io.ReadFull(f, data) + ok(t, err) + + ciphertext := make([]byte, size+restic.CiphertextExtension) + n, err := k.Encrypt(ciphertext, data) + ok(t, err) + + plaintext, err := k.Decrypt([]byte{}, ciphertext[:n]) + ok(t, err) + + equals(t, plaintext, data) + } +} + +func BenchmarkEncryptWriter(b *testing.B) { + size := 8 << 20 // 8MiB + rd := randomReader(23, size) + + be := setupBackend(b) + defer teardownBackend(b, be) + k := setupKey(b, be, testPassword) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + rd.Seek(0, 0) + wr := k.EncryptTo(ioutil.Discard) + _, err := io.Copy(wr, rd) + ok(b, err) + } +} + +func BenchmarkEncrypt(b *testing.B) { + size := 8 << 20 // 8MiB + data := make([]byte, size) + + be := setupBackend(b) + defer teardownBackend(b, be) + k := setupKey(b, be, testPassword) + + buf := make([]byte, len(data)+restic.CiphertextExtension) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + _, err := k.Encrypt(buf, data) + ok(b, err) + } +} + +func BenchmarkDecryptReader(b *testing.B) { + be := setupBackend(b) + defer teardownBackend(b, be) + k := setupKey(b, be, testPassword) + + size := 8 << 20 // 8MiB + buf := get_random(23, size) + + ciphertext := make([]byte, len(buf)+restic.CiphertextExtension) + _, err := k.Encrypt(ciphertext, buf) + ok(b, err) + + rd := bytes.NewReader(ciphertext) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + rd.Seek(0, 0) + decRd, err := k.DecryptFrom(rd) + ok(b, err) + + _, err = io.Copy(ioutil.Discard, decRd) + ok(b, err) + } +} + +func BenchmarkEncryptDecryptReader(b *testing.B) { + be := setupBackend(b) + defer teardownBackend(b, be) + k := setupKey(b, be, testPassword) + + size := 8 << 20 // 8MiB + rd := randomReader(23, size) + + b.ResetTimer() + b.SetBytes(int64(size)) + + buf := bytes.NewBuffer(nil) + for i := 0; i < b.N; i++ { + rd.Seek(0, 0) + buf.Reset() + wr := k.EncryptTo(buf) + _, err := io.Copy(wr, rd) + ok(b, err) + ok(b, wr.Close()) + + r, err := k.DecryptFrom(buf) + ok(b, err) + + _, err = io.Copy(ioutil.Discard, r) + ok(b, err) + } + + restic.PoolAlloc() +} + +func BenchmarkDecrypt(b *testing.B) { + size := 8 << 20 // 8MiB + data := make([]byte, size) + + s := setupBackend(b) + defer teardownBackend(b, s) + k := setupKey(b, s, testPassword) + + ciphertext := restic.GetChunkBuf("BenchmarkDecrypt") + defer restic.FreeChunkBuf("BenchmarkDecrypt", ciphertext) + plaintext := restic.GetChunkBuf("BenchmarkDecrypt") + defer restic.FreeChunkBuf("BenchmarkDecrypt", plaintext) + + n, err := k.Encrypt(ciphertext, data) + ok(b, err) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + plaintext, err = k.Decrypt(plaintext, ciphertext[:n]) + ok(b, err) + } +} + +func TestEncryptStreamWriter(t *testing.T) { + s := setupBackend(t) + defer teardownBackend(t, s) + k := setupKey(t, s, testPassword) + + tests := []int{5, 23, 2<<18 + 23, 1 << 20} + if *testLargeCrypto { + tests = append(tests, 7<<20+123) + } + + for _, size := range tests { + data := make([]byte, size) + _, err := io.ReadFull(randomReader(42, size), data) + ok(t, err) + + ciphertext := bytes.NewBuffer(nil) + wr := k.EncryptTo(ciphertext) + + _, err = io.Copy(wr, bytes.NewReader(data)) + ok(t, err) + ok(t, wr.Close()) + + l := len(data) + restic.CiphertextExtension + assert(t, len(ciphertext.Bytes()) == l, + "wrong ciphertext length: expected %d, got %d", + l, len(ciphertext.Bytes())) + + // decrypt with default function + plaintext, err := k.Decrypt([]byte{}, ciphertext.Bytes()) + ok(t, err) + assert(t, bytes.Equal(data, plaintext), + "wrong plaintext after decryption: expected %02x, got %02x", + data, plaintext) + } +} + +func TestDecryptStreamReader(t *testing.T) { + s := setupBackend(t) + defer teardownBackend(t, s) + k := setupKey(t, s, testPassword) + + tests := []int{5, 23, 2<<18 + 23, 1 << 20} + if *testLargeCrypto { + tests = append(tests, 7<<20+123) + } + + for _, size := range tests { + data := make([]byte, size) + _, err := io.ReadFull(randomReader(42, size), data) + ok(t, err) + + ciphertext := make([]byte, size+restic.CiphertextExtension) + + // encrypt with default function + n, err := k.Encrypt(ciphertext, data) + ok(t, err) + assert(t, n == len(data)+restic.CiphertextExtension, + "wrong number of bytes returned after encryption: expected %d, got %d", + len(data)+restic.CiphertextExtension, n) + + rd, err := k.DecryptFrom(bytes.NewReader(ciphertext)) + ok(t, err) + + plaintext, err := ioutil.ReadAll(rd) + ok(t, err) + + assert(t, bytes.Equal(data, plaintext), + "wrong plaintext after decryption: expected %02x, got %02x", + data, plaintext) + } +} + +func TestEncryptWriter(t *testing.T) { + s := setupBackend(t) + defer teardownBackend(t, s) + k := setupKey(t, s, testPassword) + + tests := []int{5, 23, 2<<18 + 23, 1 << 20} + if *testLargeCrypto { + tests = append(tests, 7<<20+123) + } + + for _, size := range tests { + data := make([]byte, size) + _, err := io.ReadFull(randomReader(42, size), data) + ok(t, err) + + buf := bytes.NewBuffer(nil) + wr := k.EncryptTo(buf) + + _, err = io.Copy(wr, bytes.NewReader(data)) + ok(t, err) + ok(t, wr.Close()) + + ciphertext := buf.Bytes() + + l := len(data) + restic.CiphertextExtension + assert(t, len(ciphertext) == l, + "wrong ciphertext length: expected %d, got %d", + l, len(ciphertext)) + + // decrypt with default function + plaintext, err := k.Decrypt([]byte{}, ciphertext) + ok(t, err) + assert(t, bytes.Equal(data, plaintext), + "wrong plaintext after decryption: expected %02x, got %02x", + data, plaintext) + } +} diff --git a/key.go b/key.go index cbaf5a92c..3e9d10c67 100644 --- a/key.go +++ b/key.go @@ -1,33 +1,25 @@ package restic import ( - "crypto/aes" - "crypto/cipher" - "crypto/hmac" "crypto/rand" - "crypto/sha256" "encoding/json" "errors" "fmt" - "hash" "io" - "io/ioutil" "os" "os/user" - "sync" "time" "github.com/restic/restic/backend" "github.com/restic/restic/chunker" - "golang.org/x/crypto/scrypt" + "golang.org/x/crypto/poly1305" ) // max size is 8MiB, defined in chunker -const ivSize = aes.BlockSize -const hmacSize = sha256.Size -const maxCiphertextSize = ivSize + chunker.MaxSize + hmacSize -const CiphertextExtension = ivSize + hmacSize +const macSize = poly1305.TagSize // Poly1305 size is 16 byte +const maxCiphertextSize = ivSize + chunker.MaxSize + macSize +const CiphertextExtension = ivSize + macSize var ( // ErrUnauthenticated is returned when ciphertext verification has failed. @@ -46,8 +38,6 @@ const ( scryptR = 8 scryptP = 1 scryptSaltsize = 64 - aesKeysize = 32 // for AES256 - hmacKeysize = 32 // for HMAC with SHA256 ) // Key represents an encrypted master key for a repository. @@ -72,8 +62,8 @@ type Key struct { // keys is a JSON structure that holds signing and encryption keys. type keys struct { - Sign []byte - Encrypt []byte + Sign *MACKey + Encrypt *AESKey } // CreateKey initializes a master key in the given backend and encrypts it with @@ -95,7 +85,7 @@ func OpenKey(s Server, id backend.ID, password string) (*Key, error) { } // derive user key - k.user, err = k.scrypt(password) + k.user, err = kdf(k, password) if err != nil { return nil, err } @@ -186,18 +176,15 @@ func AddKey(s Server, password string, template *Key) (*Key, error) { panic("unable to read enough random bytes for salt") } - // call scrypt() to derive user key - newkey.user, err = newkey.scrypt(password) + // call KDF to derive user key + newkey.user, err = kdf(newkey, password) if err != nil { return nil, err } if template == nil { // generate new random master keys - newkey.master, err = newkey.newKeys() - if err != nil { - return nil, err - } + newkey.master = newkey.newKeys() } else { // copy master keys from old key newkey.master = template.master @@ -247,43 +234,11 @@ func AddKey(s Server, password string, template *Key) (*Key, error) { return newkey, nil } -func (k *Key) scrypt(password string) (*keys, error) { - if len(k.Salt) == 0 { - return nil, fmt.Errorf("scrypt() called with empty salt") +func (k *Key) newKeys() *keys { + return &keys{ + Encrypt: generateRandomAESKey(), + Sign: generateRandomMACKey(), } - - keybytes := hmacKeysize + aesKeysize - scryptKeys, err := scrypt.Key([]byte(password), k.Salt, k.N, k.R, k.P, keybytes) - if err != nil { - return nil, fmt.Errorf("error deriving keys from password: %v", err) - } - - if len(scryptKeys) != keybytes { - return nil, fmt.Errorf("invalid numbers of bytes expanded from scrypt(): %d", len(scryptKeys)) - } - - ks := &keys{ - Encrypt: scryptKeys[:aesKeysize], - Sign: scryptKeys[aesKeysize:], - } - return ks, nil -} - -func (k *Key) newKeys() (*keys, error) { - ks := &keys{ - Encrypt: make([]byte, aesKeysize), - Sign: make([]byte, hmacKeysize), - } - n, err := rand.Read(ks.Encrypt) - if n != aesKeysize || err != nil { - panic("unable to read enough random bytes for encryption key") - } - n, err = rand.Read(ks.Sign) - if n != hmacKeysize || err != nil { - panic("unable to read enough random bytes for signing key") - } - - return ks, nil } func (k *Key) newIV(buf []byte) error { @@ -296,364 +251,62 @@ func (k *Key) newIV(buf []byte) error { return nil } -// Encrypt encrypts and signs data. Stored in ciphertext is IV || Ciphertext || -// HMAC. Encrypt returns the ciphertext's length. For the hash function, SHA256 -// is used, so the overhead is 16+32=48 byte. -func (k *Key) encrypt(ks *keys, ciphertext, plaintext []byte) (int, error) { - if cap(ciphertext) < len(plaintext)+ivSize+hmacSize { - return 0, ErrBufferTooSmall - } - - _, err := io.ReadFull(rand.Reader, ciphertext[:ivSize]) - if err != nil { - panic(fmt.Sprintf("unable to generate new random iv: %v", err)) - } - - c, err := aes.NewCipher(ks.Encrypt) - if err != nil { - panic(fmt.Sprintf("unable to create cipher: %v", err)) - } - - e := cipher.NewCTR(c, ciphertext[:ivSize]) - e.XORKeyStream(ciphertext[ivSize:cap(ciphertext)], plaintext) - ciphertext = ciphertext[:ivSize+len(plaintext)] - - hm := hmac.New(sha256.New, ks.Sign) - - n, err := hm.Write(ciphertext) - if err != nil || n != len(ciphertext) { - panic(fmt.Sprintf("unable to calculate hmac of ciphertext: %v", err)) - } - - ciphertext = hm.Sum(ciphertext) - - return len(ciphertext), nil -} - // EncryptUser encrypts and signs data with the user key. Stored in ciphertext -// is IV || Ciphertext || HMAC. Returns the ciphertext length. For the hash -// function, SHA256 is used, so the overhead is 16+32=48 byte. +// is IV || Ciphertext || MAC. func (k *Key) EncryptUser(ciphertext, plaintext []byte) (int, error) { - return k.encrypt(k.user, ciphertext, plaintext) + return Encrypt(k.user, ciphertext, plaintext) } // Encrypt encrypts and signs data with the master key. Stored in ciphertext is -// IV || Ciphertext || HMAC. Returns the ciphertext length. For the hash -// function, SHA256 is used, so the overhead is 16+32=48 byte. +// IV || Ciphertext || MAC. Returns the ciphertext length. func (k *Key) Encrypt(ciphertext, plaintext []byte) (int, error) { - return k.encrypt(k.master, ciphertext, plaintext) -} - -type encryptWriter struct { - iv []byte - wroteIV bool - h hash.Hash - s cipher.Stream - w io.Writer - origWr io.Writer - err error // remember error writing iv -} - -func (e *encryptWriter) Close() error { - // write hmac - _, err := e.origWr.Write(e.h.Sum(nil)) - if err != nil { - return err - } - - return nil -} - -const encryptWriterChunkSize = 512 * 1024 // 512 KiB -var encryptWriterBufPool = sync.Pool{ - New: func() interface{} { - return make([]byte, encryptWriterChunkSize) - }, -} - -func (e *encryptWriter) Write(p []byte) (int, error) { - // write iv first - if !e.wroteIV { - _, e.err = e.origWr.Write(e.iv) - e.wroteIV = true - } - - if e.err != nil { - return 0, e.err - } - - buf := encryptWriterBufPool.Get().([]byte) - defer encryptWriterBufPool.Put(buf) - - written := 0 - for len(p) > 0 { - max := len(p) - if max > encryptWriterChunkSize { - max = encryptWriterChunkSize - } - - e.s.XORKeyStream(buf, p[:max]) - n, err := e.w.Write(buf[:max]) - if n != max { - if err == nil { // should never happen - err = io.ErrShortWrite - } - } - - written += n - p = p[n:] - - if err != nil { - e.err = err - return written, err - } - } - - return written, nil -} - -func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser { - ew := &encryptWriter{ - iv: make([]byte, ivSize), - h: hmac.New(sha256.New, ks.Sign), - origWr: wr, - } - - _, err := io.ReadFull(rand.Reader, ew.iv) - if err != nil { - panic(fmt.Sprintf("unable to generate new random iv: %v", err)) - } - - // write iv to hmac - _, err = ew.h.Write(ew.iv) - if err != nil { - panic(err) - } - - c, err := aes.NewCipher(ks.Encrypt) - if err != nil { - panic(fmt.Sprintf("unable to create cipher: %v", err)) - } - - ew.s = cipher.NewCTR(c, ew.iv) - ew.w = io.MultiWriter(ew.h, wr) - - return ew + return Encrypt(k.master, ciphertext, plaintext) } // EncryptTo encrypts and signs data with the master key. The returned // io.Writer writes IV || Ciphertext || HMAC. For the hash function, SHA256 is // used. func (k *Key) EncryptTo(wr io.Writer) io.WriteCloser { - return k.encryptTo(k.master, wr) + return EncryptTo(k.master, wr) } // EncryptUserTo encrypts and signs data with the user key. The returned // io.Writer writes IV || Ciphertext || HMAC. For the hash function, SHA256 is // used. func (k *Key) EncryptUserTo(wr io.Writer) io.WriteCloser { - return k.encryptTo(k.user, wr) -} - -// Decrypt verifes and decrypts the ciphertext. Ciphertext must be in the form -// IV || Ciphertext || HMAC. -func (k *Key) decrypt(ks *keys, plaintext, ciphertext []byte) ([]byte, error) { - // check for plausible length - if len(ciphertext) < ivSize+hmacSize { - panic("trying to decrypt invalid data: ciphertext too small") - } - - if cap(plaintext) < len(ciphertext) { - // extend plaintext - plaintext = append(plaintext, make([]byte, len(ciphertext)-cap(plaintext))...) - } - - hm := hmac.New(sha256.New, ks.Sign) - - // extract hmac - l := len(ciphertext) - hm.Size() - ciphertext, mac := ciphertext[:l], ciphertext[l:] - - // calculate new hmac - n, err := hm.Write(ciphertext) - if err != nil || n != len(ciphertext) { - panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err)) - } - - // verify hmac - mac2 := hm.Sum(nil) - - if !hmac.Equal(mac, mac2) { - return nil, ErrUnauthenticated - } - - // extract iv - iv, ciphertext := ciphertext[:aes.BlockSize], ciphertext[aes.BlockSize:] - - // decrypt data - c, err := aes.NewCipher(ks.Encrypt) - if err != nil { - panic(fmt.Sprintf("unable to create cipher: %v", err)) - } - - // decrypt - e := cipher.NewCTR(c, iv) - plaintext = plaintext[:len(ciphertext)] - e.XORKeyStream(plaintext, ciphertext) - - return plaintext, nil + return EncryptTo(k.user, wr) } // Decrypt verifes and decrypts the ciphertext with the master key. Ciphertext -// must be in the form IV || Ciphertext || HMAC. +// must be in the form IV || Ciphertext || MAC. func (k *Key) Decrypt(plaintext, ciphertext []byte) ([]byte, error) { - return k.decrypt(k.master, plaintext, ciphertext) + return Decrypt(k.master, plaintext, ciphertext) } // DecryptUser verifes and decrypts the ciphertext with the user key. Ciphertext -// must be in the form IV || Ciphertext || HMAC. +// must be in the form IV || Ciphertext || MAC. func (k *Key) DecryptUser(plaintext, ciphertext []byte) ([]byte, error) { - return k.decrypt(k.user, plaintext, ciphertext) -} - -type decryptReader struct { - buf []byte - pos int -} - -func (d *decryptReader) Read(dst []byte) (int, error) { - if d.buf == nil { - return 0, io.EOF - } - - if len(dst) == 0 { - return 0, nil - } - - remaining := len(d.buf) - d.pos - if len(dst) >= remaining { - n := copy(dst, d.buf[d.pos:]) - d.Close() - return n, io.EOF - } - - n := copy(dst, d.buf[d.pos:d.pos+len(dst)]) - d.pos += n - - return n, nil -} - -func (d *decryptReader) ReadByte() (c byte, err error) { - if d.buf == nil { - return 0, io.EOF - } - - remaining := len(d.buf) - d.pos - if remaining == 1 { - c = d.buf[d.pos] - d.Close() - return c, io.EOF - } - - c = d.buf[d.pos] - d.pos++ - - return -} - -func (d *decryptReader) Close() error { - if d.buf == nil { - return nil - } - - FreeChunkBuf("decryptReader", d.buf) - d.buf = nil - return nil -} - -// decryptFrom verifies and decrypts the ciphertext read from rd with ks and -// makes it available on the returned Reader. Ciphertext must be in the form IV -// || Ciphertext || HMAC. In order to correctly verify the ciphertext, rd is -// drained, locally buffered and made available on the returned Reader -// afterwards. If an HMAC verification failure is observed, it is returned -// immediately. -func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.ReadCloser, error) { - ciphertext := GetChunkBuf("decryptReader") - ciphertext = ciphertext[0:cap(ciphertext)] - n, err := io.ReadFull(rd, ciphertext) - if err != io.ErrUnexpectedEOF { - // read remaining data - buf, e := ioutil.ReadAll(rd) - ciphertext = append(ciphertext, buf...) - n += len(buf) - err = e - } else { - err = nil - } - - if err != nil { - return nil, err - } - - ciphertext = ciphertext[:n] - - // check for plausible length - if len(ciphertext) < ivSize+hmacSize { - panic("trying to decrypt invalid data: ciphertext too small") - } - - hm := hmac.New(sha256.New, ks.Sign) - - // extract hmac - l := len(ciphertext) - hm.Size() - ciphertext, mac := ciphertext[:l], ciphertext[l:] - - // calculate new hmac - n, err = hm.Write(ciphertext) - if err != nil || n != len(ciphertext) { - panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err)) - } - - // verify hmac - mac2 := hm.Sum(nil) - - if !hmac.Equal(mac, mac2) { - return nil, ErrUnauthenticated - } - - // extract iv - iv, ciphertext := ciphertext[:aes.BlockSize], ciphertext[aes.BlockSize:] - - // decrypt data - c, err := aes.NewCipher(ks.Encrypt) - if err != nil { - panic(fmt.Sprintf("unable to create cipher: %v", err)) - } - - stream := cipher.NewCTR(c, iv) - stream.XORKeyStream(ciphertext, ciphertext) - - return &decryptReader{buf: ciphertext}, nil + return Decrypt(k.user, plaintext, ciphertext) } // DecryptFrom verifies and decrypts the ciphertext read from rd and makes it // available on the returned Reader. Ciphertext must be in the form IV || -// Ciphertext || HMAC. In order to correctly verify the ciphertext, rd is +// Ciphertext || MAC. In order to correctly verify the ciphertext, rd is // drained, locally buffered and made available on the returned Reader -// afterwards. If an HMAC verification failure is observed, it is returned +// afterwards. If an MAC verification failure is observed, it is returned // immediately. func (k *Key) DecryptFrom(rd io.Reader) (io.ReadCloser, error) { - return k.decryptFrom(k.master, rd) + return DecryptFrom(k.master, rd) } // DecryptFrom verifies and decrypts the ciphertext read from rd with the user // key and makes it available on the returned Reader. Ciphertext must be in the -// form IV || Ciphertext || HMAC. In order to correctly verify the ciphertext, +// form IV || Ciphertext || MAC. In order to correctly verify the ciphertext, // rd is drained, locally buffered and made available on the returned Reader -// afterwards. If an HMAC verification failure is observed, it is returned +// afterwards. If an MAC verification failure is observed, it is returned // immediately. func (k *Key) DecryptUserFrom(rd io.Reader) (io.ReadCloser, error) { - return k.decryptFrom(k.user, rd) + return DecryptFrom(k.user, rd) } func (k *Key) String() string { diff --git a/key_int_test.go b/key_int_test.go deleted file mode 100644 index a1fb8fd5e..000000000 --- a/key_int_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package restic - -import ( - "bytes" - "encoding/hex" - "testing" -) - -var test_values = []struct { - ekey, skey []byte - ciphertext []byte - plaintext []byte - should_panic bool -}{ - { - ekey: decode_hex("303e8687b1d7db18421bdc6bb8588ccadac4d59ee87b8ff70c44e635790cafef"), - skey: decode_hex("cc8d4b948ee0ebfe1d415de921d10353ef4d8824cb80b2bcc5fbff8a9b12a42c"), - ciphertext: decode_hex("fe85b32b108308f6f8834a96e463b66e0eae6a0f1e9809da0773a2db12a24528bce3220e6a5700b40bd45ef2a2ce96a7fc0a895a019d4a77eef5fc9579297059c6d0"), - plaintext: []byte("Dies ist ein Test!"), - }, -} - -func decode_hex(s string) []byte { - d, _ := hex.DecodeString(s) - return d -} - -// returns true if function called panic -func should_panic(f func()) (did_panic bool) { - defer func() { - if r := recover(); r != nil { - did_panic = true - } - }() - - f() - - return false -} - -func TestCrypto(t *testing.T) { - r := &Key{} - - for _, tv := range test_values { - // test encryption - r.master = &keys{ - Encrypt: tv.ekey, - Sign: tv.skey, - } - - msg := make([]byte, maxCiphertextSize) - n, err := r.encrypt(r.master, msg, tv.plaintext) - if err != nil { - t.Fatal(err) - } - msg = msg[:n] - - // decrypt message - _, err = r.decrypt(r.master, []byte{}, msg) - if err != nil { - t.Fatal(err) - } - - // change hmac, this must fail - msg[len(msg)-8] ^= 0x23 - - if _, err = r.decrypt(r.master, []byte{}, msg); err != ErrUnauthenticated { - t.Fatal("wrong HMAC value not detected") - } - - // test decryption - p, err := r.decrypt(r.master, []byte{}, tv.ciphertext) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(p, tv.plaintext) { - t.Fatalf("wrong plaintext: expected %q but got %q\n", tv.plaintext, p) - } - } -} - -func TestHmac(t *testing.T) { - -} diff --git a/key_test.go b/key_test.go index 6c22e7e87..99eb5941b 100644 --- a/key_test.go +++ b/key_test.go @@ -1,9 +1,7 @@ package restic_test import ( - "bytes" "flag" - "io" "io/ioutil" "os" "path/filepath" @@ -11,7 +9,6 @@ import ( "github.com/restic/restic" "github.com/restic/restic/backend" - "github.com/restic/restic/chunker" ) var testPassword = "foobar" @@ -56,309 +53,3 @@ func TestRepo(t *testing.T) { defer teardownBackend(t, s) _ = setupKey(t, s, testPassword) } - -func TestEncryptDecrypt(t *testing.T) { - s := setupBackend(t) - defer teardownBackend(t, s) - k := setupKey(t, s, testPassword) - - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { - tests = append(tests, 7<<20+123) - } - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(randomReader(42, size), data) - ok(t, err) - - ciphertext := restic.GetChunkBuf("TestEncryptDecrypt") - n, err := k.Encrypt(ciphertext, data) - ok(t, err) - - plaintext, err := k.Decrypt(nil, ciphertext[:n]) - ok(t, err) - - restic.FreeChunkBuf("TestEncryptDecrypt", ciphertext) - - equals(t, plaintext, data) - } -} - -func TestSmallBuffer(t *testing.T) { - s := setupBackend(t) - defer teardownBackend(t, s) - k := setupKey(t, s, testPassword) - - size := 600 - data := make([]byte, size) - f, err := os.Open("/dev/urandom") - ok(t, err) - - _, err = io.ReadFull(f, data) - ok(t, err) - - ciphertext := make([]byte, size/2) - _, err = k.Encrypt(ciphertext, data) - // this must throw an error, since the target slice is too small - assert(t, err != nil && err == restic.ErrBufferTooSmall, - "expected restic.ErrBufferTooSmall, got %#v", err) -} - -func TestLargeEncrypt(t *testing.T) { - if !*testLargeCrypto { - t.SkipNow() - } - - s := setupBackend(t) - defer teardownBackend(t, s) - k := setupKey(t, s, testPassword) - - for _, size := range []int{chunker.MaxSize, chunker.MaxSize + 1, chunker.MaxSize + 1<<20} { - data := make([]byte, size) - f, err := os.Open("/dev/urandom") - ok(t, err) - - _, err = io.ReadFull(f, data) - ok(t, err) - - ciphertext := make([]byte, size+restic.CiphertextExtension) - n, err := k.Encrypt(ciphertext, data) - ok(t, err) - - plaintext, err := k.Decrypt([]byte{}, ciphertext[:n]) - ok(t, err) - - equals(t, plaintext, data) - } -} - -func BenchmarkEncryptWriter(b *testing.B) { - size := 8 << 20 // 8MiB - rd := randomReader(23, size) - - be := setupBackend(b) - defer teardownBackend(b, be) - k := setupKey(b, be, testPassword) - - b.ResetTimer() - b.SetBytes(int64(size)) - - for i := 0; i < b.N; i++ { - rd.Seek(0, 0) - wr := k.EncryptTo(ioutil.Discard) - _, err := io.Copy(wr, rd) - ok(b, err) - } -} - -func BenchmarkEncrypt(b *testing.B) { - size := 8 << 20 // 8MiB - data := make([]byte, size) - - be := setupBackend(b) - defer teardownBackend(b, be) - k := setupKey(b, be, testPassword) - - buf := make([]byte, len(data)+restic.CiphertextExtension) - - b.ResetTimer() - b.SetBytes(int64(size)) - - for i := 0; i < b.N; i++ { - _, err := k.Encrypt(buf, data) - ok(b, err) - } -} - -func BenchmarkDecryptReader(b *testing.B) { - be := setupBackend(b) - defer teardownBackend(b, be) - k := setupKey(b, be, testPassword) - - size := 8 << 20 // 8MiB - buf := get_random(23, size) - - ciphertext := make([]byte, len(buf)+restic.CiphertextExtension) - _, err := k.Encrypt(ciphertext, buf) - ok(b, err) - - rd := bytes.NewReader(ciphertext) - - b.ResetTimer() - b.SetBytes(int64(size)) - - for i := 0; i < b.N; i++ { - rd.Seek(0, 0) - decRd, err := k.DecryptFrom(rd) - ok(b, err) - - _, err = io.Copy(ioutil.Discard, decRd) - ok(b, err) - } -} - -func BenchmarkEncryptDecryptReader(b *testing.B) { - be := setupBackend(b) - defer teardownBackend(b, be) - k := setupKey(b, be, testPassword) - - size := 8 << 20 // 8MiB - rd := randomReader(23, size) - - b.ResetTimer() - b.SetBytes(int64(size)) - - buf := bytes.NewBuffer(nil) - for i := 0; i < b.N; i++ { - rd.Seek(0, 0) - buf.Reset() - wr := k.EncryptTo(buf) - _, err := io.Copy(wr, rd) - ok(b, err) - ok(b, wr.Close()) - - r, err := k.DecryptFrom(buf) - ok(b, err) - - _, err = io.Copy(ioutil.Discard, r) - ok(b, err) - } - - restic.PoolAlloc() -} - -func BenchmarkDecrypt(b *testing.B) { - size := 8 << 20 // 8MiB - data := make([]byte, size) - - s := setupBackend(b) - defer teardownBackend(b, s) - k := setupKey(b, s, testPassword) - - ciphertext := restic.GetChunkBuf("BenchmarkDecrypt") - defer restic.FreeChunkBuf("BenchmarkDecrypt", ciphertext) - plaintext := restic.GetChunkBuf("BenchmarkDecrypt") - defer restic.FreeChunkBuf("BenchmarkDecrypt", plaintext) - - n, err := k.Encrypt(ciphertext, data) - ok(b, err) - - b.ResetTimer() - b.SetBytes(int64(size)) - - for i := 0; i < b.N; i++ { - plaintext, err = k.Decrypt(plaintext, ciphertext[:n]) - ok(b, err) - } -} - -func TestEncryptStreamWriter(t *testing.T) { - s := setupBackend(t) - defer teardownBackend(t, s) - k := setupKey(t, s, testPassword) - - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { - tests = append(tests, 7<<20+123) - } - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(randomReader(42, size), data) - ok(t, err) - - ciphertext := bytes.NewBuffer(nil) - wr := k.EncryptTo(ciphertext) - - _, err = io.Copy(wr, bytes.NewReader(data)) - ok(t, err) - ok(t, wr.Close()) - - l := len(data) + restic.CiphertextExtension - assert(t, len(ciphertext.Bytes()) == l, - "wrong ciphertext length: expected %d, got %d", - l, len(ciphertext.Bytes())) - - // decrypt with default function - plaintext, err := k.Decrypt([]byte{}, ciphertext.Bytes()) - ok(t, err) - assert(t, bytes.Equal(data, plaintext), - "wrong plaintext after decryption: expected %02x, got %02x", - data, plaintext) - } -} - -func TestDecryptStreamReader(t *testing.T) { - s := setupBackend(t) - defer teardownBackend(t, s) - k := setupKey(t, s, testPassword) - - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { - tests = append(tests, 7<<20+123) - } - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(randomReader(42, size), data) - ok(t, err) - - ciphertext := make([]byte, size+restic.CiphertextExtension) - - // encrypt with default function - n, err := k.Encrypt(ciphertext, data) - ok(t, err) - assert(t, n == len(data)+restic.CiphertextExtension, - "wrong number of bytes returned after encryption: expected %d, got %d", - len(data)+restic.CiphertextExtension, n) - - rd, err := k.DecryptFrom(bytes.NewReader(ciphertext)) - ok(t, err) - - plaintext, err := ioutil.ReadAll(rd) - ok(t, err) - - assert(t, bytes.Equal(data, plaintext), - "wrong plaintext after decryption: expected %02x, got %02x", - data, plaintext) - } -} - -func TestEncryptWriter(t *testing.T) { - s := setupBackend(t) - defer teardownBackend(t, s) - k := setupKey(t, s, testPassword) - - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { - tests = append(tests, 7<<20+123) - } - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(randomReader(42, size), data) - ok(t, err) - - buf := bytes.NewBuffer(nil) - wr := k.EncryptTo(buf) - - _, err = io.Copy(wr, bytes.NewReader(data)) - ok(t, err) - ok(t, wr.Close()) - - ciphertext := buf.Bytes() - - l := len(data) + restic.CiphertextExtension - assert(t, len(ciphertext) == l, - "wrong ciphertext length: expected %d, got %d", - l, len(ciphertext)) - - // decrypt with default function - plaintext, err := k.Decrypt([]byte{}, ciphertext) - ok(t, err) - assert(t, bytes.Equal(data, plaintext), - "wrong plaintext after decryption: expected %02x, got %02x", - data, plaintext) - } -} diff --git a/pools.go b/pools.go index 1bf7a0cc8..cf8e899a9 100644 --- a/pools.go +++ b/pools.go @@ -80,7 +80,7 @@ func newChunkBuf() interface{} { defer chunkStats.m.Unlock() chunkStats.new++ - // create buffer for iv, data and hmac + // create buffer for iv, data and mac return make([]byte, maxCiphertextSize) } diff --git a/server.go b/server.go index 0e3220cd5..bf822ded7 100644 --- a/server.go +++ b/server.go @@ -149,11 +149,11 @@ func (s Server) Save(t backend.Type, data []byte, id backend.ID) (Blob, error) { var ciphertext []byte // if the data is small enough, use a slice from the pool - if len(data) <= maxCiphertextSize-ivSize-hmacSize { + if len(data) <= maxCiphertextSize-ivSize-macSize { ciphertext = GetChunkBuf("ch.Save()") defer FreeChunkBuf("ch.Save()", ciphertext) } else { - l := len(data) + ivSize + hmacSize + l := len(data) + ivSize + macSize debug.Log("Server.Save", "create large slice of %d bytes for ciphertext", l)