Linux/MacOSX crypto: report XTS optimization implemented previously on Windows
c
This commit is contained in:
parent
d8d92357b0
commit
ce0a34941a
|
@ -10,9 +10,41 @@
|
|||
code distribution packages.
|
||||
*/
|
||||
|
||||
#include "Crypto/cpu.h"
|
||||
#include "Crypto/misc.h"
|
||||
#include "EncryptionModeXTS.h"
|
||||
#include "Common/Crypto.h"
|
||||
|
||||
#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64)
|
||||
|
||||
#define XorBlocks(result,ptr,len,start,end) \
|
||||
while (len >= 2) \
|
||||
{ \
|
||||
__m128i xmm1 = _mm_loadu_si128((const __m128i*) ptr); \
|
||||
__m128i xmm2 = _mm_loadu_si128((__m128i*)result); \
|
||||
__m128i xmm3 = _mm_loadu_si128((const __m128i*) (ptr + 2)); \
|
||||
__m128i xmm4 = _mm_loadu_si128((__m128i*)(result + 2)); \
|
||||
\
|
||||
_mm_storeu_si128((__m128i*)result, _mm_xor_si128(xmm1, xmm2)); \
|
||||
_mm_storeu_si128((__m128i*)(result + 2), _mm_xor_si128(xmm3, xmm4)); \
|
||||
ptr+= 4; \
|
||||
result+= 4; \
|
||||
len -= 2; \
|
||||
} \
|
||||
\
|
||||
if (len) \
|
||||
{ \
|
||||
__m128i xmm1 = _mm_loadu_si128((const __m128i*)ptr); \
|
||||
__m128i xmm2 = _mm_loadu_si128((__m128i*)result); \
|
||||
\
|
||||
_mm_storeu_si128((__m128i*)result, _mm_xor_si128(xmm1, xmm2)); \
|
||||
ptr+= 2; \
|
||||
result+= 2; \
|
||||
} \
|
||||
len = end - start;
|
||||
|
||||
#endif
|
||||
|
||||
namespace VeraCrypt
|
||||
{
|
||||
void EncryptionModeXTS::Encrypt (byte *data, uint64 length) const
|
||||
|
@ -45,9 +77,8 @@ namespace VeraCrypt
|
|||
uint64 *whiteningValuePtr64 = (uint64 *) whiteningValue;
|
||||
uint64 *bufPtr = (uint64 *) buffer;
|
||||
uint64 *dataUnitBufPtr;
|
||||
unsigned int startBlock = startCipherBlockNo, endBlock, block;
|
||||
uint64 *const finalInt64WhiteningValuesPtr = whiteningValuesPtr64 + sizeof (whiteningValues) / sizeof (*whiteningValuesPtr64) - 1;
|
||||
uint64 blockCount, dataUnitNo;
|
||||
unsigned int startBlock = startCipherBlockNo, endBlock, block, countBlock;
|
||||
uint64 remainingBlocks, dataUnitNo;
|
||||
|
||||
startDataUnitNo += SectorOffset;
|
||||
|
||||
|
@ -67,17 +98,18 @@ namespace VeraCrypt
|
|||
if (length % BYTES_PER_XTS_BLOCK)
|
||||
TC_THROW_FATAL_EXCEPTION;
|
||||
|
||||
blockCount = length / BYTES_PER_XTS_BLOCK;
|
||||
remainingBlocks = length / BYTES_PER_XTS_BLOCK;
|
||||
|
||||
// Process all blocks in the buffer
|
||||
while (blockCount > 0)
|
||||
while (remainingBlocks > 0)
|
||||
{
|
||||
if (blockCount < BLOCKS_PER_XTS_DATA_UNIT)
|
||||
endBlock = startBlock + (unsigned int) blockCount;
|
||||
if (remainingBlocks < BLOCKS_PER_XTS_DATA_UNIT)
|
||||
endBlock = startBlock + (unsigned int) remainingBlocks;
|
||||
else
|
||||
endBlock = BLOCKS_PER_XTS_DATA_UNIT;
|
||||
countBlock = endBlock - startBlock;
|
||||
|
||||
whiteningValuesPtr64 = finalInt64WhiteningValuesPtr;
|
||||
whiteningValuesPtr64 = (uint64 *) whiteningValues;
|
||||
whiteningValuePtr64 = (uint64 *) whiteningValue;
|
||||
|
||||
// Encrypt the data unit number using the secondary key (in order to generate the first
|
||||
|
@ -87,13 +119,13 @@ namespace VeraCrypt
|
|||
secondaryCipher.EncryptBlock (whiteningValue);
|
||||
|
||||
// Generate subsequent whitening values for blocks in this data unit. Note that all generated 128-bit
|
||||
// whitening values are stored in memory as a sequence of 64-bit integers in reverse order.
|
||||
// whitening values are stored in memory as a sequence of 64-bit integers.
|
||||
for (block = 0; block < endBlock; block++)
|
||||
{
|
||||
if (block >= startBlock)
|
||||
{
|
||||
*whiteningValuesPtr64-- = *whiteningValuePtr64++;
|
||||
*whiteningValuesPtr64-- = *whiteningValuePtr64;
|
||||
*whiteningValuesPtr64++ = *whiteningValuePtr64++;
|
||||
*whiteningValuesPtr64++ = *whiteningValuePtr64;
|
||||
}
|
||||
else
|
||||
whiteningValuePtr64++;
|
||||
|
@ -136,31 +168,36 @@ namespace VeraCrypt
|
|||
}
|
||||
|
||||
dataUnitBufPtr = bufPtr;
|
||||
whiteningValuesPtr64 = finalInt64WhiteningValuesPtr;
|
||||
whiteningValuesPtr64 = (uint64 *) whiteningValues;
|
||||
|
||||
// Encrypt all blocks in this data unit
|
||||
|
||||
for (block = startBlock; block < endBlock; block++)
|
||||
#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64)
|
||||
XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock);
|
||||
#else
|
||||
for (block = 0; block < countBlock; block++)
|
||||
{
|
||||
// Pre-whitening
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
}
|
||||
|
||||
#endif
|
||||
// Actual encryption
|
||||
cipher.EncryptBlocks ((byte *) dataUnitBufPtr, endBlock - startBlock);
|
||||
cipher.EncryptBlocks ((byte *) dataUnitBufPtr, countBlock);
|
||||
|
||||
bufPtr = dataUnitBufPtr;
|
||||
whiteningValuesPtr64 = finalInt64WhiteningValuesPtr;
|
||||
whiteningValuesPtr64 = (uint64 *) whiteningValues;
|
||||
|
||||
for (block = startBlock; block < endBlock; block++)
|
||||
#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64)
|
||||
XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock);
|
||||
#else
|
||||
for (block = 0; block < countBlock; block++)
|
||||
{
|
||||
// Post-whitening
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
}
|
||||
|
||||
blockCount -= endBlock - startBlock;
|
||||
#endif
|
||||
remainingBlocks -= countBlock;
|
||||
startBlock = 0;
|
||||
dataUnitNo++;
|
||||
*((uint64 *) byteBufUnitNo) = Endian::Little (dataUnitNo);
|
||||
|
@ -219,9 +256,8 @@ namespace VeraCrypt
|
|||
uint64 *whiteningValuePtr64 = (uint64 *) whiteningValue;
|
||||
uint64 *bufPtr = (uint64 *) buffer;
|
||||
uint64 *dataUnitBufPtr;
|
||||
unsigned int startBlock = startCipherBlockNo, endBlock, block;
|
||||
uint64 *const finalInt64WhiteningValuesPtr = whiteningValuesPtr64 + sizeof (whiteningValues) / sizeof (*whiteningValuesPtr64) - 1;
|
||||
uint64 blockCount, dataUnitNo;
|
||||
unsigned int startBlock = startCipherBlockNo, endBlock, block, countBlock;
|
||||
uint64 remainingBlocks, dataUnitNo;
|
||||
|
||||
startDataUnitNo += SectorOffset;
|
||||
|
||||
|
@ -234,17 +270,18 @@ namespace VeraCrypt
|
|||
if (length % BYTES_PER_XTS_BLOCK)
|
||||
TC_THROW_FATAL_EXCEPTION;
|
||||
|
||||
blockCount = length / BYTES_PER_XTS_BLOCK;
|
||||
remainingBlocks = length / BYTES_PER_XTS_BLOCK;
|
||||
|
||||
// Process all blocks in the buffer
|
||||
while (blockCount > 0)
|
||||
while (remainingBlocks > 0)
|
||||
{
|
||||
if (blockCount < BLOCKS_PER_XTS_DATA_UNIT)
|
||||
endBlock = startBlock + (unsigned int) blockCount;
|
||||
if (remainingBlocks < BLOCKS_PER_XTS_DATA_UNIT)
|
||||
endBlock = startBlock + (unsigned int) remainingBlocks;
|
||||
else
|
||||
endBlock = BLOCKS_PER_XTS_DATA_UNIT;
|
||||
countBlock = endBlock - startBlock;
|
||||
|
||||
whiteningValuesPtr64 = finalInt64WhiteningValuesPtr;
|
||||
whiteningValuesPtr64 = (uint64 *) whiteningValues;
|
||||
whiteningValuePtr64 = (uint64 *) whiteningValue;
|
||||
|
||||
// Encrypt the data unit number using the secondary key (in order to generate the first
|
||||
|
@ -254,13 +291,13 @@ namespace VeraCrypt
|
|||
secondaryCipher.EncryptBlock (whiteningValue);
|
||||
|
||||
// Generate subsequent whitening values for blocks in this data unit. Note that all generated 128-bit
|
||||
// whitening values are stored in memory as a sequence of 64-bit integers in reverse order.
|
||||
// whitening values are stored in memory as a sequence of 64-bit integers.
|
||||
for (block = 0; block < endBlock; block++)
|
||||
{
|
||||
if (block >= startBlock)
|
||||
{
|
||||
*whiteningValuesPtr64-- = *whiteningValuePtr64++;
|
||||
*whiteningValuesPtr64-- = *whiteningValuePtr64;
|
||||
*whiteningValuesPtr64++ = *whiteningValuePtr64++;
|
||||
*whiteningValuesPtr64++ = *whiteningValuePtr64;
|
||||
}
|
||||
else
|
||||
whiteningValuePtr64++;
|
||||
|
@ -303,28 +340,32 @@ namespace VeraCrypt
|
|||
}
|
||||
|
||||
dataUnitBufPtr = bufPtr;
|
||||
whiteningValuesPtr64 = finalInt64WhiteningValuesPtr;
|
||||
whiteningValuesPtr64 = (uint64 *) whiteningValues;
|
||||
|
||||
// Decrypt blocks in this data unit
|
||||
|
||||
for (block = startBlock; block < endBlock; block++)
|
||||
#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64)
|
||||
XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock);
|
||||
#else
|
||||
for (block = 0; block < countBlock; block++)
|
||||
{
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
}
|
||||
|
||||
cipher.DecryptBlocks ((byte *) dataUnitBufPtr, endBlock - startBlock);
|
||||
#endif
|
||||
cipher.DecryptBlocks ((byte *) dataUnitBufPtr, countBlock);
|
||||
|
||||
bufPtr = dataUnitBufPtr;
|
||||
whiteningValuesPtr64 = finalInt64WhiteningValuesPtr;
|
||||
|
||||
for (block = startBlock; block < endBlock; block++)
|
||||
whiteningValuesPtr64 = (uint64 *) whiteningValues;
|
||||
#if (CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && CRYPTOPP_BOOL_X64)
|
||||
XorBlocks (bufPtr, whiteningValuesPtr64, countBlock, startBlock, endBlock);
|
||||
#else
|
||||
for (block = 0; block < countBlock; block++)
|
||||
{
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64--;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
*bufPtr++ ^= *whiteningValuesPtr64++;
|
||||
}
|
||||
|
||||
blockCount -= endBlock - startBlock;
|
||||
#endif
|
||||
remainingBlocks -= countBlock;
|
||||
startBlock = 0;
|
||||
dataUnitNo++;
|
||||
|
||||
|
|
Loading…
Reference in New Issue