Crypto++  5.6.3
Free C++ class library of cryptographic schemes
gcm.cpp
1 // gcm.cpp - written and placed in the public domain by Wei Dai
2 
3 // use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM gcm.cpp" to generate MASM code
4 
5 #include "pch.h"
6 #include "config.h"
7 
8 #if CRYPTOPP_MSC_VERSION
9 # pragma warning(disable: 4189)
10 #endif
11 
12 #ifndef CRYPTOPP_IMPORTS
13 #ifndef CRYPTOPP_GENERATE_X64_MASM
14 
15 #include "gcm.h"
16 #include "cpu.h"
17 
18 NAMESPACE_BEGIN(CryptoPP)
19 
20 word16 GCM_Base::s_reductionTable[256];
21 volatile bool GCM_Base::s_reductionTableInitialized = false;
22 
23 void GCM_Base::GCTR::IncrementCounterBy256()
24 {
25  IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
26 }
27 
28 #if 0
29 // preserved for testing
30 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
31 {
32  word64 Z0=0, Z1=0, V0, V1;
33 
35  Block::Get(a)(V0)(V1);
36 
37  for (int i=0; i<16; i++)
38  {
39  for (int j=0x80; j!=0; j>>=1)
40  {
41  int x = b[i] & j;
42  Z0 ^= x ? V0 : 0;
43  Z1 ^= x ? V1 : 0;
44  x = (int)V1 & 1;
45  V1 = (V1>>1) | (V0<<63);
46  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
47  }
48  }
49  Block::Put(NULL, c)(Z0)(Z1);
50 }
51 
52 __m128i _mm_clmulepi64_si128(const __m128i &a, const __m128i &b, int i)
53 {
54  word64 A[1] = {ByteReverse(((word64*)&a)[i&1])};
55  word64 B[1] = {ByteReverse(((word64*)&b)[i>>4])};
56 
57  PolynomialMod2 pa((byte *)A, 8);
58  PolynomialMod2 pb((byte *)B, 8);
59  PolynomialMod2 c = pa*pb;
60 
61  __m128i output;
62  for (int i=0; i<16; i++)
63  ((byte *)&output)[i] = c.GetByte(i);
64  return output;
65 }
66 #endif
67 
68 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
69 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
70 {
71 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
72  *(__m128i *)a = _mm_xor_si128(*(__m128i *)b, *(__m128i *)c);
73 #else
74  asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
75 #endif
76 }
77 #endif
78 
79 inline static void Xor16(byte *a, const byte *b, const byte *c)
80 {
81  ((word64 *)a)[0] = ((word64 *)b)[0] ^ ((word64 *)c)[0];
82  ((word64 *)a)[1] = ((word64 *)b)[1] ^ ((word64 *)c)[1];
83 }
84 
85 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
86 static CRYPTOPP_ALIGN_DATA(16) const word64 s_clmulConstants64[] = {
87  W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
88  W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
89  W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)};
90 static const __m128i *s_clmulConstants = (const __m128i *)s_clmulConstants64;
91 static const unsigned int s_clmulTableSizeInBlocks = 8;
92 
93 inline __m128i CLMUL_Reduce(__m128i c0, __m128i c1, __m128i c2, const __m128i &r)
94 {
95  /*
96  The polynomial to be reduced is c0 * x^128 + c1 * x^64 + c2. c0t below refers to the most
97  significant half of c0 as a polynomial, which, due to GCM's bit reflection, are in the
98  rightmost bit positions, and the lowest byte addresses.
99 
100  c1 ^= c0t * 0xc200000000000000
101  c2t ^= c0t
102  t = shift (c1t ^ c0b) left 1 bit
103  c2 ^= t * 0xe100000000000000
104  c2t ^= c1b
105  shift c2 left 1 bit and xor in lowest bit of c1t
106  */
107 #if 0 // MSVC 2010 workaround: see http://connect.microsoft.com/VisualStudio/feedback/details/575301
108  c2 = _mm_xor_si128(c2, _mm_move_epi64(c0));
109 #else
110  c1 = _mm_xor_si128(c1, _mm_slli_si128(c0, 8));
111 #endif
112  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(c0, r, 0x10));
113  c0 = _mm_srli_si128(c0, 8);
114  c0 = _mm_xor_si128(c0, c1);
115  c0 = _mm_slli_epi64(c0, 1);
116  c0 = _mm_clmulepi64_si128(c0, r, 0);
117  c2 = _mm_xor_si128(c2, c0);
118  c2 = _mm_xor_si128(c2, _mm_srli_si128(c1, 8));
119  c1 = _mm_unpacklo_epi64(c1, c2);
120  c1 = _mm_srli_epi64(c1, 63);
121  c2 = _mm_slli_epi64(c2, 1);
122  return _mm_xor_si128(c2, c1);
123 }
124 
125 inline __m128i CLMUL_GF_Mul(const __m128i &x, const __m128i &h, const __m128i &r)
126 {
127  __m128i c0 = _mm_clmulepi64_si128(x,h,0);
128  __m128i c1 = _mm_xor_si128(_mm_clmulepi64_si128(x,h,1), _mm_clmulepi64_si128(x,h,0x10));
129  __m128i c2 = _mm_clmulepi64_si128(x,h,0x11);
130 
131  return CLMUL_Reduce(c0, c1, c2, r);
132 }
133 #endif
134 
135 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs &params)
136 {
137  BlockCipher &blockCipher = AccessBlockCipher();
138  blockCipher.SetKey(userKey, keylength, params);
139 
140  if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
141  throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
142 
143  int tableSize, i, j, k;
144 
145 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
146  if (HasCLMUL())
147  {
148  // Avoid "parameter not used" error and suppress Coverity finding
149  (void)params.GetIntValue(Name::TableSize(), tableSize);
150  tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
151  }
152  else
153 #endif
154  {
155  if (params.GetIntValue(Name::TableSize(), tableSize))
156  tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
157  else
158  tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
159 
160 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400)
161  // VC 2003 workaround: compiler generates bad code for 64K tables
162  tableSize = 2*1024;
163 #endif
164  }
165 
166  m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
167  byte *table = MulTable();
168  byte *hashKey = HashKey();
169  memset(hashKey, 0, REQUIRED_BLOCKSIZE);
170  blockCipher.ProcessBlock(hashKey);
171 
172 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
173  if (HasCLMUL())
174  {
175  const __m128i r = s_clmulConstants[0];
176  __m128i h0 = _mm_shuffle_epi8(_mm_load_si128((__m128i *)hashKey), s_clmulConstants[1]);
177  __m128i h = h0;
178 
179  for (i=0; i<tableSize; i+=32)
180  {
181  __m128i h1 = CLMUL_GF_Mul(h, h0, r);
182  _mm_storel_epi64((__m128i *)(table+i), h);
183  _mm_storeu_si128((__m128i *)(table+i+16), h1);
184  _mm_storeu_si128((__m128i *)(table+i+8), h);
185  _mm_storel_epi64((__m128i *)(table+i+8), h1);
186  h = CLMUL_GF_Mul(h1, h0, r);
187  }
188 
189  return;
190  }
191 #endif
192 
193  word64 V0, V1;
194  typedef BlockGetAndPut<word64, BigEndian> Block;
195  Block::Get(hashKey)(V0)(V1);
196 
197  if (tableSize == 64*1024)
198  {
199  for (i=0; i<128; i++)
200  {
201  k = i%8;
202  Block::Put(NULL, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
203 
204  int x = (int)V1 & 1;
205  V1 = (V1>>1) | (V0<<63);
206  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
207  }
208 
209  for (i=0; i<16; i++)
210  {
211  memset(table+i*256*16, 0, 16);
212 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
213  if (HasSSE2())
214  for (j=2; j<=0x80; j*=2)
215  for (k=1; k<j; k++)
216  SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
217  else
218 #endif
219  for (j=2; j<=0x80; j*=2)
220  for (k=1; k<j; k++)
221  Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
222  }
223  }
224  else
225  {
226  if (!s_reductionTableInitialized)
227  {
228  s_reductionTable[0] = 0;
229  word16 x = 0x01c2;
230  s_reductionTable[1] = ByteReverse(x);
231  for (unsigned int ii=2; ii<=0x80; ii*=2)
232  {
233  x <<= 1;
234  s_reductionTable[ii] = ByteReverse(x);
235  for (unsigned int jj=1; jj<ii; jj++)
236  s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
237  }
238  s_reductionTableInitialized = true;
239  }
240 
241  for (i=0; i<128-24; i++)
242  {
243  k = i%32;
244  if (k < 4)
245  Block::Put(NULL, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
246  else if (k < 8)
247  Block::Put(NULL, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
248 
249  int x = (int)V1 & 1;
250  V1 = (V1>>1) | (V0<<63);
251  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
252  }
253 
254  for (i=0; i<4; i++)
255  {
256  memset(table+i*256, 0, 16);
257  memset(table+1024+i*256, 0, 16);
258 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
259  if (HasSSE2())
260  for (j=2; j<=8; j*=2)
261  for (k=1; k<j; k++)
262  {
263  SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
264  SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
265  }
266  else
267 #endif
268  for (j=2; j<=8; j*=2)
269  for (k=1; k<j; k++)
270  {
271  Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
272  Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
273  }
274  }
275  }
276 }
277 
278 inline void GCM_Base::ReverseHashBufferIfNeeded()
279 {
280 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
281  if (HasCLMUL())
282  {
283  __m128i &x = *(__m128i *)HashBuffer();
284  x = _mm_shuffle_epi8(x, s_clmulConstants[1]);
285  }
286 #endif
287 }
288 
289 void GCM_Base::Resync(const byte *iv, size_t len)
290 {
291  BlockCipher &cipher = AccessBlockCipher();
292  byte *hashBuffer = HashBuffer();
293 
294  if (len == 12)
295  {
296  memcpy(hashBuffer, iv, len);
297  memset(hashBuffer+len, 0, 3);
298  hashBuffer[len+3] = 1;
299  }
300  else
301  {
302  size_t origLen = len;
303  memset(hashBuffer, 0, HASH_BLOCKSIZE);
304 
305  if (len >= HASH_BLOCKSIZE)
306  {
307  len = GCM_Base::AuthenticateBlocks(iv, len);
308  iv += (origLen - len);
309  }
310 
311  if (len > 0)
312  {
313  memcpy(m_buffer, iv, len);
314  memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
315  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
316  }
317 
318  PutBlock<word64, BigEndian, true>(NULL, m_buffer)(0)(origLen*8);
319  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
320 
321  ReverseHashBufferIfNeeded();
322  }
323 
324  if (m_state >= State_IVSet)
325  m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
326  else
327  m_ctr.SetCipherWithIV(cipher, hashBuffer);
328 
329  m_ctr.Seek(HASH_BLOCKSIZE);
330 
331  memset(hashBuffer, 0, HASH_BLOCKSIZE);
332 }
333 
334 unsigned int GCM_Base::OptimalDataAlignment() const
335 {
336  return
337 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
338  HasSSE2() ? 16 :
339 #endif
340  GetBlockCipher().OptimalDataAlignment();
341 }
342 
343 #if CRYPTOPP_MSC_VERSION
344 # pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
345 #endif
346 
347 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
348 
349 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
350 extern "C" {
351 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
352 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
353 }
354 #endif
355 
356 #ifndef CRYPTOPP_GENERATE_X64_MASM
357 
358 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
359 {
360 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
361  if (HasCLMUL())
362  {
363  const __m128i *table = (const __m128i *)MulTable();
364  __m128i x = _mm_load_si128((__m128i *)HashBuffer());
365  const __m128i r = s_clmulConstants[0], bswapMask = s_clmulConstants[1], bswapMask2 = s_clmulConstants[2];
366 
367  while (len >= 16)
368  {
369  size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
370  __m128i d, d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data+(s-1)*16)), bswapMask2);;
371  __m128i c0 = _mm_setzero_si128();
372  __m128i c1 = _mm_setzero_si128();
373  __m128i c2 = _mm_setzero_si128();
374 
375  while (true)
376  {
377  __m128i h0 = _mm_load_si128(table+i);
378  __m128i h1 = _mm_load_si128(table+i+1);
379  __m128i h01 = _mm_xor_si128(h0, h1);
380 
381  if (++i == s)
382  {
383  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)data), bswapMask);
384  d = _mm_xor_si128(d, x);
385  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0));
386  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
387  d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
388  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0));
389  break;
390  }
391 
392  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data+(s-i)*16-8)), bswapMask2);
393  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d2, h0, 1));
394  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
395  d2 = _mm_xor_si128(d2, d);
396  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d2, h01, 1));
397 
398  if (++i == s)
399  {
400  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)data), bswapMask);
401  d = _mm_xor_si128(d, x);
402  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
403  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 0x11));
404  d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
405  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0x10));
406  break;
407  }
408 
409  d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(data+(s-i)*16-8)), bswapMask);
410  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
411  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d2, h1, 0x10));
412  d = _mm_xor_si128(d, d2);
413  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h01, 0x10));
414  }
415  data += s*16;
416  len -= s*16;
417 
418  c1 = _mm_xor_si128(_mm_xor_si128(c1, c0), c2);
419  x = CLMUL_Reduce(c0, c1, c2, r);
420  }
421 
422  _mm_store_si128((__m128i *)HashBuffer(), x);
423  return len;
424  }
425 #endif
426 
428  word64 *hashBuffer = (word64 *)HashBuffer();
429 
430  switch (2*(m_buffer.size()>=64*1024)
431 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
432  + HasSSE2()
433 #endif
434  )
435  {
436  case 0: // non-SSE2 and 2K tables
437  {
438  byte *table = MulTable();
439  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
440 
441  do
442  {
443  word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
444  Block::Get(data)(y0)(y1);
445  x0 ^= y0;
446  x1 ^= y1;
447 
448  data += HASH_BLOCKSIZE;
449  len -= HASH_BLOCKSIZE;
450 
451  #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(table+(a*1024)+(b*256)+c+d*8)
452 
453  #ifdef IS_LITTLE_ENDIAN
454  #if CRYPTOPP_BOOL_SLOW_WORD64
455  word32 z0 = (word32)x0;
456  word32 z1 = (word32)(x0>>32);
457  word32 z2 = (word32)x1;
458  word32 z3 = (word32)(x1>>32);
459  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
460  #else
461  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
462  #endif
463  #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
464  #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
465  #else
466  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
467  #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
468  #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
469  #endif
470 
471  #define GF_MUL_32BY128(op, a, b, c) \
472  a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\
473  a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\
474  b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\
475  b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\
476  c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\
477  c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\
478  d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\
479  d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\
480 
481  GF_MUL_32BY128(=, 0, 0, 0)
482  GF_MUL_32BY128(^=, 0, 1, 1)
483  GF_MUL_32BY128(^=, 1, 0, 2)
484  GF_MUL_32BY128(^=, 1, 1, 3)
485 
486  word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
487  GF_SHIFT_8(d)
488  c0 ^= d0; c1 ^= d1;
489  r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
490  GF_SHIFT_8(c)
491  b0 ^= c0; b1 ^= c1;
492  r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
493  GF_SHIFT_8(b)
494  a0 ^= b0; a1 ^= b1;
495  a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
496  x0 = a0; x1 = a1;
497  }
498  while (len >= HASH_BLOCKSIZE);
499 
500  hashBuffer[0] = x0; hashBuffer[1] = x1;
501  return len;
502  }
503 
504  case 2: // non-SSE2 and 64K tables
505  {
506  byte *table = MulTable();
507  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
508 
509  do
510  {
511  word64 y0, y1, a0, a1;
512  Block::Get(data)(y0)(y1);
513  x0 ^= y0;
514  x1 ^= y1;
515 
516  data += HASH_BLOCKSIZE;
517  len -= HASH_BLOCKSIZE;
518 
519  #undef READ_TABLE_WORD64_COMMON
520  #undef READ_TABLE_WORD64
521 
522  #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(table+(a)*256*16+(c)+(d)*8)
523 
524  #ifdef IS_LITTLE_ENDIAN
525  #if CRYPTOPP_BOOL_SLOW_WORD64
526  word32 z0 = (word32)x0;
527  word32 z1 = (word32)(x0>>32);
528  word32 z2 = (word32)x1;
529  word32 z3 = (word32)(x1>>32);
530  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
531  #else
532  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
533  #endif
534  #else
535  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
536  #endif
537 
538  #define GF_MUL_8BY128(op, b, c, d) \
539  a0 op READ_TABLE_WORD64(b, c, d, 0);\
540  a1 op READ_TABLE_WORD64(b, c, d, 1);\
541 
542  GF_MUL_8BY128(=, 0, 0, 0)
543  GF_MUL_8BY128(^=, 0, 0, 1)
544  GF_MUL_8BY128(^=, 0, 0, 2)
545  GF_MUL_8BY128(^=, 0, 0, 3)
546  GF_MUL_8BY128(^=, 0, 1, 0)
547  GF_MUL_8BY128(^=, 0, 1, 1)
548  GF_MUL_8BY128(^=, 0, 1, 2)
549  GF_MUL_8BY128(^=, 0, 1, 3)
550  GF_MUL_8BY128(^=, 1, 2, 0)
551  GF_MUL_8BY128(^=, 1, 2, 1)
552  GF_MUL_8BY128(^=, 1, 2, 2)
553  GF_MUL_8BY128(^=, 1, 2, 3)
554  GF_MUL_8BY128(^=, 1, 3, 0)
555  GF_MUL_8BY128(^=, 1, 3, 1)
556  GF_MUL_8BY128(^=, 1, 3, 2)
557  GF_MUL_8BY128(^=, 1, 3, 3)
558 
559  x0 = a0; x1 = a1;
560  }
561  while (len >= HASH_BLOCKSIZE);
562 
563  hashBuffer[0] = x0; hashBuffer[1] = x1;
564  return len;
565  }
566 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
567 
568 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
569  case 1: // SSE2 and 2K tables
570  GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
571  return len % 16;
572  case 3: // SSE2 and 64K tables
573  GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
574  return len % 16;
575 #endif
576 
577 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
578  case 1: // SSE2 and 2K tables
579  {
580  #ifdef __GNUC__
581  __asm__ __volatile__
582  (
583  INTEL_NOPREFIX
584  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
585  ALIGN 8
586  GCM_AuthenticateBlocks_2K PROC FRAME
587  rex_push_reg rsi
588  push_reg rdi
589  push_reg rbx
590  .endprolog
591  mov rsi, r8
592  mov r11, r9
593  #else
594  AS2( mov WORD_REG(cx), data )
595  AS2( mov WORD_REG(dx), len )
596  AS2( mov WORD_REG(si), hashBuffer )
597  AS2( shr WORD_REG(dx), 4 )
598  #endif
599 
600  #if CRYPTOPP_BOOL_X32
601  AS1(push rbx)
602  AS1(push rbp)
603  #else
604  AS_PUSH_IF86( bx)
605  AS_PUSH_IF86( bp)
606  #endif
607 
608  #ifdef __GNUC__
609  AS2( mov AS_REG_7, WORD_REG(di))
610  #elif CRYPTOPP_BOOL_X86
611  AS2( lea AS_REG_7, s_reductionTable)
612  #endif
613 
614  AS2( movdqa xmm0, [WORD_REG(si)] )
615 
616  #define MUL_TABLE_0 WORD_REG(si) + 32
617  #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
618  #define RED_TABLE AS_REG_7
619 
620  ASL(0)
621  AS2( movdqu xmm4, [WORD_REG(cx)] )
622  AS2( pxor xmm0, xmm4 )
623 
624  AS2( movd ebx, xmm0 )
625  AS2( mov eax, AS_HEX(f0f0f0f0) )
626  AS2( and eax, ebx )
627  AS2( shl ebx, 4 )
628  AS2( and ebx, AS_HEX(f0f0f0f0) )
629  AS2( movzx edi, ah )
630  AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
631  AS2( movzx edi, al )
632  AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
633  AS2( shr eax, 16 )
634  AS2( movzx edi, ah )
635  AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
636  AS2( movzx edi, al )
637  AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
638 
639  #define SSE2_MUL_32BITS(i) \
640  AS2( psrldq xmm0, 4 )\
641  AS2( movd eax, xmm0 )\
642  AS2( and eax, AS_HEX(f0f0f0f0) )\
643  AS2( movzx edi, bh )\
644  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
645  AS2( movzx edi, bl )\
646  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
647  AS2( shr ebx, 16 )\
648  AS2( movzx edi, bh )\
649  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
650  AS2( movzx edi, bl )\
651  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
652  AS2( movd ebx, xmm0 )\
653  AS2( shl ebx, 4 )\
654  AS2( and ebx, AS_HEX(f0f0f0f0) )\
655  AS2( movzx edi, ah )\
656  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
657  AS2( movzx edi, al )\
658  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
659  AS2( shr eax, 16 )\
660  AS2( movzx edi, ah )\
661  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
662  AS2( movzx edi, al )\
663  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
664 
665  SSE2_MUL_32BITS(1)
666  SSE2_MUL_32BITS(2)
667  SSE2_MUL_32BITS(3)
668 
669  AS2( movzx edi, bh )
670  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
671  AS2( movzx edi, bl )
672  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
673  AS2( shr ebx, 16 )
674  AS2( movzx edi, bh )
675  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
676  AS2( movzx edi, bl )
677  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
678 
679  AS2( movdqa xmm0, xmm3 )
680  AS2( pslldq xmm3, 1 )
681  AS2( pxor xmm2, xmm3 )
682  AS2( movdqa xmm1, xmm2 )
683  AS2( pslldq xmm2, 1 )
684  AS2( pxor xmm5, xmm2 )
685 
686  AS2( psrldq xmm0, 15 )
687 #if (CRYPTOPP_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
688  AS2( movd edi, xmm0 )
689 #elif defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
690  AS2( mov WORD_REG(di), xmm0 )
691 #else // GNU Assembler
692  AS2( movd WORD_REG(di), xmm0 )
693 #endif
694  AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
695  AS2( shl eax, 8 )
696 
697  AS2( movdqa xmm0, xmm5 )
698  AS2( pslldq xmm5, 1 )
699  AS2( pxor xmm4, xmm5 )
700 
701  AS2( psrldq xmm1, 15 )
702 #if (CRYPTOPP_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
703  AS2( movd edi, xmm1 )
704 #elif defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
705  AS2( mov WORD_REG(di), xmm1 )
706 #else
707  AS2( movd WORD_REG(di), xmm1 )
708 #endif
709  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
710  AS2( shl eax, 8 )
711 
712  AS2( psrldq xmm0, 15 )
713 #if (CRYPTOPP_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
714  AS2( movd edi, xmm0 )
715 #elif defined(CRYPTOPP_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)
716  AS2( mov WORD_REG(di), xmm0 )
717 #else
718  AS2( movd WORD_REG(di), xmm0 )
719 #endif
720  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
721 
722  AS2( movd xmm0, eax )
723  AS2( pxor xmm0, xmm4 )
724 
725  AS2( add WORD_REG(cx), 16 )
726  AS2( sub WORD_REG(dx), 1 )
727  ASJ( jnz, 0, b )
728  AS2( movdqa [WORD_REG(si)], xmm0 )
729 
730  #if CRYPTOPP_BOOL_X32
731  AS1(pop rbp)
732  AS1(pop rbx)
733  #else
734  AS_POP_IF86( bp)
735  AS_POP_IF86( bx)
736  #endif
737 
738  #ifdef __GNUC__
739  ATT_PREFIX
740  :
741  : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
742  : "memory", "cc", "%eax"
743  #if CRYPTOPP_BOOL_X64
744  , "%ebx", "%r11"
745  #endif
746  );
747  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
748  pop rbx
749  pop rdi
750  pop rsi
751  ret
752  GCM_AuthenticateBlocks_2K ENDP
753  #endif
754 
755  return len%16;
756  }
757  case 3: // SSE2 and 64K tables
758  {
759  #ifdef __GNUC__
760  __asm__ __volatile__
761  (
762  INTEL_NOPREFIX
763  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
764  ALIGN 8
765  GCM_AuthenticateBlocks_64K PROC FRAME
766  rex_push_reg rsi
767  push_reg rdi
768  .endprolog
769  mov rsi, r8
770  #else
771  AS2( mov WORD_REG(cx), data )
772  AS2( mov WORD_REG(dx), len )
773  AS2( mov WORD_REG(si), hashBuffer )
774  AS2( shr WORD_REG(dx), 4 )
775  #endif
776 
777  AS2( movdqa xmm0, [WORD_REG(si)] )
778 
779  #undef MUL_TABLE
780  #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
781 
782  ASL(1)
783  AS2( movdqu xmm1, [WORD_REG(cx)] )
784  AS2( pxor xmm1, xmm0 )
785  AS2( pxor xmm0, xmm0 )
786 
787  #undef SSE2_MUL_32BITS
788  #define SSE2_MUL_32BITS(i) \
789  AS2( movd eax, xmm1 )\
790  AS2( psrldq xmm1, 4 )\
791  AS2( movzx edi, al )\
792  AS2( add WORD_REG(di), WORD_REG(di) )\
793  AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
794  AS2( movzx edi, ah )\
795  AS2( add WORD_REG(di), WORD_REG(di) )\
796  AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
797  AS2( shr eax, 16 )\
798  AS2( movzx edi, al )\
799  AS2( add WORD_REG(di), WORD_REG(di) )\
800  AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
801  AS2( movzx edi, ah )\
802  AS2( add WORD_REG(di), WORD_REG(di) )\
803  AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
804 
805  SSE2_MUL_32BITS(0)
806  SSE2_MUL_32BITS(1)
807  SSE2_MUL_32BITS(2)
808  SSE2_MUL_32BITS(3)
809 
810  AS2( add WORD_REG(cx), 16 )
811  AS2( sub WORD_REG(dx), 1 )
812  ASJ( jnz, 1, b )
813  AS2( movdqa [WORD_REG(si)], xmm0 )
814 
815  #ifdef __GNUC__
816  ATT_PREFIX
817  :
818  : "c" (data), "d" (len/16), "S" (hashBuffer)
819  : "memory", "cc", "%edi", "%eax"
820  );
821  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
822  pop rdi
823  pop rsi
824  ret
825  GCM_AuthenticateBlocks_64K ENDP
826  #endif
827 
828  return len%16;
829  }
830 #endif
831 #ifndef CRYPTOPP_GENERATE_X64_MASM
832  }
833 
834  return len%16;
835 }
836 
837 void GCM_Base::AuthenticateLastHeaderBlock()
838 {
839  if (m_bufferedDataLength > 0)
840  {
841  memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
842  m_bufferedDataLength = 0;
843  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
844  }
845 }
846 
847 void GCM_Base::AuthenticateLastConfidentialBlock()
848 {
849  GCM_Base::AuthenticateLastHeaderBlock();
850  PutBlock<word64, BigEndian, true>(NULL, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
851  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
852 }
853 
854 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
855 {
856  m_ctr.Seek(0);
857  ReverseHashBufferIfNeeded();
858  m_ctr.ProcessData(mac, HashBuffer(), macSize);
859 }
860 
861 NAMESPACE_END
862 
863 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
864 #endif
CCM block cipher mode of operation.
Definition: gcm.h:22
An invalid argument was detected.
Definition: cryptlib.h:166
virtual void SetKey(const byte *key, size_t length, const NameValuePairs &params=g_nullNameValuePairs)
Sets or reset the key of this object.
Definition: cryptlib.cpp:100
void IncrementCounterByOne(byte *inout, unsigned int size)
Performs an addition with carry on a block of bytes.
Definition: misc.h:905
virtual unsigned int BlockSize() const =0
Provides the block size of the cipher.
Library configuration file.
byte GetByte(size_t n) const
return the n-th byte
Definition: gf2n.cpp:77
bool GetIntValue(const char *name, int &value) const
Get a named value with type int.
Definition: cryptlib.h:355
Polynomial with Coefficients in GF(2)
Definition: gf2n.h:18
Interface for one direction (encryption or decryption) of a block cipher.
Definition: cryptlib.h:1013
const char * TableSize()
int, in bytes
Definition: argnames.h:80
unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: gcm.cpp:334
void ProcessBlock(const byte *inBlock, byte *outBlock) const
Encrypt or decrypt a block.
Definition: cryptlib.h:685
const T1 UnsignedMin(const T1 &a, const T2 &b)
Safe comparison of values that could be neagtive and incorrectly promoted.
Definition: misc.h:422
const char * BlockSize()
int, in bytes
Definition: argnames.h:26
Classes, functions, intrinsics and features for X86, X32 nd X64 assembly.
virtual unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: cryptlib.cpp:232
std::string AlgorithmName() const
Provides the name of this algorithm.
Definition: gcm.h:26
GCM block cipher mode of operation.
Crypto++ library namespace.
byte ByteReverse(byte value)
Reverses bytes in a 8-bit value.
Definition: misc.h:1551
Interface for retrieving values given their names.
Definition: cryptlib.h:261