Messages in this thread Patch in this message | | | From | Denis Vlasenko <> | Subject | [PATCH 2/4] whirlpool gcc bug fix | Date | Sun, 3 Jul 2005 16:05:23 +0300 |
| |
wp was not usable: stack overflow with certain gcc versions. -- vda diff -urpN linux-2.6.12.1.n/crypto/wp512.c linux-2.6.12.2.n/crypto/wp512.c --- linux-2.6.12.1.n/crypto/wp512.c Sun Jul 3 15:52:55 2005 +++ linux-2.6.12.2.n/crypto/wp512.c Sun Jul 3 15:52:59 2005 @@ -793,80 +793,84 @@ static void wp512_process_buffer(struct state[6] = block[6] ^ (K[6] = wctx->hash[6]); state[7] = block[7] ^ (K[7] = wctx->hash[7]); +/* we do not use L[0] = C0[...] ^ C1[...] ^ ... ^ rc[r] +** due to gcc -O2 (3.4.3) optimizer bug: +** this will cause excessive spills (~3K stack used) +** See http://gcc.gnu.org/bugzilla/show_bug.cgi?id=21141 */ for (r = 1; r <= WHIRLPOOL_ROUNDS; r++) { - L[0] = C0[(int)(K[0] >> 56) ] ^ - C1[(int)(K[7] >> 48) & 0xff] ^ - C2[(int)(K[6] >> 40) & 0xff] ^ - C3[(int)(K[5] >> 32) & 0xff] ^ - C4[(int)(K[4] >> 24) & 0xff] ^ - C5[(int)(K[3] >> 16) & 0xff] ^ - C6[(int)(K[2] >> 8) & 0xff] ^ - C7[(int)(K[1] ) & 0xff] ^ - rc[r]; - - L[1] = C0[(int)(K[1] >> 56) ] ^ - C1[(int)(K[0] >> 48) & 0xff] ^ - C2[(int)(K[7] >> 40) & 0xff] ^ - C3[(int)(K[6] >> 32) & 0xff] ^ - C4[(int)(K[5] >> 24) & 0xff] ^ - C5[(int)(K[4] >> 16) & 0xff] ^ - C6[(int)(K[3] >> 8) & 0xff] ^ - C7[(int)(K[2] ) & 0xff]; - - L[2] = C0[(int)(K[2] >> 56) ] ^ - C1[(int)(K[1] >> 48) & 0xff] ^ - C2[(int)(K[0] >> 40) & 0xff] ^ - C3[(int)(K[7] >> 32) & 0xff] ^ - C4[(int)(K[6] >> 24) & 0xff] ^ - C5[(int)(K[5] >> 16) & 0xff] ^ - C6[(int)(K[4] >> 8) & 0xff] ^ - C7[(int)(K[3] ) & 0xff]; - - L[3] = C0[(int)(K[3] >> 56) ] ^ - C1[(int)(K[2] >> 48) & 0xff] ^ - C2[(int)(K[1] >> 40) & 0xff] ^ - C3[(int)(K[0] >> 32) & 0xff] ^ - C4[(int)(K[7] >> 24) & 0xff] ^ - C5[(int)(K[6] >> 16) & 0xff] ^ - C6[(int)(K[5] >> 8) & 0xff] ^ - C7[(int)(K[4] ) & 0xff]; - - L[4] = C0[(int)(K[4] >> 56) ] ^ - C1[(int)(K[3] >> 48) & 0xff] ^ - C2[(int)(K[2] >> 40) & 0xff] ^ - C3[(int)(K[1] >> 32) & 0xff] ^ - C4[(int)(K[0] >> 24) & 0xff] ^ - C5[(int)(K[7] >> 16) & 0xff] ^ - C6[(int)(K[6] >> 8) & 0xff] ^ - C7[(int)(K[5] ) & 0xff]; - - L[5] = C0[(int)(K[5] >> 56) ] ^ - C1[(int)(K[4] >> 48) & 0xff] ^ - C2[(int)(K[3] >> 40) & 0xff] ^ - C3[(int)(K[2] >> 32) & 0xff] ^ - C4[(int)(K[1] >> 24) & 0xff] ^ - C5[(int)(K[0] >> 16) & 0xff] ^ - C6[(int)(K[7] >> 8) & 0xff] ^ - C7[(int)(K[6] ) & 0xff]; - - L[6] = C0[(int)(K[6] >> 56) ] ^ - C1[(int)(K[5] >> 48) & 0xff] ^ - C2[(int)(K[4] >> 40) & 0xff] ^ - C3[(int)(K[3] >> 32) & 0xff] ^ - C4[(int)(K[2] >> 24) & 0xff] ^ - C5[(int)(K[1] >> 16) & 0xff] ^ - C6[(int)(K[0] >> 8) & 0xff] ^ - C7[(int)(K[7] ) & 0xff]; - - L[7] = C0[(int)(K[7] >> 56) ] ^ - C1[(int)(K[6] >> 48) & 0xff] ^ - C2[(int)(K[5] >> 40) & 0xff] ^ - C3[(int)(K[4] >> 32) & 0xff] ^ - C4[(int)(K[3] >> 24) & 0xff] ^ - C5[(int)(K[2] >> 16) & 0xff] ^ - C6[(int)(K[1] >> 8) & 0xff] ^ - C7[(int)(K[0] ) & 0xff]; + L[0] = C0[BYTE7(K[0])]; + L[0] ^= C1[BYTE6(K[7])]; + L[0] ^= C2[BYTE5(K[6])]; + L[0] ^= C3[BYTE4(K[5])]; + L[0] ^= C4[BYTE3(K[4])]; + L[0] ^= C5[BYTE2(K[3])]; + L[0] ^= C6[BYTE1(K[2])]; + L[0] ^= C7[BYTE0(K[1])]; + L[0] ^= rc[r]; + + L[1] = C0[BYTE7(K[1])]; + L[1] ^= C1[BYTE6(K[0])]; + L[1] ^= C2[BYTE5(K[7])]; + L[1] ^= C3[BYTE4(K[6])]; + L[1] ^= C4[BYTE3(K[5])]; + L[1] ^= C5[BYTE2(K[4])]; + L[1] ^= C6[BYTE1(K[3])]; + L[1] ^= C7[BYTE0(K[2])]; + + L[2] = C0[BYTE7(K[2])]; + L[2] ^= C1[BYTE6(K[1])]; + L[2] ^= C2[BYTE5(K[0])]; + L[2] ^= C3[BYTE4(K[7])]; + L[2] ^= C4[BYTE3(K[6])]; + L[2] ^= C5[BYTE2(K[5])]; + L[2] ^= C6[BYTE1(K[4])]; + L[2] ^= C7[BYTE0(K[3])]; + + L[3] = C0[BYTE7(K[3])]; + L[3] ^= C1[BYTE6(K[2])]; + L[3] ^= C2[BYTE5(K[1])]; + L[3] ^= C3[BYTE4(K[0])]; + L[3] ^= C4[BYTE3(K[7])]; + L[3] ^= C5[BYTE2(K[6])]; + L[3] ^= C6[BYTE1(K[5])]; + L[3] ^= C7[BYTE0(K[4])]; + + L[4] = C0[BYTE7(K[4])]; + L[4] ^= C1[BYTE6(K[3])]; + L[4] ^= C2[BYTE5(K[2])]; + L[4] ^= C3[BYTE4(K[1])]; + L[4] ^= C4[BYTE3(K[0])]; + L[4] ^= C5[BYTE2(K[7])]; + L[4] ^= C6[BYTE1(K[6])]; + L[4] ^= C7[BYTE0(K[5])]; + + L[5] = C0[BYTE7(K[5])]; + L[5] ^= C1[BYTE6(K[4])]; + L[5] ^= C2[BYTE5(K[3])]; + L[5] ^= C3[BYTE4(K[2])]; + L[5] ^= C4[BYTE3(K[1])]; + L[5] ^= C5[BYTE2(K[0])]; + L[5] ^= C6[BYTE1(K[7])]; + L[5] ^= C7[BYTE0(K[6])]; + + L[6] = C0[BYTE7(K[6])]; + L[6] ^= C1[BYTE6(K[5])]; + L[6] ^= C2[BYTE5(K[4])]; + L[6] ^= C3[BYTE4(K[3])]; + L[6] ^= C4[BYTE3(K[2])]; + L[6] ^= C5[BYTE2(K[1])]; + L[6] ^= C6[BYTE1(K[0])]; + L[6] ^= C7[BYTE0(K[7])]; + + L[7] = C0[BYTE7(K[7])]; + L[7] ^= C1[BYTE6(K[6])]; + L[7] ^= C2[BYTE5(K[5])]; + L[7] ^= C3[BYTE4(K[4])]; + L[7] ^= C4[BYTE3(K[3])]; + L[7] ^= C5[BYTE2(K[2])]; + L[7] ^= C6[BYTE1(K[1])]; + L[7] ^= C7[BYTE0(K[0])]; K[0] = L[0]; K[1] = L[1]; @@ -877,85 +881,85 @@ static void wp512_process_buffer(struct K[6] = L[6]; K[7] = L[7]; - L[0] = C0[(int)(state[0] >> 56) ] ^ - C1[(int)(state[7] >> 48) & 0xff] ^ - C2[(int)(state[6] >> 40) & 0xff] ^ - C3[(int)(state[5] >> 32) & 0xff] ^ - C4[(int)(state[4] >> 24) & 0xff] ^ - C5[(int)(state[3] >> 16) & 0xff] ^ - C6[(int)(state[2] >> 8) & 0xff] ^ - C7[(int)(state[1] ) & 0xff] ^ - K[0]; - - L[1] = C0[(int)(state[1] >> 56) ] ^ - C1[(int)(state[0] >> 48) & 0xff] ^ - C2[(int)(state[7] >> 40) & 0xff] ^ - C3[(int)(state[6] >> 32) & 0xff] ^ - C4[(int)(state[5] >> 24) & 0xff] ^ - C5[(int)(state[4] >> 16) & 0xff] ^ - C6[(int)(state[3] >> 8) & 0xff] ^ - C7[(int)(state[2] ) & 0xff] ^ - K[1]; - - L[2] = C0[(int)(state[2] >> 56) ] ^ - C1[(int)(state[1] >> 48) & 0xff] ^ - C2[(int)(state[0] >> 40) & 0xff] ^ - C3[(int)(state[7] >> 32) & 0xff] ^ - C4[(int)(state[6] >> 24) & 0xff] ^ - C5[(int)(state[5] >> 16) & 0xff] ^ - C6[(int)(state[4] >> 8) & 0xff] ^ - C7[(int)(state[3] ) & 0xff] ^ - K[2]; - - L[3] = C0[(int)(state[3] >> 56) ] ^ - C1[(int)(state[2] >> 48) & 0xff] ^ - C2[(int)(state[1] >> 40) & 0xff] ^ - C3[(int)(state[0] >> 32) & 0xff] ^ - C4[(int)(state[7] >> 24) & 0xff] ^ - C5[(int)(state[6] >> 16) & 0xff] ^ - C6[(int)(state[5] >> 8) & 0xff] ^ - C7[(int)(state[4] ) & 0xff] ^ - K[3]; - - L[4] = C0[(int)(state[4] >> 56) ] ^ - C1[(int)(state[3] >> 48) & 0xff] ^ - C2[(int)(state[2] >> 40) & 0xff] ^ - C3[(int)(state[1] >> 32) & 0xff] ^ - C4[(int)(state[0] >> 24) & 0xff] ^ - C5[(int)(state[7] >> 16) & 0xff] ^ - C6[(int)(state[6] >> 8) & 0xff] ^ - C7[(int)(state[5] ) & 0xff] ^ - K[4]; - - L[5] = C0[(int)(state[5] >> 56) ] ^ - C1[(int)(state[4] >> 48) & 0xff] ^ - C2[(int)(state[3] >> 40) & 0xff] ^ - C3[(int)(state[2] >> 32) & 0xff] ^ - C4[(int)(state[1] >> 24) & 0xff] ^ - C5[(int)(state[0] >> 16) & 0xff] ^ - C6[(int)(state[7] >> 8) & 0xff] ^ - C7[(int)(state[6] ) & 0xff] ^ - K[5]; - - L[6] = C0[(int)(state[6] >> 56) ] ^ - C1[(int)(state[5] >> 48) & 0xff] ^ - C2[(int)(state[4] >> 40) & 0xff] ^ - C3[(int)(state[3] >> 32) & 0xff] ^ - C4[(int)(state[2] >> 24) & 0xff] ^ - C5[(int)(state[1] >> 16) & 0xff] ^ - C6[(int)(state[0] >> 8) & 0xff] ^ - C7[(int)(state[7] ) & 0xff] ^ - K[6]; - - L[7] = C0[(int)(state[7] >> 56) ] ^ - C1[(int)(state[6] >> 48) & 0xff] ^ - C2[(int)(state[5] >> 40) & 0xff] ^ - C3[(int)(state[4] >> 32) & 0xff] ^ - C4[(int)(state[3] >> 24) & 0xff] ^ - C5[(int)(state[2] >> 16) & 0xff] ^ - C6[(int)(state[1] >> 8) & 0xff] ^ - C7[(int)(state[0] ) & 0xff] ^ - K[7]; + L[0] = C0[BYTE7(state[0])]; + L[0] ^= C1[BYTE6(state[7])]; + L[0] ^= C2[BYTE5(state[6])]; + L[0] ^= C3[BYTE4(state[5])]; + L[0] ^= C4[BYTE3(state[4])]; + L[0] ^= C5[BYTE2(state[3])]; + L[0] ^= C6[BYTE1(state[2])]; + L[0] ^= C7[BYTE0(state[1])]; + L[0] ^= K[0]; + + L[1] = C0[BYTE7(state[1])]; + L[1] ^= C1[BYTE6(state[0])]; + L[1] ^= C2[BYTE5(state[7])]; + L[1] ^= C3[BYTE4(state[6])]; + L[1] ^= C4[BYTE3(state[5])]; + L[1] ^= C5[BYTE2(state[4])]; + L[1] ^= C6[BYTE1(state[3])]; + L[1] ^= C7[BYTE0(state[2])]; + L[1] ^= K[1]; + + L[2] = C0[BYTE7(state[2])]; + L[2] ^= C1[BYTE6(state[1])]; + L[2] ^= C2[BYTE5(state[0])]; + L[2] ^= C3[BYTE4(state[7])]; + L[2] ^= C4[BYTE3(state[6])]; + L[2] ^= C5[BYTE2(state[5])]; + L[2] ^= C6[BYTE1(state[4])]; + L[2] ^= C7[BYTE0(state[3])]; + L[2] ^= K[2]; + + L[3] = C0[BYTE7(state[3])]; + L[3] ^= C1[BYTE6(state[2])]; + L[3] ^= C2[BYTE5(state[1])]; + L[3] ^= C3[BYTE4(state[0])]; + L[3] ^= C4[BYTE3(state[7])]; + L[3] ^= C5[BYTE2(state[6])]; + L[3] ^= C6[BYTE1(state[5])]; + L[3] ^= C7[BYTE0(state[4])]; + L[3] ^= K[3]; + + L[4] = C0[BYTE7(state[4])]; + L[4] ^= C1[BYTE6(state[3])]; + L[4] ^= C2[BYTE5(state[2])]; + L[4] ^= C3[BYTE4(state[1])]; + L[4] ^= C4[BYTE3(state[0])]; + L[4] ^= C5[BYTE2(state[7])]; + L[4] ^= C6[BYTE1(state[6])]; + L[4] ^= C7[BYTE0(state[5])]; + L[4] ^= K[4]; + + L[5] = C0[BYTE7(state[5])]; + L[5] ^= C1[BYTE6(state[4])]; + L[5] ^= C2[BYTE5(state[3])]; + L[5] ^= C3[BYTE4(state[2])]; + L[5] ^= C4[BYTE3(state[1])]; + L[5] ^= C5[BYTE2(state[0])]; + L[5] ^= C6[BYTE1(state[7])]; + L[5] ^= C7[BYTE0(state[6])]; + L[5] ^= K[5]; + + L[6] = C0[BYTE7(state[6])]; + L[6] ^= C1[BYTE6(state[5])]; + L[6] ^= C2[BYTE5(state[4])]; + L[6] ^= C3[BYTE4(state[3])]; + L[6] ^= C4[BYTE3(state[2])]; + L[6] ^= C5[BYTE2(state[1])]; + L[6] ^= C6[BYTE1(state[0])]; + L[6] ^= C7[BYTE0(state[7])]; + L[6] ^= K[6]; + + L[7] = C0[BYTE7(state[7])]; + L[7] ^= C1[BYTE6(state[6])]; + L[7] ^= C2[BYTE5(state[5])]; + L[7] ^= C3[BYTE4(state[4])]; + L[7] ^= C4[BYTE3(state[3])]; + L[7] ^= C5[BYTE2(state[2])]; + L[7] ^= C6[BYTE1(state[1])]; + L[7] ^= C7[BYTE0(state[0])]; + L[7] ^= K[7]; state[0] = L[0]; state[1] = L[1]; | |