Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change SHA256 byte counter from size_t to uint64_t #731

Merged
merged 4 commits into from
Mar 24, 2022
Merged

Change SHA256 byte counter from size_t to uint64_t #731

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eb28464
Change SHA256 byte counter from size_t to uint64_t
real-or-random Mar 31, 2020
8e3dde1
Simplify struct initializer for SHA256 padding
real-or-random Mar 31, 2020
9b514ce
Add test vector for very long SHA256 messages
real-or-random Apr 16, 2020
f8d9174
Add SHA256 bit counter tests
real-or-random Apr 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/hash.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
typedef struct {
uint32_t s[8];
uint32_t buf[16]; /* In big endian */
size_t bytes;
uint64_t bytes;
} secp256k1_sha256;

static void secp256k1_sha256_initialize(secp256k1_sha256 *hash);
Expand Down
4 changes: 3 additions & 1 deletion src/hash_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,12 @@ static void secp256k1_sha256_write(secp256k1_sha256 *hash, const unsigned char *
}

static void secp256k1_sha256_finalize(secp256k1_sha256 *hash, unsigned char *out32) {
static const unsigned char pad[64] = {0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
static const unsigned char pad[64] = {0x80};
uint32_t sizedesc[2];
uint32_t out[8];
int i = 0;
/* The maximum message size of SHA256 is 2^64-1 bits. */
VERIFY_CHECK(hash->bytes < ((uint64_t)1 << 61));
sizedesc[0] = BE32(hash->bytes >> 29);
sizedesc[1] = BE32(hash->bytes << 3);
secp256k1_sha256_write(hash, pad, 1 + ((119 - (hash->bytes % 64)) % 64));
Expand Down
146 changes: 135 additions & 11 deletions src/tests.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,42 +451,165 @@ void run_ctz_tests(void) {

/***** HASH TESTS *****/

void run_sha256_tests(void) {
static const char *inputs[8] = {
void run_sha256_known_output_tests(void) {
static const char *inputs[] = {
"", "abc", "message digest", "secure hash algorithm", "SHA256 is considered to be safe",
"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
"For this sample, this 63-byte string will be used as input data",
"This is exactly 64 bytes long, not counting the terminating byte"
"This is exactly 64 bytes long, not counting the terminating byte",
"aaaaa",
};
static const unsigned char outputs[8][32] = {
static const unsigned int repeat[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1000000/5
};
static const unsigned char outputs[][32] = {
{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55},
{0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad},
{0xf7, 0x84, 0x6f, 0x55, 0xcf, 0x23, 0xe1, 0x4e, 0xeb, 0xea, 0xb5, 0xb4, 0xe1, 0x55, 0x0c, 0xad, 0x5b, 0x50, 0x9e, 0x33, 0x48, 0xfb, 0xc4, 0xef, 0xa3, 0xa1, 0x41, 0x3d, 0x39, 0x3c, 0xb6, 0x50},
{0xf3, 0x0c, 0xeb, 0x2b, 0xb2, 0x82, 0x9e, 0x79, 0xe4, 0xca, 0x97, 0x53, 0xd3, 0x5a, 0x8e, 0xcc, 0x00, 0x26, 0x2d, 0x16, 0x4c, 0xc0, 0x77, 0x08, 0x02, 0x95, 0x38, 0x1c, 0xbd, 0x64, 0x3f, 0x0d},
{0x68, 0x19, 0xd9, 0x15, 0xc7, 0x3f, 0x4d, 0x1e, 0x77, 0xe4, 0xe1, 0xb5, 0x2d, 0x1f, 0xa0, 0xf9, 0xcf, 0x9b, 0xea, 0xea, 0xd3, 0x93, 0x9f, 0x15, 0x87, 0x4b, 0xd9, 0x88, 0xe2, 0xa2, 0x36, 0x30},
{0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, 0xe5, 0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39, 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff, 0x21, 0x67, 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1},
{0xf0, 0x8a, 0x78, 0xcb, 0xba, 0xee, 0x08, 0x2b, 0x05, 0x2a, 0xe0, 0x70, 0x8f, 0x32, 0xfa, 0x1e, 0x50, 0xc5, 0xc4, 0x21, 0xaa, 0x77, 0x2b, 0xa5, 0xdb, 0xb4, 0x06, 0xa2, 0xea, 0x6b, 0xe3, 0x42},
{0xab, 0x64, 0xef, 0xf7, 0xe8, 0x8e, 0x2e, 0x46, 0x16, 0x5e, 0x29, 0xf2, 0xbc, 0xe4, 0x18, 0x26, 0xbd, 0x4c, 0x7b, 0x35, 0x52, 0xf6, 0xb3, 0x82, 0xa9, 0xe7, 0xd3, 0xaf, 0x47, 0xc2, 0x45, 0xf8}
{0xab, 0x64, 0xef, 0xf7, 0xe8, 0x8e, 0x2e, 0x46, 0x16, 0x5e, 0x29, 0xf2, 0xbc, 0xe4, 0x18, 0x26, 0xbd, 0x4c, 0x7b, 0x35, 0x52, 0xf6, 0xb3, 0x82, 0xa9, 0xe7, 0xd3, 0xaf, 0x47, 0xc2, 0x45, 0xf8},
{0xcd, 0xc7, 0x6e, 0x5c, 0x99, 0x14, 0xfb, 0x92, 0x81, 0xa1, 0xc7, 0xe2, 0x84, 0xd7, 0x3e, 0x67, 0xf1, 0x80, 0x9a, 0x48, 0xa4, 0x97, 0x20, 0x0e, 0x04, 0x6d, 0x39, 0xcc, 0xc7, 0x11, 0x2c, 0xd0},
};
int i;
for (i = 0; i < 8; i++) {
unsigned int i, ninputs;

/* Skip last input vector for low iteration counts */
ninputs = sizeof(inputs)/sizeof(inputs[0]) - 1;
CONDITIONAL_TEST(16, "run_sha256_known_output_tests 1000000") ninputs++;

for (i = 0; i < ninputs; i++) {
unsigned char out[32];
secp256k1_sha256 hasher;
unsigned int j;
/* 1. Run: simply write the input bytestrings */
j = repeat[i];
secp256k1_sha256_initialize(&hasher);
secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), strlen(inputs[i]));
while (j > 0) {
secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), strlen(inputs[i]));
j--;
}
secp256k1_sha256_finalize(&hasher, out);
CHECK(secp256k1_memcmp_var(out, outputs[i], 32) == 0);
/* 2. Run: split the input bytestrings randomly before writing */
if (strlen(inputs[i]) > 0) {
int split = secp256k1_testrand_int(strlen(inputs[i]));
secp256k1_sha256_initialize(&hasher);
secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), split);
secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i] + split), strlen(inputs[i]) - split);
j = repeat[i];
while (j > 0) {
secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i]), split);
secp256k1_sha256_write(&hasher, (const unsigned char*)(inputs[i] + split), strlen(inputs[i]) - split);
j--;
}
secp256k1_sha256_finalize(&hasher, out);
CHECK(secp256k1_memcmp_var(out, outputs[i], 32) == 0);
}
}
}

/** SHA256 counter tests

The tests verify that the SHA256 counter doesn't wrap around at message length
2^i bytes for i = 20, ..., 33. This wide range aims at being independent of the
implementation of the counter and it catches multiple natural 32-bit overflows
(e.g., counting bits, counting bytes, counting blocks, ...).

The test vectors have been generated using following Python script which relies
on https://github.com/cloudtools/sha256/ (v0.3 on Python v3.10.2).

```
from sha256 import sha256
from copy import copy

def midstate_c_definition(hasher):
ret = ' {{0x' + hasher.state[0].hex('_', 4).replace('_', ', 0x') + '},\n'
ret += ' {0x00}, ' + str(hex(hasher.state[1])) + '}'
return ret

def output_c_literal(hasher):
return '{0x' + hasher.digest().hex('_').replace('_', ', 0x') + '}'

MESSAGE = b'abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno'
assert(len(MESSAGE) == 64)
BYTE_BOUNDARIES = [(2**b)//len(MESSAGE) - 1 for b in range(20, 34)]

midstates = []
digests = []
hasher = sha256()
for i in range(BYTE_BOUNDARIES[-1] + 1):
if i in BYTE_BOUNDARIES:
midstates.append(midstate_c_definition(hasher))
hasher_copy = copy(hasher)
hasher_copy.update(MESSAGE)
digests.append(output_c_literal(hasher_copy))
hasher.update(MESSAGE)

for x in midstates:
print(x + ',')

for x in digests:
print(x + ',')
```
*/
void run_sha256_counter_tests(void) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would make sense to add a comment here that there exists code that generates this and where it is.

Fwiw I can't run it. After installing sha256 with pip I get the error

Traceback (most recent call last):
  File "py.py", line 22, in <module>
    hasher_copy = copy(hasher)
  File "/usr/lib/python3.8/copy.py", line 92, in copy
    rv = reductor(4)
TypeError: cannot pickle 'sha256.sha256' object

That's why everyone should use nix by the way :) I don't think it's particularly important to recreate the test cases as they shouldn't change and it's unlikely that we want to add more (otherwise I would have also suggested to move the code from the commit message into the contrib directory).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I get the same error.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe retry pip install sha256. They recently released a new version which claims to have Python 3 support. Maybe that fixes the problem (though I have no idea why this was working for me....)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Works for me now (python 3).

static const char *input = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno";
static const secp256k1_sha256 midstates[] = {
{{0xa2b5c8bb, 0x26c88bb3, 0x2abdc3d2, 0x9def99a3, 0xdfd21a6e, 0x41fe585b, 0x7ef2c440, 0x2b79adda},
{0x00}, 0xfffc0},
{{0xa0d29445, 0x9287de66, 0x76aabd71, 0x41acd765, 0x0c7528b4, 0x84e14906, 0x942faec6, 0xcc5a7b26},
{0x00}, 0x1fffc0},
{{0x50449526, 0xb9f1d657, 0xa0fc13e9, 0x50860f10, 0xa550c431, 0x3fbc97c1, 0x7bbb2d89, 0xdb67bac1},
{0x00}, 0x3fffc0},
{{0x54a6efdc, 0x46762e7b, 0x88bfe73f, 0xbbd149c7, 0x41620c43, 0x1168da7b, 0x2c5960f9, 0xeccffda6},
{0x00}, 0x7fffc0},
{{0x2515a8f5, 0x5faa2977, 0x3a850486, 0xac858cad, 0x7b7276ee, 0x235c0385, 0xc53a157c, 0x7cb3e69c},
{0x00}, 0xffffc0},
{{0x34f39828, 0x409fedb7, 0x4bbdd0fb, 0x3b643634, 0x7806bf2e, 0xe0d1b713, 0xca3f2e1e, 0xe38722c2},
{0x00}, 0x1ffffc0},
{{0x389ef5c5, 0x38c54167, 0x8f5d56ab, 0x582a75cc, 0x8217caef, 0xf10947dd, 0x6a1998a8, 0x048f0b8c},
{0x00}, 0x3ffffc0},
{{0xd6c3f394, 0x0bee43b9, 0x6783f497, 0x29fa9e21, 0x6ce491c1, 0xa81fe45e, 0x2fc3859a, 0x269012d0},
{0x00}, 0x7ffffc0},
{{0x6dd3c526, 0x44d88aa0, 0x806a1bae, 0xfbcc0d32, 0x9d6144f3, 0x9d2bd757, 0x9851a957, 0xb50430ad},
{0x00}, 0xfffffc0},
{{0x2add4021, 0xdfe8a9e6, 0xa56317c6, 0x7a15f5bb, 0x4a48aacd, 0x5d368414, 0x4f00e6f0, 0xd9355023},
{0x00}, 0x1fffffc0},
{{0xb66666b4, 0xdbeac32b, 0x0ea351ae, 0xcba9da46, 0x6278b874, 0x8c508e23, 0xe16ca776, 0x8465bac1},
{0x00}, 0x3fffffc0},
{{0xb6744789, 0x9cce87aa, 0xc4c478b7, 0xf38404d8, 0x2e38ba62, 0xa3f7019b, 0x50458fe7, 0x3047dbec},
{0x00}, 0x7fffffc0},
{{0x8b1297ba, 0xba261a80, 0x2ba1b0dd, 0xfbc67d6d, 0x61072c4e, 0x4b5a2a0f, 0x52872760, 0x2dfeb162},
{0x00}, 0xffffffc0},
{{0x24f33cf7, 0x41ad6583, 0x41c8ff5d, 0xca7ef35f, 0x50395756, 0x021b743e, 0xd7126cd7, 0xd037473a},
{0x00}, 0x1ffffffc0},
};
static const unsigned char outputs[][32] = {
{0x0e, 0x83, 0xe2, 0xc9, 0x4f, 0xb2, 0xb8, 0x2b, 0x89, 0x06, 0x92, 0x78, 0x04, 0x03, 0x48, 0x5c, 0x48, 0x44, 0x67, 0x61, 0x77, 0xa4, 0xc7, 0x90, 0x9e, 0x92, 0x55, 0x10, 0x05, 0xfe, 0x39, 0x15},
{0x1d, 0x1e, 0xd7, 0xb8, 0xa3, 0xa7, 0x8a, 0x79, 0xfd, 0xa0, 0x05, 0x08, 0x9c, 0xeb, 0xf0, 0xec, 0x67, 0x07, 0x9f, 0x8e, 0x3c, 0x0d, 0x8e, 0xf9, 0x75, 0x55, 0x13, 0xc1, 0xe8, 0x77, 0xf8, 0xbb},
{0x66, 0x95, 0x6c, 0xc9, 0xe0, 0x39, 0x65, 0xb6, 0xb0, 0x05, 0xd1, 0xaf, 0xaf, 0xf3, 0x1d, 0xb9, 0xa4, 0xda, 0x6f, 0x20, 0xcd, 0x3a, 0xae, 0x64, 0xc2, 0xdb, 0xee, 0xf5, 0xb8, 0x8d, 0x57, 0x0e},
{0x3c, 0xbb, 0x1c, 0x12, 0x5e, 0x17, 0xfd, 0x54, 0x90, 0x45, 0xa7, 0x7b, 0x61, 0x6c, 0x1d, 0xfe, 0xe6, 0xcc, 0x7f, 0xee, 0xcf, 0xef, 0x33, 0x35, 0x50, 0x62, 0x16, 0x70, 0x2f, 0x87, 0xc3, 0xc9},
{0x53, 0x4d, 0xa8, 0xe7, 0x1e, 0x98, 0x73, 0x8d, 0xd9, 0xa3, 0x54, 0xa5, 0x0e, 0x59, 0x2c, 0x25, 0x43, 0x6f, 0xaa, 0xa2, 0xf5, 0x21, 0x06, 0x3e, 0xc9, 0x82, 0x06, 0x94, 0x98, 0x72, 0x9d, 0xa7},
{0xef, 0x7e, 0xe9, 0x6b, 0xd3, 0xe5, 0xb7, 0x41, 0x4c, 0xc8, 0xd3, 0x07, 0x52, 0x9a, 0x5a, 0x8b, 0x4e, 0x1e, 0x75, 0xa4, 0x17, 0x78, 0xc8, 0x36, 0xcd, 0xf8, 0x2e, 0xd9, 0x57, 0xe3, 0xd7, 0x07},
{0x87, 0x16, 0xfb, 0xf9, 0xa5, 0xf8, 0xc4, 0x56, 0x2b, 0x48, 0x52, 0x8e, 0x2d, 0x30, 0x85, 0xb6, 0x4c, 0x56, 0xb5, 0xd1, 0x16, 0x9c, 0xcf, 0x32, 0x95, 0xad, 0x03, 0xe8, 0x05, 0x58, 0x06, 0x76},
{0x75, 0x03, 0x80, 0x28, 0xf2, 0xa7, 0x63, 0x22, 0x1a, 0x26, 0x9c, 0x68, 0xe0, 0x58, 0xfc, 0x73, 0xeb, 0x42, 0xf6, 0x86, 0x16, 0x24, 0x4b, 0xbc, 0x24, 0xf7, 0x02, 0xc8, 0x3d, 0x90, 0xe2, 0xb0},
{0xdf, 0x49, 0x0f, 0x15, 0x7b, 0x7d, 0xbf, 0xe0, 0xd4, 0xcf, 0x47, 0xc0, 0x80, 0x93, 0x4a, 0x61, 0xaa, 0x03, 0x07, 0x66, 0xb3, 0x38, 0x5d, 0xc8, 0xc9, 0x07, 0x61, 0xfb, 0x97, 0x10, 0x2f, 0xd8},
{0x77, 0x19, 0x40, 0x56, 0x41, 0xad, 0xbc, 0x59, 0xda, 0x1e, 0xc5, 0x37, 0x14, 0x63, 0x7b, 0xfb, 0x79, 0xe2, 0x7a, 0xb1, 0x55, 0x42, 0x99, 0x42, 0x56, 0xfe, 0x26, 0x9d, 0x0f, 0x7e, 0x80, 0xc6},
{0x50, 0xe7, 0x2a, 0x0e, 0x26, 0x44, 0x2f, 0xe2, 0x55, 0x2d, 0xc3, 0x93, 0x8a, 0xc5, 0x86, 0x58, 0x22, 0x8c, 0x0c, 0xbf, 0xb1, 0xd2, 0xca, 0x87, 0x2a, 0xe4, 0x35, 0x26, 0x6f, 0xcd, 0x05, 0x5e},
{0xe4, 0x80, 0x6f, 0xdb, 0x3d, 0x7d, 0xba, 0xde, 0x50, 0x3f, 0xea, 0x00, 0x3d, 0x46, 0x59, 0x64, 0xfd, 0x58, 0x1c, 0xa1, 0xb8, 0x7d, 0x5f, 0xac, 0x94, 0x37, 0x9e, 0xa0, 0xc0, 0x9c, 0x93, 0x8b},
{0x2c, 0xf3, 0xa9, 0xf6, 0x15, 0x25, 0x80, 0x70, 0x76, 0x99, 0x7d, 0xf1, 0xc3, 0x2f, 0xa3, 0x31, 0xff, 0x92, 0x35, 0x2e, 0x8d, 0x04, 0x13, 0x33, 0xd8, 0x0d, 0xdb, 0x4a, 0xf6, 0x8c, 0x03, 0x34},
{0xec, 0x12, 0x24, 0x9f, 0x35, 0xa4, 0x29, 0x8b, 0x9e, 0x4a, 0x95, 0xf8, 0x61, 0xaf, 0x61, 0xc5, 0x66, 0x55, 0x3e, 0x3f, 0x2a, 0x98, 0xea, 0x71, 0x16, 0x6b, 0x1c, 0xd9, 0xe4, 0x09, 0xd2, 0x8e},
};
unsigned int i;
for (i = 0; i < sizeof(midstates)/sizeof(midstates[0]); i++) {
unsigned char out[32];
secp256k1_sha256 hasher = midstates[i];
secp256k1_sha256_write(&hasher, (const unsigned char*)input, strlen(input));
secp256k1_sha256_finalize(&hasher, out);
CHECK(secp256k1_memcmp_var(out, outputs[i], 32) == 0);
}
}

void run_hmac_sha256_tests(void) {
static const char *keys[6] = {
"\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b",
Expand Down Expand Up @@ -6964,7 +7087,8 @@ int main(int argc, char **argv) {
run_modinv_tests();
run_inverse_tests();

run_sha256_tests();
run_sha256_known_output_tests();
run_sha256_counter_tests();
run_hmac_sha256_tests();
run_rfc6979_hmac_sha256_tests();
run_tagged_sha256_tests();
Expand Down