Merge #1554: [Core] Big endian support

25224a8 Replace CBlockHeader::GetHash with call to SerializeHash (random-zebra)
49a7716 Add serialize float/double tests (random-zebra)
83c6d13 src/txmempool.cpp: make numEntries a uint32_t (random-zebra)
70c6606 src/serialize.h: base serialization level endianness neutrality (random-zebra)
4ad83fe src/script/script.h: endian compatibility for PUSHDATA sizes (random-zebra)
74f1699 src/primitives/transaction.h: endian compatibility in serialization (random-zebra)
b5e5de6 src/primitives/block.cpp: endian compatibility in GetHash (random-zebra)
b6b4b3e src/net.cpp: endian compatibility in EndMessage (random-zebra)
1d84b73 src/netbase.h: Fix endian in CNetAddr serialization (random-zebra)
4d98939 src/main.cpp: endian compatibility in packet checksum check (random-zebra)
fe2727b src/hash.cpp: endian compatibility (random-zebra)

Pull request description:

  First step in getting our serialization code up to par with upstream.
  Backports bitcoin#5510 with only minor adjustments.

  > Fix issue bitcoin#888.
  >
  > This has been structured so that each compatibility change is one commit that touches only one file. After the initial build change, they are independent.
  >
  > Most extensive changes are in 'src/serialize.h: base serialization level endianness neutrality'. I had to replace READDATA and WRITEDATA with functions that take sized integer types to make use of the proper endian.h functions. I'm confident that the end result is the same, although this may require more tests.
  >
  > I've tested this on mipsbe32.
  >
  > All tests pass
  > Testnet syncs correctly
  > Node can successfully function on P2P mainnet
  Checked that data directory can be copied between endians with no adverse results (only peers.dat required special attention here)
  > Known issues (to be fixed before merge):
  >
  > DNS seeding always comes with 0 results on BE (confirmed as working by @paveljanik on real hardware, must have been issue with my qemu-user setup)

ACKs for top commit:
  furszy:
    Everything working here, utACK 25224a8
  Fuzzbawls:
    utACK 25224a8

Tree-SHA512: 047e3bd6a7c6bbbd341da1ec3fb7698e19d52e170ba0d475e6f451cbb52e2c55f23a15999da663a8022ddc9df3a5c168f825b779c4afb1fd210b9da680503297

src/hash.cpp

@@ -4,6 +4,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "hash.h"
+#include "crypto/common.h"
 #include "crypto/hmac_sha512.h"
 #include "crypto/scrypt.h"
 
@@ -24,10 +25,10 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char
 
         //----------
         // body
-        const uint32_t* blocks = (const uint32_t*)(&vDataToHash[0] + nblocks * 4);
+        const uint8_t* blocks = &vDataToHash[0] + nblocks * 4;
 
         for (int i = -nblocks; i; i++) {
-            uint32_t k1 = blocks[i];
+            uint32_t k1 = ReadLE32(blocks + i*4);
 
             k1 *= c1;
             k1 = ROTL32(k1, 15);

src/main.cpp

@@ -6129,8 +6129,7 @@ bool ProcessMessages(CNode* pfrom)
         // Checksum
         CDataStream& vRecv = msg.vRecv;
         uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize);
-        unsigned int nChecksum = 0;
-        memcpy(&nChecksum, &hash, sizeof(nChecksum));
+        unsigned int nChecksum = ReadLE32((unsigned char*)&hash);
         if (nChecksum != hdr.nChecksum) {
             LogPrintf("ProcessMessages(%s, %u bytes): CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n",
                 SanitizeString(strCommand), nMessageSize, nChecksum, hdr.nChecksum);

src/net.cpp

@@ -14,6 +14,7 @@
 #include "addrman.h"
 #include "chainparams.h"
 #include "clientversion.h"
+#include "crypto/common.h"
 #include "guiinterface.h"
 #include "main.h"
 #include "miner.h"
@@ -2255,7 +2256,7 @@ void CNode::EndMessage() UNLOCK_FUNCTION(cs_vSend)
 
     // Set the size
     unsigned int nSize = ssSend.size() - CMessageHeader::HEADER_SIZE;
-    memcpy((char*)&ssSend[CMessageHeader::MESSAGE_SIZE_OFFSET], &nSize, sizeof(nSize));
+    WriteLE32((uint8_t*)&ssSend[CMessageHeader::MESSAGE_SIZE_OFFSET], nSize);
 
     // Set the checksum
     uint256 hash = Hash(ssSend.begin() + CMessageHeader::HEADER_SIZE, ssSend.end());

src/netbase.h

@@ -181,7 +181,7 @@ class CService : public CNetAddr
     {
         READWRITE(FLATDATA(ip));
         unsigned short portN = htons(port);
-        READWRITE(portN);
+        READWRITE(FLATDATA(portN));
         if (ser_action.ForRead())
             port = ntohs(portN);
     }

src/primitives/block.cpp

@@ -15,13 +15,22 @@
 
 uint256 CBlockHeader::GetHash() const
 {
-    if (nVersion < 4)
+    if (nVersion < 4)  {
+#if defined(WORDS_BIGENDIAN)
+        uint8_t data[80];
+        WriteLE32(&data[0], nVersion);
+        memcpy(&data[4], hashPrevBlock.begin(), hashPrevBlock.size());
+        memcpy(&data[36], hashMerkleRoot.begin(), hashMerkleRoot.size());
+        WriteLE32(&data[68], nTime);
+        WriteLE32(&data[72], nBits);
+        WriteLE32(&data[76], nNonce);
+        return HashQuark(data, data + 80);
+#else // Can take shortcut for little endian
         return HashQuark(BEGIN(nVersion), END(nNonce));
-
-    if (nVersion < 7)
-        return Hash(BEGIN(nVersion), END(nAccumulatorCheckpoint));
-
-    return Hash(BEGIN(nVersion), END(nNonce));
+#endif
+    }
+    // version >= 4
+    return SerializeHash(*this);
 }
 
 std::string CBlock::ToString() const

src/primitives/transaction.h

@@ -30,7 +30,8 @@ class COutPoint
 
     template <typename Stream, typename Operation>
     inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
-        READWRITE(FLATDATA(*this));
+        READWRITE(hash);
+        READWRITE(n);
     }
 
     void SetNull() { hash.SetNull(); n = (uint32_t) -1; }

src/script/script.h

@@ -18,6 +18,8 @@
 #include <string>
 #include <vector>
 
+#include "crypto/common.h"
+
 typedef std::vector<unsigned char> valtype;
 
 static const unsigned int MAX_SCRIPT_ELEMENT_SIZE = 520; // bytes
@@ -436,14 +438,16 @@ class CScript : public std::vector<unsigned char>
         else if (b.size() <= 0xffff)
         {
             insert(end(), OP_PUSHDATA2);
-            unsigned short nSize = b.size();
-            insert(end(), (unsigned char*)&nSize, (unsigned char*)&nSize + sizeof(nSize));
+            uint8_t data[2];
+            WriteLE16(data, b.size());
+            insert(end(), data, data + sizeof(data));
         }
         else
         {
             insert(end(), OP_PUSHDATA4);
-            unsigned int nSize = b.size();
-            insert(end(), (unsigned char*)&nSize, (unsigned char*)&nSize + sizeof(nSize));
+            uint8_t data[4];
+            WriteLE32(data, b.size());
+            insert(end(), data, data + sizeof(data));
         }
         insert(end(), b.begin(), b.end());
         return *this;
@@ -522,15 +526,14 @@ class CScript : public std::vector<unsigned char>
             {
                 if (end() - pc < 2)
                     return false;
-                nSize = 0;
-                memcpy(&nSize, &pc[0], 2);
+                nSize = ReadLE16(&pc[0]);
                 pc += 2;
             }
             else if (opcode == OP_PUSHDATA4)
             {
                 if (end() - pc < 4)
                     return false;
-                memcpy(&nSize, &pc[0], 4);
+                nSize = ReadLE32(&pc[0]);
                 pc += 4;
             }
             if (end() - pc < 0 || (unsigned int)(end() - pc) < nSize)