Merge pull request #100 from Chia-Network/alpha-1.4

Alpha 1.4

1. Compiling and execution now detect AES-NI, or a lack of it, and fall back to a software AES implementation. This adds support for Raspberry Pi 4, related ARM processors and Celeron processors. 
2. Added install instructions for CentOS/RHEL 8.1. Thanks to @A-Caccese for fixes to Windows WSL instructions.
3. Due to changes to the sqlite database that are not backwards compatible so re-synch will be required. Loading the blockchain only loads headers into memory instead of header blocks (header + proofs), speeds up the startup, and reduces normal operation memory usage by 80%. Memory access is now synchronous to reduce use of locks and speed up block processing.
4. Chia fullnode, farmer and harvester now default to logging to chia.log in the chia-blockchain directory. This is configured in config.yaml and due to config.yaml changes it is recommended to edit the new template config instead of using older config.yaml’s from previous versions.
5. Plotting working directory and final directory can both be specified in config.yaml
6. uvloop is now an optional add on.
7. Plotting now logs a timestamp at each major step. Thanks @dkackman who also fixed some compiler warnings. Additionally harvester/farmer will not try to farm plots that they don’t have the key for.
8. Added support for Python 3.8.

.github/workflows/pythonpackage.yml

@@ -9,34 +9,40 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
+        python-version: [3.7, 3.8]
         os: [ubuntu-latest, macOS-latest]
 
     steps:
     - uses: actions/checkout@v1
     - name: Setup Python environment
       uses: actions/setup-python@v1.1.1
       with:
-        python-version: 3.7 # optional, default is 3.x
+        python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      env:
-        CHIA_MACHINE_SSH_KEY: ${{ secrets.CHIA_MACHINE_SSH_KEY }}
-        GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"
       run: |
-        eval "$(ssh-agent -s)"
-        ssh-add - <<< "${CHIA_MACHINE_SSH_KEY}"
-        git submodule update --init --recursive
-        brew update && brew install gmp || echo ""
-        python3 -m venv .venv
+        brew update && brew install gmp boost || echo ""
+        sh install.sh
+    - name: Test proof of space
+      run: |
+        cd lib/chiapos
+        mkdir -p build && cd build
+        cmake ../
+        cmake --build . -- -j 6
+        ./RunTests
+        cd ../../../
+    - name: Test vdf
+      run: |
         . .venv/bin/activate
-        pip install -e .
-        pip install -r requirements.txt
-    - name: Lint with flake8
+        cd lib/chiavdf/fast_vdf
+        python python_bindings/test_verifier.py
+        cd ../../../
+    - name: Lint source with flake8
       run: |
         ./.venv/bin/flake8 src
-    - name: Lint with mypy
+    - name: Lint source with mypy
       run: |
         ./.venv/bin/mypy src tests
-    - name: Test with pytest
+    - name: Test blockchain code with pytest
       run: |
         ./.venv/bin/py.test tests -s -v
 

.gitignore

@@ -14,6 +14,7 @@ mongod.log*
 fndb_test*
 blockchain_test*
 *.db
+*.db-journal
 
 # Logs
 *.log

README.md

@@ -40,7 +40,7 @@ sh install.sh
 
 . .venv/bin/activate
 ```
-### CentOS 7
+### CentOS 7.7
 
 ```bash
 sudo yum update
@@ -63,9 +63,28 @@ git clone https://github.com/Chia-Network/chia-blockchain.git
 cd chia-blockchain
 
 sh install.sh
+
 . .venv/bin/activate
 ```
+### RHEL 8.1
+
+```bash
+sudo yum update
+sudo yum install gcc-c++ cmake3 git openssl openssl-devel
+sudo yum install wget make libffi-devel gmp-devel sqlite-devel
+
+# Install Python 3.7.5 (current rpm's are 3.6.x)
+wget https://www.python.org/ftp/python/3.7.5/Python-3.7.5.tgz
+tar -zxvf Python-3.7.5.tgz; cd Python-3.7.5
+./configure --enable-optimizations; sudo make install; cd ..
+
+git clone https://github.com/Chia-Network/chia-blockchain.git
+cd chia-blockchain
 
+sh install.sh
+
+. .venv/bin/activate
+```
 ### Windows (WSL + Ubuntu)
 #### Install WSL + Ubuntu 18.04 LTS, upgrade to Ubuntu 19.x
 
@@ -89,13 +108,14 @@ git clone https://github.com/Chia-Network/chia-blockchain.git
 cd chia-blockchain
 
 sudo sh install.sh
+
 . .venv/bin/activate
 ```
 
 #### Alternate method for Ubuntu 18.04 LTS
 In `./install.sh`:
 Change `python3` to `python3.7`
-Each line that starts with `pip ...` becomes `python -m pip ...`
+Each line that starts with `pip ...` becomes `python3.7 -m pip ...`
 
 ```bash
 sudo apt-get -y update
@@ -129,14 +149,14 @@ sh install.sh
 
 ## Step 2: Install timelord (optional)
 Note: this step is needed only if you intend to run a timelord or a local simulation.
-These assume you've already successfully installed harvester, farmer, plotting, and full node above. boost 1.67 or newer is required on all platforms.
+These assume you've already successfully installed harvester, farmer, plotting, and full node above. boost 1.66 or newer is required on all platforms.
 ### Ubuntu/Debian
 ```bash
 cd chia-blockchain
 
 sh install_timelord.sh
 ```
-### Amazon Linux 2 and CentOS 7
+### Amazon Linux 2 and CentOS 7.7
 ```bash
 #Only for Amazon Linux 2
 sudo amazon-linux-extras install epel
@@ -149,12 +169,20 @@ tar -zxvf boost_1_72_0.tar.gz
 cd boost_1_72_0
 ./bootstrap.sh --prefix=/usr/local
 sudo ./b2 install --prefix=/usr/local --with=all; cd ..
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 
 cd chia-blockchain
 
 sh install_timelord.sh
 ```
+### RHEL 8.1
+```bash
+sudo yum install mpfr-devel boost boost-devel
+
+cd chia-blockchain
 
+sh install_timelord.sh
+```
 ### Windows (WSL + Ubuntu)
 #### Install WSL + Ubuntu upgraded to 19.x
 ```bash
@@ -189,7 +217,9 @@ python -m scripts.regenerate_keys
 ## Step 4a: Run a full node
 To run a full node on port 8444, and connect to the testnet, run the following command.
 This will also start an ssh server in port 8222 for the UI, which you can connect to
-to see the state of the node.
+to see the state of the node. If you want to see std::out log output, modify the logging.std_out
+variable in ./config/config.yaml.
+
 ```bash
 ./scripts/run_full_node.sh
 ssh -p 8222 localhost
@@ -201,7 +231,8 @@ Farmers are entities in the network who use their hard drive space to try to cre
 blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which
 can take a long time depending on the [size of the plots](https://github.com/Chia-Network/chia-blockchain/wiki/k-sizes)
 (the k variable). Then, run the farmer + full node with the following script. A full node is also started,
-which you can ssh into to view the node UI (previous ssh command).
+which you can ssh into to view the node UI (previous ssh command). You can also change the working directory and
+final directory for plotting, with the "-t" and "-d" arguments to the create_plots script.
 ```bash
 python -m scripts.create_plots -k 20 -n 10
 sh ./scripts/run_farming.sh
@@ -229,8 +260,6 @@ Due to the nature of proof of space lookups by the harvester in the current alph
 the number of plots on a physical drive to 50 or less. This limit should significantly increase before beta.
 
 You can also run the simulation, which runs all servers and multiple full nodes, locally, at once.
-If you want to run the simulation, change the introducer ip in ./config/config.yaml so that the
-full node points to the local introducer (127.0.0.1:8445).
 
 Note the the simulation is local only and requires installation of timelords and VDFs.
 
@@ -240,3 +269,15 @@ ips to external peers.
 ```bash
 sh ./scripts/run_all_simulation.sh
 ```
+
+For increased networking performance, install uvloop:
+```bash
+pip install -e ".[uvloop]"
+```
+
+You can also use the [HTTP RPC](https://github.com/Chia-Network/chia-blockchain/wiki/Networking-and-Serialization#rpc) api to access information and control the full node:
+
+
+```bash
+curl -X POST  http://localhost:8555/get_blockchain_state
+```

config/config.yaml

@@ -2,15 +2,21 @@ network_id: testnet  # testnet/mainnet
 # Send a ping to all peers after ping_interval seconds
 ping_interval: 120
 
+# Controls logging of all servers (harvester, farmer, etc..). Each one can be overriden.
+logging: &logging
+  log_stdout: False  # If True, outputs to stdout instead of a file
+  log_filename: "chia.log"
+
 harvester:
   # The harvester server (if run) will run on this host and port
   host: 127.0.0.1
   port: 8448
   farmer_peer:
     host: 127.0.0.1
     port: 8447
-  # Location of all the plots, default ./plots
+  # Location of all the plots, default ./plots, for relative paths in plots.yaml.
   # plot_root: "/mnt/pos"
+  logging: *logging
 
 farmer:
   # The farmer server (if run) will run on this host and port
@@ -26,7 +32,9 @@ farmer:
   # To send a share to a pool, a block must be faster than this, in seconds
   pool_share_threshold: 12000
   # To send to the full node, a block must be faster than this, in seconds
+
   propagate_threshold: 10000
+  logging: *logging
 
 timelord:
   # The timelord server (if run) will run on this host and port
@@ -49,11 +57,20 @@ timelord:
   full_node_peer:
       host: 127.0.0.1
       port: 8444
+  logging: *logging
 
 full_node:
   # The full node server (if run) will run on this host and port
   host: 127.0.0.1
   port: 8444
+
+  # Run multiple nodes with different databases by changing the database_id
+  database_id: 1
+
+  # If True, starts an RPC server at the following port
+  start_rpc_server: True
+  rpc_port: 8555
+
   enable_upnp: True
   # Don't send any more than these number of headers and blocks, in one message
   max_headers_to_send: 25
@@ -63,27 +80,38 @@ full_node:
   # If node is more than these blocks behind, will do a sync
   sync_blocks_behind_threshold: 20
 
-  # This SSH key is for the ui SSH server
-  ssh_filename: config/ssh_host_key
   # How often to connect to introducer if we need to learn more peers
   introducer_connect_interval: 500
   # Continue trying to connect to more peers until this number of connections
   target_peer_count: 10
   # Only connect to peers who we have heard about in the last recent_peer_threshold seconds
   recent_peer_threshold: 6000
 
+  connect_to_farmer: False
+  connect_to_timelord: False
+
   farmer_peer:
       host: 127.0.0.1
       port: 8447
   timelord_peer:
       host: 127.0.0.1
       port: 8446
   introducer_peer:
-      # To run the simulation, set host to 127.0.0.1 and port to 8445
-      # host: 127.0.0.1
-      # port: 8445
       host: introducer.chia.net  # Chia AWS introducer IPv4/IPv6
       port: 8444
+  logging: *logging
+
+ui:
+   # The ui node server (if run) will run on this host and port
+  host: 127.0.0.1
+  port: 8222
+
+  # Which port to use to communicate with the full node
+  rpc_port: 8555
+
+  # This SSH key is for the ui SSH server
+  ssh_filename: config/ssh_host_key
+  logging: *logging
 
 introducer:
   host: 127.0.0.1
@@ -92,3 +120,4 @@ introducer:
   # The introducer will only return peers who it has seen in the last
   # recent_peer_threshold seconds
   recent_peer_threshold: 6000
+  logging: *logging

lib/chiapos/CMakeLists.txt

@@ -31,7 +31,21 @@ add_subdirectory(lib/pybind11)
 
 pybind11_add_module(chiapos ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiapos.cpp)
 
-set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++11 -maes")
+set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++1z -maes")
+try_run(CMAKE_AESNI_TEST_RUN_RESULT
+    CMAKE_AESNI_TEST_COMPILE_RESULT
+    ${CMAKE_CURRENT_BINARY_DIR}/cmake_aesni_test
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/cmake_aesni_test.cpp)
+
+# Did compilation succeed and process return 0 (success)?
+IF("${CMAKE_AESNI_TEST_COMPILE_RESULT}" AND ("${CMAKE_AESNI_TEST_RUN_RESULT}" EQUAL 0))
+    message(STATUS "AESNI Enabled")
+    set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++17 -maes")
+ELSE()
+    message(STATUS "AESNI Disabled")
+    add_compile_definitions (DISABLE_AESNI)
+    set (CMAKE_CXX_FLAGS "-g -O3 -Wall -march=native -std=c++17")
+ENDIF()
 
 add_executable(ProofOfSpace
     src/cli.cpp
@@ -45,7 +59,15 @@ add_executable(RunTests
     tests/test-main.cpp
     tests/test.cpp
 )
-target_link_libraries(chiapos PRIVATE fse)
-target_link_libraries(ProofOfSpace fse)
-target_link_libraries(HellmanAttacks fse)
-target_link_libraries(RunTests fse)
+
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  target_link_libraries(chiapos PRIVATE fse)
+  target_link_libraries(ProofOfSpace fse)
+  target_link_libraries(HellmanAttacks fse)
+  target_link_libraries(RunTests fse)
+else()
+  target_link_libraries(chiapos PRIVATE fse stdc++fs)
+  target_link_libraries(ProofOfSpace fse stdc++fs)
+  target_link_libraries(HellmanAttacks fse stdc++fs)
+  target_link_libraries(RunTests fse stdc++fs)
+endif()

lib/chiapos/README.md

@@ -8,8 +8,7 @@ Only runs on 64 bit architectures with AES-NI support. Read the [Proof of Space
 ### Compile
 
 ```bash
-git submodule update --init --recursive
-mkdir build && cd build
+mkdir -p build && cd build
 cmake ../
 cmake --build . -- -j 6
 ```

lib/chiapos/prepare.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+git submodule update --init --recursive
+mkdir build -p
+cd build
+cmake ../

lib/chiapos/python-bindings/chiapos.cpp

@@ -34,13 +34,14 @@ PYBIND11_MODULE(chiapos, m) {
 
     py::class_<DiskPlotter>(m, "DiskPlotter")
         .def(py::init<>())
-        .def("create_plot_disk", [](DiskPlotter &dp, const std::string filename, uint8_t k,
+        .def("create_plot_disk", [](DiskPlotter &dp, const std::string tmp_dir, const std::string final_dir,
+                                    const std::string filename, uint8_t k,
                                     const py::bytes &memo, const py::bytes &id) {
             std::string memo_str(memo);
             const uint8_t* memo_ptr = reinterpret_cast<const uint8_t*>(memo_str.data());
             std::string id_str(id);
             const uint8_t* id_ptr = reinterpret_cast<const uint8_t*>(id_str.data());
-            dp.CreatePlotDisk(filename, k, memo_ptr, len(memo), id_ptr, len(id));
+            dp.CreatePlotDisk(tmp_dir, final_dir, filename, k, memo_ptr, len(memo), id_ptr, len(id));
         });
 
     py::class_<DiskProver>(m, "DiskProver")