RVV: add cmake toolchain for use Spike (#395)

* RVV: add cmake toolchain for use Spike

* tests: replace iostream with bare printf

Use of std::cout or std:cerr makes impossible to run programs
under Spike & proxy kernel (`pk`). The initialization of these
standard streams depends on futexes (in glibc) and `pk` does
not implement all system calls.

A side effect is being ready for use the `fmt` library.

* tests: fix ASSERT_EQUAL

* singleheader: don't use cout in amalgamation demo

* Disable testcases that do not run in Spike

---------

Co-authored-by: Daniel Lemire <daniel@lemire.me>

.github/workflows/rvv-1024-clang-17.yml

@@ -21,7 +21,7 @@ jobs:
         sudo apt-get install -y cmake make g++-riscv64-linux-gnu qemu-user-static clang-17
     - name: Build
       run: |
-        CXX=clang++-17 CXXFLAGS="--target=riscv64-linux-gnu -march=rv64gcv" cmake -DQEMU=qemu-riscv64-static -DCMAKE_SYSTEM_PROCESSOR=riscv64 -DCMAKE_BUILD_TYPE=Release -B build
+        CXX=clang++-17 CXXFLAGS="--target=riscv64-linux-gnu -march=rv64gcv" cmake -DCMAKE_CROSSCOMPILING_EMULATOR=qemu-riscv64-static -DCMAKE_SYSTEM_PROCESSOR=riscv64 -DCMAKE_BUILD_TYPE=Release -B build
         cmake --build build/ -j$(nproc)
     - name: Test VLEN=1024
       run: |

.github/workflows/rvv-128-clang-17.yml

@@ -21,7 +21,7 @@ jobs:
         sudo apt-get install -y cmake make g++-riscv64-linux-gnu qemu-user-static clang-17
     - name: Build
       run: |
-        CXX=clang++-17 CXXFLAGS="--target=riscv64-linux-gnu -march=rv64gcv" cmake -DQEMU=qemu-riscv64-static -DCMAKE_SYSTEM_PROCESSOR=riscv64 -DCMAKE_BUILD_TYPE=Release -B build
+        CXX=clang++-17 CXXFLAGS="--target=riscv64-linux-gnu -march=rv64gcv" cmake -DCMAKE_CROSSCOMPILING_EMULATOR=qemu-riscv64-static -DCMAKE_SYSTEM_PROCESSOR=riscv64 -DCMAKE_BUILD_TYPE=Release -B build
         cmake --build build/ -j$(nproc)
     - name: Test VLEN=128
       run: |

README-RVV.md

@@ -0,0 +1,16 @@
+Developing for RISC-V Vector extension (RVV)
+--------------------------------------------------------------------------------
+
+Since RVV is not widely available, we still can check validity of implementation
+using either QEMU or Spike emulator.
+
+To use Spike one has to specify the toolchain:
+
+```
+$ cmake -B rvvbuild -DCMAKE_TOOLCHAIN_FILE=toolchain-rvv-spike.cmake
+$ cmake --build rvvbuild
+$ cd rvvbuild
+```
+
+To use QEMU, please refer to `.github/workflows/rvv-128-clang-17.yml` for
+command line options.

cmake/add_cpp_test.cmake

@@ -30,8 +30,8 @@ function(add_cpp_test TEST_NAME)
     )
     set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
   else()
-    if (QEMU)
-      add_test(${TEST_NAME} ${QEMU} ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME})
+    if (CMAKE_CROSSCOMPILING_EMULATOR)
+      add_test(${TEST_NAME} ${CMAKE_CROSSCOMPILING_EMULATOR} ${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME})
     else()
       add_test(${TEST_NAME} ${TEST_NAME})
     endif()

singleheader/CMakeLists.txt

@@ -50,8 +50,8 @@ if (Python3_Interpreter_FOUND)
   if (SIMDUTF_TESTS)
     add_executable(amalgamation_demo $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/amalgamation_demo.cpp>)
     target_link_libraries(amalgamation_demo simdutf-singleheader-include-source)
-    if (QEMU)
-      add_test(amalgamation_demo ${QEMU} amalgamation_demo)
+    if (CMAKE_CROSSCOMPILING_EMULATOR)
+      add_test(amalgamation_demo ${QCMAKE_CROSSCOMPILING_EMULATOR} amalgamation_demo)
     else()
       add_test(amalgamation_demo amalgamation_demo)
     endif()

singleheader/amalgamation_demo.cpp

@@ -1,4 +1,3 @@
-#include <iostream>
 #include <memory>
 
 #include "simdutf.cpp"
@@ -9,9 +8,9 @@ int main(int , char *[]) {
   // 4 == strlen(source)
   bool validutf8 = simdutf::validate_utf8(source, 4);
   if (validutf8) {
-    std::cout << "valid UTF-8" << std::endl;
+    puts("valid UTF-8");
   } else {
-    std::cerr << "invalid UTF-8" << std::endl;
+    puts("invalid UTF-8");
     return EXIT_FAILURE;
   }
   // We need a buffer of size where to write the UTF-16LE words.
@@ -20,13 +19,13 @@ int main(int , char *[]) {
   // convert to UTF-16LE
   size_t utf16words =
       simdutf::convert_utf8_to_utf16le(source, 4, utf16_output.get());
-  std::cout << "wrote " << utf16words << " UTF-16LE words." << std::endl;
+  printf("wrote %llu UTF-16LE words.", utf16words);
   // It wrote utf16words * sizeof(char16_t) bytes.
   bool validutf16 = simdutf::validate_utf16le(utf16_output.get(), utf16words);
   if (validutf16) {
-    std::cout << "valid UTF-16LE" << std::endl;
+    puts("valid UTF-16LE");
   } else {
-    std::cerr << "invalid UTF-16LE" << std::endl;
+    puts("invalid UTF-16LE");
     return EXIT_FAILURE;
   }
   // convert it back:
@@ -37,14 +36,14 @@ int main(int , char *[]) {
   // convert to UTF-8
   size_t utf8words = simdutf::convert_utf16le_to_utf8(
       utf16_output.get(), utf16words, utf8_output.get());
-  std::cout << "wrote " << utf8words << " UTF-8 words." << std::endl;
+  printf("wrote %llu UTF-8 words.", utf8words);
   std::string final_string(utf8_output.get(), utf8words);
-  std::cout << final_string << std::endl;
+  puts(final_string.c_str());
   if (final_string != source) {
-    std::cerr << "bad conversion" << std::endl;
+    puts("bad conversion");
     return EXIT_FAILURE;
   } else {
-    std::cerr << "perfect round trip" << std::endl;
+    puts("perfect round trip");
   }
   return EXIT_SUCCESS;
 }

tests/base64_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <memory>
 #include <tests/helpers/random_int.h>
@@ -374,8 +373,7 @@ TEST(roundtrip_base64) {
         printf("=====input size %zu\n", len);
         for (size_t i = 0; i < len; i++) {
           if (back[i] != source[i]) {
-            std::cerr << "Mismatch at position " << i << " trial " << trial
-                      << std::endl;
+            printf("Mismatch at position %llu trial %llu\n", i, trial);
           }
           printf("%zu: %02x %02x\n", i, uint8_t(back[i]), uint8_t(source[i]));
         }
@@ -419,8 +417,7 @@ TEST(roundtrip_base64_16) {
         printf("=====input size %zu\n", len);
         for (size_t i = 0; i < len; i++) {
           if (back[i] != source[i]) {
-            std::cerr << "Mismatch at position " << i << " trial " << trial
-                      << std::endl;
+            printf("Mismatch at position %llu trial %llu\n", i, trial);
           }
           printf("%zu: %02x %02x\n", i, uint8_t(back[i]), uint8_t(source[i]));
         }
@@ -460,8 +457,7 @@ TEST(roundtrip_base64url) {
         printf("=====input size %zu\n", len);
         for (size_t i = 0; i < len; i++) {
           if (back[i] != source[i]) {
-            std::cerr << "Mismatch at position " << i << " trial " << trial
-                      << std::endl;
+            printf("Mismatch at position %llu trial %llu\n", i, trial);
           }
           printf("%zu: %02x %02x\n", i, uint8_t(back[i]), uint8_t(source[i]));
         }
@@ -505,8 +501,7 @@ TEST(roundtrip_base64url_16) {
         printf("=====input size %zu\n", len);
         for (size_t i = 0; i < len; i++) {
           if (back[i] != source[i]) {
-            std::cerr << "Mismatch at position " << i << " trial " << trial
-                      << std::endl;
+            printf("Mismatch at position %llu trial %llu\n", i, trial);
           }
           printf("%zu: %02x %02x\n", i, uint8_t(back[i]), uint8_t(source[i]));
         }
@@ -760,8 +755,7 @@ TEST(roundtrip_base64_16_with_spaces) {
         printf("=====input size %zu\n", len);
         for (size_t i = 0; i < len; i++) {
           if (back[i] != source[i]) {
-            std::cerr << "Mismatch at position " << i << " trial " << trial
-                      << std::endl;
+            printf("Mismatch at position %llu trial %llu\n", i, trial);
           }
           printf("%zu: %02x %02x\n", i, uint8_t(back[i]), uint8_t(source[i]));
         }
@@ -1029,17 +1023,15 @@ TEST(readme_test) {
     simdutf::result r = simdutf::base64_to_binary(base64.data() + pos, count,
                                                   back.data() + outpos);
     if (r.error == simdutf::error_code::INVALID_BASE64_CHARACTER) {
-      std::cerr << "Invalid base64 character at position " << pos + r.count
-                << std::endl;
+      printf("Invalid base64 character at position %llu\n", pos + r.count);
       return;
     }
     // If we arrived at the end of the base64 input, we must check that the
     // number of characters processed is a multiple of 4, or that we have a
     // remainder of 0, 2 or 3.
     if (count + pos == base64.size() &&
         r.error == simdutf::error_code::BASE64_INPUT_REMAINDER) {
-      std::cerr << "The base64 input contained an invalid number of characters "
-                << std::endl;
+      puts("The base64 input contained an invalid number of characters");
     }
     // If we are not at then end, we may have to reprocess either 1, 2 or 3
     // bytes, and to drop the last 0, 2 or 3 bytes decoded.

tests/basic_fuzzer.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 #include <fstream>
 #include <stdexcept>
 #include <random>
@@ -55,17 +54,17 @@ SIMDUTF_POP_DISABLE_WARNINGS
   buffer[buf_size - 2] = '"';
   buffer[buf_size - 1] = '\0';
   log << std::boolalpha;
-  log << "Input: " << buffer << std::endl;
-  if (is_ok_utf8.first) { log << "validate_utf8:" << is_ok_utf8.second << std::endl; }
-  if (is_ok_utf16.first) { log << "validate_utf16le:" << is_ok_utf16.second << std::endl; }
-  if (is_ok_utf32.first) { log << "validate_utf32:" << is_ok_utf32.second << std::endl; }
-  if (utf8_to_utf16.first) { log << "convert_utf8_to_utf16le:" << utf8_to_utf16.second << std::endl; }
-  if (utf8_to_utf32.first) { log << "convert_utf8_to_utf32:" << utf8_to_utf32.second << std::endl; }
-  if (utf16_to_utf8.first) { log << "convert_utf16le_to_utf8:" << utf16_to_utf8.second << std::endl; }
-  if (utf16_to_utf32.first) { log << "convert_utf16le_to_utf32:" << utf16_to_utf32.second << std::endl; }
-  if (utf32_to_utf8.first) { log << "convert_utf32_to_utf8:" << utf32_to_utf8.second << std::endl; }
-  if (utf32_to_utf16.first) { log << "convert_utf32_to_utf16le:" << utf32_to_utf16.second << std::endl; }
-  log << std::endl;
+  log << "Input: " << buffer << '\n';
+  if (is_ok_utf8.first) { log << "validate_utf8:" << is_ok_utf8.second << '\n'; }
+  if (is_ok_utf16.first) { log << "validate_utf16le:" << is_ok_utf16.second << '\n'; }
+  if (is_ok_utf32.first) { log << "validate_utf32:" << is_ok_utf32.second << '\n'; }
+  if (utf8_to_utf16.first) { log << "convert_utf8_to_utf16le:" << utf8_to_utf16.second << '\n'; }
+  if (utf8_to_utf32.first) { log << "convert_utf8_to_utf32:" << utf8_to_utf32.second << '\n'; }
+  if (utf16_to_utf8.first) { log << "convert_utf16le_to_utf8:" << utf16_to_utf8.second << '\n'; }
+  if (utf16_to_utf32.first) { log << "convert_utf16le_to_utf32:" << utf16_to_utf32.second << '\n'; }
+  if (utf32_to_utf8.first) { log << "convert_utf32_to_utf8:" << utf32_to_utf8.second << '\n'; }
+  if (utf32_to_utf16.first) { log << "convert_utf32_to_utf16le:" << utf32_to_utf16.second << '\n'; }
+  log << '\n';
   log.close();
 }
 }
@@ -633,24 +632,24 @@ TEST(basic_fuzz) {
             reinterpret_cast<char32_t *>(input.data()),
             input.size() / sizeof(char32_t), reinterpret_cast<char16_t *>(output.data()));
         if(is_ok_utf8.second ? (utf8_to_utf16.second == 0 || utf8_to_utf32.second == 0) : (utf8_to_utf16.second > 0 || utf8_to_utf32.second > 0)) {
-          std::cout << (is_ok_utf8.second ? "UTF-8 is ok" : "UTF-8 is not ok") << std::endl;
-          std::cout << " size = " << input.size() << std::endl;
-          std::cout << "  implementation.convert_utf8_to_utf16.second return " << utf8_to_utf16.second << std::endl;
-          std::cout << "  implementation.convert_utf8_to_utf32.second return " << utf8_to_utf32.second << std::endl;
+          printf("%s\n", (is_ok_utf8.second ? "UTF-8 is ok" : "UTF-8 is not ok"));
+          printf(" size = %llu\n", input.size());
+          printf("  implementation.convert_utf8_to_utf16.second return %llu\n", utf8_to_utf16.second);
+          printf("  implementation.convert_utf8_to_utf32.second return %lli\n", utf8_to_utf32.second);
         }
         ASSERT_TRUE(is_ok_utf8.second ? (utf8_to_utf16.second > 0 && utf8_to_utf32.second > 0) : (utf8_to_utf16.second == 0 && utf8_to_utf32.second == 0));
         if(is_ok_utf16.second ? (utf16_to_utf8.second == 0 || utf16_to_utf32.second == 0) : (utf16_to_utf8.second > 0 || utf16_to_utf32.second > 0)) {
-          std::cout << (is_ok_utf16.second ? "UTF-16 is ok" : "UTF-16 is not ok") << std::endl;
-          std::cout << " size = " << input.size() / sizeof(char16_t) << std::endl;
-          std::cout << "  implementation.convert_utf16_to_utf8.second return " << utf16_to_utf8.second << std::endl;
-          std::cout << "  implementation.convert_utf16_to_utf32.second return " << utf16_to_utf32.second << std::endl;
+          printf("%s\n", (is_ok_utf16.second ? "UTF-16 is ok" : "UTF-16 is not ok"));
+          printf(" size = %llu\n", input.size() / sizeof(char16_t));
+          printf("  implementation.convert_utf16_to_utf8.second return %llu\n", utf16_to_utf8.second);
+          printf("  implementation.convert_utf16_to_utf32.second return %llu\n",utf16_to_utf32.second);
         }
         ASSERT_TRUE(is_ok_utf16.second ? (utf16_to_utf8.second > 0 && utf16_to_utf32.second > 0) : (utf16_to_utf8.second == 0 && utf16_to_utf32.second == 0));
         if(is_ok_utf32.second ? (utf32_to_utf8.second == 0 || utf32_to_utf16.second == 0) : (utf32_to_utf8.second > 0 || utf32_to_utf16.second > 0)) {
-          std::cout << (is_ok_utf32.second ? "UTF-32 is ok" : "UTF-32 is not ok") << std::endl;
-          std::cout << " size = " << input.size() / sizeof(char32_t) << std::endl;
-          std::cout << "  implementation.convert_utf32_to_utf8.second return " << utf32_to_utf8.second << std::endl;
-          std::cout << "  implementation.convert_utf32_to_utf16.second return " << utf32_to_utf16.second << std::endl;
+          printf("%s\n", (is_ok_utf32.second ? "UTF-32 is ok" : "UTF-32 is not ok"));
+          printf(" size = %llu\n", input.size() / sizeof(char32_t));
+          printf("  implementation.convert_utf32_to_utf8.second return %llu\n", utf32_to_utf8.second);
+          printf("  implementation.convert_utf32_to_utf16.second return %llu\n", utf32_to_utf16.second);
         }
         ASSERT_TRUE(is_ok_utf32.second ? (utf32_to_utf8.second > 0 && utf32_to_utf16.second > 0) : (utf32_to_utf8.second == 0 && utf32_to_utf16.second == 0));
       }

tests/convert_latin1_to_utf16be_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf32.h>
 #include <tests/reference/decode_utf32.h>

tests/convert_latin1_to_utf16le_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf32.h>
 #include <tests/reference/decode_utf32.h>

tests/convert_latin1_to_utf32_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/helpers/transcode_test_base.h>
 #include <tests/helpers/random_int.h>

tests/convert_latin1_to_utf8_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/helpers/transcode_test_base.h>
 #include <tests/helpers/random_int.h>

tests/convert_utf16be_to_latin1_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16be_to_latin1_tests_with_errors.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16be_to_utf32_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16be_to_utf32_with_errors_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16be_to_utf8_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16be_to_utf8_with_errors_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16le_to_latin1_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16le_to_latin1_tests_with_errors.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16le_to_utf32_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16le_to_utf32_with_errors_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16le_to_utf8_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf16le_to_utf8_with_errors_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/reference/validate_utf16.h>
 #include <tests/reference/decode_utf16.h>

tests/convert_utf32_to_latin1_tests.cpp

@@ -1,7 +1,6 @@
 #include "simdutf.h"
 
 #include <array>
-#include <iostream>
 
 #include <tests/helpers/transcode_test_base.h>
 #include <tests/helpers/random_int.h>