8317721: RISC-V: Implement CRC32 intrinsic

Reviewed-by: vkempik, rehn

Author: Arseny Bochkarev

src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp

@@ -1607,7 +1607,22 @@ void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
   __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
 }
 
-void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); }
+void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
+  assert(op->crc()->is_single_cpu(),  "crc must be register");
+  assert(op->val()->is_single_cpu(),  "byte value must be register");
+  assert(op->result_opr()->is_single_cpu(), "result must be register");
+  Register crc = op->crc()->as_register();
+  Register val = op->val()->as_register();
+  Register res = op->result_opr()->as_register();
+
+  assert_different_registers(val, crc, res);
+  __ la(res, ExternalAddress(StubRoutines::crc_table_addr()));
+
+  __ notr(crc, crc); // ~crc
+  __ zero_extend(crc, crc, 32);
+  __ update_byte_crc32(crc, val, res);
+  __ notr(res, crc); // ~crc
+}
 
 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass,
                                    Register tmp, Label &next, Label &none,

src/hotspot/cpu/riscv/c1_LIRGenerator_riscv.cpp

@@ -781,7 +781,79 @@ void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
 }
 
 void LIRGenerator::do_update_CRC32(Intrinsic* x) {
-  ShouldNotReachHere();
+  assert(UseCRC32Intrinsics, "why are we here?");
+  // Make all state_for calls early since they can emit code
+  LIR_Opr result = rlock_result(x);
+  switch (x->id()) {
+    case vmIntrinsics::_updateCRC32: {
+      LIRItem crc(x->argument_at(0), this);
+      LIRItem val(x->argument_at(1), this);
+      // val is destroyed by update_crc32
+      val.set_destroys_register();
+      crc.load_item();
+      val.load_item();
+      __ update_crc32(crc.result(), val.result(), result);
+      break;
+    }
+    case vmIntrinsics::_updateBytesCRC32:
+    case vmIntrinsics::_updateByteBufferCRC32: {
+      bool is_updateBytes = (x->id() == vmIntrinsics::_updateBytesCRC32);
+
+      LIRItem crc(x->argument_at(0), this);
+      LIRItem buf(x->argument_at(1), this);
+      LIRItem off(x->argument_at(2), this);
+      LIRItem len(x->argument_at(3), this);
+      buf.load_item();
+      off.load_nonconstant();
+
+      LIR_Opr index = off.result();
+      int offset = is_updateBytes ? arrayOopDesc::base_offset_in_bytes(T_BYTE) : 0;
+      if (off.result()->is_constant()) {
+        index = LIR_OprFact::illegalOpr;
+        offset += off.result()->as_jint();
+      }
+      LIR_Opr base_op = buf.result();
+
+      if (index->is_valid()) {
+        LIR_Opr tmp = new_register(T_LONG);
+        __ convert(Bytecodes::_i2l, index, tmp);
+        index = tmp;
+      }
+
+      if (offset) {
+        LIR_Opr tmp = new_pointer_register();
+        __ add(base_op, LIR_OprFact::intConst(offset), tmp);
+        base_op = tmp;
+        offset = 0;
+      }
+
+      LIR_Address* a = new LIR_Address(base_op,
+                                       index,
+                                       offset,
+                                       T_BYTE);
+      BasicTypeList signature(3);
+      signature.append(T_INT);
+      signature.append(T_ADDRESS);
+      signature.append(T_INT);
+      CallingConvention* cc = frame_map()->c_calling_convention(&signature);
+      const LIR_Opr result_reg = result_register_for(x->type());
+
+      LIR_Opr addr = new_pointer_register();
+      __ leal(LIR_OprFact::address(a), addr);
+
+      crc.load_item_force(cc->at(0));
+      __ move(addr, cc->at(1));
+      len.load_item_force(cc->at(2));
+
+      __ call_runtime_leaf(StubRoutines::updateBytesCRC32(), getThreadTemp(), result_reg, cc->args());
+      __ move(result_reg, result);
+
+      break;
+    }
+    default: {
+      ShouldNotReachHere();
+    }
+  }
 }
 
 void LIRGenerator::do_update_CRC32C(Intrinsic* x) {

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -1428,6 +1428,175 @@ int MacroAssembler::pop_fp(unsigned int bitset, Register stack) {
   return count;
 }
 
+static const int64_t right_32_bits = right_n_bits(32);
+static const int64_t right_8_bits = right_n_bits(8);
+
+/**
+ * Emits code to update CRC-32 with a byte value according to constants in table
+ *
+ * @param [in,out]crc   Register containing the crc.
+ * @param [in]val       Register containing the byte to fold into the CRC.
+ * @param [in]table     Register containing the table of crc constants.
+ *
+ * uint32_t crc;
+ * val = crc_table[(val ^ crc) & 0xFF];
+ * crc = val ^ (crc >> 8);
+ *
+ */
+void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
+  assert_different_registers(crc, val, table);
+
+  xorr(val, val, crc);
+  andi(val, val, right_8_bits);
+  shadd(val, val, table, val, 2);
+  lwu(val, Address(val));
+  srli(crc, crc, 8);
+  xorr(crc, val, crc);
+}
+
+/**
+ * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3
+ *
+ * @param [in,out]crc   Register containing the crc.
+ * @param [in]v         Register containing the 32-bit to fold into the CRC.
+ * @param [in]table0    Register containing table 0 of crc constants.
+ * @param [in]table1    Register containing table 1 of crc constants.
+ * @param [in]table2    Register containing table 2 of crc constants.
+ * @param [in]table3    Register containing table 3 of crc constants.
+ *
+ * uint32_t crc;
+ *   v = crc ^ v
+ *   crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24]
+ *
+ */
+void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp1, Register tmp2, Register tmp3,
+        Register table0, Register table1, Register table2, Register table3, bool upper) {
+  assert_different_registers(crc, v, tmp1, tmp2, tmp3, table0, table1, table2, table3);
+
+  if (upper)
+    srli(v, v, 32);
+  xorr(v, v, crc);
+
+  andi(tmp1, v, right_8_bits);
+  shadd(tmp1, tmp1, table3, tmp2, 2);
+  lwu(crc, Address(tmp1));
+
+  slli(tmp1, v, 16);
+  slli(tmp3, v, 8);
+
+  srliw(tmp1, tmp1, 24);
+  srliw(tmp3, tmp3, 24);
+
+  shadd(tmp1, tmp1, table2, tmp1, 2);
+  lwu(tmp2, Address(tmp1));
+
+  shadd(tmp3, tmp3, table1, tmp3, 2);
+  xorr(crc, crc, tmp2);
+
+  lwu(tmp2, Address(tmp3));
+  if (upper) {
+    tmp1 = v;
+    srli(tmp1, v, 24);
+  }
+  else
+    srliw(tmp1, v, 24);
+
+  // no need to clear bits other than lowest two
+  shadd(tmp1, tmp1, table0, tmp1, 2);
+  xorr(crc, crc, tmp2);
+  lwu(tmp2, Address(tmp1));
+  xorr(crc, crc, tmp2);
+}
+
+/**
+ * @param crc   register containing existing CRC (32-bit)
+ * @param buf   register pointing to input byte buffer (byte*)
+ * @param len   register containing number of bytes
+ * @param table register that will contain address of CRC table
+ * @param tmp   scratch registers
+ */
+void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
+        Register table0, Register table1, Register table2, Register table3,
+        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6) {
+  assert_different_registers(crc, buf, len, table0, table1, table2, table3, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
+  Label L_by16_loop, L_unroll_loop, L_unroll_loop_entry, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit;
+
+  const int64_t unroll = 16;
+  const int64_t unroll_words = unroll*wordSize;
+  mv(tmp5, right_32_bits);
+  subw(len, len, unroll_words);
+  andn(crc, tmp5, crc);
+
+  const ExternalAddress table_addr = StubRoutines::crc_table_addr();
+  la(table0, table_addr);
+  add(table1, table0, 1*256*sizeof(juint), tmp1);
+  add(table2, table0, 2*256*sizeof(juint), tmp1);
+  add(table3, table2, 1*256*sizeof(juint), tmp1);
+
+  bge(len, zr, L_unroll_loop_entry);
+  addiw(len, len, unroll_words-4);
+  bge(len, zr, L_by4_loop);
+  addiw(len, len, 4);
+  bgt(len, zr, L_by1_loop);
+  j(L_exit);
+
+  align(CodeEntryAlignment);
+  bind(L_unroll_loop_entry);
+    const Register buf_end = tmp3;
+    add(buf_end, buf, len); // buf_end will be used as endpoint for loop below
+    andi(len, len, unroll_words-1); // len = (len % unroll_words)
+    sub(len, len, unroll_words); // Length after all iterations
+  bind(L_unroll_loop);
+    for (int i = 0; i < unroll; i++) {
+      ld(tmp1, Address(buf, i*wordSize));
+      update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, false);
+      update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, true);
+    }
+
+    addi(buf, buf, unroll_words);
+    ble(buf, buf_end, L_unroll_loop);
+    addiw(len, len, unroll_words-4);
+    bge(len, zr, L_by4_loop);
+    addiw(len, len, 4);
+    bgt(len, zr, L_by1_loop);
+    j(L_exit);
+
+  bind(L_by4_loop);
+    lwu(tmp1, Address(buf));
+    update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, false);
+    subw(len, len, 4);
+    addi(buf, buf, 4);
+    bge(len, zr, L_by4_loop);
+    addiw(len, len, 4);
+    ble(len, zr, L_exit);
+
+  bind(L_by1_loop);
+    subw(len, len, 1);
+    lwu(tmp1, Address(buf));
+    andi(tmp2, tmp1, right_8_bits);
+    update_byte_crc32(crc, tmp2, table0);
+    ble(len, zr, L_exit);
+
+    subw(len, len, 1);
+    srli(tmp2, tmp1, 8);
+    andi(tmp2, tmp2, right_8_bits);
+    update_byte_crc32(crc, tmp2, table0);
+    ble(len, zr, L_exit);
+
+    subw(len, len, 1);
+    srli(tmp2, tmp1, 16);
+    andi(tmp2, tmp2, right_8_bits);
+    update_byte_crc32(crc, tmp2, table0);
+    ble(len, zr, L_exit);
+
+    srli(tmp2, tmp1, 24);
+    andi(tmp2, tmp2, right_8_bits);
+    update_byte_crc32(crc, tmp2, table0);
+
+  bind(L_exit);
+    andn(crc, tmp5, crc);
+}
+
 #ifdef COMPILER2
 // Push vector registers in the bitset supplied.
 // Return the number of words pushed

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -1288,6 +1288,15 @@ class MacroAssembler: public Assembler {
   void compute_match_mask(Register src, Register pattern, Register match_mask,
                           Register mask1, Register mask2);
 
+  // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
+  void kernel_crc32(Register crc, Register buf, Register len,
+        Register table0, Register table1, Register table2, Register table3,
+        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6);
+  void update_word_crc32(Register crc, Register v, Register tmp1, Register tmp2, Register tmp3,
+        Register table0, Register table1, Register table2, Register table3,
+        bool upper);
+  void update_byte_crc32(Register crc, Register val, Register table);
+
 #ifdef COMPILER2
   void mul_add(Register out, Register in, Register offset,
                Register len, Register k, Register tmp);
@@ -1317,6 +1326,7 @@ class MacroAssembler: public Assembler {
                        Register z, Register tmp0,
                        Register tmp1, Register tmp2, Register tmp3, Register tmp4,
                        Register tmp5, Register tmp6, Register product_hi);
+
 #endif
 
   void inflate_lo32(Register Rd, Register Rs, Register tmp1 = t0, Register tmp2 = t1);

src/hotspot/cpu/riscv/stubGenerator_riscv.cpp

@@ -5313,6 +5313,52 @@ static const int64_t right_3_bits = right_n_bits(3);
 
 #endif // COMPILER2
 
+  /**
+   *  Arguments:
+   *
+   * Inputs:
+   *   c_rarg0   - int crc
+   *   c_rarg1   - byte* buf
+   *   c_rarg2   - int length
+   *
+   * Output:
+   *   c_rarg0   - int crc result
+   */
+  address generate_updateBytesCRC32() {
+    assert(UseCRC32Intrinsics, "what are we doing here?");
+
+    __ align(CodeEntryAlignment);
+    StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
+
+    address start = __ pc();
+
+    const Register crc    = c_rarg0;  // crc
+    const Register buf    = c_rarg1;  // source java byte array address
+    const Register len    = c_rarg2;  // length
+    const Register table0 = c_rarg3;  // crc_table address
+    const Register table1 = c_rarg4;
+    const Register table2 = c_rarg5;
+    const Register table3 = c_rarg6;
+
+    const Register tmp1 = c_rarg7;
+    const Register tmp2 = t2;
+    const Register tmp3 = x28; // t3
+    const Register tmp4 = x29; // t4
+    const Register tmp5 = x30; // t5
+    const Register tmp6 = x31; // t6
+
+    BLOCK_COMMENT("Entry:");
+    __ enter(); // required for proper stackwalking of RuntimeStub frame
+
+    __ kernel_crc32(crc, buf, len, table0, table1, table2,
+                    table3, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
+
+    __ leave(); // required for proper stackwalking of RuntimeStub frame
+    __ ret();
+
+    return start;
+  }
+
 #if INCLUDE_JFR
 
   static void jfr_prologue(address the_pc, MacroAssembler* _masm, Register thread) {
@@ -5559,6 +5605,12 @@ static const int64_t right_3_bits = right_n_bits(3);
       generate_throw_exception("delayed StackOverflowError throw_exception",
                                CAST_FROM_FN_PTR(address,
                                                 SharedRuntime::throw_delayed_StackOverflowError));
+
+    if (UseCRC32Intrinsics) {
+      // set table address before stub generation which use it
+      StubRoutines::_crc_table_adr = (address)StubRoutines::riscv::_crc_table;
+      StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
+    }
   }
 
   void generate_continuation_stubs() {