86 changes: 62 additions & 24 deletions target/arm/cpu64.c
View file
Expand Up @@ -473,43 +473,80 @@ void aarch64_add_sme_properties(Object *obj)

void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp)
{
int arch_val = 0, impdef_val = 0;
uint64_t t;
ARMPauthFeature features = cpu_isar_feature(pauth_feature, cpu);
uint64_t isar1, isar2;

/*
* These properties enable or disable Pauth as a whole, or change
* the pauth algorithm, but do not change the set of features that
* are present. We have saved a copy of those features above and
* will now place it into the field that chooses the algorithm.
*
* Begin by disabling all fields.
*/
isar1 = cpu->isar.id_aa64isar1;
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, APA, 0);
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, GPA, 0);
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, API, 0);
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, GPI, 0);

isar2 = cpu->isar.id_aa64isar2;
isar2 = FIELD_DP64(isar2, ID_AA64ISAR2, APA3, 0);
isar2 = FIELD_DP64(isar2, ID_AA64ISAR2, GPA3, 0);

/* Exit early if PAuth is enabled, and fall through to disable it */
if ((kvm_enabled() || hvf_enabled()) && cpu->prop_pauth) {
if (!cpu_isar_feature(aa64_pauth, cpu)) {
error_setg(errp, "'pauth' feature not supported by %s on this host",
kvm_enabled() ? "KVM" : "hvf");
if (kvm_enabled() || hvf_enabled()) {
/*
* Exit early if PAuth is enabled and fall through to disable it.
* The algorithm selection properties are not present.
*/
if (cpu->prop_pauth) {
if (features == 0) {
error_setg(errp, "'pauth' feature not supported by "
"%s on this host", current_accel_name());
}
return;
}
} else {
/* Pauth properties are only present when the model supports it. */
if (features == 0) {
assert(!cpu->prop_pauth);
return;
}

return;
}
if (cpu->prop_pauth) {
if (cpu->prop_pauth_impdef && cpu->prop_pauth_qarma3) {
error_setg(errp,
"cannot enable both pauth-impdef and pauth-qarma3");
return;
}

/* TODO: Handle HaveEnhancedPAC, HaveEnhancedPAC2, HaveFPAC. */
if (cpu->prop_pauth) {
if (cpu->prop_pauth_impdef) {
impdef_val = 1;
} else {
arch_val = 1;
if (cpu->prop_pauth_impdef) {
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, API, features);
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, GPI, 1);
} else if (cpu->prop_pauth_qarma3) {
isar2 = FIELD_DP64(isar2, ID_AA64ISAR2, APA3, features);
isar2 = FIELD_DP64(isar2, ID_AA64ISAR2, GPA3, 1);
} else {
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, APA, features);
isar1 = FIELD_DP64(isar1, ID_AA64ISAR1, GPA, 1);
}
} else if (cpu->prop_pauth_impdef || cpu->prop_pauth_qarma3) {
error_setg(errp, "cannot enable pauth-impdef or "
"pauth-qarma3 without pauth");
error_append_hint(errp, "Add pauth=on to the CPU property list.\n");
}
} else if (cpu->prop_pauth_impdef) {
error_setg(errp, "cannot enable pauth-impdef without pauth");
error_append_hint(errp, "Add pauth=on to the CPU property list.\n");
}

t = cpu->isar.id_aa64isar1;
t = FIELD_DP64(t, ID_AA64ISAR1, APA, arch_val);
t = FIELD_DP64(t, ID_AA64ISAR1, GPA, arch_val);
t = FIELD_DP64(t, ID_AA64ISAR1, API, impdef_val);
t = FIELD_DP64(t, ID_AA64ISAR1, GPI, impdef_val);
cpu->isar.id_aa64isar1 = t;
cpu->isar.id_aa64isar1 = isar1;
cpu->isar.id_aa64isar2 = isar2;
}

static Property arm_cpu_pauth_property =
DEFINE_PROP_BOOL("pauth", ARMCPU, prop_pauth, true);
static Property arm_cpu_pauth_impdef_property =
DEFINE_PROP_BOOL("pauth-impdef", ARMCPU, prop_pauth_impdef, false);
static Property arm_cpu_pauth_qarma3_property =
DEFINE_PROP_BOOL("pauth-qarma3", ARMCPU, prop_pauth_qarma3, false);

void aarch64_add_pauth_properties(Object *obj)
{
Expand All @@ -529,6 +566,7 @@ void aarch64_add_pauth_properties(Object *obj)
cpu->prop_pauth = cpu_isar_feature(aa64_pauth, cpu);
} else {
qdev_property_add_static(DEVICE(obj), &arm_cpu_pauth_impdef_property);
qdev_property_add_static(DEVICE(obj), &arm_cpu_pauth_qarma3_property);
}
}

Expand Down
68 changes: 43 additions & 25 deletions target/arm/helper.c
View file
Expand Up @@ -8435,11 +8435,11 @@ void register_cp_regs_for_features(ARMCPU *cpu)
.access = PL1_R, .type = ARM_CP_CONST,
.accessfn = access_aa64_tid3,
.resetvalue = cpu->isar.id_aa64isar1 },
{ .name = "ID_AA64ISAR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
{ .name = "ID_AA64ISAR2_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
.access = PL1_R, .type = ARM_CP_CONST,
.accessfn = access_aa64_tid3,
.resetvalue = 0 },
.resetvalue = cpu->isar.id_aa64isar2 },
{ .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
.access = PL1_R, .type = ARM_CP_CONST,
Expand Down Expand Up @@ -8682,16 +8682,25 @@ void register_cp_regs_for_features(ARMCPU *cpu)
};
modify_arm_cp_regs(v8_idregs, v8_user_idregs);
#endif
/* RVBAR_EL1 is only implemented if EL1 is the highest EL */
/*
* RVBAR_EL1 and RMR_EL1 only implemented if EL1 is the highest EL.
* TODO: For RMR, a write with bit 1 set should do something with
* cpu_reset(). In the meantime, "the bit is strictly a request",
* so we are in spec just ignoring writes.
*/
if (!arm_feature(env, ARM_FEATURE_EL3) &&
!arm_feature(env, ARM_FEATURE_EL2)) {
ARMCPRegInfo rvbar = {
.name = "RVBAR_EL1", .state = ARM_CP_STATE_BOTH,
.opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL1_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar),
ARMCPRegInfo el1_reset_regs[] = {
{ .name = "RVBAR_EL1", .state = ARM_CP_STATE_BOTH,
.opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL1_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar) },
{ .name = "RMR_EL1", .state = ARM_CP_STATE_BOTH,
.opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
.access = PL1_RW, .type = ARM_CP_CONST,
.resetvalue = arm_feature(env, ARM_FEATURE_AARCH64) }
};
define_one_arm_cp_reg(cpu, &rvbar);
define_arm_cp_regs(cpu, el1_reset_regs);
}
define_arm_cp_regs(cpu, v8_idregs);
define_arm_cp_regs(cpu, v8_cp_reginfo);
Expand Down Expand Up @@ -8775,22 +8784,25 @@ void register_cp_regs_for_features(ARMCPU *cpu)
if (cpu_isar_feature(aa64_sel2, cpu)) {
define_arm_cp_regs(cpu, el2_sec_cp_reginfo);
}
/* RVBAR_EL2 is only implemented if EL2 is the highest EL */
/*
* RVBAR_EL2 and RMR_EL2 only implemented if EL2 is the highest EL.
* See commentary near RMR_EL1.
*/
if (!arm_feature(env, ARM_FEATURE_EL3)) {
ARMCPRegInfo rvbar[] = {
{
.name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL2_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar),
},
{ .name = "RVBAR", .type = ARM_CP_ALIAS,
.cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL2_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar),
},
static const ARMCPRegInfo el2_reset_regs[] = {
{ .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL2_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar) },
{ .name = "RVBAR", .type = ARM_CP_ALIAS,
.cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL2_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar) },
{ .name = "RMR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 2,
.access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 1 },
};
define_arm_cp_regs(cpu, rvbar);
define_arm_cp_regs(cpu, el2_reset_regs);
}
}

Expand All @@ -8801,8 +8813,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
{ .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
.access = PL3_R,
.fieldoffset = offsetof(CPUARMState, cp15.rvbar),
},
.fieldoffset = offsetof(CPUARMState, cp15.rvbar), },
{ .name = "RMR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 2,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 1 },
{ .name = "RMR", .state = ARM_CP_STATE_AA32,
.cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
.access = PL3_RW, .type = ARM_CP_CONST,
.resetvalue = arm_feature(env, ARM_FEATURE_AARCH64) },
{ .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
.access = PL3_RW,
Expand Down
2 changes: 2 additions & 0 deletions target/arm/helper.h
View file
Expand Up @@ -81,6 +81,8 @@ DEF_HELPER_FLAGS_2(check_bxj_trap, TCG_CALL_NO_WG, void, env, i32)

DEF_HELPER_4(access_check_cp_reg, cptr, env, i32, i32, i32)
DEF_HELPER_FLAGS_2(lookup_cp_reg, TCG_CALL_NO_RWG_SE, cptr, env, i32)
DEF_HELPER_FLAGS_2(tidcp_el0, TCG_CALL_NO_WG, void, env, i32)
DEF_HELPER_FLAGS_2(tidcp_el1, TCG_CALL_NO_WG, void, env, i32)
DEF_HELPER_3(set_cp_reg, void, env, cptr, i32)
DEF_HELPER_2(get_cp_reg, i32, env, cptr)
DEF_HELPER_3(set_cp_reg64, void, env, cptr, i64)
Expand Down
1 change: 1 addition & 0 deletions target/arm/hvf/hvf.c
View file
Expand Up @@ -847,6 +847,7 @@ static bool hvf_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
{ HV_SYS_REG_ID_AA64DFR1_EL1, &host_isar.id_aa64dfr1 },
{ HV_SYS_REG_ID_AA64ISAR0_EL1, &host_isar.id_aa64isar0 },
{ HV_SYS_REG_ID_AA64ISAR1_EL1, &host_isar.id_aa64isar1 },
/* Add ID_AA64ISAR2_EL1 here when HVF supports it */
{ HV_SYS_REG_ID_AA64MMFR0_EL1, &host_isar.id_aa64mmfr0 },
{ HV_SYS_REG_ID_AA64MMFR1_EL1, &host_isar.id_aa64mmfr1 },
{ HV_SYS_REG_ID_AA64MMFR2_EL1, &host_isar.id_aa64mmfr2 },
Expand Down
61 changes: 61 additions & 0 deletions target/arm/kvm.c
View file
Expand Up @@ -30,6 +30,7 @@
#include "exec/address-spaces.h"
#include "hw/boards.h"
#include "hw/irq.h"
#include "qapi/visitor.h"
#include "qemu/log.h"

const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
Expand Down Expand Up @@ -287,6 +288,26 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
}
}

if (s->kvm_eager_split_size) {
uint32_t sizes;

sizes = kvm_vm_check_extension(s, KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES);
if (!sizes) {
s->kvm_eager_split_size = 0;
warn_report("Eager Page Split support not available");
} else if (!(s->kvm_eager_split_size & sizes)) {
error_report("Eager Page Split requested chunk size not valid");
ret = -EINVAL;
} else {
ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE, 0,
s->kvm_eager_split_size);
if (ret < 0) {
error_report("Enabling of Eager Page Split failed: %s",
strerror(-ret));
}
}
}

kvm_arm_init_debug(s);

return ret;
Expand Down Expand Up @@ -1069,6 +1090,46 @@ bool kvm_arch_cpu_check_are_resettable(void)
return true;
}

static void kvm_arch_get_eager_split_size(Object *obj, Visitor *v,
const char *name, void *opaque,
Error **errp)
{
KVMState *s = KVM_STATE(obj);
uint64_t value = s->kvm_eager_split_size;

visit_type_size(v, name, &value, errp);
}

static void kvm_arch_set_eager_split_size(Object *obj, Visitor *v,
const char *name, void *opaque,
Error **errp)
{
KVMState *s = KVM_STATE(obj);
uint64_t value;

if (s->fd != -1) {
error_setg(errp, "Unable to set early-split-size after KVM has been initialized");
return;
}

if (!visit_type_size(v, name, &value, errp)) {
return;
}

if (value && !is_power_of_2(value)) {
error_setg(errp, "early-split-size must be a power of two");
return;
}

s->kvm_eager_split_size = value;
}

void kvm_arch_accel_class_init(ObjectClass *oc)
{
object_class_property_add(oc, "eager-split-size", "size",
kvm_arch_get_eager_split_size,
kvm_arch_set_eager_split_size, NULL, NULL);

object_class_property_set_description(oc, "eager-split-size",
"Eager Page Split chunk size for hugepages. (default: 0, disabled)");
}
3 changes: 3 additions & 0 deletions target/arm/kvm64.c
View file
Expand Up @@ -304,6 +304,8 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
ARM64_SYS_REG(3, 0, 0, 6, 0));
err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1,
ARM64_SYS_REG(3, 0, 0, 6, 1));
err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar2,
ARM64_SYS_REG(3, 0, 0, 6, 2));
err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0,
ARM64_SYS_REG(3, 0, 0, 7, 0));
err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1,
Expand Down Expand Up @@ -672,6 +674,7 @@ typedef struct CPRegStateLevel {
*/
static const CPRegStateLevel non_runtime_cpregs[] = {
{ KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE },
{ KVM_REG_ARM_PTIMER_CNT, KVM_PUT_FULL_STATE },
};

int kvm_arm_cpreg_level(uint64_t regidx)
Expand Down
7 changes: 7 additions & 0 deletions target/arm/syndrome.h
View file
Expand Up @@ -49,6 +49,7 @@ enum arm_exception_class {
EC_SYSTEMREGISTERTRAP = 0x18,
EC_SVEACCESSTRAP = 0x19,
EC_ERETTRAP = 0x1a,
EC_PACFAIL = 0x1c,
EC_SMETRAP = 0x1d,
EC_GPC = 0x1e,
EC_INSNABORT = 0x20,
Expand Down Expand Up @@ -232,6 +233,12 @@ static inline uint32_t syn_smetrap(SMEExceptionType etype, bool is_16bit)
| (is_16bit ? 0 : ARM_EL_IL) | etype;
}

static inline uint32_t syn_pacfail(bool data, int keynumber)
{
int error_code = (data << 1) | keynumber;
return (EC_PACFAIL << ARM_EL_EC_SHIFT) | ARM_EL_IL | error_code;
}

static inline uint32_t syn_pactrap(void)
{
return EC_PACTRAP << ARM_EL_EC_SHIFT;
Expand Down
215 changes: 215 additions & 0 deletions target/arm/tcg/cpu64.c
View file
Expand Up @@ -745,6 +745,217 @@ static void aarch64_neoverse_v1_initfn(Object *obj)
aarch64_add_sve_properties(obj);
}

static const ARMCPRegInfo cortex_a710_cp_reginfo[] = {
{ .name = "CPUACTLR_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 1, .opc2 = 0,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUACTLR2_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 1, .opc2 = 1,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUACTLR3_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 1, .opc2 = 2,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUACTLR4_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 1, .opc2 = 3,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUECTLR_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 1, .opc2 = 4,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUECTLR2_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 1, .opc2 = 5,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUPPMCR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 2, .opc2 = 4,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPWRCTLR_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 2, .opc2 = 7,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "ATCR_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 7, .opc2 = 0,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUACTLR5_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 8, .opc2 = 0,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUACTLR6_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 8, .opc2 = 1,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "CPUACTLR7_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 15, .crm = 8, .opc2 = 2,
.access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0,
.accessfn = access_actlr_w },
{ .name = "ATCR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 15, .crm = 7, .opc2 = 0,
.access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "AVTCR_EL2", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 4, .crn = 15, .crm = 7, .opc2 = 1,
.access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPPMCR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 2, .opc2 = 0,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPPMCR2_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 2, .opc2 = 1,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPPMCR4_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 2, .opc2 = 4,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPPMCR5_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 2, .opc2 = 5,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPPMCR6_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 2, .opc2 = 6,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUACTLR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 4, .opc2 = 0,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "ATCR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 7, .opc2 = 0,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPSELR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 0,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPCR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 1,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPOR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 2,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPMR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 3,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPOR2_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 4,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPMR2_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 5,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "CPUPFR_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 8, .opc2 = 6,
.access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 0 },

/*
* Stub RAMINDEX, as we don't actually implement caches, BTB,
* or anything else with cpu internal memory.
* "Read" zeros into the IDATA* and DDATA* output registers.
*/
{ .name = "RAMINDEX_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 6, .crn = 15, .crm = 0, .opc2 = 0,
.access = PL3_W, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "IDATA0_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 0, .opc2 = 0,
.access = PL3_R, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "IDATA1_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 0, .opc2 = 1,
.access = PL3_R, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "IDATA2_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 0, .opc2 = 2,
.access = PL3_R, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "DDATA0_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 1, .opc2 = 0,
.access = PL3_R, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "DDATA1_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 1, .opc2 = 1,
.access = PL3_R, .type = ARM_CP_CONST, .resetvalue = 0 },
{ .name = "DDATA2_EL3", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 6, .crn = 15, .crm = 1, .opc2 = 2,
.access = PL3_R, .type = ARM_CP_CONST, .resetvalue = 0 },
};

static void aarch64_a710_initfn(Object *obj)
{
ARMCPU *cpu = ARM_CPU(obj);

cpu->dtb_compatible = "arm,cortex-a710";
set_feature(&cpu->env, ARM_FEATURE_V8);
set_feature(&cpu->env, ARM_FEATURE_NEON);
set_feature(&cpu->env, ARM_FEATURE_GENERIC_TIMER);
set_feature(&cpu->env, ARM_FEATURE_AARCH64);
set_feature(&cpu->env, ARM_FEATURE_CBAR_RO);
set_feature(&cpu->env, ARM_FEATURE_EL2);
set_feature(&cpu->env, ARM_FEATURE_EL3);
set_feature(&cpu->env, ARM_FEATURE_PMU);

/* Ordered by Section B.4: AArch64 registers */
cpu->midr = 0x412FD471; /* r2p1 */
cpu->revidr = 0;
cpu->isar.id_pfr0 = 0x21110131;
cpu->isar.id_pfr1 = 0x00010000; /* GIC filled in later */
cpu->isar.id_dfr0 = 0x16011099;
cpu->id_afr0 = 0;
cpu->isar.id_mmfr0 = 0x10201105;
cpu->isar.id_mmfr1 = 0x40000000;
cpu->isar.id_mmfr2 = 0x01260000;
cpu->isar.id_mmfr3 = 0x02122211;
cpu->isar.id_isar0 = 0x02101110;
cpu->isar.id_isar1 = 0x13112111;
cpu->isar.id_isar2 = 0x21232042;
cpu->isar.id_isar3 = 0x01112131;
cpu->isar.id_isar4 = 0x00010142;
cpu->isar.id_isar5 = 0x11011121; /* with Crypto */
cpu->isar.id_mmfr4 = 0x21021110;
cpu->isar.id_isar6 = 0x01111111;
cpu->isar.mvfr0 = 0x10110222;
cpu->isar.mvfr1 = 0x13211111;
cpu->isar.mvfr2 = 0x00000043;
cpu->isar.id_pfr2 = 0x00000011;
cpu->isar.id_aa64pfr0 = 0x1201111120111112ull; /* GIC filled in later */
cpu->isar.id_aa64pfr1 = 0x0000000000000221ull;
cpu->isar.id_aa64zfr0 = 0x0000110100110021ull; /* with Crypto */
cpu->isar.id_aa64dfr0 = 0x000011f010305611ull;
cpu->isar.id_aa64dfr1 = 0;
cpu->id_aa64afr0 = 0;
cpu->id_aa64afr1 = 0;
cpu->isar.id_aa64isar0 = 0x0221111110212120ull; /* with Crypto */
cpu->isar.id_aa64isar1 = 0x0010111101211032ull;
cpu->isar.id_aa64mmfr0 = 0x0000022200101122ull;
cpu->isar.id_aa64mmfr1 = 0x0000000010212122ull;
cpu->isar.id_aa64mmfr2 = 0x1221011110101011ull;
cpu->clidr = 0x0000001482000023ull;
cpu->gm_blocksize = 4;
cpu->ctr = 0x000000049444c004ull;
cpu->dcz_blocksize = 4;
/* TODO FEAT_MPAM: mpamidr_el1 = 0x0000_0001_0006_003f */

/* Section B.5.2: PMCR_EL0 */
cpu->isar.reset_pmcr_el0 = 0xa000; /* with 20 counters */

/* Section B.6.7: ICH_VTR_EL2 */
cpu->gic_num_lrs = 4;
cpu->gic_vpribits = 5;
cpu->gic_vprebits = 5;
cpu->gic_pribits = 5;

/* Section 14: Scalable Vector Extensions support */
cpu->sve_vq.supported = 1 << 0; /* 128bit */

/*
* The cortex-a710 TRM does not list CCSIDR values. The layout of
* the caches are in text in Table 7-1, Table 8-1, and Table 9-1.
*
* L1: 4-way set associative 64-byte line size, total either 32K or 64K.
* L2: 8-way set associative 64 byte line size, total either 256K or 512K.
*/
cpu->ccsidr[0] = make_ccsidr64(4, 64, 64 * KiB); /* L1 dcache */
cpu->ccsidr[1] = cpu->ccsidr[0]; /* L1 icache */
cpu->ccsidr[2] = make_ccsidr64(8, 64, 512 * KiB); /* L2 cache */

/* FIXME: Not documented -- copied from neoverse-v1 */
cpu->reset_sctlr = 0x30c50838;

define_arm_cp_regs(cpu, cortex_a710_cp_reginfo);

aarch64_add_pauth_properties(obj);
aarch64_add_sve_properties(obj);
}

/*
* -cpu max: a CPU with as many features enabled as our emulation supports.
* The version of '-cpu max' for qemu-system-arm is defined in cpu32.c;
Expand Down Expand Up @@ -803,6 +1014,8 @@ void aarch64_max_tcg_initfn(Object *obj)

t = cpu->isar.id_aa64isar1;
t = FIELD_DP64(t, ID_AA64ISAR1, DPB, 2); /* FEAT_DPB2 */
t = FIELD_DP64(t, ID_AA64ISAR1, APA, PauthFeat_FPACCOMBINED);
t = FIELD_DP64(t, ID_AA64ISAR1, API, 1);
t = FIELD_DP64(t, ID_AA64ISAR1, JSCVT, 1); /* FEAT_JSCVT */
t = FIELD_DP64(t, ID_AA64ISAR1, FCMA, 1); /* FEAT_FCMA */
t = FIELD_DP64(t, ID_AA64ISAR1, LRCPC, 2); /* FEAT_LRCPC2 */
Expand Down Expand Up @@ -858,6 +1071,7 @@ void aarch64_max_tcg_initfn(Object *obj)
t = FIELD_DP64(t, ID_AA64MMFR1, XNX, 1); /* FEAT_XNX */
t = FIELD_DP64(t, ID_AA64MMFR1, ETS, 1); /* FEAT_ETS */
t = FIELD_DP64(t, ID_AA64MMFR1, HCX, 1); /* FEAT_HCX */
t = FIELD_DP64(t, ID_AA64MMFR1, TIDCP1, 1); /* FEAT_TIDCP1 */
cpu->isar.id_aa64mmfr1 = t;

t = cpu->isar.id_aa64mmfr2;
Expand Down Expand Up @@ -934,6 +1148,7 @@ static const ARMCPUInfo aarch64_cpus[] = {
{ .name = "cortex-a55", .initfn = aarch64_a55_initfn },
{ .name = "cortex-a72", .initfn = aarch64_a72_initfn },
{ .name = "cortex-a76", .initfn = aarch64_a76_initfn },
{ .name = "cortex-a710", .initfn = aarch64_a710_initfn },
{ .name = "a64fx", .initfn = aarch64_a64fx_initfn },
{ .name = "neoverse-n1", .initfn = aarch64_neoverse_n1_initfn },
{ .name = "neoverse-v1", .initfn = aarch64_neoverse_v1_initfn },
Expand Down
10 changes: 2 additions & 8 deletions target/arm/tcg/crypto_helper.c
View file
Expand Up @@ -614,10 +614,7 @@ static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
CR_ST_WORD(d, (i + 3) % 4) ^
CR_ST_WORD(n, i);

t = sm4_sbox[t & 0xff] |
sm4_sbox[(t >> 8) & 0xff] << 8 |
sm4_sbox[(t >> 16) & 0xff] << 16 |
sm4_sbox[(t >> 24) & 0xff] << 24;
t = sm4_subword(t);

CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
rol32(t, 24);
Expand Down Expand Up @@ -651,10 +648,7 @@ static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
CR_ST_WORD(d, (i + 3) % 4) ^
CR_ST_WORD(m, i);

t = sm4_sbox[t & 0xff] |
sm4_sbox[(t >> 8) & 0xff] << 8 |
sm4_sbox[(t >> 16) & 0xff] << 16 |
sm4_sbox[(t >> 24) & 0xff] << 24;
t = sm4_subword(t);

CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
}
Expand Down
4 changes: 4 additions & 0 deletions target/arm/tcg/helper-a64.h
View file
Expand Up @@ -90,9 +90,13 @@ DEF_HELPER_FLAGS_3(pacda, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(pacdb, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(pacga, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autia, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autia_combined, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autib, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autib_combined, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autda, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autda_combined, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autdb, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_3(autdb_combined, TCG_CALL_NO_WG, i64, env, i64, i64)
DEF_HELPER_FLAGS_2(xpaci, TCG_CALL_NO_RWG_SE, i64, env, i64)
DEF_HELPER_FLAGS_2(xpacd, TCG_CALL_NO_RWG_SE, i64, env, i64)

Expand Down
33 changes: 33 additions & 0 deletions target/arm/tcg/op_helper.c
View file
Expand Up @@ -764,6 +764,39 @@ const void *HELPER(lookup_cp_reg)(CPUARMState *env, uint32_t key)
return ri;
}

/*
* Test for HCR_EL2.TIDCP at EL1.
* Since implementation defined registers are rare, and within QEMU
* most of them are no-op, do not waste HFLAGS space for this and
* always use a helper.
*/
void HELPER(tidcp_el1)(CPUARMState *env, uint32_t syndrome)
{
if (arm_hcr_el2_eff(env) & HCR_TIDCP) {
raise_exception_ra(env, EXCP_UDEF, syndrome, 2, GETPC());
}
}

/*
* Similarly, for FEAT_TIDCP1 at EL0.
* We have already checked for the presence of the feature.
*/
void HELPER(tidcp_el0)(CPUARMState *env, uint32_t syndrome)
{
/* See arm_sctlr(), but we also need the sctlr el. */
ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, 0);
int target_el = mmu_idx == ARMMMUIdx_E20_0 ? 2 : 1;

/*
* The bit is not valid unless the target el is aa64, but since the
* bit test is simpler perform that first and check validity after.
*/
if ((env->cp15.sctlr_el[target_el] & SCTLR_TIDCP)
&& arm_el_is_aa64(env, target_el)) {
raise_exception_ra(env, EXCP_UDEF, syndrome, target_el, GETPC());
}
}

void HELPER(set_cp_reg)(CPUARMState *env, const void *rip, uint32_t value)
{
const ARMCPRegInfo *ri = rip;
Expand Down
178 changes: 146 additions & 32 deletions target/arm/tcg/pauth_helper.c
View file
Expand Up @@ -96,6 +96,21 @@ static uint64_t pac_sub(uint64_t i)
return o;
}

static uint64_t pac_sub1(uint64_t i)
{
static const uint8_t sub1[16] = {
0xa, 0xd, 0xe, 0x6, 0xf, 0x7, 0x3, 0x5,
0x9, 0x8, 0x0, 0xc, 0xb, 0x1, 0x2, 0x4,
};
uint64_t o = 0;
int b;

for (b = 0; b < 64; b += 4) {
o |= (uint64_t)sub1[(i >> b) & 0xf] << b;
}
return o;
}

static uint64_t pac_inv_sub(uint64_t i)
{
static const uint8_t inv_sub[16] = {
Expand Down Expand Up @@ -209,7 +224,7 @@ static uint64_t tweak_inv_shuffle(uint64_t i)
}

static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier,
ARMPACKey key)
ARMPACKey key, bool isqarma3)
{
static const uint64_t RC[5] = {
0x0000000000000000ull,
Expand All @@ -219,6 +234,7 @@ static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier,
0x452821E638D01377ull,
};
const uint64_t alpha = 0xC0AC29B7C97C50DDull;
int iterations = isqarma3 ? 2 : 4;
/*
* Note that in the ARM pseudocode, key0 contains bits <127:64>
* and key1 contains bits <63:0> of the 128-bit key.
Expand All @@ -231,40 +247,56 @@ static uint64_t pauth_computepac_architected(uint64_t data, uint64_t modifier,
runningmod = modifier;
workingval = data ^ key0;

for (i = 0; i <= 4; ++i) {
for (i = 0; i <= iterations; ++i) {
roundkey = key1 ^ runningmod;
workingval ^= roundkey;
workingval ^= RC[i];
if (i > 0) {
workingval = pac_cell_shuffle(workingval);
workingval = pac_mult(workingval);
}
workingval = pac_sub(workingval);
if (isqarma3) {
workingval = pac_sub1(workingval);
} else {
workingval = pac_sub(workingval);
}
runningmod = tweak_shuffle(runningmod);
}
roundkey = modk0 ^ runningmod;
workingval ^= roundkey;
workingval = pac_cell_shuffle(workingval);
workingval = pac_mult(workingval);
workingval = pac_sub(workingval);
if (isqarma3) {
workingval = pac_sub1(workingval);
} else {
workingval = pac_sub(workingval);
}
workingval = pac_cell_shuffle(workingval);
workingval = pac_mult(workingval);
workingval ^= key1;
workingval = pac_cell_inv_shuffle(workingval);
workingval = pac_inv_sub(workingval);
if (isqarma3) {
workingval = pac_sub1(workingval);
} else {
workingval = pac_inv_sub(workingval);
}
workingval = pac_mult(workingval);
workingval = pac_cell_inv_shuffle(workingval);
workingval ^= key0;
workingval ^= runningmod;
for (i = 0; i <= 4; ++i) {
workingval = pac_inv_sub(workingval);
if (i < 4) {
for (i = 0; i <= iterations; ++i) {
if (isqarma3) {
workingval = pac_sub1(workingval);
} else {
workingval = pac_inv_sub(workingval);
}
if (i < iterations) {
workingval = pac_mult(workingval);
workingval = pac_cell_inv_shuffle(workingval);
}
runningmod = tweak_inv_shuffle(runningmod);
roundkey = key1 ^ runningmod;
workingval ^= RC[4 - i];
workingval ^= RC[iterations - i];
workingval ^= roundkey;
workingval ^= alpha;
}
Expand All @@ -282,8 +314,10 @@ static uint64_t pauth_computepac_impdef(uint64_t data, uint64_t modifier,
static uint64_t pauth_computepac(CPUARMState *env, uint64_t data,
uint64_t modifier, ARMPACKey key)
{
if (cpu_isar_feature(aa64_pauth_arch, env_archcpu(env))) {
return pauth_computepac_architected(data, modifier, key);
if (cpu_isar_feature(aa64_pauth_qarma5, env_archcpu(env))) {
return pauth_computepac_architected(data, modifier, key, false);
} else if (cpu_isar_feature(aa64_pauth_qarma3, env_archcpu(env))) {
return pauth_computepac_architected(data, modifier, key, true);
} else {
return pauth_computepac_impdef(data, modifier, key);
}
Expand All @@ -292,8 +326,10 @@ static uint64_t pauth_computepac(CPUARMState *env, uint64_t data,
static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier,
ARMPACKey *key, bool data)
{
ARMCPU *cpu = env_archcpu(env);
ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false);
ARMPauthFeature pauth_feature = cpu_isar_feature(pauth_feature, cpu);
uint64_t pac, ext_ptr, ext, test;
int bot_bit, top_bit;

Expand All @@ -317,17 +353,26 @@ static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier,
*/
test = sextract64(ptr, bot_bit, top_bit - bot_bit);
if (test != 0 && test != -1) {
/*
* Note that our top_bit is one greater than the pseudocode's
* version, hence "- 2" here.
*/
pac ^= MAKE_64BIT_MASK(top_bit - 2, 1);
if (pauth_feature >= PauthFeat_2) {
/* No action required */
} else if (pauth_feature == PauthFeat_EPAC) {
pac = 0;
} else {
/*
* Note that our top_bit is one greater than the pseudocode's
* version, hence "- 2" here.
*/
pac ^= MAKE_64BIT_MASK(top_bit - 2, 1);
}
}

/*
* Preserve the determination between upper and lower at bit 55,
* and insert pointer authentication code.
*/
if (pauth_feature >= PauthFeat_2) {
pac ^= ptr;
}
if (param.tbi) {
ptr &= ~MAKE_64BIT_MASK(bot_bit, 55 - bot_bit + 1);
pac &= MAKE_64BIT_MASK(bot_bit, 54 - bot_bit + 1);
Expand All @@ -351,21 +396,46 @@ static uint64_t pauth_original_ptr(uint64_t ptr, ARMVAParameters param)
}
}

static G_NORETURN
void pauth_fail_exception(CPUARMState *env, bool data,
int keynumber, uintptr_t ra)
{
raise_exception_ra(env, EXCP_UDEF, syn_pacfail(data, keynumber),
exception_target_el(env), ra);
}

static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier,
ARMPACKey *key, bool data, int keynumber)
ARMPACKey *key, bool data, int keynumber,
uintptr_t ra, bool is_combined)
{
ARMCPU *cpu = env_archcpu(env);
ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env);
ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data, false);
ARMPauthFeature pauth_feature = cpu_isar_feature(pauth_feature, cpu);
int bot_bit, top_bit;
uint64_t pac, orig_ptr, test;
uint64_t pac, orig_ptr, cmp_mask;

orig_ptr = pauth_original_ptr(ptr, param);
pac = pauth_computepac(env, orig_ptr, modifier, *key);
bot_bit = 64 - param.tsz;
top_bit = 64 - 8 * param.tbi;

test = (pac ^ ptr) & ~MAKE_64BIT_MASK(55, 1);
if (unlikely(extract64(test, bot_bit, top_bit - bot_bit))) {
cmp_mask = MAKE_64BIT_MASK(bot_bit, top_bit - bot_bit);
cmp_mask &= ~MAKE_64BIT_MASK(55, 1);

if (pauth_feature >= PauthFeat_2) {
ARMPauthFeature fault_feature =
is_combined ? PauthFeat_FPACCOMBINED : PauthFeat_FPAC;
uint64_t result = ptr ^ (pac & cmp_mask);

if (pauth_feature >= fault_feature
&& ((result ^ sextract64(result, 55, 1)) & cmp_mask)) {
pauth_fail_exception(env, data, keynumber, ra);
}
return result;
}

if ((pac ^ ptr) & cmp_mask) {
int error_code = (keynumber << 1) | (keynumber ^ 1);
if (param.tbi) {
return deposit64(orig_ptr, 53, 2, error_code);
Expand Down Expand Up @@ -466,44 +536,88 @@ uint64_t HELPER(pacga)(CPUARMState *env, uint64_t x, uint64_t y)
return pac & 0xffffffff00000000ull;
}

uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y)
static uint64_t pauth_autia(CPUARMState *env, uint64_t x, uint64_t y,
uintptr_t ra, bool is_combined)
{
int el = arm_current_el(env);
if (!pauth_key_enabled(env, el, SCTLR_EnIA)) {
return x;
}
pauth_check_trap(env, el, GETPC());
return pauth_auth(env, x, y, &env->keys.apia, false, 0);
pauth_check_trap(env, el, ra);
return pauth_auth(env, x, y, &env->keys.apia, false, 0, ra, is_combined);
}

uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y)
uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autia(env, x, y, GETPC(), false);
}

uint64_t HELPER(autia_combined)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autia(env, x, y, GETPC(), true);
}

static uint64_t pauth_autib(CPUARMState *env, uint64_t x, uint64_t y,
uintptr_t ra, bool is_combined)
{
int el = arm_current_el(env);
if (!pauth_key_enabled(env, el, SCTLR_EnIB)) {
return x;
}
pauth_check_trap(env, el, GETPC());
return pauth_auth(env, x, y, &env->keys.apib, false, 1);
pauth_check_trap(env, el, ra);
return pauth_auth(env, x, y, &env->keys.apib, false, 1, ra, is_combined);
}

uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y)
uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autib(env, x, y, GETPC(), false);
}

uint64_t HELPER(autib_combined)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autib(env, x, y, GETPC(), true);
}

static uint64_t pauth_autda(CPUARMState *env, uint64_t x, uint64_t y,
uintptr_t ra, bool is_combined)
{
int el = arm_current_el(env);
if (!pauth_key_enabled(env, el, SCTLR_EnDA)) {
return x;
}
pauth_check_trap(env, el, GETPC());
return pauth_auth(env, x, y, &env->keys.apda, true, 0);
pauth_check_trap(env, el, ra);
return pauth_auth(env, x, y, &env->keys.apda, true, 0, ra, is_combined);
}

uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y)
uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autda(env, x, y, GETPC(), false);
}

uint64_t HELPER(autda_combined)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autda(env, x, y, GETPC(), true);
}

static uint64_t pauth_autdb(CPUARMState *env, uint64_t x, uint64_t y,
uintptr_t ra, bool is_combined)
{
int el = arm_current_el(env);
if (!pauth_key_enabled(env, el, SCTLR_EnDB)) {
return x;
}
pauth_check_trap(env, el, GETPC());
return pauth_auth(env, x, y, &env->keys.apdb, true, 1);
pauth_check_trap(env, el, ra);
return pauth_auth(env, x, y, &env->keys.apdb, true, 1, ra, is_combined);
}

uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autdb(env, x, y, GETPC(), false);
}

uint64_t HELPER(autdb_combined)(CPUARMState *env, uint64_t x, uint64_t y)
{
return pauth_autdb(env, x, y, GETPC(), true);
}

uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a)
Expand Down
74 changes: 40 additions & 34 deletions target/arm/tcg/translate-a64.c
View file
Expand Up @@ -1530,9 +1530,9 @@ static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst,

truedst = tcg_temp_new_i64();
if (use_key_a) {
gen_helper_autia(truedst, cpu_env, dst, modifier);
gen_helper_autia_combined(truedst, cpu_env, dst, modifier);
} else {
gen_helper_autib(truedst, cpu_env, dst, modifier);
gen_helper_autib_combined(truedst, cpu_env, dst, modifier);
}
return truedst;
}
Expand Down Expand Up @@ -2154,6 +2154,25 @@ static void handle_sys(DisasContext *s, bool isread,
bool need_exit_tb = false;
TCGv_ptr tcg_ri = NULL;
TCGv_i64 tcg_rt;
uint32_t syndrome;

if (crn == 11 || crn == 15) {
/*
* Check for TIDCP trap, which must take precedence over
* the UNDEF for "no such register" etc.
*/
syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
switch (s->current_el) {
case 0:
if (dc_isar_feature(aa64_tidcp1, s)) {
gen_helper_tidcp_el0(cpu_env, tcg_constant_i32(syndrome));
}
break;
case 1:
gen_helper_tidcp_el1(cpu_env, tcg_constant_i32(syndrome));
break;
}
}

if (!ri) {
/* Unknown register; this might be a guest error or a QEMU
Expand All @@ -2176,8 +2195,6 @@ static void handle_sys(DisasContext *s, bool isread,
/* Emit code to perform further access permissions checks at
* runtime; this may result in an exception.
*/
uint32_t syndrome;

syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
gen_a64_update_pc(s, 0);
tcg_ri = tcg_temp_new_ptr();
Expand Down Expand Up @@ -3020,37 +3037,17 @@ static bool trans_STGP(DisasContext *s, arg_ldstpair *a)
tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
}

if (!s->ata) {
/*
* TODO: We could rely on the stores below, at least for
* system mode, if we arrange to add MO_ALIGN_16.
*/
gen_helper_stg_stub(cpu_env, dirty_addr);
} else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
gen_helper_stg_parallel(cpu_env, dirty_addr, dirty_addr);
} else {
gen_helper_stg(cpu_env, dirty_addr, dirty_addr);
}

mop = finalize_memop(s, MO_64);
clean_addr = gen_mte_checkN(s, dirty_addr, true, false, 2 << MO_64, mop);

clean_addr = clean_data_tbi(s, dirty_addr);
tcg_rt = cpu_reg(s, a->rt);
tcg_rt2 = cpu_reg(s, a->rt2);

/*
* STGP is defined as two 8-byte memory operations and one tag operation.
* We implement it as one single 16-byte memory operation for convenience.
* Rebuild mop as for STP.
* TODO: The atomicity with LSE2 is stronger than required.
* Need a form of MO_ATOM_WITHIN16_PAIR that never requires
* 16-byte atomicity.
* STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
* and one tag operation. We implement it as one single aligned 16-byte
* memory operation for convenience. Note that the alignment ensures
* MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
*/
mop = MO_128;
if (s->align_mem) {
mop |= MO_ALIGN_8;
}
mop = finalize_memop_pair(s, mop);
mop = finalize_memop_atom(s, MO_128 | MO_ALIGN, MO_ATOM_IFALIGN_PAIR);

tmp = tcg_temp_new_i128();
if (s->be_data == MO_LE) {
Expand All @@ -3060,6 +3057,15 @@ static bool trans_STGP(DisasContext *s, arg_ldstpair *a)
}
tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);

/* Perform the tag store, if tag access enabled. */
if (s->ata) {
if (tb_cflags(s->base.tb) & CF_PARALLEL) {
gen_helper_stg_parallel(cpu_env, dirty_addr, dirty_addr);
} else {
gen_helper_stg(cpu_env, dirty_addr, dirty_addr);
}
}

op_addr_ldstpair_post(s, a, dirty_addr, offset);
return true;
}
Expand Down Expand Up @@ -3352,11 +3358,11 @@ static bool trans_LDRA(DisasContext *s, arg_LDRA *a)

if (s->pauth_active) {
if (!a->m) {
gen_helper_autda(dirty_addr, cpu_env, dirty_addr,
tcg_constant_i64(0));
gen_helper_autda_combined(dirty_addr, cpu_env, dirty_addr,
tcg_constant_i64(0));
} else {
gen_helper_autdb(dirty_addr, cpu_env, dirty_addr,
tcg_constant_i64(0));
gen_helper_autdb_combined(dirty_addr, cpu_env, dirty_addr,
tcg_constant_i64(0));
}
}

Expand Down
33 changes: 33 additions & 0 deletions target/arm/tcg/translate.c
View file
Expand Up @@ -4538,6 +4538,20 @@ void gen_gvec_uaba(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
}

static bool aa32_cpreg_encoding_in_impdef_space(uint8_t crn, uint8_t crm)
{
static const uint16_t mask[3] = {
0b0000000111100111, /* crn == 9, crm == {c0-c2, c5-c8} */
0b0000000100010011, /* crn == 10, crm == {c0, c1, c4, c8} */
0b1000000111111111, /* crn == 11, crm == {c0-c8, c15} */
};

if (crn >= 9 && crn <= 11) {
return (mask[crn - 9] >> crm) & 1;
}
return false;
}

static void do_coproc_insn(DisasContext *s, int cpnum, int is64,
int opc1, int crn, int crm, int opc2,
bool isread, int rt, int rt2)
Expand Down Expand Up @@ -4619,6 +4633,25 @@ static void do_coproc_insn(DisasContext *s, int cpnum, int is64,
}
}

if (cpnum == 15 && aa32_cpreg_encoding_in_impdef_space(crn, crm)) {
/*
* Check for TIDCP trap, which must take precedence over the UNDEF
* for "no such register" etc. It shares precedence with HSTR,
* but raises the same exception, so order doesn't matter.
*/
switch (s->current_el) {
case 0:
if (arm_dc_feature(s, ARM_FEATURE_AARCH64)
&& dc_isar_feature(aa64_tidcp1, s)) {
gen_helper_tidcp_el0(cpu_env, tcg_constant_i32(syndrome));
}
break;
case 1:
gen_helper_tidcp_el1(cpu_env, tcg_constant_i32(syndrome));
break;
}
}

if (!ri) {
/*
* Unknown register; this might be a guest error or a QEMU
Expand Down
83 changes: 76 additions & 7 deletions target/riscv/cpu.c
View file
Expand Up @@ -87,7 +87,9 @@ static const struct isa_ext_data isa_edata_arr[] = {
ISA_EXT_DATA_ENTRY(zicond, PRIV_VERSION_1_12_0, ext_zicond),
ISA_EXT_DATA_ENTRY(zicsr, PRIV_VERSION_1_10_0, ext_icsr),
ISA_EXT_DATA_ENTRY(zifencei, PRIV_VERSION_1_10_0, ext_ifencei),
ISA_EXT_DATA_ENTRY(zihintntl, PRIV_VERSION_1_10_0, ext_zihintntl),
ISA_EXT_DATA_ENTRY(zihintpause, PRIV_VERSION_1_10_0, ext_zihintpause),
ISA_EXT_DATA_ENTRY(zmmul, PRIV_VERSION_1_12_0, ext_zmmul),
ISA_EXT_DATA_ENTRY(zawrs, PRIV_VERSION_1_12_0, ext_zawrs),
ISA_EXT_DATA_ENTRY(zfa, PRIV_VERSION_1_12_0, ext_zfa),
ISA_EXT_DATA_ENTRY(zfbfmin, PRIV_VERSION_1_12_0, ext_zfbfmin),
Expand Down Expand Up @@ -119,16 +121,25 @@ static const struct isa_ext_data isa_edata_arr[] = {
ISA_EXT_DATA_ENTRY(zksed, PRIV_VERSION_1_12_0, ext_zksed),
ISA_EXT_DATA_ENTRY(zksh, PRIV_VERSION_1_12_0, ext_zksh),
ISA_EXT_DATA_ENTRY(zkt, PRIV_VERSION_1_12_0, ext_zkt),
ISA_EXT_DATA_ENTRY(zvbb, PRIV_VERSION_1_12_0, ext_zvbb),
ISA_EXT_DATA_ENTRY(zvbc, PRIV_VERSION_1_12_0, ext_zvbc),
ISA_EXT_DATA_ENTRY(zve32f, PRIV_VERSION_1_10_0, ext_zve32f),
ISA_EXT_DATA_ENTRY(zve64f, PRIV_VERSION_1_10_0, ext_zve64f),
ISA_EXT_DATA_ENTRY(zve64d, PRIV_VERSION_1_10_0, ext_zve64d),
ISA_EXT_DATA_ENTRY(zvfbfmin, PRIV_VERSION_1_12_0, ext_zvfbfmin),
ISA_EXT_DATA_ENTRY(zvfbfwma, PRIV_VERSION_1_12_0, ext_zvfbfwma),
ISA_EXT_DATA_ENTRY(zvfh, PRIV_VERSION_1_12_0, ext_zvfh),
ISA_EXT_DATA_ENTRY(zvfhmin, PRIV_VERSION_1_12_0, ext_zvfhmin),
ISA_EXT_DATA_ENTRY(zvkg, PRIV_VERSION_1_12_0, ext_zvkg),
ISA_EXT_DATA_ENTRY(zvkned, PRIV_VERSION_1_12_0, ext_zvkned),
ISA_EXT_DATA_ENTRY(zvknha, PRIV_VERSION_1_12_0, ext_zvknha),
ISA_EXT_DATA_ENTRY(zvknhb, PRIV_VERSION_1_12_0, ext_zvknhb),
ISA_EXT_DATA_ENTRY(zvksed, PRIV_VERSION_1_12_0, ext_zvksed),
ISA_EXT_DATA_ENTRY(zvksh, PRIV_VERSION_1_12_0, ext_zvksh),
ISA_EXT_DATA_ENTRY(zhinx, PRIV_VERSION_1_12_0, ext_zhinx),
ISA_EXT_DATA_ENTRY(zhinxmin, PRIV_VERSION_1_12_0, ext_zhinxmin),
ISA_EXT_DATA_ENTRY(smaia, PRIV_VERSION_1_12_0, ext_smaia),
ISA_EXT_DATA_ENTRY(smepmp, PRIV_VERSION_1_12_0, epmp),
ISA_EXT_DATA_ENTRY(smstateen, PRIV_VERSION_1_12_0, ext_smstateen),
ISA_EXT_DATA_ENTRY(ssaia, PRIV_VERSION_1_12_0, ext_ssaia),
ISA_EXT_DATA_ENTRY(sscofpmf, PRIV_VERSION_1_12_0, ext_sscofpmf),
Expand Down Expand Up @@ -298,6 +309,17 @@ static uint8_t satp_mode_from_str(const char *satp_mode_str)

uint8_t satp_mode_max_from_map(uint32_t map)
{
/*
* 'map = 0' will make us return (31 - 32), which C will
* happily overflow to UINT_MAX. There's no good result to
* return if 'map = 0' (e.g. returning 0 will be ambiguous
* with the result for 'map = 1').
*
* Assert out if map = 0. Callers will have to deal with
* it outside of this function.
*/
g_assert(map > 0);

/* map here has at least one bit set, so no problem with clz */
return 31 - __builtin_clz(map);
}
Expand Down Expand Up @@ -875,9 +897,9 @@ static void riscv_cpu_reset_hold(Object *obj)
env->two_stage_lookup = false;

env->menvcfg = (cpu->cfg.ext_svpbmt ? MENVCFG_PBMTE : 0) |
(cpu->cfg.ext_svadu ? MENVCFG_HADE : 0);
(cpu->cfg.ext_svadu ? MENVCFG_ADUE : 0);
env->henvcfg = (cpu->cfg.ext_svpbmt ? HENVCFG_PBMTE : 0) |
(cpu->cfg.ext_svadu ? HENVCFG_HADE : 0);
(cpu->cfg.ext_svadu ? HENVCFG_ADUE : 0);

/* Initialized default priorities of local interrupts. */
for (i = 0; i < ARRAY_SIZE(env->miprio); i++) {
Expand All @@ -904,7 +926,7 @@ static void riscv_cpu_reset_hold(Object *obj)

#ifndef CONFIG_USER_ONLY
if (cpu->cfg.debug) {
riscv_trigger_init(env);
riscv_trigger_reset_hold(env);
}

if (kvm_enabled()) {
Expand Down Expand Up @@ -1269,6 +1291,25 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp)
return;
}

/*
* In principle Zve*x would also suffice here, were they supported
* in qemu
*/
if ((cpu->cfg.ext_zvbb || cpu->cfg.ext_zvkg || cpu->cfg.ext_zvkned ||
cpu->cfg.ext_zvknha || cpu->cfg.ext_zvksed || cpu->cfg.ext_zvksh) &&
!cpu->cfg.ext_zve32f) {
error_setg(errp,
"Vector crypto extensions require V or Zve* extensions");
return;
}

if ((cpu->cfg.ext_zvbc || cpu->cfg.ext_zvknhb) && !cpu->cfg.ext_zve64f) {
error_setg(
errp,
"Zvbc and Zvknhb extensions require V or Zve64{f,d} extensions");
return;
}

if (cpu->cfg.ext_zk) {
cpu->cfg.ext_zkn = true;
cpu->cfg.ext_zkr = true;
Expand Down Expand Up @@ -1303,9 +1344,15 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp)
static void riscv_cpu_satp_mode_finalize(RISCVCPU *cpu, Error **errp)
{
bool rv32 = riscv_cpu_mxl(&cpu->env) == MXL_RV32;
uint8_t satp_mode_map_max;
uint8_t satp_mode_supported_max =
satp_mode_max_from_map(cpu->cfg.satp_mode.supported);
uint8_t satp_mode_map_max, satp_mode_supported_max;

/* The CPU wants the OS to decide which satp mode to use */
if (cpu->cfg.satp_mode.supported == 0) {
return;
}

satp_mode_supported_max =
satp_mode_max_from_map(cpu->cfg.satp_mode.supported);

if (cpu->cfg.satp_mode.map == 0) {
if (cpu->cfg.satp_mode.init == 0) {
Expand Down Expand Up @@ -1395,6 +1442,11 @@ static void riscv_cpu_realize_tcg(DeviceState *dev, Error **errp)
CPURISCVState *env = &cpu->env;
Error *local_err = NULL;

if (object_dynamic_cast(OBJECT(dev), TYPE_RISCV_CPU_HOST)) {
error_setg(errp, "'host' CPU is not compatible with TCG acceleration");
return;
}

riscv_cpu_validate_misa_mxl(cpu, &local_err);
if (local_err != NULL) {
error_propagate(errp, local_err);
Expand Down Expand Up @@ -1473,6 +1525,12 @@ static void riscv_cpu_realize(DeviceState *dev, Error **errp)

riscv_cpu_register_gdb_regs_for_features(cs);

#ifndef CONFIG_USER_ONLY
if (cpu->cfg.debug) {
riscv_trigger_realize(&cpu->env);
}
#endif

qemu_init_vcpu(cs);
cpu_reset(cs);

Expand Down Expand Up @@ -1756,6 +1814,7 @@ static Property riscv_cpu_extensions[] = {
DEFINE_PROP_BOOL("sscofpmf", RISCVCPU, cfg.ext_sscofpmf, false),
DEFINE_PROP_BOOL("Zifencei", RISCVCPU, cfg.ext_ifencei, true),
DEFINE_PROP_BOOL("Zicsr", RISCVCPU, cfg.ext_icsr, true),
DEFINE_PROP_BOOL("Zihintntl", RISCVCPU, cfg.ext_zihintntl, true),
DEFINE_PROP_BOOL("Zihintpause", RISCVCPU, cfg.ext_zihintpause, true),
DEFINE_PROP_BOOL("Zawrs", RISCVCPU, cfg.ext_zawrs, true),
DEFINE_PROP_BOOL("Zfa", RISCVCPU, cfg.ext_zfa, true),
Expand Down Expand Up @@ -1816,6 +1875,7 @@ static Property riscv_cpu_extensions[] = {
DEFINE_PROP_BOOL("zcf", RISCVCPU, cfg.ext_zcf, false),
DEFINE_PROP_BOOL("zcmp", RISCVCPU, cfg.ext_zcmp, false),
DEFINE_PROP_BOOL("zcmt", RISCVCPU, cfg.ext_zcmt, false),
DEFINE_PROP_BOOL("zicond", RISCVCPU, cfg.ext_zicond, false),

/* Vendor-specific custom extensions */
DEFINE_PROP_BOOL("xtheadba", RISCVCPU, cfg.ext_xtheadba, false),
Expand All @@ -1832,7 +1892,6 @@ static Property riscv_cpu_extensions[] = {
DEFINE_PROP_BOOL("xventanacondops", RISCVCPU, cfg.ext_XVentanaCondOps, false),

/* These are experimental so mark with 'x-' */
DEFINE_PROP_BOOL("x-zicond", RISCVCPU, cfg.ext_zicond, false),

/* ePMP 0.9.3 */
DEFINE_PROP_BOOL("x-epmp", RISCVCPU, cfg.epmp, false),
Expand All @@ -1846,6 +1905,16 @@ static Property riscv_cpu_extensions[] = {
DEFINE_PROP_BOOL("x-zvfbfmin", RISCVCPU, cfg.ext_zvfbfmin, false),
DEFINE_PROP_BOOL("x-zvfbfwma", RISCVCPU, cfg.ext_zvfbfwma, false),

/* Vector cryptography extensions */
DEFINE_PROP_BOOL("x-zvbb", RISCVCPU, cfg.ext_zvbb, false),
DEFINE_PROP_BOOL("x-zvbc", RISCVCPU, cfg.ext_zvbc, false),
DEFINE_PROP_BOOL("x-zvkg", RISCVCPU, cfg.ext_zvkg, false),
DEFINE_PROP_BOOL("x-zvkned", RISCVCPU, cfg.ext_zvkned, false),
DEFINE_PROP_BOOL("x-zvknha", RISCVCPU, cfg.ext_zvknha, false),
DEFINE_PROP_BOOL("x-zvknhb", RISCVCPU, cfg.ext_zvknhb, false),
DEFINE_PROP_BOOL("x-zvksed", RISCVCPU, cfg.ext_zvksed, false),
DEFINE_PROP_BOOL("x-zvksh", RISCVCPU, cfg.ext_zvksh, false),

DEFINE_PROP_END_OF_LIST(),
};

Expand Down
8 changes: 4 additions & 4 deletions target/riscv/cpu_bits.h
View file
Expand Up @@ -745,12 +745,12 @@ typedef enum RISCVException {
#define MENVCFG_CBIE (3UL << 4)
#define MENVCFG_CBCFE BIT(6)
#define MENVCFG_CBZE BIT(7)
#define MENVCFG_HADE (1ULL << 61)
#define MENVCFG_ADUE (1ULL << 61)
#define MENVCFG_PBMTE (1ULL << 62)
#define MENVCFG_STCE (1ULL << 63)

/* For RV32 */
#define MENVCFGH_HADE BIT(29)
#define MENVCFGH_ADUE BIT(29)
#define MENVCFGH_PBMTE BIT(30)
#define MENVCFGH_STCE BIT(31)

Expand All @@ -763,12 +763,12 @@ typedef enum RISCVException {
#define HENVCFG_CBIE MENVCFG_CBIE
#define HENVCFG_CBCFE MENVCFG_CBCFE
#define HENVCFG_CBZE MENVCFG_CBZE
#define HENVCFG_HADE MENVCFG_HADE
#define HENVCFG_ADUE MENVCFG_ADUE
#define HENVCFG_PBMTE MENVCFG_PBMTE
#define HENVCFG_STCE MENVCFG_STCE

/* For RV32 */
#define HENVCFGH_HADE MENVCFGH_HADE
#define HENVCFGH_ADUE MENVCFGH_ADUE
#define HENVCFGH_PBMTE MENVCFGH_PBMTE
#define HENVCFGH_STCE MENVCFGH_STCE

Expand Down
9 changes: 9 additions & 0 deletions target/riscv/cpu_cfg.h
View file
Expand Up @@ -66,6 +66,7 @@ struct RISCVCPUConfig {
bool ext_icbom;
bool ext_icboz;
bool ext_zicond;
bool ext_zihintntl;
bool ext_zihintpause;
bool ext_smstateen;
bool ext_sstc;
Expand All @@ -85,6 +86,14 @@ struct RISCVCPUConfig {
bool ext_zve32f;
bool ext_zve64f;
bool ext_zve64d;
bool ext_zvbb;
bool ext_zvbc;
bool ext_zvkg;
bool ext_zvkned;
bool ext_zvknha;
bool ext_zvknhb;
bool ext_zvksed;
bool ext_zvksh;
bool ext_zmmul;
bool ext_zvfbfmin;
bool ext_zvfbfwma;
Expand Down
6 changes: 3 additions & 3 deletions target/riscv/cpu_helper.c
View file
Expand Up @@ -861,11 +861,11 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
}

bool pbmte = env->menvcfg & MENVCFG_PBMTE;
bool hade = env->menvcfg & MENVCFG_HADE;
bool adue = env->menvcfg & MENVCFG_ADUE;

if (first_stage && two_stage && env->virt_enabled) {
pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
hade = hade && (env->henvcfg & HENVCFG_HADE);
adue = adue && (env->henvcfg & HENVCFG_ADUE);
}

int ptshift = (levels - 1) * ptidxbits;
Expand Down Expand Up @@ -1026,7 +1026,7 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,

/* Page table updates need to be atomic with MTTCG enabled */
if (updated_pte != pte && !is_debug) {
if (!hade) {
if (!adue) {
return TRANSLATE_FAIL;
}

Expand Down
51 changes: 9 additions & 42 deletions target/riscv/crypto_helper.c
View file
Expand Up @@ -25,53 +25,27 @@
#include "crypto/aes-round.h"
#include "crypto/sm4.h"

#define AES_XTIME(a) \
((a << 1) ^ ((a & 0x80) ? 0x1b : 0))

#define AES_GFMUL(a, b) (( \
(((b) & 0x1) ? (a) : 0) ^ \
(((b) & 0x2) ? AES_XTIME(a) : 0) ^ \
(((b) & 0x4) ? AES_XTIME(AES_XTIME(a)) : 0) ^ \
(((b) & 0x8) ? AES_XTIME(AES_XTIME(AES_XTIME(a))) : 0)) & 0xFF)

static inline uint32_t aes_mixcolumn_byte(uint8_t x, bool fwd)
{
uint32_t u;

if (fwd) {
u = (AES_GFMUL(x, 3) << 24) | (x << 16) | (x << 8) |
(AES_GFMUL(x, 2) << 0);
} else {
u = (AES_GFMUL(x, 0xb) << 24) | (AES_GFMUL(x, 0xd) << 16) |
(AES_GFMUL(x, 0x9) << 8) | (AES_GFMUL(x, 0xe) << 0);
}
return u;
}

#define sext32_xlen(x) (target_ulong)(int32_t)(x)

static inline target_ulong aes32_operation(target_ulong shamt,
target_ulong rs1, target_ulong rs2,
bool enc, bool mix)
{
uint8_t si = rs2 >> shamt;
uint8_t so;
uint32_t mixed;
target_ulong res;

if (enc) {
so = AES_sbox[si];
if (mix) {
mixed = aes_mixcolumn_byte(so, true);
mixed = be32_to_cpu(AES_Te0[si]);
} else {
mixed = so;
mixed = AES_sbox[si];
}
} else {
so = AES_isbox[si];
if (mix) {
mixed = aes_mixcolumn_byte(so, false);
mixed = be32_to_cpu(AES_Td0[si]);
} else {
mixed = so;
mixed = AES_isbox[si];
}
}
mixed = rol32(mixed, shamt);
Expand Down Expand Up @@ -174,24 +148,17 @@ target_ulong HELPER(aes64ks1i)(target_ulong rs1, target_ulong rnum)

uint8_t enc_rnum = rnum;
uint32_t temp = (RS1 >> 32) & 0xFFFFFFFF;
uint8_t rcon_ = 0;
target_ulong result;
AESState t, rc = {};

if (enc_rnum != 0xA) {
temp = ror32(temp, 8); /* Rotate right by 8 */
rcon_ = round_consts[enc_rnum];
rc.w[0] = rc.w[1] = round_consts[enc_rnum];
}

temp = ((uint32_t)AES_sbox[(temp >> 24) & 0xFF] << 24) |
((uint32_t)AES_sbox[(temp >> 16) & 0xFF] << 16) |
((uint32_t)AES_sbox[(temp >> 8) & 0xFF] << 8) |
((uint32_t)AES_sbox[(temp >> 0) & 0xFF] << 0);

temp ^= rcon_;
t.w[0] = t.w[1] = t.w[2] = t.w[3] = temp;
aesenc_SB_SR_AK(&t, &t, &rc, false);

result = ((uint64_t)temp << 32) | temp;

return result;
return t.d[0];
}

target_ulong HELPER(aes64im)(target_ulong rs1)
Expand Down
54 changes: 35 additions & 19 deletions target/riscv/csr.c
View file
Expand Up @@ -1684,7 +1684,7 @@ static int rmw_iprio(target_ulong xlen,
static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val,
target_ulong new_val, target_ulong wr_mask)
{
bool virt;
bool virt, isel_reserved;
uint8_t *iprio;
int ret = -EINVAL;
target_ulong priv, isel, vgein;
Expand All @@ -1694,6 +1694,7 @@ static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val,

/* Decode register details from CSR number */
virt = false;
isel_reserved = false;
switch (csrno) {
case CSR_MIREG:
iprio = env->miprio;
Expand Down Expand Up @@ -1738,11 +1739,13 @@ static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val,
riscv_cpu_mxl_bits(env)),
val, new_val, wr_mask);
}
} else {
isel_reserved = true;
}

done:
if (ret) {
return (env->virt_enabled && virt) ?
return (env->virt_enabled && virt && !isel_reserved) ?
RISCV_EXCP_VIRT_INSTRUCTION_FAULT : RISCV_EXCP_ILLEGAL_INST;
}
return RISCV_EXCP_NONE;
Expand Down Expand Up @@ -1833,8 +1836,11 @@ static RISCVException write_mcountinhibit(CPURISCVState *env, int csrno,
{
int cidx;
PMUCTRState *counter;
RISCVCPU *cpu = env_archcpu(env);

env->mcountinhibit = val;
/* WARL register - disable unavailable counters; TM bit is always 0 */
env->mcountinhibit =
val & (cpu->pmu_avail_ctrs | COUNTEREN_CY | COUNTEREN_IR);

/* Check if any other counter is also monitoring cycles/instructions */
for (cidx = 0; cidx < RV_MAX_MHPMCOUNTERS; cidx++) {
Expand All @@ -1857,7 +1863,11 @@ static RISCVException read_mcounteren(CPURISCVState *env, int csrno,
static RISCVException write_mcounteren(CPURISCVState *env, int csrno,
target_ulong val)
{
env->mcounteren = val;
RISCVCPU *cpu = env_archcpu(env);

/* WARL register - disable unavailable counters */
env->mcounteren = val & (cpu->pmu_avail_ctrs | COUNTEREN_CY | COUNTEREN_TM |
COUNTEREN_IR);
return RISCV_EXCP_NONE;
}

Expand Down Expand Up @@ -1950,7 +1960,7 @@ static RISCVException write_menvcfg(CPURISCVState *env, int csrno,
if (riscv_cpu_mxl(env) == MXL_RV64) {
mask |= (cfg->ext_svpbmt ? MENVCFG_PBMTE : 0) |
(cfg->ext_sstc ? MENVCFG_STCE : 0) |
(cfg->ext_svadu ? MENVCFG_HADE : 0);
(cfg->ext_svadu ? MENVCFG_ADUE : 0);
}
env->menvcfg = (env->menvcfg & ~mask) | (val & mask);

Expand All @@ -1970,7 +1980,7 @@ static RISCVException write_menvcfgh(CPURISCVState *env, int csrno,
const RISCVCPUConfig *cfg = riscv_cpu_cfg(env);
uint64_t mask = (cfg->ext_svpbmt ? MENVCFG_PBMTE : 0) |
(cfg->ext_sstc ? MENVCFG_STCE : 0) |
(cfg->ext_svadu ? MENVCFG_HADE : 0);
(cfg->ext_svadu ? MENVCFG_ADUE : 0);
uint64_t valh = (uint64_t)val << 32;

env->menvcfg = (env->menvcfg & ~mask) | (valh & mask);
Expand Down Expand Up @@ -2022,7 +2032,7 @@ static RISCVException read_henvcfg(CPURISCVState *env, int csrno,
* henvcfg.stce is read_only 0 when menvcfg.stce = 0
* henvcfg.hade is read_only 0 when menvcfg.hade = 0
*/
*val = env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE) |
*val = env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_ADUE) |
env->menvcfg);
return RISCV_EXCP_NONE;
}
Expand All @@ -2039,7 +2049,7 @@ static RISCVException write_henvcfg(CPURISCVState *env, int csrno,
}

if (riscv_cpu_mxl(env) == MXL_RV64) {
mask |= env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE);
mask |= env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_ADUE);
}

env->henvcfg = (env->henvcfg & ~mask) | (val & mask);
Expand All @@ -2057,7 +2067,7 @@ static RISCVException read_henvcfgh(CPURISCVState *env, int csrno,
return ret;
}

*val = (env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE) |
*val = (env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_ADUE) |
env->menvcfg)) >> 32;
return RISCV_EXCP_NONE;
}
Expand All @@ -2066,7 +2076,7 @@ static RISCVException write_henvcfgh(CPURISCVState *env, int csrno,
target_ulong val)
{
uint64_t mask = env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE |
HENVCFG_HADE);
HENVCFG_ADUE);
uint64_t valh = (uint64_t)val << 32;
RISCVException ret;

Expand Down Expand Up @@ -3907,21 +3917,27 @@ static RISCVException riscv_csrrw_do64(CPURISCVState *env, int csrno,
target_ulong write_mask)
{
RISCVException ret;
target_ulong old_value;
target_ulong old_value = 0;

/* execute combined read/write operation if it exists */
if (csr_ops[csrno].op) {
return csr_ops[csrno].op(env, csrno, ret_value, new_value, write_mask);
}

/* if no accessor exists then return failure */
if (!csr_ops[csrno].read) {
return RISCV_EXCP_ILLEGAL_INST;
}
/* read old value */
ret = csr_ops[csrno].read(env, csrno, &old_value);
if (ret != RISCV_EXCP_NONE) {
return ret;
/*
* ret_value == NULL means that rd=x0 and we're coming from helper_csrw()
* and we can't throw side effects caused by CSR reads.
*/
if (ret_value) {
/* if no accessor exists then return failure */
if (!csr_ops[csrno].read) {
return RISCV_EXCP_ILLEGAL_INST;
}
/* read old value */
ret = csr_ops[csrno].read(env, csrno, &old_value);
if (ret != RISCV_EXCP_NONE) {
return ret;
}
}

/* write value if writable and write mask set, otherwise drop writes */
Expand Down
15 changes: 12 additions & 3 deletions target/riscv/debug.c
View file
Expand Up @@ -903,7 +903,17 @@ bool riscv_cpu_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
return false;
}

void riscv_trigger_init(CPURISCVState *env)
void riscv_trigger_realize(CPURISCVState *env)
{
int i;

for (i = 0; i < RV_MAX_TRIGGERS; i++) {
env->itrigger_timer[i] = timer_new_ns(QEMU_CLOCK_VIRTUAL,
riscv_itrigger_timer_cb, env);
}
}

void riscv_trigger_reset_hold(CPURISCVState *env)
{
target_ulong tdata1 = build_tdata1(env, TRIGGER_TYPE_AD_MATCH, 0, 0);
int i;
Expand All @@ -928,7 +938,6 @@ void riscv_trigger_init(CPURISCVState *env)
env->tdata3[i] = 0;
env->cpu_breakpoint[i] = NULL;
env->cpu_watchpoint[i] = NULL;
env->itrigger_timer[i] = timer_new_ns(QEMU_CLOCK_VIRTUAL,
riscv_itrigger_timer_cb, env);
timer_del(env->itrigger_timer[i]);
}
}
3 changes: 2 additions & 1 deletion target/riscv/debug.h
View file
Expand Up @@ -143,7 +143,8 @@ void riscv_cpu_debug_excp_handler(CPUState *cs);
bool riscv_cpu_debug_check_breakpoint(CPUState *cs);
bool riscv_cpu_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp);

void riscv_trigger_init(CPURISCVState *env);
void riscv_trigger_realize(CPURISCVState *env);
void riscv_trigger_reset_hold(CPURISCVState *env);

bool riscv_itrigger_enabled(CPURISCVState *env);
void riscv_itrigger_update_priv(CPURISCVState *env);
Expand Down
98 changes: 98 additions & 0 deletions target/riscv/helper.h
View file
Expand Up @@ -1182,3 +1182,101 @@ DEF_HELPER_5(vfwcvtbf16_f_f_v, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vfwmaccbf16_vv, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vfwmaccbf16_vf, void, ptr, ptr, i64, ptr, env, i32)

/* Vector crypto functions */
DEF_HELPER_6(vclmul_vv, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vclmul_vx, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vclmulh_vv, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vclmulh_vx, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_6(vror_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vror_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vror_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vror_vv_d, void, ptr, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vror_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vror_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vror_vx_w, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vror_vx_d, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_6(vrol_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vrol_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vrol_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vrol_vv_d, void, ptr, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vrol_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vrol_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vrol_vx_w, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vrol_vx_d, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_5(vrev8_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vrev8_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vrev8_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vrev8_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_d, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_5(vclz_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vclz_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vclz_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vclz_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_d, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vwsll_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vwsll_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vwsll_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vwsll_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vwsll_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vwsll_vx_w, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_6(vandn_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vandn_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vandn_vx_w, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vandn_vx_d, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_2(egs_check, void, i32, env)

DEF_HELPER_4(vaesef_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesef_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdf_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdf_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesem_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesem_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdm_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdm_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesz_vs, void, ptr, ptr, env, i32)
DEF_HELPER_5(vaeskf1_vi, void, ptr, ptr, i32, env, i32)
DEF_HELPER_5(vaeskf2_vi, void, ptr, ptr, i32, env, i32)

DEF_HELPER_5(vsha2ms_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2ch32_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2ch64_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2cl32_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2cl64_vv, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_5(vsm3me_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsm3c_vi, void, ptr, ptr, i32, env, i32)

DEF_HELPER_5(vghsh_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_4(vgmul_vv, void, ptr, ptr, env, i32)

DEF_HELPER_5(vsm4k_vi, void, ptr, ptr, i32, env, i32)
DEF_HELPER_4(vsm4r_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vsm4r_vs, void, ptr, ptr, env, i32)
58 changes: 58 additions & 0 deletions target/riscv/insn32.decode
View file
Expand Up @@ -37,6 +37,7 @@
%imm_u 12:s20 !function=ex_shift_12
%imm_bs 30:2 !function=ex_shift_3
%imm_rnum 20:4
%imm_z6 26:1 15:5

# Argument sets:
&empty
Expand Down Expand Up @@ -74,6 +75,7 @@
@r_rm ....... ..... ..... ... ..... ....... %rs2 %rs1 %rm %rd
@r2_rm ....... ..... ..... ... ..... ....... %rs1 %rm %rd
@r2 ....... ..... ..... ... ..... ....... &r2 %rs1 %rd
@r2_vm_1 ...... . ..... ..... ... ..... ....... &rmr vm=1 %rs2 %rd
@r2_nfvm ... ... vm:1 ..... ..... ... ..... ....... &r2nfvm %nf %rs1 %rd
@r2_vm ...... vm:1 ..... ..... ... ..... ....... &rmr %rs2 %rd
@r1_vm ...... vm:1 ..... ..... ... ..... ....... %rd
Expand All @@ -82,6 +84,7 @@
@r_vm ...... vm:1 ..... ..... ... ..... ....... &rmrr %rs2 %rs1 %rd
@r_vm_1 ...... . ..... ..... ... ..... ....... &rmrr vm=1 %rs2 %rs1 %rd
@r_vm_0 ...... . ..... ..... ... ..... ....... &rmrr vm=0 %rs2 %rs1 %rd
@r2_zimm6 ..... . vm:1 ..... ..... ... ..... ....... &rmrr %rs2 rs1=%imm_z6 %rd
@r2_zimm11 . zimm:11 ..... ... ..... ....... %rs1 %rd
@r2_zimm10 .. zimm:10 ..... ... ..... ....... %rs1 %rd
@r2_s ....... ..... ..... ... ..... ....... %rs2 %rs1
Expand Down Expand Up @@ -946,3 +949,58 @@ vfwcvtbf16_f_f_v 010010 . ..... 01101 001 ..... 1010111 @r2_vm
# *** Zvfbfwma Standard Extension ***
vfwmaccbf16_vv 111011 . ..... ..... 001 ..... 1010111 @r_vm
vfwmaccbf16_vf 111011 . ..... ..... 101 ..... 1010111 @r_vm

# *** Zvbc vector crypto extension ***
vclmul_vv 001100 . ..... ..... 010 ..... 1010111 @r_vm
vclmul_vx 001100 . ..... ..... 110 ..... 1010111 @r_vm
vclmulh_vv 001101 . ..... ..... 010 ..... 1010111 @r_vm
vclmulh_vx 001101 . ..... ..... 110 ..... 1010111 @r_vm

# *** Zvbb vector crypto extension ***
vrol_vv 010101 . ..... ..... 000 ..... 1010111 @r_vm
vrol_vx 010101 . ..... ..... 100 ..... 1010111 @r_vm
vror_vv 010100 . ..... ..... 000 ..... 1010111 @r_vm
vror_vx 010100 . ..... ..... 100 ..... 1010111 @r_vm
vror_vi 01010. . ..... ..... 011 ..... 1010111 @r2_zimm6
vbrev8_v 010010 . ..... 01000 010 ..... 1010111 @r2_vm
vrev8_v 010010 . ..... 01001 010 ..... 1010111 @r2_vm
vandn_vv 000001 . ..... ..... 000 ..... 1010111 @r_vm
vandn_vx 000001 . ..... ..... 100 ..... 1010111 @r_vm
vbrev_v 010010 . ..... 01010 010 ..... 1010111 @r2_vm
vclz_v 010010 . ..... 01100 010 ..... 1010111 @r2_vm
vctz_v 010010 . ..... 01101 010 ..... 1010111 @r2_vm
vcpop_v 010010 . ..... 01110 010 ..... 1010111 @r2_vm
vwsll_vv 110101 . ..... ..... 000 ..... 1010111 @r_vm
vwsll_vx 110101 . ..... ..... 100 ..... 1010111 @r_vm
vwsll_vi 110101 . ..... ..... 011 ..... 1010111 @r_vm

# *** Zvkned vector crypto extension ***
vaesef_vv 101000 1 ..... 00011 010 ..... 1110111 @r2_vm_1
vaesef_vs 101001 1 ..... 00011 010 ..... 1110111 @r2_vm_1
vaesdf_vv 101000 1 ..... 00001 010 ..... 1110111 @r2_vm_1
vaesdf_vs 101001 1 ..... 00001 010 ..... 1110111 @r2_vm_1
vaesem_vv 101000 1 ..... 00010 010 ..... 1110111 @r2_vm_1
vaesem_vs 101001 1 ..... 00010 010 ..... 1110111 @r2_vm_1
vaesdm_vv 101000 1 ..... 00000 010 ..... 1110111 @r2_vm_1
vaesdm_vs 101001 1 ..... 00000 010 ..... 1110111 @r2_vm_1
vaesz_vs 101001 1 ..... 00111 010 ..... 1110111 @r2_vm_1
vaeskf1_vi 100010 1 ..... ..... 010 ..... 1110111 @r_vm_1
vaeskf2_vi 101010 1 ..... ..... 010 ..... 1110111 @r_vm_1

# *** Zvknh vector crypto extension ***
vsha2ms_vv 101101 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsha2ch_vv 101110 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsha2cl_vv 101111 1 ..... ..... 010 ..... 1110111 @r_vm_1

# *** Zvksh vector crypto extension ***
vsm3me_vv 100000 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsm3c_vi 101011 1 ..... ..... 010 ..... 1110111 @r_vm_1

# *** Zvkg vector crypto extension ***
vghsh_vv 101100 1 ..... ..... 010 ..... 1110111 @r_vm_1
vgmul_vv 101000 1 ..... 10001 010 ..... 1110111 @r2_vm_1

# *** Zvksed vector crypto extension ***
vsm4k_vi 100001 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsm4r_vv 101000 1 ..... 10000 010 ..... 1110111 @r2_vm_1
vsm4r_vs 101001 1 ..... 10000 010 ..... 1110111 @r2_vm_1
171 changes: 67 additions & 104 deletions target/riscv/insn_trans/trans_rvv.c.inc
View file

Large diffs are not rendered by default.

606 changes: 606 additions & 0 deletions target/riscv/insn_trans/trans_rvvk.c.inc
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions target/riscv/insn_trans/trans_rvzfa.c.inc
View file
Expand Up @@ -470,7 +470,7 @@ bool trans_fleq_d(DisasContext *ctx, arg_fleq_d *a)
TCGv_i64 src1 = get_fpr_hs(ctx, a->rs1);
TCGv_i64 src2 = get_fpr_hs(ctx, a->rs2);

gen_helper_fltq_s(dest, cpu_env, src1, src2);
gen_helper_fleq_d(dest, cpu_env, src1, src2);
gen_set_gpr(ctx, a->rd, dest);
return true;
}
Expand All @@ -485,7 +485,7 @@ bool trans_fltq_d(DisasContext *ctx, arg_fltq_d *a)
TCGv_i64 src1 = get_fpr_hs(ctx, a->rs1);
TCGv_i64 src2 = get_fpr_hs(ctx, a->rs2);

gen_helper_fltq_s(dest, cpu_env, src1, src2);
gen_helper_fltq_d(dest, cpu_env, src1, src2);
gen_set_gpr(ctx, a->rd, dest);
return true;
}
Expand Down
201 changes: 200 additions & 1 deletion target/riscv/kvm.c
View file
Expand Up @@ -36,13 +36,20 @@
#include "exec/address-spaces.h"
#include "hw/boards.h"
#include "hw/irq.h"
#include "hw/intc/riscv_imsic.h"
#include "qemu/log.h"
#include "hw/loader.h"
#include "kvm_riscv.h"
#include "sbi_ecall_interface.h"
#include "chardev/char-fe.h"
#include "migration/migration.h"
#include "sysemu/runstate.h"
#include "hw/riscv/numa.h"

void riscv_kvm_aplic_request(void *opaque, int irq, int level)
{
kvm_set_irq(kvm_state, irq, !!level);
}

static uint64_t kvm_riscv_reg_id(CPURISCVState *env, uint64_t type,
uint64_t idx)
Expand Down Expand Up @@ -926,7 +933,15 @@ int kvm_arch_init(MachineState *ms, KVMState *s)

int kvm_arch_irqchip_create(KVMState *s)
{
return 0;
if (kvm_kernel_irqchip_split()) {
error_report("-machine kernel_irqchip=split is not supported on RISC-V.");
exit(1);
}

/*
* We can create the VAIA using the newer device control API.
*/
return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
}

int kvm_arch_process_async_events(CPUState *cs)
Expand Down Expand Up @@ -1027,6 +1042,190 @@ bool kvm_arch_cpu_check_are_resettable(void)
return true;
}

static int aia_mode;

static const char *kvm_aia_mode_str(uint64_t mode)
{
switch (mode) {
case KVM_DEV_RISCV_AIA_MODE_EMUL:
return "emul";
case KVM_DEV_RISCV_AIA_MODE_HWACCEL:
return "hwaccel";
case KVM_DEV_RISCV_AIA_MODE_AUTO:
default:
return "auto";
};
}

static char *riscv_get_kvm_aia(Object *obj, Error **errp)
{
return g_strdup(kvm_aia_mode_str(aia_mode));
}

static void riscv_set_kvm_aia(Object *obj, const char *val, Error **errp)
{
if (!strcmp(val, "emul")) {
aia_mode = KVM_DEV_RISCV_AIA_MODE_EMUL;
} else if (!strcmp(val, "hwaccel")) {
aia_mode = KVM_DEV_RISCV_AIA_MODE_HWACCEL;
} else if (!strcmp(val, "auto")) {
aia_mode = KVM_DEV_RISCV_AIA_MODE_AUTO;
} else {
error_setg(errp, "Invalid KVM AIA mode");
error_append_hint(errp, "Valid values are emul, hwaccel, and auto.\n");
}
}

void kvm_arch_accel_class_init(ObjectClass *oc)
{
object_class_property_add_str(oc, "riscv-aia", riscv_get_kvm_aia,
riscv_set_kvm_aia);
object_class_property_set_description(oc, "riscv-aia",
"Set KVM AIA mode. Valid values are "
"emul, hwaccel, and auto. Default "
"is auto.");
object_property_set_default_str(object_class_property_find(oc, "riscv-aia"),
"auto");
}

void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift,
uint64_t aia_irq_num, uint64_t aia_msi_num,
uint64_t aplic_base, uint64_t imsic_base,
uint64_t guest_num)
{
int ret, i;
int aia_fd = -1;
uint64_t default_aia_mode;
uint64_t socket_count = riscv_socket_count(machine);
uint64_t max_hart_per_socket = 0;
uint64_t socket, base_hart, hart_count, socket_imsic_base, imsic_addr;
uint64_t socket_bits, hart_bits, guest_bits;

aia_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_RISCV_AIA, false);

if (aia_fd < 0) {
error_report("Unable to create in-kernel irqchip");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_MODE,
&default_aia_mode, false, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to get current KVM AIA mode");
exit(1);
}
qemu_log("KVM AIA: default mode is %s\n",
kvm_aia_mode_str(default_aia_mode));

if (default_aia_mode != aia_mode) {
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_MODE,
&aia_mode, true, NULL);
if (ret < 0)
warn_report("KVM AIA: failed to set KVM AIA mode");
else
qemu_log("KVM AIA: set current mode to %s\n",
kvm_aia_mode_str(aia_mode));
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_SRCS,
&aia_irq_num, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set number of input irq lines");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_IDS,
&aia_msi_num, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set number of msi");
exit(1);
}

socket_bits = find_last_bit(&socket_count, BITS_PER_LONG) + 1;
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS,
&socket_bits, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set group_bits");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT,
&group_shift, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set group_shift");
exit(1);
}

guest_bits = guest_num == 0 ? 0 :
find_last_bit(&guest_num, BITS_PER_LONG) + 1;
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS,
&guest_bits, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set guest_bits");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
KVM_DEV_RISCV_AIA_ADDR_APLIC,
&aplic_base, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set the base address of APLIC");
exit(1);
}

for (socket = 0; socket < socket_count; socket++) {
socket_imsic_base = imsic_base + socket * (1U << group_shift);
hart_count = riscv_socket_hart_count(machine, socket);
base_hart = riscv_socket_first_hartid(machine, socket);

if (max_hart_per_socket < hart_count) {
max_hart_per_socket = hart_count;
}

for (i = 0; i < hart_count; i++) {
imsic_addr = socket_imsic_base + i * IMSIC_HART_SIZE(guest_bits);
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
KVM_DEV_RISCV_AIA_ADDR_IMSIC(i + base_hart),
&imsic_addr, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set the IMSIC address for hart %d", i);
exit(1);
}
}
}

hart_bits = find_last_bit(&max_hart_per_socket, BITS_PER_LONG) + 1;
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
&hart_bits, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set hart_bits");
exit(1);
}

if (kvm_has_gsi_routing()) {
for (uint64_t idx = 0; idx < aia_irq_num + 1; ++idx) {
/* KVM AIA only has one APLIC instance */
kvm_irqchip_add_irq_route(kvm_state, idx, 0, idx);
}
kvm_gsi_routing_allowed = true;
kvm_irqchip_commit_routes(kvm_state);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CTRL,
KVM_DEV_RISCV_AIA_CTRL_INIT,
NULL, true, NULL);
if (ret < 0) {
error_report("KVM AIA: initialized fail");
exit(1);
}

kvm_msi_via_irqfd_allowed = kvm_irqfds_enabled();
}
5 changes: 5 additions & 0 deletions target/riscv/kvm_riscv.h
View file
Expand Up @@ -22,5 +22,10 @@
void kvm_riscv_init_user_properties(Object *cpu_obj);
void kvm_riscv_reset_vcpu(RISCVCPU *cpu);
void kvm_riscv_set_irq(RISCVCPU *cpu, int irq, int level);
void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift,
uint64_t aia_irq_num, uint64_t aia_msi_num,
uint64_t aplic_base, uint64_t imsic_base,
uint64_t guest_num);
void riscv_kvm_aplic_request(void *opaque, int irq, int level);

#endif
4 changes: 3 additions & 1 deletion target/riscv/meson.build
View file
Expand Up @@ -16,11 +16,13 @@ riscv_ss.add(files(
'gdbstub.c',
'op_helper.c',
'vector_helper.c',
'vector_internals.c',
'bitmanip_helper.c',
'translate.c',
'm128_helper.c',
'crypto_helper.c',
'zce_helper.c'
'zce_helper.c',
'vcrypto_helper.c'
))
riscv_ss.add(when: 'CONFIG_KVM', if_true: files('kvm.c'), if_false: files('kvm-stub.c'))

Expand Down
4 changes: 4 additions & 0 deletions target/riscv/pmp.c
View file
Expand Up @@ -44,6 +44,10 @@ static inline uint8_t pmp_get_a_field(uint8_t cfg)
*/
static inline int pmp_is_locked(CPURISCVState *env, uint32_t pmp_index)
{
/* mseccfg.RLB is set */
if (MSECCFG_RLB_ISSET(env)) {
return 0;
}

if (env->pmp_state.pmp[pmp_index].cfg_reg & PMP_LOCK) {
return 1;
Expand Down
1 change: 1 addition & 0 deletions target/riscv/translate.c
View file
Expand Up @@ -1094,6 +1094,7 @@ static uint32_t opcode_at(DisasContextBase *dcbase, target_ulong pc)
#include "insn_trans/trans_rvzfa.c.inc"
#include "insn_trans/trans_rvzfh.c.inc"
#include "insn_trans/trans_rvk.c.inc"
#include "insn_trans/trans_rvvk.c.inc"
#include "insn_trans/trans_privileged.c.inc"
#include "insn_trans/trans_svinval.c.inc"
#include "insn_trans/trans_rvbf16.c.inc"
Expand Down