133 changes: 73 additions & 60 deletions qemu-nbd.c
View file
Expand Up @@ -73,8 +73,6 @@

#define MBR_SIZE 512

static char *srcpath;
static SocketAddress *saddr;
static int persistent = 0;
static enum { RUNNING, TERMINATE, TERMINATED } state;
static int shared = 1;
Expand Down Expand Up @@ -253,6 +251,29 @@ static int qemu_nbd_client_list(SocketAddress *saddr, QCryptoTLSCreds *tls,
}


struct NbdClientOpts {
char *device;
char *srcpath;
SocketAddress *saddr;
int stderr;
bool fork_process;
bool verbose;
};

static void nbd_client_release_pipe(int old_stderr)
{
/* Close stderr so that the qemu-nbd process exits. */
if (dup2(old_stderr, STDERR_FILENO) < 0) {
error_report("Could not release pipe to parent: %s",
strerror(errno));
exit(EXIT_FAILURE);
}
if (old_stderr != STDOUT_FILENO && close(old_stderr) < 0) {
error_report("Could not release qemu-nbd: %s", strerror(errno));
exit(EXIT_FAILURE);
}
}

#if HAVE_NBD_DEVICE
static void *show_parts(void *arg)
{
Expand All @@ -271,12 +292,6 @@ static void *show_parts(void *arg)
return NULL;
}

struct NbdClientOpts {
char *device;
bool fork_process;
bool verbose;
};

static void *nbd_client_thread(void *arg)
{
struct NbdClientOpts *opts = arg;
Expand All @@ -289,14 +304,14 @@ static void *nbd_client_thread(void *arg)

sioc = qio_channel_socket_new();
if (qio_channel_socket_connect_sync(sioc,
saddr,
opts->saddr,
&local_error) < 0) {
error_report_err(local_error);
goto out;
}

if (nbd_receive_negotiate(NULL, QIO_CHANNEL(sioc),
NULL, NULL, NULL, &info, &local_error) < 0) {
if (nbd_receive_negotiate(QIO_CHANNEL(sioc), NULL, NULL, NULL,
&info, &local_error) < 0) {
if (local_error) {
error_report_err(local_error);
}
Expand All @@ -320,14 +335,9 @@ static void *nbd_client_thread(void *arg)

if (opts->verbose && !opts->fork_process) {
fprintf(stderr, "NBD device %s is now connected to %s\n",
opts->device, srcpath);
opts->device, opts->srcpath);
} else {
/* Close stderr so that the qemu-nbd process exits. */
if (dup2(STDOUT_FILENO, STDERR_FILENO) < 0) {
error_report("Could not set stderr to /dev/null: %s",
strerror(errno));
exit(EXIT_FAILURE);
}
nbd_client_release_pipe(opts->stderr);
}

if (nbd_client(fd) < 0) {
Expand Down Expand Up @@ -519,7 +529,6 @@ int main(int argc, char **argv)
const char *bindto = NULL;
const char *port = NULL;
char *sockpath = NULL;
char *device = NULL;
QemuOpts *sn_opts = NULL;
const char *sn_id_or_name = NULL;
const char *sopt = "hVb:o:p:rsnc:dvk:e:f:tl:x:T:D:AB:L";
Expand Down Expand Up @@ -582,16 +591,19 @@ int main(int argc, char **argv)
const char *tlshostname = NULL;
bool imageOpts = false;
bool writethrough = false; /* Client will flush as needed. */
bool verbose = false;
bool fork_process = false;
bool list = false;
unsigned socket_activation;
const char *pid_file_name = NULL;
const char *selinux_label = NULL;
BlockExportOptions *export_opts;
#if HAVE_NBD_DEVICE
struct NbdClientOpts opts;
#endif
struct NbdClientOpts opts = {
.fork_process = false,
.verbose = false,
.device = NULL,
.srcpath = NULL,
.saddr = NULL,
.stderr = STDOUT_FILENO,
};

#ifdef CONFIG_POSIX
os_setup_early_signal_handling();
Expand Down Expand Up @@ -719,7 +731,7 @@ int main(int argc, char **argv)
disconnect = true;
break;
case 'c':
device = optarg;
opts.device = optarg;
break;
case 'e':
if (qemu_strtoi(optarg, NULL, 0, &shared) < 0 ||
Expand Down Expand Up @@ -750,7 +762,7 @@ int main(int argc, char **argv)
}
break;
case 'v':
verbose = true;
opts.verbose = true;
break;
case 'V':
version(argv[0]);
Expand Down Expand Up @@ -782,7 +794,7 @@ int main(int argc, char **argv)
tlsauthz = optarg;
break;
case QEMU_NBD_OPT_FORK:
fork_process = true;
opts.fork_process = true;
break;
case 'L':
list = true;
Expand All @@ -802,12 +814,12 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE);
}
if (export_name || export_description || dev_offset ||
device || disconnect || fmt || sn_id_or_name || bitmaps ||
opts.device || disconnect || fmt || sn_id_or_name || bitmaps ||
alloc_depth || seen_aio || seen_discard || seen_cache) {
error_report("List mode is incompatible with per-device settings");
exit(EXIT_FAILURE);
}
if (fork_process) {
if (opts.fork_process) {
error_report("List mode is incompatible with forking");
exit(EXIT_FAILURE);
}
Expand All @@ -832,7 +844,8 @@ int main(int argc, char **argv)
}
} else {
/* Using socket activation - check user didn't use -p etc. */
const char *err_msg = socket_activation_validate_opts(device, sockpath,
const char *err_msg = socket_activation_validate_opts(opts.device,
sockpath,
bindto, port,
selinux_label,
list);
Expand All @@ -850,7 +863,7 @@ int main(int argc, char **argv)
}

if (tlscredsid) {
if (device) {
if (opts.device) {
error_report("TLS is not supported with a host device");
exit(EXIT_FAILURE);
}
Expand Down Expand Up @@ -880,7 +893,7 @@ int main(int argc, char **argv)

if (selinux_label) {
#ifdef CONFIG_SELINUX
if (sockpath == NULL && device == NULL) {
if (sockpath == NULL && opts.device == NULL) {
error_report("--selinux-label is not permitted without --socket");
exit(EXIT_FAILURE);
}
Expand All @@ -891,13 +904,13 @@ int main(int argc, char **argv)
}

if (list) {
saddr = nbd_build_socket_address(sockpath, bindto, port);
return qemu_nbd_client_list(saddr, tlscreds,
opts.saddr = nbd_build_socket_address(sockpath, bindto, port);
return qemu_nbd_client_list(opts.saddr, tlscreds,
tlshostname ? tlshostname : bindto);
}

#if !HAVE_NBD_DEVICE
if (disconnect || device) {
if (disconnect || opts.device) {
error_report("Kernel /dev/nbdN support not available");
exit(EXIT_FAILURE);
}
Expand All @@ -919,7 +932,7 @@ int main(int argc, char **argv)
}
#endif

if ((device && !verbose) || fork_process) {
if ((opts.device && !opts.verbose) || opts.fork_process) {
#ifndef WIN32
g_autoptr(GError) err = NULL;
int stderr_fd[2];
Expand All @@ -944,6 +957,16 @@ int main(int argc, char **argv)

close(stderr_fd[0]);

/* Remember parent's stderr if we will be restoring it. */
if (opts.verbose /* fork_process is set */) {
opts.stderr = dup(STDERR_FILENO);
if (opts.stderr < 0) {
error_report("Could not dup original stderr: %s",
strerror(errno));
exit(EXIT_FAILURE);
}
}

ret = qemu_daemon(1, 0);
saved_errno = errno; /* dup2 will overwrite error below */

Expand Down Expand Up @@ -1002,9 +1025,9 @@ int main(int argc, char **argv)
#endif /* WIN32 */
}

if (device != NULL && sockpath == NULL) {
if (opts.device != NULL && sockpath == NULL) {
sockpath = g_malloc(128);
snprintf(sockpath, 128, SOCKET_PATH, basename(device));
snprintf(sockpath, 128, SOCKET_PATH, basename(opts.device));
}

server = qio_net_listener_new();
Expand All @@ -1023,8 +1046,8 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE);
}
#endif
saddr = nbd_build_socket_address(sockpath, bindto, port);
if (qio_net_listener_open_sync(server, saddr, backlog,
opts.saddr = nbd_build_socket_address(sockpath, bindto, port);
if (qio_net_listener_open_sync(server, opts.saddr, backlog,
&local_err) < 0) {
object_unref(OBJECT(server));
error_report_err(local_err);
Expand Down Expand Up @@ -1059,27 +1082,27 @@ int main(int argc, char **argv)
bdrv_init();
atexit(qemu_nbd_shutdown);

srcpath = argv[optind];
opts.srcpath = argv[optind];
if (imageOpts) {
QemuOpts *opts;
QemuOpts *o;
if (fmt) {
error_report("--image-opts and -f are mutually exclusive");
exit(EXIT_FAILURE);
}
opts = qemu_opts_parse_noisily(&file_opts, srcpath, true);
if (!opts) {
o = qemu_opts_parse_noisily(&file_opts, opts.srcpath, true);
if (!o) {
qemu_opts_reset(&file_opts);
exit(EXIT_FAILURE);
}
options = qemu_opts_to_qdict(opts, NULL);
options = qemu_opts_to_qdict(o, NULL);
qemu_opts_reset(&file_opts);
blk = blk_new_open(NULL, NULL, options, flags, &local_err);
} else {
if (fmt) {
options = qdict_new();
qdict_put_str(options, "driver", fmt);
}
blk = blk_new_open(srcpath, NULL, options, flags, &local_err);
blk = blk_new_open(opts.srcpath, NULL, options, flags, &local_err);
}

if (!blk) {
Expand Down Expand Up @@ -1145,15 +1168,9 @@ int main(int argc, char **argv)
blk_exp_add(export_opts, &error_fatal);
qapi_free_BlockExportOptions(export_opts);

if (device) {
if (opts.device) {
#if HAVE_NBD_DEVICE
int ret;
opts = (struct NbdClientOpts) {
.device = device,
.fork_process = fork_process,
.verbose = verbose,
};

ret = pthread_create(&client_thread, NULL, nbd_client_thread, &opts);
if (ret != 0) {
error_report("Failed to create client thread: %s", strerror(ret));
Expand All @@ -1179,12 +1196,8 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE);
}

if (fork_process) {
if (dup2(STDOUT_FILENO, STDERR_FILENO) < 0) {
error_report("Could not set stderr to /dev/null: %s",
strerror(errno));
exit(EXIT_FAILURE);
}
if (opts.fork_process) {
nbd_client_release_pipe(opts.stderr);
}

state = RUNNING;
Expand All @@ -1203,7 +1216,7 @@ int main(int argc, char **argv)

qemu_opts_del(sn_opts);

if (device) {
if (opts.device) {
void *ret;
pthread_join(client_thread, &ret);
exit(ret != NULL);
Expand Down
70 changes: 69 additions & 1 deletion qemu-options.hx
View file
Expand Up @@ -2863,6 +2863,19 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
" VALE port (created on the fly) called 'name' ('nmname' is name of the \n"
" netmap device, defaults to '/dev/netmap')\n"
#endif
#ifdef CONFIG_AF_XDP
"-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off]\n"
" [,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]\n"
" attach to the existing network interface 'name' with AF_XDP socket\n"
" use 'mode=MODE' to specify an XDP program attach mode\n"
" use 'force-copy=on|off' to force XDP copy mode even if device supports zero-copy (default: off)\n"
" use 'inhibit=on|off' to inhibit loading of a default XDP program (default: off)\n"
" with inhibit=on,\n"
" use 'sock-fds' to provide file descriptors for already open AF_XDP sockets\n"
" added to a socket map in XDP program. One socket per queue.\n"
" use 'queues=n' to specify how many queues of a multiqueue interface should be used\n"
" use 'start-queue=m' to specify the first queue that should be used\n"
#endif
#ifdef CONFIG_POSIX
"-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
" configure a vhost-user network, backed by a chardev 'dev'\n"
Expand Down Expand Up @@ -2908,6 +2921,9 @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
#ifdef CONFIG_NETMAP
"netmap|"
#endif
#ifdef CONFIG_AF_XDP
"af-xdp|"
#endif
#ifdef CONFIG_POSIX
"vhost-user|"
#endif
Expand Down Expand Up @@ -2936,14 +2952,17 @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
#ifdef CONFIG_NETMAP
"netmap|"
#endif
#ifdef CONFIG_AF_XDP
"af-xdp|"
#endif
#ifdef CONFIG_VMNET
"vmnet-host|vmnet-shared|vmnet-bridged|"
#endif
"socket][,option][,option][,...]\n"
" old way to initialize a host network interface\n"
" (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
SRST
``-nic [tap|bridge|user|l2tpv3|vde|netmap|vhost-user|socket][,...][,mac=macaddr][,model=mn]``
``-nic [tap|bridge|user|l2tpv3|vde|netmap|af-xdp|vhost-user|socket][,...][,mac=macaddr][,model=mn]``
This option is a shortcut for configuring both the on-board
(default) guest NIC hardware and the host network backend in one go.
The host backend options are the same as with the corresponding
Expand Down Expand Up @@ -3357,6 +3376,55 @@ SRST
# launch QEMU instance
|qemu_system| linux.img -nic vde,sock=/tmp/myswitch
``-netdev af-xdp,id=str,ifname=name[,mode=native|skb][,force-copy=on|off][,queues=n][,start-queue=m][,inhibit=on|off][,sock-fds=x:y:...:z]``
Configure AF_XDP backend to connect to a network interface 'name'
using AF_XDP socket. A specific program attach mode for a default
XDP program can be forced with 'mode', defaults to best-effort,
where the likely most performant mode will be in use. Number of queues
'n' should generally match the number or queues in the interface,
defaults to 1. Traffic arriving on non-configured device queues will
not be delivered to the network backend.
.. parsed-literal::
# set number of queues to 4
ethtool -L eth0 combined 4
# launch QEMU instance
|qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
-netdev af-xdp,id=n1,ifname=eth0,queues=4
'start-queue' option can be specified if a particular range of queues
[m, m + n] should be in use. For example, this is may be necessary in
order to use certain NICs in native mode. Kernel allows the driver to
create a separate set of XDP queues on top of regular ones, and only
these queues can be used for AF_XDP sockets. NICs that work this way
may also require an additional traffic redirection with ethtool to these
special queues.
.. parsed-literal::
# set number of queues to 1
ethtool -L eth0 combined 1
# redirect all the traffic to the second queue (id: 1)
# note: drivers may require non-empty key/mask pair.
ethtool -N eth0 flow-type ether \\
dst 00:00:00:00:00:00 m FF:FF:FF:FF:FF:FE action 1
ethtool -N eth0 flow-type ether \\
dst 00:00:00:00:00:01 m FF:FF:FF:FF:FF:FE action 1
# launch QEMU instance
|qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
-netdev af-xdp,id=n1,ifname=eth0,queues=1,start-queue=1
XDP program can also be loaded externally. In this case 'inhibit' option
should be set to 'on' and 'sock-fds' provided with file descriptors for
already open but not bound XDP sockets already added to a socket map for
corresponding queues. One socket per queue.
.. parsed-literal::
|qemu_system| linux.img -device virtio-net-pci,netdev=n1 \\
-netdev af-xdp,id=n1,ifname=eth0,queues=3,inhibit=on,sock-fds=15:16:17
``-netdev vhost-user,chardev=id[,vhostforce=on|off][,queues=n]``
Establish a vhost-user netdev, backed by a chardev id. The chardev
should be a unix domain socket backed one. The vhost-user uses a
Expand Down
1 change: 1 addition & 0 deletions scripts/ci/org.centos/stream/8/x86_64/configure
View file
Expand Up @@ -35,6 +35,7 @@
--block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
--with-coroutine=ucontext \
--tls-priority=@QEMU,SYSTEM \
--disable-af-xdp \
--disable-attr \
--disable-auth-pam \
--disable-avx2 \
Expand Down
3 changes: 3 additions & 0 deletions scripts/meson-buildoptions.sh
View file
Expand Up @@ -76,6 +76,7 @@ meson_options_help() {
printf "%s\n" 'disabled with --disable-FEATURE, default is enabled if available'
printf "%s\n" '(unless built with --without-default-features):'
printf "%s\n" ''
printf "%s\n" ' af-xdp AF_XDP network backend support'
printf "%s\n" ' alsa ALSA sound support'
printf "%s\n" ' attr attr/xattr support'
printf "%s\n" ' auth-pam PAM access control'
Expand Down Expand Up @@ -208,6 +209,8 @@ meson_options_help() {
}
_meson_option_parse() {
case $1 in
--enable-af-xdp) printf "%s" -Daf_xdp=enabled ;;
--disable-af-xdp) printf "%s" -Daf_xdp=disabled ;;
--enable-alsa) printf "%s" -Dalsa=enabled ;;
--disable-alsa) printf "%s" -Dalsa=disabled ;;
--enable-attr) printf "%s" -Dattr=enabled ;;
Expand Down
4 changes: 1 addition & 3 deletions scsi/qemu-pr-helper.c
View file
Expand Up @@ -735,8 +735,7 @@ static void coroutine_fn prh_co_entry(void *opaque)

qio_channel_set_blocking(QIO_CHANNEL(client->ioc),
false, NULL);
qio_channel_attach_aio_context(QIO_CHANNEL(client->ioc),
qemu_get_aio_context());
qio_channel_set_follow_coroutine_ctx(QIO_CHANNEL(client->ioc), true);

/* A very simple negotiation for future extensibility. No features
* are defined so write 0.
Expand Down Expand Up @@ -796,7 +795,6 @@ static void coroutine_fn prh_co_entry(void *opaque)
}

out:
qio_channel_detach_aio_context(QIO_CHANNEL(client->ioc));
object_unref(OBJECT(client->ioc));
g_free(client);
}
Expand Down
10 changes: 2 additions & 8 deletions target/arm/tcg/crypto_helper.c
View file
Expand Up @@ -614,10 +614,7 @@ static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm)
CR_ST_WORD(d, (i + 3) % 4) ^
CR_ST_WORD(n, i);

t = sm4_sbox[t & 0xff] |
sm4_sbox[(t >> 8) & 0xff] << 8 |
sm4_sbox[(t >> 16) & 0xff] << 16 |
sm4_sbox[(t >> 24) & 0xff] << 24;
t = sm4_subword(t);

CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
rol32(t, 24);
Expand Down Expand Up @@ -651,10 +648,7 @@ static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm)
CR_ST_WORD(d, (i + 3) % 4) ^
CR_ST_WORD(m, i);

t = sm4_sbox[t & 0xff] |
sm4_sbox[(t >> 8) & 0xff] << 8 |
sm4_sbox[(t >> 16) & 0xff] << 16 |
sm4_sbox[(t >> 24) & 0xff] << 24;
t = sm4_subword(t);

CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
}
Expand Down
1 change: 1 addition & 0 deletions target/riscv/cpu-qom.h
View file
Expand Up @@ -30,6 +30,7 @@
#define CPU_RESOLVING_TYPE TYPE_RISCV_CPU

#define TYPE_RISCV_CPU_ANY RISCV_CPU_TYPE_NAME("any")
#define TYPE_RISCV_CPU_MAX RISCV_CPU_TYPE_NAME("max")
#define TYPE_RISCV_CPU_BASE32 RISCV_CPU_TYPE_NAME("rv32")
#define TYPE_RISCV_CPU_BASE64 RISCV_CPU_TYPE_NAME("rv64")
#define TYPE_RISCV_CPU_BASE128 RISCV_CPU_TYPE_NAME("x-rv128")
Expand Down
625 changes: 465 additions & 160 deletions target/riscv/cpu.c
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions target/riscv/cpu.h
View file
Expand Up @@ -62,6 +62,8 @@
const char *riscv_get_misa_ext_name(uint32_t bit);
const char *riscv_get_misa_ext_description(uint32_t bit);

#define CPU_CFG_OFFSET(_prop) offsetof(struct RISCVCPUConfig, _prop)

/* Privileged specification version */
enum {
PRIV_VERSION_1_10_0 = 0,
Expand Down
8 changes: 4 additions & 4 deletions target/riscv/cpu_bits.h
View file
Expand Up @@ -745,12 +745,12 @@ typedef enum RISCVException {
#define MENVCFG_CBIE (3UL << 4)
#define MENVCFG_CBCFE BIT(6)
#define MENVCFG_CBZE BIT(7)
#define MENVCFG_HADE (1ULL << 61)
#define MENVCFG_ADUE (1ULL << 61)
#define MENVCFG_PBMTE (1ULL << 62)
#define MENVCFG_STCE (1ULL << 63)

/* For RV32 */
#define MENVCFGH_HADE BIT(29)
#define MENVCFGH_ADUE BIT(29)
#define MENVCFGH_PBMTE BIT(30)
#define MENVCFGH_STCE BIT(31)

Expand All @@ -763,12 +763,12 @@ typedef enum RISCVException {
#define HENVCFG_CBIE MENVCFG_CBIE
#define HENVCFG_CBCFE MENVCFG_CBCFE
#define HENVCFG_CBZE MENVCFG_CBZE
#define HENVCFG_HADE MENVCFG_HADE
#define HENVCFG_ADUE MENVCFG_ADUE
#define HENVCFG_PBMTE MENVCFG_PBMTE
#define HENVCFG_STCE MENVCFG_STCE

/* For RV32 */
#define HENVCFGH_HADE MENVCFGH_HADE
#define HENVCFGH_ADUE MENVCFGH_ADUE
#define HENVCFGH_PBMTE MENVCFGH_PBMTE
#define HENVCFGH_STCE MENVCFGH_STCE

Expand Down
9 changes: 9 additions & 0 deletions target/riscv/cpu_cfg.h
View file
Expand Up @@ -66,6 +66,7 @@ struct RISCVCPUConfig {
bool ext_icbom;
bool ext_icboz;
bool ext_zicond;
bool ext_zihintntl;
bool ext_zihintpause;
bool ext_smstateen;
bool ext_sstc;
Expand All @@ -85,6 +86,14 @@ struct RISCVCPUConfig {
bool ext_zve32f;
bool ext_zve64f;
bool ext_zve64d;
bool ext_zvbb;
bool ext_zvbc;
bool ext_zvkg;
bool ext_zvkned;
bool ext_zvknha;
bool ext_zvknhb;
bool ext_zvksed;
bool ext_zvksh;
bool ext_zmmul;
bool ext_zvfbfmin;
bool ext_zvfbfwma;
Expand Down
6 changes: 3 additions & 3 deletions target/riscv/cpu_helper.c
View file
Expand Up @@ -861,11 +861,11 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,
}

bool pbmte = env->menvcfg & MENVCFG_PBMTE;
bool hade = env->menvcfg & MENVCFG_HADE;
bool adue = env->menvcfg & MENVCFG_ADUE;

if (first_stage && two_stage && env->virt_enabled) {
pbmte = pbmte && (env->henvcfg & HENVCFG_PBMTE);
hade = hade && (env->henvcfg & HENVCFG_HADE);
adue = adue && (env->henvcfg & HENVCFG_ADUE);
}

int ptshift = (levels - 1) * ptidxbits;
Expand Down Expand Up @@ -1026,7 +1026,7 @@ static int get_physical_address(CPURISCVState *env, hwaddr *physical,

/* Page table updates need to be atomic with MTTCG enabled */
if (updated_pte != pte && !is_debug) {
if (!hade) {
if (!adue) {
return TRANSLATE_FAIL;
}

Expand Down
51 changes: 9 additions & 42 deletions target/riscv/crypto_helper.c
View file
Expand Up @@ -25,53 +25,27 @@
#include "crypto/aes-round.h"
#include "crypto/sm4.h"

#define AES_XTIME(a) \
((a << 1) ^ ((a & 0x80) ? 0x1b : 0))

#define AES_GFMUL(a, b) (( \
(((b) & 0x1) ? (a) : 0) ^ \
(((b) & 0x2) ? AES_XTIME(a) : 0) ^ \
(((b) & 0x4) ? AES_XTIME(AES_XTIME(a)) : 0) ^ \
(((b) & 0x8) ? AES_XTIME(AES_XTIME(AES_XTIME(a))) : 0)) & 0xFF)

static inline uint32_t aes_mixcolumn_byte(uint8_t x, bool fwd)
{
uint32_t u;

if (fwd) {
u = (AES_GFMUL(x, 3) << 24) | (x << 16) | (x << 8) |
(AES_GFMUL(x, 2) << 0);
} else {
u = (AES_GFMUL(x, 0xb) << 24) | (AES_GFMUL(x, 0xd) << 16) |
(AES_GFMUL(x, 0x9) << 8) | (AES_GFMUL(x, 0xe) << 0);
}
return u;
}

#define sext32_xlen(x) (target_ulong)(int32_t)(x)

static inline target_ulong aes32_operation(target_ulong shamt,
target_ulong rs1, target_ulong rs2,
bool enc, bool mix)
{
uint8_t si = rs2 >> shamt;
uint8_t so;
uint32_t mixed;
target_ulong res;

if (enc) {
so = AES_sbox[si];
if (mix) {
mixed = aes_mixcolumn_byte(so, true);
mixed = be32_to_cpu(AES_Te0[si]);
} else {
mixed = so;
mixed = AES_sbox[si];
}
} else {
so = AES_isbox[si];
if (mix) {
mixed = aes_mixcolumn_byte(so, false);
mixed = be32_to_cpu(AES_Td0[si]);
} else {
mixed = so;
mixed = AES_isbox[si];
}
}
mixed = rol32(mixed, shamt);
Expand Down Expand Up @@ -174,24 +148,17 @@ target_ulong HELPER(aes64ks1i)(target_ulong rs1, target_ulong rnum)

uint8_t enc_rnum = rnum;
uint32_t temp = (RS1 >> 32) & 0xFFFFFFFF;
uint8_t rcon_ = 0;
target_ulong result;
AESState t, rc = {};

if (enc_rnum != 0xA) {
temp = ror32(temp, 8); /* Rotate right by 8 */
rcon_ = round_consts[enc_rnum];
rc.w[0] = rc.w[1] = round_consts[enc_rnum];
}

temp = ((uint32_t)AES_sbox[(temp >> 24) & 0xFF] << 24) |
((uint32_t)AES_sbox[(temp >> 16) & 0xFF] << 16) |
((uint32_t)AES_sbox[(temp >> 8) & 0xFF] << 8) |
((uint32_t)AES_sbox[(temp >> 0) & 0xFF] << 0);

temp ^= rcon_;
t.w[0] = t.w[1] = t.w[2] = t.w[3] = temp;
aesenc_SB_SR_AK(&t, &t, &rc, false);

result = ((uint64_t)temp << 32) | temp;

return result;
return t.d[0];
}

target_ulong HELPER(aes64im)(target_ulong rs1)
Expand Down
54 changes: 35 additions & 19 deletions target/riscv/csr.c
View file
Expand Up @@ -1684,7 +1684,7 @@ static int rmw_iprio(target_ulong xlen,
static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val,
target_ulong new_val, target_ulong wr_mask)
{
bool virt;
bool virt, isel_reserved;
uint8_t *iprio;
int ret = -EINVAL;
target_ulong priv, isel, vgein;
Expand All @@ -1694,6 +1694,7 @@ static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val,

/* Decode register details from CSR number */
virt = false;
isel_reserved = false;
switch (csrno) {
case CSR_MIREG:
iprio = env->miprio;
Expand Down Expand Up @@ -1738,11 +1739,13 @@ static int rmw_xireg(CPURISCVState *env, int csrno, target_ulong *val,
riscv_cpu_mxl_bits(env)),
val, new_val, wr_mask);
}
} else {
isel_reserved = true;
}

done:
if (ret) {
return (env->virt_enabled && virt) ?
return (env->virt_enabled && virt && !isel_reserved) ?
RISCV_EXCP_VIRT_INSTRUCTION_FAULT : RISCV_EXCP_ILLEGAL_INST;
}
return RISCV_EXCP_NONE;
Expand Down Expand Up @@ -1833,8 +1836,11 @@ static RISCVException write_mcountinhibit(CPURISCVState *env, int csrno,
{
int cidx;
PMUCTRState *counter;
RISCVCPU *cpu = env_archcpu(env);

env->mcountinhibit = val;
/* WARL register - disable unavailable counters; TM bit is always 0 */
env->mcountinhibit =
val & (cpu->pmu_avail_ctrs | COUNTEREN_CY | COUNTEREN_IR);

/* Check if any other counter is also monitoring cycles/instructions */
for (cidx = 0; cidx < RV_MAX_MHPMCOUNTERS; cidx++) {
Expand All @@ -1857,7 +1863,11 @@ static RISCVException read_mcounteren(CPURISCVState *env, int csrno,
static RISCVException write_mcounteren(CPURISCVState *env, int csrno,
target_ulong val)
{
env->mcounteren = val;
RISCVCPU *cpu = env_archcpu(env);

/* WARL register - disable unavailable counters */
env->mcounteren = val & (cpu->pmu_avail_ctrs | COUNTEREN_CY | COUNTEREN_TM |
COUNTEREN_IR);
return RISCV_EXCP_NONE;
}

Expand Down Expand Up @@ -1950,7 +1960,7 @@ static RISCVException write_menvcfg(CPURISCVState *env, int csrno,
if (riscv_cpu_mxl(env) == MXL_RV64) {
mask |= (cfg->ext_svpbmt ? MENVCFG_PBMTE : 0) |
(cfg->ext_sstc ? MENVCFG_STCE : 0) |
(cfg->ext_svadu ? MENVCFG_HADE : 0);
(cfg->ext_svadu ? MENVCFG_ADUE : 0);
}
env->menvcfg = (env->menvcfg & ~mask) | (val & mask);

Expand All @@ -1970,7 +1980,7 @@ static RISCVException write_menvcfgh(CPURISCVState *env, int csrno,
const RISCVCPUConfig *cfg = riscv_cpu_cfg(env);
uint64_t mask = (cfg->ext_svpbmt ? MENVCFG_PBMTE : 0) |
(cfg->ext_sstc ? MENVCFG_STCE : 0) |
(cfg->ext_svadu ? MENVCFG_HADE : 0);
(cfg->ext_svadu ? MENVCFG_ADUE : 0);
uint64_t valh = (uint64_t)val << 32;

env->menvcfg = (env->menvcfg & ~mask) | (valh & mask);
Expand Down Expand Up @@ -2022,7 +2032,7 @@ static RISCVException read_henvcfg(CPURISCVState *env, int csrno,
* henvcfg.stce is read_only 0 when menvcfg.stce = 0
* henvcfg.hade is read_only 0 when menvcfg.hade = 0
*/
*val = env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE) |
*val = env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_ADUE) |
env->menvcfg);
return RISCV_EXCP_NONE;
}
Expand All @@ -2039,7 +2049,7 @@ static RISCVException write_henvcfg(CPURISCVState *env, int csrno,
}

if (riscv_cpu_mxl(env) == MXL_RV64) {
mask |= env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE);
mask |= env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_ADUE);
}

env->henvcfg = (env->henvcfg & ~mask) | (val & mask);
Expand All @@ -2057,7 +2067,7 @@ static RISCVException read_henvcfgh(CPURISCVState *env, int csrno,
return ret;
}

*val = (env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_HADE) |
*val = (env->henvcfg & (~(HENVCFG_PBMTE | HENVCFG_STCE | HENVCFG_ADUE) |
env->menvcfg)) >> 32;
return RISCV_EXCP_NONE;
}
Expand All @@ -2066,7 +2076,7 @@ static RISCVException write_henvcfgh(CPURISCVState *env, int csrno,
target_ulong val)
{
uint64_t mask = env->menvcfg & (HENVCFG_PBMTE | HENVCFG_STCE |
HENVCFG_HADE);
HENVCFG_ADUE);
uint64_t valh = (uint64_t)val << 32;
RISCVException ret;

Expand Down Expand Up @@ -3907,21 +3917,27 @@ static RISCVException riscv_csrrw_do64(CPURISCVState *env, int csrno,
target_ulong write_mask)
{
RISCVException ret;
target_ulong old_value;
target_ulong old_value = 0;

/* execute combined read/write operation if it exists */
if (csr_ops[csrno].op) {
return csr_ops[csrno].op(env, csrno, ret_value, new_value, write_mask);
}

/* if no accessor exists then return failure */
if (!csr_ops[csrno].read) {
return RISCV_EXCP_ILLEGAL_INST;
}
/* read old value */
ret = csr_ops[csrno].read(env, csrno, &old_value);
if (ret != RISCV_EXCP_NONE) {
return ret;
/*
* ret_value == NULL means that rd=x0 and we're coming from helper_csrw()
* and we can't throw side effects caused by CSR reads.
*/
if (ret_value) {
/* if no accessor exists then return failure */
if (!csr_ops[csrno].read) {
return RISCV_EXCP_ILLEGAL_INST;
}
/* read old value */
ret = csr_ops[csrno].read(env, csrno, &old_value);
if (ret != RISCV_EXCP_NONE) {
return ret;
}
}

/* write value if writable and write mask set, otherwise drop writes */
Expand Down
15 changes: 12 additions & 3 deletions target/riscv/debug.c
View file
Expand Up @@ -903,7 +903,17 @@ bool riscv_cpu_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
return false;
}

void riscv_trigger_init(CPURISCVState *env)
void riscv_trigger_realize(CPURISCVState *env)
{
int i;

for (i = 0; i < RV_MAX_TRIGGERS; i++) {
env->itrigger_timer[i] = timer_new_ns(QEMU_CLOCK_VIRTUAL,
riscv_itrigger_timer_cb, env);
}
}

void riscv_trigger_reset_hold(CPURISCVState *env)
{
target_ulong tdata1 = build_tdata1(env, TRIGGER_TYPE_AD_MATCH, 0, 0);
int i;
Expand All @@ -928,7 +938,6 @@ void riscv_trigger_init(CPURISCVState *env)
env->tdata3[i] = 0;
env->cpu_breakpoint[i] = NULL;
env->cpu_watchpoint[i] = NULL;
env->itrigger_timer[i] = timer_new_ns(QEMU_CLOCK_VIRTUAL,
riscv_itrigger_timer_cb, env);
timer_del(env->itrigger_timer[i]);
}
}
3 changes: 2 additions & 1 deletion target/riscv/debug.h
View file
Expand Up @@ -143,7 +143,8 @@ void riscv_cpu_debug_excp_handler(CPUState *cs);
bool riscv_cpu_debug_check_breakpoint(CPUState *cs);
bool riscv_cpu_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp);

void riscv_trigger_init(CPURISCVState *env);
void riscv_trigger_realize(CPURISCVState *env);
void riscv_trigger_reset_hold(CPURISCVState *env);

bool riscv_itrigger_enabled(CPURISCVState *env);
void riscv_itrigger_update_priv(CPURISCVState *env);
Expand Down
98 changes: 98 additions & 0 deletions target/riscv/helper.h
View file
Expand Up @@ -1182,3 +1182,101 @@ DEF_HELPER_5(vfwcvtbf16_f_f_v, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vfwmaccbf16_vv, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vfwmaccbf16_vf, void, ptr, ptr, i64, ptr, env, i32)

/* Vector crypto functions */
DEF_HELPER_6(vclmul_vv, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vclmul_vx, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vclmulh_vv, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vclmulh_vx, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_6(vror_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vror_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vror_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vror_vv_d, void, ptr, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vror_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vror_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vror_vx_w, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vror_vx_d, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_6(vrol_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vrol_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vrol_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vrol_vv_d, void, ptr, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vrol_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vrol_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vrol_vx_w, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vrol_vx_d, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_5(vrev8_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vrev8_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vrev8_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vrev8_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev8_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vbrev_v_d, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_5(vclz_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vclz_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vclz_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vclz_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vctz_v_d, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_b, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_h, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_w, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vcpop_v_d, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_6(vwsll_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vwsll_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vwsll_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vwsll_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vwsll_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vwsll_vx_w, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_6(vandn_vv_b, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vv_h, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vv_w, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vv_d, void, ptr, ptr, ptr, ptr, env, i32)
DEF_HELPER_6(vandn_vx_b, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vandn_vx_h, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vandn_vx_w, void, ptr, ptr, tl, ptr, env, i32)
DEF_HELPER_6(vandn_vx_d, void, ptr, ptr, tl, ptr, env, i32)

DEF_HELPER_2(egs_check, void, i32, env)

DEF_HELPER_4(vaesef_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesef_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdf_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdf_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesem_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesem_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdm_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesdm_vs, void, ptr, ptr, env, i32)
DEF_HELPER_4(vaesz_vs, void, ptr, ptr, env, i32)
DEF_HELPER_5(vaeskf1_vi, void, ptr, ptr, i32, env, i32)
DEF_HELPER_5(vaeskf2_vi, void, ptr, ptr, i32, env, i32)

DEF_HELPER_5(vsha2ms_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2ch32_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2ch64_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2cl32_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsha2cl64_vv, void, ptr, ptr, ptr, env, i32)

DEF_HELPER_5(vsm3me_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_5(vsm3c_vi, void, ptr, ptr, i32, env, i32)

DEF_HELPER_5(vghsh_vv, void, ptr, ptr, ptr, env, i32)
DEF_HELPER_4(vgmul_vv, void, ptr, ptr, env, i32)

DEF_HELPER_5(vsm4k_vi, void, ptr, ptr, i32, env, i32)
DEF_HELPER_4(vsm4r_vv, void, ptr, ptr, env, i32)
DEF_HELPER_4(vsm4r_vs, void, ptr, ptr, env, i32)
58 changes: 58 additions & 0 deletions target/riscv/insn32.decode
View file
Expand Up @@ -37,6 +37,7 @@
%imm_u 12:s20 !function=ex_shift_12
%imm_bs 30:2 !function=ex_shift_3
%imm_rnum 20:4
%imm_z6 26:1 15:5

# Argument sets:
&empty
Expand Down Expand Up @@ -74,6 +75,7 @@
@r_rm ....... ..... ..... ... ..... ....... %rs2 %rs1 %rm %rd
@r2_rm ....... ..... ..... ... ..... ....... %rs1 %rm %rd
@r2 ....... ..... ..... ... ..... ....... &r2 %rs1 %rd
@r2_vm_1 ...... . ..... ..... ... ..... ....... &rmr vm=1 %rs2 %rd
@r2_nfvm ... ... vm:1 ..... ..... ... ..... ....... &r2nfvm %nf %rs1 %rd
@r2_vm ...... vm:1 ..... ..... ... ..... ....... &rmr %rs2 %rd
@r1_vm ...... vm:1 ..... ..... ... ..... ....... %rd
Expand All @@ -82,6 +84,7 @@
@r_vm ...... vm:1 ..... ..... ... ..... ....... &rmrr %rs2 %rs1 %rd
@r_vm_1 ...... . ..... ..... ... ..... ....... &rmrr vm=1 %rs2 %rs1 %rd
@r_vm_0 ...... . ..... ..... ... ..... ....... &rmrr vm=0 %rs2 %rs1 %rd
@r2_zimm6 ..... . vm:1 ..... ..... ... ..... ....... &rmrr %rs2 rs1=%imm_z6 %rd
@r2_zimm11 . zimm:11 ..... ... ..... ....... %rs1 %rd
@r2_zimm10 .. zimm:10 ..... ... ..... ....... %rs1 %rd
@r2_s ....... ..... ..... ... ..... ....... %rs2 %rs1
Expand Down Expand Up @@ -946,3 +949,58 @@ vfwcvtbf16_f_f_v 010010 . ..... 01101 001 ..... 1010111 @r2_vm
# *** Zvfbfwma Standard Extension ***
vfwmaccbf16_vv 111011 . ..... ..... 001 ..... 1010111 @r_vm
vfwmaccbf16_vf 111011 . ..... ..... 101 ..... 1010111 @r_vm

# *** Zvbc vector crypto extension ***
vclmul_vv 001100 . ..... ..... 010 ..... 1010111 @r_vm
vclmul_vx 001100 . ..... ..... 110 ..... 1010111 @r_vm
vclmulh_vv 001101 . ..... ..... 010 ..... 1010111 @r_vm
vclmulh_vx 001101 . ..... ..... 110 ..... 1010111 @r_vm

# *** Zvbb vector crypto extension ***
vrol_vv 010101 . ..... ..... 000 ..... 1010111 @r_vm
vrol_vx 010101 . ..... ..... 100 ..... 1010111 @r_vm
vror_vv 010100 . ..... ..... 000 ..... 1010111 @r_vm
vror_vx 010100 . ..... ..... 100 ..... 1010111 @r_vm
vror_vi 01010. . ..... ..... 011 ..... 1010111 @r2_zimm6
vbrev8_v 010010 . ..... 01000 010 ..... 1010111 @r2_vm
vrev8_v 010010 . ..... 01001 010 ..... 1010111 @r2_vm
vandn_vv 000001 . ..... ..... 000 ..... 1010111 @r_vm
vandn_vx 000001 . ..... ..... 100 ..... 1010111 @r_vm
vbrev_v 010010 . ..... 01010 010 ..... 1010111 @r2_vm
vclz_v 010010 . ..... 01100 010 ..... 1010111 @r2_vm
vctz_v 010010 . ..... 01101 010 ..... 1010111 @r2_vm
vcpop_v 010010 . ..... 01110 010 ..... 1010111 @r2_vm
vwsll_vv 110101 . ..... ..... 000 ..... 1010111 @r_vm
vwsll_vx 110101 . ..... ..... 100 ..... 1010111 @r_vm
vwsll_vi 110101 . ..... ..... 011 ..... 1010111 @r_vm

# *** Zvkned vector crypto extension ***
vaesef_vv 101000 1 ..... 00011 010 ..... 1110111 @r2_vm_1
vaesef_vs 101001 1 ..... 00011 010 ..... 1110111 @r2_vm_1
vaesdf_vv 101000 1 ..... 00001 010 ..... 1110111 @r2_vm_1
vaesdf_vs 101001 1 ..... 00001 010 ..... 1110111 @r2_vm_1
vaesem_vv 101000 1 ..... 00010 010 ..... 1110111 @r2_vm_1
vaesem_vs 101001 1 ..... 00010 010 ..... 1110111 @r2_vm_1
vaesdm_vv 101000 1 ..... 00000 010 ..... 1110111 @r2_vm_1
vaesdm_vs 101001 1 ..... 00000 010 ..... 1110111 @r2_vm_1
vaesz_vs 101001 1 ..... 00111 010 ..... 1110111 @r2_vm_1
vaeskf1_vi 100010 1 ..... ..... 010 ..... 1110111 @r_vm_1
vaeskf2_vi 101010 1 ..... ..... 010 ..... 1110111 @r_vm_1

# *** Zvknh vector crypto extension ***
vsha2ms_vv 101101 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsha2ch_vv 101110 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsha2cl_vv 101111 1 ..... ..... 010 ..... 1110111 @r_vm_1

# *** Zvksh vector crypto extension ***
vsm3me_vv 100000 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsm3c_vi 101011 1 ..... ..... 010 ..... 1110111 @r_vm_1

# *** Zvkg vector crypto extension ***
vghsh_vv 101100 1 ..... ..... 010 ..... 1110111 @r_vm_1
vgmul_vv 101000 1 ..... 10001 010 ..... 1110111 @r2_vm_1

# *** Zvksed vector crypto extension ***
vsm4k_vi 100001 1 ..... ..... 010 ..... 1110111 @r_vm_1
vsm4r_vv 101000 1 ..... 10000 010 ..... 1110111 @r2_vm_1
vsm4r_vs 101001 1 ..... 10000 010 ..... 1110111 @r2_vm_1
171 changes: 67 additions & 104 deletions target/riscv/insn_trans/trans_rvv.c.inc
View file

Large diffs are not rendered by default.

606 changes: 606 additions & 0 deletions target/riscv/insn_trans/trans_rvvk.c.inc
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions target/riscv/insn_trans/trans_rvzfa.c.inc
View file
Expand Up @@ -470,7 +470,7 @@ bool trans_fleq_d(DisasContext *ctx, arg_fleq_d *a)
TCGv_i64 src1 = get_fpr_hs(ctx, a->rs1);
TCGv_i64 src2 = get_fpr_hs(ctx, a->rs2);

gen_helper_fltq_s(dest, cpu_env, src1, src2);
gen_helper_fleq_d(dest, cpu_env, src1, src2);
gen_set_gpr(ctx, a->rd, dest);
return true;
}
Expand All @@ -485,7 +485,7 @@ bool trans_fltq_d(DisasContext *ctx, arg_fltq_d *a)
TCGv_i64 src1 = get_fpr_hs(ctx, a->rs1);
TCGv_i64 src2 = get_fpr_hs(ctx, a->rs2);

gen_helper_fltq_s(dest, cpu_env, src1, src2);
gen_helper_fltq_d(dest, cpu_env, src1, src2);
gen_set_gpr(ctx, a->rd, dest);
return true;
}
Expand Down
209 changes: 203 additions & 6 deletions target/riscv/kvm.c
View file
Expand Up @@ -36,13 +36,20 @@
#include "exec/address-spaces.h"
#include "hw/boards.h"
#include "hw/irq.h"
#include "hw/intc/riscv_imsic.h"
#include "qemu/log.h"
#include "hw/loader.h"
#include "kvm_riscv.h"
#include "sbi_ecall_interface.h"
#include "chardev/char-fe.h"
#include "migration/migration.h"
#include "sysemu/runstate.h"
#include "hw/riscv/numa.h"

void riscv_kvm_aplic_request(void *opaque, int irq, int level)
{
kvm_set_irq(kvm_state, irq, !!level);
}

static uint64_t kvm_riscv_reg_id(CPURISCVState *env, uint64_t type,
uint64_t idx)
Expand Down Expand Up @@ -198,10 +205,8 @@ static void kvm_riscv_update_cpu_misa_ext(RISCVCPU *cpu, CPUState *cs)
}
}

#define CPUCFG(_prop) offsetof(struct RISCVCPUConfig, _prop)

#define KVM_EXT_CFG(_name, _prop, _reg_id) \
{.name = _name, .offset = CPUCFG(_prop), \
{.name = _name, .offset = CPU_CFG_OFFSET(_prop), \
.kvm_reg_id = _reg_id}

static KVMCPUConfig kvm_multi_ext_cfgs[] = {
Expand Down Expand Up @@ -278,13 +283,13 @@ static void kvm_cpu_set_multi_ext_cfg(Object *obj, Visitor *v,

static KVMCPUConfig kvm_cbom_blocksize = {
.name = "cbom_blocksize",
.offset = CPUCFG(cbom_blocksize),
.offset = CPU_CFG_OFFSET(cbom_blocksize),
.kvm_reg_id = KVM_REG_RISCV_CONFIG_REG(zicbom_block_size)
};

static KVMCPUConfig kvm_cboz_blocksize = {
.name = "cboz_blocksize",
.offset = CPUCFG(cboz_blocksize),
.offset = CPU_CFG_OFFSET(cboz_blocksize),
.kvm_reg_id = KVM_REG_RISCV_CONFIG_REG(zicboz_block_size)
};

Expand Down Expand Up @@ -926,7 +931,15 @@ int kvm_arch_init(MachineState *ms, KVMState *s)

int kvm_arch_irqchip_create(KVMState *s)
{
return 0;
if (kvm_kernel_irqchip_split()) {
error_report("-machine kernel_irqchip=split is not supported on RISC-V.");
exit(1);
}

/*
* We can create the VAIA using the newer device control API.
*/
return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
}

int kvm_arch_process_async_events(CPUState *cs)
Expand Down Expand Up @@ -1027,6 +1040,190 @@ bool kvm_arch_cpu_check_are_resettable(void)
return true;
}

static int aia_mode;

static const char *kvm_aia_mode_str(uint64_t mode)
{
switch (mode) {
case KVM_DEV_RISCV_AIA_MODE_EMUL:
return "emul";
case KVM_DEV_RISCV_AIA_MODE_HWACCEL:
return "hwaccel";
case KVM_DEV_RISCV_AIA_MODE_AUTO:
default:
return "auto";
};
}

static char *riscv_get_kvm_aia(Object *obj, Error **errp)
{
return g_strdup(kvm_aia_mode_str(aia_mode));
}

static void riscv_set_kvm_aia(Object *obj, const char *val, Error **errp)
{
if (!strcmp(val, "emul")) {
aia_mode = KVM_DEV_RISCV_AIA_MODE_EMUL;
} else if (!strcmp(val, "hwaccel")) {
aia_mode = KVM_DEV_RISCV_AIA_MODE_HWACCEL;
} else if (!strcmp(val, "auto")) {
aia_mode = KVM_DEV_RISCV_AIA_MODE_AUTO;
} else {
error_setg(errp, "Invalid KVM AIA mode");
error_append_hint(errp, "Valid values are emul, hwaccel, and auto.\n");
}
}

void kvm_arch_accel_class_init(ObjectClass *oc)
{
object_class_property_add_str(oc, "riscv-aia", riscv_get_kvm_aia,
riscv_set_kvm_aia);
object_class_property_set_description(oc, "riscv-aia",
"Set KVM AIA mode. Valid values are "
"emul, hwaccel, and auto. Default "
"is auto.");
object_property_set_default_str(object_class_property_find(oc, "riscv-aia"),
"auto");
}

void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift,
uint64_t aia_irq_num, uint64_t aia_msi_num,
uint64_t aplic_base, uint64_t imsic_base,
uint64_t guest_num)
{
int ret, i;
int aia_fd = -1;
uint64_t default_aia_mode;
uint64_t socket_count = riscv_socket_count(machine);
uint64_t max_hart_per_socket = 0;
uint64_t socket, base_hart, hart_count, socket_imsic_base, imsic_addr;
uint64_t socket_bits, hart_bits, guest_bits;

aia_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_RISCV_AIA, false);

if (aia_fd < 0) {
error_report("Unable to create in-kernel irqchip");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_MODE,
&default_aia_mode, false, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to get current KVM AIA mode");
exit(1);
}
qemu_log("KVM AIA: default mode is %s\n",
kvm_aia_mode_str(default_aia_mode));

if (default_aia_mode != aia_mode) {
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_MODE,
&aia_mode, true, NULL);
if (ret < 0)
warn_report("KVM AIA: failed to set KVM AIA mode");
else
qemu_log("KVM AIA: set current mode to %s\n",
kvm_aia_mode_str(aia_mode));
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_SRCS,
&aia_irq_num, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set number of input irq lines");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_IDS,
&aia_msi_num, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set number of msi");
exit(1);
}

socket_bits = find_last_bit(&socket_count, BITS_PER_LONG) + 1;
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS,
&socket_bits, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set group_bits");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT,
&group_shift, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set group_shift");
exit(1);
}

guest_bits = guest_num == 0 ? 0 :
find_last_bit(&guest_num, BITS_PER_LONG) + 1;
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS,
&guest_bits, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set guest_bits");
exit(1);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
KVM_DEV_RISCV_AIA_ADDR_APLIC,
&aplic_base, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set the base address of APLIC");
exit(1);
}

for (socket = 0; socket < socket_count; socket++) {
socket_imsic_base = imsic_base + socket * (1U << group_shift);
hart_count = riscv_socket_hart_count(machine, socket);
base_hart = riscv_socket_first_hartid(machine, socket);

if (max_hart_per_socket < hart_count) {
max_hart_per_socket = hart_count;
}

for (i = 0; i < hart_count; i++) {
imsic_addr = socket_imsic_base + i * IMSIC_HART_SIZE(guest_bits);
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_ADDR,
KVM_DEV_RISCV_AIA_ADDR_IMSIC(i + base_hart),
&imsic_addr, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set the IMSIC address for hart %d", i);
exit(1);
}
}
}

hart_bits = find_last_bit(&max_hart_per_socket, BITS_PER_LONG) + 1;
ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CONFIG,
KVM_DEV_RISCV_AIA_CONFIG_HART_BITS,
&hart_bits, true, NULL);
if (ret < 0) {
error_report("KVM AIA: failed to set hart_bits");
exit(1);
}

if (kvm_has_gsi_routing()) {
for (uint64_t idx = 0; idx < aia_irq_num + 1; ++idx) {
/* KVM AIA only has one APLIC instance */
kvm_irqchip_add_irq_route(kvm_state, idx, 0, idx);
}
kvm_gsi_routing_allowed = true;
kvm_irqchip_commit_routes(kvm_state);
}

ret = kvm_device_access(aia_fd, KVM_DEV_RISCV_AIA_GRP_CTRL,
KVM_DEV_RISCV_AIA_CTRL_INIT,
NULL, true, NULL);
if (ret < 0) {
error_report("KVM AIA: initialized fail");
exit(1);
}

kvm_msi_via_irqfd_allowed = kvm_irqfds_enabled();
}
5 changes: 5 additions & 0 deletions target/riscv/kvm_riscv.h
View file
Expand Up @@ -22,5 +22,10 @@
void kvm_riscv_init_user_properties(Object *cpu_obj);
void kvm_riscv_reset_vcpu(RISCVCPU *cpu);
void kvm_riscv_set_irq(RISCVCPU *cpu, int irq, int level);
void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift,
uint64_t aia_irq_num, uint64_t aia_msi_num,
uint64_t aplic_base, uint64_t imsic_base,
uint64_t guest_num);
void riscv_kvm_aplic_request(void *opaque, int irq, int level);

#endif
4 changes: 3 additions & 1 deletion target/riscv/meson.build
View file
Expand Up @@ -16,11 +16,13 @@ riscv_ss.add(files(
'gdbstub.c',
'op_helper.c',
'vector_helper.c',
'vector_internals.c',
'bitmanip_helper.c',
'translate.c',
'm128_helper.c',
'crypto_helper.c',
'zce_helper.c'
'zce_helper.c',
'vcrypto_helper.c'
))
riscv_ss.add(when: 'CONFIG_KVM', if_true: files('kvm.c'), if_false: files('kvm-stub.c'))

Expand Down
4 changes: 4 additions & 0 deletions target/riscv/pmp.c
View file
Expand Up @@ -44,6 +44,10 @@ static inline uint8_t pmp_get_a_field(uint8_t cfg)
*/
static inline int pmp_is_locked(CPURISCVState *env, uint32_t pmp_index)
{
/* mseccfg.RLB is set */
if (MSECCFG_RLB_ISSET(env)) {
return 0;
}

if (env->pmp_state.pmp[pmp_index].cfg_reg & PMP_LOCK) {
return 1;
Expand Down
1 change: 1 addition & 0 deletions target/riscv/translate.c
View file
Expand Up @@ -1094,6 +1094,7 @@ static uint32_t opcode_at(DisasContextBase *dcbase, target_ulong pc)
#include "insn_trans/trans_rvzfa.c.inc"
#include "insn_trans/trans_rvzfh.c.inc"
#include "insn_trans/trans_rvk.c.inc"
#include "insn_trans/trans_rvvk.c.inc"
#include "insn_trans/trans_privileged.c.inc"
#include "insn_trans/trans_svinval.c.inc"
#include "insn_trans/trans_rvbf16.c.inc"
Expand Down
970 changes: 970 additions & 0 deletions target/riscv/vcrypto_helper.c
View file

Large diffs are not rendered by default.

245 changes: 2 additions & 243 deletions target/riscv/vector_helper.c
View file
Expand Up @@ -27,6 +27,7 @@
#include "fpu/softfloat.h"
#include "tcg/tcg-gvec-desc.h"
#include "internals.h"
#include "vector_internals.h"
#include <math.h>

target_ulong HELPER(vsetvl)(CPURISCVState *env, target_ulong s1,
Expand Down Expand Up @@ -73,68 +74,6 @@ target_ulong HELPER(vsetvl)(CPURISCVState *env, target_ulong s1,
return vl;
}

/*
* Note that vector data is stored in host-endian 64-bit chunks,
* so addressing units smaller than that needs a host-endian fixup.
*/
#if HOST_BIG_ENDIAN
#define H1(x) ((x) ^ 7)
#define H1_2(x) ((x) ^ 6)
#define H1_4(x) ((x) ^ 4)
#define H2(x) ((x) ^ 3)
#define H4(x) ((x) ^ 1)
#define H8(x) ((x))
#else
#define H1(x) (x)
#define H1_2(x) (x)
#define H1_4(x) (x)
#define H2(x) (x)
#define H4(x) (x)
#define H8(x) (x)
#endif

static inline uint32_t vext_nf(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, NF);
}

static inline uint32_t vext_vm(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VM);
}

/*
* Encode LMUL to lmul as following:
* LMUL vlmul lmul
* 1 000 0
* 2 001 1
* 4 010 2
* 8 011 3
* - 100 -
* 1/8 101 -3
* 1/4 110 -2
* 1/2 111 -1
*/
static inline int32_t vext_lmul(uint32_t desc)
{
return sextract32(FIELD_EX32(simd_data(desc), VDATA, LMUL), 0, 3);
}

static inline uint32_t vext_vta(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VTA);
}

static inline uint32_t vext_vma(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VMA);
}

static inline uint32_t vext_vta_all_1s(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VTA_ALL_1S);
}

/*
* Get the maximum number of elements can be operated.
*
Expand All @@ -153,21 +92,6 @@ static inline uint32_t vext_max_elems(uint32_t desc, uint32_t log2_esz)
return scale < 0 ? vlenb >> -scale : vlenb << scale;
}

/*
* Get number of total elements, including prestart, body and tail elements.
* Note that when LMUL < 1, the tail includes the elements past VLMAX that
* are held in the same vector register.
*/
static inline uint32_t vext_get_total_elems(CPURISCVState *env, uint32_t desc,
uint32_t esz)
{
uint32_t vlenb = simd_maxsz(desc);
uint32_t sew = 1 << FIELD_EX64(env->vtype, VTYPE, VSEW);
int8_t emul = ctzl(esz) - ctzl(sew) + vext_lmul(desc) < 0 ? 0 :
ctzl(esz) - ctzl(sew) + vext_lmul(desc);
return (vlenb << emul) / esz;
}

static inline target_ulong adjust_addr(CPURISCVState *env, target_ulong addr)
{
return (addr & ~env->cur_pmmask) | env->cur_pmbase;
Expand Down Expand Up @@ -200,20 +124,6 @@ static void probe_pages(CPURISCVState *env, target_ulong addr,
}
}

/* set agnostic elements to 1s */
static void vext_set_elems_1s(void *base, uint32_t is_agnostic, uint32_t cnt,
uint32_t tot)
{
if (is_agnostic == 0) {
/* policy undisturbed */
return;
}
if (tot - cnt == 0) {
return;
}
memset(base + cnt, -1, tot - cnt);
}

static inline void vext_set_elem_mask(void *v0, int index,
uint8_t value)
{
Expand All @@ -223,18 +133,6 @@ static inline void vext_set_elem_mask(void *v0, int index,
((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value);
}

/*
* Earlier designs (pre-0.9) had a varying number of bits
* per mask value (MLEN). In the 0.9 design, MLEN=1.
* (Section 4.5)
*/
static inline int vext_elem_mask(void *v0, int index)
{
int idx = index / 64;
int pos = index % 64;
return (((uint64_t *)v0)[idx] >> pos) & 1;
}

/* elements operations for load and store */
typedef void vext_ldst_elem_fn(CPURISCVState *env, abi_ptr addr,
uint32_t idx, void *vd, uintptr_t retaddr);
Expand Down Expand Up @@ -584,7 +482,7 @@ vext_ldff(void *vd, void *v0, target_ulong base,
cpu_mmu_index(env, false));
if (host) {
#ifdef CONFIG_USER_ONLY
if (page_check_range(addr, offset, PAGE_READ)) {
if (!page_check_range(addr, offset, PAGE_READ)) {
vl = i;
goto ProbeSuccess;
}
Expand Down Expand Up @@ -729,25 +627,15 @@ GEN_VEXT_ST_WHOLE(vs8r_v, int8_t, ste_b)
* Vector Integer Arithmetic Instructions
*/

/* expand macro args before macro */
#define RVVCALL(macro, ...) macro(__VA_ARGS__)

/* (TD, T1, T2, TX1, TX2) */
#define OP_SSS_B int8_t, int8_t, int8_t, int8_t, int8_t
#define OP_SSS_H int16_t, int16_t, int16_t, int16_t, int16_t
#define OP_SSS_W int32_t, int32_t, int32_t, int32_t, int32_t
#define OP_SSS_D int64_t, int64_t, int64_t, int64_t, int64_t
#define OP_UUU_B uint8_t, uint8_t, uint8_t, uint8_t, uint8_t
#define OP_UUU_H uint16_t, uint16_t, uint16_t, uint16_t, uint16_t
#define OP_UUU_W uint32_t, uint32_t, uint32_t, uint32_t, uint32_t
#define OP_UUU_D uint64_t, uint64_t, uint64_t, uint64_t, uint64_t
#define OP_SUS_B int8_t, uint8_t, int8_t, uint8_t, int8_t
#define OP_SUS_H int16_t, uint16_t, int16_t, uint16_t, int16_t
#define OP_SUS_W int32_t, uint32_t, int32_t, uint32_t, int32_t
#define OP_SUS_D int64_t, uint64_t, int64_t, uint64_t, int64_t
#define WOP_UUU_B uint16_t, uint8_t, uint8_t, uint16_t, uint16_t
#define WOP_UUU_H uint32_t, uint16_t, uint16_t, uint32_t, uint32_t
#define WOP_UUU_W uint64_t, uint32_t, uint32_t, uint64_t, uint64_t
#define WOP_SSS_B int16_t, int8_t, int8_t, int16_t, int16_t
#define WOP_SSS_H int32_t, int16_t, int16_t, int32_t, int32_t
#define WOP_SSS_W int64_t, int32_t, int32_t, int64_t, int64_t
Expand All @@ -764,16 +652,6 @@ GEN_VEXT_ST_WHOLE(vs8r_v, int8_t, ste_b)
#define NOP_UUU_H uint16_t, uint16_t, uint32_t, uint16_t, uint32_t
#define NOP_UUU_W uint32_t, uint32_t, uint64_t, uint32_t, uint64_t

/* operation of two vector elements */
typedef void opivv2_fn(void *vd, void *vs1, void *vs2, int i);

#define OPIVV2(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
static void do_##NAME(void *vd, void *vs1, void *vs2, int i) \
{ \
TX1 s1 = *((T1 *)vs1 + HS1(i)); \
TX2 s2 = *((T2 *)vs2 + HS2(i)); \
*((TD *)vd + HD(i)) = OP(s2, s1); \
}
#define DO_SUB(N, M) (N - M)
#define DO_RSUB(N, M) (M - N)

Expand All @@ -786,40 +664,6 @@ RVVCALL(OPIVV2, vsub_vv_h, OP_SSS_H, H2, H2, H2, DO_SUB)
RVVCALL(OPIVV2, vsub_vv_w, OP_SSS_W, H4, H4, H4, DO_SUB)
RVVCALL(OPIVV2, vsub_vv_d, OP_SSS_D, H8, H8, H8, DO_SUB)

static void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2,
CPURISCVState *env, uint32_t desc,
opivv2_fn *fn, uint32_t esz)
{
uint32_t vm = vext_vm(desc);
uint32_t vl = env->vl;
uint32_t total_elems = vext_get_total_elems(env, desc, esz);
uint32_t vta = vext_vta(desc);
uint32_t vma = vext_vma(desc);
uint32_t i;

for (i = env->vstart; i < vl; i++) {
if (!vm && !vext_elem_mask(v0, i)) {
/* set masked-off elements to 1s */
vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
continue;
}
fn(vd, vs1, vs2, i);
}
env->vstart = 0;
/* set tail elements to 1s */
vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
}

/* generate the helpers for OPIVV */
#define GEN_VEXT_VV(NAME, ESZ) \
void HELPER(NAME)(void *vd, void *v0, void *vs1, \
void *vs2, CPURISCVState *env, \
uint32_t desc) \
{ \
do_vext_vv(vd, v0, vs1, vs2, env, desc, \
do_##NAME, ESZ); \
}

GEN_VEXT_VV(vadd_vv_b, 1)
GEN_VEXT_VV(vadd_vv_h, 2)
GEN_VEXT_VV(vadd_vv_w, 4)
Expand All @@ -829,18 +673,6 @@ GEN_VEXT_VV(vsub_vv_h, 2)
GEN_VEXT_VV(vsub_vv_w, 4)
GEN_VEXT_VV(vsub_vv_d, 8)

typedef void opivx2_fn(void *vd, target_long s1, void *vs2, int i);

/*
* (T1)s1 gives the real operator type.
* (TX1)(T1)s1 expands the operator type of widen or narrow operations.
*/
#define OPIVX2(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
static void do_##NAME(void *vd, target_long s1, void *vs2, int i) \
{ \
TX2 s2 = *((T2 *)vs2 + HS2(i)); \
*((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)s1); \
}

RVVCALL(OPIVX2, vadd_vx_b, OP_SSS_B, H1, H1, DO_ADD)
RVVCALL(OPIVX2, vadd_vx_h, OP_SSS_H, H2, H2, DO_ADD)
Expand All @@ -855,40 +687,6 @@ RVVCALL(OPIVX2, vrsub_vx_h, OP_SSS_H, H2, H2, DO_RSUB)
RVVCALL(OPIVX2, vrsub_vx_w, OP_SSS_W, H4, H4, DO_RSUB)
RVVCALL(OPIVX2, vrsub_vx_d, OP_SSS_D, H8, H8, DO_RSUB)

static void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2,
CPURISCVState *env, uint32_t desc,
opivx2_fn fn, uint32_t esz)
{
uint32_t vm = vext_vm(desc);
uint32_t vl = env->vl;
uint32_t total_elems = vext_get_total_elems(env, desc, esz);
uint32_t vta = vext_vta(desc);
uint32_t vma = vext_vma(desc);
uint32_t i;

for (i = env->vstart; i < vl; i++) {
if (!vm && !vext_elem_mask(v0, i)) {
/* set masked-off elements to 1s */
vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
continue;
}
fn(vd, s1, vs2, i);
}
env->vstart = 0;
/* set tail elements to 1s */
vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
}

/* generate the helpers for OPIVX */
#define GEN_VEXT_VX(NAME, ESZ) \
void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \
void *vs2, CPURISCVState *env, \
uint32_t desc) \
{ \
do_vext_vx(vd, v0, s1, vs2, env, desc, \
do_##NAME, ESZ); \
}

GEN_VEXT_VX(vadd_vx_b, 1)
GEN_VEXT_VX(vadd_vx_h, 2)
GEN_VEXT_VX(vadd_vx_w, 4)
Expand Down Expand Up @@ -3637,11 +3435,6 @@ GEN_VEXT_VF(vfwnmsac_vf_h, 4)
GEN_VEXT_VF(vfwnmsac_vf_w, 8)

/* Vector Floating-Point Square-Root Instruction */
/* (TD, T2, TX2) */
#define OP_UU_H uint16_t, uint16_t, uint16_t
#define OP_UU_W uint32_t, uint32_t, uint32_t
#define OP_UU_D uint64_t, uint64_t, uint64_t

#define OPFVV1(NAME, TD, T2, TX2, HD, HS2, OP) \
static void do_##NAME(void *vd, void *vs2, int i, \
CPURISCVState *env) \
Expand Down Expand Up @@ -4338,40 +4131,6 @@ GEN_VEXT_CMP_VF(vmfge_vf_w, uint32_t, H4, vmfge32)
GEN_VEXT_CMP_VF(vmfge_vf_d, uint64_t, H8, vmfge64)

/* Vector Floating-Point Classify Instruction */
#define OPIVV1(NAME, TD, T2, TX2, HD, HS2, OP) \
static void do_##NAME(void *vd, void *vs2, int i) \
{ \
TX2 s2 = *((T2 *)vs2 + HS2(i)); \
*((TD *)vd + HD(i)) = OP(s2); \
}

#define GEN_VEXT_V(NAME, ESZ) \
void HELPER(NAME)(void *vd, void *v0, void *vs2, \
CPURISCVState *env, uint32_t desc) \
{ \
uint32_t vm = vext_vm(desc); \
uint32_t vl = env->vl; \
uint32_t total_elems = \
vext_get_total_elems(env, desc, ESZ); \
uint32_t vta = vext_vta(desc); \
uint32_t vma = vext_vma(desc); \
uint32_t i; \
\
for (i = env->vstart; i < vl; i++) { \
if (!vm && !vext_elem_mask(v0, i)) { \
/* set masked-off elements to 1s */ \
vext_set_elems_1s(vd, vma, i * ESZ, \
(i + 1) * ESZ); \
continue; \
} \
do_##NAME(vd, vs2, i); \
} \
env->vstart = 0; \
/* set tail elements to 1s */ \
vext_set_elems_1s(vd, vta, vl * ESZ, \
total_elems * ESZ); \
}

target_ulong fclass_h(uint64_t frs1)
{
float16 f = frs1;
Expand Down
81 changes: 81 additions & 0 deletions target/riscv/vector_internals.c
View file
@@ -0,0 +1,81 @@
/*
* RISC-V Vector Extension Internals
*
* Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2 or later, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include "vector_internals.h"

/* set agnostic elements to 1s */
void vext_set_elems_1s(void *base, uint32_t is_agnostic, uint32_t cnt,
uint32_t tot)
{
if (is_agnostic == 0) {
/* policy undisturbed */
return;
}
if (tot - cnt == 0) {
return ;
}
memset(base + cnt, -1, tot - cnt);
}

void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2,
CPURISCVState *env, uint32_t desc,
opivv2_fn *fn, uint32_t esz)
{
uint32_t vm = vext_vm(desc);
uint32_t vl = env->vl;
uint32_t total_elems = vext_get_total_elems(env, desc, esz);
uint32_t vta = vext_vta(desc);
uint32_t vma = vext_vma(desc);
uint32_t i;

for (i = env->vstart; i < vl; i++) {
if (!vm && !vext_elem_mask(v0, i)) {
/* set masked-off elements to 1s */
vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
continue;
}
fn(vd, vs1, vs2, i);
}
env->vstart = 0;
/* set tail elements to 1s */
vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
}

void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2,
CPURISCVState *env, uint32_t desc,
opivx2_fn fn, uint32_t esz)
{
uint32_t vm = vext_vm(desc);
uint32_t vl = env->vl;
uint32_t total_elems = vext_get_total_elems(env, desc, esz);
uint32_t vta = vext_vta(desc);
uint32_t vma = vext_vma(desc);
uint32_t i;

for (i = env->vstart; i < vl; i++) {
if (!vm && !vext_elem_mask(v0, i)) {
/* set masked-off elements to 1s */
vext_set_elems_1s(vd, vma, i * esz, (i + 1) * esz);
continue;
}
fn(vd, s1, vs2, i);
}
env->vstart = 0;
/* set tail elements to 1s */
vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);
}
228 changes: 228 additions & 0 deletions target/riscv/vector_internals.h
View file
@@ -0,0 +1,228 @@
/*
* RISC-V Vector Extension Internals
*
* Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2 or later, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program. If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef TARGET_RISCV_VECTOR_INTERNALS_H
#define TARGET_RISCV_VECTOR_INTERNALS_H

#include "qemu/osdep.h"
#include "qemu/bitops.h"
#include "cpu.h"
#include "tcg/tcg-gvec-desc.h"
#include "internals.h"

static inline uint32_t vext_nf(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, NF);
}

/*
* Note that vector data is stored in host-endian 64-bit chunks,
* so addressing units smaller than that needs a host-endian fixup.
*/
#if HOST_BIG_ENDIAN
#define H1(x) ((x) ^ 7)
#define H1_2(x) ((x) ^ 6)
#define H1_4(x) ((x) ^ 4)
#define H2(x) ((x) ^ 3)
#define H4(x) ((x) ^ 1)
#define H8(x) ((x))
#else
#define H1(x) (x)
#define H1_2(x) (x)
#define H1_4(x) (x)
#define H2(x) (x)
#define H4(x) (x)
#define H8(x) (x)
#endif

/*
* Encode LMUL to lmul as following:
* LMUL vlmul lmul
* 1 000 0
* 2 001 1
* 4 010 2
* 8 011 3
* - 100 -
* 1/8 101 -3
* 1/4 110 -2
* 1/2 111 -1
*/
static inline int32_t vext_lmul(uint32_t desc)
{
return sextract32(FIELD_EX32(simd_data(desc), VDATA, LMUL), 0, 3);
}

static inline uint32_t vext_vm(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VM);
}

static inline uint32_t vext_vma(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VMA);
}

static inline uint32_t vext_vta(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VTA);
}

static inline uint32_t vext_vta_all_1s(uint32_t desc)
{
return FIELD_EX32(simd_data(desc), VDATA, VTA_ALL_1S);
}

/*
* Earlier designs (pre-0.9) had a varying number of bits
* per mask value (MLEN). In the 0.9 design, MLEN=1.
* (Section 4.5)
*/
static inline int vext_elem_mask(void *v0, int index)
{
int idx = index / 64;
int pos = index % 64;
return (((uint64_t *)v0)[idx] >> pos) & 1;
}

/*
* Get number of total elements, including prestart, body and tail elements.
* Note that when LMUL < 1, the tail includes the elements past VLMAX that
* are held in the same vector register.
*/
static inline uint32_t vext_get_total_elems(CPURISCVState *env, uint32_t desc,
uint32_t esz)
{
uint32_t vlenb = simd_maxsz(desc);
uint32_t sew = 1 << FIELD_EX64(env->vtype, VTYPE, VSEW);
int8_t emul = ctzl(esz) - ctzl(sew) + vext_lmul(desc) < 0 ? 0 :
ctzl(esz) - ctzl(sew) + vext_lmul(desc);
return (vlenb << emul) / esz;
}

/* set agnostic elements to 1s */
void vext_set_elems_1s(void *base, uint32_t is_agnostic, uint32_t cnt,
uint32_t tot);

/* expand macro args before macro */
#define RVVCALL(macro, ...) macro(__VA_ARGS__)

/* (TD, T2, TX2) */
#define OP_UU_B uint8_t, uint8_t, uint8_t
#define OP_UU_H uint16_t, uint16_t, uint16_t
#define OP_UU_W uint32_t, uint32_t, uint32_t
#define OP_UU_D uint64_t, uint64_t, uint64_t

/* (TD, T1, T2, TX1, TX2) */
#define OP_UUU_B uint8_t, uint8_t, uint8_t, uint8_t, uint8_t
#define OP_UUU_H uint16_t, uint16_t, uint16_t, uint16_t, uint16_t
#define OP_UUU_W uint32_t, uint32_t, uint32_t, uint32_t, uint32_t
#define OP_UUU_D uint64_t, uint64_t, uint64_t, uint64_t, uint64_t

#define OPIVV1(NAME, TD, T2, TX2, HD, HS2, OP) \
static void do_##NAME(void *vd, void *vs2, int i) \
{ \
TX2 s2 = *((T2 *)vs2 + HS2(i)); \
*((TD *)vd + HD(i)) = OP(s2); \
}

#define GEN_VEXT_V(NAME, ESZ) \
void HELPER(NAME)(void *vd, void *v0, void *vs2, \
CPURISCVState *env, uint32_t desc) \
{ \
uint32_t vm = vext_vm(desc); \
uint32_t vl = env->vl; \
uint32_t total_elems = \
vext_get_total_elems(env, desc, ESZ); \
uint32_t vta = vext_vta(desc); \
uint32_t vma = vext_vma(desc); \
uint32_t i; \
\
for (i = env->vstart; i < vl; i++) { \
if (!vm && !vext_elem_mask(v0, i)) { \
/* set masked-off elements to 1s */ \
vext_set_elems_1s(vd, vma, i * ESZ, \
(i + 1) * ESZ); \
continue; \
} \
do_##NAME(vd, vs2, i); \
} \
env->vstart = 0; \
/* set tail elements to 1s */ \
vext_set_elems_1s(vd, vta, vl * ESZ, \
total_elems * ESZ); \
}

/* operation of two vector elements */
typedef void opivv2_fn(void *vd, void *vs1, void *vs2, int i);

#define OPIVV2(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \
static void do_##NAME(void *vd, void *vs1, void *vs2, int i) \
{ \
TX1 s1 = *((T1 *)vs1 + HS1(i)); \
TX2 s2 = *((T2 *)vs2 + HS2(i)); \
*((TD *)vd + HD(i)) = OP(s2, s1); \
}

void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2,
CPURISCVState *env, uint32_t desc,
opivv2_fn *fn, uint32_t esz);

/* generate the helpers for OPIVV */
#define GEN_VEXT_VV(NAME, ESZ) \
void HELPER(NAME)(void *vd, void *v0, void *vs1, \
void *vs2, CPURISCVState *env, \
uint32_t desc) \
{ \
do_vext_vv(vd, v0, vs1, vs2, env, desc, \
do_##NAME, ESZ); \
}

typedef void opivx2_fn(void *vd, target_long s1, void *vs2, int i);

/*
* (T1)s1 gives the real operator type.
* (TX1)(T1)s1 expands the operator type of widen or narrow operations.
*/
#define OPIVX2(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \
static void do_##NAME(void *vd, target_long s1, void *vs2, int i) \
{ \
TX2 s2 = *((T2 *)vs2 + HS2(i)); \
*((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)s1); \
}

void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2,
CPURISCVState *env, uint32_t desc,
opivx2_fn fn, uint32_t esz);

/* generate the helpers for OPIVX */
#define GEN_VEXT_VX(NAME, ESZ) \
void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \
void *vs2, CPURISCVState *env, \
uint32_t desc) \
{ \
do_vext_vx(vd, v0, s1, vs2, env, desc, \
do_##NAME, ESZ); \
}

/* Three of the widening shortening macros: */
/* (TD, T1, T2, TX1, TX2) */
#define WOP_UUU_B uint16_t, uint8_t, uint8_t, uint16_t, uint16_t
#define WOP_UUU_H uint32_t, uint16_t, uint16_t, uint32_t, uint32_t
#define WOP_UUU_W uint64_t, uint32_t, uint32_t, uint64_t, uint64_t

#endif /* TARGET_RISCV_VECTOR_INTERNALS_H */
32 changes: 32 additions & 0 deletions tests/avocado/tuxrun_baselines.py
View file
Expand Up @@ -501,6 +501,38 @@ def test_riscv64(self):

self.common_tuxrun(csums=sums)

def test_riscv32_maxcpu(self):
"""
:avocado: tags=arch:riscv32
:avocado: tags=machine:virt
:avocado: tags=cpu:max
:avocado: tags=tuxboot:riscv32
"""
sums = { "Image" :
"89599407d7334de629a40e7ad6503c73670359eb5f5ae9d686353a3d6deccbd5",
"fw_jump.elf" :
"f2ef28a0b77826f79d085d3e4aa686f1159b315eff9099a37046b18936676985",
"rootfs.ext4.zst" :
"7168d296d0283238ea73cd5a775b3dd608e55e04c7b92b76ecce31bb13108cba" }

self.common_tuxrun(csums=sums)

def test_riscv64_maxcpu(self):
"""
:avocado: tags=arch:riscv64
:avocado: tags=machine:virt
:avocado: tags=cpu:max
:avocado: tags=tuxboot:riscv64
"""
sums = { "Image" :
"cd634badc65e52fb63465ec99e309c0de0369f0841b7d9486f9729e119bac25e",
"fw_jump.elf" :
"6e3373abcab4305fe151b564a4c71110d833c21f2c0a1753b7935459e36aedcf",
"rootfs.ext4.zst" :
"b18e3a3bdf27be03da0b285e84cb71bf09eca071c3a087b42884b6982ed679eb" }

self.common_tuxrun(csums=sums)

def test_s390(self):
"""
:avocado: tags=arch:s390x
Expand Down
1 change: 1 addition & 0 deletions tests/docker/dockerfiles/debian-amd64.docker
View file
Expand Up @@ -98,6 +98,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
libvirglrenderer-dev \
libvte-2.91-dev \
libxen-dev \
libxdp-dev \
libzstd-dev \
llvm \
locales \
Expand Down
8 changes: 4 additions & 4 deletions tests/qemu-iotests/197
View file
Expand Up @@ -136,18 +136,18 @@ IMGPROTO=file IMGFMT=qcow2 TEST_IMG_FILE="$TEST_WRAP" \
$QEMU_IO -c "write -P 0xaa 0 64k" "$TEST_IMG" | _filter_qemu_io

# Allocate individual subclusters in the top image, and not the whole cluster
$QEMU_IO -c "write -P 0xbb 28K 2K" -c "write -P 0xcc 34K 2K" "$TEST_WRAP" \
$QEMU_IO -f qcow2 -c "write -P 0xbb 28K 2K" -c "write -P 0xcc 34K 2K" "$TEST_WRAP" \
| _filter_qemu_io

# Only 2 subclusters should be allocated in the top image at this point
$QEMU_IMG map "$TEST_WRAP" | _filter_qemu_img_map
$QEMU_IO -f qcow2 -c map "$TEST_WRAP"

# Actual copy-on-read operation
$QEMU_IO -C -c "read -P 0xaa 30K 4K" "$TEST_WRAP" | _filter_qemu_io
$QEMU_IO -f qcow2 -C -c "read -P 0xaa 30K 4K" "$TEST_WRAP" | _filter_qemu_io

# And here we should have 4 subclusters allocated right in the middle of the
# top image. Make sure the whole cluster remains unallocated
$QEMU_IMG map "$TEST_WRAP" | _filter_qemu_img_map
$QEMU_IO -f qcow2 -c map "$TEST_WRAP"

_check_test_img

Expand Down
18 changes: 8 additions & 10 deletions tests/qemu-iotests/197.out
View file
Expand Up @@ -42,17 +42,15 @@ wrote 2048/2048 bytes at offset 28672
2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 2048/2048 bytes at offset 34816
2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
Offset Length File
0 0x7000 TEST_DIR/t.IMGFMT
0x7000 0x800 TEST_DIR/t.wrap.IMGFMT
0x7800 0x1000 TEST_DIR/t.IMGFMT
0x8800 0x800 TEST_DIR/t.wrap.IMGFMT
0x9000 0x7000 TEST_DIR/t.IMGFMT
28 KiB (0x7000) bytes not allocated at offset 0 bytes (0x0)
2 KiB (0x800) bytes allocated at offset 28 KiB (0x7000)
4 KiB (0x1000) bytes not allocated at offset 30 KiB (0x7800)
2 KiB (0x800) bytes allocated at offset 34 KiB (0x8800)
28 KiB (0x7000) bytes not allocated at offset 36 KiB (0x9000)
read 4096/4096 bytes at offset 30720
4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
Offset Length File
0 0x7000 TEST_DIR/t.IMGFMT
0x7000 0x2000 TEST_DIR/t.wrap.IMGFMT
0x9000 0x7000 TEST_DIR/t.IMGFMT
28 KiB (0x7000) bytes not allocated at offset 0 bytes (0x0)
8 KiB (0x2000) bytes allocated at offset 28 KiB (0x7000)
28 KiB (0x7000) bytes not allocated at offset 36 KiB (0x9000)
No errors were found on the image.
*** done
5 changes: 5 additions & 0 deletions tests/qtest/libqos/igb.c
View file
Expand Up @@ -109,6 +109,11 @@ static void igb_pci_start_hw(QOSGraphObject *obj)
E1000_RAH_AV | E1000_RAH_POOL_1 |
le16_to_cpu(*(uint16_t *)(address + 4)));

/* Set supported receive descriptor mode */
e1000e_macreg_write(&d->e1000e,
E1000_SRRCTL(0),
E1000_SRRCTL_DESCTYPE_ADV_ONEBUF);

/* Enable receive */
e1000e_macreg_write(&d->e1000e, E1000_RFCTL, E1000_RFCTL_EXTEN);
e1000e_macreg_write(&d->e1000e, E1000_RCTL, E1000_RCTL_EN);
Expand Down
1 change: 1 addition & 0 deletions tests/qtest/meson.build
View file
Expand Up @@ -269,6 +269,7 @@ qos_test_ss.add(
'virtio-iommu-test.c',
'vmxnet3-test.c',
'igb-test.c',
'ufs-test.c',
)

if config_all_devices.has_key('CONFIG_VIRTIO_SERIAL')
Expand Down
587 changes: 587 additions & 0 deletions tests/qtest/ufs-test.c
View file

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion tools/ebpf/rss.bpf.c
View file
Expand Up @@ -81,20 +81,23 @@ struct {
__uint(key_size, sizeof(__u32));
__uint(value_size, sizeof(struct rss_config_t));
__uint(max_entries, 1);
__uint(map_flags, BPF_F_MMAPABLE);
} tap_rss_map_configurations SEC(".maps");

struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(key_size, sizeof(__u32));
__uint(value_size, sizeof(struct toeplitz_key_data_t));
__uint(max_entries, 1);
__uint(map_flags, BPF_F_MMAPABLE);
} tap_rss_map_toeplitz_key SEC(".maps");

struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(key_size, sizeof(__u32));
__uint(value_size, sizeof(__u16));
__uint(max_entries, INDIRECTION_TABLE_SIZE);
__uint(map_flags, BPF_F_MMAPABLE);
} tap_rss_map_indirection_table SEC(".maps");

static inline void net_rx_rss_add_chunk(__u8 *rss_input, size_t *bytes_written,
Expand Down Expand Up @@ -528,7 +531,7 @@ static inline __u32 calculate_rss_hash(struct __sk_buff *skb,
return result;
}

SEC("tun_rss_steering")
SEC("socket")
int tun_rss_steering_prog(struct __sk_buff *skb)
{

Expand Down
2 changes: 1 addition & 1 deletion util/iov.c
View file
Expand Up @@ -571,7 +571,7 @@ static int sortelem_cmp_src_index(const void *a, const void *b)
*/
void qemu_iovec_clone(QEMUIOVector *dest, const QEMUIOVector *src, void *buf)
{
IOVectorSortElem sortelems[src->niov];
g_autofree IOVectorSortElem *sortelems = g_new(IOVectorSortElem, src->niov);
void *last_end;
int i;

Expand Down
27 changes: 20 additions & 7 deletions util/vhost-user-server.c
View file
Expand Up @@ -127,7 +127,14 @@ vu_message_read(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)
if (rc < 0) {
if (rc == QIO_CHANNEL_ERR_BLOCK) {
assert(local_err == NULL);
qio_channel_yield(ioc, G_IO_IN);
if (server->ctx) {
server->in_qio_channel_yield = true;
qio_channel_yield(ioc, G_IO_IN);
server->in_qio_channel_yield = false;
} else {
/* Wait until attached to an AioContext again */
qemu_coroutine_yield();
}
continue;
} else {
error_report_err(local_err);
Expand Down Expand Up @@ -278,7 +285,7 @@ set_watch(VuDev *vu_dev, int fd, int vu_evt,
vu_fd_watch->fd = fd;
vu_fd_watch->cb = cb;
qemu_socket_set_nonblock(fd);
aio_set_fd_handler(server->ioc->ctx, fd, kick_handler,
aio_set_fd_handler(server->ctx, fd, kick_handler,
NULL, NULL, NULL, vu_fd_watch);
vu_fd_watch->vu_dev = vu_dev;
vu_fd_watch->pvt = pvt;
Expand All @@ -299,7 +306,7 @@ static void remove_watch(VuDev *vu_dev, int fd)
if (!vu_fd_watch) {
return;
}
aio_set_fd_handler(server->ioc->ctx, fd, NULL, NULL, NULL, NULL, NULL);
aio_set_fd_handler(server->ctx, fd, NULL, NULL, NULL, NULL, NULL);

QTAILQ_REMOVE(&server->vu_fd_watches, vu_fd_watch, next);
g_free(vu_fd_watch);
Expand Down Expand Up @@ -344,6 +351,8 @@ static void vu_accept(QIONetListener *listener, QIOChannelSocket *sioc,
/* TODO vu_message_write() spins if non-blocking! */
qio_channel_set_blocking(server->ioc, false, NULL);

qio_channel_set_follow_coroutine_ctx(server->ioc, true);

server->co_trip = qemu_coroutine_create(vu_client_trip, server);

aio_context_acquire(server->ctx);
Expand Down Expand Up @@ -399,13 +408,12 @@ void vhost_user_server_attach_aio_context(VuServer *server, AioContext *ctx)
return;
}

qio_channel_attach_aio_context(server->ioc, ctx);

QTAILQ_FOREACH(vu_fd_watch, &server->vu_fd_watches, next) {
aio_set_fd_handler(ctx, vu_fd_watch->fd, kick_handler, NULL,
NULL, NULL, vu_fd_watch);
}

assert(!server->in_qio_channel_yield);
aio_co_schedule(ctx, server->co_trip);
}

Expand All @@ -419,11 +427,16 @@ void vhost_user_server_detach_aio_context(VuServer *server)
aio_set_fd_handler(server->ctx, vu_fd_watch->fd,
NULL, NULL, NULL, NULL, vu_fd_watch);
}

qio_channel_detach_aio_context(server->ioc);
}

server->ctx = NULL;

if (server->ioc) {
if (server->in_qio_channel_yield) {
/* Stop receiving the next vhost-user message */
qio_channel_wake_read(server->ioc);
}
}
}

bool vhost_user_server_start(VuServer *server,
Expand Down