From 11f4892e26ddae4b6c525e3d056ab166e6d96f2e Mon Sep 17 00:00:00 2001 From: Eric Chanudet Date: Thu, 25 Apr 2019 11:04:09 -0400 Subject: [PATCH 1/3] vsock/module: Initial datagram implementation. Based on Xen 4.12.0 released headers. An Argo implementation for VSock for datagram transfers. Support basic packet communication across domains. Rings are registered at socket naming (bind(3p)). There is one ring created per partner in the domain. This includes ANY, which Xen will allow only in mac-permissive mode for Argo. Send is synchronous: does an hypercall directly, like V4V used to do. Recv is handled in a tasklet that will make skbs with data recovered from the ring. Ring size is hardcoded to 32 pages currently. Signed-off-by: Eric Chanudet OXT-1473 --- vsock-argo/module/.gitignore | 10 + vsock-argo/module/Kbuild | 5 + vsock-argo/module/Makefile | 20 + vsock-argo/module/argo_ring.c | 401 +++++++++++++++++ vsock-argo/module/argo_ring.h | 73 ++++ vsock-argo/module/argo_transport.c | 559 ++++++++++++++++++++++++ vsock-argo/module/asm-x86/argo-compat.h | 43 ++ vsock-argo/module/include/Kbuild | 1 + vsock-argo/module/include/xen/Kbuild | 1 + vsock-argo/module/include/xen/argo.h | 279 ++++++++++++ 10 files changed, 1392 insertions(+) create mode 100644 vsock-argo/module/.gitignore create mode 100644 vsock-argo/module/Kbuild create mode 100644 vsock-argo/module/Makefile create mode 100644 vsock-argo/module/argo_ring.c create mode 100644 vsock-argo/module/argo_ring.h create mode 100644 vsock-argo/module/argo_transport.c create mode 100644 vsock-argo/module/asm-x86/argo-compat.h create mode 100644 vsock-argo/module/include/Kbuild create mode 100644 vsock-argo/module/include/xen/Kbuild create mode 100644 vsock-argo/module/include/xen/argo.h diff --git a/vsock-argo/module/.gitignore b/vsock-argo/module/.gitignore new file mode 100644 index 0000000..327fae1 --- /dev/null +++ b/vsock-argo/module/.gitignore @@ -0,0 +1,10 @@ +*.ko +*.ko.cmd +*.mod.o +*.mod.c +modules.order +Module.symvers +*.o +*.o.d +*.o.cmd +.tmp_versions diff --git a/vsock-argo/module/Kbuild b/vsock-argo/module/Kbuild new file mode 100644 index 0000000..4aa2d4d --- /dev/null +++ b/vsock-argo/module/Kbuild @@ -0,0 +1,5 @@ +obj-m += vsock_argo_transport.o +vsock_argo_transport-y += argo_transport.o argo_ring.o + +ccflags-y := -I$(src)/include +ccflags-$(CONFIG_X86) += -I$(src)/asm-x86 diff --git a/vsock-argo/module/Makefile b/vsock-argo/module/Makefile new file mode 100644 index 0000000..4a9cce3 --- /dev/null +++ b/vsock-argo/module/Makefile @@ -0,0 +1,20 @@ +ifneq ($(KERNELRELEASE),) +# kbuild part of makefile +include Kbuild + +else +# normal makefile +KERNEL_VERSION ?= `uname -r` +KERNEL_SRC ?= /lib/modules/$(KERNEL_VERSION)/build +INSTALL_HDR_PATH ?= /usr + +default: + $(MAKE) -C $(KERNEL_SRC) M=$$PWD + +clean: + $(MAKE) -C $(KERNEL_SRC) M=$$PWD clean + +modules_install: + $(MAKE) -C $(KERNEL_SRC) M=$$PWD modules_install + +endif diff --git a/vsock-argo/module/argo_ring.c b/vsock-argo/module/argo_ring.c new file mode 100644 index 0000000..af604fc --- /dev/null +++ b/vsock-argo/module/argo_ring.c @@ -0,0 +1,401 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include "argo_ring.h" + +/* + * Global ring list. + */ +struct list_head argo_rings; +rwlock_t argo_rings_lock; + +/* + * Ring management helpers. + */ +static void argo_ring_free(xen_argo_ring_t *r) +{ + vfree(r); +} + +static xen_argo_ring_t *argo_ring_alloc(size_t len) +{ + xen_argo_ring_t *r; + + if (unlikely(len < + sizeof (struct xen_argo_ring_message_header) + + ARGO_RING_ALIGN(1) + ARGO_RING_ALIGN(1))) + return ERR_PTR(-EINVAL); + + if (len > XEN_ARGO_MAX_RING_SIZE) + return ERR_PTR(-E2BIG); + if (len != ARGO_RING_ALIGN(len)) + return ERR_PTR(-EINVAL); + + r = vmalloc(sizeof (*r) + len); + if (!r) + return ERR_PTR(-ENOMEM); + + r->rx_ptr = 0; + r->tx_ptr = 0; + + return r; +} + +static void argo_gfn_array_free(struct argo_gfn_array *ga) +{ + kfree(ga); +} +static struct argo_gfn_array * +argo_gfn_array_alloc(volatile void *ring_ptr, size_t n) +{ + struct argo_gfn_array *ga; + unsigned char *p = (void*)ring_ptr; + size_t i; + + ga = kmalloc(sizeof (*ga) + n * sizeof (ga->gfns[0]), GFP_KERNEL); + if (!ga) + return ERR_PTR(-ENOMEM); + + ga->n = n; + for (i = 0; i < n; ++i) + ga->gfns[i] = pfn_to_mfn(vmalloc_to_pfn(p + i * PAGE_SIZE)); + + return ga; +} + +/* + * Ring interface. + */ +void argo_ring_handle_free(struct argo_ring_hnd *h) +{ + list_del(&h->l); + + argo_gfn_array_free(h->gfns); + argo_ring_free(h->ring); + + kfree(h); +} + +struct argo_ring_hnd * +argo_ring_handle_alloc(domid_t domain, unsigned int port, + argo_recv_skb_cb recv_cb, void *priv) +{ + struct argo_ring_hnd *h; + size_t ring_npages; + int rc; + + h = kmalloc(sizeof (*h), GFP_KERNEL); + if (!h) + return ERR_PTR(-ENOMEM); + + h->ring = argo_ring_alloc(ring_len); + if (IS_ERR(h->ring)) { + rc = PTR_ERR(h->ring); + goto fail_alloc; + } + h->ring_len = ring_len; + ring_npages = round_up( + ARGO_RING_ALIGN(ring_len) + sizeof (xen_argo_ring_t), + PAGE_SIZE) >> PAGE_SHIFT; + + h->gfns = argo_gfn_array_alloc(h->ring->ring, ring_npages); + if (IS_ERR(h->gfns)) { + rc = PTR_ERR(h->gfns); + goto fail_gfns; + } + + /* FIXME: ring_lock. */ + spin_lock_init(&h->ring_lock); + write_lock(&argo_rings_lock); + list_add_tail(&h->l, &argo_rings); + write_unlock(&argo_rings_lock); + + h->partner_id = domain; + h->aport = port; + + h->recv_cb = recv_cb; + h->priv = priv; + + pr_debug("New ring for partner dom%u:%u, %uB.\n", + h->partner_id, h->aport, h->ring_len); + + return h; + +fail_gfns: + argo_ring_free(h->ring); + h->ring = NULL; +fail_alloc: + kfree(h); + return ERR_PTR(rc); +} + +void argo_ring_unregister(struct argo_ring_hnd *h) +{ + xen_argo_unregister_ring_t unreg = { + .aport = h->aport, + .partner_id = h->partner_id, + .pad = 0, + }; + int rc; + + if (!h->ring || !h->gfns) + return; + + rc = HYPERVISOR_argo_op(XEN_ARGO_OP_unregister_ring, + &unreg, NULL, 0, 0); + if (rc) + pr_warn("Failed to unregister argo ring for dom%u:%u (%d).\n", + h->partner_id, h->aport, rc); + else + pr_debug("Ring for dom%u:%u unregistered.\n", + h->partner_id, h->aport); +} + +int argo_ring_register(struct argo_ring_hnd *h) +{ + int rc; + xen_argo_register_ring_t reg = { + .aport = h->aport, + .partner_id = h->partner_id, + .pad = 0, + .len = h->ring_len, + }; + + rc = HYPERVISOR_argo_op(XEN_ARGO_OP_register_ring, + ®, h->gfns->gfns, h->gfns->n, 0); + if (rc) + pr_warn("Failed to register argo ring for dom%u:%u (%d).\n", + h->partner_id, h->aport, rc); + else + pr_debug("Ring for dom%u:%u registered.\n", + h->partner_id, h->aport); + + return rc; +} + +/* + * Ring arithmetic helpers. + * Argo ring never fill up completely, so tx == rx means the ring is empty. + */ +static inline size_t argo_ring_has_data(const struct argo_ring_hnd *h) +{ + const xen_argo_ring_t *r = h->ring; + const size_t rx = r->rx_ptr; + const size_t tx = r->tx_ptr; + + if (rx > tx) + return h->ring_len - (rx - tx); + + return tx - rx; +} + +static inline size_t argo_ring_has_data_no_wrap(const struct argo_ring_hnd *h) +{ + const xen_argo_ring_t *r = h->ring; + const size_t rx = r->rx_ptr; + const size_t tx = r->tx_ptr; + + if (rx > tx) + return h->ring_len - rx; + + return tx - rx; +} + +static inline size_t argo_ring_has_space(const struct argo_ring_hnd *h) +{ + return h->ring_len - argo_ring_has_data(h) - ARGO_RING_ALIGN(1); +} + +static int argo_ring_recv(struct argo_ring_hnd *h, void *buf, size_t len) +{ + xen_argo_ring_t *r = h->ring; + unsigned char *p = buf; + size_t chunk; + size_t rx = r->rx_ptr; + + if (len > argo_ring_has_data(h)) + return -E2BIG; + + pr_debug("receive %zuB: ring_len:%uB data:%zuB data-no-wrap:%zuB " + "space-left:%zuB, rx:%zu, tx:%u.\n", + len, h->ring_len, argo_ring_has_data(h), + argo_ring_has_data_no_wrap(h), + argo_ring_has_space(h), + rx, r->tx_ptr); + + chunk = argo_ring_has_data_no_wrap(h); + if (len > chunk) { + memcpy(p, (void*)&r->ring[rx], chunk); + memcpy(&p[chunk], (void*)&r->ring[0], len - chunk); + } else + memcpy(p, (void*)&r->ring[rx], len); + + rx = ARGO_RING_ALIGN(ARGO_RING_ALIGN(rx + len) % h->ring_len); + + mb(); /* rx cannot be set out-of-order, thank you. */ + r->rx_ptr = rx; + + return len; +} + +static struct sk_buff *argo_ring_recv_skb(struct argo_ring_hnd *h) +{ + const struct xen_argo_ring_message_header *mh; + struct sk_buff *skb; + size_t msg_len; + int rc = 0; + + /* There is at least a header. */ + skb = alloc_skb(sizeof (*mh), GFP_ATOMIC); + if (!skb) + return ERR_PTR(-ENOMEM); + + /* FIXME: ring_lock. + * Tasklet can only be ran on one CPU (while it may be scheduled by + * more than one). This lock is useless, right? + */ + spin_lock(&h->ring_lock); + argo_ring_recv(h, skb_put(skb, sizeof (*mh)), sizeof (*mh)); + + mh = (void*)skb->data; + msg_len = mh->len - sizeof (*mh); + + if (unlikely(msg_len > argo_ring_has_data(h))) { + pr_debug("Invalid packet, message size exceeds ring capacity."); + rc = E2BIG; + goto out; + } + + if (msg_len) { + /* Data in this packet. */ + if (pskb_expand_head(skb, 0, msg_len, GFP_ATOMIC)) { + pr_debug("Failed to allocate skb to receive message."); + rc = ENOMEM; + goto out; + } + argo_ring_recv(h, skb_put(skb, msg_len), msg_len); + } + /* FIXME: See ring_lock above. */ + spin_unlock(&h->ring_lock); + + return skb; + +out: + spin_unlock(&h->ring_lock); + kfree_skb(skb); + return ERR_PTR(-rc); +} + +int argo_ring_send_skb(struct argo_ring_hnd *h, const struct sk_buff *skb, + xen_argo_send_addr_t *send) +{ + xen_argo_iov_t iov; + int rc; + + if (argo_ring_has_space(h) < + ARGO_RING_ALIGN(skb->len + + sizeof (struct xen_argo_ring_message_header))) { + pr_debug("%s: Insuficient space in target ring.\n", __func__); + return -ENOBUFS; + } + + iov.iov_hnd = (uint64_t)skb->data; + iov.iov_len = skb->len; + iov.pad = 0; + + /* TODO: Message-type is forced to 0 here. */ + rc = HYPERVISOR_argo_op(XEN_ARGO_OP_sendv, send, (void *)&iov, 1, 0); + if (rc < 0) + pr_warn("Failed to send packet (%uB) through Argo to dom%u:%u (%d).\n", + skb->len, send->dst.domain_id, send->dst.aport, -rc); + + return rc; +} + +/* + * Tasklet handling packets reception. + */ +static void argo_handle_event(unsigned long data) +{ + struct argo_ring_hnd *h, *tmp; + int rc; + + read_lock(&argo_rings_lock); + list_for_each_entry_safe(h, tmp, &argo_rings, l) { + struct sk_buff *skb; + + while (argo_ring_has_data(h) >= + sizeof (struct xen_argo_ring_message_header)) { + skb = argo_ring_recv_skb(h); + if (IS_ERR(skb)) { + pr_warn("Failed to retrieve packet from Argo " + "ring (%ld).\n", -PTR_ERR(skb)); + break; + } + rc = h->recv_cb(h->priv, skb); + if (rc) { + pr_warn("Failed to queue received packet, dropping.\n"); + kfree_skb(skb); + break; + } + } + } + read_unlock(&argo_rings_lock); +} + +DECLARE_TASKLET(argo_event, argo_handle_event, 0); + +/* + * IRQ handler scheduling tasklet. + */ +static irqreturn_t +argo_interrupt(int irq, void *dev_id) +{ + tasklet_schedule(&argo_event); + return IRQ_HANDLED; +} + +/* + * Initialisation and cleanup of the VIRQ. + */ +static int argo_irq = -1; +int argo_core_init(void) +{ + int rc; + + argo_ring_check_sizes(); + INIT_LIST_HEAD(&argo_rings); + rwlock_init(&argo_rings_lock); + + rc = bind_virq_to_irqhandler(VIRQ_ARGO, 0, argo_interrupt, 0, + "argo", NULL); + if (rc < 0) + return rc; + + argo_irq = rc; + + return 0; +} + +void argo_core_cleanup(void) +{ + if (argo_irq < 0) + return; + + unbind_from_irqhandler(argo_irq, NULL); +} diff --git a/vsock-argo/module/argo_ring.h b/vsock-argo/module/argo_ring.h new file mode 100644 index 0000000..40aee04 --- /dev/null +++ b/vsock-argo/module/argo_ring.h @@ -0,0 +1,73 @@ +#ifndef _ARGO_RING_H_ +# define _ARGO_RING_H_ + +#include + +#include + +/* + * Ring GFN management. + */ +struct argo_gfn_array { + size_t n; + xen_argo_gfn_t gfns[0]; +}; + +/* + * xen_argo_ring_t does not have a length field. + * Arbitrary fixed ring size for now. + * This size is chosen to match other Argo driver implementations. + */ +static const size_t ring_len = 32 * PAGE_SIZE; + +/* + * Messages on the ring are aligned on XEN_ARGO_MSG_SLOT_SIZE. + * XEN_ARGO_MSG_SLOT_SIZE needs to be a power of 2. + */ +#define ARGO_RING_ALIGN(a) round_up((a), XEN_ARGO_MSG_SLOT_SIZE) + +static inline void argo_ring_check_sizes(void) +{ + BUILD_BUG_ON_NOT_POWER_OF_2(XEN_ARGO_MSG_SLOT_SIZE); +} + +/* + * Ring management. + */ +typedef int (*argo_recv_skb_cb)(void *priv, struct sk_buff *skb); +struct argo_ring_hnd { + struct list_head l; + spinlock_t ring_lock; + xen_argo_ring_t *ring; + unsigned int ring_len; + struct argo_gfn_array *gfns; + xen_argo_port_t aport; + domid_t partner_id; + argo_recv_skb_cb recv_cb; + void *priv; /* TODO: Do better. Opaque to get struct + vsock_sock/struct sock to recv_cb */ +}; + +/* + * Ring handle primitives. + */ +void argo_ring_handle_free(struct argo_ring_hnd *h); +struct argo_ring_hnd *argo_ring_handle_alloc(domid_t domain, unsigned int port, + argo_recv_skb_cb recv_cb, void *priv); + +/* + * Ring primitives, hypercalls to Xen. + */ +void argo_ring_unregister(struct argo_ring_hnd *h); +int argo_ring_register(struct argo_ring_hnd *h); + +/* + * Ring "send" primitive. send is synchronous, direct hypercall to Xen. + */ +int argo_ring_send_skb(struct argo_ring_hnd *h, const struct sk_buff *skb, + xen_argo_send_addr_t *send); + +int argo_core_init(void); +void argo_core_cleanup(void); + +#endif /* !_ARGO_RING_H_ */ diff --git a/vsock-argo/module/argo_transport.c b/vsock-argo/module/argo_transport.c new file mode 100644 index 0000000..87fdeca --- /dev/null +++ b/vsock-argo/module/argo_transport.c @@ -0,0 +1,559 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "argo_ring.h" + +/* + * Argo auto-bind default address. + */ +static const struct sockaddr_vm addr_auto = { + .svm_family = AF_VSOCK, + .svm_cid = XEN_ARGO_DOMID_ANY, + .svm_port = 0, + .svm_zero = { 0 }, +}; + +/* + * Argo private data. + */ +struct argo_transport { + struct list_head sockets; /* List of all argo_transport. */ + struct argo_ring_hnd *h; /* Argo ring handle. */ + struct vsock_sock *vsk; /* Parent vsock struct. */ +}; + +/* + * Global socket list. + */ +struct list_head sockets = LIST_HEAD_INIT(sockets); + +/* + * Private data helpers. + */ +#define argo_trans(vsk) ((struct argo_transport *)((vsk)->trans)) + +/* + * Initialize/Tear-down socket. + */ +static int argo_transport_socket_init( + struct vsock_sock *vsk, struct vsock_sock *psk) +{ + vsk->trans = kmalloc(sizeof (struct argo_transport), GFP_KERNEL); + if (!vsk->trans) + return -ENOMEM; + INIT_LIST_HEAD(&argo_trans(vsk)->sockets); + list_add_tail(&argo_trans(vsk)->sockets, &sockets); + argo_trans(vsk)->vsk = vsk; + argo_trans(vsk)->h = NULL; + + return 0; +} + +static void argo_transport_destruct(struct vsock_sock *vsk) +{ + argo_trans(vsk)->vsk = NULL; + list_del_init(&argo_trans(vsk)->sockets); + kfree(argo_trans(vsk)); + vsk->trans = NULL; + + return; +} + +static void argo_transport_release(struct vsock_sock *vsk) +{ + struct argo_transport *t = argo_trans(vsk); + + vsock_remove_sock(vsk); + + /* + * Disconnect/Detach before release of resources: + * TODO: Send RST for STREAM. + */ + + if (t->h) { + argo_ring_unregister(t->h); + argo_ring_handle_free(t->h); + } +} + +/* + * VSock VMADDR_CID_ANY & VMADDR_PORT_ANY do not match Argo definitions. + * Convert sockaddr_vm to sockaddr_vm argo compatible: + * - CID: 0 -> XEN_ARGO_DOMID_ANY. + * - PORT: 0 -> ~0U - 1 ? + */ +static inline int sockaddr_vm_normalize(struct sockaddr_vm *addr) +{ + if (addr->svm_cid == VMADDR_CID_ANY) + addr->svm_cid = addr_auto.svm_cid; + if (addr->svm_port == VMADDR_PORT_ANY) + addr->svm_port = addr_auto.svm_port; + + if (addr->svm_cid > XEN_ARGO_DOMID_ANY) + return EINVAL; + + return 0; +} +static inline int sockaddrvm_to_argo(const struct sockaddr_vm *s, xen_argo_addr_t *d) +{ + struct sockaddr_vm c = *s; + + if (sockaddr_vm_normalize(&c)) + return EINVAL; + + d->domain_id = s->svm_cid; + d->aport = s->svm_port; + d->pad = 0; + + return 0; +} + +static inline bool sockaddr_vm_match(const struct sockaddr_vm *src, + const struct sockaddr_vm *dst) +{ + return ((src->svm_cid == dst->svm_cid) && + (src->svm_port == dst->svm_port)); +} + +/* + * Connections. + */ +static int argo_transport_connect(struct vsock_sock *vsk) +{ + if (!vsock_addr_bound(&vsk->local_addr)) + return -EINVAL; + if (!vsock_addr_bound(&vsk->remote_addr)) + return -EINVAL; + + /* TODO: STREAM will require SYN/ACK dance here. + * DGRAM requires nothing right? */ + + return -ECONNREFUSED; +} + +/* + * DGRAM. + */ +static int argo_transport_recv_dgram_cb(void *priv, struct sk_buff *skb); +static int argo_transport_dgram_bind(struct vsock_sock *vsk, + struct sockaddr_vm *addr) +{ + struct argo_transport *t = argo_trans(vsk); + int rc; + + if (sockaddr_vm_normalize(addr)) + return EINVAL; + + /* Auto-bind local_addr. */ + memcpy(&vsk->local_addr, addr, sizeof (*addr)); + + t->h = argo_ring_handle_alloc(addr->svm_cid, addr->svm_port, + argo_transport_recv_dgram_cb, vsk); + if (IS_ERR(t->h)) { + rc = PTR_ERR(t->h); + pr_debug("argo_ring_handle_alloc(dom%u:%u) %s (%d).\n", + addr->svm_cid, addr->svm_port, + rc ? "failed" : "succeed", -rc); + goto failed_alloc; + } + + rc = argo_ring_register(t->h); + if (rc) { + pr_debug("argo_ring_register(dom%u:%u) %s (%d).\n", + addr->svm_cid, addr->svm_port, + rc ? "failed" : "succeed", -rc); + goto failed_register; + } + + return 0; + +failed_register: + argo_ring_handle_free(t->h); + t->h = NULL; +failed_alloc: + return rc; +} + +static int argo_transport_dgram_enqueue(struct vsock_sock *vsk, + struct sockaddr_vm *remote_addr, struct msghdr *msg, size_t len) +{ + int rc = 0; + struct sk_buff *skb; + xen_argo_send_addr_t sendaddr; + + /* TODO: Auto-bind already done? */ + if (sockaddrvm_to_argo(&vsk->local_addr, &sendaddr.src) || + sockaddrvm_to_argo(remote_addr, &sendaddr.dst)) + return EINVAL; + + skb = alloc_skb(len, GFP_ATOMIC); + if (!skb) { + pr_debug("%s: alloc_skb failed.\n", __func__); + return -ENOMEM; + } + + if (memcpy_from_msg(skb_put(skb, len), msg, len)) { + pr_debug("%s: memcpy_from_msg failed.\n", __func__); + rc = -EMSGSIZE; + goto out; + } + + rc = argo_ring_send_skb(argo_trans(vsk)->h, skb, &sendaddr); + if (rc < 0) { + pr_debug("%s: argo_ring_send_skb failed.\n", __func__); + goto out; + } + + pr_debug("enqueued %zuB dom%u:%u to dom%u:%u.\n", len, + sendaddr.src.domain_id, sendaddr.src.aport, + sendaddr.dst.domain_id, sendaddr.dst.aport); + +out: + kfree_skb(skb); + return rc; +} + +static int argo_transport_recv_dgram_cb(void *priv, struct sk_buff *skb) +{ + struct vsock_sock *vsk = priv; + struct sock *sk = &vsk->sk; + int rc; + + /* sk_receive_skb() does sock_put(). */ + sock_hold(sk); + rc = sk_receive_skb(sk, skb, 0); + if (rc != NET_RX_SUCCESS) + pr_warn("dom%u:%u cannot queue packet, dropping.", + vsk->local_addr.svm_cid, + vsk->local_addr.svm_port); + return rc == NET_RX_SUCCESS ? 0 : -1; +} + +static int argo_transport_dgram_dequeue(struct vsock_sock *vsk, + struct msghdr *msg, size_t len, int flags) +{ + struct sk_buff *skb; + struct xen_argo_ring_message_header *mh; + size_t msg_len; + int rc = 0; + + skb = skb_recv_datagram(&vsk->sk, flags, flags & MSG_DONTWAIT, &rc); + if (!skb) { + pr_debug("skb_recv_datagram failed (%d).\n", rc); + goto out; + } + + /* Assume skb is always in linear data area for now. */ + mh = (void*)skb->data; + if (!mh) { + pr_debug("could not access sk_buff data to read message header, dropping packet.\n"); + goto out; + } + + msg_len = mh->len - sizeof (*mh); + rc = skb_copy_datagram_msg(skb, sizeof (*mh), msg, msg_len); + if (rc) + goto out; + + if (msg->msg_name) { + DECLARE_SOCKADDR(struct sockaddr_vm *, vm_addr, msg->msg_name); + vsock_addr_init(vm_addr, mh->source.domain_id, mh->source.aport); + msg->msg_namelen = sizeof (*vm_addr); + pr_debug("dequeued: report source as dom%u:%u\n", + vm_addr->svm_cid, vm_addr->svm_port); + } + pr_debug("dequeued skb: %uB (%zuB data) from dom%u:%u\n", + mh->len, msg_len, mh->source.domain_id, mh->source.aport); + + rc = msg_len; +out: + skb_free_datagram(&vsk->sk, skb); + return rc; +} + +static bool argo_transport_dgram_allow(u32 cid, u32 port) +{ + return true; +} + + +/* + * TODO: STREAM. + */ +#ifdef TODO_STREAM +static ssize_t argo_transport_stream_dequeue( + struct vsock_sock *vsk, + struct msghdr *msg, + size_t len, + int flags) +{ + return -ENOTSUP; +} + +static ssize_t argo_transport_stream_enqueue( + struct vsock_sock *vsk, + struct msghdr *msg, + size_t len) +{ + return -ENOTSUP; +} + +static s64 argo_transport_stream_has_data(struct vsock_sock *vsk) +{ + return -ENOTSUP; +} + +static s64 argo_transport_stream_has_space(struct vsock_sock *vsk) +{ + return -ENOTSUP; +} + +static u64 argo_transport_stream_rcvhiwat(struct vsock_sock *vsk) +{ + return -ENOTSUP; + /* TODO: Return high-watermark... probably something to frob around + with. */ +} + +static bool argo_transport_stream_is_active(struct vsock_sock *vsk) +{ + return false; +} + +static bool argo_transport_stream_allow(u32 cid, u32 port) +{ + /* TODO: Pre-filtering can be done in here. */ + return false; +} +#endif /* TODO_STREAM */ + +/* + * Notification. + */ +static int argo_transport_notify_poll_in( + struct vsock_sock *vsk, + size_t target, + bool *data_ready_now) +{ + *data_ready_now = vsock_stream_has_data(vsk); + return 0; +} + +static int argo_transport_notify_poll_out( + struct vsock_sock *vsk, + size_t target, + bool *space_available_now) +{ + *space_available_now = vsock_stream_has_space(vsk); + return 0; +} + +static int argo_transport_notify_recv_init( + struct vsock_sock *vsk, + size_t target, + struct vsock_transport_recv_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_recv_pre_block( + struct vsock_sock *vsk, + size_t target, + struct vsock_transport_recv_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_recv_pre_dequeue( + struct vsock_sock *vsk, + size_t target, + struct vsock_transport_recv_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_recv_post_dequeue( + struct vsock_sock *vsk, + size_t target, + ssize_t copied, + bool data_read, + struct vsock_transport_recv_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_send_init( + struct vsock_sock *vsk, + struct vsock_transport_send_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_send_pre_block( + struct vsock_sock *vsk, + struct vsock_transport_send_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_send_pre_enqueue( + struct vsock_sock *vsk, + struct vsock_transport_send_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +static int argo_transport_notify_send_post_enqueue( + struct vsock_sock *vsk, + ssize_t written, + struct vsock_transport_send_notify_data *data) +{ + /* TODO: Not sure... */ + return 0; +} + +/* + * Shutdown. + */ +static int argo_transport_shutdown(struct vsock_sock *vsk, int mode) +{ + /* TODO: That might be where we want to send RST instead... */ + return 0; +} + +/* + * Buffer sizes. + */ +static void argo_transport_set_buffer_size(struct vsock_sock *vsk, u64 val) +{ + /* TODO: Probably not usable in our case. */ +} + +static void argo_transport_set_min_buffer_size(struct vsock_sock *vsk, u64 val) +{ +} + +static void argo_transport_set_max_buffer_size(struct vsock_sock *vsk, u64 val) +{ +} + +static u64 argo_transport_get_buffer_size(struct vsock_sock *vsk) +{ + return 0ULL; +} + +static u64 argo_transport_get_min_buffer_size(struct vsock_sock *vsk) +{ + return 0ULL; +} + +static u64 argo_transport_get_max_buffer_size(struct vsock_sock *vsk) +{ + return 0ULL; +} + +static u32 argo_transport_get_local_cid(void) +{ + /* TODO: May require svm_cid format instead of Argo. */ + return XEN_ARGO_DOMID_ANY; +} + +static struct vsock_transport argo_transport = { + .init = argo_transport_socket_init, + .destruct = argo_transport_destruct, + .release = argo_transport_release, + + .connect = argo_transport_connect, + + .dgram_bind = argo_transport_dgram_bind, + .dgram_dequeue = argo_transport_dgram_dequeue, + .dgram_enqueue = argo_transport_dgram_enqueue, + .dgram_allow = argo_transport_dgram_allow, + +#ifdef TODO_STREAM + .stream_dequeue = argo_transport_stream_dequeue, + .stream_enqueue = argo_transport_stream_enqueue, + .stream_has_data = argo_transport_stream_has_data, + .stream_has_space = argo_transport_stream_has_space, + .stream_rcvhiwat = argo_transport_stream_rcvhiwat, + .stream_is_active = argo_transport_stream_is_active, + .stream_allow = argo_transport_stream_allow, +#endif /* TODO_STREAM */ + .notify_poll_in = argo_transport_notify_poll_in, + .notify_poll_out = argo_transport_notify_poll_out, + .notify_recv_init = argo_transport_notify_recv_init, + .notify_recv_pre_block = argo_transport_notify_recv_pre_block, + .notify_recv_pre_dequeue = argo_transport_notify_recv_pre_dequeue, + .notify_recv_post_dequeue = argo_transport_notify_recv_post_dequeue, + .notify_send_init = argo_transport_notify_send_init, + .notify_send_pre_block = argo_transport_notify_send_pre_block, + .notify_send_pre_enqueue = argo_transport_notify_send_pre_enqueue, + .notify_send_post_enqueue = argo_transport_notify_send_post_enqueue, + + .shutdown = argo_transport_shutdown, + + .set_buffer_size = argo_transport_set_buffer_size, + .set_min_buffer_size = argo_transport_set_min_buffer_size, + .set_max_buffer_size = argo_transport_set_max_buffer_size, + .get_buffer_size = argo_transport_get_buffer_size, + .get_min_buffer_size = argo_transport_get_min_buffer_size, + .get_max_buffer_size = argo_transport_get_max_buffer_size, + + .get_local_cid = argo_transport_get_local_cid, +}; + +static int __init argo_transport_init(void) +{ + int rc; + + rc = vsock_core_init(&argo_transport); + if (rc) { + pr_err("vsock_core_init() failed (%d).\n", rc); + return rc; + } + rc = argo_core_init(); + if (rc) { + pr_err("argo_core_init() failed (%d).\n", rc); + vsock_core_exit(); + return rc; + } + pr_info("vsock_argo_transport registered.\n"); + + return 0; +} +module_init(argo_transport_init); + +static void __exit argo_transport_exit(void) +{ + /* TODO: Flush sockets... */ + + pr_info("vsock_argo_transport unregistered.\n"); + argo_core_cleanup(); + vsock_core_exit(); + return; +} +module_exit(argo_transport_exit); + +MODULE_AUTHOR("Assured Information Security, Inc."); +MODULE_DESCRIPTION("Argo transport for Virtual Socket."); +MODULE_VERSION("1.0.0"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("argo_vsock"); +MODULE_ALIAS_NETPROTO(argo_vsock); diff --git a/vsock-argo/module/asm-x86/argo-compat.h b/vsock-argo/module/asm-x86/argo-compat.h new file mode 100644 index 0000000..36178cc --- /dev/null +++ b/vsock-argo/module/asm-x86/argo-compat.h @@ -0,0 +1,43 @@ +#include +#include +#include +#include + +#ifndef HYPERVISOR_argo_op +#define __HYPERVISOR_argo_op 39 + +#ifndef _hypercall5 +/* + * Removed because unused in: + * https://lkml.org/lkml/2018/8/20/267 + */ +#define _hypercall5(type, name, a1, a2, a3, a4, a5) \ +({ \ + __HYPERCALL_DECLS; \ + __HYPERCALL_5ARG(a1, a2, a3, a4, a5); \ + asm volatile (__HYPERCALL \ + : __HYPERCALL_5PARAM \ + : __HYPERCALL_ENTRY(name) \ + : __HYPERCALL_CLOBBER5); \ + (type)__res; \ +}) +#endif /* _hypercall5*/ + +static inline int __must_check +HYPERVISOR_argo_op(int cmd, void *arg1, void *arg2, uint32_t arg3, + uint32_t arg4) +{ + int ret; + + stac(); + ret = _hypercall5(int, argo_op, cmd, arg1, arg2, arg3, arg4); + clac(); + + return ret; +} + +#ifndef VIRQ_ARGO +#define VIRQ_ARGO 11 /* G. (DOM0) ARGO interdomain communication */ +#endif + +#endif /* _ARGO_H_ */ diff --git a/vsock-argo/module/include/Kbuild b/vsock-argo/module/include/Kbuild new file mode 100644 index 0000000..768209a --- /dev/null +++ b/vsock-argo/module/include/Kbuild @@ -0,0 +1 @@ +header-y += xen/ diff --git a/vsock-argo/module/include/xen/Kbuild b/vsock-argo/module/include/xen/Kbuild new file mode 100644 index 0000000..93caae4 --- /dev/null +++ b/vsock-argo/module/include/xen/Kbuild @@ -0,0 +1 @@ +header-y += argo.h diff --git a/vsock-argo/module/include/xen/argo.h b/vsock-argo/module/include/xen/argo.h new file mode 100644 index 0000000..0b598a8 --- /dev/null +++ b/vsock-argo/module/include/xen/argo.h @@ -0,0 +1,279 @@ +/****************************************************************************** + * Argo : Hypervisor-Mediated data eXchange + * + * Derived from v4v, the version 2 of v2v. + * + * Copyright (c) 2010, Citrix Systems + * Copyright (c) 2018-2019, BAE Systems + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __XEN_PUBLIC_ARGO_H__ +#define __XEN_PUBLIC_ARGO_H__ + +#ifdef __XEN__ +#include "xen.h" +#else +#include +#endif + +#define XEN_ARGO_DOMID_ANY DOMID_INVALID + +/* The maximum size of an Argo ring is defined to be: 16MB (0x1000000 bytes). */ +#define XEN_ARGO_MAX_RING_SIZE (0x1000000ULL) + +/* Fixed-width type for "argo port" number. Nothing to do with evtchns. */ +typedef uint32_t xen_argo_port_t; + +/* gfn type: 64-bit fixed-width on all architectures */ +typedef uint64_t xen_argo_gfn_t; + +/* + * XEN_ARGO_MAXIOV : maximum number of iovs accepted in a single sendv. + * Caution is required if this value is increased: this determines the size of + * an array of xen_argo_iov_t structs on the hypervisor stack, so could cause + * stack overflow if the value is too large. + * The Linux Argo driver never passes more than two iovs. +*/ +#define XEN_ARGO_MAXIOV 8U + +#ifdef DEFINE_XEN_GUEST_HANDLE +DEFINE_XEN_GUEST_HANDLE(uint8_t); +#endif + +typedef struct xen_argo_iov +{ +#ifdef XEN_GUEST_HANDLE_64 + XEN_GUEST_HANDLE_64(uint8_t) iov_hnd; +#else + uint64_t iov_hnd; +#endif + uint32_t iov_len; + uint32_t pad; +} xen_argo_iov_t; + +typedef struct xen_argo_addr +{ + xen_argo_port_t aport; + domid_t domain_id; + uint16_t pad; +} xen_argo_addr_t; + +typedef struct xen_argo_send_addr +{ + struct xen_argo_addr src; + struct xen_argo_addr dst; +} xen_argo_send_addr_t; + +typedef struct xen_argo_ring +{ + /* Guests should use atomic operations to access rx_ptr */ + uint32_t rx_ptr; + /* Guests should use atomic operations to access tx_ptr */ + uint32_t tx_ptr; + /* + * Header space reserved for later use. Align the start of the ring to a + * multiple of the message slot size. + */ + uint8_t reserved[56]; +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + uint8_t ring[]; +#elif defined(__GNUC__) + uint8_t ring[0]; +#endif +} xen_argo_ring_t; + +typedef struct xen_argo_register_ring +{ + xen_argo_port_t aport; + domid_t partner_id; + uint16_t pad; + uint32_t len; +} xen_argo_register_ring_t; + +typedef struct xen_argo_unregister_ring +{ + xen_argo_port_t aport; + domid_t partner_id; + uint16_t pad; +} xen_argo_unregister_ring_t; + +/* Messages on the ring are padded to a multiple of this size. */ +#define XEN_ARGO_MSG_SLOT_SIZE 0x10 + +/* + * Notify flags + */ +/* Ring exists */ +#define XEN_ARGO_RING_EXISTS (1U << 0) +/* Ring is shared, not unicast */ +#define XEN_ARGO_RING_SHARED (1U << 1) +/* Ring is empty */ +#define XEN_ARGO_RING_EMPTY (1U << 2) +/* Sufficient space to queue space_required bytes might exist */ +#define XEN_ARGO_RING_SUFFICIENT (1U << 3) +/* Insufficient ring size for space_required bytes */ +#define XEN_ARGO_RING_EMSGSIZE (1U << 4) +/* Too many domains waiting for available space signals for this ring */ +#define XEN_ARGO_RING_EBUSY (1U << 5) + +typedef struct xen_argo_ring_data_ent +{ + struct xen_argo_addr ring; + uint16_t flags; + uint16_t pad; + uint32_t space_required; + uint32_t max_message_size; +} xen_argo_ring_data_ent_t; + +typedef struct xen_argo_ring_data +{ + uint32_t nent; + uint32_t pad; +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + struct xen_argo_ring_data_ent data[]; +#elif defined(__GNUC__) + struct xen_argo_ring_data_ent data[0]; +#endif +} xen_argo_ring_data_t; + +struct xen_argo_ring_message_header +{ + uint32_t len; + struct xen_argo_addr source; + uint32_t message_type; +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + uint8_t data[]; +#elif defined(__GNUC__) + uint8_t data[0]; +#endif +}; + +/* + * Hypercall operations + */ + +/* + * XEN_ARGO_OP_register_ring + * + * Register a ring using the guest-supplied memory pages. + * Also used to reregister an existing ring (eg. after resume from hibernate). + * + * The first argument struct indicates the port number for the ring to register + * and the partner domain, if any, that is to be allowed to send to the ring. + * A wildcard (XEN_ARGO_DOMID_ANY) may be supplied instead of a partner domid, + * and if the hypervisor has wildcard sender rings enabled, this will allow + * any domain (XSM notwithstanding) to send to the ring. + * + * The second argument is an array of guest frame numbers and the third argument + * indicates the size of the array. This operation only supports 4K-sized pages. + * + * arg1: XEN_GUEST_HANDLE(xen_argo_register_ring_t) + * arg2: XEN_GUEST_HANDLE(xen_argo_gfn_t) + * arg3: unsigned long npages + * arg4: unsigned long flags (32-bit value) + */ +#define XEN_ARGO_OP_register_ring 1 + +/* Register op flags */ +/* + * Fail exist: + * If set, reject attempts to (re)register an existing established ring. + * If clear, reregistration occurs if the ring exists, with the new ring + * taking the place of the old, preserving tx_ptr if it remains valid. + */ +#define XEN_ARGO_REGISTER_FLAG_FAIL_EXIST 0x1 + +#ifdef __XEN__ +/* Mask for all defined flags. */ +#define XEN_ARGO_REGISTER_FLAG_MASK XEN_ARGO_REGISTER_FLAG_FAIL_EXIST +#endif + +/* + * XEN_ARGO_OP_unregister_ring + * + * Unregister a previously-registered ring, ending communication. + * + * arg1: XEN_GUEST_HANDLE(xen_argo_unregister_ring_t) + * arg2: NULL + * arg3: 0 (ZERO) + * arg4: 0 (ZERO) + */ +#define XEN_ARGO_OP_unregister_ring 2 + +/* + * XEN_ARGO_OP_sendv + * + * Send a list of buffers contained in iovs. + * + * The send address struct specifies the source and destination addresses + * for the message being sent, which are used to find the destination ring: + * Xen first looks for a most-specific match with a registered ring with + * (id.addr == dst) and (id.partner == sending_domain) ; + * if that fails, it then looks for a wildcard match (aka multicast receiver) + * where (id.addr == dst) and (id.partner == DOMID_ANY). + * + * For each iov entry, send iov_len bytes from iov_base to the destination ring. + * If insufficient space exists in the destination ring, it will return -EAGAIN + * and Xen will notify the caller when sufficient space becomes available. + * + * The message type is a 32-bit data field available to communicate message + * context data (eg. kernel-to-kernel, rather than application layer). + * + * arg1: XEN_GUEST_HANDLE(xen_argo_send_addr_t) source and dest addresses + * arg2: XEN_GUEST_HANDLE(xen_argo_iov_t) iovs + * arg3: unsigned long niov + * arg4: unsigned long message type (32-bit value) + */ +#define XEN_ARGO_OP_sendv 3 + +/* + * XEN_ARGO_OP_notify + * + * Asks Xen for information about other rings in the system. + * + * ent->ring is the xen_argo_addr_t of the ring you want information on. + * Uses the same ring matching rules as XEN_ARGO_OP_sendv. + * + * ent->space_required : if this field is not null then Xen will check + * that there is space in the destination ring for this many bytes of payload. + * If the ring is too small for the requested space_required, it will set the + * XEN_ARGO_RING_EMSGSIZE flag on return. + * If sufficient space is available, it will set XEN_ARGO_RING_SUFFICIENT + * and CANCEL any pending notification for that ent->ring; otherwise it + * will schedule a notification event and the flag will not be set. + * + * These flags are set by Xen when notify replies: + * XEN_ARGO_RING_EXISTS ring exists + * XEN_ARGO_RING_SHARED ring is registered for wildcard partner + * XEN_ARGO_RING_EMPTY ring is empty + * XEN_ARGO_RING_SUFFICIENT sufficient space for space_required is there + * XEN_ARGO_RING_EMSGSIZE space_required is too large for the ring size + * XEN_ARGO_RING_EBUSY too many domains waiting for available space signals + * + * arg1: XEN_GUEST_HANDLE(xen_argo_ring_data_t) ring_data (may be NULL) + * arg2: NULL + * arg3: 0 (ZERO) + * arg4: 0 (ZERO) + */ +#define XEN_ARGO_OP_notify 4 + +#endif From 8ea0f5b77cf5795fea2f4e2d62c8946a8b2f0090 Mon Sep 17 00:00:00 2001 From: Eric Chanudet Date: Thu, 25 Apr 2019 11:05:04 -0400 Subject: [PATCH 2/3] vsock/test: Netcat-like utility for Argo DGRAM. Very simple tool to send/recv things through VSock. Basic usage: hatch domid port hatch -r -p local_port Options: -r, --recv receive mode, wait for incoming packets. -p, --port set local port number. -D enable debug output on stderr. Debug is logged on stderr to allow stdout redirection. If vsock_argo_transport.ko was not loaded, vmw_vsock_vmci_transport.ko will likely be loaded by default (depends on the installation). You need to remove vmw_vsock_vmci_transport.ko before you can load vsock_argo_transport.ko again. Signed-off-by: Eric Chanudet OXT-1473 --- vsock-argo/test/hatch/.gitignore | 72 ++++++++ vsock-argo/test/hatch/Makefile.am | 1 + vsock-argo/test/hatch/configure.ac | 46 +++++ vsock-argo/test/hatch/src/Makefile.am | 5 + vsock-argo/test/hatch/src/hatch.c | 249 ++++++++++++++++++++++++++ vsock-argo/test/hatch/src/project.h | 53 ++++++ vsock-argo/test/hatch/src/utils.h | 84 +++++++++ 7 files changed, 510 insertions(+) create mode 100644 vsock-argo/test/hatch/.gitignore create mode 100644 vsock-argo/test/hatch/Makefile.am create mode 100644 vsock-argo/test/hatch/configure.ac create mode 100644 vsock-argo/test/hatch/src/Makefile.am create mode 100644 vsock-argo/test/hatch/src/hatch.c create mode 100644 vsock-argo/test/hatch/src/project.h create mode 100644 vsock-argo/test/hatch/src/utils.h diff --git a/vsock-argo/test/hatch/.gitignore b/vsock-argo/test/hatch/.gitignore new file mode 100644 index 0000000..d6e9f46 --- /dev/null +++ b/vsock-argo/test/hatch/.gitignore @@ -0,0 +1,72 @@ +## Autotools. +# http://www.gnu.org/software/automake +Makefile +Makefile.in +/ar-lib +/mdate-sh +/py-compile +/test-driver +/ylwrap + +# http://www.gnu.org/software/autoconf +/autom4te.cache +/autoscan.log +/autoscan-*.log +/aclocal.m4 +/compile +/config.guess +config.h +config.h.in +config.h.in~ +/config.sub +/configure +/configure.scan +/depcomp +/install-sh +/missing +stamp-h1 +/config.log +/config.status + +# https://www.gnu.org/software/libtool/ +/ltmain.sh +.libs +.deps +libtool + +# http://www.gnu.org/software/texinfo +/texinfo.tex + +## C. +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects +*.so +*.so.* +*.dylib + +## Misc +cscope.* +hatch diff --git a/vsock-argo/test/hatch/Makefile.am b/vsock-argo/test/hatch/Makefile.am new file mode 100644 index 0000000..af437a6 --- /dev/null +++ b/vsock-argo/test/hatch/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = src diff --git a/vsock-argo/test/hatch/configure.ac b/vsock-argo/test/hatch/configure.ac new file mode 100644 index 0000000..31cd904 --- /dev/null +++ b/vsock-argo/test/hatch/configure.ac @@ -0,0 +1,46 @@ +# +# Autoconf initialisation. +# +AC_INIT(hatch, 1.0) +AC_PREREQ(2.13) +AC_CONFIG_HEADERS(src/config.h) +AC_CONFIG_SRCDIR(src/project.h) +AM_INIT_AUTOMAKE([foreign -Wall -Werror]) + +# +# Version management. +# +1 : 0 Interface changes breaking retro-compatibility. +# 0 : +1 Internal changes without lost of retro-compatibility. +# +VERSION_MAJOR=0 +VERSION_MINOR=1 +AC_DEFINE_UNQUOTED([VERSION_MAJOR], [$VERSION_MAJOR], [major version number]) +AC_DEFINE_UNQUOTED([VERSION_MINOR], [$VERSION_MINOR], [minor version number]) + +# +# Standard checks. +# +AC_PROG_CC +AC_PROG_CPP +AC_PROG_INSTALL +AC_PROG_LN_S +AC_PROG_MAKE_SET +AM_PROG_AR + +# Compiler capabilities. +AC_C_INLINE +AC_C_CONST + +# Required headers. +AC_CHECK_HEADERS([sys/ioctl.h sys/socket.h linux/socket.h]) +AC_CHECK_HEADERS([linux/vm_sockets.h], [], + AC_MSG_ERROR("VSock support not available"), + [#include + #include + #include + ]) + +# Output files. +AC_OUTPUT([Makefile + src/Makefile + ]) diff --git a/vsock-argo/test/hatch/src/Makefile.am b/vsock-argo/test/hatch/src/Makefile.am new file mode 100644 index 0000000..1b00e7a --- /dev/null +++ b/vsock-argo/test/hatch/src/Makefile.am @@ -0,0 +1,5 @@ +AM_CFLAGS = -W -Wall -Werror + +bin_PROGRAMS = hatch + +hatch_SOURCES = hatch.c project.h utils.h diff --git a/vsock-argo/test/hatch/src/hatch.c b/vsock-argo/test/hatch/src/hatch.c new file mode 100644 index 0000000..b80f5d7 --- /dev/null +++ b/vsock-argo/test/hatch/src/hatch.c @@ -0,0 +1,249 @@ +#include "project.h" + +/* + * Parameter agregation. + */ +struct hatch_args { + int recv; + unsigned int local_port; + unsigned int port; + unsigned int domid; +}; + +/* + * Sanity check helpers. + */ +static int hatch_sanity(const struct hatch_args *args) +{ + assert(args != NULL); + + if (!args->recv && (args->domid == VMADDR_CID_ANY)) { + printe("Missing domid."); + return -EINVAL; + } + if (args->recv && !args->local_port) { + printe("Missing local port."); + return -EINVAL; + } + if (!args->recv && !args->port) { + printe("Missing port."); + return -EINVAL; + } + + return 0; +} + +/* + * Recving end. + */ +static int hatch_recvfrom_run(unsigned long port) +{ + struct sockaddr_vm sa = { 0 }; + struct sockaddr_vm sar = { + .svm_family = AF_VSOCK, + .svm_port = port, + .svm_cid = VMADDR_CID_ANY, + .svm_zero = { 0 }, + }; + socklen_t sa_len = 0; + char msg[1025] = { 0 }; + int s, rc; + + s = __vsock_bdgram(&sar); + + while (1) { + rc = recvfrom(s, msg, sizeof (msg) - 1, 0, + (struct sockaddr *)&sa, &sa_len); + if (rc < 0) { + rc = -errno; + perror("recvfrom"); + goto out; + } + printd("received from dom%u:%u %dB.", sa.svm_cid, sa.svm_port, rc); + msg[rc] = '\0'; + rc = write(STDOUT_FILENO, msg, rc); + if (rc < 0) { + rc = -errno; + perror("write"); + goto out; + } + } + +out: + close(s); + return rc; +} + +/* + * Sending side. + */ +static int hatch_sendto_run(unsigned int domid, unsigned long port) +{ + int s, rc; +#define AUTOBIND +#ifndef AUTOBIND + struct sockaddr_vm sal = { + .svm_family = AF_VSOCK, + .svm_cid = VMADDR_CID_ANY, + .svm_port = VMADDR_PORT_ANY, + .svm_zero = { 0 }, + }; +#endif /* AUTOBIND */ + struct sockaddr_vm sa = { + .svm_family = AF_VSOCK, + .svm_cid = domid, + .svm_port = port, + .svm_zero = { 0 }, + }; + char msg[1025] = { 0 }; + size_t offset, msg_len = 0; + +#ifdef AUTOBIND + s = __vsock_dgram(); +#else /* !AUTOBIND */ + s = __vsock_bdgram(&sal); +#endif /* AUTOBIND */ + + while (1) { + do { + rc = read(0, msg, sizeof (msg) - 1); + if (rc < 0) { + rc = -errno; + perror("read"); + goto out; + } else if (rc == 0) + goto out; + + msg_len = rc; + } while (msg_len == 0); + + printd("%zuB read from stdin.", msg_len); + + offset = 0; + do { + rc = sendto(s, msg + offset, msg_len, 0, + (struct sockaddr *)&sa, sizeof (sa)); + if (rc < 0) { + rc = -errno; + perror("sendto"); + goto out; + } + printd("%dB sent to dom%u:%u.", rc, sa.svm_cid, sa.svm_port); + offset += rc; + msg_len -= rc; + } while (msg_len != 0); + } + +out: + close(s); + return rc; +} + +static int hatch_run(const struct hatch_args *args) +{ + int rc; + + assert(args != NULL); + + rc = hatch_sanity(args); + if (rc < 0) + return -rc; + + if (args->recv) + rc = hatch_recvfrom_run(args->local_port); + else + rc = hatch_sendto_run(args->domid, args->port); + + return rc; +} + +/* + * Usage. + */ +static int usage(int rc) +{ + printi("Basic usage:"); + printi(" hatch domid port"); + printi(" hatch -r -p local_port"); + printi("Options:"); + printi(" -r, --recv receive mode, wait for incoming packets."); + printi(" -p, --port set local port number."); + printi(" -D enable debug output on stderr."); + + return rc; +} + +/* + * Option handling. + */ +#define OPT_STR "hrp:D" +static struct option long_options[] = { + { "help", no_argument, 0, 'h' }, + { "recv", no_argument, 0, 'r' }, + { "port", required_argument, 0, 'p' }, + { "" , no_argument, 0, 'D' }, + { 0, 0, 0, 0 } +}; + +int main(int argc, char *argv[]) +{ + int rc; + struct hatch_args args = { + .recv = 0, + .local_port = 0, + .port = 0, + .domid = VMADDR_CID_ANY, + }; + + if (argc < 1) + return usage(EINVAL); + + while (1) { + int opt, longindex; + + opt = getopt_long(argc, argv, OPT_STR, long_options, &longindex); + switch (opt) { + case -1: + goto getopt_done; + case 0: + printe("Malformed option \"%s\", please fix the code.", + long_options[longindex].name); + return EINVAL; + case 'h': + return usage(0); + case 'r': + args.recv = 1; + continue; + case 'p': + rc = parse_uint(optarg, &args.local_port); + if (rc || !is_valid_port(args.local_port)) { + printe("Invalid local port %u.", args.local_port); + return -rc; + } + continue; + case 'D': + __enable_debug_print = 1; + continue; + + default: + printe("Unknown option '%c'.", opt); + return usage(EINVAL); + } + } + +getopt_done: + while (optind < argc) { + if (args.domid == VMADDR_CID_ANY) + parse_domid(argv[optind++], &args.domid); + else if (!args.port) { + rc = parse_uint(argv[optind++], &args.port); + if (rc || !is_valid_port(args.port)) { + printe("Invalid port %s.", argv[optind - 1]); + return -rc; + } + } else + printw("Argument \"%s\" not handled.", argv[optind++]); + } + + return hatch_run(&args); +} diff --git a/vsock-argo/test/hatch/src/project.h b/vsock-argo/test/hatch/src/project.h new file mode 100644 index 0000000..28a8694 --- /dev/null +++ b/vsock-argo/test/hatch/src/project.h @@ -0,0 +1,53 @@ +#ifndef __PROJECT_H__ +# define __PROJECT_H__ + +# include "config.h" + +# include +# include +# include +# include + +# include + +# include +# include + +# include +# include + +# include +# include + +# include + +# include +# include +# include +# include + +/* + * Output macro helpers. + */ +static unsigned int __enable_debug_print = 0; +#define printd(fmt, ...) \ + do { \ + if (__enable_debug_print) \ + fprintf(stderr, "debug:%s:%d: " fmt "\n", \ + __func__, __LINE__, ##__VA_ARGS__); \ + } while (0) +#define printi(fmt, ...) \ + fprintf(stdout, fmt "\n", ##__VA_ARGS__) +#define printw(fmt, ...) \ + fprintf(stderr, "warning: " fmt "\n", ##__VA_ARGS__) +#define printe(fmt, ...) \ + fprintf(stderr, "error: " fmt "\n", ##__VA_ARGS__) + +/* + * GCC macro helpers. + */ +#define unused(x) ((void)(x)) + +# include "utils.h" + +#endif /* __PROJECT_H__ */ diff --git a/vsock-argo/test/hatch/src/utils.h b/vsock-argo/test/hatch/src/utils.h new file mode 100644 index 0000000..8eca55e --- /dev/null +++ b/vsock-argo/test/hatch/src/utils.h @@ -0,0 +1,84 @@ +#ifndef _UTILS_H_ +# define _UTILS_H_ + +# include "project.h" + +/* + * Socket helpers. + */ +static inline int __vsock_dgram(void) +{ + int s; + + s = socket(AF_VSOCK, SOCK_DGRAM, 0); + if (s < 0) { + perror("socket"); + exit(errno); + } + + return s; +} + +static inline int __vsock_bdgram(struct sockaddr_vm *sa) +{ + int s; + + s = __vsock_dgram(); + if (bind(s, (struct sockaddr *)sa, sizeof (*sa))) { + perror("bind"); + exit(errno); + } + + return s; +} + +/* + * Sanity checks. + */ +static inline int is_valid_port(unsigned long port) +{ + return (port > 0) && (port < 65536); +} + +/* + * Parsing helpers. + */ +static inline int parse_uint(const char *nptr, unsigned int *ui) +{ + char *end; + unsigned long ul; + + assert(nptr != NULL); + assert(ui != NULL); + + ul = strtoul(nptr, &end, 0); + if (end == nptr) + return -EINVAL; + + if (ul >= UINT_MAX) + return -ERANGE; + + *ui = ul; + + return 0; +} + +static inline int parse_domid(const char *nptr, unsigned int *domid) +{ + int rc; + + assert(nptr != NULL); + assert(domid != NULL); + + rc = parse_uint(nptr, domid); + if (rc < 0) + return rc; + + if (*domid >= VMADDR_CID_ANY) + return -ERANGE; + + return 0; +} + +#endif /* !_UTILS_H_ */ + From 0b1f6e7d0820d9c39bb8ec45820ad462b47eca63 Mon Sep 17 00:00:00 2001 From: Eric Chanudet Date: Thu, 25 Apr 2019 11:05:40 -0400 Subject: [PATCH 3/3] README: Initial documentation. Walk through how to test VSock/Argo. Signed-off-by: Eric Chanudet --- vsock-argo/README.md | 134 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 vsock-argo/README.md diff --git a/vsock-argo/README.md b/vsock-argo/README.md new file mode 100644 index 0000000..faa3af6 --- /dev/null +++ b/vsock-argo/README.md @@ -0,0 +1,134 @@ +--- + +# Work In Progress + +This software is still under development and should be considered experimental. +It is currently developed for x86-64 and has not been built or tested for ARM +targets. + +--- + +# VSOCK module for Xen/Argo + +[vsock(7)](http://man7.org/linux/man-pages/man7/vsock.7.html) is an address +family to facilitate communication between virtual machines. +[Argo](https://xenbits.xenproject.org/docs/unstable/designs/argo.html) is an +interdomain communication mechanism for Xen. + +This module implements Argo primitives under the Vsock address family to allow +socket communication between Xen domains. + +## Getting started + +### Linux module + +Building `vsock_argo_transport` will require your Linux kernel headers. It can +be built as an out-of-tree module. A convenience Makefile is provided: +```bash + $ make -C module +``` + +The module itself will require VSock support in the kernel: +```bash + $ zgrep CONFIG_VSOCKETS /proc/config.gz +CONFIG_VSOCKETS=m +CONFIG_VSOCKETS_DIAG=m + $ sudo modprobe vsock +``` + +It is possible that another VSock transport driver was already loaded, you will +need to remove it, e.g: +```bash + $ lsmod | grep vsock +vmw_vsock_vmci_transport 32768 0 +vmw_vmci 77824 1 vmw_vsock_vmci_transport +vsock 40960 1 vmw_vsock_vmci_transport + $ sudo rmmod vmw_vsock_vmci_transport +``` +Then load the module and confirm the operation succeeded. +```bash + $ sudo insmod module/vsock_argo_transport.ko + $ dmesg -H | grep argo +[ +0.000017] vsock_argo_transport registered. +``` + +### Test client + +A test `netcat` like utility is provided to simplify send and receive operation +using VSock datagram over Argo, the tool is call `hatch` and roughly follows +the `netcat` usage. + +Ship cargo, packets, are usualy passed through a `hatch`, hence the +name... Since Argo is an old mythological ship. + +It builds using autotools: +```bash + $ cd test/hatch + $ autoreconf -i + $ mkdir dist ; ./configure --prefix=$(pwd)/dist + ... + $ make install + $ ./dist/bin/hatch -h +Basic usage: + hatch domid port + hatch -r -p local_port +Options: + -r, --recv receive mode, wait for incoming packets. + -p, --port set local port number. + -D enable debug output on stderr. +``` + +## Testing + +Argo was released with Xen 4.12.0, but is still experimental feature. You will +likely need to rebuild Xen to enable argo and change its cmdline. I recommend +following the way your distribution packages Xen (PKGBUILD, sbuild, etc) and +amend with the changes described below. The following is only provided as an +example: +```bash + $ git clone git://xenbits.xen.org/xen.git + $ cd xen + $ git checkout -b stable-4.12 origin/stable-4.12 + $ echo 'CONFIG_ARGO=y' >> xen/.config + $ PYTHON=/usr/bin/python2 ./configure --prefix=/usr --sbindir=/usr/bin --with-sysconfig-leaf-dir=conf.d --with-rundir=/run --enable-systemd--with-extra-qemuu-configure-args="--disable-bluez --disable-gtk --disable-vte --disable-werror --disable-virglrenderer --disable-libnfs --disable-glusterfs --disable-numa --disable-smartcard --disable-fdt --enable-spice --enable-usb-redir --with-sdlabi=1.2" + $ yes "" | make XEN_CONFIG_EXPERT=y -C xen oldconfig + $ make XEN_CONFIG_EXPERT=y LANG=C PYTHON=python2 dist +``` + +Install and configure the previously built Xen to suit your distribution. This +is beyond the scope of this document. + +Add `argo=true,mac-permissive=1` to your Xen cmdline and reboot on the freshly +built Xen with Argo support. + +You will need at least 1 Xen guest, other than dom0, to test +`vsock_argo_transport.ko`. No guest specific guest configuration is required, +e.g: +``` +type = "pvh" +kernel="/var/lib/machines/vm0/boot/vmlinuz-linux" +ramdisk="/var/lib/machines/vm0/boot/initramfs-linux.img" +root="/dev/xvda" +extra="earlyprintk=xen console=tty0 console=hvc0,115200n8 rw quiet" +name = "vm0.pvh" +memory = 512 +maxmem = 512 +vcpus = 2 +vfb = [ "type=vfb, vnc=1, vnclisten=0.0.0.0, vncdisplay=1" ] +vif = [ 'mac=00:16:3e:01:01:01, bridge=br0' ] +disk = [ '/dev/vg0/vm0,raw,xvda,rw' ] +``` + +Start and log into the guest to load `vsock_argo_transport.ko`. `hatch` can be +used as netcat using the domain-id as address and a port. +```bash +user@dom0 $ hatch -r -p 1234 > /tmp/file + ^C +user@dom0 $ md5sum file +b061b7954fbcb8b17cc7e2e8a5754269 file +``` +```bash +user@vm0 $ md5sum file +b061b7954fbcb8b17cc7e2e8a5754269 file +user@vm0 $ hatch 0 1234 < file +```