intc_intel_vtd.c

/*
 * Copyright (c) 2020 Intel Corporation
 * SPDX-License-Identifier: Apache-2.0
 */

#define DT_DRV_COMPAT intel_vt_d

#include <errno.h>

#include <zephyr/kernel.h>
#include <zephyr/arch/cpu.h>

#include <soc.h>
#include <zephyr/device.h>
#include <zephyr/init.h>
#include <string.h>


#include <zephyr/cache.h>

#include <zephyr/arch/x86/intel_vtd.h>
#include <zephyr/drivers/interrupt_controller/intel_vtd.h>
#include <zephyr/drivers/interrupt_controller/ioapic.h>
#include <zephyr/drivers/interrupt_controller/loapic.h>
#include <zephyr/drivers/pcie/msi.h>

#include <kernel_arch_func.h>

#include "intc_intel_vtd.h"

static inline void vtd_pause_cpu(void)
{
	__asm__ volatile("pause" ::: "memory");
}

static void vtd_write_reg32(const struct device *dev,
			    uint16_t reg, uint32_t value)
{
	uintptr_t base_address = DEVICE_MMIO_GET(dev);

	sys_write32(value, (base_address + reg));
}

static uint32_t vtd_read_reg32(const struct device *dev, uint16_t reg)
{
	uintptr_t base_address = DEVICE_MMIO_GET(dev);

	return sys_read32(base_address + reg);
}

static void vtd_write_reg64(const struct device *dev,
			    uint16_t reg, uint64_t value)
{
	uintptr_t base_address = DEVICE_MMIO_GET(dev);

	sys_write64(value, (base_address + reg));
}

static uint64_t vtd_read_reg64(const struct device *dev, uint16_t reg)
{
	uintptr_t base_address = DEVICE_MMIO_GET(dev);

	return sys_read64(base_address + reg);
}

static void vtd_send_cmd(const struct device *dev,
			 uint16_t cmd_bit, uint16_t status_bit)
{
	uintptr_t base_address = DEVICE_MMIO_GET(dev);
	uint32_t value;

	value = vtd_read_reg32(dev, VTD_GSTS_REG);
	value |= BIT(cmd_bit);

	vtd_write_reg32(dev, VTD_GCMD_REG, value);

	while (!sys_test_bit((base_address + VTD_GSTS_REG),
			     status_bit)) {
		/* Do nothing */
	}
}

static void vtd_flush_irte_from_cache(const struct device *dev,
				      uint8_t irte_idx)
{
	struct vtd_ictl_data *data = dev->data;

	if (!data->pwc) {
		cache_data_flush_range(&data->irte[irte_idx],
				       sizeof(union vtd_irte));
	}
}

static void vtd_qi_init(const struct device *dev)
{
	struct vtd_ictl_data *data = dev->data;
	uint64_t value;

	vtd_write_reg64(dev, VTD_IQT_REG, 0);
	data->qi_tail = 0;

	value = VTD_IQA_REG_GEN_CONTENT((uintptr_t)data->qi,
					VTD_IQA_WIDTH_128_BIT, QI_SIZE);
	vtd_write_reg64(dev, VTD_IQA_REG, value);

	vtd_send_cmd(dev, VTD_GCMD_QIE, VTD_GSTS_QIES);
}

static inline void vtd_qi_tail_inc(const struct device *dev)
{
	struct vtd_ictl_data *data = dev->data;

	data->qi_tail += sizeof(struct qi_descriptor);
	data->qi_tail %= (QI_NUM * sizeof(struct qi_descriptor));
}

static int vtd_qi_send(const struct device *dev,
		       struct qi_descriptor *descriptor)
{
	struct vtd_ictl_data *data = dev->data;
	union qi_wait_descriptor wait_desc = { 0 };
	struct qi_descriptor *desc;
	uint32_t wait_status;
	uint32_t wait_count;

	desc = (struct qi_descriptor *)((uintptr_t)data->qi + data->qi_tail);

	desc->low = descriptor->low;
	desc->high = descriptor->high;

	vtd_qi_tail_inc(dev);

	desc++;

	wait_status = QI_WAIT_STATUS_INCOMPLETE;

	wait_desc.wait.type = QI_TYPE_WAIT;
	wait_desc.wait.status_write = 1;
	wait_desc.wait.status_data = QI_WAIT_STATUS_COMPLETE;
	wait_desc.wait.address = ((uintptr_t)&wait_status) >> 2;

	desc->low = wait_desc.desc.low;
	desc->high = wait_desc.desc.high;

	vtd_qi_tail_inc(dev);

	vtd_write_reg64(dev, VTD_IQT_REG, data->qi_tail);

	wait_count = 0;

	while (wait_status != QI_WAIT_STATUS_COMPLETE) {
		/* We cannot use timeout here, this function being called
		 * at init time, it might result that the system clock
		 * is not initialized yet since VT-D init comes first.
		 */
		if (wait_count > QI_WAIT_COUNT_LIMIT) {
			printk("QI timeout\n");
			return -ETIME;
		}

		if (vtd_read_reg32(dev, VTD_FSTS_REG) & VTD_FSTS_IQE) {
			printk("QI error\n");
			return -EIO;
		}

		vtd_pause_cpu();
		wait_count++;
	}

	return 0;
}

static int vtd_global_cc_invalidate(const struct device *dev)
{
	union qi_icc_descriptor iec_desc = { 0 };

	iec_desc.icc.type = QI_TYPE_ICC;
	iec_desc.icc.granularity = 1; /* Global Invalidation requested */

	return vtd_qi_send(dev, &iec_desc.desc);
}

static int vtd_global_iec_invalidate(const struct device *dev)
{
	union qi_iec_descriptor iec_desc = { 0 };

	iec_desc.iec.type = QI_TYPE_IEC;
	iec_desc.iec.granularity = 0; /* Global Invalidation requested */

	return vtd_qi_send(dev, &iec_desc.desc);
}

static int vtd_index_iec_invalidate(const struct device *dev, uint8_t irte_idx)
{
	union qi_iec_descriptor iec_desc = { 0 };

	iec_desc.iec.type = QI_TYPE_IEC;
	iec_desc.iec.granularity = 1; /* Index based invalidation requested */

	iec_desc.iec.interrupt_index = irte_idx;
	iec_desc.iec.index_mask = 0;

	return vtd_qi_send(dev, &iec_desc.desc);
}

static void fault_status_description(uint32_t status)
{
	if (status & VTD_FSTS_PFO) {
		printk("Primary Fault Overflow (PFO)\n");
	}

	if (status & VTD_FSTS_AFO) {
		printk("Advanced Fault Overflow (AFO)\n");
	}

	if (status & VTD_FSTS_APF) {
		printk("Advanced Primary Fault (APF)\n");
	}

	if (status & VTD_FSTS_IQE) {
		printk("Invalidation Queue Error (IQE)\n");
	}

	if (status & VTD_FSTS_ICE) {
		printk("Invalidation Completion Error (ICE)\n");
	}

	if (status & VTD_FSTS_ITE) {
		printk("Invalidation Timeout Error\n");
	}

	if (status & VTD_FSTS_PPF) {
		printk("Primary Pending Fault (PPF) %u\n",
		       VTD_FSTS_FRI(status));
	}
}

static void fault_record_description(uint64_t low, uint64_t high)
{
	printk("Fault %s request: Reason 0x%x info 0x%llx src 0x%x\n",
	       (high & VTD_FRCD_T) ? "Read/Atomic" : "Write/Page",
	       VTD_FRCD_FR(high), VTD_FRCD_FI(low), VTD_FRCD_SID(high));
}

static void fault_event_isr(const void *arg)
{
	const struct device *dev = arg;
	struct vtd_ictl_data *data = dev->data;
	uint32_t status;
	uint8_t f_idx;

	status = vtd_read_reg32(dev, VTD_FSTS_REG);
	fault_status_description(status);

	if (!(status & VTD_FSTS_PPF)) {
		goto out;
	}

	f_idx = VTD_FSTS_FRI(status);
	while (f_idx < data->fault_record_num) {
		uint64_t fault_l, fault_h;

		/* Reading fault's 64 lowest bits */
		fault_l = vtd_read_reg64(dev, data->fault_record_reg +
					 (VTD_FRCD_REG_SIZE * f_idx));
		/* Reading fault's 64 highest bits */
		fault_h = vtd_read_reg64(dev, data->fault_record_reg +
				       (VTD_FRCD_REG_SIZE * f_idx) + 8);

		if (fault_h & VTD_FRCD_F) {
			fault_record_description(fault_l, fault_h);
		}

		/* Clearing the fault */
		vtd_write_reg64(dev, data->fault_record_reg +
				(VTD_FRCD_REG_SIZE * f_idx), fault_l);
		vtd_write_reg64(dev, data->fault_record_reg +
				(VTD_FRCD_REG_SIZE * f_idx) + 8, fault_h);
		f_idx++;
	}
out:
	/* Clearing fault status */
	vtd_write_reg32(dev, VTD_FSTS_REG, VTD_FSTS_CLEAR(status));
}

static void vtd_fault_event_init(const struct device *dev)
{
	struct vtd_ictl_data *data = dev->data;
	uint64_t value;
	uint32_t reg;

	value = vtd_read_reg64(dev, VTD_CAP_REG);
	data->fault_record_num = VTD_CAP_NFR(value) + 1;
	data->fault_record_reg = DEVICE_MMIO_GET(dev) +
		(uintptr_t)(16 * VTD_CAP_FRO(value));

	/* Allocating IRQ & vector and connecting the ISR handler,
	 * by-passing remapping by using x86 functions directly.
	 */
	data->fault_irq = arch_irq_allocate();
	data->fault_vector = z_x86_allocate_vector(0, -1);

	vtd_write_reg32(dev, VTD_FEDATA_REG, data->fault_vector);
	vtd_write_reg32(dev, VTD_FEADDR_REG,
			pcie_msi_map(data->fault_irq, NULL, 0));
	vtd_write_reg32(dev, VTD_FEUADDR_REG, 0);

	z_x86_irq_connect_on_vector(data->fault_irq, data->fault_vector,
				    fault_event_isr, dev);

	vtd_write_reg32(dev, VTD_FSTS_REG,
			VTD_FSTS_CLEAR(vtd_read_reg32(dev, VTD_FSTS_REG)));

	/* Unmasking interrupts */
	reg = vtd_read_reg32(dev, VTD_FECTL_REG);
	reg &= ~BIT(VTD_FECTL_REG_IM);
	vtd_write_reg32(dev, VTD_FECTL_REG, reg);
}

static int vtd_ictl_allocate_entries(const struct device *dev,
				     uint8_t n_entries)
{
	struct vtd_ictl_data *data = dev->data;
	int irte_idx_start;

	if ((data->irte_num_used + n_entries) > IRTE_NUM) {
		return -EBUSY;
	}

	irte_idx_start = data->irte_num_used;
	data->irte_num_used += n_entries;

	return irte_idx_start;
}

static uint32_t vtd_ictl_remap_msi(const struct device *dev,
				   msi_vector_t *vector,
				   uint8_t n_vector)
{
	uint32_t shv = (n_vector > 1) ? VTD_INT_SHV : 0;

	return VTD_MSI_MAP(vector->arch.irte, shv);
}

static int vtd_ictl_remap(const struct device *dev,
			  uint8_t irte_idx,
			  uint16_t vector,
			  uint32_t flags,
			  uint16_t src_id)
{
	struct vtd_ictl_data *data = dev->data;
	union vtd_irte irte = { 0 };
	uint32_t delivery_mode;

	irte.bits.vector = vector;

	if (IS_ENABLED(CONFIG_X2APIC)) {
		/* Getting the logical APIC ID */
		irte.bits.dst_id = x86_read_loapic(LOAPIC_LDR);
	} else {
		/* As for IOAPIC: let's mask all possible IDs */
		irte.bits.dst_id = 0xFF << 8;
	}

	if (src_id != USHRT_MAX &&
	    !IS_ENABLED(CONFIG_INTEL_VTD_ICTL_NO_SRC_ID_CHECK)) {
		irte.bits.src_validation_type = 1;
		irte.bits.src_id = src_id;
	}

	delivery_mode = (flags & IOAPIC_DELIVERY_MODE_MASK);
	if ((delivery_mode != IOAPIC_FIXED) ||
	    (delivery_mode != IOAPIC_LOW)) {
		delivery_mode = IOAPIC_LOW;
	}

	irte.bits.trigger_mode = (flags & IOAPIC_TRIGGER_MASK) >> 15;
	irte.bits.delivery_mode = delivery_mode >> 8;
	irte.bits.redirection_hint = 1;
	irte.bits.dst_mode = 1; /* Always logical */
	irte.bits.present = 1;

	data->irte[irte_idx].parts.low = irte.parts.low;
	data->irte[irte_idx].parts.high = irte.parts.high;

	vtd_index_iec_invalidate(dev, irte_idx);

	vtd_flush_irte_from_cache(dev, irte_idx);

	return 0;
}

static int vtd_ictl_set_irte_vector(const struct device *dev,
				    uint8_t irte_idx,
				    uint16_t vector)
{
	struct vtd_ictl_data *data = dev->data;

	data->vectors[irte_idx] = vector;

	return 0;
}

static int vtd_ictl_get_irte_by_vector(const struct device *dev,
				       uint16_t vector)
{
	struct vtd_ictl_data *data = dev->data;
	int irte_idx;

	for (irte_idx = 0; irte_idx < IRTE_NUM; irte_idx++) {
		if (data->vectors[irte_idx] == vector) {
			return irte_idx;
		}
	}

	return -EINVAL;
}

static uint16_t vtd_ictl_get_irte_vector(const struct device *dev,
					 uint8_t irte_idx)
{
	struct vtd_ictl_data *data = dev->data;

	return data->vectors[irte_idx];
}

static int vtd_ictl_set_irte_irq(const struct device *dev,
				 uint8_t irte_idx,
				 unsigned int irq)
{
	struct vtd_ictl_data *data = dev->data;

	data->irqs[irte_idx] = irq;

	return 0;
}

static int vtd_ictl_get_irte_by_irq(const struct device *dev,
				    unsigned int irq)
{
	struct vtd_ictl_data *data = dev->data;
	int irte_idx;

	for (irte_idx = 0; irte_idx < IRTE_NUM; irte_idx++) {
		if (data->irqs[irte_idx] == irq) {
			return irte_idx;
		}
	}

	return -EINVAL;
}

static void vtd_ictl_set_irte_msi(const struct device *dev,
				  uint8_t irte_idx, bool msi)
{
	struct vtd_ictl_data *data = dev->data;

	data->msi[irte_idx] = msi;
}

static bool vtd_ictl_irte_is_msi(const struct device *dev,
				 uint8_t irte_idx)
{
	struct vtd_ictl_data *data = dev->data;

	return data->msi[irte_idx];
}

static int vtd_ictl_init(const struct device *dev)
{
	struct vtd_ictl_data *data = dev->data;
	unsigned int key = irq_lock();
	uint64_t eime = 0;
	uint64_t value;
	int ret = 0;

	DEVICE_MMIO_MAP(dev, K_MEM_CACHE_NONE);

	if (vtd_read_reg64(dev, VTD_ECAP_REG) & VTD_ECAP_C) {
		printk("Page walk coherency supported\n");
		data->pwc = true;
	}

	vtd_fault_event_init(dev);

	vtd_qi_init(dev);

	if (vtd_global_cc_invalidate(dev) != 0) {
		printk("Could not perform ICC invalidation\n");
		ret = -EIO;
		goto out;
	}

	if (IS_ENABLED(CONFIG_X2APIC)) {
		eime = VTD_IRTA_EIME;
	}

	value = VTD_IRTA_REG_GEN_CONTENT((uintptr_t)data->irte,
					 IRTA_SIZE, eime);

	vtd_write_reg64(dev, VTD_IRTA_REG, value);

	if (vtd_global_iec_invalidate(dev) != 0) {
		printk("Could not perform IEC invalidation\n");
		ret = -EIO;
		goto out;
	}

	if (!IS_ENABLED(CONFIG_X2APIC) &&
	    IS_ENABLED(CONFIG_INTEL_VTD_ICTL_XAPIC_PASSTHROUGH)) {
		vtd_send_cmd(dev, VTD_GCMD_CFI, VTD_GSTS_CFIS);
	}

	vtd_send_cmd(dev, VTD_GCMD_SIRTP, VTD_GSTS_SIRTPS);
	vtd_send_cmd(dev, VTD_GCMD_IRE, VTD_GSTS_IRES);

	printk("Intel VT-D up and running (status 0x%x)\n",
	       vtd_read_reg32(dev, VTD_GSTS_REG));

out:
	irq_unlock(key);

	return ret;
}

static const struct vtd_driver_api vtd_api = {
	.allocate_entries = vtd_ictl_allocate_entries,
	.remap_msi = vtd_ictl_remap_msi,
	.remap = vtd_ictl_remap,
	.set_irte_vector = vtd_ictl_set_irte_vector,
	.get_irte_by_vector = vtd_ictl_get_irte_by_vector,
	.get_irte_vector = vtd_ictl_get_irte_vector,
	.set_irte_irq = vtd_ictl_set_irte_irq,
	.get_irte_by_irq = vtd_ictl_get_irte_by_irq,
	.set_irte_msi = vtd_ictl_set_irte_msi,
	.irte_is_msi = vtd_ictl_irte_is_msi
};

static struct vtd_ictl_data vtd_ictl_data_0 = {
	.irqs = { -EINVAL },
	.vectors = { -EINVAL },
};

static const struct vtd_ictl_cfg vtd_ictl_cfg_0 = {
	DEVICE_MMIO_ROM_INIT(DT_DRV_INST(0)),
};

DEVICE_DT_INST_DEFINE(0,
	      vtd_ictl_init, NULL,
	      &vtd_ictl_data_0, &vtd_ictl_cfg_0,
	      PRE_KERNEL_1, CONFIG_INTEL_VTD_ICTL_INIT_PRIORITY, &vtd_api);