From a47389c635100e85f13786ddcfed5249f418c405 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Mon, 3 Apr 2023 16:19:19 +0200 Subject: [PATCH 001/635] dt-bindings: pwm: Add Apple PWM controller Apple SoCs such as the M1 contain a PWM controller used among other things to control the keyboard backlight. Signed-off-by: Sasha Finkelstein Reviewed-by: Krzysztof Kozlowski Acked-by: Sven Peter --- .../bindings/pwm/apple,s5l-fpwm.yaml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml new file mode 100644 index 00000000000000..142157bff0cd85 --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/apple,s5l-fpwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple FPWM controller + +maintainers: + - asahi@lists.linux.dev + - Sasha Finkelstein + +description: PWM controller used for keyboard backlight on ARM Macs + +properties: + compatible: + items: + - enum: + - apple,t8103-fpwm + - apple,t6000-fpwm + - apple,t8112-fpwm + - const: apple,s5l-fpwm + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + power-domains: + maxItems: 1 + + "#pwm-cells": + const: 2 + +required: + - compatible + - reg + - clocks + +additionalProperties: false + +examples: + - | + pwm@235044000 { + compatible = "apple,t8103-fpwm", "apple,s5l-fpwm"; + reg = <0x35044000 0x4000>; + power-domains = <&ps_fpwm1>; + clocks = <&clkref>; + #pwm-cells = <2>; + }; From 48a2d57ee4261d5c32a025b86ad908e3853c9d70 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 12 Dec 2021 12:28:41 +0900 Subject: [PATCH 002/635] MAINTAINERS: Add Apple dwc3 bindings to ARM/APPLE This Apple dwc3 controller variance is present on Apple ARM SoCs (t8103/t600x). Splitting this change from the binding/driver commits to avoid merge conflicts with other things touching this section, as usual. Signed-off-by: Hector Martin --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index c6545eb541047e..52df7ad2c67857 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1955,6 +1955,7 @@ F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml F: Documentation/devicetree/bindings/power/apple* +F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml F: Documentation/devicetree/bindings/watchdog/apple,wdt.yaml F: arch/arm64/boot/dts/apple/ F: drivers/bluetooth/hci_bcm4377.c From 5518860299d5f9fe0ca064d5e3963dfaf0cc8f90 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 12 Dec 2021 12:28:41 +0900 Subject: [PATCH 003/635] MAINTAINERS: Add apple-spi driver & binding files This Apple SPI controller is present on Apple ARM SoCs (t8103/t6000). Splitting this change from the binding/driver commits to avoid merge conflicts with other things touching this section, as usual. Signed-off-by: Hector Martin --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 52df7ad2c67857..31e0cd24d7676d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1955,6 +1955,7 @@ F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml F: Documentation/devicetree/bindings/power/apple* +F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml F: Documentation/devicetree/bindings/watchdog/apple,wdt.yaml F: arch/arm64/boot/dts/apple/ @@ -1972,6 +1973,7 @@ F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/soc/apple/* +F: drivers/spi/spi-apple.c F: drivers/watchdog/apple_wdt.c F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h From 2a90cbe6c7f8b1c690d811a1ddb8a9076ff506e6 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 29 Oct 2022 00:16:54 +0300 Subject: [PATCH 004/635] MAINTAINERS: Add entries for Apple PWM driver Add the MAINTAINERS entries for the driver Signed-off-by: Sasha Finkelstein --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 31e0cd24d7676d..bca266a6ca8fd3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1955,6 +1955,7 @@ F: Documentation/devicetree/bindings/nvmem/apple,efuses.yaml F: Documentation/devicetree/bindings/pci/apple,pcie.yaml F: Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml F: Documentation/devicetree/bindings/power/apple* +F: Documentation/devicetree/bindings/pwm/pwm-apple.yaml F: Documentation/devicetree/bindings/spi/apple,spi.yaml F: Documentation/devicetree/bindings/usb/apple,dwc3.yaml F: Documentation/devicetree/bindings/watchdog/apple,wdt.yaml @@ -1972,6 +1973,7 @@ F: drivers/mailbox/apple-mailbox.c F: drivers/nvme/host/apple.c F: drivers/nvmem/apple-efuses.c F: drivers/pinctrl/pinctrl-apple-gpio.c +F: drivers/pwm/pwm-apple.c F: drivers/soc/apple/* F: drivers/spi/spi-apple.c F: drivers/watchdog/apple_wdt.c From 6cb6ce47b2ee3983980593fde5147a85be59e093 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 15 Oct 2022 21:24:57 +0900 Subject: [PATCH 005/635] soc: apple: rtkit: Check for failure to send mgmt messages & log Signed-off-by: Asahi Lina --- drivers/soc/apple/rtkit.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 35ec35aa500d67..ed0ba2b4bb5431 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -102,12 +102,20 @@ bool apple_rtkit_is_crashed(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_is_crashed); -static void apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, +static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 msg) { + int ret; + msg &= ~APPLE_RTKIT_MGMT_TYPE; msg |= FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, type); - apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false); + ret = apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_MGMT, msg, NULL, false); + + if (ret) { + dev_err(rtk->dev, "RTKit: Failed to send management message: %d\n", ret); + } + + return ret; } static void apple_rtkit_management_rx_hello(struct apple_rtkit *rtk, u64 msg) @@ -807,8 +815,10 @@ static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk, reinit_completion(&rtk->ap_pwr_ack_completion); msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state); - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, - msg); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_AP_PWR_STATE, + msg); + if (ret) + return ret; ret = apple_rtkit_wait_for_completion(&rtk->ap_pwr_ack_completion); if (ret) @@ -828,8 +838,10 @@ static int apple_rtkit_set_iop_power_state(struct apple_rtkit *rtk, reinit_completion(&rtk->iop_pwr_ack_completion); msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, state); - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, - msg); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + msg); + if (ret) + return ret; ret = apple_rtkit_wait_for_completion(&rtk->iop_pwr_ack_completion); if (ret) @@ -930,6 +942,7 @@ EXPORT_SYMBOL_GPL(apple_rtkit_quiesce); int apple_rtkit_wake(struct apple_rtkit *rtk) { u64 msg; + int ret; if (apple_rtkit_is_running(rtk)) return -EINVAL; @@ -941,8 +954,10 @@ int apple_rtkit_wake(struct apple_rtkit *rtk) * will wait for the completion anyway. */ msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON); - apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, - msg); + ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, + msg); + if (ret) + return ret; return apple_rtkit_boot(rtk); } From db16bfc4a5ec68be1910d7916425bec57f184541 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Tue, 27 Sep 2022 05:10:17 +0900 Subject: [PATCH 006/635] soc: apple: rtkit: Log failure to send messages Signed-off-by: Asahi Lina --- drivers/soc/apple/rtkit.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index ed0ba2b4bb5431..60d992515d3c37 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -601,11 +601,18 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, int ret; gfp_t flags; - if (rtk->crashed) + if (rtk->crashed) { + dev_warn(rtk->dev, + "RTKit: Device is crashed, cannot send message\n"); return -EINVAL; + } + if (ep >= APPLE_RTKIT_APP_ENDPOINT_START && - !apple_rtkit_is_running(rtk)) + !apple_rtkit_is_running(rtk)) { + dev_warn(rtk->dev, + "RTKit: Endpoint 0x%02x is not running, cannot send message\n", ep); return -EINVAL; + } if (atomic) flags = GFP_ATOMIC; From bfeb8da7df9804fcc509df063dcc6da47f13af0c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 18 Aug 2022 02:06:54 +0900 Subject: [PATCH 007/635] soc: apple: rtkit: Fix buffer address field width Signed-off-by: Asahi Lina --- drivers/soc/apple/rtkit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 60d992515d3c37..97bd93b7729265 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -55,7 +55,7 @@ enum { #define APPLE_RTKIT_BUFFER_REQUEST 1 #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK_ULL(51, 44) -#define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(41, 0) +#define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK_ULL(43, 0) #define APPLE_RTKIT_SYSLOG_TYPE GENMASK_ULL(59, 52) From 339d10d1bd1ffbd35122e0c4d0547854860f5d11 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 18 Aug 2022 02:13:03 +0900 Subject: [PATCH 008/635] soc: apple: rtkit: Log failed buffer requests Signed-off-by: Asahi Lina --- drivers/soc/apple/rtkit.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 97bd93b7729265..6653a2e211b212 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -308,6 +308,9 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, return 0; error: + dev_err(rtk->dev, "RTKit: failed buffer request for 0x%zx bytes (%d)\n", + buffer->size, err); + buffer->buffer = NULL; buffer->iomem = NULL; buffer->iova = 0; From 749d90f337d468eb34c4b0a0cbb76fd7a1c71dc4 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 25 Jul 2022 21:38:56 +0200 Subject: [PATCH 009/635] soc: apple: rtkit: Add APPLE_RTKIT_PWR_STATE_INIT This state is needed to wake the dcp IOP after m1n1 shut it down. Signed-off-by: Janne Grunau --- drivers/soc/apple/rtkit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 6653a2e211b212..4587a8fea3cc96 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -12,6 +12,7 @@ enum { APPLE_RTKIT_PWR_STATE_IDLE = 0x201, /* sleeping, retain state */ APPLE_RTKIT_PWR_STATE_QUIESCED = 0x10, /* running but no communication */ APPLE_RTKIT_PWR_STATE_ON = 0x20, /* normal operating state */ + APPLE_RTKIT_PWR_STATE_INIT = 0x220, /* init after starting the coproc */ }; enum { @@ -963,7 +964,7 @@ int apple_rtkit_wake(struct apple_rtkit *rtk) * Use open-coded apple_rtkit_set_iop_power_state since apple_rtkit_boot * will wait for the completion anyway. */ - msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON); + msg = FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_INIT); ret = apple_rtkit_management_send(rtk, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE, msg); if (ret) From 3248c87361763716e69c5cd2383eb6bf8c97a6c0 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 14 Mar 2023 19:45:45 +0900 Subject: [PATCH 010/635] soc: apple: mailbox: Add ASC/M3 mailbox driver Signed-off-by: Hector Martin --- drivers/soc/apple/Kconfig | 14 ++ drivers/soc/apple/Makefile | 3 + drivers/soc/apple/mailbox.c | 433 ++++++++++++++++++++++++++++++++++++ drivers/soc/apple/mailbox.h | 48 ++++ 4 files changed, 498 insertions(+) create mode 100644 drivers/soc/apple/mailbox.c create mode 100644 drivers/soc/apple/mailbox.h diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index a1596fefacffe1..f785fb37096ad0 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -17,6 +17,20 @@ config APPLE_PMGR_PWRSTATE controls for SoC devices. This driver manages them through the generic power domain framework, and also provides reset support. +config APPLE_MBOX + tristate "Apple SoC mailboxes" + depends on PM + depends on ARCH_APPLE || (64BIT && COMPILE_TEST) + depends on !APPLE_MAILBOX + default ARCH_APPLE + help + Apple SoCs have various co-processors required for certain + peripherals to work (NVMe, display controller, etc.). This + driver adds support for the mailbox controller used to + communicate with those. + + Say Y here if you have a Apple SoC. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on MAILBOX diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index e293770cf66d73..e52edf6a73dafd 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,6 +1,9 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += apple-pmgr-pwrstate.o +obj-$(CONFIG_APPLE_MBOX) += apple-mailbox.o +apple-mailbox-y = mailbox.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c new file mode 100644 index 00000000000000..eeeeccd6cfd30b --- /dev/null +++ b/drivers/soc/apple/mailbox.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple mailbox driver + * + * Copyright The Asahi Linux Contributors + * + * This driver adds support for two mailbox variants (called ASC and M3 by + * Apple) found in Apple SoCs such as the M1. It consists of two FIFOs used to + * exchange 64+32 bit messages between the main CPU and a co-processor. + * Various coprocessors implement different IPC protocols based on these simple + * messages and shared memory buffers. + * + * Both the main CPU and the co-processor see the same set of registers but + * the first FIFO (A2I) is always used to transfer messages from the application + * processor (us) to the I/O processor and the second one (I2A) for the + * other direction. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mailbox.h" + +#define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) +#define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) + +#define APPLE_ASC_MBOX_A2I_CONTROL 0x110 +#define APPLE_ASC_MBOX_A2I_SEND0 0x800 +#define APPLE_ASC_MBOX_A2I_SEND1 0x808 +#define APPLE_ASC_MBOX_A2I_RECV0 0x810 +#define APPLE_ASC_MBOX_A2I_RECV1 0x818 + +#define APPLE_ASC_MBOX_I2A_CONTROL 0x114 +#define APPLE_ASC_MBOX_I2A_SEND0 0x820 +#define APPLE_ASC_MBOX_I2A_SEND1 0x828 +#define APPLE_ASC_MBOX_I2A_RECV0 0x830 +#define APPLE_ASC_MBOX_I2A_RECV1 0x838 + +#define APPLE_M3_MBOX_CONTROL_FULL BIT(16) +#define APPLE_M3_MBOX_CONTROL_EMPTY BIT(17) + +#define APPLE_M3_MBOX_A2I_CONTROL 0x50 +#define APPLE_M3_MBOX_A2I_SEND0 0x60 +#define APPLE_M3_MBOX_A2I_SEND1 0x68 +#define APPLE_M3_MBOX_A2I_RECV0 0x70 +#define APPLE_M3_MBOX_A2I_RECV1 0x78 + +#define APPLE_M3_MBOX_I2A_CONTROL 0x80 +#define APPLE_M3_MBOX_I2A_SEND0 0x90 +#define APPLE_M3_MBOX_I2A_SEND1 0x98 +#define APPLE_M3_MBOX_I2A_RECV0 0xa0 +#define APPLE_M3_MBOX_I2A_RECV1 0xa8 + +#define APPLE_M3_MBOX_IRQ_ENABLE 0x48 +#define APPLE_M3_MBOX_IRQ_ACK 0x4c +#define APPLE_M3_MBOX_IRQ_A2I_EMPTY BIT(0) +#define APPLE_M3_MBOX_IRQ_A2I_NOT_EMPTY BIT(1) +#define APPLE_M3_MBOX_IRQ_I2A_EMPTY BIT(2) +#define APPLE_M3_MBOX_IRQ_I2A_NOT_EMPTY BIT(3) + +#define APPLE_MBOX_MSG1_OUTCNT GENMASK(56, 52) +#define APPLE_MBOX_MSG1_INCNT GENMASK(51, 48) +#define APPLE_MBOX_MSG1_OUTPTR GENMASK(47, 44) +#define APPLE_MBOX_MSG1_INPTR GENMASK(43, 40) +#define APPLE_MBOX_MSG1_MSG GENMASK(31, 0) + +#define APPLE_MBOX_TX_TIMEOUT 500 + +struct apple_mbox_hw { + unsigned int control_full; + unsigned int control_empty; + + unsigned int a2i_control; + unsigned int a2i_send0; + unsigned int a2i_send1; + + unsigned int i2a_control; + unsigned int i2a_recv0; + unsigned int i2a_recv1; + + bool has_irq_controls; + unsigned int irq_enable; + unsigned int irq_ack; + unsigned int irq_bit_recv_not_empty; + unsigned int irq_bit_send_empty; +}; + +int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, + bool atomic) +{ + unsigned long flags; + int ret; + u32 mbox_ctrl; + long t; + + spin_lock_irqsave(&mbox->tx_lock, flags); + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); + + while (mbox_ctrl & mbox->hw->control_full) { + if (atomic) { + ret = readl_poll_timeout_atomic( + mbox->regs + mbox->hw->a2i_control, mbox_ctrl, + !(mbox_ctrl & mbox->hw->control_full), 100, + APPLE_MBOX_TX_TIMEOUT * 1000); + + if (ret) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return ret; + } + + break; + } + /* + * The interrupt is level triggered and will keep firing as long as the + * FIFO is empty. It will also keep firing if the FIFO was empty + * at any point in the past until it has been acknowledged at the + * mailbox level. By acknowledging it here we can ensure that we will + * only get the interrupt once the FIFO has been cleared again. + * If the FIFO is already empty before the ack it will fire again + * immediately after the ack. + */ + if (mbox->hw->has_irq_controls) { + writel_relaxed(mbox->hw->irq_bit_send_empty, + mbox->regs + mbox->hw->irq_ack); + } + enable_irq(mbox->irq_send_empty); + reinit_completion(&mbox->tx_empty); + spin_unlock_irqrestore(&mbox->tx_lock, flags); + + t = wait_for_completion_interruptible_timeout( + &mbox->tx_empty, + msecs_to_jiffies(APPLE_MBOX_TX_TIMEOUT)); + if (t < 0) + return t; + else if (t == 0) + return -ETIMEDOUT; + + spin_lock_irqsave(&mbox->tx_lock, flags); + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); + } + + writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); + writeq_relaxed(FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1), + mbox->regs + mbox->hw->a2i_send1); + + spin_unlock_irqrestore(&mbox->tx_lock, flags); + + return 0; +} +EXPORT_SYMBOL(apple_mbox_send); + +static irqreturn_t apple_mbox_send_empty_irq(int irq, void *data) +{ + struct apple_mbox *mbox = data; + + /* + * We don't need to acknowledge the interrupt at the mailbox level + * here even if supported by the hardware. It will keep firing but that + * doesn't matter since it's disabled at the main interrupt controller. + * apple_mbox_send will acknowledge it before enabling + * it at the main controller again. + */ + spin_lock(&mbox->tx_lock); + disable_irq_nosync(mbox->irq_send_empty); + complete(&mbox->tx_empty); + spin_unlock(&mbox->tx_lock); + + return IRQ_HANDLED; +} + +static int apple_mbox_poll_locked(struct apple_mbox *mbox) +{ + struct apple_mbox_msg msg; + int ret = 0; + + u32 mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + + while (!(mbox_ctrl & mbox->hw->control_empty)) { + msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); + msg.msg1 = FIELD_GET( + APPLE_MBOX_MSG1_MSG, + readq_relaxed(mbox->regs + mbox->hw->i2a_recv1)); + + mbox->rx(mbox, msg, mbox->cookie); + ret++; + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); + } + + /* + * The interrupt will keep firing even if there are no more messages + * unless we also acknowledge it at the mailbox level here. + * There's no race if a message comes in between the check in the while + * loop above and the ack below: If a new messages arrives inbetween + * those two the interrupt will just fire again immediately after the + * ack since it's level triggered. + */ + if (mbox->hw->has_irq_controls) { + writel_relaxed(mbox->hw->irq_bit_recv_not_empty, + mbox->regs + mbox->hw->irq_ack); + } + + return ret; +} + +static irqreturn_t apple_mbox_recv_irq(int irq, void *data) +{ + struct apple_mbox *mbox = data; + + spin_lock(&mbox->rx_lock); + apple_mbox_poll_locked(mbox); + spin_unlock(&mbox->rx_lock); + + return IRQ_HANDLED; +} + +int apple_mbox_poll(struct apple_mbox *mbox) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&mbox->rx_lock, flags); + ret = apple_mbox_poll_locked(mbox); + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + return ret; +} +EXPORT_SYMBOL(apple_mbox_poll); + +int apple_mbox_start(struct apple_mbox *mbox) +{ + int ret; + + if (mbox->active) + return 0; + + ret = pm_runtime_resume_and_get(mbox->dev); + if (ret) + return ret; + + /* + * Only some variants of this mailbox HW provide interrupt control + * at the mailbox level. We therefore need to handle enabling/disabling + * interrupts at the main interrupt controller anyway for hardware that + * doesn't. Just always keep the interrupts we care about enabled at + * the mailbox level so that both hardware revisions behave almost + * the same. + */ + if (mbox->hw->has_irq_controls) { + writel_relaxed(mbox->hw->irq_bit_recv_not_empty | + mbox->hw->irq_bit_send_empty, + mbox->regs + mbox->hw->irq_enable); + } + + enable_irq(mbox->irq_recv_not_empty); + mbox->active = true; + return 0; +} +EXPORT_SYMBOL(apple_mbox_start); + +void apple_mbox_stop(struct apple_mbox *mbox) +{ + if (!mbox->active) + return; + + mbox->active = false; + disable_irq(mbox->irq_recv_not_empty); + pm_runtime_mark_last_busy(mbox->dev); + pm_runtime_put_autosuspend(mbox->dev); +} +EXPORT_SYMBOL(apple_mbox_stop); + +struct apple_mbox *apple_mbox_get(struct device *dev, int index) +{ + struct of_phandle_args args; + struct platform_device *pdev; + struct apple_mbox *mbox; + int ret; + + ret = of_parse_phandle_with_args(dev->of_node, "mboxes", "#mbox-cells", + index, &args); + if (ret || !args.np) + return ERR_PTR(ret); + + pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + + if (!pdev) + return ERR_PTR(EPROBE_DEFER); + + mbox = platform_get_drvdata(pdev); + if (!mbox) + return ERR_PTR(EPROBE_DEFER); + + if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) + return ERR_PTR(ENODEV); + + return mbox; +} +EXPORT_SYMBOL(apple_mbox_get); + +struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name) +{ + int index; + index = of_property_match_string(dev->of_node, "mbox-names", name); + if (index < 0) + return ERR_PTR(index); + + return apple_mbox_get(dev, index); +} +EXPORT_SYMBOL(apple_mbox_get_byname); + +static int apple_mbox_probe(struct platform_device *pdev) +{ + int ret; + char *irqname; + struct apple_mbox *mbox; + struct device *dev = &pdev->dev; + + mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); + if (!mbox) + return -ENOMEM; + + mbox->dev = &pdev->dev; + mbox->hw = of_device_get_match_data(dev); + if (!mbox->hw) + return -EINVAL; + + mbox->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + + mbox->irq_recv_not_empty = + platform_get_irq_byname(pdev, "recv-not-empty"); + if (mbox->irq_recv_not_empty < 0) + return -ENODEV; + + mbox->irq_send_empty = platform_get_irq_byname(pdev, "send-empty"); + if (mbox->irq_send_empty < 0) + return -ENODEV; + + spin_lock_init(&mbox->rx_lock); + spin_lock_init(&mbox->tx_lock); + init_completion(&mbox->tx_empty); + + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); + if (!irqname) + return -ENOMEM; + + ret = devm_request_irq(dev, mbox->irq_recv_not_empty, + apple_mbox_recv_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (ret) + return ret; + + irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", dev_name(dev)); + if (!irqname) + return -ENOMEM; + + ret = devm_request_irq(dev, mbox->irq_send_empty, + apple_mbox_send_empty_irq, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND, irqname, mbox); + if (ret) + return ret; + + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + platform_set_drvdata(pdev, mbox); + return 0; +} + +static const struct apple_mbox_hw apple_mbox_asc_hw = { + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + +static const struct apple_mbox_hw apple_mbox_m3_hw = { + .control_full = APPLE_M3_MBOX_CONTROL_FULL, + .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_M3_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_M3_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_M3_MBOX_A2I_SEND1, + + .i2a_control = APPLE_M3_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_M3_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_M3_MBOX_I2A_RECV1, + + .has_irq_controls = true, + .irq_enable = APPLE_M3_MBOX_IRQ_ENABLE, + .irq_ack = APPLE_M3_MBOX_IRQ_ACK, + .irq_bit_recv_not_empty = APPLE_M3_MBOX_IRQ_I2A_NOT_EMPTY, + .irq_bit_send_empty = APPLE_M3_MBOX_IRQ_A2I_EMPTY, +}; + +static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, + { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_mbox_of_match); + +static struct platform_driver apple_mbox_driver = { + .driver = { + .name = "apple-mailbox", + .of_match_table = apple_mbox_of_match, + }, + .probe = apple_mbox_probe, +}; +module_platform_driver(apple_mbox_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_AUTHOR("Sven Peter "); +MODULE_DESCRIPTION("Apple Mailbox driver"); diff --git a/drivers/soc/apple/mailbox.h b/drivers/soc/apple/mailbox.h new file mode 100644 index 00000000000000..f73a8913da9510 --- /dev/null +++ b/drivers/soc/apple/mailbox.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple mailbox message format + * + * Copyright The Asahi Linux Contributors + */ + +#ifndef _APPLE_MAILBOX_H_ +#define _APPLE_MAILBOX_H_ + +#include +#include + +/* encodes a single 96bit message sent over the single channel */ +struct apple_mbox_msg { + u64 msg0; + u32 msg1; +}; + +struct apple_mbox { + struct device *dev; + void __iomem *regs; + const struct apple_mbox_hw *hw; + bool active; + + int irq_recv_not_empty; + int irq_send_empty; + + spinlock_t rx_lock; + spinlock_t tx_lock; + + struct completion tx_empty; + + /** Receive callback for incoming messages */ + void (*rx)(struct apple_mbox *mbox, struct apple_mbox_msg msg, void *cookie); + void *cookie; +}; + +struct apple_mbox *apple_mbox_get(struct device *dev, int index); +struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); + +int apple_mbox_start(struct apple_mbox *mbox); +void apple_mbox_stop(struct apple_mbox *mbox); +int apple_mbox_poll(struct apple_mbox *mbox); +int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, + bool atomic); + +#endif From 2cbd92404e35b6cc8c7019cc4a44ae62072d2f5d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 14 Mar 2023 23:53:07 +0900 Subject: [PATCH 011/635] soc: apple: rtkit: Port to the internal mailbox driver Signed-off-by: Hector Martin --- drivers/soc/apple/Kconfig | 2 +- drivers/soc/apple/rtkit-internal.h | 8 +-- drivers/soc/apple/rtkit.c | 101 ++++++++--------------------- 3 files changed, 31 insertions(+), 80 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index f785fb37096ad0..b3879a89b5ed92 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -33,7 +33,7 @@ config APPLE_MBOX config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" - depends on MAILBOX + depends on APPLE_MBOX depends on ARCH_APPLE || COMPILE_TEST default ARCH_APPLE help diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 24bd619ec5e487..27c9fa745fd528 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -7,18 +7,17 @@ #ifndef _APPLE_RTKIT_INTERAL_H #define _APPLE_RTKIT_INTERAL_H -#include #include #include #include #include #include #include -#include #include #include #include #include +#include "mailbox.h" #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 @@ -28,10 +27,7 @@ struct apple_rtkit { const struct apple_rtkit_ops *ops; struct device *dev; - const char *mbox_name; - int mbox_idx; - struct mbox_client mbox_cl; - struct mbox_chan *mbox_chan; + struct apple_mbox *mbox; struct completion epmap_completion; struct completion iop_pwr_ack_completion; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 4587a8fea3cc96..64bd696ae00f71 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -73,11 +73,6 @@ enum { #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 -struct apple_rtkit_msg { - struct completion *completion; - struct apple_mbox_msg mbox_msg; -}; - struct apple_rtkit_rx_work { struct apple_rtkit *rtk; u8 ep; @@ -550,12 +545,12 @@ static void apple_rtkit_rx_work(struct work_struct *work) kfree(rtk_work); } -static void apple_rtkit_rx(struct mbox_client *cl, void *mssg) +static void apple_rtkit_rx(struct apple_mbox *mbox, struct apple_mbox_msg msg, + void *cookie) { - struct apple_rtkit *rtk = container_of(cl, struct apple_rtkit, mbox_cl); - struct apple_mbox_msg *msg = mssg; + struct apple_rtkit *rtk = cookie; struct apple_rtkit_rx_work *work; - u8 ep = msg->msg1; + u8 ep = msg.msg1; /* * The message was read from a MMIO FIFO and we have to make @@ -571,7 +566,7 @@ static void apple_rtkit_rx(struct mbox_client *cl, void *mssg) if (ep >= APPLE_RTKIT_APP_ENDPOINT_START && rtk->ops->recv_message_early && - rtk->ops->recv_message_early(rtk->cookie, ep, msg->msg0)) + rtk->ops->recv_message_early(rtk->cookie, ep, msg.msg0)) return; work = kzalloc(sizeof(*work), GFP_ATOMIC); @@ -580,30 +575,18 @@ static void apple_rtkit_rx(struct mbox_client *cl, void *mssg) work->rtk = rtk; work->ep = ep; - work->msg = msg->msg0; + work->msg = msg.msg0; INIT_WORK(&work->work, apple_rtkit_rx_work); queue_work(rtk->wq, &work->work); } -static void apple_rtkit_tx_done(struct mbox_client *cl, void *mssg, int r) -{ - struct apple_rtkit_msg *msg = - container_of(mssg, struct apple_rtkit_msg, mbox_msg); - - if (r == -ETIME) - return; - - if (msg->completion) - complete(msg->completion); - kfree(msg); -} - int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, struct completion *completion, bool atomic) { - struct apple_rtkit_msg *msg; - int ret; - gfp_t flags; + struct apple_mbox_msg msg = { + .msg0 = message, + .msg1 = ep, + }; if (rtk->crashed) { dev_warn(rtk->dev, @@ -618,19 +601,6 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, return -EINVAL; } - if (atomic) - flags = GFP_ATOMIC; - else - flags = GFP_KERNEL; - - msg = kzalloc(sizeof(*msg), flags); - if (!msg) - return -ENOMEM; - - msg->mbox_msg.msg0 = message; - msg->mbox_msg.msg1 = ep; - msg->completion = completion; - /* * The message will be sent with a MMIO write. We need the barrier * here to ensure any previous writes to buffers are visible to the @@ -638,13 +608,7 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ dma_wmb(); - ret = mbox_send_message(rtk->mbox_chan, &msg->mbox_msg); - if (ret < 0) { - kfree(msg); - return ret; - } - - return 0; + return apple_mbox_send(rtk->mbox, msg, atomic); } EXPORT_SYMBOL_GPL(apple_rtkit_send_message); @@ -682,7 +646,7 @@ EXPORT_SYMBOL_GPL(apple_rtkit_send_message_wait); int apple_rtkit_poll(struct apple_rtkit *rtk) { - return mbox_client_peek_data(rtk->mbox_chan); + return apple_mbox_poll(rtk->mbox); } EXPORT_SYMBOL_GPL(apple_rtkit_poll); @@ -704,20 +668,6 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) } EXPORT_SYMBOL_GPL(apple_rtkit_start_ep); -static int apple_rtkit_request_mbox_chan(struct apple_rtkit *rtk) -{ - if (rtk->mbox_name) - rtk->mbox_chan = mbox_request_channel_byname(&rtk->mbox_cl, - rtk->mbox_name); - else - rtk->mbox_chan = - mbox_request_channel(&rtk->mbox_cl, rtk->mbox_idx); - - if (IS_ERR(rtk->mbox_chan)) - return PTR_ERR(rtk->mbox_chan); - return 0; -} - struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops) @@ -743,13 +693,18 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, bitmap_zero(rtk->endpoints, APPLE_RTKIT_MAX_ENDPOINTS); set_bit(APPLE_RTKIT_EP_MGMT, rtk->endpoints); - rtk->mbox_name = mbox_name; - rtk->mbox_idx = mbox_idx; - rtk->mbox_cl.dev = dev; - rtk->mbox_cl.tx_block = false; - rtk->mbox_cl.knows_txdone = false; - rtk->mbox_cl.rx_callback = &apple_rtkit_rx; - rtk->mbox_cl.tx_done = &apple_rtkit_tx_done; + if (mbox_name) + rtk->mbox = apple_mbox_get_byname(dev, mbox_name); + else + rtk->mbox = apple_mbox_get(dev, mbox_idx); + + if (IS_ERR(rtk->mbox)) { + ret = PTR_ERR(rtk->mbox); + goto free_rtk; + } + + rtk->mbox->rx = apple_rtkit_rx; + rtk->mbox->cookie = rtk; rtk->wq = alloc_ordered_workqueue("rtkit-%s", WQ_MEM_RECLAIM, dev_name(rtk->dev)); @@ -758,7 +713,7 @@ struct apple_rtkit *apple_rtkit_init(struct device *dev, void *cookie, goto free_rtk; } - ret = apple_rtkit_request_mbox_chan(rtk); + ret = apple_mbox_start(rtk->mbox); if (ret) goto destroy_wq; @@ -789,7 +744,7 @@ static int apple_rtkit_wait_for_completion(struct completion *c) int apple_rtkit_reinit(struct apple_rtkit *rtk) { /* make sure we don't handle any messages while reinitializing */ - mbox_free_channel(rtk->mbox_chan); + apple_mbox_stop(rtk->mbox); flush_workqueue(rtk->wq); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); @@ -813,7 +768,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) rtk->iop_power_state = APPLE_RTKIT_PWR_STATE_OFF; rtk->ap_power_state = APPLE_RTKIT_PWR_STATE_OFF; - return apple_rtkit_request_mbox_chan(rtk); + return apple_mbox_start(rtk->mbox); } EXPORT_SYMBOL_GPL(apple_rtkit_reinit); @@ -976,7 +931,7 @@ EXPORT_SYMBOL_GPL(apple_rtkit_wake); void apple_rtkit_free(struct apple_rtkit *rtk) { - mbox_free_channel(rtk->mbox_chan); + apple_mbox_stop(rtk->mbox); destroy_workqueue(rtk->wq); apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); From 718fdf42575923c879b29752487020a8add88766 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 5 Mar 2022 01:08:33 +0900 Subject: [PATCH 012/635] mailbox: apple: Delete driver Signed-off-by: Hector Martin --- drivers/mailbox/Kconfig | 12 - drivers/mailbox/Makefile | 2 - drivers/mailbox/apple-mailbox.c | 441 -------------------------------- 3 files changed, 455 deletions(-) delete mode 100644 drivers/mailbox/apple-mailbox.c diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig index 1495965bc394c0..73952108cc524b 100644 --- a/drivers/mailbox/Kconfig +++ b/drivers/mailbox/Kconfig @@ -8,18 +8,6 @@ menuconfig MAILBOX if MAILBOX -config APPLE_MAILBOX - tristate "Apple Mailbox driver" - depends on ARCH_APPLE || (ARM64 && COMPILE_TEST) - default ARCH_APPLE - help - Apple SoCs have various co-processors required for certain - peripherals to work (NVMe, display controller, etc.). This - driver adds support for the mailbox controller used to - communicate with those. - - Say Y here if you have a Apple SoC. - config ARM_MHU tristate "ARM MHU Mailbox" depends on ARM_AMBA diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile index fc93761171113e..18793e6caa2f1c 100644 --- a/drivers/mailbox/Makefile +++ b/drivers/mailbox/Makefile @@ -60,5 +60,3 @@ obj-$(CONFIG_SUN6I_MSGBOX) += sun6i-msgbox.o obj-$(CONFIG_SPRD_MBOX) += sprd-mailbox.o obj-$(CONFIG_QCOM_IPCC) += qcom-ipcc.o - -obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o diff --git a/drivers/mailbox/apple-mailbox.c b/drivers/mailbox/apple-mailbox.c deleted file mode 100644 index 2a3e8d8ff8b5e6..00000000000000 --- a/drivers/mailbox/apple-mailbox.c +++ /dev/null @@ -1,441 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Apple mailbox driver - * - * Copyright (C) 2021 The Asahi Linux Contributors - * - * This driver adds support for two mailbox variants (called ASC and M3 by - * Apple) found in Apple SoCs such as the M1. It consists of two FIFOs used to - * exchange 64+32 bit messages between the main CPU and a co-processor. - * Various coprocessors implement different IPC protocols based on these simple - * messages and shared memory buffers. - * - * Both the main CPU and the co-processor see the same set of registers but - * the first FIFO (A2I) is always used to transfer messages from the application - * processor (us) to the I/O processor and the second one (I2A) for the - * other direction. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) -#define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) - -#define APPLE_ASC_MBOX_A2I_CONTROL 0x110 -#define APPLE_ASC_MBOX_A2I_SEND0 0x800 -#define APPLE_ASC_MBOX_A2I_SEND1 0x808 -#define APPLE_ASC_MBOX_A2I_RECV0 0x810 -#define APPLE_ASC_MBOX_A2I_RECV1 0x818 - -#define APPLE_ASC_MBOX_I2A_CONTROL 0x114 -#define APPLE_ASC_MBOX_I2A_SEND0 0x820 -#define APPLE_ASC_MBOX_I2A_SEND1 0x828 -#define APPLE_ASC_MBOX_I2A_RECV0 0x830 -#define APPLE_ASC_MBOX_I2A_RECV1 0x838 - -#define APPLE_M3_MBOX_CONTROL_FULL BIT(16) -#define APPLE_M3_MBOX_CONTROL_EMPTY BIT(17) - -#define APPLE_M3_MBOX_A2I_CONTROL 0x50 -#define APPLE_M3_MBOX_A2I_SEND0 0x60 -#define APPLE_M3_MBOX_A2I_SEND1 0x68 -#define APPLE_M3_MBOX_A2I_RECV0 0x70 -#define APPLE_M3_MBOX_A2I_RECV1 0x78 - -#define APPLE_M3_MBOX_I2A_CONTROL 0x80 -#define APPLE_M3_MBOX_I2A_SEND0 0x90 -#define APPLE_M3_MBOX_I2A_SEND1 0x98 -#define APPLE_M3_MBOX_I2A_RECV0 0xa0 -#define APPLE_M3_MBOX_I2A_RECV1 0xa8 - -#define APPLE_M3_MBOX_IRQ_ENABLE 0x48 -#define APPLE_M3_MBOX_IRQ_ACK 0x4c -#define APPLE_M3_MBOX_IRQ_A2I_EMPTY BIT(0) -#define APPLE_M3_MBOX_IRQ_A2I_NOT_EMPTY BIT(1) -#define APPLE_M3_MBOX_IRQ_I2A_EMPTY BIT(2) -#define APPLE_M3_MBOX_IRQ_I2A_NOT_EMPTY BIT(3) - -#define APPLE_MBOX_MSG1_OUTCNT GENMASK(56, 52) -#define APPLE_MBOX_MSG1_INCNT GENMASK(51, 48) -#define APPLE_MBOX_MSG1_OUTPTR GENMASK(47, 44) -#define APPLE_MBOX_MSG1_INPTR GENMASK(43, 40) -#define APPLE_MBOX_MSG1_MSG GENMASK(31, 0) - -struct apple_mbox_hw { - unsigned int control_full; - unsigned int control_empty; - - unsigned int a2i_control; - unsigned int a2i_send0; - unsigned int a2i_send1; - - unsigned int i2a_control; - unsigned int i2a_recv0; - unsigned int i2a_recv1; - - bool has_irq_controls; - unsigned int irq_enable; - unsigned int irq_ack; - unsigned int irq_bit_recv_not_empty; - unsigned int irq_bit_send_empty; -}; - -struct apple_mbox { - void __iomem *regs; - const struct apple_mbox_hw *hw; - - int irq_recv_not_empty; - int irq_send_empty; - - struct mbox_chan chan; - - struct device *dev; - struct mbox_controller controller; - spinlock_t rx_lock; -}; - -static const struct of_device_id apple_mbox_of_match[]; - -static bool apple_mbox_hw_can_send(struct apple_mbox *apple_mbox) -{ - u32 mbox_ctrl = - readl_relaxed(apple_mbox->regs + apple_mbox->hw->a2i_control); - - return !(mbox_ctrl & apple_mbox->hw->control_full); -} - -static bool apple_mbox_hw_send_empty(struct apple_mbox *apple_mbox) -{ - u32 mbox_ctrl = - readl_relaxed(apple_mbox->regs + apple_mbox->hw->a2i_control); - - return mbox_ctrl & apple_mbox->hw->control_empty; -} - -static int apple_mbox_hw_send(struct apple_mbox *apple_mbox, - struct apple_mbox_msg *msg) -{ - if (!apple_mbox_hw_can_send(apple_mbox)) - return -EBUSY; - - dev_dbg(apple_mbox->dev, "> TX %016llx %08x\n", msg->msg0, msg->msg1); - - writeq_relaxed(msg->msg0, apple_mbox->regs + apple_mbox->hw->a2i_send0); - writeq_relaxed(FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg->msg1), - apple_mbox->regs + apple_mbox->hw->a2i_send1); - - return 0; -} - -static bool apple_mbox_hw_can_recv(struct apple_mbox *apple_mbox) -{ - u32 mbox_ctrl = - readl_relaxed(apple_mbox->regs + apple_mbox->hw->i2a_control); - - return !(mbox_ctrl & apple_mbox->hw->control_empty); -} - -static int apple_mbox_hw_recv(struct apple_mbox *apple_mbox, - struct apple_mbox_msg *msg) -{ - if (!apple_mbox_hw_can_recv(apple_mbox)) - return -ENOMSG; - - msg->msg0 = readq_relaxed(apple_mbox->regs + apple_mbox->hw->i2a_recv0); - msg->msg1 = FIELD_GET( - APPLE_MBOX_MSG1_MSG, - readq_relaxed(apple_mbox->regs + apple_mbox->hw->i2a_recv1)); - - dev_dbg(apple_mbox->dev, "< RX %016llx %08x\n", msg->msg0, msg->msg1); - - return 0; -} - -static int apple_mbox_chan_send_data(struct mbox_chan *chan, void *data) -{ - struct apple_mbox *apple_mbox = chan->con_priv; - struct apple_mbox_msg *msg = data; - int ret; - - ret = apple_mbox_hw_send(apple_mbox, msg); - if (ret) - return ret; - - /* - * The interrupt is level triggered and will keep firing as long as the - * FIFO is empty. It will also keep firing if the FIFO was empty - * at any point in the past until it has been acknowledged at the - * mailbox level. By acknowledging it here we can ensure that we will - * only get the interrupt once the FIFO has been cleared again. - * If the FIFO is already empty before the ack it will fire again - * immediately after the ack. - */ - if (apple_mbox->hw->has_irq_controls) { - writel_relaxed(apple_mbox->hw->irq_bit_send_empty, - apple_mbox->regs + apple_mbox->hw->irq_ack); - } - enable_irq(apple_mbox->irq_send_empty); - - return 0; -} - -static irqreturn_t apple_mbox_send_empty_irq(int irq, void *data) -{ - struct apple_mbox *apple_mbox = data; - - /* - * We don't need to acknowledge the interrupt at the mailbox level - * here even if supported by the hardware. It will keep firing but that - * doesn't matter since it's disabled at the main interrupt controller. - * apple_mbox_chan_send_data will acknowledge it before enabling - * it at the main controller again. - */ - disable_irq_nosync(apple_mbox->irq_send_empty); - mbox_chan_txdone(&apple_mbox->chan, 0); - return IRQ_HANDLED; -} - -static int apple_mbox_poll(struct apple_mbox *apple_mbox) -{ - struct apple_mbox_msg msg; - int ret = 0; - - while (apple_mbox_hw_recv(apple_mbox, &msg) == 0) { - mbox_chan_received_data(&apple_mbox->chan, (void *)&msg); - ret++; - } - - /* - * The interrupt will keep firing even if there are no more messages - * unless we also acknowledge it at the mailbox level here. - * There's no race if a message comes in between the check in the while - * loop above and the ack below: If a new messages arrives inbetween - * those two the interrupt will just fire again immediately after the - * ack since it's level triggered. - */ - if (apple_mbox->hw->has_irq_controls) { - writel_relaxed(apple_mbox->hw->irq_bit_recv_not_empty, - apple_mbox->regs + apple_mbox->hw->irq_ack); - } - - return ret; -} - -static irqreturn_t apple_mbox_recv_irq(int irq, void *data) -{ - struct apple_mbox *apple_mbox = data; - - spin_lock(&apple_mbox->rx_lock); - apple_mbox_poll(apple_mbox); - spin_unlock(&apple_mbox->rx_lock); - - return IRQ_HANDLED; -} - -static bool apple_mbox_chan_peek_data(struct mbox_chan *chan) -{ - struct apple_mbox *apple_mbox = chan->con_priv; - unsigned long flags; - int ret; - - spin_lock_irqsave(&apple_mbox->rx_lock, flags); - ret = apple_mbox_poll(apple_mbox); - spin_unlock_irqrestore(&apple_mbox->rx_lock, flags); - - return ret > 0; -} - -static int apple_mbox_chan_flush(struct mbox_chan *chan, unsigned long timeout) -{ - struct apple_mbox *apple_mbox = chan->con_priv; - unsigned long deadline = jiffies + msecs_to_jiffies(timeout); - - while (time_before(jiffies, deadline)) { - if (apple_mbox_hw_send_empty(apple_mbox)) { - mbox_chan_txdone(&apple_mbox->chan, 0); - return 0; - } - - udelay(1); - } - - return -ETIME; -} - -static int apple_mbox_chan_startup(struct mbox_chan *chan) -{ - struct apple_mbox *apple_mbox = chan->con_priv; - - /* - * Only some variants of this mailbox HW provide interrupt control - * at the mailbox level. We therefore need to handle enabling/disabling - * interrupts at the main interrupt controller anyway for hardware that - * doesn't. Just always keep the interrupts we care about enabled at - * the mailbox level so that both hardware revisions behave almost - * the same. - */ - if (apple_mbox->hw->has_irq_controls) { - writel_relaxed(apple_mbox->hw->irq_bit_recv_not_empty | - apple_mbox->hw->irq_bit_send_empty, - apple_mbox->regs + apple_mbox->hw->irq_enable); - } - - enable_irq(apple_mbox->irq_recv_not_empty); - return 0; -} - -static void apple_mbox_chan_shutdown(struct mbox_chan *chan) -{ - struct apple_mbox *apple_mbox = chan->con_priv; - - disable_irq(apple_mbox->irq_recv_not_empty); -} - -static const struct mbox_chan_ops apple_mbox_ops = { - .send_data = apple_mbox_chan_send_data, - .peek_data = apple_mbox_chan_peek_data, - .flush = apple_mbox_chan_flush, - .startup = apple_mbox_chan_startup, - .shutdown = apple_mbox_chan_shutdown, -}; - -static struct mbox_chan *apple_mbox_of_xlate(struct mbox_controller *mbox, - const struct of_phandle_args *args) -{ - if (args->args_count != 0) - return ERR_PTR(-EINVAL); - - return &mbox->chans[0]; -} - -static int apple_mbox_probe(struct platform_device *pdev) -{ - int ret; - const struct of_device_id *match; - char *irqname; - struct apple_mbox *mbox; - struct device *dev = &pdev->dev; - - match = of_match_node(apple_mbox_of_match, pdev->dev.of_node); - if (!match) - return -EINVAL; - if (!match->data) - return -EINVAL; - - mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); - if (!mbox) - return -ENOMEM; - platform_set_drvdata(pdev, mbox); - - mbox->dev = dev; - mbox->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(mbox->regs)) - return PTR_ERR(mbox->regs); - - mbox->hw = match->data; - mbox->irq_recv_not_empty = - platform_get_irq_byname(pdev, "recv-not-empty"); - if (mbox->irq_recv_not_empty < 0) - return -ENODEV; - - mbox->irq_send_empty = platform_get_irq_byname(pdev, "send-empty"); - if (mbox->irq_send_empty < 0) - return -ENODEV; - - mbox->controller.dev = mbox->dev; - mbox->controller.num_chans = 1; - mbox->controller.chans = &mbox->chan; - mbox->controller.ops = &apple_mbox_ops; - mbox->controller.txdone_irq = true; - mbox->controller.of_xlate = apple_mbox_of_xlate; - mbox->chan.con_priv = mbox; - spin_lock_init(&mbox->rx_lock); - - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); - if (!irqname) - return -ENOMEM; - - ret = devm_request_threaded_irq(dev, mbox->irq_recv_not_empty, NULL, - apple_mbox_recv_irq, - IRQF_NO_AUTOEN | IRQF_ONESHOT, irqname, - mbox); - if (ret) - return ret; - - irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-send", dev_name(dev)); - if (!irqname) - return -ENOMEM; - - ret = devm_request_irq(dev, mbox->irq_send_empty, - apple_mbox_send_empty_irq, IRQF_NO_AUTOEN, - irqname, mbox); - if (ret) - return ret; - - return devm_mbox_controller_register(dev, &mbox->controller); -} - -static const struct apple_mbox_hw apple_mbox_asc_hw = { - .control_full = APPLE_ASC_MBOX_CONTROL_FULL, - .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, - - .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, - .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, - .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, - - .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, - .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, - .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, - - .has_irq_controls = false, -}; - -static const struct apple_mbox_hw apple_mbox_m3_hw = { - .control_full = APPLE_M3_MBOX_CONTROL_FULL, - .control_empty = APPLE_M3_MBOX_CONTROL_EMPTY, - - .a2i_control = APPLE_M3_MBOX_A2I_CONTROL, - .a2i_send0 = APPLE_M3_MBOX_A2I_SEND0, - .a2i_send1 = APPLE_M3_MBOX_A2I_SEND1, - - .i2a_control = APPLE_M3_MBOX_I2A_CONTROL, - .i2a_recv0 = APPLE_M3_MBOX_I2A_RECV0, - .i2a_recv1 = APPLE_M3_MBOX_I2A_RECV1, - - .has_irq_controls = true, - .irq_enable = APPLE_M3_MBOX_IRQ_ENABLE, - .irq_ack = APPLE_M3_MBOX_IRQ_ACK, - .irq_bit_recv_not_empty = APPLE_M3_MBOX_IRQ_I2A_NOT_EMPTY, - .irq_bit_send_empty = APPLE_M3_MBOX_IRQ_A2I_EMPTY, -}; - -static const struct of_device_id apple_mbox_of_match[] = { - { .compatible = "apple,asc-mailbox-v4", .data = &apple_mbox_asc_hw }, - { .compatible = "apple,m3-mailbox-v2", .data = &apple_mbox_m3_hw }, - {} -}; -MODULE_DEVICE_TABLE(of, apple_mbox_of_match); - -static struct platform_driver apple_mbox_driver = { - .driver = { - .name = "apple-mailbox", - .of_match_table = apple_mbox_of_match, - }, - .probe = apple_mbox_probe, -}; -module_platform_driver(apple_mbox_driver); - -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_AUTHOR("Sven Peter "); -MODULE_DESCRIPTION("Apple Mailbox driver"); From 1ad28c1bc998db51066d4f5a37581c9e5f0efcc0 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 14 Mar 2023 20:03:44 +0900 Subject: [PATCH 013/635] soc: apple: Drop !APPLE_MAILBOX dependency Signed-off-by: Hector Martin --- drivers/soc/apple/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index b3879a89b5ed92..0bee99d0e7d975 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -21,7 +21,6 @@ config APPLE_MBOX tristate "Apple SoC mailboxes" depends on PM depends on ARCH_APPLE || (64BIT && COMPILE_TEST) - depends on !APPLE_MAILBOX default ARCH_APPLE help Apple SoCs have various co-processors required for certain From c229f8f53079ae8fd280c89ea3d301c0eed78ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 24 Feb 2023 17:21:59 +0100 Subject: [PATCH 014/635] soc: apple: rtkit: Crop syslog messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crop trailing whitespace, null, and newline characters in syslog messages received from coprocessors. Notably DCP sends its messages including a trailing newline, so prior to this change we would end up cluttering the kernel log by repeated newlines at the end of messages. Signed-off-by: Martin Povišer Reviewed-by: Hector Martin Signed-off-by: Hector Martin --- drivers/soc/apple/rtkit.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 64bd696ae00f71..d16c98e6540e2b 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -416,11 +416,17 @@ static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg) rtk->syslog_n_entries, rtk->syslog_msg_size); } +static bool should_crop_syslog_char(char c) +{ + return c == '\n' || c == '\r' || c == ' ' || c == '\0'; +} + static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg) { u8 idx = msg & 0xff; char log_context[24]; size_t entry_size = 0x20 + rtk->syslog_msg_size; + int msglen; if (!rtk->syslog_msg_buffer) { dev_warn( @@ -453,7 +459,13 @@ static void apple_rtkit_syslog_rx_log(struct apple_rtkit *rtk, u64 msg) rtk->syslog_msg_size); log_context[sizeof(log_context) - 1] = 0; - rtk->syslog_msg_buffer[rtk->syslog_msg_size - 1] = 0; + + msglen = rtk->syslog_msg_size - 1; + while (msglen > 0 && + should_crop_syslog_char(rtk->syslog_msg_buffer[msglen - 1])) + msglen--; + + rtk->syslog_msg_buffer[msglen] = 0; dev_info(rtk->dev, "RTKit: syslog message: %s: %s\n", log_context, rtk->syslog_msg_buffer); From 612cb2861e4c86843d94eb52cd5ef528bb4ec3ab Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 16 Apr 2023 21:21:17 +0900 Subject: [PATCH 015/635] soc: apple: rtkit: Implement OSLog buffers properly Apparently nobody can figure out where the old logic came from, but it seems like it has never been actually used on any supported firmware to this day. OSLog buffers were apparently never requested. But starting with 13.3, we actually need this implemented properly for MTP (and later AOP) to work, so let's actually do that. Signed-off-by: Hector Martin --- drivers/soc/apple/rtkit-internal.h | 1 + drivers/soc/apple/rtkit.c | 50 ++++++++++++++++++------------ 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 27c9fa745fd528..b8d5244678f010 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -44,6 +44,7 @@ struct apple_rtkit { struct apple_rtkit_shmem ioreport_buffer; struct apple_rtkit_shmem crashlog_buffer; + struct apple_rtkit_shmem oslog_buffer; struct apple_rtkit_shmem syslog_buffer; char *syslog_msg_buffer; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d16c98e6540e2b..f8e2a5a910ef89 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -67,8 +67,9 @@ enum { #define APPLE_RTKIT_SYSLOG_MSG_SIZE GENMASK_ULL(31, 24) #define APPLE_RTKIT_OSLOG_TYPE GENMASK_ULL(63, 56) -#define APPLE_RTKIT_OSLOG_INIT 1 -#define APPLE_RTKIT_OSLOG_ACK 3 +#define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1 +#define APPLE_RTKIT_OSLOG_SIZE GENMASK_ULL(55, 48) +#define APPLE_RTKIT_OSLOG_IOVA GENMASK_ULL(47, 0) #define APPLE_RTKIT_MIN_SUPPORTED_VERSION 11 #define APPLE_RTKIT_MAX_SUPPORTED_VERSION 12 @@ -260,10 +261,15 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct apple_rtkit_shmem *buffer, u8 ep, u64 msg) { - size_t n_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg); + size_t n_4kpages; u64 reply; int err; + if (ep == APPLE_RTKIT_EP_OSLOG) + n_4kpages = FIELD_GET(APPLE_RTKIT_OSLOG_SIZE, msg); + else + n_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg); + buffer->buffer = NULL; buffer->iomem = NULL; buffer->is_mapped = false; @@ -293,11 +299,20 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, } if (!buffer->is_mapped) { - reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE, - APPLE_RTKIT_BUFFER_REQUEST); - reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, n_4kpages); - reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, - buffer->iova); + /* oslog uses different fields */ + if (ep == APPLE_RTKIT_EP_OSLOG) { + reply = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE, + APPLE_RTKIT_OSLOG_BUFFER_REQUEST); + reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_SIZE, n_4kpages); + reply |= FIELD_PREP(APPLE_RTKIT_OSLOG_IOVA, + buffer->iova >> 12); + } else { + reply = FIELD_PREP(APPLE_RTKIT_SYSLOG_TYPE, + APPLE_RTKIT_BUFFER_REQUEST); + reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, n_4kpages); + reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, + buffer->iova); + } apple_rtkit_send_message(rtk, ep, reply, NULL, false); } @@ -494,25 +509,18 @@ static void apple_rtkit_syslog_rx(struct apple_rtkit *rtk, u64 msg) } } -static void apple_rtkit_oslog_rx_init(struct apple_rtkit *rtk, u64 msg) -{ - u64 ack; - - dev_dbg(rtk->dev, "RTKit: oslog init: msg: 0x%llx\n", msg); - ack = FIELD_PREP(APPLE_RTKIT_OSLOG_TYPE, APPLE_RTKIT_OSLOG_ACK); - apple_rtkit_send_message(rtk, APPLE_RTKIT_EP_OSLOG, ack, NULL, false); -} - static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); switch (type) { - case APPLE_RTKIT_OSLOG_INIT: - apple_rtkit_oslog_rx_init(rtk, msg); + case APPLE_RTKIT_OSLOG_BUFFER_REQUEST: + apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer, + APPLE_RTKIT_EP_OSLOG, msg); break; default: - dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", msg); + dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", + msg); } } @@ -761,6 +769,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); + apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); kfree(rtk->syslog_msg_buffer); @@ -948,6 +957,7 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->ioreport_buffer); apple_rtkit_free_buffer(rtk, &rtk->crashlog_buffer); + apple_rtkit_free_buffer(rtk, &rtk->oslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); kfree(rtk->syslog_msg_buffer); From ac98ef9ff59e60618a1116034f9dedbb971234a3 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Apr 2023 04:19:44 +0900 Subject: [PATCH 016/635] soc: apple: Add driver for Apple PMGR misc controls Apple SoCs have PMGR blocks that control a bunch of power-related features. Besides the existing device power state controls (which are very uniform and handled by apple-pmgr-pwrstate), we also need to manage more random registers such as SoC-wide fabric and memory controller power states, which have a different interface. Add a driver for these kitchen sink controls. Right now it implements fabric and memory controller power state switching on system standby/s2idle, which saves about 1W of power or so on t60xx platforms. Signed-off-by: Hector Martin --- drivers/soc/apple/Kconfig | 9 ++ drivers/soc/apple/Makefile | 1 + drivers/soc/apple/apple-pmgr-misc.c | 158 ++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 drivers/soc/apple/apple-pmgr-misc.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 0bee99d0e7d975..70120ee1565ce0 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -17,6 +17,15 @@ config APPLE_PMGR_PWRSTATE controls for SoC devices. This driver manages them through the generic power domain framework, and also provides reset support. +config APPLE_PMGR_MISC + bool "Apple SoC PMGR miscellaneous support" + depends on PM + default ARCH_APPLE + help + The PMGR block in Apple SoCs provides high-level power state + controls for SoC devices. This driver manages miscellaneous + power controls. + config APPLE_MBOX tristate "Apple SoC mailboxes" depends on PM diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index e52edf6a73dafd..d80162e1899adb 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += apple-pmgr-pwrstate.o +obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o obj-$(CONFIG_APPLE_MBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c new file mode 100644 index 00000000000000..e768f34aacc586 --- /dev/null +++ b/drivers/soc/apple/apple-pmgr-misc.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMGR device power state driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_CLKGEN_PSTATE 0 +#define APPLE_CLKGEN_PSTATE_DESIRED GENMASK(3, 0) + +#define SYS_DEV_PSTATE_SUSPEND 1 + +enum sys_device { + DEV_FABRIC, + DEV_DCS, + DEV_MAX, +}; + +struct apple_pmgr_sys_device { + void __iomem *base; + u32 active_state; + u32 suspend_state; +}; + +struct apple_pmgr_misc { + struct device *dev; + struct apple_pmgr_sys_device devices[DEV_MAX]; +}; + +static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, + enum sys_device dev, bool active) +{ + u32 pstate; + u32 val; + + if (!misc->devices[dev].base) + return; + + if (active) + pstate = misc->devices[dev].active_state; + else + pstate = misc->devices[dev].suspend_state; + + printk("set %d ps to pstate %d\n", dev, pstate); + + val = readl_relaxed(misc->devices[dev].base + APPLE_CLKGEN_PSTATE); + val &= ~APPLE_CLKGEN_PSTATE_DESIRED; + val |= FIELD_PREP(APPLE_CLKGEN_PSTATE_DESIRED, pstate); + writel_relaxed(val, misc->devices[dev].base); +} + +static int __maybe_unused apple_pmgr_misc_suspend_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, false); + + return 0; +} + +static int __maybe_unused apple_pmgr_misc_resume_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, true); + + return 0; +} + +static bool apple_pmgr_init_device(struct apple_pmgr_misc *misc, + enum sys_device dev, const char *device_name) +{ + void __iomem *base; + char name[32]; + u32 val; + + snprintf(name, sizeof(name), "%s-ps", device_name); + + base = devm_platform_ioremap_resource_byname( + to_platform_device(misc->dev), name); + if (!base) + return false; + + val = readl_relaxed(base + APPLE_CLKGEN_PSTATE); + + misc->devices[dev].base = base; + misc->devices[dev].active_state = + FIELD_GET(APPLE_CLKGEN_PSTATE_DESIRED, val); + misc->devices[dev].suspend_state = SYS_DEV_PSTATE_SUSPEND; + + snprintf(name, sizeof(name), "apple,%s-min-ps", device_name); + of_property_read_u32(misc->dev->of_node, name, + &misc->devices[dev].suspend_state); + + return true; +} + +static int apple_pmgr_misc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_pmgr_misc *misc; + int ret = -ENODEV; + + misc = devm_kzalloc(dev, sizeof(*misc), GFP_KERNEL); + if (!misc) + return -ENOMEM; + + misc->dev = dev; + + if (apple_pmgr_init_device(misc, DEV_FABRIC, "fabric")) + ret = 0; + + if (apple_pmgr_init_device(misc, DEV_DCS, "dcs")) + ret = 0; + + platform_set_drvdata(pdev, misc); + + return ret; +} + +static const struct of_device_id apple_pmgr_misc_of_match[] = { + { .compatible = "apple,t6000-pmgr-misc" }, + {} +}; + +MODULE_DEVICE_TABLE(of, apple_pmgr_misc_of_match); + +static const struct dev_pm_ops apple_pmgr_misc_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_misc_suspend_noirq, + apple_pmgr_misc_resume_noirq) +}; + +static struct platform_driver apple_pmgr_misc_driver = { + .probe = apple_pmgr_misc_probe, + .driver = { + .name = "apple-pmgr-misc", + .of_match_table = apple_pmgr_misc_of_match, + .pm = pm_ptr(&apple_pmgr_misc_pm_ops), + }, +}; + +MODULE_AUTHOR("Hector Martin "); +MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs"); +MODULE_LICENSE("GPL v2"); + +module_platform_driver(apple_pmgr_misc_driver); From db1cc8afaef4423ec1ac5f44e5df729cef8abe73 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 17 Apr 2023 20:41:13 +0900 Subject: [PATCH 017/635] cpuidle: apple: Add Apple SoC cpuidle driver May the PSCI conversation happen some day. Until it does, this will make the user experience a lot less painful in downstream kernels. Signed-off-by: Hector Martin --- drivers/cpuidle/Kconfig.arm | 8 ++ drivers/cpuidle/Makefile | 1 + drivers/cpuidle/cpuidle-apple.c | 151 ++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 drivers/cpuidle/cpuidle-apple.c diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index a1ee475d180dac..c6870f08457632 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -130,3 +130,11 @@ config ARM_QCOM_SPM_CPUIDLE The Subsystem Power Manager (SPM) controls low power modes for the CPU and L2 cores. It interface with various system drivers to put the cores in low power modes. + +config ARM_APPLE_CPUIDLE + bool "Apple SoC CPU idle driver" + depends on ARM64 + default ARCH_APPLE + select CPU_IDLE_MULTIPLE_DRIVERS + help + Select this to enable cpuidle on Apple SoCs. diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index d103342b7cfc21..972b49aec88903 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_ARM_PSCI_CPUIDLE) += cpuidle-psci.o obj-$(CONFIG_ARM_PSCI_CPUIDLE_DOMAIN) += cpuidle-psci-domain.o obj-$(CONFIG_ARM_TEGRA_CPUIDLE) += cpuidle-tegra.o obj-$(CONFIG_ARM_QCOM_SPM_CPUIDLE) += cpuidle-qcom-spm.o +obj-$(CONFIG_ARM_APPLE_CPUIDLE) += cpuidle-apple.o ############################################################################### # MIPS drivers diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c new file mode 100644 index 00000000000000..b430174c724bbf --- /dev/null +++ b/drivers/cpuidle/cpuidle-apple.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * CPU idle support for Apple SoCs + */ + +#include +#include +#include +#include +#include +#include + +enum idle_state { + STATE_WFI, + STATE_PWRDOWN, + STATE_COUNT +}; + +asm( + ".type apple_cpu_deep_wfi, @function\n" + "apple_cpu_deep_wfi:\n" + "str x30, [sp, #-16]!\n" + "stp x28, x29, [sp, #-16]!\n" + "stp x26, x27, [sp, #-16]!\n" + "stp x24, x25, [sp, #-16]!\n" + "stp x22, x23, [sp, #-16]!\n" + "stp x20, x21, [sp, #-16]!\n" + "stp x18, x19, [sp, #-16]!\n" + + "mrs x0, s3_5_c15_c5_0\n" + "orr x0, x0, #(3L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "dsb sy\n" + "wfi\n" + + "mrs x0, s3_5_c15_c5_0\n" + "bic x0, x0, #(1L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "ldp x18, x19, [sp], #16\n" + "ldp x20, x21, [sp], #16\n" + "ldp x22, x23, [sp], #16\n" + "ldp x24, x25, [sp], #16\n" + "ldp x26, x27, [sp], #16\n" + "ldp x28, x29, [sp], #16\n" + "ldr x30, [sp], #16\n" + + "ret\n" +); + +void apple_cpu_deep_wfi(void); + +static __cpuidle int apple_enter_idle(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + /* + * Deep WFI will clobber FP state, among other things. + * The CPU PM notifier will take care of saving that and anything else + * that needs to be notified of the CPU powering down. + */ + if (cpu_pm_enter()) + return -1; + + switch(index) { + case STATE_WFI: + cpu_do_idle(); + break; + case STATE_PWRDOWN: + /* + * If we probed this driver in EL1, that's the m1n1 hypervisor. + * Go through the motions but do a normal WFI idle, since it will + * trap out the WFI idle stuff anyway. This avoids noise and slowness. + */ + if (is_kernel_in_hyp_mode()) + apple_cpu_deep_wfi(); + else + cpu_do_idle(); + + break; + default: + WARN_ON(1); + break; + } + + cpu_pm_exit(); + + return index; +} + +static struct cpuidle_driver apple_idle_driver = { + .name = "apple_idle", + .owner = THIS_MODULE, + .states = { + [STATE_WFI] = { + .enter = apple_enter_idle, + .enter_s2idle = apple_enter_idle, + .exit_latency = 1, + .target_residency = 1, + .power_usage = UINT_MAX, + .name = "WFI", + .desc = "CPU clock-gated", + }, + [STATE_PWRDOWN] = { + .enter = apple_enter_idle, + .enter_s2idle = apple_enter_idle, + .exit_latency = 10, + .target_residency = 10000, + .power_usage = 0, + .name = "CPU PD", + .desc = "CPU/cluster powered down", + }, + }, + .safe_state_index = STATE_WFI, + .state_count = STATE_COUNT, +}; + +static int apple_cpuidle_probe(struct platform_device *pdev) +{ + return cpuidle_register(&apple_idle_driver, NULL); +} + +static struct platform_driver apple_cpuidle_driver = { + .driver = { + .name = "cpuidle-apple", + }, + .probe = apple_cpuidle_probe, +}; + +static int __init apple_cpuidle_init(void) +{ + struct platform_device *pdev; + int ret; + + ret = platform_driver_register(&apple_cpuidle_driver); + if (ret) + return ret; + + if (!of_machine_is_compatible("apple,arm-platform")) + return 0; + + pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&apple_cpuidle_driver); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(apple_cpuidle_init); From 64314818271dd6e78606e3d5197a5d74bf5d9e7c Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 2 Jan 2023 19:20:55 +0100 Subject: [PATCH 018/635] soc: apple: rtkit: Add devm_apple_rtkit_free() To be used to free a RTKit interface while the associated device remains alive. Probably useless since it's unknown how or if RTKit based co-processors can be restarted. Signed-off-by: Janne Grunau --- drivers/soc/apple/rtkit.c | 6 ++++++ include/linux/soc/apple/rtkit.h | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index f8e2a5a910ef89..50e137a14e8f90 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -989,6 +989,12 @@ struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, } EXPORT_SYMBOL_GPL(devm_apple_rtkit_init); +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk) +{ + devm_release_action(dev, apple_rtkit_free_wrapper, rtk); +} +EXPORT_SYMBOL_GPL(devm_apple_rtkit_free); + MODULE_LICENSE("Dual MIT/GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple RTKit driver"); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index fc456f75c13192..f8d9cc0691de03 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -78,6 +78,13 @@ struct apple_rtkit; struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Frees internal RTKit state allocated by devm_apple_rtkit_init(). + * + * @dev: Pointer to the device node this coprocessor is assocated with + * @rtk: Internal RTKit state initialized by devm_apple_rtkit_init() + */ +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk); /* * Non-devm version of devm_apple_rtkit_init. Must be freed with From 7f228ef0075fa439dec4bcf1c52e7ce007b41fe8 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 22 Oct 2022 16:01:01 +0200 Subject: [PATCH 019/635] iommu/io-pgtable: dart: Remove unused __dart_alloc_pages() paramater The parameter became unused during refactoring but was missed. Fixes: 745ef1092bcf ("iommu/io-pgtable: Move Apple DART support to its own file") Signed-off-by: Janne Grunau --- drivers/iommu/io-pgtable-dart.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 74b1ef2b96bee1..a8a81b55b4d26b 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -106,8 +106,7 @@ static phys_addr_t iopte_to_paddr(dart_iopte pte, return paddr; } -static void *__dart_alloc_pages(size_t size, gfp_t gfp, - struct io_pgtable_cfg *cfg) +static void *__dart_alloc_pages(size_t size, gfp_t gfp) { int order = get_order(size); struct page *p; @@ -262,7 +261,7 @@ static int dart_map_pages(struct io_pgtable_ops *ops, unsigned long iova, /* no L2 table present */ if (!pte) { - cptep = __dart_alloc_pages(tblsz, gfp, cfg); + cptep = __dart_alloc_pages(tblsz, gfp); if (!cptep) return -ENOMEM; @@ -419,8 +418,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) { - data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL, - cfg); + data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL); if (!data->pgd[i]) goto out_free_data; cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); From 096ad8b1826846885eb9d215998a0d1fbfec4e9b Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 22 Oct 2022 12:00:21 +0200 Subject: [PATCH 020/635] iommu: Add IOMMU_RESV_TRANSLATED for non 1:1 mapped reserved regions The display controller in Apple silicon SoCs uses bootloader mappings which require IOMMU translation. Signed-off-by: Janne Grunau --- drivers/iommu/iommu.c | 24 ++++++++++++++++++++---- include/linux/iommu.h | 10 ++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 10db680acaed5a..0288f35733d573 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -80,6 +80,7 @@ static const char * const iommu_group_resv_type_string[] = { [IOMMU_RESV_RESERVED] = "reserved", [IOMMU_RESV_MSI] = "msi", [IOMMU_RESV_SW_MSI] = "msi", + [IOMMU_RESV_TRANSLATED] = "translated", }; #define IOMMU_CMD_LINE_DMA_API BIT(0) @@ -2667,10 +2668,11 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) } EXPORT_SYMBOL(iommu_put_resv_regions); -struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, - size_t length, int prot, - enum iommu_resv_type type, - gfp_t gfp) +struct iommu_resv_region *iommu_alloc_resv_region_tr(phys_addr_t start, + dma_addr_t dva_start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) { struct iommu_resv_region *region; @@ -2680,11 +2682,25 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, INIT_LIST_HEAD(®ion->list); region->start = start; + if (type == IOMMU_RESV_TRANSLATED) + region->dva = dva_start; region->length = length; region->prot = prot; region->type = type; return region; } +EXPORT_SYMBOL_GPL(iommu_alloc_resv_region_tr); + +struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) +{ + if (type == IOMMU_RESV_TRANSLATED) + return NULL; + + return iommu_alloc_resv_region_tr(start, 0, length, prot, type, gfp); +} EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); void iommu_set_default_passthrough(bool cmd_line) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 6595454d4f48de..4b6f30d3980879 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -146,12 +146,18 @@ enum iommu_resv_type { IOMMU_RESV_MSI, /* Software-managed MSI translation window */ IOMMU_RESV_SW_MSI, + /* + * Memory regions which must be mapped with the specified mapping + * at all times. + */ + IOMMU_RESV_TRANSLATED, }; /** * struct iommu_resv_region - descriptor for a reserved memory region * @list: Linked list pointers * @start: System physical start address of the region + * @start: Device virtual start address of the region for IOMMU_RESV_TRANSLATED * @length: Length of the region in bytes * @prot: IOMMU Protection flags (READ/WRITE/...) * @type: Type of the reserved region @@ -160,6 +166,7 @@ enum iommu_resv_type { struct iommu_resv_region { struct list_head list; phys_addr_t start; + dma_addr_t dva; size_t length; int prot; enum iommu_resv_type type; @@ -492,6 +499,9 @@ extern bool iommu_default_passthrough(void); extern struct iommu_resv_region * iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, enum iommu_resv_type type, gfp_t gfp); +extern struct iommu_resv_region * +iommu_alloc_resv_region_tr(phys_addr_t start, dma_addr_t dva_start, size_t length, + int prot, enum iommu_resv_type type, gfp_t gfp); extern int iommu_get_group_resv_regions(struct iommu_group *group, struct list_head *head); From 519a3cab68f6ae24892bd4bef8b42529ffa8466d Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 22 Oct 2022 12:24:54 +0200 Subject: [PATCH 021/635] iommu: Parse translated reserved regions These regions are setup by the boot loader and require an iommu to translate arbitray physical to device VA mappings. Signed-off-by: Janne Grunau --- drivers/iommu/dma-iommu.c | 9 +++++++-- drivers/iommu/of_iommu.c | 11 +++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 99b2646cb5c7ad..317a600cedbb20 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -500,8 +500,13 @@ static int iova_reserve_iommu_regions(struct device *dev, if (region->type == IOMMU_RESV_SW_MSI) continue; - lo = iova_pfn(iovad, region->start); - hi = iova_pfn(iovad, region->start + region->length - 1); + if (region->type == IOMMU_RESV_TRANSLATED) { + lo = iova_pfn(iovad, region->dva); + hi = iova_pfn(iovad, region->dva + region->length - 1); + } else { + lo = iova_pfn(iovad, region->start); + hi = iova_pfn(iovad, region->start + region->length - 1); + } reserve_iova(iovad, lo, hi); if (region->type == IOMMU_RESV_MSI) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 40f57d293a79d4..943c56d962048a 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -191,9 +191,7 @@ iommu_resv_region_get_type(struct device *dev, if (start == phys->start && end == phys->end) return IOMMU_RESV_DIRECT; - dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", &phys, - &start, &end); - return IOMMU_RESV_RESERVED; + return IOMMU_RESV_TRANSLATED; } /** @@ -257,8 +255,13 @@ void of_iommu_get_resv_regions(struct device *dev, struct list_head *list) maps = of_translate_dma_region(np, maps, &iova, &length); type = iommu_resv_region_get_type(dev, &phys, iova, length); - region = iommu_alloc_resv_region(iova, length, prot, type, + if (type == IOMMU_RESV_TRANSLATED) + region = iommu_alloc_resv_region_tr(phys.start, iova, length, prot, type, + GFP_KERNEL); + else + region = iommu_alloc_resv_region(iova, length, prot, type, GFP_KERNEL); + if (region) list_add_tail(®ion->list, list); } From 9ee5d1a279754c7a62c69e67389c1a578d2e9aca Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Thu, 26 Aug 2021 12:16:52 -0400 Subject: [PATCH 022/635] iommu/dart: Track if the DART is locked Locked DARTs require special handling. This can be detected with the configuration register. Check this when probing and save the result. Signed-off-by: Alyssa Rosenzweig --- drivers/iommu/apple-dart.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 06169d36eab829..2b2e135790e7fa 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -197,6 +197,7 @@ struct apple_dart_hw { * @pgsize: pagesize supported by this DART * @supports_bypass: indicates if this DART supports bypass mode * @force_bypass: force bypass mode due to pagesize mismatch? + * @locked: indicates if this DART is locked * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ @@ -218,6 +219,7 @@ struct apple_dart { u32 num_streams; u32 supports_bypass : 1; u32 force_bypass : 1; + u32 locked : 1; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; @@ -1045,6 +1047,11 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) return IRQ_HANDLED; } +static bool apple_dart_is_locked(struct apple_dart *dart) +{ + return !!(readl(dart->regs + dart->hw->lock) & dart->hw->lock_bit); +} + static int apple_dart_probe(struct platform_device *pdev) { int ret; @@ -1114,6 +1121,7 @@ static int apple_dart_probe(struct platform_device *pdev) dart->force_bypass = dart->pgsize > PAGE_SIZE; + dart->locked = apple_dart_is_locked(dart); ret = apple_dart_hw_reset(dart); if (ret) goto err_clk_disable; @@ -1136,8 +1144,8 @@ static int apple_dart_probe(struct platform_device *pdev) dev_info( &pdev->dev, - "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", - dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass); + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d] initialized\n", + dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass, dart->locked); return 0; err_sysfs_remove: From 3a14d5c7580cbd8383fde2b4f3d8ccf9aaa2e315 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Thu, 26 Aug 2021 12:18:28 -0400 Subject: [PATCH 023/635] iommu/dart: Allow locked DARTs to probe Instead of bailing from reset() if the DART is locked, simply skip the reset for locked DARTs. Signed-off-by: Alyssa Rosenzweig --- drivers/iommu/apple-dart.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 2b2e135790e7fa..9fc7f5db3baf97 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -451,17 +451,9 @@ apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_hw_reset(struct apple_dart *dart) { - u32 config; struct apple_dart_stream_map stream_map; int i; - config = readl(dart->regs + dart->hw->lock); - if (config & dart->hw->lock_bit) { - dev_err(dart->dev, "DART is locked down until reboot: %08x\n", - config); - return -EINVAL; - } - stream_map.dart = dart; bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); @@ -1122,9 +1114,11 @@ static int apple_dart_probe(struct platform_device *pdev) dart->force_bypass = dart->pgsize > PAGE_SIZE; dart->locked = apple_dart_is_locked(dart); - ret = apple_dart_hw_reset(dart); - if (ret) - goto err_clk_disable; + if (!dart->locked) { + ret = apple_dart_hw_reset(dart); + if (ret) + goto err_clk_disable; + } ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, "apple-dart fault handler", dart); From d8690b50dc581327817af76945ff50dc92873cca Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Thu, 26 Aug 2021 12:47:11 -0400 Subject: [PATCH 024/635] iommu/dart: Factor out apple_dart* expression In the next patch this will be used a third time. Signed-off-by: Alyssa Rosenzweig --- drivers/iommu/apple-dart.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 9fc7f5db3baf97..147832d7961097 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -652,12 +652,11 @@ static int apple_dart_attach_dev(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_domain *dart_domain = to_dart_domain(domain); + struct apple_dart *dart0 = cfg->stream_maps[0].dart; - if (cfg->stream_maps[0].dart->force_bypass && - domain->type != IOMMU_DOMAIN_IDENTITY) + if (dart0->force_bypass && domain->type != IOMMU_DOMAIN_IDENTITY) return -EINVAL; - if (!cfg->stream_maps[0].dart->supports_bypass && - domain->type == IOMMU_DOMAIN_IDENTITY) + if (!dart0->supports_bypass && domain->type == IOMMU_DOMAIN_IDENTITY) return -EINVAL; ret = apple_dart_finalize_domain(domain, cfg); @@ -907,10 +906,11 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) static int apple_dart_def_domain_type(struct device *dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; - if (cfg->stream_maps[0].dart->force_bypass) + if (dart->force_bypass) return IOMMU_DOMAIN_IDENTITY; - if (!cfg->stream_maps[0].dart->supports_bypass) + if (dart->supports_bypass) return IOMMU_DOMAIN_DMA; return 0; From 3ee165b24d07ea8b27df1064d2ced5d82f629e0d Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Thu, 26 Aug 2021 12:48:59 -0400 Subject: [PATCH 025/635] iommu/dart: Set DMA domain for locked DARTs This is required. Signed-off-by: Alyssa Rosenzweig --- drivers/iommu/apple-dart.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 147832d7961097..b9ee79652685db 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -908,8 +908,12 @@ static int apple_dart_def_domain_type(struct device *dev) struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart *dart = cfg->stream_maps[0].dart; + WARN_ON(dart->force_bypass && dart->locked); + if (dart->force_bypass) return IOMMU_DOMAIN_IDENTITY; + if (dart->locked) + return IOMMU_DOMAIN_DMA; if (dart->supports_bypass) return IOMMU_DOMAIN_DMA; From 6ff858c91eeca7f2c5db789ba09e4421d2d238e7 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Thu, 26 Aug 2021 12:50:07 -0400 Subject: [PATCH 026/635] iommu/dart: Require DMA domain for locked DARTs Signed-off-by: Alyssa Rosenzweig --- drivers/iommu/apple-dart.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b9ee79652685db..524bacf389f866 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -658,6 +658,8 @@ static int apple_dart_attach_dev(struct iommu_domain *domain, return -EINVAL; if (!dart0->supports_bypass && domain->type == IOMMU_DOMAIN_IDENTITY) return -EINVAL; + if (dart0->locked && domain->type != IOMMU_DOMAIN_DMA) + return -EINVAL; ret = apple_dart_finalize_domain(domain, cfg); if (ret) From 1d060f8d68ee3b1b3ece5f7eeab4c2a4052c65f7 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Thu, 26 Aug 2021 13:19:51 -0400 Subject: [PATCH 027/635] iommu/dart: Assert !locked when reconfiguring Signed-off-by: Alyssa Rosenzweig --- drivers/iommu/apple-dart.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 524bacf389f866..1b69368be552bd 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -310,6 +310,7 @@ apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(stream_map->dart->locked); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); } @@ -319,6 +320,7 @@ static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(stream_map->dart->locked); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); } @@ -329,6 +331,7 @@ apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(stream_map->dart->locked); WARN_ON(!stream_map->dart->supports_bypass); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(dart->hw->tcr_bypass, @@ -341,6 +344,7 @@ static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(stream_map->dart->locked); WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(dart->hw->ttbr_valid | @@ -354,6 +358,7 @@ static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(stream_map->dart->locked); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(0, dart->regs + DART_TTBR(dart, sid, idx)); } From 891eae8a836e3224f6daf327e3abd1c9e0227b18 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 27 Aug 2021 12:54:09 -0400 Subject: [PATCH 028/635] iommu: io-pgtable: Add locked DART quirk Apple DARTs can be locked by the bootloader. In particular, we cannot change the TTBR address for these DARTs. However, we may take over the existing allocated page. We clear the existing page table. Important mappings will be restored by reserved regions. Add a quirk for this case and handle it. Signed-off-by: Alyssa Rosenzweig Co-developed-by: Janne Grunau Signed-off-by: Janne Grunau --- drivers/iommu/apple-dart.c | 23 +++++++++++++++++++++++ drivers/iommu/io-pgtable-dart.c | 31 +++++++++++++++++++++++++++++-- include/linux/io-pgtable.h | 4 ++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1b69368be552bd..31f126d14378b6 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -596,6 +596,29 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, .iommu_dev = dart->dev, }; + if (dart->locked) { + unsigned long *sidmap; + int sid; + phys_addr_t phys; + u32 ttbr; + + /* Locked DARTs can only have a single stream bound */ + sidmap = cfg->stream_maps[0].sidmap; + sid = find_first_bit(sidmap, dart->num_streams); + + WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); + ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + + WARN_ON(!(ttbr & dart->hw->ttbr_valid)); + ttbr &= ~dart->hw->ttbr_valid; + if (dart->hw->ttbr_addr_field_shift) + ttbr >>= dart->hw->ttbr_addr_field_shift; + + phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift; + pgtbl_cfg.apple_dart_cfg.ttbr[0] = phys; + pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_APPLE_LOCKED; + } + dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, domain); if (!dart_domain->pgtbl_ops) { diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index a8a81b55b4d26b..81129e6ca4704f 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -417,29 +418,55 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; + /* Locked DARTs can not modify the TTBR registers. The known locked + * DARTs (dcp, dcpext0) use just a single TTBR so we do not have to + * worry whether the pages are consecutive. + */ + if (cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED) { + size_t size = cfg->pgsize_bitmap; + if (cfg->apple_dart_cfg.n_ttbrs > 1) + goto out_free_data; + + data->pgd[0] = devm_memremap(cfg->iommu_dev, + cfg->apple_dart_cfg.ttbr[0], + size, MEMREMAP_WB); + if (!data->pgd[0]) + goto out_free_data; + /* start with an empty table */ + memset(data->pgd[0], 0, size); + return &data->iop; + } + for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) { data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL); if (!data->pgd[i]) - goto out_free_data; + goto out_free_pages; cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); } return &data->iop; -out_free_data: +out_free_pages: while (--i >= 0) free_pages((unsigned long)data->pgd[i], get_order(DART_GRANULE(data))); +out_free_data: kfree(data); return NULL; } static void apple_dart_free_pgtable(struct io_pgtable *iop) { + struct io_pgtable_cfg *cfg = &iop->cfg; struct dart_io_pgtable *data = io_pgtable_to_data(iop); dart_iopte *ptep, *end; int i; + if (cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED) { + kfree(data); + return; + } + for (i = 0; i < (1 << data->tbl_bits) && data->pgd[i]; ++i) { ptep = data->pgd[i]; end = (void *)ptep + DART_GRANULE(data); diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 1b7a44b35616c7..e4697ff48d3ad4 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -85,6 +85,9 @@ struct io_pgtable_cfg { * * IO_PGTABLE_QUIRK_ARM_OUTER_WBWA: Override the outer-cacheability * attributes set in the TCR for a non-coherent page-table walker. + * + * IO_PGTABLE_QUIRK_APPLE_LOCKED: Cannot modify the TTBR pointer, must + * inherit mappings from the bootloader. */ #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) @@ -92,6 +95,7 @@ struct io_pgtable_cfg { #define IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT BIT(4) #define IO_PGTABLE_QUIRK_ARM_TTBR1 BIT(5) #define IO_PGTABLE_QUIRK_ARM_OUTER_WBWA BIT(6) + #define IO_PGTABLE_QUIRK_APPLE_LOCKED BIT(7) unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; From b94ca3c63eeb9a18b43271d0cd6f50ff65e7679f Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 22 Oct 2022 14:30:15 +0200 Subject: [PATCH 029/635] iommu: apple-dart: Install IOMMU_RESV_TRANSLATED mappings [insert rant about locked TTBR] The iommus for the display processors on Apple silicon machines have locked TTBR registers. This makes it a little akward to install the boot loader's translated reserved mappings on iommu domain init. Handle this via a shadow L1 pgtable apple_dart_alloc_pgtable(). apple-dart will after that map the reserved regions and let io-pgtable-dart transfer the shadow L1 table to the locked HW L1 table. This has to be done this way since HW L1 table still holds the bootloader allocated L2 tables. Switching the L2 PTE has to happen atomically since the display processor is the whole time active with scan out of the boot frame buffer. TODO: find a less hacky way for upstream Signed-off-by: Janne Grunau --- drivers/iommu/Kconfig | 1 + drivers/iommu/apple-dart.c | 58 +++++++++++++++++++++++++++++---- drivers/iommu/io-pgtable-dart.c | 44 ++++++++++++++++++++----- drivers/iommu/io-pgtable-dart.h | 15 +++++++++ 4 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 drivers/iommu/io-pgtable-dart.h diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 889c7efd050bc8..0134342292db5b 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -311,6 +311,7 @@ config APPLE_DART depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_DART select IOMMU_API select IOMMU_IO_PGTABLE_DART + select OF_IOMMU default ARCH_APPLE help Support for Apple DART (Device Address Resolution Table) IOMMUs diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 31f126d14378b6..ed1bae7c5d8e34 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -33,6 +33,7 @@ #include #include "dma-iommu.h" +#include "io-pgtable-dart.h" #define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 @@ -557,17 +558,55 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) - apple_dart_hw_set_ttbr(stream_map, i, - pgtbl_cfg->apple_dart_cfg.ttbr[i]); - for (; i < stream_map->dart->hw->ttbr_count; ++i) - apple_dart_hw_clear_ttbr(stream_map, i); + /* Locked DARTs are set up by the bootloader. */ + if (!stream_map->dart->locked) { + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) + apple_dart_hw_set_ttbr(stream_map, i, + pgtbl_cfg->apple_dart_cfg.ttbr[i]); + for (; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_clear_ttbr(stream_map, i); - apple_dart_hw_enable_translation(stream_map); + apple_dart_hw_enable_translation(stream_map); + } stream_map->dart->hw->invalidate_tlb(stream_map); } +static int apple_dart_setup_resv_locked(struct iommu_domain *domain, + struct device *dev, size_t pgsize) +{ + struct iommu_resv_region *region; + LIST_HEAD(resv_regions); + int ret = 0; + + of_iommu_get_resv_regions(dev, &resv_regions); + list_for_each_entry(region, &resv_regions, list) { + size_t mapped = 0; + + /* Only map translated reserved regions */ + if (region->type != IOMMU_RESV_TRANSLATED) + continue; + + while (mapped < region->length) { + phys_addr_t paddr = region->start + mapped; + unsigned long iova = region->dva + mapped; + size_t length = region->length - mapped; + size_t pgcount = length / pgsize; + + ret = apple_dart_map_pages(domain, iova, + paddr, pgsize, pgcount, + region->prot, GFP_KERNEL, &mapped); + + if (ret) + goto end_put; + } + } +end_put: + iommu_put_resv_regions(dev, &resv_regions); + return ret; +} + static int apple_dart_finalize_domain(struct iommu_domain *domain, + struct device *dev, struct apple_dart_master_cfg *cfg) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); @@ -633,6 +672,11 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, dart_domain->finalized = true; + if (dart->locked) { + /* TODO: error handling */ + ret = apple_dart_setup_resv_locked(domain, dev, dart->pgsize); + io_pgtable_dart_setup_locked(dart_domain->pgtbl_ops); + } done: mutex_unlock(&dart_domain->init_lock); return ret; @@ -689,7 +733,7 @@ static int apple_dart_attach_dev(struct iommu_domain *domain, if (dart0->locked && domain->type != IOMMU_DOMAIN_DMA) return -EINVAL; - ret = apple_dart_finalize_domain(domain, cfg); + ret = apple_dart_finalize_domain(domain, dev, cfg); if (ret) return ret; diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 81129e6ca4704f..8459772bfefeb6 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -16,6 +16,7 @@ #include #include #include +#include "linux/export.h" #include #include #include @@ -25,6 +26,8 @@ #include +#include "io-pgtable-dart.h" + #define DART1_MAX_ADDR_BITS 36 #define DART_MAX_TABLES 4 @@ -363,6 +366,32 @@ static phys_addr_t dart_iova_to_phys(struct io_pgtable_ops *ops, return 0; } +int io_pgtable_dart_setup_locked(struct io_pgtable_ops *ops) +{ + void *l1tbl; + struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + size_t size; + + if (!(cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED)) + return 0; + + size = cfg->pgsize_bitmap; + l1tbl = devm_memremap(cfg->iommu_dev, cfg->apple_dart_cfg.ttbr[0], size, + MEMREMAP_WB); + if (!l1tbl) + return -ENOMEM; + + for (int entry = 0; entry < DART_PTES_PER_TABLE(data); entry++) + ((dart_iopte *)l1tbl)[entry] = ((dart_iopte *)data->pgd[0])[entry]; + + free_pages((unsigned long)data->pgd[0], get_order(DART_GRANULE(data))); + data->pgd[0] = l1tbl; + + return 0; +} +EXPORT_SYMBOL(io_pgtable_dart_setup_locked); + static struct dart_io_pgtable * dart_alloc_pgtable(struct io_pgtable_cfg *cfg) { @@ -418,22 +447,19 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; - /* Locked DARTs can not modify the TTBR registers. The known locked - * DARTs (dcp, dcpext0) use just a single TTBR so we do not have to - * worry whether the pages are consecutive. + /* Locked DARTs can not modify the TTBR registers. Allocate first a shadow + * page table so locked DARTs (disp0, dcp, dcpext*) can map their reserved + * memory regions. They will be later in io_pgtable_dart_setup_locked() + * copied to the locked L1 table. */ if (cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED) { - size_t size = cfg->pgsize_bitmap; if (cfg->apple_dart_cfg.n_ttbrs > 1) goto out_free_data; - data->pgd[0] = devm_memremap(cfg->iommu_dev, - cfg->apple_dart_cfg.ttbr[0], - size, MEMREMAP_WB); + data->pgd[0] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL); if (!data->pgd[0]) goto out_free_data; - /* start with an empty table */ - memset(data->pgd[0], 0, size); + return &data->iop; } diff --git a/drivers/iommu/io-pgtable-dart.h b/drivers/iommu/io-pgtable-dart.h new file mode 100644 index 00000000000000..90fd1035e9f1d7 --- /dev/null +++ b/drivers/iommu/io-pgtable-dart.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple DART page table allocator. + * + * Copyright (C) 2022 The Asahi Linux Contributors + */ + +/* This will go away on the next iteration of locked DART handling */ + +#ifndef IO_PGTABLE_DART_H_ +#define IO_PGTABLE_DART_H_ + +int io_pgtable_dart_setup_locked(struct io_pgtable_ops *ops); + +#endif /* IO_PGTABLE_DART_H_ */ From 282ab49995a711ce9d16a3ab2ffa2d45fa75de7f Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 17 Nov 2022 21:31:23 +0900 Subject: [PATCH 030/635] iommu: apple-dart: Don't attempt to reset/restore locked DARTs This can't work, and should not be needed in these cases. Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index ed1bae7c5d8e34..b03d362362c24c 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1234,7 +1234,9 @@ static int apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); - apple_dart_hw_reset(dart); + if (!dart->locked) + apple_dart_hw_reset(dart); + free_irq(dart->irq, dart); iommu_device_unregister(&dart->iommu); @@ -1342,6 +1344,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* Locked DARTs can't be restored, and they should not need it */ + if (dart->locked) + return 0; + ret = apple_dart_hw_reset(dart); if (ret) { dev_err(dev, "Failed to reset DART on resume\n"); From 7edfa962bd38fc79c38b845c7cc87cc2f762cb42 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 3 Jan 2023 16:20:01 +0100 Subject: [PATCH 031/635] HACK: iommu: apple-dart: workaround broken runtime PM with PCIe devices Runtime PM behaves strangely with PCIe devices, pm_runtime_enabled() returns true here although the devices do not support runtime PM. This seems to confuse the PM core and DART is runtime suspended while the device is in use. The dart_hw_reset() in the resume path ends up breaking PCIe devices eventually. Signed-off-by: Janne Grunau --- drivers/iommu/apple-dart.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b03d362362c24c..2efb48a7028a18 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -764,15 +764,24 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_stream_map *stream_map; + u32 dl_flags = DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER; int i; if (!cfg) return ERR_PTR(-ENODEV); + /* + * Runtime PM behaves strangely with PCIe devices, pm_runtime_enabled() + * returns true here although the devices do not support runtime PM. + * This seems to confuse the PM core and DART is runtime suspended while + * the device is in use. The dart_hw_reset() in the resume path ends up + * breaking PCIe devices eventually. + */ + if (dev_is_pci(dev)) + dl_flags |= DL_FLAG_RPM_ACTIVE; + for_each_stream_map(i, cfg, stream_map) - device_link_add( - dev, stream_map->dart->dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + device_link_add(dev, stream_map->dart->dev, dl_flags); return &cfg->stream_maps[0].dart->iommu; } From e8284ec399aa873fc0a9ab786a3a1f890ce5e8d2 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 12 Dec 2022 23:53:23 +0900 Subject: [PATCH 032/635] iommu: apple-dart: Enable runtime PM Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 2efb48a7028a18..b7daed5e02bfc5 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -491,7 +492,9 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); + WARN_ON(pm_runtime_get_sync(stream_map.dart->dev) < 0); stream_map.dart->hw->invalidate_tlb(&stream_map); + pm_runtime_put(stream_map.dart->dev); } } @@ -733,16 +736,19 @@ static int apple_dart_attach_dev(struct iommu_domain *domain, if (dart0->locked && domain->type != IOMMU_DOMAIN_DMA) return -EINVAL; + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + ret = apple_dart_finalize_domain(domain, dev, cfg); if (ret) - return ret; + goto err; switch (domain->type) { case IOMMU_DOMAIN_DMA: case IOMMU_DOMAIN_UNMANAGED: ret = apple_dart_domain_add_streams(dart_domain, cfg); if (ret) - return ret; + goto err; for_each_stream_map(i, cfg, stream_map) apple_dart_setup_translation(dart_domain, stream_map); @@ -757,6 +763,9 @@ static int apple_dart_attach_dev(struct iommu_domain *domain, break; } +err: + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return ret; } @@ -1169,6 +1178,14 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) return ret; + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_irq_safe(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + goto err_clk_disable; + dart_params[0] = readl(dart->regs + DART_PARAMS1); dart_params[1] = readl(dart->regs + DART_PARAMS2); dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); @@ -1223,6 +1240,8 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_sysfs_remove; + pm_runtime_put(dev); + dev_info( &pdev->dev, "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d] initialized\n", @@ -1234,6 +1253,7 @@ static int apple_dart_probe(struct platform_device *pdev) err_free_irq: free_irq(dart->irq, dart); err_clk_disable: + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -1373,7 +1393,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) return 0; } -DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); +DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -1388,7 +1408,7 @@ static struct platform_driver apple_dart_driver = { .name = "apple-dart", .of_match_table = apple_dart_of_match, .suppress_bind_attrs = true, - .pm = pm_sleep_ptr(&apple_dart_pm_ops), + .pm = pm_ptr(&apple_dart_pm_ops), }, .probe = apple_dart_probe, .remove = apple_dart_remove, From dcf43a427ffcbc79686412189e790ae5f406b748 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 20:02:26 +0900 Subject: [PATCH 033/635] iommu: io-pgtable: Add 4-level page table support DARTs on t602x SoCs are of the t8110 variant but have an IAS of 42, which means optional support for an extra page table level. Refactor the PTE management to support an arbitrary level count, and then calculate how many levels we need for any given configuration. Signed-off-by: Hector Martin --- drivers/iommu/io-pgtable-dart.c | 150 +++++++++++++++++++------------- include/linux/io-pgtable.h | 1 + 2 files changed, 91 insertions(+), 60 deletions(-) diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index 8459772bfefeb6..06febd787d416f 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -30,8 +30,9 @@ #define DART1_MAX_ADDR_BITS 36 -#define DART_MAX_TABLES 4 -#define DART_LEVELS 2 +#define DART_MAX_TABLE_BITS 2 +#define DART_MAX_TABLES BIT(DART_MAX_TABLE_BITS) +#define DART_MAX_LEVELS 4 /* Includes TTBR level */ /* Struct accessors */ #define io_pgtable_to_data(x) \ @@ -71,6 +72,7 @@ struct dart_io_pgtable { struct io_pgtable iop; + int levels; int tbl_bits; int bits_per_level; @@ -173,44 +175,45 @@ static dart_iopte dart_install_table(dart_iopte *table, return old; } -static int dart_get_table(struct dart_io_pgtable *data, unsigned long iova) +static int dart_get_index(struct dart_io_pgtable *data, unsigned long iova, int level) { - return (iova >> (3 * data->bits_per_level + ilog2(sizeof(dart_iopte)))) & - ((1 << data->tbl_bits) - 1); + return (iova >> (level * data->bits_per_level + ilog2(sizeof(dart_iopte)))) & + ((1 << data->bits_per_level) - 1); } -static int dart_get_l1_index(struct dart_io_pgtable *data, unsigned long iova) -{ - - return (iova >> (2 * data->bits_per_level + ilog2(sizeof(dart_iopte)))) & - ((1 << data->bits_per_level) - 1); -} - -static int dart_get_l2_index(struct dart_io_pgtable *data, unsigned long iova) +static int dart_get_last_index(struct dart_io_pgtable *data, unsigned long iova) { return (iova >> (data->bits_per_level + ilog2(sizeof(dart_iopte)))) & ((1 << data->bits_per_level) - 1); } -static dart_iopte *dart_get_l2(struct dart_io_pgtable *data, unsigned long iova) +static dart_iopte *dart_get_last(struct dart_io_pgtable *data, unsigned long iova) { dart_iopte pte, *ptep; - int tbl = dart_get_table(data, iova); + int level = data->levels; + int tbl = dart_get_index(data, iova, level); + + if (tbl > (1 << data->tbl_bits)) + return NULL; ptep = data->pgd[tbl]; if (!ptep) return NULL; - ptep += dart_get_l1_index(data, iova); - pte = READ_ONCE(*ptep); + while (--level > 1) { + ptep += dart_get_index(data, iova, level); + pte = READ_ONCE(*ptep); - /* Valid entry? */ - if (!pte) - return NULL; + /* Valid entry? */ + if (!pte) + return NULL; + + /* Deref to get next level table */ + ptep = iopte_deref(pte, data); + } - /* Deref to get level 2 table */ - return iopte_deref(pte, data); + return ptep; } static dart_iopte dart_prot_to_pte(struct dart_io_pgtable *data, @@ -246,6 +249,7 @@ static int dart_map_pages(struct io_pgtable_ops *ops, unsigned long iova, int ret = 0, tbl, num_entries, max_entries, map_idx_start; dart_iopte pte, *cptep, *ptep; dart_iopte prot; + int level = data->levels; if (WARN_ON(pgsize != cfg->pgsize_bitmap)) return -EINVAL; @@ -257,31 +261,36 @@ static int dart_map_pages(struct io_pgtable_ops *ops, unsigned long iova, if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) return 0; - tbl = dart_get_table(data, iova); + tbl = dart_get_index(data, iova, level); + + if (tbl > (1 << data->tbl_bits)) + return -ENOMEM; ptep = data->pgd[tbl]; - ptep += dart_get_l1_index(data, iova); - pte = READ_ONCE(*ptep); + while (--level > 1) { + ptep += dart_get_index(data, iova, level); + pte = READ_ONCE(*ptep); - /* no L2 table present */ - if (!pte) { - cptep = __dart_alloc_pages(tblsz, gfp); - if (!cptep) - return -ENOMEM; + /* no table present */ + if (!pte) { + cptep = __dart_alloc_pages(tblsz, gfp); + if (!cptep) + return -ENOMEM; - pte = dart_install_table(cptep, ptep, 0, data); - if (pte) - free_pages((unsigned long)cptep, get_order(tblsz)); + pte = dart_install_table(cptep, ptep, 0, data); + if (pte) + free_pages((unsigned long)cptep, get_order(tblsz)); - /* L2 table is present (now) */ - pte = READ_ONCE(*ptep); - } + /* L2 table is present (now) */ + pte = READ_ONCE(*ptep); + } - ptep = iopte_deref(pte, data); + ptep = iopte_deref(pte, data); + } /* install a leaf entries into L2 table */ prot = dart_prot_to_pte(data, iommu_prot); - map_idx_start = dart_get_l2_index(data, iova); + map_idx_start = dart_get_last_index(data, iova); max_entries = DART_PTES_PER_TABLE(data) - map_idx_start; num_entries = min_t(int, pgcount, max_entries); ptep += map_idx_start; @@ -310,13 +319,13 @@ static size_t dart_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, if (WARN_ON(pgsize != cfg->pgsize_bitmap || !pgcount)) return 0; - ptep = dart_get_l2(data, iova); + ptep = dart_get_last(data, iova); /* Valid L2 IOPTE pointer? */ if (WARN_ON(!ptep)) return 0; - unmap_idx_start = dart_get_l2_index(data, iova); + unmap_idx_start = dart_get_last_index(data, iova); ptep += unmap_idx_start; max_entries = DART_PTES_PER_TABLE(data) - unmap_idx_start; @@ -347,13 +356,13 @@ static phys_addr_t dart_iova_to_phys(struct io_pgtable_ops *ops, struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops); dart_iopte pte, *ptep; - ptep = dart_get_l2(data, iova); + ptep = dart_get_last(data, iova); /* Valid L2 IOPTE pointer? */ if (!ptep) return 0; - ptep += dart_get_l2_index(data, iova); + ptep += dart_get_last_index(data, iova); pte = READ_ONCE(*ptep); /* Found translation */ @@ -396,21 +405,37 @@ static struct dart_io_pgtable * dart_alloc_pgtable(struct io_pgtable_cfg *cfg) { struct dart_io_pgtable *data; - int tbl_bits, bits_per_level, va_bits, pg_shift; + int levels, max_tbl_bits, tbl_bits, bits_per_level, va_bits, pg_shift; + + /* + * Old 4K page DARTs can use up to 4 top-level tables. + * Newer ones only ever use a maximum of 1. + */ + if (cfg->pgsize_bitmap == SZ_4K) + max_tbl_bits = DART_MAX_TABLE_BITS; + else + max_tbl_bits = 0; pg_shift = __ffs(cfg->pgsize_bitmap); bits_per_level = pg_shift - ilog2(sizeof(dart_iopte)); va_bits = cfg->ias - pg_shift; - tbl_bits = max_t(int, 0, va_bits - (bits_per_level * DART_LEVELS)); - if ((1 << tbl_bits) > DART_MAX_TABLES) + levels = max_t(int, 2, (va_bits - max_tbl_bits + bits_per_level - 1) / bits_per_level); + + if (levels > (DART_MAX_LEVELS - 1)) + return NULL; + + tbl_bits = max_t(int, 0, va_bits - (bits_per_level * levels)); + + if (tbl_bits > max_tbl_bits) return NULL; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; + data->levels = levels + 1; /* Table level counts as one level */ data->tbl_bits = tbl_bits; data->bits_per_level = bits_per_level; @@ -446,6 +471,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) return NULL; cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; + cfg->apple_dart_cfg.n_levels = data->levels; /* Locked DARTs can not modify the TTBR registers. Allocate first a shadow * page table so locked DARTs (disp0, dcp, dcpext*) can map their reserved @@ -481,11 +507,28 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) return NULL; } +static void apple_dart_free_pgtables(struct dart_io_pgtable *data, dart_iopte *ptep, int level) +{ + dart_iopte *end; + + if (level > 1) { + end = (void *)ptep + DART_GRANULE(data); + + while (ptep != end) { + dart_iopte pte = *ptep++; + + if (pte) + apple_dart_free_pgtables(data, iopte_deref(pte, data), level - 1); + } + } + free_pages((unsigned long)ptep, get_order(DART_GRANULE(data))); +} + static void apple_dart_free_pgtable(struct io_pgtable *iop) { struct io_pgtable_cfg *cfg = &iop->cfg; struct dart_io_pgtable *data = io_pgtable_to_data(iop); - dart_iopte *ptep, *end; + dart_iopte *ptep; int i; if (cfg->quirks & IO_PGTABLE_QUIRK_APPLE_LOCKED) { @@ -495,20 +538,7 @@ static void apple_dart_free_pgtable(struct io_pgtable *iop) for (i = 0; i < (1 << data->tbl_bits) && data->pgd[i]; ++i) { ptep = data->pgd[i]; - end = (void *)ptep + DART_GRANULE(data); - - while (ptep != end) { - dart_iopte pte = *ptep++; - - if (pte) { - unsigned long page = - (unsigned long)iopte_deref(pte, data); - - free_pages(page, get_order(DART_GRANULE(data))); - } - } - free_pages((unsigned long)data->pgd[i], - get_order(DART_GRANULE(data))); + apple_dart_free_pgtables(data, data->pgd[i], data->levels - 1); } kfree(data); diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index e4697ff48d3ad4..7c84de9a3a2844 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -147,6 +147,7 @@ struct io_pgtable_cfg { struct { u64 ttbr[4]; u32 n_ttbrs; + u32 n_levels; } apple_dart_cfg; }; }; From 077218f0a3b169ecd66a0fe3df01021dfadf1641 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 20:04:05 +0900 Subject: [PATCH 034/635] iommu: apple-dart: Clear stream error indicator bits for T8110 DARTs These registers exist at least on the t602x variant, and if not cleared the IRQ will never clear. Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index b7daed5e02bfc5..36bb952f6d7529 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -123,6 +123,8 @@ #define DART_T8110_ERROR_ADDR_LO 0x170 #define DART_T8110_ERROR_ADDR_HI 0x174 +#define DART_T8110_ERROR_STREAMS 0x1c0 + #define DART_T8110_PROTECT 0x200 #define DART_T8110_UNPROTECT 0x204 #define DART_T8110_PROTECT_LOCK 0x208 @@ -1106,6 +1108,7 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); u64 addr = addr_lo | (((u64)addr_hi) << 32); u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); + int i; if (!(error & DART_T8110_ERROR_FLAG)) return IRQ_NONE; @@ -1132,6 +1135,9 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) error, stream_idx, error_code, fault_name, addr); writel(error, dart->regs + DART_T8110_ERROR); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + writel(U32_MAX, dart->regs + DART_T8110_ERROR_STREAMS + 4 * i); + return IRQ_HANDLED; } From aec4e006db12c43a48c8aa41c3bddf259fbc5f21 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 20:05:20 +0900 Subject: [PATCH 035/635] iommu: apple-dart: Make the hw register fields u32s The registers are 32-bit and the offsets definitely don't need 64 bits either, these should've been u32s. Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 36bb952f6d7529..1043dc9a521228 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -169,22 +169,22 @@ struct apple_dart_hw { int max_sid_count; - u64 lock; - u64 lock_bit; + u32 lock; + u32 lock_bit; - u64 error; + u32 error; - u64 enable_streams; + u32 enable_streams; - u64 tcr; - u64 tcr_enabled; - u64 tcr_disabled; - u64 tcr_bypass; + u32 tcr; + u32 tcr_enabled; + u32 tcr_disabled; + u32 tcr_bypass; - u64 ttbr; - u64 ttbr_valid; - u64 ttbr_addr_field_shift; - u64 ttbr_shift; + u32 ttbr; + u32 ttbr_valid; + u32 ttbr_addr_field_shift; + u32 ttbr_shift; int ttbr_count; }; From 5829a0f24ca3b5d3224fd18be3ecd3f674fb8283 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 20:06:44 +0900 Subject: [PATCH 036/635] iommu: apple-dart: Add 4-level page table support The T8110 variant DART implementation on T602x SoCs indicates an IAS of 42, which requires an extra page table level. The extra level is optional, but let's implement it. Later it might be useful to restrict this based on the actual attached devices, since most won't need that much address space anyway. Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1043dc9a521228..c6530444bc2274 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -136,6 +136,7 @@ #define DART_T8110_TCR 0x1000 #define DART_T8110_TCR_REMAP GENMASK(11, 8) #define DART_T8110_TCR_REMAP_EN BIT(7) +#define DART_T8110_TCR_FOUR_LEVEL BIT(3) #define DART_T8110_TCR_BYPASS_DAPF BIT(2) #define DART_T8110_TCR_BYPASS_DART BIT(1) #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) @@ -180,6 +181,7 @@ struct apple_dart_hw { u32 tcr_enabled; u32 tcr_disabled; u32 tcr_bypass; + u32 tcr_4level; u32 ttbr; u32 ttbr_valid; @@ -224,6 +226,7 @@ struct apple_dart { u32 supports_bypass : 1; u32 force_bypass : 1; u32 locked : 1; + u32 four_level : 1; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; @@ -309,14 +312,17 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) } static void -apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) +apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels) { struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(levels != 3 && levels != 4); + WARN_ON(levels == 4 && !dart->four_level); WARN_ON(stream_map->dart->locked); for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) - writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); + writel(dart->hw->tcr_enabled | (levels == 4 ? dart->hw->tcr_4level : 0), + dart->regs + DART_TCR(dart, sid)); } static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) @@ -571,7 +577,8 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, for (; i < stream_map->dart->hw->ttbr_count; ++i) apple_dart_hw_clear_ttbr(stream_map, i); - apple_dart_hw_enable_translation(stream_map); + apple_dart_hw_enable_translation(stream_map, + pgtbl_cfg->apple_dart_cfg.n_levels); } stream_map->dart->hw->invalidate_tlb(stream_map); } @@ -661,6 +668,19 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift; pgtbl_cfg.apple_dart_cfg.ttbr[0] = phys; pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_APPLE_LOCKED; + + /* If the DART is locked, we need to keep the translation level count. */ + if (dart->hw->tcr_4level && dart->ias > 36) { + if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) { + if (dart->ias < 37) { + dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); + pgtbl_cfg.ias = 37; + } + } else if (dart->ias > 36) { + dev_info(dart->dev, "Limited to ias=36 due to lock\n"); + pgtbl_cfg.ias = 36; + } + } } dart_domain->pgtbl_ops = @@ -672,7 +692,7 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; domain->geometry.aperture_start = 0; - domain->geometry.aperture_end = (dma_addr_t)DMA_BIT_MASK(dart->ias); + domain->geometry.aperture_end = (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); domain->geometry.force_aperture = true; dart_domain->finalized = true; @@ -1211,6 +1231,7 @@ static int apple_dart_probe(struct platform_device *pdev) dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); + dart->four_level = dart->ias > 36; break; } @@ -1250,8 +1271,9 @@ static int apple_dart_probe(struct platform_device *pdev) dev_info( &pdev->dev, - "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d] initialized\n", - dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass, dart->locked); + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d, AS %d -> %d] initialized\n", + dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass, dart->locked, + dart->ias, dart->oas); return 0; err_sysfs_remove: @@ -1350,6 +1372,7 @@ static const struct apple_dart_hw apple_dart_hw_t8110 = { .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, .tcr_disabled = 0, .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, + .tcr_4level = DART_T8110_TCR_FOUR_LEVEL, .ttbr = DART_T8110_TTBR, .ttbr_valid = DART_T8110_TTBR_VALID, From 59096bb2482b2826a05a5f84a765e36b7aeb96a0 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 01:32:06 +0900 Subject: [PATCH 037/635] iommu: apple-dart: Support specifying the DMA aperture in the DT Apple DARTs are often connected directly to devices that expect only a portion of their address space to be used for DMA (for example, because other ranges are mapped directly to something else). Add an apple,dma-range property to allow specifying this range. This range *can* be outside of the DART's IAS. In that case, it is assumed that the hardware truncates addresses and the page tables will only map the lower bits of the address. However, the specified range cannot straddle an IAS boundary (you cannot cover more than IAS worth of address space nor wrap). This corresponds to the vm-base and vm-size properties on the Apple device tree side of things. Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 62 ++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index c6530444bc2274..922254383a95ba 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -228,6 +229,9 @@ struct apple_dart { u32 locked : 1; u32 four_level : 1; + dma_addr_t dma_min; + dma_addr_t dma_max; + struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; @@ -272,6 +276,7 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + u64 mask; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -532,7 +537,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova); + return ops->iova_to_phys(ops, iova & dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -546,8 +551,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, - mapped); + return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + pgcount, prot, gfp, mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -558,7 +563,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + gather); } static void @@ -624,6 +630,8 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; + dma_addr_t dma_max = dart->dma_max; + u32 ias = min_t(u32, dart->ias, fls64(dma_max)); int ret = 0; int i, j; @@ -641,7 +649,7 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, - .ias = dart->ias, + .ias = ias, .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, @@ -672,13 +680,21 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, /* If the DART is locked, we need to keep the translation level count. */ if (dart->hw->tcr_4level && dart->ias > 36) { if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) { - if (dart->ias < 37) { + if (ias < 37) { dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); pgtbl_cfg.ias = 37; } - } else if (dart->ias > 36) { + } else if (ias > 36) { dev_info(dart->dev, "Limited to ias=36 due to lock\n"); pgtbl_cfg.ias = 36; + if (dart->dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) { + dma_max = DMA_BIT_MASK(pgtbl_cfg.ias); + } else if ((dart->dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) { + dev_err(dart->dev, + "Invalid DMA range for locked 3-level PT\n"); + ret = -ENOMEM; + goto done; + } } } } @@ -690,9 +706,16 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, goto done; } + if (pgtbl_cfg.pgsize_bitmap == SZ_4K) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 32)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 3) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 36)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 4) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 47)); + domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - domain->geometry.aperture_start = 0; - domain->geometry.aperture_end = (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); + domain->geometry.aperture_start = dart->dma_min; + domain->geometry.aperture_end = dma_max; domain->geometry.force_aperture = true; dart_domain->finalized = true; @@ -1173,6 +1196,7 @@ static int apple_dart_probe(struct platform_device *pdev) struct resource *res; struct apple_dart *dart; struct device *dev = &pdev->dev; + u64 dma_range[2]; dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); if (!dart) @@ -1235,6 +1259,26 @@ static int apple_dart_probe(struct platform_device *pdev) break; } + dart->dma_min = 0; + dart->dma_max = DMA_BIT_MASK(dart->ias); + + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", dma_range, 2); + if (ret == -EINVAL) { + ret = 0; + } else if (ret) { + goto err_clk_disable; + } else { + dart->dma_min = dma_range[0]; + dart->dma_max = dma_range[0] + dma_range[1] - 1; + if ((dart->dma_min ^ dart->dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(&pdev->dev, "Invalid DMA range for ias=%d\n", + dart->ias); + goto err_clk_disable; + } + dev_info(&pdev->dev, "Limiting DMA range to %pad..%pad\n", + &dart->dma_min, &dart->dma_max); + } + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); From 532039d599356dfa557c508f7a77166cffdfb259 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Sep 2021 02:23:11 +0900 Subject: [PATCH 038/635] tty: serial: samsung_tty: Support runtime PM This allows idle UART devices to be suspended using the standard runtime-PM framework. The logic is modeled after stm32-usart. Signed-off-by: Hector Martin --- drivers/tty/serial/samsung_tty.c | 91 ++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index 2a7520ad3abd92..7d82c97f642b9e 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -40,6 +40,7 @@ #include #include #include +#include #include /* UART name and device definitions */ @@ -1360,30 +1361,49 @@ static int apple_s5l_serial_startup(struct uart_port *port) /* power power management control */ +static int __maybe_unused s3c24xx_serial_runtime_suspend(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + int timeout = 10000; + + while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) + udelay(100); + + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + clk_disable_unprepare(ourport->clk); + return 0; +}; + +static int __maybe_unused s3c24xx_serial_runtime_resume(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + + clk_prepare_enable(ourport->clk); + + if (!IS_ERR(ourport->baudclk)) + clk_prepare_enable(ourport->baudclk); + return 0; +}; + static void s3c24xx_serial_pm(struct uart_port *port, unsigned int level, unsigned int old) { struct s3c24xx_uart_port *ourport = to_ourport(port); - int timeout = 10000; ourport->pm_level = level; switch (level) { - case 3: - while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) - udelay(100); - - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); - - clk_disable_unprepare(ourport->clk); + case UART_PM_STATE_OFF: + pm_runtime_mark_last_busy(port->dev); + pm_runtime_put_sync(port->dev); break; - case 0: - clk_prepare_enable(ourport->clk); - - if (!IS_ERR(ourport->baudclk)) - clk_prepare_enable(ourport->baudclk); + case UART_PM_STATE_ON: + pm_runtime_get_sync(port->dev); break; default: dev_err(port->dev, "s3c24xx_serial: unknown pm %d\n", level); @@ -2129,18 +2149,15 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) } } + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + dev_dbg(&pdev->dev, "%s: adding port\n", __func__); uart_add_one_port(&s3c24xx_uart_drv, &ourport->port); platform_set_drvdata(pdev, &ourport->port); - /* - * Deactivate the clock enabled in s3c24xx_serial_init_port here, - * so that a potential re-enablement through the pm-callback overlaps - * and keeps the clock enabled in this case. - */ - clk_disable_unprepare(ourport->clk); - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); + pm_runtime_put_sync(&pdev->dev); probe_index++; @@ -2150,9 +2167,19 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) static int s3c24xx_serial_remove(struct platform_device *dev) { struct uart_port *port = s3c24xx_dev_to_port(&dev->dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); if (port) { + pm_runtime_get_sync(&dev->dev); uart_remove_one_port(&s3c24xx_uart_drv, port); + + clk_disable_unprepare(ourport->clk); + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + pm_runtime_disable(&dev->dev); + pm_runtime_set_suspended(&dev->dev); + pm_runtime_put_noidle(&dev->dev); } uart_unregister_driver(&s3c24xx_uart_drv); @@ -2161,8 +2188,8 @@ static int s3c24xx_serial_remove(struct platform_device *dev) } /* UART power management code */ -#ifdef CONFIG_PM_SLEEP -static int s3c24xx_serial_suspend(struct device *dev) + +static int __maybe_unused s3c24xx_serial_suspend(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); @@ -2172,7 +2199,7 @@ static int s3c24xx_serial_suspend(struct device *dev) return 0; } -static int s3c24xx_serial_resume(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2192,7 +2219,7 @@ static int s3c24xx_serial_resume(struct device *dev) return 0; } -static int s3c24xx_serial_resume_noirq(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume_noirq(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2262,16 +2289,14 @@ static int s3c24xx_serial_resume_noirq(struct device *dev) } static const struct dev_pm_ops s3c24xx_serial_pm_ops = { +#ifdef CONFIG_PM_SLEEP .suspend = s3c24xx_serial_suspend, .resume = s3c24xx_serial_resume, .resume_noirq = s3c24xx_serial_resume_noirq, +#endif + SET_RUNTIME_PM_OPS(s3c24xx_serial_runtime_suspend, + s3c24xx_serial_runtime_resume, NULL) }; -#define SERIAL_SAMSUNG_PM_OPS (&s3c24xx_serial_pm_ops) - -#else /* !CONFIG_PM_SLEEP */ - -#define SERIAL_SAMSUNG_PM_OPS NULL -#endif /* CONFIG_PM_SLEEP */ /* Console code */ @@ -2709,7 +2734,7 @@ static struct platform_driver samsung_serial_driver = { .id_table = s3c24xx_serial_driver_ids, .driver = { .name = "samsung-uart", - .pm = SERIAL_SAMSUNG_PM_OPS, + .pm = &s3c24xx_serial_pm_ops, .of_match_table = of_match_ptr(s3c24xx_uart_dt_match), }, }; From 1aba7bb187e665d59abae94492773c0789dab0af Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Mar 2022 19:19:38 +0900 Subject: [PATCH 039/635] drm/simpledrm: Add backlight support Allows devicetrees to link the simplefb node to a backlight device, and toggles power to the backlight when the display pipe is enabled/disabled. This is sufficient for basic DPMS style functionality in trivial devices. Signed-off-by: Hector Martin --- drivers/gpu/drm/tiny/Kconfig | 1 + drivers/gpu/drm/tiny/simpledrm.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/tiny/Kconfig b/drivers/gpu/drm/tiny/Kconfig index f6889f649bc182..45d09dc38a0aa2 100644 --- a/drivers/gpu/drm/tiny/Kconfig +++ b/drivers/gpu/drm/tiny/Kconfig @@ -85,6 +85,7 @@ config DRM_SIMPLEDRM select APERTURE_HELPERS select DRM_GEM_SHMEM_HELPER select DRM_KMS_HELPER + select BACKLIGHT_CLASS_DEVICE help DRM driver for simple platform-provided framebuffers. diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c index 63881a3754f8a0..2ac85eefa8bcc3 100644 --- a/drivers/gpu/drm/tiny/simpledrm.c +++ b/drivers/gpu/drm/tiny/simpledrm.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only +#include #include #include #include @@ -243,6 +244,8 @@ struct simpledrm_device { struct drm_crtc crtc; struct drm_encoder encoder; struct drm_connector connector; + /* backlight */ + struct backlight_device *backlight; }; static struct simpledrm_device *simpledrm_device_of_dev(struct drm_device *dev) @@ -553,6 +556,26 @@ static enum drm_mode_status simpledrm_crtc_helper_mode_valid(struct drm_crtc *cr return drm_crtc_helper_mode_valid_fixed(crtc, mode, &sdev->mode); } +static void simpledrm_crtc_helper_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_device *dev = crtc->dev; + struct simpledrm_device *sdev = simpledrm_device_of_dev(dev); + + if (sdev->backlight) + backlight_enable(sdev->backlight); +} + +static void simpledrm_crtc_helper_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_device *dev = crtc->dev; + struct simpledrm_device *sdev = simpledrm_device_of_dev(dev); + + if (sdev->backlight) + backlight_disable(sdev->backlight); +} + /* * The CRTC is always enabled. Screen updates are performed by * the primary plane's atomic_update function. Disabling clears @@ -561,6 +584,8 @@ static enum drm_mode_status simpledrm_crtc_helper_mode_valid(struct drm_crtc *cr static const struct drm_crtc_helper_funcs simpledrm_crtc_helper_funcs = { .mode_valid = simpledrm_crtc_helper_mode_valid, .atomic_check = drm_crtc_helper_atomic_check, + .atomic_enable = simpledrm_crtc_helper_atomic_enable, + .atomic_disable = simpledrm_crtc_helper_atomic_disable, }; static const struct drm_crtc_funcs simpledrm_crtc_funcs = { @@ -649,6 +674,10 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, * Hardware settings */ + sdev->backlight = devm_of_find_backlight(&pdev->dev); + if (IS_ERR(sdev->backlight)) + sdev->backlight = NULL; + ret = simpledrm_device_init_clocks(sdev); if (ret) return ERR_PTR(ret); From 45f714b63cd98b688331ee681f00eac40d1a2615 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 12 Mar 2022 00:07:09 +0900 Subject: [PATCH 040/635] of: Demote "Bad cell count" to debug message This happens on the SPMI bus... TODO: figure out what the right solution is here. Signed-off-by: Hector Martin --- drivers/of/address.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/of/address.c b/drivers/of/address.c index 4c0b169ef9bfba..fef88e6e12dde9 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -538,7 +538,7 @@ static u64 __of_translate_address(struct device_node *dev, pbus = of_match_bus(parent); pbus->count_cells(dev, &pna, &pns); if (!OF_CHECK_COUNTS(pna, pns)) { - pr_err("Bad cell count for %pOF\n", dev); + pr_debug("Bad cell count for %pOF\n", dev); break; } From 571bfb1fa48d43034825506d6613adf9c7eb378d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 5 May 2022 01:40:31 +0900 Subject: [PATCH 041/635] mmc: sdhci-pci: Support external CD GPIO on all OF systems Allow OF systems to specify an external CD GPIO on all devices, even if they have an internal CD feature. Signed-off-by: Hector Martin --- drivers/mmc/host/sdhci-pci-core.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 01975d14520066..5a426674dbbdde 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -2117,6 +2117,15 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( dev_warn(&pdev->dev, "failed to setup card detect gpio\n"); slot->cd_idx = -1; } + } else if (is_of_node(pdev->dev.fwnode)) { + /* Allow all OF systems to use a CD GPIO if provided */ + + ret = mmc_gpiod_request_cd(host->mmc, "cd", 0, + slot->cd_override_level, 0); + if (ret == -EPROBE_DEFER) + goto remove; + else if (ret == 0) + slot->cd_idx = 0; } if (chip->fixes && chip->fixes->add_host) From a1e0878450f8c1aaf71b3e2095b446861588c3fb Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 5 May 2022 02:27:35 +0900 Subject: [PATCH 042/635] mmc: sdhci-pci: Support setting CD debounce delay Some systems (e.g. 2021 MacBook Pro 14/16") have noncompliant connectors where CD activates before the card is fully inserted. We need debounce delay support on these to avoid detection failures when the card isn't inserted very quickly. Set the default to 200ms for all systems instead of 0. This is the default on non-PCI platforms, and will probably help other systems too. The naughty MacBooks will need closer to 750ms in the device tree to be reliable... Signed-off-by: Hector Martin --- drivers/mmc/host/sdhci-pci-core.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 5a426674dbbdde..03a3a769481e86 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -2036,6 +2036,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( struct sdhci_host *host; int ret, bar = first_bar + slotno; size_t priv_size = chip->fixes ? chip->fixes->priv_size : 0; + u32 cd_debounce_delay_ms; if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "BAR %d is not iomem. Aborting.\n", bar); @@ -2102,6 +2103,10 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( if (host->mmc->caps & MMC_CAP_CD_WAKE) device_init_wakeup(&pdev->dev, true); + if (device_property_read_u32(&pdev->dev, "cd-debounce-delay-ms", + &cd_debounce_delay_ms)) + cd_debounce_delay_ms = 200; + if (slot->cd_idx >= 0) { ret = mmc_gpiod_request_cd(host->mmc, "cd", slot->cd_idx, slot->cd_override_level, 0); @@ -2109,7 +2114,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( ret = mmc_gpiod_request_cd(host->mmc, NULL, slot->cd_idx, slot->cd_override_level, - 0); + cd_debounce_delay_ms * 1000); if (ret == -EPROBE_DEFER) goto remove; @@ -2121,7 +2126,8 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( /* Allow all OF systems to use a CD GPIO if provided */ ret = mmc_gpiod_request_cd(host->mmc, "cd", 0, - slot->cd_override_level, 0); + slot->cd_override_level, + cd_debounce_delay_ms * 1000); if (ret == -EPROBE_DEFER) goto remove; else if (ret == 0) From 31a5ec72b9bf9f711ede4a722e110d31e6f58d33 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 7 Aug 2022 23:45:03 +0200 Subject: [PATCH 043/635] drivers/perf: Add Apple blizzard/avalanche to Apple M1 CPU PMU driver Signed-off-by: Janne Grunau --- drivers/perf/apple_m1_cpu_pmu.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/perf/apple_m1_cpu_pmu.c b/drivers/perf/apple_m1_cpu_pmu.c index 979a7c2b4f5693..03e1e81b94b9bd 100644 --- a/drivers/perf/apple_m1_cpu_pmu.c +++ b/drivers/perf/apple_m1_cpu_pmu.c @@ -559,9 +559,23 @@ static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu) return m1_pmu_init(cpu_pmu); } +static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) +{ + cpu_pmu->name = "apple_blizzard_pmu"; + return m1_pmu_init(cpu_pmu); +} + +static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) +{ + cpu_pmu->name = "apple_avalanche_pmu"; + return m1_pmu_init(cpu_pmu); +} + static const struct of_device_id m1_pmu_of_device_ids[] = { { .compatible = "apple,icestorm-pmu", .data = m1_pmu_ice_init, }, { .compatible = "apple,firestorm-pmu", .data = m1_pmu_fire_init, }, + { .compatible = "apple,blizzard-pmu", .data = m2_pmu_blizzard_init, }, + { .compatible = "apple,avalanche-pmu", .data = m2_pmu_avalanche_init, }, { }, }; MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids); From c7906f5f8fba8cdda64f97ebda2cbbe2f3e5b700 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 9 Sep 2022 19:52:19 +0200 Subject: [PATCH 044/635] PCI: apple: Add depends on ARM64_PAGE_SHIFT = 14 The iommu on Apple's M1 and M2 supports only a page size of 16kB and is mandatory for PCIe devices. The PCI controller itself is not affeccted by the CPU page size the page size mismatch devices are renderer useless due to non-working DMA. While the the iommu prints a warning in this scenario it seems a common and hard to debug problem. Signed-off-by: Janne Grunau --- drivers/pci/controller/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 42654035654a1f..d0eb73092d2c44 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -326,6 +326,7 @@ config PCIE_APPLE depends on ARCH_APPLE || COMPILE_TEST depends on OF depends on PCI_MSI + depends on ARM64_PAGE_SHIFT = 14 || COMPILE_TEST select PCI_HOST_COMMON help Say Y here if you want to enable PCIe controller support on Apple From cee25d7284793679d1ed3d4d532c49fbe9a3927a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 24 Oct 2022 22:49:17 +0900 Subject: [PATCH 045/635] drm/simpledrm: Make backlight support a Kconfig option And make it mutually exclusive with DCP. Signed-off-by: Hector Martin --- drivers/gpu/drm/tiny/Kconfig | 7 +++++++ drivers/gpu/drm/tiny/simpledrm.c | 11 ++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/tiny/Kconfig b/drivers/gpu/drm/tiny/Kconfig index 45d09dc38a0aa2..097c2fa5979ac8 100644 --- a/drivers/gpu/drm/tiny/Kconfig +++ b/drivers/gpu/drm/tiny/Kconfig @@ -97,6 +97,13 @@ config DRM_SIMPLEDRM On x86 BIOS or UEFI systems, you should also select SYSFB_SIMPLEFB to use UEFI and VESA framebuffers. +config DRM_SIMPLEDRM_BACKLIGHT + bool "Backlight support for simpledrm" + depends on DRM_SIMPLEDRM && !DRM_APPLE + select BACKLIGHT_CLASS_DEVICE + help + Enable backlight support for simpledrm. + config TINYDRM_HX8357D tristate "DRM support for HX8357D display panels" depends on DRM && SPI diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c index 2ac85eefa8bcc3..800de38aa31bef 100644 --- a/drivers/gpu/drm/tiny/simpledrm.c +++ b/drivers/gpu/drm/tiny/simpledrm.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only +#if defined CONFIG_DRM_SIMPLEDRM_BACKLIGHT #include +#endif #include #include #include @@ -244,8 +246,10 @@ struct simpledrm_device { struct drm_crtc crtc; struct drm_encoder encoder; struct drm_connector connector; +#if defined CONFIG_DRM_SIMPLEDRM_BACKLIGHT /* backlight */ struct backlight_device *backlight; +#endif }; static struct simpledrm_device *simpledrm_device_of_dev(struct drm_device *dev) @@ -556,6 +560,7 @@ static enum drm_mode_status simpledrm_crtc_helper_mode_valid(struct drm_crtc *cr return drm_crtc_helper_mode_valid_fixed(crtc, mode, &sdev->mode); } +#if defined CONFIG_DRM_SIMPLEDRM_BACKLIGHT static void simpledrm_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) { @@ -575,6 +580,7 @@ static void simpledrm_crtc_helper_atomic_disable(struct drm_crtc *crtc, if (sdev->backlight) backlight_disable(sdev->backlight); } +#endif /* * The CRTC is always enabled. Screen updates are performed by @@ -584,8 +590,10 @@ static void simpledrm_crtc_helper_atomic_disable(struct drm_crtc *crtc, static const struct drm_crtc_helper_funcs simpledrm_crtc_helper_funcs = { .mode_valid = simpledrm_crtc_helper_mode_valid, .atomic_check = drm_crtc_helper_atomic_check, +#if defined CONFIG_DRM_SIMPLEDRM_BACKLIGHT .atomic_enable = simpledrm_crtc_helper_atomic_enable, .atomic_disable = simpledrm_crtc_helper_atomic_disable, +#endif }; static const struct drm_crtc_funcs simpledrm_crtc_funcs = { @@ -674,10 +682,11 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, * Hardware settings */ +#if defined CONFIG_DRM_SIMPLEDRM_BACKLIGHT sdev->backlight = devm_of_find_backlight(&pdev->dev); if (IS_ERR(sdev->backlight)) sdev->backlight = NULL; - +#endif ret = simpledrm_device_init_clocks(sdev); if (ret) return ERR_PTR(ret); From 26b75ce22683b1258ad2cbf38eb7ea3bf46eabb7 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 25 Oct 2022 01:12:17 +0900 Subject: [PATCH 046/635] firmware_loader: Add /lib/firmware/vendor path Signed-off-by: Hector Martin --- drivers/base/firmware_loader/main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index 017c4cdb219eb1..63b751b191419c 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -470,6 +470,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, static char fw_path_para[256]; static const char * const fw_path[] = { fw_path_para, + "/lib/firmware/vendor/" UTS_RELEASE, + "/lib/firmware/vendor", "/lib/firmware/updates/" UTS_RELEASE, "/lib/firmware/updates", "/lib/firmware/" UTS_RELEASE, From 43d0300c9f42772dda31f52703c37fcf5add6f36 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Wed, 2 Nov 2022 01:50:55 +0900 Subject: [PATCH 047/635] i2c: pasemi: Improve timeout handling and error recovery The hardware (supposedly) has a 25ms timeout for clock stretching, but the driver uses a 10ms timeout, which is too low (and actually gets hit with the tipd controllers on Apple Silicon machines sporadically). Increase the timeout to 100ms, which should be plenty, and then add handling for all the missing error condition, and better recovery in pasemi_smb_clear(). Since this needs a bunch more bit defines, take the change to switch to bitfield.h macros, which is much more readable. Signed-off-by: Hector Martin --- drivers/i2c/busses/i2c-pasemi-core.c | 105 ++++++++++++++++++++------- 1 file changed, 78 insertions(+), 27 deletions(-) diff --git a/drivers/i2c/busses/i2c-pasemi-core.c b/drivers/i2c/busses/i2c-pasemi-core.c index 7d54a9f34c74b5..457a4095756822 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.c +++ b/drivers/i2c/busses/i2c-pasemi-core.c @@ -5,6 +5,7 @@ * SMBus host driver for PA Semi PWRficient */ +#include #include #include #include @@ -26,21 +27,31 @@ #define REG_REV 0x28 /* Register defs */ -#define MTXFIFO_READ 0x00000400 -#define MTXFIFO_STOP 0x00000200 -#define MTXFIFO_START 0x00000100 -#define MTXFIFO_DATA_M 0x000000ff - -#define MRXFIFO_EMPTY 0x00000100 -#define MRXFIFO_DATA_M 0x000000ff - -#define SMSTA_XEN 0x08000000 -#define SMSTA_MTN 0x00200000 - -#define CTL_MRR 0x00000400 -#define CTL_MTR 0x00000200 -#define CTL_EN 0x00000800 -#define CTL_CLK_M 0x000000ff +#define MTXFIFO_READ BIT(10) +#define MTXFIFO_STOP BIT(9) +#define MTXFIFO_START BIT(8) +#define MTXFIFO_DATA_M GENMASK(7, 0) + +#define MRXFIFO_EMPTY BIT(8) +#define MRXFIFO_DATA_M GENMASK(7, 0) + +#define SMSTA_XIP BIT(28) +#define SMSTA_XEN BIT(27) +#define SMSTA_JMD BIT(25) +#define SMSTA_JAM BIT(24) +#define SMSTA_MTO BIT(23) +#define SMSTA_MTA BIT(22) +#define SMSTA_MTN BIT(21) +#define SMSTA_MRNE BIT(19) +#define SMSTA_MTE BIT(16) +#define SMSTA_TOM BIT(6) + +#define CTL_EN BIT(11) +#define CTL_MRR BIT(10) +#define CTL_MTR BIT(9) +#define CTL_CLK_M GENMASK(7, 0) + +#define TRANSFER_TIMEOUT_MS 100 static inline void reg_write(struct pasemi_smbus *smbus, int reg, int val) { @@ -70,23 +81,45 @@ static void pasemi_reset(struct pasemi_smbus *smbus) reinit_completion(&smbus->irq_completion); } -static void pasemi_smb_clear(struct pasemi_smbus *smbus) +static int pasemi_smb_clear(struct pasemi_smbus *smbus) { unsigned int status; + int timeout = TRANSFER_TIMEOUT_MS; status = reg_read(smbus, REG_SMSTA); + + /* First wait for the bus to go idle */ + while ((status & (SMSTA_XIP | SMSTA_JAM)) && timeout--) { + msleep(1); + status = reg_read(smbus, REG_SMSTA); + } + + if (timeout < 0) { + dev_warn(smbus->dev, "Bus is still stuck (status 0x%08x)\n", status); + return -EIO; + } + + /* If any badness happened or there is data in the FIFOs, reset the FIFOs */ + if ((status & (SMSTA_MRNE | SMSTA_JMD | SMSTA_MTO | SMSTA_TOM | SMSTA_MTN | SMSTA_MTA)) || + !(status & SMSTA_MTE)) + pasemi_reset(smbus); + + /* Clear the flags */ reg_write(smbus, REG_SMSTA, status); + + return 0; } static int pasemi_smb_waitready(struct pasemi_smbus *smbus) { - int timeout = 100; + int timeout = TRANSFER_TIMEOUT_MS; unsigned int status; if (smbus->use_irq) { reinit_completion(&smbus->irq_completion); - reg_write(smbus, REG_IMASK, SMSTA_XEN | SMSTA_MTN); - wait_for_completion_timeout(&smbus->irq_completion, msecs_to_jiffies(100)); + /* XEN should be set when a transaction terminates, whether due to error or not */ + reg_write(smbus, REG_IMASK, SMSTA_XEN); + wait_for_completion_timeout(&smbus->irq_completion, msecs_to_jiffies(timeout)); reg_write(smbus, REG_IMASK, 0); status = reg_read(smbus, REG_SMSTA); } else { @@ -97,16 +130,32 @@ static int pasemi_smb_waitready(struct pasemi_smbus *smbus) } } - /* Got NACK? */ - if (status & SMSTA_MTN) - return -ENXIO; + /* Controller timeout? */ + if (status & SMSTA_TOM) { + dev_warn(smbus->dev, "Controller timeout, status 0x%08x\n", status); + return -EIO; + } - if (timeout < 0) { - dev_warn(smbus->dev, "Timeout, status 0x%08x\n", status); - reg_write(smbus, REG_SMSTA, status); + /* Peripheral timeout? */ + if (status & SMSTA_MTO) { + dev_warn(smbus->dev, "Peripheral timeout, status 0x%08x\n", status); return -ETIME; } + /* Still stuck in a transaction? */ + if (status & SMSTA_XIP) { + dev_warn(smbus->dev, "Bus stuck, status 0x%08x\n", status); + return -EIO; + } + + /* Arbitration loss? */ + if (status & SMSTA_MTA) + return -EBUSY; + + /* Got NACK? */ + if (status & SMSTA_MTN) + return -ENXIO; + /* Clear XEN */ reg_write(smbus, REG_SMSTA, SMSTA_XEN); @@ -167,7 +216,8 @@ static int pasemi_i2c_xfer(struct i2c_adapter *adapter, struct pasemi_smbus *smbus = adapter->algo_data; int ret, i; - pasemi_smb_clear(smbus); + if (pasemi_smb_clear(smbus)) + return -EIO; ret = 0; @@ -190,7 +240,8 @@ static int pasemi_smb_xfer(struct i2c_adapter *adapter, addr <<= 1; read_flag = read_write == I2C_SMBUS_READ; - pasemi_smb_clear(smbus); + if (pasemi_smb_clear(smbus)) + return -EIO; switch (size) { case I2C_SMBUS_QUICK: From 9f6f28d7fe18e12199b87ab44b51e5f7c13375c5 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Wed, 2 Nov 2022 02:07:16 +0900 Subject: [PATCH 048/635] usb: typec: tipd: Be more verbose about errors Make sure to print out error codes and log exactly what packet sizes were received in case of a mismatch. This is very useful for debugging badness. Signed-off-by: Hector Martin --- drivers/usb/typec/tipd/core.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 485b90c13078a9..324c18ef65a93c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -130,11 +130,15 @@ tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) return regmap_raw_read(tps->regmap, reg, val, len); ret = regmap_raw_read(tps->regmap, reg, data, len + 1); - if (ret) + if (ret) { + dev_err(tps->dev, "regmap_raw_read returned %d\n", ret); return ret; + } - if (data[0] < len) + if (data[0] < len) { + dev_err(tps->dev, "expected %zu bytes, got %d\n", len, data[0]); return -EIO; + } memcpy(val, &data[1], len); return 0; @@ -423,7 +427,7 @@ static bool tps6598x_read_status(struct tps6598x *tps, u32 *status) ret = tps6598x_read32(tps, TPS_REG_STATUS, status); if (ret) { - dev_err(tps->dev, "%s: failed to read status\n", __func__); + dev_err(tps->dev, "%s: failed to read status: %d\n", __func__, ret); return false; } trace_tps6598x_status(*status); From 271457a92d72e9326232a5756231b801e156e528 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Nov 2022 19:09:49 +0900 Subject: [PATCH 049/635] Add 'asahi' localversion to 05-asahi.localversion Signed-off-by: Hector Martin --- localversion.05-asahi | 1 + 1 file changed, 1 insertion(+) create mode 100644 localversion.05-asahi diff --git a/localversion.05-asahi b/localversion.05-asahi new file mode 100644 index 00000000000000..6742ba757f12ac --- /dev/null +++ b/localversion.05-asahi @@ -0,0 +1 @@ +-asahi From 946c11897a974b5c9918837bc6fd5da84f1d6e7e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Wed, 23 Nov 2022 02:41:37 +0900 Subject: [PATCH 050/635] of: HACK: disable backlight supplier binding if not needed This is just to get -edge kernels with DCP and the backlight stuff compiled out working with the same DTs as regular ones, and avoid blocking here. Will go away once the simpledrm backlight stuff goes away when DCP is stable. Signed-off-by: Hector Martin H# with '#' will be ignored, and an empty message aborts the commit. --- drivers/of/property.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/of/property.c b/drivers/of/property.c index ddc75cd50825e9..c4080d86046e38 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -1353,7 +1353,9 @@ static const struct supplier_bindings of_supplier_bindings[] = { { .parse_prop = parse_pwms, }, { .parse_prop = parse_resets, }, { .parse_prop = parse_leds, }, +#ifdef CONFIG_DRM_SIMPLEDRM_BACKLIGHT { .parse_prop = parse_backlight, }, +#endif { .parse_prop = parse_gpio_compat, }, { .parse_prop = parse_interrupts, }, { .parse_prop = parse_regulators, }, From 9d2b29d75316d980a8e3c391a4fb033aea39d327 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 14 Jan 2023 13:01:31 +0900 Subject: [PATCH 051/635] i2c: pasemi: Enable the unjam machine The I2C bus can get stuck under some conditions (desync between controller and device). The pasemi controllers include an unjam feature that is enabled on reset, but was being disabled by the driver. Keep it enabled by explicitly setting the UJM bit in the CTL register. This should help recover the bus from certain conditions, which would otherwise remain stuck forever. Signed-off-by: Hector Martin --- drivers/i2c/busses/i2c-pasemi-core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/i2c/busses/i2c-pasemi-core.c b/drivers/i2c/busses/i2c-pasemi-core.c index 457a4095756822..85c9b12ead401b 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.c +++ b/drivers/i2c/busses/i2c-pasemi-core.c @@ -49,6 +49,7 @@ #define CTL_EN BIT(11) #define CTL_MRR BIT(10) #define CTL_MTR BIT(9) +#define CTL_UJM BIT(8) #define CTL_CLK_M GENMASK(7, 0) #define TRANSFER_TIMEOUT_MS 100 @@ -72,7 +73,7 @@ static inline int reg_read(struct pasemi_smbus *smbus, int reg) static void pasemi_reset(struct pasemi_smbus *smbus) { - u32 val = (CTL_MTR | CTL_MRR | (smbus->clk_div & CTL_CLK_M)); + u32 val = (CTL_MTR | CTL_MRR | CTL_UJM | (smbus->clk_div & CTL_CLK_M)); if (smbus->hw_rev >= 6) val |= CTL_EN; From 1b0ceb53d70d8aa5b3700181d0d575b24a02c3f8 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 15 Jan 2023 20:29:40 +0900 Subject: [PATCH 052/635] i2c: pasemi: Log bus reset causes Signed-off-by: Hector Martin --- drivers/i2c/busses/i2c-pasemi-core.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/busses/i2c-pasemi-core.c b/drivers/i2c/busses/i2c-pasemi-core.c index 85c9b12ead401b..a76f063d18cf2e 100644 --- a/drivers/i2c/busses/i2c-pasemi-core.c +++ b/drivers/i2c/busses/i2c-pasemi-core.c @@ -21,6 +21,7 @@ /* Register offsets */ #define REG_MTXFIFO 0x00 #define REG_MRXFIFO 0x04 +#define REG_XFSTA 0x0c #define REG_SMSTA 0x14 #define REG_IMASK 0x18 #define REG_CTL 0x1c @@ -84,7 +85,7 @@ static void pasemi_reset(struct pasemi_smbus *smbus) static int pasemi_smb_clear(struct pasemi_smbus *smbus) { - unsigned int status; + unsigned int status, xfstatus; int timeout = TRANSFER_TIMEOUT_MS; status = reg_read(smbus, REG_SMSTA); @@ -95,15 +96,21 @@ static int pasemi_smb_clear(struct pasemi_smbus *smbus) status = reg_read(smbus, REG_SMSTA); } + xfstatus = reg_read(smbus, REG_XFSTA); + if (timeout < 0) { - dev_warn(smbus->dev, "Bus is still stuck (status 0x%08x)\n", status); + dev_warn(smbus->dev, "Bus is still stuck (status 0x%08x xfstatus 0x%08x)\n", + status, xfstatus); return -EIO; } /* If any badness happened or there is data in the FIFOs, reset the FIFOs */ if ((status & (SMSTA_MRNE | SMSTA_JMD | SMSTA_MTO | SMSTA_TOM | SMSTA_MTN | SMSTA_MTA)) || - !(status & SMSTA_MTE)) + !(status & SMSTA_MTE)) { + dev_warn(smbus->dev, "Issuing reset due to status 0x%08x (xfstatus 0x%08x)\n", + status, xfstatus); pasemi_reset(smbus); + } /* Clear the flags */ reg_write(smbus, REG_SMSTA, status); From c269595f22725f58750efb4246b1a4856c1eab64 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Sun, 7 Nov 2021 11:21:19 +0100 Subject: [PATCH 053/635] dt-bindings: usb: Add Apple dwc3 bindings Apple Silicon SoCs such as the M1 have multiple USB controllers based on the Synopsys DesignWare USB3 controller. References to the ATC PHY required for SuperSpeed are left out for now until support has been upstreamed as well. Signed-off-by: Sven Peter --- .../devicetree/bindings/usb/apple,dwc3.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Documentation/devicetree/bindings/usb/apple,dwc3.yaml diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml new file mode 100644 index 00000000000000..fb3b3489e6b263 --- /dev/null +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/usb/apple,dwc3.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon DWC3 USB controller + +maintainers: + - Sven Peter + +description: + On Apple Silicon SoCs such as the M1 each Type-C port has a corresponding + USB controller based on the Synopsys DesignWare USB3 controller. + + The common content of this binding is defined in snps,dwc3.yaml. + +allOf: + - $ref: snps,dwc3.yaml# + +select: + properties: + compatible: + contains: + const: apple,dwc3 + required: + - compatible + +properties: + compatible: + items: + - enum: + - apple,t8103-dwc3 + - apple,t6000-dwc3 + - const: apple,dwc3 + - const: snps,dwc3 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + +examples: + - | + #include + #include + + usb@82280000 { + compatible = "apple,t8103-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x82280000 0x10000>; + interrupts = ; + + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + }; From 9765fd1ea82ce3935c158b3a13bed36fe8f8f9a1 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 30 Nov 2022 22:10:59 +0100 Subject: [PATCH 054/635] usb: dwc3: Add support for Apple DWC3 As mad as it sounds, the dwc3 controller present on the Apple M1 must be reset and reinitialized whenever a device is unplugged from the root port or when the PHY mode is changed. This is required for at least the following reasons: - The USB2 D+/D- lines are connected through a stateful eUSB2 repeater which in turn is controlled by a variant of the TI TPS6598x USB PD chip. When the USB PD controller detects a hotplug event it resets the eUSB2 repeater. Afterwards, no new device is recognized before the DWC3 core and PHY are reset as well because the eUSB2 repeater and the PHY/dwc3 block disagree about the current state. - It's possible to completely break the dwc3 controller by switching it to device mode and unplugging the cable at just the wrong time. If this happens dwc3 behaves as if no device is connected. CORESOFTRESET will also never clear after it has been set. The only workaround is to trigger a hard reset of the entire dwc3 core with its external reset line. - Whenever the PHY mode is changed (to e.g. transition to DisplayPort alternate mode or USB4) dwc3 has to be shutdown and reinitialized. Otherwise the Type-C port will not be useable until the entire SoC has been reset. All of this can be easily worked around by respecting transitions to USB_ROLE_NONE and making sure the external reset line is asserted when switching roles. Signed-off-by: Sven Peter --- drivers/usb/dwc3/core.c | 56 ++++++++++++++++++++++++++++++++++++++--- drivers/usb/dwc3/core.h | 3 +++ drivers/usb/dwc3/drd.c | 11 +++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 476b6361851166..fdc0d4718b275e 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -116,6 +116,9 @@ void dwc3_set_prtcap(struct dwc3 *dwc, u32 mode) dwc->current_dr_role = mode; } +static void dwc3_core_exit(struct dwc3 *dwc); +static int dwc3_core_init_for_resume(struct dwc3 *dwc); + static void __dwc3_set_mode(struct work_struct *work) { struct dwc3 *dwc = work_to_dwc(work); @@ -134,7 +137,7 @@ static void __dwc3_set_mode(struct work_struct *work) if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_OTG) dwc3_otg_update(dwc, 0); - if (!desired_dr_role) + if (!desired_dr_role && !dwc->role_switch_reset_quirk) goto out; if (desired_dr_role == dwc->current_dr_role) @@ -162,13 +165,32 @@ static void __dwc3_set_mode(struct work_struct *work) break; } + if (dwc->role_switch_reset_quirk) { + if (dwc->current_dr_role) { + dwc->current_dr_role = 0; + dwc3_core_exit(dwc); + } + + if (desired_dr_role) { + ret = dwc3_core_init_for_resume(dwc); + if (ret) { + dev_err(dwc->dev, + "failed to reinitialize core\n"); + goto out; + } + } else { + goto out; + } + } + /* * When current_dr_role is not set, there's no role switching. * Only perform GCTL.CoreSoftReset when there's DRD role switching. */ - if (dwc->current_dr_role && ((DWC3_IP_IS(DWC3) || + if (dwc->role_switch_reset_quirk || + (dwc->current_dr_role && ((DWC3_IP_IS(DWC3) || DWC3_VER_IS_PRIOR(DWC31, 190A)) && - desired_dr_role != DWC3_GCTL_PRTCAP_OTG)) { + desired_dr_role != DWC3_GCTL_PRTCAP_OTG))) { reg = dwc3_readl(dwc->regs, DWC3_GCTL); reg |= DWC3_GCTL_CORESOFTRESET; dwc3_writel(dwc->regs, DWC3_GCTL, reg); @@ -1405,6 +1427,18 @@ static int dwc3_core_init_mode(struct dwc3 *dwc) ret = dwc3_drd_init(dwc); if (ret) return dev_err_probe(dev, ret, "failed to initialize dual-role\n"); + + /* + * If the role switch reset quirk is required the first role + * switch notification will initialize the core such that we + * have to shut it down here. Make sure that the __dwc3_set_mode + * queued by dwc3_drd_init has completed before since it + * may still try to access MMIO. + */ + if (dwc->role_switch_reset_quirk) { + flush_work(&dwc->drd_work); + dwc3_core_exit(dwc); + } break; default: dev_err(dev, "Unsupported mode of operation %d\n", dwc->dr_mode); @@ -1854,6 +1888,20 @@ static int dwc3_probe(struct platform_device *pdev) goto put_usb_psy; } } + + if (of_device_is_compatible(dev->of_node, "apple,dwc3")) { + if (!IS_ENABLED(CONFIG_USB_ROLE_SWITCH) || + !IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE)) { + dev_err(dev, + "Apple DWC3 requires role switch support.\n" + ); + ret = -EINVAL; + goto put_usb_psy; + } + + dwc->dr_mode = USB_DR_MODE_OTG; + dwc->role_switch_reset_quirk = true; + } } ret = reset_control_deassert(dwc->reset); @@ -1996,7 +2044,6 @@ static int dwc3_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM static int dwc3_core_init_for_resume(struct dwc3 *dwc) { int ret; @@ -2023,6 +2070,7 @@ static int dwc3_core_init_for_resume(struct dwc3 *dwc) return ret; } +#ifdef CONFIG_PM static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg) { unsigned long flags; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 4743e918dcafa9..d56bbfb7171d69 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -1110,6 +1110,7 @@ struct dwc3_scratchpad_array { * 3 - Reserved * @dis_metastability_quirk: set to disable metastability quirk. * @dis_split_quirk: set to disable split boundary. + * @role_switch_reset_quirk: set to force reinitialization after any role switch * @imod_interval: set the interrupt moderation interval in 250ns * increments or 0 to disable. * @max_cfg_eps: current max number of IN eps used across all USB configs. @@ -1328,6 +1329,8 @@ struct dwc3 { unsigned dis_split_quirk:1; unsigned async_callbacks:1; + unsigned role_switch_reset_quirk:1; + u16 imod_interval; int max_cfg_eps; diff --git a/drivers/usb/dwc3/drd.c b/drivers/usb/dwc3/drd.c index 039bf241769afb..e30f048f3ee664 100644 --- a/drivers/usb/dwc3/drd.c +++ b/drivers/usb/dwc3/drd.c @@ -461,6 +461,9 @@ static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, break; } + if (dwc->role_switch_reset_quirk && role == USB_ROLE_NONE) + mode = 0; + dwc3_set_mode(dwc, mode); return 0; } @@ -489,6 +492,10 @@ static enum usb_role dwc3_usb_role_switch_get(struct usb_role_switch *sw) role = USB_ROLE_DEVICE; break; } + + if (dwc->role_switch_reset_quirk && !dwc->current_dr_role) + role = USB_ROLE_NONE; + spin_unlock_irqrestore(&dwc->lock, flags); return role; } @@ -499,7 +506,9 @@ static int dwc3_setup_role_switch(struct dwc3 *dwc) u32 mode; dwc->role_switch_default_mode = usb_get_role_switch_default_mode(dwc->dev); - if (dwc->role_switch_default_mode == USB_DR_MODE_HOST) { + if (dwc->role_switch_reset_quirk) { + mode = 0; + } else if (dwc->role_switch_default_mode == USB_DR_MODE_HOST) { mode = DWC3_GCTL_PRTCAP_HOST; } else { dwc->role_switch_default_mode = USB_DR_MODE_PERIPHERAL; From f78b8daefe6ae48712d3b6395ccc01682be73b59 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Feb 2022 12:17:58 -0700 Subject: [PATCH 055/635] apple-nvme: defer cache flushes by a specified amount Cache flushes on the M1 nvme are really slow, taking 17-18 msec to complete. This can slow down workloads considerably, pure random writes end up being bound by the flush latency and hence run at 55-60 IOPS. Add a deferred flush work around to provide better performance, at a minimal risk. By default, flushes are delayed at most 1 second, but this is configurable. With this work-around, a pure random write workload runs at ~12K IOPS rather than 56 IOPS. Signed-off-by: Jens Axboe --- drivers/nvme/host/apple.c | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index b317ce6c4ec318..be86b0bedb5dd8 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -195,8 +195,20 @@ struct apple_nvme { int irq; spinlock_t lock; + + /* + * Delayed cache flush handling state + */ + struct nvme_ns *flush_ns; + unsigned long flush_interval; + unsigned long last_flush; + struct delayed_work flush_dwork; }; +unsigned int flush_interval = 1000; +module_param(flush_interval, uint, 0644); +MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); + static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); @@ -729,6 +741,26 @@ static int apple_nvme_remove_sq(struct apple_nvme *anv) return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); } +static bool apple_nvme_delayed_flush(struct apple_nvme *anv, struct nvme_ns *ns, + struct request *req) +{ + if (!anv->flush_interval || req_op(req) != REQ_OP_FLUSH) + return false; + if (delayed_work_pending(&anv->flush_dwork)) + return true; + if (time_before(jiffies, anv->last_flush + anv->flush_interval)) { + kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &anv->flush_dwork, + anv->flush_interval); + if (WARN_ON_ONCE(anv->flush_ns && anv->flush_ns != ns)) + goto out; + anv->flush_ns = ns; + return true; + } +out: + anv->last_flush = jiffies; + return false; +} + static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -764,6 +796,12 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, } nvme_start_request(req); + + if (apple_nvme_delayed_flush(anv, ns, req)) { + blk_mq_complete_request(req); + return BLK_STS_OK; + } + apple_nvme_submit_cmd(q, cmnd); return BLK_STS_OK; @@ -1387,6 +1425,28 @@ static void devm_apple_nvme_mempool_destroy(void *data) mempool_destroy(data); } +static void apple_nvme_flush_work(struct work_struct *work) +{ + struct nvme_command c = { }; + struct apple_nvme *anv; + struct nvme_ns *ns; + int err; + + anv = container_of(work, struct apple_nvme, flush_dwork.work); + ns = anv->flush_ns; + if (WARN_ON_ONCE(!ns)) + return; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(anv->flush_ns->head->ns_id); + err = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (err) { + dev_err(anv->dev, "Deferred flush failed: %d\n", err); + } else { + anv->last_flush = jiffies; + } +} + static int apple_nvme_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1521,6 +1581,14 @@ static int apple_nvme_probe(struct platform_device *pdev) goto put_dev; } + if (flush_interval) { + anv->flush_interval = msecs_to_jiffies(flush_interval); + anv->flush_ns = NULL; + anv->last_flush = jiffies - anv->flush_interval; + } + + INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -1554,6 +1622,7 @@ static void apple_nvme_shutdown(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); if (apple_rtkit_is_running(anv->rtk)) apple_rtkit_shutdown(anv->rtk); From 7ce29162846d3c00a1f0d13b5639d453a5664621 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 27 Jun 2022 21:47:43 +0900 Subject: [PATCH 056/635] apple-nvme: Release power domains when probe fails Signed-off-by: Hector Martin --- drivers/nvme/host/apple.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index be86b0bedb5dd8..a3d676677cb2da 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1595,6 +1595,7 @@ static int apple_nvme_probe(struct platform_device *pdev) return 0; put_dev: + apple_nvme_detach_genpd(anv); put_device(anv->dev); return ret; } From 2639296b62678ecc4d475edad328134798050063 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 02:56:59 +0900 Subject: [PATCH 057/635] apple-nvme: Support coprocessors left idle iBoot on at least some firmwares/machines leaves ANS2 running, requiring a wake command instead of a CPU boot (and if we reset ANS2 in that state, everything breaks). Only stop the CPU if RTKit was running, and only do the reset dance if the CPU is stopped. Normal shutdown handoff: - RTKit not yet running - CPU detected not running - Reset - CPU powerup - RTKit boot wait ANS2 left running/idle: - RTKit not yet running - CPU detected running - RTKit wake message Sleep/resume cycle: - RTKit shutdown - CPU stopped - (sleep here) - CPU detected not running - Reset - CPU powerup - RTKit boot wait Shutdown or device removal: - RTKit shutdown - CPU stopped Therefore, the CPU running bit serves as a consistent flag of whether the coprocessor is fully stopped or just idle. Signed-off-by: Hector Martin --- drivers/nvme/host/apple.c | 53 ++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index a3d676677cb2da..3c81a5dddb559b 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1048,25 +1048,37 @@ static void apple_nvme_reset_work(struct work_struct *work) ret = apple_rtkit_shutdown(anv->rtk); if (ret) goto out; + + writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); } - writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + /* + * Only do the soft-reset if the CPU is not running, which means either we + * or the previous stage shut it down cleanly. + */ + if (!(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & + APPLE_ANS_COPROC_CPU_CONTROL_RUN)) { - ret = reset_control_assert(anv->reset); - if (ret) - goto out; + ret = reset_control_assert(anv->reset); + if (ret) + goto out; - ret = apple_rtkit_reinit(anv->rtk); - if (ret) - goto out; + ret = apple_rtkit_reinit(anv->rtk); + if (ret) + goto out; - ret = reset_control_deassert(anv->reset); - if (ret) - goto out; + ret = reset_control_deassert(anv->reset); + if (ret) + goto out; + + writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN, + anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + + ret = apple_rtkit_boot(anv->rtk); + } else { + ret = apple_rtkit_wake(anv->rtk); + } - writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN, - anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); - ret = apple_rtkit_boot(anv->rtk); if (ret) { dev_err(anv->dev, "ANS did not boot"); goto out; @@ -1611,9 +1623,12 @@ static int apple_nvme_remove(struct platform_device *pdev) apple_nvme_disable(anv, true); nvme_uninit_ctrl(&anv->ctrl); - if (apple_rtkit_is_running(anv->rtk)) + if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); + writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + } + apple_nvme_detach_genpd(anv); return 0; @@ -1625,8 +1640,11 @@ static void apple_nvme_shutdown(struct platform_device *pdev) flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); - if (apple_rtkit_is_running(anv->rtk)) + if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); + + writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + } } static int apple_nvme_resume(struct device *dev) @@ -1643,10 +1661,11 @@ static int apple_nvme_suspend(struct device *dev) apple_nvme_disable(anv, true); - if (apple_rtkit_is_running(anv->rtk)) + if (apple_rtkit_is_running(anv->rtk)) { ret = apple_rtkit_shutdown(anv->rtk); - writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); + } return ret; } From 21e345394f08149274d2a3917ad130b1d1fef7dd Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 12 Dec 2021 11:46:33 +0900 Subject: [PATCH 058/635] dt-bindings: spi: apple,spi: Add binding for Apple SPI controllers The Apple SPI controller is present in SoCs such as the M1 (t8103) and M1 Pro/Max (t600x). This controller uses one IRQ and one clock, and doesn't need any special properties, so the binding is trivial. Signed-off-by: Hector Martin --- .../devicetree/bindings/spi/apple,spi.yaml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 Documentation/devicetree/bindings/spi/apple,spi.yaml diff --git a/Documentation/devicetree/bindings/spi/apple,spi.yaml b/Documentation/devicetree/bindings/spi/apple,spi.yaml new file mode 100644 index 00000000000000..bcbdc8943e92a3 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/apple,spi.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/apple,spi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple ARM SoC SPI controller + +allOf: + - $ref: "spi-controller.yaml#" + +maintainers: + - Hector Martin + +properties: + compatible: + items: + - enum: + - apple,t8103-spi + - apple,t6000-spi + - const: apple,spi + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - interrupts + - '#address-cells' + - '#size-cells' + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + spi: spi@39b104000 { + compatible = "apple,t6000-spi", "apple,spi"; + reg = <0x3 0x9b104000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk>; + }; + }; From dfb0bd4a56714916a20a80a68042f02f208ab39a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 9 Dec 2021 21:55:49 +0900 Subject: [PATCH 059/635] spi: apple: Add driver for Apple SPI controller This SPI controller is present in Apple SoCs such as the M1 (t8103) and M1 Pro/Max (t600x). It is a relatively straightforward design with two 16-entry FIFOs, arbitrary transfer sizes (up to 2**32 - 1) and fully configurable word size up to 32 bits. It supports one hardware CS line which can also be driven via the pinctrl/GPIO driver instead, if desired. TX and RX can be independently enabled. There are a surprising number of knobs for tweaking details of the transfer, most of which we do not use right now. Hardware CS control is available, but we haven't found a way to make it stay low across multiple logical transfers, so we just use software CS control for now. There is also a shared DMA offload coprocessor that can be used to handle larger transfers without requiring an IRQ every 8-16 words, but that feature depends on a bunch of scaffolding that isn't ready to be upstreamed yet, so leave it for later. The hardware shares some register bit definitions with spi-s3c24xx which suggests it has a shared legacy with Samsung SoCs, but it is too different to warrant sharing a driver. Signed-off-by: Hector Martin --- drivers/spi/Kconfig | 8 + drivers/spi/Makefile | 1 + drivers/spi/spi-apple.c | 544 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 553 insertions(+) create mode 100644 drivers/spi/spi-apple.c diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 47bbba04fe3aed..eec66011cdb000 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -79,6 +79,14 @@ config SPI_ALTERA_DFL Altera SPI master controller. The SPI master is connected to a SPI slave to Avalon bridge in a Intel MAX BMC. +config SPI_APPLE + tristate "Apple SoC SPI Controller platform driver" + depends on ARCH_APPLE || COMPILE_TEST + help + This enables support for the SPI controller present on + many Apple SoCs, including the t8103 (M1) and t600x + (M1 Pro/Max). + config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" depends on ATH79 || COMPILE_TEST diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index d87cf75bee6a45..a2ccfa030c4ce5 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_SPI_LOOPBACK_TEST) += spi-loopback-test.o obj-$(CONFIG_SPI_ALTERA) += spi-altera-platform.o obj-$(CONFIG_SPI_ALTERA_CORE) += spi-altera-core.o obj-$(CONFIG_SPI_ALTERA_DFL) += spi-altera-dfl.o +obj-$(CONFIG_SPI_APPLE) += spi-apple.o obj-$(CONFIG_SPI_AR934X) += spi-ar934x.o obj-$(CONFIG_SPI_ARMADA_3700) += spi-armada-3700.o obj-$(CONFIG_SPI_ASPEED_SMC) += spi-aspeed-smc.o diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c new file mode 100644 index 00000000000000..c483ad3f69ef5e --- /dev/null +++ b/drivers/spi/spi-apple.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Apple SoC SPI device driver + * + * Copyright The Asahi Linux Contributors + * + * Based on spi-sifive.c, Copyright 2018 SiFive, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_SPI_CTRL 0x000 +#define APPLE_SPI_CTRL_RUN BIT(0) +#define APPLE_SPI_CTRL_TX_RESET BIT(2) +#define APPLE_SPI_CTRL_RX_RESET BIT(3) + +#define APPLE_SPI_CFG 0x004 +#define APPLE_SPI_CFG_CPHA BIT(1) +#define APPLE_SPI_CFG_CPOL BIT(2) +#define APPLE_SPI_CFG_MODE GENMASK(6, 5) +#define APPLE_SPI_CFG_MODE_POLLED 0 +#define APPLE_SPI_CFG_MODE_IRQ 1 +#define APPLE_SPI_CFG_MODE_DMA 2 +#define APPLE_SPI_CFG_IE_RXCOMPLETE BIT(7) +#define APPLE_SPI_CFG_IE_TXRXTHRESH BIT(8) +#define APPLE_SPI_CFG_LSB_FIRST BIT(13) +#define APPLE_SPI_CFG_WORD_SIZE GENMASK(16, 15) +#define APPLE_SPI_CFG_WORD_SIZE_8B 0 +#define APPLE_SPI_CFG_WORD_SIZE_16B 1 +#define APPLE_SPI_CFG_WORD_SIZE_32B 2 +#define APPLE_SPI_CFG_FIFO_THRESH GENMASK(18, 17) +#define APPLE_SPI_CFG_FIFO_THRESH_8B 0 +#define APPLE_SPI_CFG_FIFO_THRESH_4B 1 +#define APPLE_SPI_CFG_FIFO_THRESH_1B 2 +#define APPLE_SPI_CFG_IE_TXCOMPLETE BIT(21) + +#define APPLE_SPI_STATUS 0x008 +#define APPLE_SPI_STATUS_RXCOMPLETE BIT(0) +#define APPLE_SPI_STATUS_TXRXTHRESH BIT(1) +#define APPLE_SPI_STATUS_TXCOMPLETE BIT(2) + +#define APPLE_SPI_PIN 0x00c +#define APPLE_SPI_PIN_KEEP_MOSI BIT(0) +#define APPLE_SPI_PIN_CS BIT(1) + +#define APPLE_SPI_TXDATA 0x010 +#define APPLE_SPI_RXDATA 0x020 +#define APPLE_SPI_CLKDIV 0x030 +#define APPLE_SPI_CLKDIV_MAX 0x7ff +#define APPLE_SPI_RXCNT 0x034 +#define APPLE_SPI_WORD_DELAY 0x038 +#define APPLE_SPI_TXCNT 0x04c + +#define APPLE_SPI_FIFOSTAT 0x10c +#define APPLE_SPI_FIFOSTAT_TXFULL BIT(4) +#define APPLE_SPI_FIFOSTAT_LEVEL_TX GENMASK(15, 8) +#define APPLE_SPI_FIFOSTAT_RXEMPTY BIT(20) +#define APPLE_SPI_FIFOSTAT_LEVEL_RX GENMASK(31, 24) + +#define APPLE_SPI_IE_XFER 0x130 +#define APPLE_SPI_IF_XFER 0x134 +#define APPLE_SPI_XFER_RXCOMPLETE BIT(0) +#define APPLE_SPI_XFER_TXCOMPLETE BIT(1) + +#define APPLE_SPI_IE_FIFO 0x138 +#define APPLE_SPI_IF_FIFO 0x13c +#define APPLE_SPI_FIFO_RXTHRESH BIT(4) +#define APPLE_SPI_FIFO_TXTHRESH BIT(5) +#define APPLE_SPI_FIFO_RXFULL BIT(8) +#define APPLE_SPI_FIFO_TXEMPTY BIT(9) +#define APPLE_SPI_FIFO_RXUNDERRUN BIT(16) +#define APPLE_SPI_FIFO_TXOVERFLOW BIT(17) + +#define APPLE_SPI_SHIFTCFG 0x150 +#define APPLE_SPI_SHIFTCFG_CLK_ENABLE BIT(0) +#define APPLE_SPI_SHIFTCFG_CS_ENABLE BIT(1) +#define APPLE_SPI_SHIFTCFG_AND_CLK_DATA BIT(8) +#define APPLE_SPI_SHIFTCFG_CS_AS_DATA BIT(9) +#define APPLE_SPI_SHIFTCFG_TX_ENABLE BIT(10) +#define APPLE_SPI_SHIFTCFG_RX_ENABLE BIT(11) +#define APPLE_SPI_SHIFTCFG_BITS GENMASK(21, 16) +#define APPLE_SPI_SHIFTCFG_OVERRIDE_CS BIT(24) + +#define APPLE_SPI_PINCFG 0x154 +#define APPLE_SPI_PINCFG_KEEP_CLK BIT(0) +#define APPLE_SPI_PINCFG_KEEP_CS BIT(1) +#define APPLE_SPI_PINCFG_KEEP_MOSI BIT(2) +#define APPLE_SPI_PINCFG_CLK_IDLE_VAL BIT(8) +#define APPLE_SPI_PINCFG_CS_IDLE_VAL BIT(9) +#define APPLE_SPI_PINCFG_MOSI_IDLE_VAL BIT(10) + +#define APPLE_SPI_DELAY_PRE 0x160 +#define APPLE_SPI_DELAY_POST 0x168 +#define APPLE_SPI_DELAY_ENABLE BIT(0) +#define APPLE_SPI_DELAY_NO_INTERBYTE BIT(1) +#define APPLE_SPI_DELAY_SET_SCK BIT(4) +#define APPLE_SPI_DELAY_SET_MOSI BIT(6) +#define APPLE_SPI_DELAY_SCK_VAL BIT(8) +#define APPLE_SPI_DELAY_MOSI_VAL BIT(12) + +#define APPLE_SPI_FIFO_DEPTH 16 + +/* + * The slowest refclock available is 24MHz, the highest divider is 0x7ff, + * the largest word size is 32 bits, the FIFO depth is 16, the maximum + * intra-word delay is 0xffff refclocks. So the maximum time a transfer + * cycle can take is: + * + * (0x7ff * 32 + 0xffff) * 16 / 24e6 Hz ~= 87ms + * + * Double it and round it up to 200ms for good measure. + */ +#define APPLE_SPI_TIMEOUT_MS 200 + +struct apple_spi { + void __iomem *regs; /* MMIO register address */ + struct clk *clk; /* bus clock */ + struct completion done; /* wake-up from interrupt */ +}; + +static inline void reg_write(struct apple_spi *spi, int offset, u32 value) +{ + writel_relaxed(value, spi->regs + offset); +} + +static inline u32 reg_read(struct apple_spi *spi, int offset) +{ + return readl_relaxed(spi->regs + offset); +} + +static inline void reg_mask(struct apple_spi *spi, int offset, u32 clear, u32 set) +{ + u32 val = reg_read(spi, offset); + + val &= ~clear; + val |= set; + reg_write(spi, offset, val); +} + +static void apple_spi_init(struct apple_spi *spi) +{ + /* Set CS high (inactive) and disable override and auto-CS */ + reg_write(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS); + reg_mask(spi, APPLE_SPI_SHIFTCFG, APPLE_SPI_SHIFTCFG_OVERRIDE_CS, 0); + reg_mask(spi, APPLE_SPI_PINCFG, APPLE_SPI_PINCFG_CS_IDLE_VAL, APPLE_SPI_PINCFG_KEEP_CS); + + /* Reset FIFOs */ + reg_write(spi, APPLE_SPI_CTRL, APPLE_SPI_CTRL_RX_RESET | APPLE_SPI_CTRL_TX_RESET); + + /* Configure defaults */ + reg_write(spi, APPLE_SPI_CFG, + FIELD_PREP(APPLE_SPI_CFG_FIFO_THRESH, APPLE_SPI_CFG_FIFO_THRESH_8B) | + FIELD_PREP(APPLE_SPI_CFG_MODE, APPLE_SPI_CFG_MODE_IRQ) | + FIELD_PREP(APPLE_SPI_CFG_WORD_SIZE, APPLE_SPI_CFG_WORD_SIZE_8B)); + + /* Disable IRQs */ + reg_write(spi, APPLE_SPI_IE_FIFO, 0); + reg_write(spi, APPLE_SPI_IE_XFER, 0); + + /* Disable delays */ + reg_write(spi, APPLE_SPI_DELAY_PRE, 0); + reg_write(spi, APPLE_SPI_DELAY_POST, 0); +} + +static int apple_spi_prepare_message(struct spi_controller *ctlr, struct spi_message *msg) +{ + struct apple_spi *spi = spi_controller_get_devdata(ctlr); + struct spi_device *device = msg->spi; + + u32 cfg = ((device->mode & SPI_CPHA ? APPLE_SPI_CFG_CPHA : 0) | + (device->mode & SPI_CPOL ? APPLE_SPI_CFG_CPOL : 0) | + (device->mode & SPI_LSB_FIRST ? APPLE_SPI_CFG_LSB_FIRST : 0)); + + /* Update core config */ + reg_mask(spi, APPLE_SPI_CFG, + APPLE_SPI_CFG_CPHA | APPLE_SPI_CFG_CPOL | APPLE_SPI_CFG_LSB_FIRST, cfg); + + return 0; +} + +static void apple_spi_set_cs(struct spi_device *device, bool is_high) +{ + struct apple_spi *spi = spi_controller_get_devdata(device->controller); + + reg_mask(spi, APPLE_SPI_PIN, APPLE_SPI_PIN_CS, is_high ? APPLE_SPI_PIN_CS : 0); +} + +static bool apple_spi_prep_transfer(struct apple_spi *spi, struct spi_transfer *t) +{ + u32 cr, fifo_threshold; + + /* Calculate and program the clock rate */ + cr = DIV_ROUND_UP(clk_get_rate(spi->clk), t->speed_hz); + reg_write(spi, APPLE_SPI_CLKDIV, min_t(u32, cr, APPLE_SPI_CLKDIV_MAX)); + + /* Update bits per word */ + reg_mask(spi, APPLE_SPI_SHIFTCFG, APPLE_SPI_SHIFTCFG_BITS, + FIELD_PREP(APPLE_SPI_SHIFTCFG_BITS, t->bits_per_word)); + + /* We will want to poll if the time we need to wait is + * less than the context switching time. + * Let's call that threshold 5us. The operation will take: + * bits_per_word * fifo_threshold / hz <= 5 * 10^-6 + * 200000 * bits_per_word * fifo_threshold <= hz + */ + fifo_threshold = APPLE_SPI_FIFO_DEPTH / 2; + return (200000 * t->bits_per_word * fifo_threshold) <= t->speed_hz; +} + +static irqreturn_t apple_spi_irq(int irq, void *dev_id) +{ + struct apple_spi *spi = dev_id; + u32 fifo = reg_read(spi, APPLE_SPI_IF_FIFO) & reg_read(spi, APPLE_SPI_IE_FIFO); + u32 xfer = reg_read(spi, APPLE_SPI_IF_XFER) & reg_read(spi, APPLE_SPI_IE_XFER); + + if (fifo || xfer) { + /* Disable interrupts until next transfer */ + reg_write(spi, APPLE_SPI_IE_XFER, 0); + reg_write(spi, APPLE_SPI_IE_FIFO, 0); + complete(&spi->done); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int apple_spi_wait(struct apple_spi *spi, u32 fifo_bit, u32 xfer_bit, int poll) +{ + int ret = 0; + + if (poll) { + u32 fifo, xfer; + unsigned long timeout = jiffies + APPLE_SPI_TIMEOUT_MS * HZ / 1000; + + do { + fifo = reg_read(spi, APPLE_SPI_IF_FIFO); + xfer = reg_read(spi, APPLE_SPI_IF_XFER); + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + break; + } + } while (!((fifo & fifo_bit) || (xfer & xfer_bit))); + } else { + reinit_completion(&spi->done); + reg_write(spi, APPLE_SPI_IE_XFER, xfer_bit); + reg_write(spi, APPLE_SPI_IE_FIFO, fifo_bit); + + if (!wait_for_completion_timeout(&spi->done, + msecs_to_jiffies(APPLE_SPI_TIMEOUT_MS))) + ret = -ETIMEDOUT; + + reg_write(spi, APPLE_SPI_IE_XFER, 0); + reg_write(spi, APPLE_SPI_IE_FIFO, 0); + } + + return ret; +} + +static void apple_spi_tx(struct apple_spi *spi, const void **tx_ptr, u32 *left, + unsigned int bytes_per_word) +{ + u32 inuse, words, wrote; + + if (!*tx_ptr) + return; + + inuse = FIELD_GET(APPLE_SPI_FIFOSTAT_LEVEL_TX, reg_read(spi, APPLE_SPI_FIFOSTAT)); + words = wrote = min_t(u32, *left, APPLE_SPI_FIFO_DEPTH - inuse); + + if (!words) + return; + + *left -= words; + + switch (bytes_per_word) { + case 1: { + const u8 *p = *tx_ptr; + + while (words--) + reg_write(spi, APPLE_SPI_TXDATA, *p++); + break; + } + case 2: { + const u16 *p = *tx_ptr; + + while (words--) + reg_write(spi, APPLE_SPI_TXDATA, *p++); + break; + } + case 4: { + const u32 *p = *tx_ptr; + + while (words--) + reg_write(spi, APPLE_SPI_TXDATA, *p++); + break; + } + default: + WARN_ON(1); + } + + *tx_ptr = ((u8 *)*tx_ptr) + bytes_per_word * wrote; +} + +static void apple_spi_rx(struct apple_spi *spi, void **rx_ptr, u32 *left, + unsigned int bytes_per_word) +{ + u32 words, read; + + if (!*rx_ptr) + return; + + words = read = FIELD_GET(APPLE_SPI_FIFOSTAT_LEVEL_RX, reg_read(spi, APPLE_SPI_FIFOSTAT)); + WARN_ON(words > *left); + + if (!words) + return; + + *left -= min_t(u32, *left, words); + + switch (bytes_per_word) { + case 1: { + u8 *p = *rx_ptr; + + while (words--) + *p++ = reg_read(spi, APPLE_SPI_RXDATA); + break; + } + case 2: { + u16 *p = *rx_ptr; + + while (words--) + *p++ = reg_read(spi, APPLE_SPI_RXDATA); + break; + } + case 4: { + u32 *p = *rx_ptr; + + while (words--) + *p++ = reg_read(spi, APPLE_SPI_RXDATA); + break; + } + default: + WARN_ON(1); + } + + *rx_ptr = ((u8 *)*rx_ptr) + bytes_per_word * read; +} + +static int apple_spi_transfer_one(struct spi_controller *ctlr, struct spi_device *device, + struct spi_transfer *t) +{ + struct apple_spi *spi = spi_controller_get_devdata(ctlr); + bool poll = apple_spi_prep_transfer(spi, t); + const void *tx_ptr = t->tx_buf; + void *rx_ptr = t->rx_buf; + unsigned int bytes_per_word; + u32 words, remaining_tx, remaining_rx; + u32 xfer_flags = 0; + u32 fifo_flags; + int retries = 100; + int ret = 0; + + if (t->bits_per_word > 16) + bytes_per_word = 4; + else if (t->bits_per_word > 8) + bytes_per_word = 2; + else + bytes_per_word = 1; + + words = t->len / bytes_per_word; + remaining_tx = tx_ptr ? words : 0; + remaining_rx = rx_ptr ? words : 0; + + /* Reset FIFOs */ + reg_write(spi, APPLE_SPI_CTRL, APPLE_SPI_CTRL_RX_RESET | APPLE_SPI_CTRL_TX_RESET); + + /* Clear IRQ flags */ + reg_write(spi, APPLE_SPI_IF_XFER, ~0); + reg_write(spi, APPLE_SPI_IF_FIFO, ~0); + + /* Determine transfer completion flags we wait for */ + if (tx_ptr) + xfer_flags |= APPLE_SPI_XFER_TXCOMPLETE; + if (rx_ptr) + xfer_flags |= APPLE_SPI_XFER_RXCOMPLETE; + + /* Set transfer length */ + reg_write(spi, APPLE_SPI_TXCNT, remaining_tx); + reg_write(spi, APPLE_SPI_RXCNT, remaining_rx); + + /* Prime transmit FIFO */ + apple_spi_tx(spi, &tx_ptr, &remaining_tx, bytes_per_word); + + /* Start transfer */ + reg_write(spi, APPLE_SPI_CTRL, APPLE_SPI_CTRL_RUN); + + /* TX again since a few words get popped off immediately */ + apple_spi_tx(spi, &tx_ptr, &remaining_tx, bytes_per_word); + + while (xfer_flags) { + fifo_flags = 0; + + if (remaining_tx) + fifo_flags |= APPLE_SPI_FIFO_TXTHRESH; + if (remaining_rx) + fifo_flags |= APPLE_SPI_FIFO_RXTHRESH; + + /* Wait for anything to happen */ + ret = apple_spi_wait(spi, fifo_flags, xfer_flags, poll); + if (ret) { + dev_err(&ctlr->dev, "transfer timed out (remaining %d tx, %d rx)\n", + remaining_tx, remaining_rx); + goto err; + } + + /* Stop waiting on transfer halves once they complete */ + xfer_flags &= ~reg_read(spi, APPLE_SPI_IF_XFER); + + /* Transmit and receive everything we can */ + apple_spi_tx(spi, &tx_ptr, &remaining_tx, bytes_per_word); + apple_spi_rx(spi, &rx_ptr, &remaining_rx, bytes_per_word); + } + + /* + * Sometimes the transfer completes before the last word is in the RX FIFO. + * Normally one retry is all it takes to get the last word out. + */ + while (remaining_rx && retries--) + apple_spi_rx(spi, &rx_ptr, &remaining_rx, bytes_per_word); + + if (remaining_tx) + dev_err(&ctlr->dev, "transfer completed with %d words left to transmit\n", + remaining_tx); + if (remaining_rx) + dev_err(&ctlr->dev, "transfer completed with %d words left to receive\n", + remaining_rx); + +err: + fifo_flags = reg_read(spi, APPLE_SPI_IF_FIFO); + WARN_ON(fifo_flags & APPLE_SPI_FIFO_TXOVERFLOW); + WARN_ON(fifo_flags & APPLE_SPI_FIFO_RXUNDERRUN); + + /* Stop transfer */ + reg_write(spi, APPLE_SPI_CTRL, 0); + + return ret; +} + +static void apple_spi_clk_disable_unprepare(void *data) +{ + clk_disable_unprepare(data); +} + +static int apple_spi_probe(struct platform_device *pdev) +{ + struct apple_spi *spi; + int ret, irq; + struct spi_controller *ctlr; + + ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(struct apple_spi)); + if (!ctlr) + return dev_err_probe(&pdev->dev, -ENOMEM, "out of memory\n"); + + spi = spi_controller_get_devdata(ctlr); + init_completion(&spi->done); + platform_set_drvdata(pdev, ctlr); + + spi->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(spi->regs)) + return PTR_ERR(spi->regs); + + spi->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(spi->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(spi->clk), "Unable to find bus clock\n"); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + return irq; + } + + ret = devm_request_irq(&pdev->dev, irq, apple_spi_irq, 0, + dev_name(&pdev->dev), spi); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Unable to bind to interrupt\n"); + + ret = clk_prepare_enable(spi->clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Unable to enable bus clock\n"); + + ret = devm_add_action_or_reset(&pdev->dev, apple_spi_clk_disable_unprepare, spi->clk); + if (ret) + return ret; + + ctlr->dev.of_node = pdev->dev.of_node; + ctlr->bus_num = pdev->id; + ctlr->num_chipselect = 1; + ctlr->mode_bits = SPI_CPHA | SPI_CPOL | SPI_LSB_FIRST; + ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); + ctlr->flags = 0; + ctlr->prepare_message = apple_spi_prepare_message; + ctlr->set_cs = apple_spi_set_cs; + ctlr->transfer_one = apple_spi_transfer_one; + ctlr->auto_runtime_pm = true; + + pm_runtime_set_active(&pdev->dev); + devm_pm_runtime_enable(&pdev->dev); + + apple_spi_init(spi); + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, "devm_spi_register_controller failed\n"); + + return 0; +} + +static const struct of_device_id apple_spi_of_match[] = { + { .compatible = "apple,spi", }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_spi_of_match); + +static struct platform_driver apple_spi_driver = { + .probe = apple_spi_probe, + .driver = { + .name = "apple-spi", + .owner = THIS_MODULE, + .of_match_table = apple_spi_of_match, + }, +}; +module_platform_driver(apple_spi_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_DESCRIPTION("Apple SoC SPI driver"); +MODULE_LICENSE("GPL"); From fb0d311995ebd125f70c98a86c8377c2e411e5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 18:09:39 +0200 Subject: [PATCH 060/635] Revert "ASoC: ops: Don't modify the driver's plaform_max when reading state" This reverts commit 30ac49841386f933339817771ec315a34a4c0edd. --- sound/soc/soc-ops.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index 55b009d3c68154..6b67fdeb43f043 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -176,28 +176,20 @@ int snd_soc_info_volsw(struct snd_kcontrol *kcontrol, { struct soc_mixer_control *mc = (struct soc_mixer_control *)kcontrol->private_value; - const char *vol_string = NULL; - int max; + int platform_max; - max = uinfo->value.integer.max = mc->max - mc->min; - if (mc->platform_max && mc->platform_max < max) - max = mc->platform_max; + if (!mc->platform_max) + mc->platform_max = mc->max; + platform_max = mc->platform_max; - if (max == 1) { - /* Even two value controls ending in Volume should always be integer */ - vol_string = strstr(kcontrol->id.name, " Volume"); - if (vol_string && !strcmp(vol_string, " Volume")) - uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; - else - uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN; - } else { + if (platform_max == 1 && !strstr(kcontrol->id.name, " Volume")) + uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN; + else uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; - } uinfo->count = snd_soc_volsw_is_stereo(mc) ? 2 : 1; uinfo->value.integer.min = 0; - uinfo->value.integer.max = max; - + uinfo->value.integer.max = platform_max - mc->min; return 0; } EXPORT_SYMBOL_GPL(snd_soc_info_volsw); From b3de738b45534fa4e4cb6bb05bf64c1928fd7253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 11 Mar 2022 11:55:44 +0100 Subject: [PATCH 061/635] ASoC: tas2764: Extend driver to SN012776 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SN012776 is a speaker amp chip found in Apple's 2021 laptops. It appears similar and more-or-less compatible to TAS2764. Extend the TAS2764 driver with some SN012776 specifics and configure the chip assuming it's in one of the Apple machines. Signed-off-by: Martin Povišer --- sound/soc/codecs/tas2764.c | 50 ++++++++++++++++++++++++++++++++++---- sound/soc/codecs/tas2764.h | 3 +++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 2e0ed3e68fa543..0481a74eed8ac8 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,11 @@ #include "tas2764.h" +enum tas2764_devid { + DEVID_TAS2764 = 0, + DEVID_SN012776 = 1 +}; + struct tas2764_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; @@ -32,7 +38,8 @@ struct tas2764_priv { struct regmap *regmap; struct device *dev; int irq; - + enum tas2764_devid devid; + int v_sense_slot; int i_sense_slot; @@ -528,10 +535,16 @@ static struct snd_soc_dai_driver tas2764_dai_driver[] = { }, }; +static uint8_t sn012776_bop_presets[] = { + 0x01, 0x32, 0x02, 0x22, 0x83, 0x2d, 0x80, 0x02, 0x06, + 0x32, 0x46, 0x30, 0x02, 0x06, 0x38, 0x40, 0x30, 0x02, + 0x06, 0x3e, 0x37, 0x30, 0xff, 0xe6 +}; + static int tas2764_codec_probe(struct snd_soc_component *component) { struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); - int ret; + int ret, i; tas2764->component = component; @@ -580,6 +593,23 @@ static int tas2764_codec_probe(struct snd_soc_component *component) if (ret < 0) return ret; + if (tas2764->devid == DEVID_SN012776) { + ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, + TAS2764_PWR_CTRL_BOP_SRC, + TAS2764_PWR_CTRL_BOP_SRC); + if (ret < 0) + return ret; + + for (i = 0; i < ARRAY_SIZE(sn012776_bop_presets); i++) { + ret = snd_soc_component_write(component, + TAS2764_BOP_CFG0 + i, + sn012776_bop_presets[i]); + + if (ret < 0) + return ret; + } + } + return 0; } @@ -699,9 +729,12 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) return 0; } +static const struct of_device_id tas2764_of_match[]; + static int tas2764_i2c_probe(struct i2c_client *client) { struct tas2764_priv *tas2764; + const struct of_device_id *of_id = NULL; int result; tas2764 = devm_kzalloc(&client->dev, sizeof(struct tas2764_priv), @@ -709,6 +742,14 @@ static int tas2764_i2c_probe(struct i2c_client *client) if (!tas2764) return -ENOMEM; + if (client->dev.of_node) + of_id = of_match_device(tas2764_of_match, &client->dev); + + if (of_id) + tas2764->devid = (enum tas2764_devid) of_id->data; + else + tas2764->devid = DEVID_TAS2764; + tas2764->dev = &client->dev; tas2764->irq = client->irq; i2c_set_clientdata(client, tas2764); @@ -743,13 +784,12 @@ static const struct i2c_device_id tas2764_i2c_id[] = { }; MODULE_DEVICE_TABLE(i2c, tas2764_i2c_id); -#if defined(CONFIG_OF) static const struct of_device_id tas2764_of_match[] = { - { .compatible = "ti,tas2764" }, + { .compatible = "ti,tas2764", .data = (void*) DEVID_TAS2764 }, + { .compatible = "ti,sn012776", .data = (void*) DEVID_SN012776 }, {}, }; MODULE_DEVICE_TABLE(of, tas2764_of_match); -#endif static struct i2c_driver tas2764_i2c_driver = { .driver = { diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h index 168af772a898ff..0a40166040e7e8 100644 --- a/sound/soc/codecs/tas2764.h +++ b/sound/soc/codecs/tas2764.h @@ -29,6 +29,7 @@ #define TAS2764_PWR_CTRL_ACTIVE 0x0 #define TAS2764_PWR_CTRL_MUTE BIT(0) #define TAS2764_PWR_CTRL_SHUTDOWN BIT(1) +#define TAS2764_PWR_CTRL_BOP_SRC BIT(7) #define TAS2764_VSENSE_POWER_EN 3 #define TAS2764_ISENSE_POWER_EN 4 @@ -110,4 +111,6 @@ #define TAS2764_INT_CLK_CFG TAS2764_REG(0x0, 0x5c) #define TAS2764_INT_CLK_CFG_IRQZ_CLR BIT(2) +#define TAS2764_BOP_CFG0 TAS2764_REG(0X0, 0x1d) + #endif /* __TAS2764__ */ From 58379973bbacebdcb9e0d4b9dbd69bca92eabdb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sat, 20 Aug 2022 20:13:05 +0200 Subject: [PATCH 062/635] ASoC: tas2764: Add control concerning overcurrent events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add control to expose the option of autoretry behavior on overcurrent events in the codec. Signed-off-by: Martin Povišer --- sound/soc/codecs/tas2764.c | 9 +++++++++ sound/soc/codecs/tas2764.h | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 0481a74eed8ac8..76edcbe9868030 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -625,12 +625,21 @@ static SOC_ENUM_SINGLE_DECL( tas2764_hpf_enum, TAS2764_DC_BLK0, TAS2764_DC_BLK0_HPF_FREQ_PB_SHIFT, tas2764_hpf_texts); +static const char * const tas2764_oce_texts[] = { + "Disable", "Retry", +}; + +static SOC_ENUM_SINGLE_DECL( + tas2764_oce_enum, TAS2764_MISC_CFG1, + TAS2764_MISC_CFG1_OCE_RETRY_SHIFT, tas2764_oce_texts); + static const struct snd_kcontrol_new tas2764_snd_controls[] = { SOC_SINGLE_TLV("Speaker Volume", TAS2764_DVC, 0, TAS2764_DVC_MAX, 1, tas2764_playback_volume), SOC_SINGLE_TLV("Amp Gain Volume", TAS2764_CHNL_0, 1, 0x14, 0, tas2764_digital_tlv), SOC_ENUM("HPF Corner Frequency", tas2764_hpf_enum), + SOC_ENUM("OCE Handling", tas2764_oce_enum), }; static const struct snd_soc_component_driver soc_component_driver_tas2764 = { diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h index 0a40166040e7e8..20628e51bf94f0 100644 --- a/sound/soc/codecs/tas2764.h +++ b/sound/soc/codecs/tas2764.h @@ -44,6 +44,10 @@ #define TAS2764_CHNL_0 TAS2764_REG(0X0, 0x03) +/* Miscellaneous */ +#define TAS2764_MISC_CFG1 TAS2764_REG(0x0, 0x06) +#define TAS2764_MISC_CFG1_OCE_RETRY_SHIFT 5 + /* TDM Configuration Reg0 */ #define TAS2764_TDM_CFG0 TAS2764_REG(0X0, 0x08) #define TAS2764_TDM_CFG0_SMP_MASK BIT(5) From 58318a5b979eedd83260b215c4457e4b56c5fd15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 18:38:04 +0200 Subject: [PATCH 063/635] ASoC: ops: Move guts out of snd_soc_limit_volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In advance of other changes, move the modification of the control itself into function of its own. Signed-off-by: Martin Povišer --- sound/soc/soc-ops.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index 6b67fdeb43f043..be7048a8d9d21b 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -631,6 +631,16 @@ int snd_soc_get_volsw_range(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_get_volsw_range); +static int soc_limit_volume(struct snd_kcontrol *kctl, int max) +{ + struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; + + if (max <= 0 || max > mc->max) + return -EINVAL; + mc->platform_max = max; + return 0; +} + /** * snd_soc_limit_volume - Set new limit to an existing volume control. * @@ -644,21 +654,16 @@ int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max) { struct snd_kcontrol *kctl; - int ret = -EINVAL; - /* Sanity check for name and max */ - if (unlikely(!name || max <= 0)) + /* Sanity check for name */ + if (unlikely(!name)) return -EINVAL; kctl = snd_soc_card_get_kcontrol(card, name); - if (kctl) { - struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; - if (max <= mc->max) { - mc->platform_max = max; - ret = 0; - } - } - return ret; + if (!kctl) + return -EINVAL; + + return soc_limit_volume(kctl, max); } EXPORT_SYMBOL_GPL(snd_soc_limit_volume); From a3e6f2e87e1c123319812b7ff6d6231db80d30f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 18:58:29 +0200 Subject: [PATCH 064/635] ASoC: ops: Enforce platform maximum on initial value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower the volume if it is violating the platform maximum at its initial value (i.e. at the time of the the 'snd_soc_limit_volume' call). Signed-off-by: Martin Povišer --- sound/soc/soc-ops.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index be7048a8d9d21b..7c44cfd5bfc4ed 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -631,6 +631,33 @@ int snd_soc_get_volsw_range(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_get_volsw_range); +static int soc_clip_to_platform_max(struct snd_kcontrol *kctl) +{ + struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; + struct snd_ctl_elem_value uctl; + int ret; + + if (!mc->platform_max) + return 0; + + ret = kctl->get(kctl, &uctl); + if (ret < 0) + return ret; + + if (uctl.value.integer.value[0] > mc->platform_max) + uctl.value.integer.value[0] = mc->platform_max; + + if (snd_soc_volsw_is_stereo(mc) && + uctl.value.integer.value[1] > mc->platform_max) + uctl.value.integer.value[1] = mc->platform_max; + + ret = kctl->put(kctl, &uctl); + if (ret < 0) + return ret; + + return 0; +} + static int soc_limit_volume(struct snd_kcontrol *kctl, int max) { struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; @@ -638,7 +665,8 @@ static int soc_limit_volume(struct snd_kcontrol *kctl, int max) if (max <= 0 || max > mc->max) return -EINVAL; mc->platform_max = max; - return 0; + + return soc_clip_to_platform_max(kctl); } /** From 22ec2013d4410c56daa712a6764fc6898e28b977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 19:15:54 +0200 Subject: [PATCH 065/635] ASoC: ops: Accept patterns in snd_soc_limit_volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In snd_soc_limit_volume, instead of looking up a single control by name, also understand wildcard-starting patterns like '* Amp Gain Volume' to touch many controls at one. Signed-off-by: Martin Povišer --- sound/soc/soc-ops.c | 51 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index 7c44cfd5bfc4ed..6f89bdd9575456 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -631,6 +631,29 @@ int snd_soc_get_volsw_range(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_get_volsw_range); +static bool soc_control_matches(struct snd_kcontrol *kctl, + const char *pattern) +{ + const char *name = kctl->id.name; + + if (pattern[0] == '*') { + int namelen; + int patternlen; + + pattern++; + if (pattern[0] == ' ') + pattern++; + + namelen = strlen(name); + patternlen = strlen(pattern); + + if (namelen > patternlen) + name += namelen - patternlen; + } + + return !strcmp(name, pattern); +} + static int soc_clip_to_platform_max(struct snd_kcontrol *kctl) { struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; @@ -670,28 +693,44 @@ static int soc_limit_volume(struct snd_kcontrol *kctl, int max) } /** - * snd_soc_limit_volume - Set new limit to an existing volume control. + * snd_soc_limit_volume - Set new limit to existing volume controls * * @card: where to look for the control - * @name: Name of the control + * @name: name pattern * @max: new maximum limit + * + * Finds controls matching the given name (which can be either a name + * verbatim, or a pattern starting with the wildcard '*') and sets + * a platform volume limit on them. * - * Return 0 for success, else error. + * Return number of matching controls on success, else error. At least + * one control needs to match the pattern. */ int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max) { struct snd_kcontrol *kctl; + int hits = 0; + int ret; /* Sanity check for name */ if (unlikely(!name)) return -EINVAL; - kctl = snd_soc_card_get_kcontrol(card, name); - if (!kctl) + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!soc_control_matches(kctl, name)) + continue; + + ret = soc_limit_volume(kctl, max); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) return -EINVAL; - return soc_limit_volume(kctl, max); + return hits; } EXPORT_SYMBOL_GPL(snd_soc_limit_volume); From eb4b98aff0f219e74028990697b51376b5c07027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 19:24:35 +0200 Subject: [PATCH 066/635] ASoC: ops: Introduce 'snd_soc_deactivate_kctl' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new function can be used to deactivate controls -- either a single one or in bulk by pattern. It is something a machine driver may call in fixup_controls. Signed-off-by: Martin Povišer --- include/sound/soc.h | 2 ++ sound/soc/soc-ops.c | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/include/sound/soc.h b/include/sound/soc.h index e58b43b5da7c05..8f12bd9f78502a 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -586,6 +586,8 @@ int snd_soc_get_volsw_range(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max); +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active); int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo); int snd_soc_bytes_get(struct snd_kcontrol *kcontrol, diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index 6f89bdd9575456..f4f9650871b352 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -734,6 +734,44 @@ int snd_soc_limit_volume(struct snd_soc_card *card, } EXPORT_SYMBOL_GPL(snd_soc_limit_volume); +/** + * snd_soc_deactivate_kctl - Activate/deactive controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @active: non-zero to activate, zero to deactivate + * + * Return number of matching controls on success, else error. + * No controls need to match. + */ +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!soc_control_matches(kctl, name)) + continue; + + ret = snd_ctl_activate_id(card->snd_card, &kctl->id, active); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_deactivate_kctl); + int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { From da1b763c95a09c2234123dc7745c8eec5eda96f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 19:25:36 +0200 Subject: [PATCH 067/635] ASoC: ops: Introduce 'soc_set_enum_kctl' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new function is to be used to set enumerated controls to desired values -- either a single control or many controls in bulk by pattern. It is something a machine driver may call in fixup_controls. Signed-off-by: Martin Povišer --- include/sound/soc.h | 2 ++ sound/soc/soc-ops.c | 70 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/include/sound/soc.h b/include/sound/soc.h index 8f12bd9f78502a..401c999037d237 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -588,6 +588,8 @@ int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max); int snd_soc_deactivate_kctl(struct snd_soc_card *card, const char *name, int active); +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *strval); int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo); int snd_soc_bytes_get(struct snd_kcontrol *kcontrol, diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index f4f9650871b352..11035615e2ed2b 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -772,6 +772,76 @@ int snd_soc_deactivate_kctl(struct snd_soc_card *card, } EXPORT_SYMBOL_GPL(snd_soc_deactivate_kctl); +static int soc_set_enum_kctl(struct snd_kcontrol *kctl, const char *strval) +{ + struct snd_ctl_elem_value value; + struct snd_ctl_elem_info info; + int sel, i, ret; + + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (info.type != SNDRV_CTL_ELEM_TYPE_ENUMERATED) + return -EINVAL; + + for (sel = 0; sel < info.value.enumerated.items; sel++) { + info.value.enumerated.item = sel; + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (!strcmp(strval, info.value.enumerated.name)) + break; + } + + if (sel == info.value.enumerated.items) + return -EINVAL; + + for (i = 0; i < info.count; i++) + value.value.enumerated.item[i] = sel; + + return kctl->put(kctl, &value); +} + +/** + * snd_soc_set_enum_kctl - Set enumerated controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @value: string value to set the controls to + * + * Return number of matching and set controls on success, else error. + * No controls need to match. + */ +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *value) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!soc_control_matches(kctl, name)) + continue; + + ret = soc_set_enum_kctl(kctl, value); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_set_enum_kctl); + int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { From b37d27bcaf34a4a08e6407f3533662b2a144a330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 19 Aug 2022 21:09:35 +0200 Subject: [PATCH 068/635] ASoC: card: Let 'fixup_controls' return errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let the 'fixup_controls' card method return error values which will roll back the half-done binding of the card. Signed-off-by: Martin Povišer --- include/sound/soc-card.h | 2 +- include/sound/soc.h | 2 +- sound/soc/soc-card.c | 12 +++++++++--- sound/soc/soc-core.c | 5 ++++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/sound/soc-card.h b/include/sound/soc-card.h index 9d31a5c0db33c2..ebfec0f306a6c4 100644 --- a/include/sound/soc-card.h +++ b/include/sound/soc-card.h @@ -29,7 +29,7 @@ int snd_soc_card_resume_post(struct snd_soc_card *card); int snd_soc_card_probe(struct snd_soc_card *card); int snd_soc_card_late_probe(struct snd_soc_card *card); -void snd_soc_card_fixup_controls(struct snd_soc_card *card); +int snd_soc_card_fixup_controls(struct snd_soc_card *card); int snd_soc_card_remove(struct snd_soc_card *card); int snd_soc_card_set_bias_level(struct snd_soc_card *card, diff --git a/include/sound/soc.h b/include/sound/soc.h index 401c999037d237..7bbbbbc3f893db 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -930,7 +930,7 @@ struct snd_soc_card { int (*probe)(struct snd_soc_card *card); int (*late_probe)(struct snd_soc_card *card); - void (*fixup_controls)(struct snd_soc_card *card); + int (*fixup_controls)(struct snd_soc_card *card); int (*remove)(struct snd_soc_card *card); /* the pre and post PM functions are used to do any PM work before and diff --git a/sound/soc/soc-card.c b/sound/soc/soc-card.c index 285ab4c9c71683..674a12258fc76e 100644 --- a/sound/soc/soc-card.c +++ b/sound/soc/soc-card.c @@ -197,10 +197,16 @@ int snd_soc_card_late_probe(struct snd_soc_card *card) return 0; } -void snd_soc_card_fixup_controls(struct snd_soc_card *card) +int snd_soc_card_fixup_controls(struct snd_soc_card *card) { - if (card->fixup_controls) - card->fixup_controls(card); + if (card->fixup_controls) { + int ret = card->fixup_controls(card); + + if (ret < 0) + return soc_card_ret(card, ret); + } + + return 0; } int snd_soc_card_remove(struct snd_soc_card *card) diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index 71b022f7edfd5b..ed4657b7f70fee 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -2071,7 +2071,10 @@ static int snd_soc_bind_card(struct snd_soc_card *card) goto probe_end; snd_soc_dapm_new_widgets(card); - snd_soc_card_fixup_controls(card); + + ret = snd_soc_card_fixup_controls(card); + if (ret < 0) + goto probe_end; ret = snd_card_register(card->snd_card); if (ret < 0) { From 177186f8db57dda90240d888bc5a3c29ef82e4e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Thu, 31 Mar 2022 01:16:48 +0200 Subject: [PATCH 069/635] dt-bindings: sound: Add Apple Macs sound peripherals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add binding for Apple Silicon Macs' machine-level integration of sound peripherals. Signed-off-by: Martin Povišer --- .../bindings/sound/apple,macaudio.yaml | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 Documentation/devicetree/bindings/sound/apple,macaudio.yaml diff --git a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml new file mode 100644 index 00000000000000..8fe22dec3015d6 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/apple,macaudio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon Macs integrated sound peripherals + +description: + This binding represents the overall machine-level integration of sound + peripherals on 'Apple Silicon' machines by Apple. + +maintainers: + - Martin Povišer + +properties: + compatible: + items: + - enum: + - apple,j274-macaudio + - apple,j293-macaudio + - apple,j314-macaudio + - const: apple,macaudio + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + model: + description: + Model name for presentation to users + $ref: /schemas/types.yaml#/definitions/string + +patternProperties: + "^dai-link(@[0-9a-f]+)?$": + description: | + Node for each sound peripheral such as the speaker array, headphones jack, + or microphone. + type: object + + additionalProperties: false + + properties: + reg: + maxItems: 1 + + link-name: + description: | + Name for the peripheral, expecting 'Speaker' or 'Speakers' if this is + the speaker array. + $ref: /schemas/types.yaml#/definitions/string + + cpu: + type: object + + properties: + sound-dai: + description: | + DAI list with CPU-side I2S ports involved in this peripheral. + minItems: 1 + maxItems: 2 + + required: + - sound-dai + + codec: + type: object + + properties: + sound-dai: + minItems: 1 + maxItems: 8 + description: | + DAI list with the CODEC-side DAIs connected to the above CPU-side + DAIs and involved in this sound peripheral. + + The list is in left/right order if applicable. If there are more + than one CPU-side DAIs (there can be two), the CODECs must be + listed first those connected to the first CPU, then those + connected to the second. + + In addition, on some machines with many speaker codecs, the CODECs + are listed in this fixed order: + + J293: Left Front, Left Rear, Right Front, Right Rear + J314: Left Woofer 1, Left Tweeter, Left Woofer 2, + Right Woofer 1, Right Tweeter, Right Woofer 2 + + required: + - sound-dai + + required: + - reg + - cpu + - codec + +required: + - compatible + - model + +additionalProperties: false + +examples: + - | + mca: mca@9b600000 { + compatible = "apple,t6000-mca", "apple,mca"; + reg = <0x9b600000 0x10000>, + <0x9b500000 0x20000>; + + clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; + power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, + <&ps_mca2>, <&ps_mca3>; + dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>, + <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>, + <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>, + <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>; + dma-names = "tx0a", "rx0a", "tx0b", "rx0b", + "tx1a", "rx1a", "tx1b", "rx1b", + "tx2a", "rx2a", "tx2b", "rx2b", + "tx3a", "rx3a", "tx3b", "rx3b"; + + #sound-dai-cells = <1>; + }; + + sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314 integrated audio"; + + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Headphones Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; From 4727f3c68cd923aaf22a43c83c1b8de365bbb1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sat, 19 Feb 2022 09:49:56 +0100 Subject: [PATCH 070/635] ASoC: apple: Add macaudio machine driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/Kconfig | 17 + sound/soc/apple/Makefile | 4 + sound/soc/apple/macaudio.c | 923 +++++++++++++++++++++++++++++++++++++ 3 files changed, 944 insertions(+) create mode 100644 sound/soc/apple/macaudio.c diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig index 793f7782e0d721..992e416108be5f 100644 --- a/sound/soc/apple/Kconfig +++ b/sound/soc/apple/Kconfig @@ -6,3 +6,20 @@ config SND_SOC_APPLE_MCA help This option enables an ASoC platform driver for MCA peripherals found on Apple Silicon SoCs. + +config SND_SOC_APPLE_MACAUDIO + tristate "Sound support for Apple Silicon Macs" + depends on ARCH_APPLE || COMPILE_TEST + select SND_SOC_APPLE_MCA + select SND_SIMPLE_CARD_UTILS + select APPLE_ADMAC + select COMMON_CLK_APPLE_NCO + select SND_SOC_TAS2764 + select SND_SOC_TAS2770 + select SND_SOC_CS42L83 + select SND_SOC_CS42L84 + default ARCH_APPLE + help + This option enables an ASoC machine-level driver for Apple Silicon Macs + and it also enables the required SoC and codec drivers for overall + sound support on these machines. diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile index 7a30bf452817e0..a14b8fc7f349aa 100644 --- a/sound/soc/apple/Makefile +++ b/sound/soc/apple/Makefile @@ -1,3 +1,7 @@ snd-soc-apple-mca-objs := mca.o obj-$(CONFIG_SND_SOC_APPLE_MCA) += snd-soc-apple-mca.o + +snd-soc-macaudio-objs := macaudio.o + +obj-$(CONFIG_SND_SOC_APPLE_MACAUDIO) += snd-soc-macaudio.o diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c new file mode 100644 index 00000000000000..87d58a7481a6fb --- /dev/null +++ b/sound/soc/apple/macaudio.c @@ -0,0 +1,923 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ASoC machine driver for Apple Silicon Macs + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on sound/soc/qcom/{sc7180.c|common.c} + * Copyright (c) 2018, Linaro Limited. + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * + * The platform driver has independent frontend and backend DAIs with the + * option of routing backends to any of the frontends. The platform + * driver configures the routing based on DPCM couplings in ASoC runtime + * structures, which in turn are determined from DAPM paths by ASoC. But the + * platform driver doesn't supply relevant DAPM paths and leaves that up for + * the machine driver to fill in. The filled-in virtual topology can be + * anything as long as any backend isn't connected to more than one frontend + * at any given time. (The limitation is due to the unsupported case of + * reparenting of live BEs.) + */ + +#define DEBUG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "snd-soc-macaudio" + +/* + * CPU side is bit and frame clock provider + * I2S has both clocks inverted + */ +#define MACAUDIO_DAI_FMT (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBC_CFC | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF) +#define MACAUDIO_JACK_MASK (SND_JACK_HEADSET | SND_JACK_HEADPHONE) +#define MACAUDIO_SLOTWIDTH 32 + +struct macaudio_snd_data { + struct snd_soc_card card; + struct snd_soc_jack jack; + int jack_plugin_state; + + bool has_speakers; + + struct macaudio_link_props { + /* frontend props */ + unsigned int bclk_ratio; + + /* backend props */ + bool is_speakers; + bool is_headphones; + unsigned int tdm_mask; + } *link_props; + + unsigned int speaker_nchans_array[2]; + struct snd_pcm_hw_constraint_list speaker_nchans_list; +}; + +static bool void_warranty; +module_param(void_warranty, bool, 0644); +MODULE_PARM_DESC(void_warranty, "Do not bail if safety is not assured"); + +SND_SOC_DAILINK_DEFS(primary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); // platform (filled at runtime) + +SND_SOC_DAILINK_DEFS(secondary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-1")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +static struct snd_soc_dai_link macaudio_fe_links[] = { + { + .name = "Primary", + .stream_name = "Primary", + .dynamic = 1, + .dpcm_playback = 1, + .dpcm_capture = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + SND_SOC_DAILINK_REG(primary), + }, + { + .name = "Secondary", + .stream_name = "Secondary", + .dynamic = 1, + .dpcm_playback = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + SND_SOC_DAILINK_REG(secondary), + }, +}; + +static struct macaudio_link_props macaudio_fe_link_props[] = { + { + /* + * Primary FE + * + * The bclk ratio at 64 for the primary frontend is important + * to ensure that the headphones codec's idea of left and right + * in a stereo stream over I2S fits in nicely with everyone else's. + * (This is until the headphones codec's driver supports + * set_tdm_slot.) + * + * The low bclk ratio precludes transmitting more than two + * channels over I2S, but that's okay since there is the secondary + * FE for speaker arrays anyway. + */ + .bclk_ratio = 64, + }, + { + /* + * Secondary FE + * + * Here we want frames plenty long to be able to drive all + * those fancy speaker arrays. + */ + .bclk_ratio = 256, + } +}; + +static int macaudio_copy_link(struct device *dev, struct snd_soc_dai_link *target, + struct snd_soc_dai_link *source) +{ + memcpy(target, source, sizeof(struct snd_soc_dai_link)); + + target->cpus = devm_kmemdup(dev, target->cpus, + sizeof(*target->cpus) * target->num_cpus, + GFP_KERNEL); + target->codecs = devm_kmemdup(dev, target->codecs, + sizeof(*target->codecs) * target->num_codecs, + GFP_KERNEL); + target->platforms = devm_kmemdup(dev, target->platforms, + sizeof(*target->platforms) * target->num_platforms, + GFP_KERNEL); + + if (!target->cpus || !target->codecs || !target->platforms) + return -ENOMEM; + + return 0; +} + +static int macaudio_parse_of_component(struct device_node *node, int index, + struct snd_soc_dai_link_component *comp) +{ + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_args(node, "sound-dai", "#sound-dai-cells", + index, &args); + if (ret) + return ret; + comp->of_node = args.np; + return snd_soc_get_dai_name(&args, &comp->dai_name); +} + +/* + * Parse one DPCM backend from the devicetree. This means taking one + * of the CPU DAIs and combining it with one or more CODEC DAIs. + */ +static int macaudio_parse_of_be_dai_link(struct macaudio_snd_data *ma, + struct snd_soc_dai_link *link, + int be_index, int ncodecs_per_be, + struct device_node *cpu, + struct device_node *codec) +{ + struct snd_soc_dai_link_component *comp; + struct device *dev = ma->card.dev; + int codec_base = be_index * ncodecs_per_be; + int ret, i; + + link->no_pcm = 1; + link->dpcm_playback = 1; + link->dpcm_capture = 1; + + link->dai_fmt = MACAUDIO_DAI_FMT; + + link->num_codecs = ncodecs_per_be; + link->codecs = devm_kcalloc(dev, ncodecs_per_be, + sizeof(*comp), GFP_KERNEL); + link->num_cpus = 1; + link->cpus = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL); + + if (!link->codecs || !link->cpus) + return -ENOMEM; + + link->num_platforms = 0; + + for_each_link_codecs(link, i, comp) { + ret = macaudio_parse_of_component(codec, codec_base + i, comp); + if (ret) + return ret; + } + + ret = macaudio_parse_of_component(cpu, be_index, link->cpus); + if (ret) + return ret; + + link->name = link->cpus[0].dai_name; + + return 0; +} + +static int macaudio_parse_of(struct macaudio_snd_data *ma) +{ + struct device_node *codec = NULL; + struct device_node *cpu = NULL; + struct device_node *np = NULL; + struct device_node *platform = NULL; + struct snd_soc_dai_link *link = NULL; + struct snd_soc_card *card = &ma->card; + struct device *dev = card->dev; + struct macaudio_link_props *link_props; + int ret, num_links, i; + + ret = snd_soc_of_parse_card_name(card, "model"); + if (ret) { + dev_err(dev, "Error parsing card name: %d\n", ret); + return ret; + } + + /* Populate links, start with the fixed number of FE links */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Now add together the (dynamic) number of BE links */ + for_each_available_child_of_node(dev->of_node, np) { + int num_cpus; + + cpu = of_get_child_by_name(np, "cpu"); + if (!cpu) { + dev_err(dev, "missing CPU DAI node at %pOF\n", np); + ret = -EINVAL; + goto err_free; + } + + num_cpus = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + + if (num_cpus <= 0) { + dev_err(card->dev, "missing sound-dai property at %pOF\n", cpu); + ret = -EINVAL; + goto err_free; + } + of_node_put(cpu); + cpu = NULL; + + /* Each CPU specified counts as one BE link */ + num_links += num_cpus; + } + + /* Allocate the DAI link array */ + card->dai_link = devm_kcalloc(dev, num_links, sizeof(*link), GFP_KERNEL); + ma->link_props = devm_kcalloc(dev, num_links, sizeof(*ma->link_props), GFP_KERNEL); + if (!card->dai_link || !ma->link_props) + return -ENOMEM; + + card->num_links = num_links; + link = card->dai_link; + link_props = ma->link_props; + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + ret = macaudio_copy_link(dev, link, &macaudio_fe_links[i]); + if (ret) + goto err_free; + + memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); + link++; link_props++; + } + + for (i = 0; i < num_links; i++) + card->dai_link[i].id = i; + + /* Fill in the BEs */ + for_each_available_child_of_node(dev->of_node, np) { + const char *link_name; + bool speakers; + int be_index, num_codecs, num_bes, ncodecs_per_cpu, nchannels; + unsigned int left_mask, right_mask; + + ret = of_property_read_string(np, "link-name", &link_name); + if (ret) { + dev_err(card->dev, "missing link name\n"); + goto err_free; + } + + speakers = !strcmp(link_name, "Speaker") + || !strcmp(link_name, "Speakers"); + if (speakers) + ma->has_speakers = 1; + + cpu = of_get_child_by_name(np, "cpu"); + codec = of_get_child_by_name(np, "codec"); + + if (!codec || !cpu) { + dev_err(dev, "missing DAI specifications for '%s'\n", link_name); + ret = -EINVAL; + goto err_free; + } + + num_bes = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + if (num_bes <= 0) { + dev_err(card->dev, "missing sound-dai property at %pOF\n", cpu); + ret = -EINVAL; + goto err_free; + } + + num_codecs = of_count_phandle_with_args(codec, "sound-dai", + "#sound-dai-cells"); + if (num_codecs <= 0) { + dev_err(card->dev, "missing sound-dai property at %pOF\n", codec); + ret = -EINVAL; + goto err_free; + } + + if (num_codecs % num_bes != 0) { + dev_err(card->dev, "bad combination of CODEC (%d) and CPU (%d) number at %pOF\n", + num_codecs, num_bes, np); + ret = -EINVAL; + goto err_free; + } + + /* + * Now parse the cpu/codec lists into a number of DPCM backend links. + * In each link there will be one DAI from the cpu list paired with + * an evenly distributed number of DAIs from the codec list. (As is + * the binding semantics.) + */ + ncodecs_per_cpu = num_codecs / num_bes; + nchannels = num_codecs * (speakers ? 1 : 2); + + /* + * If there is a single speaker, assign two channels to it, because + * it can do downmix. + */ + if (nchannels < 2) + nchannels = 2; + + left_mask = 0; + for (i = 0; i < nchannels; i += 2) + left_mask = left_mask << 2 | 1; + right_mask = left_mask << 1; + + for (be_index = 0; be_index < num_bes; be_index++) { + ret = macaudio_parse_of_be_dai_link(ma, link, be_index, + ncodecs_per_cpu, cpu, codec); + if (ret) + goto err_free; + + link_props->is_speakers = speakers; + link_props->is_headphones = !speakers; + + if (num_bes == 2) + /* This sound peripheral is split between left and right BE */ + link_props->tdm_mask = be_index ? right_mask : left_mask; + else + /* One BE covers all of the peripheral */ + link_props->tdm_mask = left_mask | right_mask; + + /* Steal platform OF reference for use in FE links later */ + platform = link->cpus->of_node; + + link++; link_props++; + } + + of_node_put(codec); + of_node_put(cpu); + cpu = codec = NULL; + } + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) + card->dai_link[i].platforms->of_node = platform; + + return 0; + +err_free: + of_node_put(codec); + of_node_put(cpu); + of_node_put(np); + + if (!card->dai_link) + return ret; + + for (i = 0; i < num_links; i++) { + /* + * TODO: If we don't go through this path are the references + * freed inside ASoC? + */ + snd_soc_of_put_dai_link_codecs(&card->dai_link[i]); + snd_soc_of_put_dai_link_cpus(&card->dai_link[i]); + } + + return ret; +} + +static int macaudio_get_runtime_bclk_ratio(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dpcm *dpcm; + + /* + * If this is a FE, look it up in link_props directly. + * If this is a BE, look it up in the respective FE. + */ + if (!rtd->dai_link->no_pcm) + return ma->link_props[rtd->dai_link->id].bclk_ratio; + + for_each_dpcm_fe(rtd, substream->stream, dpcm) { + int fe_id = dpcm->fe->dai_link->id; + + return ma->link_props[fe_id].bclk_ratio; + } + + return 0; +} + +static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (bclk_ratio) { + struct snd_soc_dai *dai; + int mclk = params_rate(params) * bclk_ratio; + + for_each_rtd_codec_dais(rtd, i, dai) { + snd_soc_dai_set_sysclk(dai, 0, mclk, SND_SOC_CLOCK_IN); + snd_soc_dai_set_bclk_ratio(dai, bclk_ratio); + } + + snd_soc_dai_set_sysclk(cpu_dai, 0, mclk, SND_SOC_CLOCK_OUT); + snd_soc_dai_set_bclk_ratio(cpu_dai, bclk_ratio); + } + + return 0; +} + +static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *dai; + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (bclk_ratio) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_dai_set_sysclk(dai, 0, 0, SND_SOC_CLOCK_IN); + + snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT); + } +} + +static const struct snd_soc_ops macaudio_fe_ops = { + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_dpcm_hw_params, +}; + +static const struct snd_soc_ops macaudio_be_ops = { + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_dpcm_hw_params, +}; + +static int macaudio_be_assign_tdm(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + unsigned int mask; + int nslots, ret, i; + + if (!props->tdm_mask) + return 0; + + mask = props->tdm_mask; + nslots = __fls(mask) + 1; + + if (rtd->dai_link->num_codecs == 1) { + ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 0), mask, + 0, nslots, MACAUDIO_SLOTWIDTH); + + /* + * Headphones get a pass on -ENOTSUPP (see the comment + * around bclk_ratio value for primary FE). + */ + if (ret == -ENOTSUPP && props->is_headphones) + return 0; + + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + int slot = __ffs(mask); + + mask &= ~(1 << slot); + ret = snd_soc_dai_set_tdm_slot(dai, 1 << slot, 0, nslots, + MACAUDIO_SLOTWIDTH); + if (ret) + return ret; + } + + return 0; +} + +static int macaudio_be_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i, ret; + + ret = macaudio_be_assign_tdm(rtd); + if (ret < 0) + return ret; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, &ma->jack, NULL); + } + + return 0; +} + +static void macaudio_be_exit(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, NULL, NULL); + } +} + +static int macaudio_fe_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int nslots = props->bclk_ratio / MACAUDIO_SLOTWIDTH; + + return snd_soc_dai_set_tdm_slot(asoc_rtd_to_cpu(rtd, 0), (1 << nslots) - 1, + (1 << nslots) - 1, nslots, MACAUDIO_SLOTWIDTH); +} + +static struct snd_soc_jack_pin macaudio_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, + { + .pin = "Speaker", + .mask = SND_JACK_HEADPHONE, + .invert = 1, + }, +}; + +static int macaudio_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + int ret; + + dev_dbg(card->dev, "%s!\n", __func__); + + ret = snd_soc_card_jack_new_pins(card, "Headphone Jack", + SND_JACK_HEADSET | SND_JACK_HEADPHONE, + &ma->jack, macaudio_jack_pins, + ARRAY_SIZE(macaudio_jack_pins)); + if (ret < 0) { + dev_err(card->dev, "jack creation failed: %d\n", ret); + return ret; + } + + return ret; +} + +static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_soc_dai *dai, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_route *r; + int nroutes = 0; + int ret; + + memset(routes, 0, sizeof(routes)); + + dev_dbg(card->dev, "adding routes for '%s'\n", dai->name); + + r = &routes[nroutes++]; + if (is_speakers) + r->source = "Speaker Playback"; + else + r->source = "Headphone Playback"; + r->sink = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget->name; + + /* If headphone jack, add capture path */ + if (!is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Headphone Capture"; + } + + ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + dai->name); + return ret; +} + +static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_component *component, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[1]; + struct snd_soc_dapm_route *r; + int nroutes = 0; + char buf[32]; + int ret; + + memset(routes, 0, sizeof(routes)); + + /* Connect the far ends of CODECs to pins */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = "OUT"; + if (component->name_prefix) { + snprintf(buf, sizeof(buf) - 1, "%s OUT", component->name_prefix); + r->source = buf; + } + r->sink = "Speaker Pin Demux"; + } else { + r = &routes[nroutes++]; + r->source = "Jack HP"; + r->sink = "Headphone"; + } + + + ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + component->name); + return ret; +} + +static int macaudio_late_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dai *dai; + int ret, i; + + /* Add the dynamic DAPM routes */ + for_each_card_rtds(card, rtd) { + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (!rtd->dai_link->no_pcm) + continue; + + for_each_rtd_cpu_dais(rtd, i, dai) { + ret = macaudio_add_backend_dai_route(card, dai, props->is_speakers); + + if (ret) + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + ret = macaudio_add_pin_routes(card, dai->component, + props->is_speakers); + + if (ret) + return ret; + } + } + + return 0; +} + +#define CHECK(call, pattern, value) \ + { \ + int ret = call(card, pattern, value); \ + if (ret < 1 && !void_warranty) { \ + dev_err(card->dev, "%s on '%s': %d\n", #call, pattern, ret); \ + return ret; \ + } \ + dev_dbg(card->dev, "%s on '%s': %d hits\n", #call, pattern, ret); \ + } + + +static int macaudio_j274_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (ma->has_speakers) { + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 14); // 20 set by macOS, this is 3 dB below + } + + return 0; +} + +static int macaudio_j314_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (ma->has_speakers) { + CHECK(snd_soc_set_enum_kctl, "* ASI1 Sel", "Left"); + CHECK(snd_soc_deactivate_kctl, "* ASI1 Sel", 0); + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 9); // 15 set by macOS, this is 3 dB below + CHECK(snd_soc_set_enum_kctl, "* Tweeter HPF Corner Freq", "800 Hz"); + CHECK(snd_soc_deactivate_kctl, "* Tweeter HPF Corner Freq", 0); + + /* + * The speaker amps suffer from spurious overcurrent + * events on their unmute, so enable autoretry. + */ + CHECK(snd_soc_set_enum_kctl, "* OCE Handling", "Retry"); + CHECK(snd_soc_deactivate_kctl, "* OCE Handling", 0); + + /* + * Since we don't set the right slots yet to avoid + * driver conflict on the I2S bus sending ISENSE/VSENSE + * samples from the codecs back to us, disable the + * controls. + */ + CHECK(snd_soc_deactivate_kctl, "* VSENSE Switch", 0); + CHECK(snd_soc_deactivate_kctl, "* ISENSE Switch", 0); + } + + return 0; +} + +static int macaudio_fallback_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (ma->has_speakers && !void_warranty) { + dev_err(card->dev, "driver can't assure safety on this model, refusing probe\n"); + return -EINVAL; + } + + return 0; +} + +#undef CHECK + +static const char * const macaudio_spk_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_spk_mux_enum, macaudio_spk_mux_texts); + +static const struct snd_kcontrol_new macaudio_spk_mux = + SOC_DAPM_ENUM("Speaker Playback Mux", macaudio_spk_mux_enum); + +static const char * const macaudio_hp_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_hp_mux_enum, macaudio_hp_mux_texts); + +static const struct snd_kcontrol_new macaudio_hp_mux = + SOC_DAPM_ENUM("Headphones Playback Mux", macaudio_hp_mux_enum); + +static const char *macaudio_spk_demux_texts[] = { + "Inverse Jack", "Static", +}; + +static SOC_ENUM_SINGLE_DECL(macaudio_spk_demux_enum, + SND_SOC_NOPM, 0, macaudio_spk_demux_texts); + +static const struct snd_kcontrol_new macaudio_spk_demux = + SOC_DAPM_ENUM("Speaker Pin Demux", macaudio_spk_demux_enum); + +static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { + SND_SOC_DAPM_SPK("Speaker", NULL), + SND_SOC_DAPM_SPK("Speaker (Static)", NULL), + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), + + SND_SOC_DAPM_MUX("Speaker Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_spk_mux), + SND_SOC_DAPM_MUX("Headphone Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_hp_mux), + SND_SOC_DAPM_DEMUX("Speaker Pin Demux", SND_SOC_NOPM, 0, 0, &macaudio_spk_demux), + + SND_SOC_DAPM_AIF_OUT("Speaker Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_OUT("Headphone Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + + SND_SOC_DAPM_AIF_IN("Headphone Capture", NULL, 0, SND_SOC_NOPM, 0, 0), +}; + +static const struct snd_kcontrol_new macaudio_controls[] = { + SOC_DAPM_PIN_SWITCH("Speaker"), + SOC_DAPM_PIN_SWITCH("Speaker (Static)"), + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), +}; + +static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { + /* Playback paths */ + { "Speaker Playback Mux", "Primary", "PCM0 TX" }, + { "Speaker Playback Mux", "Secondary", "PCM1 TX" }, + { "Speaker Playback", NULL, "Speaker Playback Mux"}, + + { "Headphone Playback Mux", "Primary", "PCM0 TX" }, + { "Headphone Playback Mux", "Secondary", "PCM1 TX" }, + { "Headphone Playback", NULL, "Headphone Playback Mux"}, + /* + * Additional paths (to specific I2S ports) are added dynamically. + */ + + { "Speaker", "Inverse Jack", "Speaker Pin Demux" }, + { "Speaker (Static)", "Static", "Speaker Pin Demux" }, + + /* Capture paths */ + { "PCM0 RX", NULL, "Headphone Capture" }, +}; + +static const struct of_device_id macaudio_snd_device_id[] = { + { .compatible = "apple,j274-macaudio", .data = macaudio_j274_fixup_controls }, + { .compatible = "apple,j314-macaudio", .data = macaudio_j314_fixup_controls }, + { .compatible = "apple,macaudio"}, + { } +}; +MODULE_DEVICE_TABLE(of, macaudio_snd_device_id); + +static int macaudio_snd_platform_probe(struct platform_device *pdev) +{ + struct snd_soc_card *card; + struct macaudio_snd_data *data; + struct device *dev = &pdev->dev; + struct snd_soc_dai_link *link; + const struct of_device_id *of_id; + int ret; + int i; + + of_id = of_match_device(macaudio_snd_device_id, dev); + if (!of_id) + return -EINVAL; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + card = &data->card; + snd_soc_card_set_drvdata(card, data); + + card->owner = THIS_MODULE; + card->driver_name = DRIVER_NAME; + card->dev = dev; + card->dapm_widgets = macaudio_snd_widgets; + card->num_dapm_widgets = ARRAY_SIZE(macaudio_snd_widgets); + card->dapm_routes = macaudio_dapm_routes; + card->num_dapm_routes = ARRAY_SIZE(macaudio_dapm_routes); + card->controls = macaudio_controls; + card->num_controls = ARRAY_SIZE(macaudio_controls); + card->probe = macaudio_probe; + card->late_probe = macaudio_late_probe; + card->component_chaining = true; + card->fully_routed = true; + + if (of_id->data) + card->fixup_controls = of_id->data; + else + card->fixup_controls = macaudio_fallback_fixup_controls; + + ret = macaudio_parse_of(data); + if (ret) + return dev_err_probe(&pdev->dev, ret, "failed OF parsing\n"); + + for_each_card_prelinks(card, i, link) { + if (link->no_pcm) { + link->ops = &macaudio_be_ops; + link->init = macaudio_be_init; + link->exit = macaudio_be_exit; + } else { + link->ops = &macaudio_fe_ops; + link->init = macaudio_fe_init; + } + } + + return devm_snd_soc_register_card(dev, card); +} + +static struct platform_driver macaudio_snd_driver = { + .probe = macaudio_snd_platform_probe, + .driver = { + .name = DRIVER_NAME, + .of_match_table = macaudio_snd_device_id, + .pm = &snd_soc_pm_ops, + }, +}; +module_platform_driver(macaudio_snd_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Apple Silicon Macs machine-level sound driver"); +MODULE_LICENSE("GPL"); From d760f60280f19fed31463edb74fb1cc51c2596ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Wed, 3 Aug 2022 17:25:43 +0200 Subject: [PATCH 071/635] ASoC: cs42l42: Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/codecs/cs42l42.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index e3edaa1a2761e8..a4a357f9ae6feb 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -1672,7 +1672,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) return IRQ_NONE; } - /* Read sticky registers to clear interurpt */ + /* Read sticky registers to clear interrupt */ for (i = 0; i < ARRAY_SIZE(stickies); i++) { regmap_read(cs42l42->regmap, irq_params_table[i].status_addr, &(stickies[i])); From ff7cb9d1793f2a27bd534abfe91921accf419137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Tue, 6 Sep 2022 14:51:29 +0200 Subject: [PATCH 072/635] ASoC: cs42l42: Do not advertise sample bit symmetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/codecs/cs42l42.c | 1 - 1 file changed, 1 deletion(-) diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index a4a357f9ae6feb..b4f22c8ab2a073 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -1144,7 +1144,6 @@ struct snd_soc_dai_driver cs42l42_dai = { .formats = CS42L42_FORMATS, }, .symmetric_rate = 1, - .symmetric_sample_bits = 1, .ops = &cs42l42_ops, }; EXPORT_SYMBOL_NS_GPL(cs42l42_dai, SND_SOC_CS42L42_CORE); From c17b12b88a060442d5db8b93991ed3d826ea5b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sat, 20 Aug 2022 20:50:24 +0200 Subject: [PATCH 073/635] dt-bindings: sound: Add CS42L84 codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CS42L84 is a headphone jack codec made by Cirrus Logic and seen in Apple computer models starting with 2021 Macbook Pros. It is not a publicly documented part. To a degree the part is similar to the public CS42L42. (The L84 superseded L83 seen in earlier Apple models, and the L83 was pretty much the same as L42.) Signed-off-by: Martin Povišer --- sound/soc/codecs/cirrus,cs42l84.yaml | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 sound/soc/codecs/cirrus,cs42l84.yaml diff --git a/sound/soc/codecs/cirrus,cs42l84.yaml b/sound/soc/codecs/cirrus,cs42l84.yaml new file mode 100644 index 00000000000000..12bc6dbeeddfac --- /dev/null +++ b/sound/soc/codecs/cirrus,cs42l84.yaml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/cirrus,cs42l84.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Cirrus Logic CS42L84 audio CODEC + +maintainers: + - povik+lin@cutebit.org + +description: + The CS42L84 is a headphone jack codec made by Cirrus Logic and embedded + in personal computers sold by Apple. It was first seen in 2021 Macbook Pro + models. + + It has stereo DAC for playback, mono ADC for capture, and is somewhat + similar to CS42L42 but with a different regmap. + +properties: + compatible: + enum: + - cirrus,cs42l84 + + reg: + description: + I2C address of the device + maxItems: 1 + + reset-gpios: + description: + Reset pin, asserted to reset the device, deasserted to bring + the device online + maxItems: 1 + + interrupts: + description: + Interrupt for the IRQ output line of the device + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_LOW>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + }; + }; From 55b1d8025ed4a53f0c6d35a82afe3581354380d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Wed, 29 Jun 2022 20:32:14 +0200 Subject: [PATCH 074/635] wip: ASoC: cs42l84: Start new codec driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/codecs/Kconfig | 5 + sound/soc/codecs/Makefile | 2 + sound/soc/codecs/cs42l84.c | 1045 ++++++++++++++++++++++++++++++++++++ sound/soc/codecs/cs42l84.h | 214 ++++++++ 4 files changed, 1266 insertions(+) create mode 100644 sound/soc/codecs/cs42l84.c create mode 100644 sound/soc/codecs/cs42l84.h diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index 07747565c3b51c..b1cd0de2786a40 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -74,6 +74,7 @@ config SND_SOC_ALL_CODECS imply SND_SOC_CS42L52 imply SND_SOC_CS42L56 imply SND_SOC_CS42L73 + imply SND_SOC_CS42L84 imply SND_SOC_CS4234 imply SND_SOC_CS4265 imply SND_SOC_CS4270 @@ -755,6 +756,10 @@ config SND_SOC_CS42L83 select REGMAP_I2C select SND_SOC_CS42L42_CORE +config SND_SOC_CS42L84 + tristate "Cirrus Logic CS42L84 CODEC" + depends on I2C + config SND_SOC_CS4234 tristate "Cirrus Logic CS4234 CODEC" depends on I2C diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile index f1ca18f7946cac..a39e3ca7984f80 100644 --- a/sound/soc/codecs/Makefile +++ b/sound/soc/codecs/Makefile @@ -75,6 +75,7 @@ snd-soc-cs42l52-objs := cs42l52.o snd-soc-cs42l56-objs := cs42l56.o snd-soc-cs42l73-objs := cs42l73.o snd-soc-cs42l83-i2c-objs := cs42l83-i2c.o +snd-soc-cs42l84-objs := cs42l84.o snd-soc-cs4234-objs := cs4234.o snd-soc-cs4265-objs := cs4265.o snd-soc-cs4270-objs := cs4270.o @@ -442,6 +443,7 @@ obj-$(CONFIG_SND_SOC_CS42L52) += snd-soc-cs42l52.o obj-$(CONFIG_SND_SOC_CS42L56) += snd-soc-cs42l56.o obj-$(CONFIG_SND_SOC_CS42L73) += snd-soc-cs42l73.o obj-$(CONFIG_SND_SOC_CS42L83) += snd-soc-cs42l83-i2c.o +obj-$(CONFIG_SND_SOC_CS42L84) += snd-soc-cs42l84.o obj-$(CONFIG_SND_SOC_CS4234) += snd-soc-cs4234.o obj-$(CONFIG_SND_SOC_CS4265) += snd-soc-cs4265.o obj-$(CONFIG_SND_SOC_CS4270) += snd-soc-cs4270.o diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c new file mode 100644 index 00000000000000..e65c9c80b52bc8 --- /dev/null +++ b/sound/soc/codecs/cs42l84.c @@ -0,0 +1,1045 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * cs42l84.c -- CS42L84 ALSA SoC audio driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on sound/soc/codecs/cs42l42{.c,.h} + * Copyright 2016 Cirrus Logic, Inc. + */ + +#define DEBUG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cs42l84.h" +#include "cirrus_legacy.h" + +struct cs42l84_private { + struct regmap *regmap; + struct device *dev; + struct gpio_desc *reset_gpio; + struct snd_soc_jack *jack; + struct mutex irq_lock; + u8 plug_state; + int pll_config; + int bclk; + u8 pll_mclk_f; + u32 srate; + u8 stream_use; + int hs_type; +}; + +static const struct regmap_config cs42l84_regmap = { + .reg_bits = 16, + .val_bits = 8, + + .max_register = 0xffff, + .cache_type = REGCACHE_NONE, + + .use_single_read = true, + .use_single_write = true, +}; + +static int cs42l84_put_dac_vol(struct snd_kcontrol *kctl, + struct snd_ctl_elem_value *val) +{ + struct snd_soc_component *component = snd_soc_kcontrol_component(kctl); + unsigned int vola, volb; + int ret, ret2; + + vola = val->value.integer.value[0]; + volb = val->value.integer.value[1]; + + ret = snd_soc_component_update_bits(component, CS42L84_FRZ_CTL, + CS42L84_FRZ_CTL_ENGAGE, + CS42L84_FRZ_CTL_ENGAGE); + if (ret < 0) + goto bail; + + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHA_VOL_LSB, + 0xff, vola & 0xff); + if (ret < 0) + goto bail; + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHA_VOL_MSB, + 0xff, (vola >> 8) & 0x01); + if (ret < 0) + goto bail; + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHB_VOL_LSB, + 0xff, volb & 0xff); + if (ret < 0) + goto bail; + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHB_VOL_MSB, + 0xff, (volb >> 8) & 0x01); + if (ret < 0) + goto bail; + +bail: + ret2 = snd_soc_component_update_bits(component, CS42L84_FRZ_CTL, + CS42L84_FRZ_CTL_ENGAGE, 0); + if (ret2 < 0 && ret >= 0) + ret = ret2; + + return ret; +} + +static int cs42l84_get_dac_vol(struct snd_kcontrol *kctl, + struct snd_ctl_elem_value *val) +{ + struct snd_soc_component *component = snd_soc_kcontrol_component(kctl); + unsigned int vola, volb; + int ret; + + ret = snd_soc_component_read(component, CS42L84_DAC_CHA_VOL_LSB); + if (ret < 0) + return ret; + vola = ret; + + ret = snd_soc_component_read(component, CS42L84_DAC_CHA_VOL_MSB); + if (ret < 0) + return ret; + vola |= (ret & 1) << 8; + + ret = snd_soc_component_read(component, CS42L84_DAC_CHB_VOL_LSB); + if (ret < 0) + return ret; + volb = ret; + + ret = snd_soc_component_read(component, CS42L84_DAC_CHB_VOL_MSB); + if (ret < 0) + return ret; + volb |= (ret & 1) << 8; + + val->value.integer.value[0] = vola; + val->value.integer.value[1] = volb; + + return 0; +} + +/* TODO */ +static const DECLARE_TLV_DB_SCALE(cs42l84_dac_tlv, -25600, 50, 1); + +static const struct snd_kcontrol_new cs42l84_snd_controls[] = { + SOC_DOUBLE_R_EXT_TLV("DAC Playback Volume", CS42L84_DAC_CHA_VOL_LSB, + CS42L84_DAC_CHB_VOL_LSB, 0, 511, 0, + cs42l84_get_dac_vol, cs42l84_put_dac_vol, cs42l84_dac_tlv), + SOC_SINGLE("ADC Preamp Gain", CS42L84_ADC_CTL1, + CS42L84_ADC_CTL1_PREAMP_GAIN_SHIFT, 2, 0), + SOC_SINGLE("ADC PGA Gain", CS42L84_ADC_CTL1, + CS42L84_ADC_CTL1_PGA_GAIN_SHIFT, 31, 0), + SOC_SINGLE("ADC WNF Switch", CS42L84_ADC_CTL4, + CS42L84_ADC_CTL4_WNF_EN_SHIFT, 1, 0), + SOC_SINGLE("WNF Corner Frequency", CS42L84_ADC_CTL4, + CS42L84_ADC_CTL4_WNF_CF_SHIFT, 3, 0), + SOC_SINGLE("ADC HPF Switch", CS42L84_ADC_CTL4, + CS42L84_ADC_CTL4_HPF_EN_SHIFT, 1, 0), + SOC_SINGLE("HPF Corner Frequency", CS42L84_ADC_CTL4, + CS42L84_ADC_CTL4_HPF_CF_SHIFT, 3, 0), +}; + +static const char* const cs42l84_mux_text[] = { + "Blank", "ADC", "ASP RX CH1", "ASP RX CH2", +}; + +static const unsigned int cs42l84_mux_values[] = { + 0b0000, 0b0111, 0b1101, 0b1110, +}; + +static SOC_VALUE_ENUM_SINGLE_DECL(cs42l84_daca_mux_enum, + CS42L84_BUS_DAC_SRC, CS42L84_BUS_DAC_SRC_DACA_SHIFT, + 0b1111, cs42l84_mux_text, cs42l84_mux_values); + +static SOC_VALUE_ENUM_SINGLE_DECL(cs42l84_dacb_mux_enum, + CS42L84_BUS_DAC_SRC, CS42L84_BUS_DAC_SRC_DACB_SHIFT, + 0b1111, cs42l84_mux_text, cs42l84_mux_values); + +static SOC_VALUE_ENUM_SINGLE_DECL(cs42l84_sdout1_mux_enum, + CS42L84_BUS_ASP_TX_SRC, CS42L84_BUS_ASP_TX_SRC_CH1_SHIFT, + 0b1111, cs42l84_mux_text, cs42l84_mux_values); + +static const struct snd_kcontrol_new cs42l84_daca_mux_ctrl = + SOC_DAPM_ENUM("DACA Select", cs42l84_daca_mux_enum); + +static const struct snd_kcontrol_new cs42l84_dacb_mux_ctrl = + SOC_DAPM_ENUM("DACB Select", cs42l84_dacb_mux_enum); + +static const struct snd_kcontrol_new cs42l84_sdout1_mux_ctrl = + SOC_DAPM_ENUM("SDOUT1 Select", cs42l84_sdout1_mux_enum); + +static const struct snd_soc_dapm_widget cs42l84_dapm_widgets[] = { + /* Playback Path */ + SND_SOC_DAPM_OUTPUT("HP"), + SND_SOC_DAPM_DAC("DAC", NULL, CS42L84_MSM_BLOCK_EN2, CS42L84_MSM_BLOCK_EN2_DAC_SHIFT, 0), + SND_SOC_DAPM_MUX("DACA Select", SND_SOC_NOPM, 0, 0, &cs42l84_daca_mux_ctrl), + SND_SOC_DAPM_MUX("DACB Select", SND_SOC_NOPM, 0, 0, &cs42l84_dacb_mux_ctrl), + SND_SOC_DAPM_AIF_IN("SDIN1", NULL, 0, CS42L84_ASP_RX_EN, CS42L84_ASP_RX_EN_CH1_SHIFT, 0), + SND_SOC_DAPM_AIF_IN("SDIN2", NULL, 1, CS42L84_ASP_RX_EN, CS42L84_ASP_RX_EN_CH2_SHIFT, 0), + + /* Capture Path */ + SND_SOC_DAPM_INPUT("HS"), + SND_SOC_DAPM_ADC("ADC", NULL, CS42L84_MSM_BLOCK_EN2, CS42L84_MSM_BLOCK_EN2_ADC_SHIFT, 0), + SND_SOC_DAPM_MUX("SDOUT1 Select", SND_SOC_NOPM, 0, 0, &cs42l84_sdout1_mux_ctrl), + SND_SOC_DAPM_AIF_OUT("SDOUT1", NULL, 0, CS42L84_ASP_TX_EN, CS42L84_ASP_TX_EN_CH1_SHIFT, 0), + + /* Playback/Capture Requirements */ + SND_SOC_DAPM_SUPPLY("BUS", CS42L84_MSM_BLOCK_EN2, CS42L84_MSM_BLOCK_EN2_BUS_SHIFT, 0, NULL, 0), + SND_SOC_DAPM_SUPPLY("ASP", CS42L84_MSM_BLOCK_EN2, CS42L84_MSM_BLOCK_EN2_ASP_SHIFT, 0, NULL, 0), + SND_SOC_DAPM_SUPPLY("BCLK", CS42L84_ASP_CTL, CS42L84_ASP_CTL_BCLK_EN_SHIFT, 0, NULL, 0), +}; + +static const struct snd_soc_dapm_route cs42l84_audio_map[] = { + /* Playback Path */ + {"HP", NULL, "DAC"}, + {"DAC", NULL, "DACA Select"}, + {"DAC", NULL, "DACB Select"}, + {"DACA Select", "ASP RX CH1", "SDIN1"}, + {"DACA Select", "ASP RX CH2", "SDIN2"}, + {"DACB Select", "ASP RX CH1", "SDIN1"}, + {"DACB Select", "ASP RX CH2", "SDIN2"}, + {"SDIN1", NULL, "Playback"}, + {"SDIN2", NULL, "Playback"}, + + {"ADC", NULL, "HS"}, + {"SDOUT1 Select", "ADC", "ADC"}, + {"SDOUT1", NULL, "SDOUT1 Select"}, + {"Capture", NULL, "SDOUT1"}, + + /* Playback Requirements */ + {"DAC", NULL, "BUS"}, + {"SDIN1", NULL, "ASP"}, + {"SDIN2", NULL, "ASP"}, + {"SDIN1", NULL, "BCLK"}, + {"SDIN2", NULL, "BCLK"}, + + /* Capture Requirements */ + {"SDOUT1", NULL, "BUS"}, + {"SDOUT1", NULL, "ASP"}, + {"SDOUT1", NULL, "BCLK"}, +}; + +static int cs42l84_set_jack(struct snd_soc_component *component, struct snd_soc_jack *jk, void *d) +{ + struct cs42l84_private *cs42l84 = snd_soc_component_get_drvdata(component); + + /* Prevent race with interrupt handler */ + mutex_lock(&cs42l84->irq_lock); + cs42l84->jack = jk; + snd_soc_jack_report(jk, cs42l84->hs_type, SND_JACK_HEADSET); + mutex_unlock(&cs42l84->irq_lock); + + return 0; +} + +static int cs42l84_component_probe(struct snd_soc_component *component) +{ + snd_soc_component_update_bits(component, CS42L84_ASP_CTL, + CS42L84_ASP_CTL_TDM_MODE, 0); + snd_soc_component_update_bits(component, CS42L84_HP_VOL_CTL, + CS42L84_HP_VOL_CTL_SOFT | CS42L84_HP_VOL_CTL_ZERO_CROSS, + CS42L84_HP_VOL_CTL_ZERO_CROSS); + + /* TDM settings */ + snd_soc_component_update_bits(component, CS42L84_ASP_RX_CH1_CTL1, + CS42L84_ASP_RX_CHx_CTL1_EDGE | + CS42L84_ASP_RX_CHx_CTL1_SLOT_START_LSB, 0); + snd_soc_component_update_bits(component, CS42L84_ASP_RX_CH1_CTL2, + CS42L84_ASP_RX_CHx_CTL2_SLOT_START_MSB, 0); + snd_soc_component_update_bits(component, CS42L84_ASP_RX_CH2_CTL1, + CS42L84_ASP_RX_CHx_CTL1_EDGE | + CS42L84_ASP_RX_CHx_CTL1_SLOT_START_LSB, + CS42L84_ASP_RX_CHx_CTL1_EDGE); + snd_soc_component_update_bits(component, CS42L84_ASP_RX_CH2_CTL2, + CS42L84_ASP_RX_CHx_CTL2_SLOT_START_MSB, 0); + snd_soc_component_update_bits(component, CS42L84_ASP_TX_CH1_CTL1, + CS42L84_ASP_RX_CHx_CTL1_EDGE | \ + CS42L84_ASP_RX_CHx_CTL1_SLOT_START_LSB, 0); + snd_soc_component_update_bits(component, CS42L84_ASP_TX_CH1_CTL2, + CS42L84_ASP_RX_CHx_CTL2_SLOT_START_MSB, 0); + snd_soc_component_update_bits(component, CS42L84_ASP_TX_CH2_CTL1, + CS42L84_ASP_RX_CHx_CTL1_EDGE | \ + CS42L84_ASP_RX_CHx_CTL1_SLOT_START_LSB, + CS42L84_ASP_RX_CHx_CTL1_EDGE); + snd_soc_component_update_bits(component, CS42L84_ASP_TX_CH2_CTL2, + CS42L84_ASP_RX_CHx_CTL2_SLOT_START_MSB, 0); + /* Routing defaults */ + snd_soc_component_write(component, CS42L84_BUS_DAC_SRC, + 0b1101 << CS42L84_BUS_DAC_SRC_DACA_SHIFT | + 0b1110 << CS42L84_BUS_DAC_SRC_DACB_SHIFT); + snd_soc_component_write(component, CS42L84_BUS_ASP_TX_SRC, + 0b0111 << CS42L84_BUS_ASP_TX_SRC_CH1_SHIFT); + + return 0; +} + +static const struct snd_soc_component_driver soc_component_dev_cs42l84 = { + .set_jack = cs42l84_set_jack, + .probe = cs42l84_component_probe, + .controls = cs42l84_snd_controls, + .num_controls = ARRAY_SIZE(cs42l84_snd_controls), + .dapm_widgets = cs42l84_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(cs42l84_dapm_widgets), + .dapm_routes = cs42l84_audio_map, + .num_dapm_routes = ARRAY_SIZE(cs42l84_audio_map), + .endianness = 1, +}; + +struct cs42l84_pll_params { + u32 bclk; + u8 mclk_src_sel; + u8 bclk_prediv; + u8 pll_div_int; + u32 pll_div_frac; + u8 pll_mode; + u8 pll_divout; + u32 mclk_int; +}; + +/* + * Common PLL Settings for given BCLK + */ +static const struct cs42l84_pll_params pll_ratio_table[] = { + { 3072000, 1, 0, 0x40, 0x000000, 0x03, 0x10, 12288000}, + { 6144000, 1, 1, 0x40, 0x000000, 0x03, 0x10, 12288000}, + { 12288000, 0, 0, 0, 0, 0, 0, 12288000}, + { 24576000, 1, 3, 0x40, 0x000000, 0x03, 0x10, 12288000}, +}; + +static int cs42l84_pll_config(struct snd_soc_component *component) +{ + struct cs42l84_private *cs42l84 = snd_soc_component_get_drvdata(component); + int i; + u32 clk; + u32 fsync; + + clk = cs42l84->bclk; + + /* Don't reconfigure if there is an audio stream running */ + if (cs42l84->stream_use) { + if (pll_ratio_table[cs42l84->pll_config].bclk == clk) + return 0; + else + return -EBUSY; + } + + for (i = 0; i < ARRAY_SIZE(pll_ratio_table); i++) { + if (pll_ratio_table[i].bclk == clk) { + cs42l84->pll_config = i; + break; + } + } + + if (i == ARRAY_SIZE(pll_ratio_table)) + return -EINVAL; + + /* Set up the LRCLK */ + fsync = clk / cs42l84->srate; + if (((fsync * cs42l84->srate) != clk) + || ((fsync % 2) != 0)) { + dev_err(component->dev, + "Unsupported bclk %d/sample rate %d\n", + clk, cs42l84->srate); + return -EINVAL; + } + + /* Set the LRCLK period */ + snd_soc_component_update_bits(component, CS42L84_ASP_FSYNC_CTL2, + CS42L84_ASP_FSYNC_CTL2_BCLK_PERIOD_LO, + FIELD_PREP(CS42L84_ASP_FSYNC_CTL2_BCLK_PERIOD_LO, fsync & 0x7f)); + snd_soc_component_update_bits(component, CS42L84_ASP_FSYNC_CTL3, + CS42L84_ASP_FSYNC_CTL3_BCLK_PERIOD_HI, + FIELD_PREP(CS42L84_ASP_FSYNC_CTL3_BCLK_PERIOD_HI, fsync >> 7)); + + /* Save what the MCLK will be */ + switch (pll_ratio_table[i].mclk_int) { + case 12000000: + cs42l84->pll_mclk_f = CS42L84_CCM_CTL1_MCLK_F_12MHZ; + break; + case 12288000: + cs42l84->pll_mclk_f = CS42L84_CCM_CTL1_MCLK_F_12_288KHZ; + break; + case 24000000: + cs42l84->pll_mclk_f = CS42L84_CCM_CTL1_MCLK_F_24MHZ; + break; + case 24576000: + cs42l84->pll_mclk_f = CS42L84_CCM_CTL1_MCLK_F_24_576KHZ; + break; + } + + if (pll_ratio_table[i].mclk_src_sel) { + /* Configure PLL */ + snd_soc_component_update_bits(component, + CS42L84_CCM_CTL3, CS42L84_CCM_CTL3_REFCLK_DIV, + FIELD_PREP(CS42L84_CCM_CTL3_REFCLK_DIV, pll_ratio_table[i].bclk_prediv)); + snd_soc_component_write(component, + CS42L84_PLL_DIV_INT, + pll_ratio_table[i].pll_div_int); + snd_soc_component_write(component, + CS42L84_PLL_DIV_FRAC0, + pll_ratio_table[i].pll_div_frac); + snd_soc_component_write(component, + CS42L84_PLL_DIV_FRAC1, + pll_ratio_table[i].pll_div_frac >> 8); + snd_soc_component_write(component, + CS42L84_PLL_DIV_FRAC2, + pll_ratio_table[i].pll_div_frac >> 16); + snd_soc_component_update_bits(component, + CS42L84_PLL_CTL1, CS42L84_PLL_CTL1_MODE, + FIELD_PREP(CS42L84_PLL_CTL1_MODE, pll_ratio_table[i].pll_mode)); + snd_soc_component_write(component, + CS42L84_PLL_DIVOUT, + pll_ratio_table[i].pll_divout); + + snd_soc_component_update_bits(component, + CS42L84_PLL_CTL1, CS42L84_PLL_CTL1_EN, + CS42L84_PLL_CTL1_EN); + } + + return 0; +} + +static int cs42l84_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) +{ + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: + break; + default: + return -EINVAL; + } + + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + break; + default: + return -EINVAL; + } + + /* Bitclock/frame inversion */ + switch (fmt & SND_SOC_DAIFMT_INV_MASK) { + case SND_SOC_DAIFMT_IB_IF: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int cs42l84_pcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct cs42l84_private *cs42l84 = snd_soc_component_get_drvdata(component); + int ret; + u32 ccm_samp_rate; + + cs42l84->srate = params_rate(params); + + ret = cs42l84_pll_config(component); + if (ret) + return ret; + + switch (params_rate(params)) { + case 44100: + ccm_samp_rate = CS42L84_CCM_SAMP_RATE_RATE_44K1HZ; + break; + case 48000: + ccm_samp_rate = CS42L84_CCM_SAMP_RATE_RATE_48KHZ; + break; + case 88200: + ccm_samp_rate = CS42L84_CCM_SAMP_RATE_RATE_88K2HZ; + break; + case 96000: + ccm_samp_rate = CS42L84_CCM_SAMP_RATE_RATE_96KHZ; + break; + case 176400: + ccm_samp_rate = CS42L84_CCM_SAMP_RATE_RATE_176K4HZ; + break; + case 192000: + ccm_samp_rate = CS42L84_CCM_SAMP_RATE_RATE_192KHZ; + break; + default: + return -EINVAL; + } + + snd_soc_component_write(component, CS42L84_CCM_SAMP_RATE, ccm_samp_rate); + + switch (substream->stream) { + case SNDRV_PCM_STREAM_PLAYBACK: + snd_soc_component_write(component, CS42L84_ASP_RX_CH1_WIDTH, + params_width(params) - 1); + snd_soc_component_write(component, CS42L84_ASP_RX_CH2_WIDTH, + params_width(params) - 1); + break; + + case SNDRV_PCM_STREAM_CAPTURE: + snd_soc_component_write(component, CS42L84_ASP_TX_CH1_WIDTH, + params_width(params) - 1); + snd_soc_component_write(component, CS42L84_ASP_TX_CH2_WIDTH, + params_width(params) - 1); + break; + } + + return 0; +} + +static int cs42l84_set_sysclk(struct snd_soc_dai *dai, + int clk_id, unsigned int freq, int dir) +{ + struct snd_soc_component *component = dai->component; + struct cs42l84_private *cs42l84 = snd_soc_component_get_drvdata(component); + int i; + + if (freq == 0) { + cs42l84->bclk = 0; + return 0; + } + + for (i = 0; i < ARRAY_SIZE(pll_ratio_table); i++) { + if (pll_ratio_table[i].bclk == freq) { + cs42l84->bclk = freq; + return 0; + } + } + + dev_err(component->dev, "BCLK %u not supported\n", freq); + + return -EINVAL; +} + +static int cs42l84_mute_stream(struct snd_soc_dai *dai, int mute, int stream) +{ + struct snd_soc_component *component = dai->component; + struct cs42l84_private *cs42l84 = snd_soc_component_get_drvdata(component); + unsigned int regval; + int ret; + + if (mute) { + /* Mute the headphone */ + if (stream == SNDRV_PCM_STREAM_PLAYBACK) + snd_soc_component_update_bits(component, CS42L84_DAC_CTL1, + CS42L84_DAC_CTL1_UNMUTE, 0); + cs42l84->stream_use &= ~(1 << stream); + if (!cs42l84->stream_use) { + /* Must disconnect PLL before stopping it */ + snd_soc_component_write(component, CS42L84_CCM_CTL1, + CS42L84_CCM_CTL1_RCO); + + usleep_range(150, 300); + + snd_soc_component_update_bits(component, CS42L84_PLL_CTL1, + CS42L84_PLL_CTL1_EN, 0); + + snd_soc_component_update_bits(component, CS42L84_CCM_CTL4, + CS42L84_CCM_CTL4_REFCLK_EN, 0); + } + } else { + if (!cs42l84->stream_use) { + /* SCLK must be running before codec unmute. + * + * Note carried over from CS42L42: + * + * PLL must not be started with ADC and HP both off + * otherwise the FILT+ supply will not charge properly. + * DAPM widgets power-up before stream unmute so at least + * one of the "DAC" or "ADC" widgets will already have + * powered-up. + */ + + snd_soc_component_update_bits(component, CS42L84_CCM_CTL4, + CS42L84_CCM_CTL4_REFCLK_EN, + CS42L84_CCM_CTL4_REFCLK_EN); + + if (pll_ratio_table[cs42l84->pll_config].mclk_src_sel) { + snd_soc_component_update_bits(component, CS42L84_PLL_CTL1, + CS42L84_PLL_CTL1_EN, + CS42L84_PLL_CTL1_EN); + /* TODO: should we be doing something with divout here? */ + + ret = regmap_read_poll_timeout(cs42l84->regmap, + CS42L84_PLL_LOCK_STATUS, + regval, + (regval & CS42L84_PLL_LOCK_STATUS_LOCKED), + CS42L84_PLL_LOCK_POLL_US, + CS42L84_PLL_LOCK_TIMEOUT_US); + if (ret < 0) + dev_warn(component->dev, "PLL failed to lock: %d\n", ret); + + /* PLL must be running to drive glitchless switch logic */ + snd_soc_component_update_bits(component, + CS42L84_CCM_CTL1, + CS42L84_CCM_CTL1_MCLK_SRC | CS42L84_CCM_CTL1_MCLK_FREQ, + FIELD_PREP(CS42L84_CCM_CTL1_MCLK_SRC, CS42L84_CCM_CTL1_MCLK_SRC_PLL) + | FIELD_PREP(CS42L84_CCM_CTL1_MCLK_FREQ, cs42l84->pll_mclk_f)); + usleep_range(CS42L84_CLOCK_SWITCH_DELAY_US, CS42L84_CLOCK_SWITCH_DELAY_US*2); + } else { + snd_soc_component_update_bits(component, + CS42L84_CCM_CTL1, + CS42L84_CCM_CTL1_MCLK_SRC | CS42L84_CCM_CTL1_MCLK_FREQ, + FIELD_PREP(CS42L84_CCM_CTL1_MCLK_SRC, CS42L84_CCM_CTL1_MCLK_SRC_BCLK) + | FIELD_PREP(CS42L84_CCM_CTL1_MCLK_FREQ, cs42l84->pll_mclk_f)); + usleep_range(CS42L84_CLOCK_SWITCH_DELAY_US, CS42L84_CLOCK_SWITCH_DELAY_US*2); + } + } + cs42l84->stream_use |= 1 << stream; + + if (stream == SNDRV_PCM_STREAM_PLAYBACK) + /* Un-mute the headphone */ + snd_soc_component_update_bits(component, CS42L84_DAC_CTL1, + CS42L84_DAC_CTL1_UNMUTE, + CS42L84_DAC_CTL1_UNMUTE); + } + + return 0; +} + +static const struct snd_soc_dai_ops cs42l84_ops = { + .hw_params = cs42l84_pcm_hw_params, + .set_fmt = cs42l84_set_dai_fmt, + .set_sysclk = cs42l84_set_sysclk, + .mute_stream = cs42l84_mute_stream, +}; + +#define CS42L84_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\ + SNDRV_PCM_FMTBIT_S24_LE |\ + SNDRV_PCM_FMTBIT_S32_LE) + +static struct snd_soc_dai_driver cs42l84_dai = { + .name = "cs42l84", + .playback = { + .stream_name = "Playback", + .channels_min = 1, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000, + .formats = CS42L84_FORMATS, + }, + .capture = { + .stream_name = "Capture", + .channels_min = 1, + .channels_max = 1, + .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000, + .formats = CS42L84_FORMATS, + }, + .symmetric_rate = 1, + .symmetric_sample_bits = 1, + .ops = &cs42l84_ops, +}; + +struct cs42l84_irq_params { + u16 status_addr; + u16 mask_addr; + u8 mask; +}; + +static const struct cs42l84_irq_params irq_params_table[] = { + {CS42L84_TSRS_PLUG_INT_STATUS, CS42L84_TSRS_PLUG_INT_MASK, + CS42L84_TSRS_PLUG_VAL_MASK} +}; + +static void cs42l84_detect_hs(struct cs42l84_private *cs42l84) +{ + unsigned int reg; + + /* Power up HSBIAS */ + regmap_update_bits(cs42l84->regmap, + CS42L84_MISC_DET_CTL, + CS42L84_MISC_DET_CTL_HSBIAS_CTL | CS42L84_MISC_DET_CTL_DETECT_MODE, + FIELD_PREP(CS42L84_MISC_DET_CTL_HSBIAS_CTL, 3) | /* 2.7 V */ + FIELD_PREP(CS42L84_MISC_DET_CTL_DETECT_MODE, 0)); + + /* Power up level detection circuitry */ + regmap_update_bits(cs42l84->regmap, + CS42L84_MISC_DET_CTL, + CS42L84_MISC_DET_CTL_PDN_MIC_LVL_DET, 0); + + /* TODO: Optimize */ + msleep(100); + + /* Connect HSBIAS in CTIA wiring */ + /* TODO: Should likely be subject of detection */ + regmap_write(cs42l84->regmap, + CS42L84_HS_SWITCH_CTL, + CS42L84_HS_SWITCH_CTL_REF_HS3 | \ + CS42L84_HS_SWITCH_CTL_HSB_FILT_HS3 | \ + CS42L84_HS_SWITCH_CTL_GNDHS_HS3 | \ + CS42L84_HS_SWITCH_CTL_HSB_HS4); + regmap_update_bits(cs42l84->regmap, + CS42L84_HS_DET_CTL2, + CS42L84_HS_DET_CTL2_SET, + FIELD_PREP(CS42L84_HS_DET_CTL2_SET, 0)); + + regmap_update_bits(cs42l84->regmap, + CS42L84_MISC_DET_CTL, + CS42L84_MISC_DET_CTL_DETECT_MODE, + FIELD_PREP(CS42L84_MISC_DET_CTL_DETECT_MODE, 3)); + + /* TODO: Optimize */ + msleep(100); + + regmap_read(cs42l84->regmap, CS42L84_HS_DET_STATUS2, ®); + regmap_update_bits(cs42l84->regmap, + CS42L84_MISC_DET_CTL, + CS42L84_MISC_DET_CTL_PDN_MIC_LVL_DET, + CS42L84_MISC_DET_CTL_PDN_MIC_LVL_DET); + + switch (reg & 0b11) { + case 0b11: /* shorted */ + case 0b00: /* open */ + /* Power down HSBIAS */ + regmap_update_bits(cs42l84->regmap, + CS42L84_MISC_DET_CTL, + CS42L84_MISC_DET_CTL_HSBIAS_CTL, + FIELD_PREP(CS42L84_MISC_DET_CTL_HSBIAS_CTL, 1)); /* 0.0 V */ + break; + } + + switch (reg & 0b11) { + case 0b10: /* load */ + dev_dbg(cs42l84->dev, "Detected mic\n"); + cs42l84->hs_type = SND_JACK_HEADSET; + snd_soc_jack_report(cs42l84->jack, SND_JACK_HEADSET, + SND_JACK_HEADSET); + break; + + case 0b00: /* open */ + dev_dbg(cs42l84->dev, "Detected line-in\n"); + cs42l84->hs_type = SND_JACK_HEADSET; + snd_soc_jack_report(cs42l84->jack, SND_JACK_HEADSET, + SND_JACK_HEADSET); + break; + + case 0b11: /* shorted */ + default: + snd_soc_jack_report(cs42l84->jack, SND_JACK_HEADPHONE, + SND_JACK_HEADSET); + cs42l84->hs_type = SND_JACK_HEADPHONE; + dev_dbg(cs42l84->dev, "Detected bare headphone (no mic)\n"); + } +} + +static void cs42l84_revert_hs(struct cs42l84_private *cs42l84) +{ + /* Power down HSBIAS */ + regmap_update_bits(cs42l84->regmap, + CS42L84_MISC_DET_CTL, + CS42L84_MISC_DET_CTL_HSBIAS_CTL | CS42L84_MISC_DET_CTL_DETECT_MODE, + FIELD_PREP(CS42L84_MISC_DET_CTL_HSBIAS_CTL, 1) | /* 0.0 V */ + FIELD_PREP(CS42L84_MISC_DET_CTL_DETECT_MODE, 0)); + + /* Disconnect HSBIAS */ + regmap_write(cs42l84->regmap, + CS42L84_HS_SWITCH_CTL, + CS42L84_HS_SWITCH_CTL_REF_HS3 | \ + CS42L84_HS_SWITCH_CTL_REF_HS4 | \ + CS42L84_HS_SWITCH_CTL_HSB_FILT_HS3 | \ + CS42L84_HS_SWITCH_CTL_HSB_FILT_HS4 | \ + CS42L84_HS_SWITCH_CTL_GNDHS_HS3 | \ + CS42L84_HS_SWITCH_CTL_GNDHS_HS4); + regmap_update_bits(cs42l84->regmap, + CS42L84_HS_DET_CTL2, + CS42L84_HS_DET_CTL2_SET, + FIELD_PREP(CS42L84_HS_DET_CTL2_SET, 2)); +} + +static irqreturn_t cs42l84_irq_thread(int irq, void *data) +{ + struct cs42l84_private *cs42l84 = (struct cs42l84_private *)data; + unsigned int stickies[1]; + unsigned int masks[1]; + unsigned int reg; + u8 current_plug_status; + int i; + + mutex_lock(&cs42l84->irq_lock); + /* Read sticky registers to clear interurpt */ + for (i = 0; i < ARRAY_SIZE(stickies); i++) { + regmap_read(cs42l84->regmap, irq_params_table[i].status_addr, + &(stickies[i])); + regmap_read(cs42l84->regmap, irq_params_table[i].mask_addr, + &(masks[i])); + stickies[i] = stickies[i] & (~masks[i]) & + irq_params_table[i].mask; + } + + if ((~masks[0]) & irq_params_table[0].mask) { + regmap_read(cs42l84->regmap, CS42L84_TSRS_PLUG_STATUS, ®); + current_plug_status = (((char) reg) & + (CS42L84_TS_PLUG | CS42L84_TS_UNPLUG)) >> + CS42L84_TS_PLUG_SHIFT; + + switch (current_plug_status) { + case CS42L84_PLUG: + if (cs42l84->plug_state != CS42L84_PLUG) { + cs42l84->plug_state = CS42L84_PLUG; + dev_dbg(cs42l84->dev, "Plug event\n"); + + cs42l84_detect_hs(cs42l84); + + /* + * Check the tip sense status again, and possibly invalidate + * the detection result + * + * Thanks to debounce, this should reliably indicate if the tip + * was disconnected at any point during the detection procedure. + */ + regmap_read(cs42l84->regmap, CS42L84_TSRS_PLUG_STATUS, ®); + current_plug_status = (((char) reg) & + (CS42L84_TS_PLUG | CS42L84_TS_UNPLUG)) >> + CS42L84_TS_PLUG_SHIFT; + if (current_plug_status != CS42L84_PLUG) { + dev_dbg(cs42l84->dev, "Wobbly connection, detection invalidated\n"); + cs42l84->plug_state = CS42L84_UNPLUG; + cs42l84_revert_hs(cs42l84); + } + } + break; + + case CS42L84_UNPLUG: + if (cs42l84->plug_state != CS42L84_UNPLUG) { + cs42l84->plug_state = CS42L84_UNPLUG; + dev_dbg(cs42l84->dev, "Unplug event\n"); + + cs42l84_revert_hs(cs42l84); + cs42l84->hs_type = 0; + snd_soc_jack_report(cs42l84->jack, 0, + SND_JACK_HEADSET); + } + break; + + default: + if (cs42l84->plug_state != CS42L84_TRANS) + cs42l84->plug_state = CS42L84_TRANS; + } + } + mutex_unlock(&cs42l84->irq_lock); + + return IRQ_HANDLED; +} + +static void cs42l84_set_interrupt_masks(struct cs42l84_private *cs42l84) +{ + regmap_update_bits(cs42l84->regmap, CS42L84_TSRS_PLUG_INT_MASK, + CS42L84_RS_PLUG | CS42L84_RS_UNPLUG | + CS42L84_TS_PLUG | CS42L84_TS_UNPLUG, + CS42L84_RS_PLUG | CS42L84_RS_UNPLUG); +} + +static void cs42l84_setup_plug_detect(struct cs42l84_private *cs42l84) +{ + unsigned int reg; + + /* Set up plug detection */ + regmap_update_bits(cs42l84->regmap, CS42L84_MIC_DET_CTL4, + CS42L84_MIC_DET_CTL4_LATCH_TO_VP, + CS42L84_MIC_DET_CTL4_LATCH_TO_VP); + regmap_update_bits(cs42l84->regmap, CS42L84_TIP_SENSE_CTL2, + CS42L84_TIP_SENSE_CTL2_MODE, + FIELD_PREP(CS42L84_TIP_SENSE_CTL2_MODE, CS42L84_TIP_SENSE_CTL2_MODE_SHORT_DET)); + regmap_update_bits(cs42l84->regmap, CS42L84_RING_SENSE_CTL, + CS42L84_RING_SENSE_CTL_INV | CS42L84_RING_SENSE_CTL_UNK1 | + CS42L84_RING_SENSE_CTL_RISETIME | CS42L84_RING_SENSE_CTL_FALLTIME, + CS42L84_RING_SENSE_CTL_INV | CS42L84_RING_SENSE_CTL_UNK1 | + FIELD_PREP(CS42L84_RING_SENSE_CTL_RISETIME, CS42L84_DEBOUNCE_TIME_125MS) | + FIELD_PREP(CS42L84_RING_SENSE_CTL_FALLTIME, CS42L84_DEBOUNCE_TIME_125MS)); + regmap_update_bits(cs42l84->regmap, CS42L84_TIP_SENSE_CTL, + CS42L84_TIP_SENSE_CTL_INV | + CS42L84_TIP_SENSE_CTL_RISETIME | CS42L84_TIP_SENSE_CTL_FALLTIME, + CS42L84_TIP_SENSE_CTL_INV | + FIELD_PREP(CS42L84_TIP_SENSE_CTL_RISETIME, CS42L84_DEBOUNCE_TIME_500MS) | + FIELD_PREP(CS42L84_TIP_SENSE_CTL_FALLTIME, CS42L84_DEBOUNCE_TIME_125MS)); + regmap_update_bits(cs42l84->regmap, CS42L84_MSM_BLOCK_EN3, + CS42L84_MSM_BLOCK_EN3_TR_SENSE, + CS42L84_MSM_BLOCK_EN3_TR_SENSE); + + /* Save the initial status of the tip sense */ + regmap_read(cs42l84->regmap, CS42L84_TSRS_PLUG_STATUS, ®); + cs42l84->plug_state = (((char) reg) & + (CS42L84_TS_PLUG | CS42L84_TS_UNPLUG)) >> + CS42L84_TS_PLUG_SHIFT; + + /* Set up mic detection */ + + /* Disconnect HSBIAS (initially) */ + regmap_write(cs42l84->regmap, + CS42L84_HS_SWITCH_CTL, + CS42L84_HS_SWITCH_CTL_REF_HS3 | \ + CS42L84_HS_SWITCH_CTL_REF_HS4 | \ + CS42L84_HS_SWITCH_CTL_HSB_FILT_HS3 | \ + CS42L84_HS_SWITCH_CTL_HSB_FILT_HS4 | \ + CS42L84_HS_SWITCH_CTL_GNDHS_HS3 | \ + CS42L84_HS_SWITCH_CTL_GNDHS_HS4); + regmap_update_bits(cs42l84->regmap, + CS42L84_HS_DET_CTL2, + CS42L84_HS_DET_CTL2_SET | CS42L84_HS_DET_CTL2_CTL, + FIELD_PREP(CS42L84_HS_DET_CTL2_SET, 2) | + FIELD_PREP(CS42L84_HS_DET_CTL2_CTL, 0)); + regmap_update_bits(cs42l84->regmap, + CS42L84_HS_CLAMP_DISABLE, 1, 1); + +} + +static int cs42l84_i2c_probe(struct i2c_client *i2c_client) +{ + struct cs42l84_private *cs42l84; + int ret, devid; + unsigned int reg; + + cs42l84 = devm_kzalloc(&i2c_client->dev, sizeof(struct cs42l84_private), + GFP_KERNEL); + if (!cs42l84) + return -ENOMEM; + + cs42l84->dev = &i2c_client->dev; + i2c_set_clientdata(i2c_client, cs42l84); + mutex_init(&cs42l84->irq_lock); + + cs42l84->regmap = devm_regmap_init_i2c(i2c_client, &cs42l84_regmap); + if (IS_ERR(cs42l84->regmap)) { + ret = PTR_ERR(cs42l84->regmap); + dev_err(&i2c_client->dev, "regmap_init() failed: %d\n", ret); + return ret; + } + + /* Reset the Device */ + cs42l84->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, + "reset", GPIOD_OUT_LOW); + if (IS_ERR(cs42l84->reset_gpio)) { + ret = PTR_ERR(cs42l84->reset_gpio); + goto err_disable_noreset; + } + + if (cs42l84->reset_gpio) { + dev_dbg(&i2c_client->dev, "Found reset GPIO\n"); + gpiod_set_value_cansleep(cs42l84->reset_gpio, 1); + } + usleep_range(CS42L84_BOOT_TIME_US, CS42L84_BOOT_TIME_US * 2); + + /* Request IRQ if one was specified */ + if (i2c_client->irq) { + ret = request_threaded_irq(i2c_client->irq, + NULL, cs42l84_irq_thread, + IRQF_ONESHOT, + "cs42l84", cs42l84); + if (ret == -EPROBE_DEFER) { + goto err_disable_noirq; + } else if (ret != 0) { + dev_err(&i2c_client->dev, + "Failed to request IRQ: %d\n", ret); + goto err_disable_noirq; + } + } + + /* initialize codec */ + devid = cirrus_read_device_id(cs42l84->regmap, CS42L84_DEVID); + if (devid < 0) { + ret = devid; + dev_err(&i2c_client->dev, "Failed to read device ID: %d\n", ret); + goto err_disable; + } + + if (devid != CS42L84_CHIP_ID) { + dev_err(&i2c_client->dev, + "CS42L84 Device ID (%X). Expected %X\n", + devid, CS42L84_CHIP_ID); + ret = -EINVAL; + goto err_disable; + } + + ret = regmap_read(cs42l84->regmap, CS42L84_REVID, ®); + if (ret < 0) { + dev_err(&i2c_client->dev, "Get Revision ID failed\n"); + goto err_shutdown; + } + + dev_info(&i2c_client->dev, + "Cirrus Logic CS42L84, Revision: %02X\n", reg & 0xFF); + + /* Setup plug detection */ + cs42l84_setup_plug_detect(cs42l84); + + /* Mask/Unmask Interrupts */ + cs42l84_set_interrupt_masks(cs42l84); + + /* Register codec for machine driver */ + ret = devm_snd_soc_register_component(&i2c_client->dev, + &soc_component_dev_cs42l84, &cs42l84_dai, 1); + if (ret < 0) + goto err_shutdown; + + return 0; + +err_shutdown: + /* Nothing to do */ + +err_disable: + if (i2c_client->irq) + free_irq(i2c_client->irq, cs42l84); + +err_disable_noirq: + gpiod_set_value_cansleep(cs42l84->reset_gpio, 0); +err_disable_noreset: + return ret; +} + +static void cs42l84_i2c_remove(struct i2c_client *i2c_client) +{ + struct cs42l84_private *cs42l84 = i2c_get_clientdata(i2c_client); + + if (i2c_client->irq) + free_irq(i2c_client->irq, cs42l84); + + gpiod_set_value_cansleep(cs42l84->reset_gpio, 0); +} + +static const struct of_device_id cs42l84_of_match[] = { + { .compatible = "cirrus,cs42l84", }, + {} +}; +MODULE_DEVICE_TABLE(of, cs42l84_of_match); + +static const struct i2c_device_id cs42l84_id[] = { + {"cs42l84", 0}, + {} +}; +MODULE_DEVICE_TABLE(i2c, cs42l84_id); + +static struct i2c_driver cs42l84_i2c_driver = { + .driver = { + .name = "cs42l84", + .of_match_table = of_match_ptr(cs42l84_of_match), + }, + .id_table = cs42l84_id, + .probe = cs42l84_i2c_probe, + .remove = cs42l84_i2c_remove, +}; + +module_i2c_driver(cs42l84_i2c_driver); + +MODULE_DESCRIPTION("ASoC CS42L84 driver"); +MODULE_AUTHOR("Martin Povišer "); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/codecs/cs42l84.h b/sound/soc/codecs/cs42l84.h new file mode 100644 index 00000000000000..e7cbf5f0e2d0bb --- /dev/null +++ b/sound/soc/codecs/cs42l84.h @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) The Asahi Linux Contributors + * + * Based on sound/soc/codecs/cs42l42.h + * + * Copyright 2016 Cirrus Logic, Inc. + */ + + +#ifndef __CS42L84_H__ +#define __CS42L84_H__ + +#include + +#define CS42L84_CHIP_ID 0x42a84 + +#define CS42L84_DEVID 0x0000 +#define CS42L84_REVID 0x73fe +#define CS42L84_FRZ_CTL 0x0006 +#define CS42L84_FRZ_CTL_ENGAGE BIT(0) + +#define CS42L84_TSRS_PLUG_INT_STATUS 0x0400 +#define CS42L84_TSRS_PLUG_INT_MASK 0x0418 +#define CS42L84_RS_PLUG_SHIFT 0 +#define CS42L84_RS_PLUG BIT(0) +#define CS42L84_RS_UNPLUG BIT(1) +#define CS42L84_TS_PLUG_SHIFT 2 +#define CS42L84_TS_PLUG BIT(2) +#define CS42L84_TS_UNPLUG BIT(3) +#define CS42L84_TSRS_PLUG_VAL_MASK GENMASK(3, 0) +#define CS42L84_PLL_LOCK_STATUS 0x040e // probably bit 0x10 +#define CS42L84_PLL_LOCK_STATUS_LOCKED BIT(4) + +#define CS42L84_PLUG 3 +#define CS42L84_UNPLUG 0 +#define CS42L84_TRANS 1 + +#if 0 + l84.regs.RING_SENSE_CTRL.set(INV=1, UNK1=1, + RISETIME=E_DEBOUNCE_TIME.T_125MS, FALLTIME=E_DEBOUNCE_TIME.T_125MS) + l84.regs.TIP_SENSE_CTRL.set(INV=1, + RISETIME=E_DEBOUNCE_TIME.T_500MS, FALLTIME=E_DEBOUNCE_TIME.T_125MS) + l84.regs.MSM_BLOCK_EN3.set(TR_SENSE_EN=1) +#endif + +#define CS42L84_CCM_CTL1 0x0600 +#define CS42L84_CCM_CTL1_MCLK_SRC GENMASK(1, 0) +#define CS42L84_CCM_CTL1_MCLK_SRC_RCO 0 +#define CS42L84_CCM_CTL1_MCLK_SRC_MCLK 1 +#define CS42L84_CCM_CTL1_MCLK_SRC_BCLK 2 +#define CS42L84_CCM_CTL1_MCLK_SRC_PLL 3 +#define CS42L84_CCM_CTL1_MCLK_FREQ GENMASK(3, 2) +#define CS42L84_CCM_CTL1_MCLK_F_12MHZ 0b00 +#define CS42L84_CCM_CTL1_MCLK_F_24MHZ 0b01 +#define CS42L84_CCM_CTL1_MCLK_F_12_288KHZ 0b10 +#define CS42L84_CCM_CTL1_MCLK_F_24_576KHZ 0b11 +#define CS42L84_CCM_CTL1_RCO \ + (FIELD_PREP(CS42L84_CCM_CTL1_MCLK_SRC, CS42L84_CCM_CTL1_MCLK_SRC_RCO) \ + | FIELD_PREP(CS42L84_CCM_CTL1_MCLK_FREQ, CS42L84_CCM_CTL1_MCLK_F_12MHZ)) + +#define CS42L84_CCM_SAMP_RATE 0x0601 +#define CS42L84_CCM_SAMP_RATE_RATE_48KHZ 4 +#define CS42L84_CCM_SAMP_RATE_RATE_96KHZ 5 +#define CS42L84_CCM_SAMP_RATE_RATE_192KHZ 6 +#define CS42L84_CCM_SAMP_RATE_RATE_44K1HZ 12 +#define CS42L84_CCM_SAMP_RATE_RATE_88K2HZ 13 +#define CS42L84_CCM_SAMP_RATE_RATE_176K4HZ 14 +#define CS42L84_CCM_CTL3 0x0602 +#define CS42L84_CCM_CTL3_REFCLK_DIV GENMASK(2, 1) +#define CS42L84_CCM_CTL4 0x0603 +#define CS42L84_CCM_CTL4_REFCLK_EN BIT(0) + +#define CS42L84_CCM_ASP_CLK_CTRL 0x0608 + +#define CS42L84_PLL_CTL1 0x0800 +#define CS42L84_PLL_CTL1_EN BIT(0) +#define CS42L84_PLL_CTL1_MODE GENMASK(2, 1) +#define CS42L84_PLL_DIV_FRAC0 0x0804 +#define CS42L84_PLL_DIV_FRAC1 0x0805 +#define CS42L84_PLL_DIV_FRAC2 0x0806 +#define CS42L84_PLL_DIV_INT 0x0807 +#define CS42L84_PLL_DIVOUT 0x0808 + +#define CS42L84_RING_SENSE_CTL 0x1282 +#define CS42L84_RING_SENSE_CTL_INV BIT(7) +#define CS42L84_RING_SENSE_CTL_UNK1 BIT(6) +#define CS42L84_RING_SENSE_CTL_FALLTIME GENMASK(5, 3) +#define CS42L84_RING_SENSE_CTL_RISETIME GENMASK(2, 0) +#define CS42L84_TIP_SENSE_CTL 0x1283 +#define CS42L84_TIP_SENSE_CTL_INV BIT(7) +#define CS42L84_TIP_SENSE_CTL_FALLTIME GENMASK(5, 3) +#define CS42L84_TIP_SENSE_CTL_RISETIME GENMASK(2, 0) + +#define CS42L84_TSRS_PLUG_STATUS 0x1288 + +#define CS42L84_TIP_SENSE_CTL2 0x1473 +#define CS42L84_TIP_SENSE_CTL2_MODE GENMASK(7, 6) +#define CS42L84_TIP_SENSE_CTL2_MODE_DISABLED 0b00 +#define CS42L84_TIP_SENSE_CTL2_MODE_DIG_INPUT 0b01 +#define CS42L84_TIP_SENSE_CTL2_MODE_SHORT_DET 0b11 +#define CS42L84_TIP_SENSE_CTL2_INV BIT(5) + +#define CS42L84_MISC_DET_CTL 0x1474 +#define CS42L84_MISC_DET_CTL_DETECT_MODE GENMASK(4, 3) +#define CS42L84_MISC_DET_CTL_HSBIAS_CTL GENMASK(2, 1) +#define CS42L84_MISC_DET_CTL_PDN_MIC_LVL_DET BIT(0) + +#define CS42L84_MIC_DET_CTL4 0x1477 +#define CS42L84_MIC_DET_CTL4_LATCH_TO_VP BIT(1) + +#define CS42L84_HS_DET_STATUS2 0x147d + +#define CS42L84_MSM_BLOCK_EN1 0x1800 +#define CS42L84_MSM_BLOCK_EN2 0x1801 +#define CS42L84_MSM_BLOCK_EN2_ASP_SHIFT 6 +#define CS42L84_MSM_BLOCK_EN2_BUS_SHIFT 5 +#define CS42L84_MSM_BLOCK_EN2_DAC_SHIFT 4 +#define CS42L84_MSM_BLOCK_EN2_ADC_SHIFT 3 +#define CS42L84_MSM_BLOCK_EN3 0x1802 +#define CS42L84_MSM_BLOCK_EN3_TR_SENSE BIT(3) + +#define CS42L84_HS_DET_CTL2 0x1811 +#define CS42L84_HS_DET_CTL2_CTL GENMASK(7, 6) +#define CS42L84_HS_DET_CTL2_SET GENMASK(5, 4) +#define CS42L84_HS_DET_CTL2_REF BIT(3) +#define CS42L84_HS_DET_CTL2_AUTO_TIME GENMASK(1, 0) + +#define CS42L84_HS_SWITCH_CTL 0x1812 +#define CS42L84_HS_SWITCH_CTL_REF_HS3 BIT(7) +#define CS42L84_HS_SWITCH_CTL_REF_HS4 BIT(6) +#define CS42L84_HS_SWITCH_CTL_HSB_FILT_HS3 BIT(5) +#define CS42L84_HS_SWITCH_CTL_HSB_FILT_HS4 BIT(4) +#define CS42L84_HS_SWITCH_CTL_HSB_HS3 BIT(3) +#define CS42L84_HS_SWITCH_CTL_HSB_HS4 BIT(2) +#define CS42L84_HS_SWITCH_CTL_GNDHS_HS3 BIT(1) +#define CS42L84_HS_SWITCH_CTL_GNDHS_HS4 BIT(0) + +#define CS42L84_HS_CLAMP_DISABLE 0x1813 + +#define CS42L84_ADC_CTL1 0x2000 +#define CS42L84_ADC_CTL1_PREAMP_GAIN_SHIFT 6 +#define CS42L84_ADC_CTL1_PGA_GAIN_SHIFT 0 +#define CS42L84_ADC_CTL4 0x2003 +#define CS42L84_ADC_CTL4_WNF_CF_SHIFT 4 +#define CS42L84_ADC_CTL4_WNF_EN_SHIFT 3 +#define CS42L84_ADC_CTL4_HPF_CF_SHIFT 1 +#define CS42L84_ADC_CTL4_HPF_EN_SHIFT 0 + +#define CS42L84_DAC_CTL1 0x3000 +#define CS42L84_DAC_CTL1_UNMUTE BIT(0) +//#define CS42L84_DAC_CTL1_DACB_INV_SHIFT 1 +//#define CS42L84_DAC_CTL1_DACA_INV_SHIFT 0 +#define CS42L84_DAC_CTL2 0x3001 + +#define CS42L84_DAC_CHA_VOL_LSB 0x3004 +#define CS42L84_DAC_CHA_VOL_MSB 0x3005 +#define CS42L84_DAC_CHB_VOL_LSB 0x3006 +#define CS42L84_DAC_CHB_VOL_MSB 0x3007 +#define CS42L84_HP_VOL_CTL 0x3020 +#define CS42L84_HP_VOL_CTL_ZERO_CROSS BIT(1) +#define CS42L84_HP_VOL_CTL_SOFT BIT(0) + +#define CS42L84_SRC_ASP_RX_CH1 0b1101 +#define CS42L84_SRC_ASP_RX_CH2 0b1110 + +#define CS42L84_BUS_ASP_TX_SRC 0x4000 +#define CS42L84_BUS_ASP_TX_SRC_CH1_SHIFT 0 +#define CS42L84_BUS_DAC_SRC 0x4001 +#define CS42L84_BUS_DAC_SRC_DACA_SHIFT 0 +#define CS42L84_BUS_DAC_SRC_DACB_SHIFT 4 + +#define CS42L84_ASP_CTL 0x5000 +#define CS42L84_ASP_CTL_BCLK_EN_SHIFT 1 +#define CS42L84_ASP_CTL_TDM_MODE BIT(2) +#define CS42L84_ASP_FSYNC_CTL2 0x5010 +#define CS42L84_ASP_FSYNC_CTL2_BCLK_PERIOD_LO GENMASK(7, 1) +#define CS42L84_ASP_FSYNC_CTL3 0x5011 +#define CS42L84_ASP_FSYNC_CTL3_BCLK_PERIOD_HI GENMASK(4, 0) +#define CS42L84_ASP_DATA_CTL 0x5018 + +#define CS42L84_ASP_RX_EN 0x5020 +#define CS42L84_ASP_RX_EN_CH1_SHIFT 0 +#define CS42L84_ASP_RX_EN_CH2_SHIFT 1 +#define CS42L84_ASP_TX_EN 0x5024 +#define CS42L84_ASP_TX_EN_CH1_SHIFT 0 + +#define CS42L84_ASP_RX_CH1_CTL1 0x5028 +#define CS42L84_ASP_RX_CH1_CTL2 0x5029 +#define CS42L84_ASP_RX_CH1_WIDTH 0x502a +#define CS42L84_ASP_RX_CH2_CTL1 0x502c +#define CS42L84_ASP_RX_CH2_CTL2 0x502d +#define CS42L84_ASP_RX_CH2_WIDTH 0x502e + +#define CS42L84_ASP_RX_CHx_CTL1_EDGE BIT(0) +#define CS42L84_ASP_RX_CHx_CTL1_SLOT_START_LSB GENMASK(7, 1) +#define CS42L84_ASP_RX_CHx_CTL2_SLOT_START_MSB GENMASK(2, 0) + +#define CS42L84_ASP_TX_CH1_CTL1 0x5068 +#define CS42L84_ASP_TX_CH1_CTL2 0x5069 +#define CS42L84_ASP_TX_CH1_WIDTH 0x506a +#define CS42L84_ASP_TX_CH2_CTL1 0x506c +#define CS42L84_ASP_TX_CH2_CTL2 0x506d +#define CS42L84_ASP_TX_CH2_WIDTH 0x506e + +#define CS42L84_DEBOUNCE_TIME_125MS 0b001 +#define CS42L84_DEBOUNCE_TIME_500MS 0b011 + +#define CS42L84_BOOT_TIME_US 3000 +#define CS42L84_CLOCK_SWITCH_DELAY_US 150 +#define CS42L84_PLL_LOCK_POLL_US 250 +#define CS42L84_PLL_LOCK_TIMEOUT_US 1250 + +#endif /* __CS42L84_H__ */ From 8df86c76b631a0790beab65b43ba601d4fe902e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sun, 21 Aug 2022 02:40:29 +0200 Subject: [PATCH 075/635] ASoC: macaudio: Fix headset routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 87d58a7481a6fb..a95103736b5040 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -626,7 +626,7 @@ static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_ if (!is_speakers) { r = &routes[nroutes++]; r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; - r->sink = "Headphone Capture"; + r->sink = "Headset Capture"; } ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); @@ -639,7 +639,7 @@ static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_ static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_component *component, bool is_speakers) { - struct snd_soc_dapm_route routes[1]; + struct snd_soc_dapm_route routes[2]; struct snd_soc_dapm_route *r; int nroutes = 0; char buf[32]; @@ -660,9 +660,11 @@ static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_com r = &routes[nroutes++]; r->source = "Jack HP"; r->sink = "Headphone"; + r = &routes[nroutes++]; + r->source = "Headset Mic"; + r->sink = "Jack HS"; } - ret = snd_soc_dapm_add_routes(&card->dapm, routes, nroutes); if (ret) dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", @@ -813,7 +815,7 @@ static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { SND_SOC_DAPM_AIF_OUT("Speaker Playback", NULL, 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_OUT("Headphone Playback", NULL, 0, SND_SOC_NOPM, 0, 0), - SND_SOC_DAPM_AIF_IN("Headphone Capture", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("Headset Capture", NULL, 0, SND_SOC_NOPM, 0, 0), }; static const struct snd_kcontrol_new macaudio_controls[] = { @@ -840,7 +842,7 @@ static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { { "Speaker (Static)", "Static", "Speaker Pin Demux" }, /* Capture paths */ - { "PCM0 RX", NULL, "Headphone Capture" }, + { "PCM0 RX", NULL, "Headset Capture" }, }; static const struct of_device_id macaudio_snd_device_id[] = { From 7660beb60beaa0882e6262a87eab1334265af220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sun, 21 Aug 2022 02:40:54 +0200 Subject: [PATCH 076/635] ASoC: dapm: Export new 'graph.dot' file in debugfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/soc-dapm.c | 137 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 5d9a671e50f194..f47daf1a5b4f1f 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -2194,6 +2194,139 @@ static const struct file_operations dapm_bias_fops = { .llseek = default_llseek, }; +static ssize_t dapm_graph_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct snd_soc_card *card = file->private_data; + struct snd_soc_dapm_context *dapm; + struct snd_soc_dapm_path *p; + struct snd_soc_dapm_widget *w; + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dapm_widget *wdone[16]; + struct snd_soc_dai *dai; + int i, num_wdone = 0, cluster = 0; + char *buf; + ssize_t bufsize; + ssize_t ret = 0; + + bufsize = 1024 * card->num_dapm_widgets; + buf = kmalloc(bufsize, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + mutex_lock(&card->dapm_mutex); + +#define bufprintf(...) \ + ret += scnprintf(buf + ret, bufsize - ret, __VA_ARGS__) + + bufprintf("digraph dapm {\n"); + + /* + * Print the user-visible devices of the card. + */ + bufprintf("subgraph cluster_%d {\n", cluster++); + bufprintf("label=\"Devices\";style=filled;fillcolor=gray;\n"); + for_each_card_rtds(card, rtd) { + if (rtd->dai_link->no_pcm) + continue; + + bufprintf("w%pK [label=\"%d: %s\"];\n", rtd, + rtd->pcm->device, rtd->dai_link->name); + } + bufprintf("};\n"); + + /* + * Print the playback/capture widgets of DAIs just next to + * the user-visible devices. Keep the list of already printed + * widgets in 'wdone', so they will be skipped later. + */ + for_each_card_rtds(card, rtd) { + for_each_rtd_cpu_dais(rtd, i, dai) { + if (dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget) { + w = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget; + bufprintf("w%pK [label=\"%s\"];\n", w, w->name); + if (!rtd->dai_link->no_pcm) + bufprintf("w%pK -> w%pK;\n", rtd, w); + wdone[num_wdone] = w; + if (num_wdone < ARRAY_SIZE(wdone)) + num_wdone++; + } + + if (dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget) { + w = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget; + bufprintf("w%pK [label=\"%s\"];\n", w, w->name); + if (!rtd->dai_link->no_pcm) + bufprintf("w%pK -> w%pK;\n", w, rtd); + wdone[num_wdone] = w; + if (num_wdone < ARRAY_SIZE(wdone)) + num_wdone++; + } + } + } + + for_each_card_dapms(card, dapm) { + const char *prefix = soc_dapm_prefix(dapm); + + if (dapm != &card->dapm) { + bufprintf("subgraph cluster_%d {\n", cluster++); + if (prefix) + bufprintf("label=\"%s\";\n", prefix); + else if (dapm->component) + bufprintf("label=\"%s\";\n", + dapm->component->name); + } + + for_each_card_widgets(dapm->card, w) { + const char *name = w->name; + bool skip = false; + + if (w->dapm != dapm) + continue; + + if (list_empty(&w->edges[0]) && list_empty(&w->edges[1])) + continue; + + for (i = 0; i < num_wdone; i++) + if (wdone[i] == w) + skip = true; + if (skip) + continue; + + if (prefix && strlen(name) > strlen(prefix) + 1) + name += strlen(prefix) + 1; + + bufprintf("w%pK [label=\"%s\"];\n", w, name); + } + + if (dapm != &card->dapm) + bufprintf("}\n"); + } + + list_for_each_entry(p, &card->paths, list) { + if (p->name) + bufprintf("w%pK -> w%pK [label=\"%s\"];\n", + p->source, p->sink, p->name); + else + bufprintf("w%pK -> w%pK;\n", p->source, p->sink); + } + + bufprintf("}\n"); +#undef bufprintf + + mutex_unlock(&card->dapm_mutex); + + ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret); + + kfree(buf); + return ret; +} + +static const struct file_operations dapm_graph_fops = { + .open = simple_open, + .read = dapm_graph_read_file, + .llseek = default_llseek, +}; + void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, struct dentry *parent) { @@ -2204,6 +2337,10 @@ void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, debugfs_create_file("bias_level", 0444, dapm->debugfs_dapm, dapm, &dapm_bias_fops); + + if (dapm == &dapm->card->dapm) + debugfs_create_file("graph.dot", 0444, dapm->debugfs_dapm, + dapm->card, &dapm_graph_fops); } static void dapm_debugfs_add_widget(struct snd_soc_dapm_widget *w) From a37d31a138c23cb6afdee5db710e556ada4181da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Tue, 23 Aug 2022 11:36:24 +0200 Subject: [PATCH 077/635] ASoC: macaudio: Add j375 fixup_controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index a95103736b5040..a2f3aea2e4c386 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -758,6 +758,17 @@ static int macaudio_j314_fixup_controls(struct snd_soc_card *card) return 0; } +static int macaudio_j375_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (ma->has_speakers) { + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 14); // 20 set by macOS, this is 3 dB below + } + + return 0; +} + static int macaudio_fallback_fixup_controls(struct snd_soc_card *card) { struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); @@ -848,6 +859,7 @@ static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { static const struct of_device_id macaudio_snd_device_id[] = { { .compatible = "apple,j274-macaudio", .data = macaudio_j274_fixup_controls }, { .compatible = "apple,j314-macaudio", .data = macaudio_j314_fixup_controls }, + { .compatible = "apple,j375-macaudio", .data = macaudio_j375_fixup_controls }, { .compatible = "apple,macaudio"}, { } }; From cc6a5a57ef693a0f68ff9a6bb31824d11dde3ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Tue, 30 Aug 2022 10:20:09 +0200 Subject: [PATCH 078/635] ASoC: macaudio: Add j493 fixup_controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index a2f3aea2e4c386..2a2d03a31e23c5 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -769,6 +769,17 @@ static int macaudio_j375_fixup_controls(struct snd_soc_card *card) return 0; } +static int macaudio_j493_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (ma->has_speakers) { + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 9); // 15 set by macOS, this is 3 dB below + } + + return 0; +} + static int macaudio_fallback_fixup_controls(struct snd_soc_card *card) { struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); @@ -860,6 +871,7 @@ static const struct of_device_id macaudio_snd_device_id[] = { { .compatible = "apple,j274-macaudio", .data = macaudio_j274_fixup_controls }, { .compatible = "apple,j314-macaudio", .data = macaudio_j314_fixup_controls }, { .compatible = "apple,j375-macaudio", .data = macaudio_j375_fixup_controls }, + { .compatible = "apple,j493-macaudio", .data = macaudio_j493_fixup_controls }, { .compatible = "apple,macaudio"}, { } }; From f6074e85a96308cca9cb9d3b541e3109fceb9a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sun, 4 Sep 2022 10:29:34 +0200 Subject: [PATCH 079/635] ASoC: macaudio: Rename ALSA driver to simple 'macaudio' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 2a2d03a31e23c5..2cb1534edceea0 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -898,7 +898,7 @@ static int macaudio_snd_platform_probe(struct platform_device *pdev) snd_soc_card_set_drvdata(card, data); card->owner = THIS_MODULE; - card->driver_name = DRIVER_NAME; + card->driver_name = "macaudio"; card->dev = dev; card->dapm_widgets = macaudio_snd_widgets; card->num_dapm_widgets = ARRAY_SIZE(macaudio_snd_widgets); From c43b2a43354645beae863750bba79b9685c18079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 2 Sep 2022 19:40:16 +0200 Subject: [PATCH 080/635] ASoC: macaudio: Drop the 'inverse jack' speaker stuff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 2cb1534edceea0..517616c8900ce4 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -577,11 +577,6 @@ static struct snd_soc_jack_pin macaudio_jack_pins[] = { .pin = "Headset Mic", .mask = SND_JACK_MICROPHONE, }, - { - .pin = "Speaker", - .mask = SND_JACK_HEADPHONE, - .invert = 1, - }, }; static int macaudio_probe(struct snd_soc_card *card) @@ -655,7 +650,7 @@ static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_com snprintf(buf, sizeof(buf) - 1, "%s OUT", component->name_prefix); r->source = buf; } - r->sink = "Speaker Pin Demux"; + r->sink = "Speaker"; } else { r = &routes[nroutes++]; r->source = "Jack HP"; @@ -814,16 +809,6 @@ SOC_ENUM_SINGLE_VIRT_DECL(macaudio_hp_mux_enum, macaudio_hp_mux_texts); static const struct snd_kcontrol_new macaudio_hp_mux = SOC_DAPM_ENUM("Headphones Playback Mux", macaudio_hp_mux_enum); -static const char *macaudio_spk_demux_texts[] = { - "Inverse Jack", "Static", -}; - -static SOC_ENUM_SINGLE_DECL(macaudio_spk_demux_enum, - SND_SOC_NOPM, 0, macaudio_spk_demux_texts); - -static const struct snd_kcontrol_new macaudio_spk_demux = - SOC_DAPM_ENUM("Speaker Pin Demux", macaudio_spk_demux_enum); - static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { SND_SOC_DAPM_SPK("Speaker", NULL), SND_SOC_DAPM_SPK("Speaker (Static)", NULL), @@ -832,7 +817,6 @@ static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { SND_SOC_DAPM_MUX("Speaker Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_spk_mux), SND_SOC_DAPM_MUX("Headphone Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_hp_mux), - SND_SOC_DAPM_DEMUX("Speaker Pin Demux", SND_SOC_NOPM, 0, 0, &macaudio_spk_demux), SND_SOC_DAPM_AIF_OUT("Speaker Playback", NULL, 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_OUT("Headphone Playback", NULL, 0, SND_SOC_NOPM, 0, 0), @@ -842,7 +826,6 @@ static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { static const struct snd_kcontrol_new macaudio_controls[] = { SOC_DAPM_PIN_SWITCH("Speaker"), - SOC_DAPM_PIN_SWITCH("Speaker (Static)"), SOC_DAPM_PIN_SWITCH("Headphone"), SOC_DAPM_PIN_SWITCH("Headset Mic"), }; @@ -860,9 +843,6 @@ static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { * Additional paths (to specific I2S ports) are added dynamically. */ - { "Speaker", "Inverse Jack", "Speaker Pin Demux" }, - { "Speaker (Static)", "Static", "Speaker Pin Demux" }, - /* Capture paths */ { "PCM0 RX", NULL, "Headset Capture" }, }; From 6842e184dd247b21209e4ce980929b54a940d16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Tue, 6 Sep 2022 15:16:44 +0200 Subject: [PATCH 081/635] ASoC: macaudio: s/Freq/Frequency/ in TAS2764 control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 517616c8900ce4..d7220db2c2a5ef 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -730,8 +730,8 @@ static int macaudio_j314_fixup_controls(struct snd_soc_card *card) CHECK(snd_soc_set_enum_kctl, "* ASI1 Sel", "Left"); CHECK(snd_soc_deactivate_kctl, "* ASI1 Sel", 0); CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 9); // 15 set by macOS, this is 3 dB below - CHECK(snd_soc_set_enum_kctl, "* Tweeter HPF Corner Freq", "800 Hz"); - CHECK(snd_soc_deactivate_kctl, "* Tweeter HPF Corner Freq", 0); + CHECK(snd_soc_set_enum_kctl, "* Tweeter HPF Corner Frequency", "800 Hz"); + CHECK(snd_soc_deactivate_kctl, "* Tweeter HPF Corner Frequency", 0); /* * The speaker amps suffer from spurious overcurrent From e654f31ffeb1e693518b09a0c286c66ba752ccbc Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 13 Sep 2022 19:56:12 +0900 Subject: [PATCH 082/635] ASoC: macaudio: s/void_warranty/please_blow_up_my_speakers/ We have no idea whether any of this voids warranties, but what it does do is blow up your speakers, so let's be explicit about what users are signing up for. Signed-off-by: Hector Martin --- sound/soc/apple/macaudio.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index d7220db2c2a5ef..586b238662a80e 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -67,9 +67,9 @@ struct macaudio_snd_data { struct snd_pcm_hw_constraint_list speaker_nchans_list; }; -static bool void_warranty; -module_param(void_warranty, bool, 0644); -MODULE_PARM_DESC(void_warranty, "Do not bail if safety is not assured"); +static bool please_blow_up_my_speakers; +module_param(please_blow_up_my_speakers, bool, 0644); +MODULE_PARM_DESC(please_blow_up_my_speakers, "Allow unsafe or untested operating configurations"); SND_SOC_DAILINK_DEFS(primary, DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU @@ -703,7 +703,7 @@ static int macaudio_late_probe(struct snd_soc_card *card) #define CHECK(call, pattern, value) \ { \ int ret = call(card, pattern, value); \ - if (ret < 1 && !void_warranty) { \ + if (ret < 1 && !please_blow_up_my_speakers) { \ dev_err(card->dev, "%s on '%s': %d\n", #call, pattern, ret); \ return ret; \ } \ @@ -779,7 +779,7 @@ static int macaudio_fallback_fixup_controls(struct snd_soc_card *card) { struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); - if (ma->has_speakers && !void_warranty) { + if (ma->has_speakers && !please_blow_up_my_speakers) { dev_err(card->dev, "driver can't assure safety on this model, refusing probe\n"); return -EINVAL; } From cc9b22bb705006a4c10f2a545c1ace5de4223947 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 13 Sep 2022 19:56:47 +0900 Subject: [PATCH 083/635] ASoC: macaudio: Gate off experimental platforms We know at least some machines can have their speakers blown, even with these limits, so let's play it safe for now and require that users both enable stuff in the DT *and* pass this flag. Signed-off-by: Hector Martin --- sound/soc/apple/macaudio.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 586b238662a80e..5d2230b995b4f7 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -727,6 +727,11 @@ static int macaudio_j314_fixup_controls(struct snd_soc_card *card) struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); if (ma->has_speakers) { + if (!please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, refusing probe\n"); + return -EINVAL; + } + CHECK(snd_soc_set_enum_kctl, "* ASI1 Sel", "Left"); CHECK(snd_soc_deactivate_kctl, "* ASI1 Sel", 0); CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 9); // 15 set by macOS, this is 3 dB below @@ -758,6 +763,11 @@ static int macaudio_j375_fixup_controls(struct snd_soc_card *card) struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); if (ma->has_speakers) { + if (!please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, refusing probe\n"); + return -EINVAL; + } + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 14); // 20 set by macOS, this is 3 dB below } @@ -769,6 +779,11 @@ static int macaudio_j493_fixup_controls(struct snd_soc_card *card) struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); if (ma->has_speakers) { + if (!please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, refusing probe\n"); + return -EINVAL; + } + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 9); // 15 set by macOS, this is 3 dB below } From 02baac9834b16092f4e5b2a3ff6070da41008eb0 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 13 Sep 2022 19:58:17 +0900 Subject: [PATCH 084/635] ASoC: macaudio: Alias f413 fixups to j314 This works as far as following the same intent as j314, but we *know* these limits are not sufficient, so this one really needs the module parameter gate. Signed-off-by: Hector Martin --- sound/soc/apple/macaudio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 5d2230b995b4f7..e6b7715b8e5c26 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -866,6 +866,7 @@ static const struct of_device_id macaudio_snd_device_id[] = { { .compatible = "apple,j274-macaudio", .data = macaudio_j274_fixup_controls }, { .compatible = "apple,j314-macaudio", .data = macaudio_j314_fixup_controls }, { .compatible = "apple,j375-macaudio", .data = macaudio_j375_fixup_controls }, + { .compatible = "apple,j413-macaudio", .data = macaudio_j314_fixup_controls }, { .compatible = "apple,j493-macaudio", .data = macaudio_j493_fixup_controls }, { .compatible = "apple,macaudio"}, { } From 958118b0354f45a686dd3fd23875086cb245d23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 14 Oct 2022 16:23:21 +0200 Subject: [PATCH 085/635] ASoC: cs42l84: There's no line-in on the jack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/codecs/cs42l84.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index e65c9c80b52bc8..903f69d4790efe 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -727,12 +727,8 @@ static void cs42l84_detect_hs(struct cs42l84_private *cs42l84) break; case 0b00: /* open */ - dev_dbg(cs42l84->dev, "Detected line-in\n"); - cs42l84->hs_type = SND_JACK_HEADSET; - snd_soc_jack_report(cs42l84->jack, SND_JACK_HEADSET, - SND_JACK_HEADSET); - break; - + dev_dbg(cs42l84->dev, "Detected open circuit on HS4\n"); + fallthrough; case 0b11: /* shorted */ default: snd_soc_jack_report(cs42l84->jack, SND_JACK_HEADPHONE, From fe859bd5f6bf5fcfb3430ab5eeced15457e260fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 14 Oct 2022 17:46:44 +0200 Subject: [PATCH 086/635] ASoC: cs42l84: Adjust mic-detection voltage threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise the mic-detection voltage threshold to address some mics not getting detected. Signed-off-by: Martin Povišer --- sound/soc/codecs/cs42l84.c | 5 ++++- sound/soc/codecs/cs42l84.h | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 903f69d4790efe..641481ee3836a2 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -878,7 +878,10 @@ static void cs42l84_setup_plug_detect(struct cs42l84_private *cs42l84) (CS42L84_TS_PLUG | CS42L84_TS_UNPLUG)) >> CS42L84_TS_PLUG_SHIFT; - /* Set up mic detection */ + /* Set mic-detection threshold */ + regmap_update_bits(cs42l84->regmap, + CS42L84_MIC_DET_CTL1, CS42L84_MIC_DET_CTL1_HS_DET_LEVEL, + FIELD_PREP(CS42L84_MIC_DET_CTL1_HS_DET_LEVEL, 0x2c)); /* ~1.9 V */ /* Disconnect HSBIAS (initially) */ regmap_write(cs42l84->regmap, diff --git a/sound/soc/codecs/cs42l84.h b/sound/soc/codecs/cs42l84.h index e7cbf5f0e2d0bb..9aaf19051d395f 100644 --- a/sound/soc/codecs/cs42l84.h +++ b/sound/soc/codecs/cs42l84.h @@ -106,6 +106,9 @@ #define CS42L84_MISC_DET_CTL_HSBIAS_CTL GENMASK(2, 1) #define CS42L84_MISC_DET_CTL_PDN_MIC_LVL_DET BIT(0) +#define CS42L84_MIC_DET_CTL1 0x1475 +#define CS42L84_MIC_DET_CTL1_HS_DET_LEVEL GENMASK(5, 0) + #define CS42L84_MIC_DET_CTL4 0x1477 #define CS42L84_MIC_DET_CTL4_LATCH_TO_VP BIT(1) From 0a9fe266c702a8fce9b34442a0924a062509eb86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sun, 16 Oct 2022 13:33:21 +0200 Subject: [PATCH 087/635] ASoC: cs42l84: Put the volume control in shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous limits of the volume control were placeholders, this reflects the current understanding of the register semantics. Signed-off-by: Martin Povišer --- sound/soc/codecs/cs42l84.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 641481ee3836a2..be40b97b84860b 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -70,11 +70,15 @@ static int cs42l84_put_dac_vol(struct snd_kcontrol *kctl, struct snd_ctl_elem_value *val) { struct snd_soc_component *component = snd_soc_kcontrol_component(kctl); - unsigned int vola, volb; + struct soc_mixer_control *mc = (struct soc_mixer_control *) kctl->private_value; + int vola, volb; int ret, ret2; - vola = val->value.integer.value[0]; - volb = val->value.integer.value[1]; + vola = val->value.integer.value[0] + mc->min; + volb = val->value.integer.value[1] + mc->min; + + if (vola < mc->min || vola > mc->max || volb < mc->min || volb > mc->max) + return -EINVAL; ret = snd_soc_component_update_bits(component, CS42L84_FRZ_CTL, CS42L84_FRZ_CTL_ENGAGE, @@ -112,7 +116,8 @@ static int cs42l84_get_dac_vol(struct snd_kcontrol *kctl, struct snd_ctl_elem_value *val) { struct snd_soc_component *component = snd_soc_kcontrol_component(kctl); - unsigned int vola, volb; + struct soc_mixer_control *mc = (struct soc_mixer_control *) kctl->private_value; + int vola, volb; int ret; ret = snd_soc_component_read(component, CS42L84_DAC_CHA_VOL_LSB); @@ -135,18 +140,23 @@ static int cs42l84_get_dac_vol(struct snd_kcontrol *kctl, return ret; volb |= (ret & 1) << 8; - val->value.integer.value[0] = vola; - val->value.integer.value[1] = volb; + if (vola & BIT(8)) + vola |= ~((int)(BIT(8) - 1)); + if (volb & BIT(8)) + volb |= ~((int)(BIT(8) - 1)); + + val->value.integer.value[0] = vola - mc->min; + val->value.integer.value[1] = volb - mc->min; return 0; } /* TODO */ -static const DECLARE_TLV_DB_SCALE(cs42l84_dac_tlv, -25600, 50, 1); +static const DECLARE_TLV_DB_SCALE(cs42l84_dac_tlv, -12800, 50, true); static const struct snd_kcontrol_new cs42l84_snd_controls[] = { - SOC_DOUBLE_R_EXT_TLV("DAC Playback Volume", CS42L84_DAC_CHA_VOL_LSB, - CS42L84_DAC_CHB_VOL_LSB, 0, 511, 0, + SOC_DOUBLE_R_S_EXT_TLV("DAC Playback Volume", CS42L84_DAC_CHA_VOL_LSB, + CS42L84_DAC_CHB_VOL_LSB, 0, -256, 24, 8, 0, cs42l84_get_dac_vol, cs42l84_put_dac_vol, cs42l84_dac_tlv), SOC_SINGLE("ADC Preamp Gain", CS42L84_ADC_CTL1, CS42L84_ADC_CTL1_PREAMP_GAIN_SHIFT, 2, 0), From b49c09434f4f25144c69c1fbe499f6c794fa155a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Mon, 17 Oct 2022 11:23:45 +0200 Subject: [PATCH 088/635] ASoC: apple: mca: Constrain channels according to TDM mask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't (and can't) configure the hardware correctly if the number of channels exceeds the weight of the TDM mask. Report that constraint in startup of FE. Signed-off-by: Martin Povišer --- sound/soc/apple/mca.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 64750db9b96396..409b3a716ccbcb 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -464,6 +464,28 @@ static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, return -EINVAL; } +static int mca_fe_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + unsigned int mask, nchannels; + + if (cl->tdm_slots) { + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + mask = cl->tdm_tx_mask; + else + mask = cl->tdm_rx_mask; + + nchannels = hweight32(mask); + } else { + nchannels = 2; + } + + return snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_CHANNELS, + 1, nchannels); +} + static int mca_fe_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, unsigned int rx_mask, int slots, int slot_width) { @@ -680,6 +702,7 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, } static const struct snd_soc_dai_ops mca_fe_ops = { + .startup = mca_fe_startup, .set_fmt = mca_fe_set_fmt, .set_bclk_ratio = mca_set_bclk_ratio, .set_tdm_slot = mca_fe_set_tdm_slot, From c52ccda82e534ac87ab0ab4be02985c6617713bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Mon, 17 Oct 2022 12:16:20 +0200 Subject: [PATCH 089/635] ASoC: macaudio: Improve message on opening of unrouted PCM devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index e6b7715b8e5c26..695f908e2b2a83 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -455,6 +455,29 @@ static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, return 0; } +static int macaudio_fe_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *be; + struct snd_soc_dpcm *dpcm; + + be = NULL; + for_each_dpcm_be(rtd, substream->stream, dpcm) { + be = dpcm->be; + break; + } + + if (!be) { + dev_err(rtd->dev, "opening PCM device '%s' with no audio route configured (bad settings applied to the sound card)\n", + rtd->dai_link->name); + return -EINVAL; + } + + return macaudio_dpcm_hw_params(substream, params); +} + + static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); @@ -473,7 +496,7 @@ static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) static const struct snd_soc_ops macaudio_fe_ops = { .shutdown = macaudio_dpcm_shutdown, - .hw_params = macaudio_dpcm_hw_params, + .hw_params = macaudio_fe_hw_params, }; static const struct snd_soc_ops macaudio_be_ops = { From 910fc9482aba54982534a7b6d58d422715c11837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Mon, 17 Oct 2022 12:26:27 +0200 Subject: [PATCH 090/635] ASoC: cs42l84: Enable regcache (initially) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/codecs/cs42l84.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index be40b97b84860b..fd99bed0cd2c05 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -55,12 +55,37 @@ struct cs42l84_private { int hs_type; }; +/* +static const struct reg_default cs42l84_reg_defaults[] = { +}; +*/ + +static bool cs42l84_volatile_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case CS42L84_DEVID ... CS42L84_DEVID+5: + case CS42L84_TSRS_PLUG_INT_STATUS: + case CS42L84_PLL_LOCK_STATUS: + case CS42L84_TSRS_PLUG_STATUS: + case CS42L84_HS_DET_STATUS2: + return true; + default: + return false; + } +} + static const struct regmap_config cs42l84_regmap = { .reg_bits = 16, .val_bits = 8, + .volatile_reg = cs42l84_volatile_register, + .max_register = 0xffff, - .cache_type = REGCACHE_NONE, + /* + .reg_defaults = cs42l84_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(cs42l84_reg_defaults), + */ + .cache_type = REGCACHE_RBTREE, .use_single_read = true, .use_single_write = true, From 3af3b9af09a35c0d8113c076a741a31c61db4792 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 13:29:57 +0900 Subject: [PATCH 091/635] ASoC: cs42l84: Report volume updates correctly cs42l84_put_dac_vol needs to return 1 if the volume was updated. Before this patch, it was only doing that when the MSB changed. Correctly track changes in all the registers. Signed-off-by: Hector Martin --- sound/soc/codecs/cs42l84.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index fd99bed0cd2c05..7ecfed6163dc50 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -97,7 +97,7 @@ static int cs42l84_put_dac_vol(struct snd_kcontrol *kctl, struct snd_soc_component *component = snd_soc_kcontrol_component(kctl); struct soc_mixer_control *mc = (struct soc_mixer_control *) kctl->private_value; int vola, volb; - int ret, ret2; + int ret, ret2, updated = 0; vola = val->value.integer.value[0] + mc->min; volb = val->value.integer.value[1] + mc->min; @@ -110,23 +110,31 @@ static int cs42l84_put_dac_vol(struct snd_kcontrol *kctl, CS42L84_FRZ_CTL_ENGAGE); if (ret < 0) goto bail; + updated |= ret; ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHA_VOL_LSB, 0xff, vola & 0xff); if (ret < 0) goto bail; + updated |= ret; + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHA_VOL_MSB, 0xff, (vola >> 8) & 0x01); if (ret < 0) goto bail; + updated |= ret; + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHB_VOL_LSB, 0xff, volb & 0xff); if (ret < 0) goto bail; + updated |= ret; + ret = snd_soc_component_update_bits(component, CS42L84_DAC_CHB_VOL_MSB, 0xff, (volb >> 8) & 0x01); if (ret < 0) goto bail; + ret |= updated; bail: ret2 = snd_soc_component_update_bits(component, CS42L84_FRZ_CTL, From 1bfa127a3dcd9fbb81417456738062742a63ad85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Thu, 27 Oct 2022 11:09:19 +0200 Subject: [PATCH 092/635] ASoC: macaudio: Add initial j313 fixup_controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- sound/soc/apple/macaudio.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 695f908e2b2a83..08ded9bc964622 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -745,6 +745,36 @@ static int macaudio_j274_fixup_controls(struct snd_soc_card *card) return 0; } +static int macaudio_j313_fixup_controls(struct snd_soc_card *card) { + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (ma->has_speakers) { + if (!please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, refusing probe\n"); + return -EINVAL; + } + + CHECK(snd_soc_set_enum_kctl, "* ASI1 Sel", "Left"); + CHECK(snd_soc_deactivate_kctl, "* ASI1 Sel", 0); + + /* !!! This is copied from j274, not obtained by looking at + * what macOS sets. + */ + CHECK(snd_soc_limit_volume, "* Amp Gain Volume", 14); + + /* + * Since we don't set the right slots yet to avoid + * driver conflict on the I2S bus sending ISENSE/VSENSE + * samples from the codecs back to us, disable the + * controls. + */ + CHECK(snd_soc_deactivate_kctl, "* VSENSE Switch", 0); + CHECK(snd_soc_deactivate_kctl, "* ISENSE Switch", 0); + } + + return 0; +} + static int macaudio_j314_fixup_controls(struct snd_soc_card *card) { struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); @@ -887,6 +917,7 @@ static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { static const struct of_device_id macaudio_snd_device_id[] = { { .compatible = "apple,j274-macaudio", .data = macaudio_j274_fixup_controls }, + { .compatible = "apple,j313-macaudio", .data = macaudio_j313_fixup_controls }, { .compatible = "apple,j314-macaudio", .data = macaudio_j314_fixup_controls }, { .compatible = "apple,j375-macaudio", .data = macaudio_j375_fixup_controls }, { .compatible = "apple,j413-macaudio", .data = macaudio_j314_fixup_controls }, From 3716791458245c2f88738a8d0508dd8b31b62b73 Mon Sep 17 00:00:00 2001 From: James Calligeros Date: Mon, 24 Oct 2022 21:17:31 +1000 Subject: [PATCH 093/635] ASoC: macaudio: constrain frontend channel counts In order to support the wide range of audio arrangements possible on this platform in a generic way, it is necessary for the frontend PCMs to be populated with enough TDM slots to cover all intended use cases. Userspace therefore attempts to open "phantom" channels when a frontend has more channels than its associated backend, which results in garbled audio samples and dropped frames. We must therefore dynamically constrain the frontends when they are started to ensure that userspace can never open more channels than are present on the hardware being represented by the frontend in question. Signed-off-by: James Calligeros --- sound/soc/apple/macaudio.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 08ded9bc964622..89913b42b52412 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -52,6 +52,7 @@ struct macaudio_snd_data { int jack_plugin_state; bool has_speakers; + unsigned int max_channels; struct macaudio_link_props { /* frontend props */ @@ -345,6 +346,10 @@ static int macaudio_parse_of(struct macaudio_snd_data *ma) ncodecs_per_cpu = num_codecs / num_bes; nchannels = num_codecs * (speakers ? 1 : 2); + /* Save the max number of channels on the platform */ + if (nchannels > ma->max_channels) + ma->max_channels = nchannels; + /* * If there is a single speaker, assign two channels to it, because * it can do downmix. @@ -455,6 +460,25 @@ static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, return 0; } +static int macaudio_fe_startup(struct snd_pcm_substream *substream) +{ + + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + int ret; + + /* The FEs must never have more channels than the hardware */ + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_CHANNELS, 0, ma->max_channels); + + if (ret < 0) { + dev_err(rtd->dev, "Failed to constrain FE %d! %d", rtd->dai_link->id, ret); + return ret; + } + + return 0; +} + static int macaudio_fe_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { @@ -495,6 +519,7 @@ static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) } static const struct snd_soc_ops macaudio_fe_ops = { + .startup = macaudio_fe_startup, .shutdown = macaudio_dpcm_shutdown, .hw_params = macaudio_fe_hw_params, }; From 0d8ae77c138a14323d7a4eb9db1728a6732a0742 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 15 Jan 2023 22:20:38 +0900 Subject: [PATCH 094/635] ASoC: cs42l84: Do not enable the PLL before the clocks are ready Enabling the PLL with no valid input clock leads to PLL errors. Don't do that, always disable the PLL before reconfiguring it and let unmute take care of enabling the PLL. Also monitor the PLL error bit and complain if it's set. Signed-off-by: Hector Martin --- sound/soc/codecs/cs42l84.c | 9 +++++---- sound/soc/codecs/cs42l84.h | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 7ecfed6163dc50..54f6e75c7a8a5b 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -433,6 +433,8 @@ static int cs42l84_pll_config(struct snd_soc_component *component) break; } + snd_soc_component_update_bits(component, CS42L84_PLL_CTL1, CS42L84_PLL_CTL1_EN, 0); + if (pll_ratio_table[i].mclk_src_sel) { /* Configure PLL */ snd_soc_component_update_bits(component, @@ -456,10 +458,6 @@ static int cs42l84_pll_config(struct snd_soc_component *component) snd_soc_component_write(component, CS42L84_PLL_DIVOUT, pll_ratio_table[i].pll_divout); - - snd_soc_component_update_bits(component, - CS42L84_PLL_CTL1, CS42L84_PLL_CTL1_EN, - CS42L84_PLL_CTL1_EN); } return 0; @@ -633,6 +631,9 @@ static int cs42l84_mute_stream(struct snd_soc_dai *dai, int mute, int stream) if (ret < 0) dev_warn(component->dev, "PLL failed to lock: %d\n", ret); + if (regval & CS42L84_PLL_LOCK_STATUS_ERROR) + dev_warn(component->dev, "PLL lock error\n"); + /* PLL must be running to drive glitchless switch logic */ snd_soc_component_update_bits(component, CS42L84_CCM_CTL1, diff --git a/sound/soc/codecs/cs42l84.h b/sound/soc/codecs/cs42l84.h index 9aaf19051d395f..35bd15e2ef17c9 100644 --- a/sound/soc/codecs/cs42l84.h +++ b/sound/soc/codecs/cs42l84.h @@ -31,6 +31,7 @@ #define CS42L84_TSRS_PLUG_VAL_MASK GENMASK(3, 0) #define CS42L84_PLL_LOCK_STATUS 0x040e // probably bit 0x10 #define CS42L84_PLL_LOCK_STATUS_LOCKED BIT(4) +#define CS42L84_PLL_LOCK_STATUS_ERROR BIT(5) #define CS42L84_PLUG 3 #define CS42L84_UNPLUG 0 From ea9b9d39080c939943598644cecde99f034c36b4 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 16 Apr 2023 19:27:40 +0900 Subject: [PATCH 095/635] ASoC: cs42l42: Set a faster digital ramp-up rate With the default ramp-up rate, there is a noticeable fade-in when streams start. This can be undesirable with aggressive muting for power saving, since the beginning of the stream is lost. Lower the digital output ramp-up time from 8 samples per period to 2 samples per period. This still leaves some fade-in to avoid pops, but it is a lot less noticeable and no longer feels like the stream is fading in. Signed-off-by: Hector Martin --- include/sound/cs42l42.h | 4 ++++ sound/soc/codecs/cs42l42.c | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/include/sound/cs42l42.h b/include/sound/cs42l42.h index 3994e933db1956..dcd52d64e07cd8 100644 --- a/include/sound/cs42l42.h +++ b/include/sound/cs42l42.h @@ -62,6 +62,10 @@ #define CS42L42_INTERNAL_FS_MASK (1 << CS42L42_INTERNAL_FS_SHIFT) #define CS42L42_SFTRAMP_RATE (CS42L42_PAGE_10 + 0x0A) +#define CS42L42_SFTRAMP_ASR_RATE_MASK GENMASK(7, 4) +#define CS42L42_SFTRAMP_ASR_RATE_SHIFT 4 +#define CS42L42_SFTRAMP_DSR_RATE_MASK GENMASK(3, 0) +#define CS42L42_SFTRAMP_DSR_RATE_SHIFT 0 #define CS42L42_SLOW_START_ENABLE (CS42L42_PAGE_10 + 0x0B) #define CS42L42_SLOW_START_EN_MASK GENMASK(6, 4) #define CS42L42_SLOW_START_EN_SHIFT 4 diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index b4f22c8ab2a073..96d1839b8deb98 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -2397,6 +2397,16 @@ int cs42l42_init(struct cs42l42_private *cs42l42) (1 << CS42L42_ADC_PDN_SHIFT) | (0 << CS42L42_PDN_ALL_SHIFT)); + /* + * Configure a faster digital ramp time, to avoid an audible + * fade-in when streams start. + */ + regmap_update_bits(cs42l42->regmap, CS42L42_SFTRAMP_RATE, + CS42L42_SFTRAMP_ASR_RATE_MASK | + CS42L42_SFTRAMP_DSR_RATE_MASK, + (10 << CS42L42_SFTRAMP_ASR_RATE_SHIFT) | + (1 << CS42L42_SFTRAMP_DSR_RATE_SHIFT)); + ret = cs42l42_handle_device_data(cs42l42->dev, cs42l42); if (ret != 0) goto err_shutdown; From a83cba6338dc1a40eaff3acf45fadfff71e6da6b Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 16 Apr 2023 18:53:40 +0900 Subject: [PATCH 096/635] ASoC: apple: mca: Move clock shutdown to be shutdown Codecs are set to mute after hw_free, so yanking the clock out from under them in hw_free leads to breakage. Move the clock shutdown to the shutdown op, which is late enough. Signed-off-by: Hector Martin --- sound/soc/apple/mca.c | 48 ++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index 409b3a716ccbcb..8ca40d8e87fed2 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -355,33 +355,6 @@ static int mca_be_prepare(struct snd_pcm_substream *substream, return 0; } -static int mca_be_hw_free(struct snd_pcm_substream *substream, - struct snd_soc_dai *dai) -{ - struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - - if (cl->port_driver < 0) - return -EINVAL; - - /* - * We are operating on a foreign cluster here, but since we - * belong to the same PCM, accesses should have been - * synchronized at ASoC level. - */ - fe_cl = &mca->clusters[cl->port_driver]; - if (!mca_fe_clocks_in_use(fe_cl)) - return 0; /* Nothing to do */ - - cl->clocks_in_use[substream->stream] = false; - - if (!mca_fe_clocks_in_use(fe_cl)) - mca_fe_disable_clocks(fe_cl); - - return 0; -} - static unsigned int mca_crop_mask(unsigned int mask, int nchans) { while (hweight32(mask) > nchans) @@ -779,6 +752,26 @@ static void mca_be_shutdown(struct snd_pcm_substream *substream, struct mca_cluster *cl = mca_dai_to_cluster(dai); struct mca_data *mca = cl->host; + if (cl->clocks_in_use[substream->stream] && + !WARN_ON(cl->port_driver < 0)) { + struct mca_cluster *fe_cl = &mca->clusters[cl->port_driver]; + + /* + * Typically the CODECs we are paired with will require clocks + * to be present at time of mute with the 'mute_stream' op. + * We need to disable the clocks here at the earliest (hw_free + * would be too early). + * + * We are operating on a foreign cluster here, but since we + * belong to the same PCM, accesses should have been + * synchronized at ASoC level. + */ + cl->clocks_in_use[substream->stream] = false; + + if (!mca_fe_clocks_in_use(fe_cl)) + mca_fe_disable_clocks(fe_cl); + } + cl->port_started[substream->stream] = false; if (!mca_be_started(cl)) { @@ -796,7 +789,6 @@ static void mca_be_shutdown(struct snd_pcm_substream *substream, static const struct snd_soc_dai_ops mca_be_ops = { .prepare = mca_be_prepare, - .hw_free = mca_be_hw_free, .startup = mca_be_startup, .shutdown = mca_be_shutdown, }; From 128223beb3257387d22d4ae302d9759fd7bf8075 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 23 Dec 2021 19:51:11 +0900 Subject: [PATCH 097/635] ACPI / property: Support strings in Apple _DSM props The Wi-Fi module in Apple machines has a "module-instance" device property that specifies the platform type and is used for firmware selection. Its value is a string, so add support for string values in acpi_extract_apple_properties(). Reviewed-by: Lukas Wunner Acked-by: Linus Walleij Signed-off-by: Hector Martin --- drivers/acpi/x86/apple.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/x86/apple.c b/drivers/acpi/x86/apple.c index 8812ecd03d5525..45d0f16f374f44 100644 --- a/drivers/acpi/x86/apple.c +++ b/drivers/acpi/x86/apple.c @@ -71,13 +71,16 @@ void acpi_extract_apple_properties(struct acpi_device *adev) if ( key->type != ACPI_TYPE_STRING || (val->type != ACPI_TYPE_INTEGER && - val->type != ACPI_TYPE_BUFFER)) + val->type != ACPI_TYPE_BUFFER && + val->type != ACPI_TYPE_STRING)) continue; /* skip invalid properties */ __set_bit(i, valid); newsize += key->string.length + 1; if ( val->type == ACPI_TYPE_BUFFER) newsize += val->buffer.length; + else if (val->type == ACPI_TYPE_STRING) + newsize += val->string.length + 1; } numvalid = bitmap_weight(valid, numprops); @@ -119,6 +122,12 @@ void acpi_extract_apple_properties(struct acpi_device *adev) newprops[v].type = val->type; if (val->type == ACPI_TYPE_INTEGER) { newprops[v].integer.value = val->integer.value; + } else if (val->type == ACPI_TYPE_STRING) { + newprops[v].string.length = val->string.length; + newprops[v].string.pointer = free_space; + memcpy(free_space, val->string.pointer, + val->string.length); + free_space += val->string.length + 1; } else { newprops[v].buffer.length = val->buffer.length; newprops[v].buffer.pointer = free_space; From 27ea7b8a5addfa4913c99e54c3d2eefd449bceb8 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 23 Dec 2021 19:51:36 +0900 Subject: [PATCH 098/635] brcmfmac: acpi: Add support for fetching Apple ACPI properties On DT platforms, the module-instance and antenna-sku-info properties are passed in the DT. On ACPI platforms, module-instance is passed via the analogous Apple device property mechanism, while the antenna SKU info is instead obtained via an ACPI method that grabs it from non-volatile storage. Add support for this, to allow proper firmware selection on Apple platforms. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/Makefile | 2 + .../broadcom/brcm80211/brcmfmac/acpi.c | 52 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/common.c | 1 + .../broadcom/brcm80211/brcmfmac/common.h | 9 ++++ 4 files changed, 64 insertions(+) create mode 100644 drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile index 0e996cf24f8829..dc6d27a36faa97 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile @@ -48,6 +48,8 @@ brcmfmac-$(CONFIG_OF) += \ of.o brcmfmac-$(CONFIG_DMI) += \ dmi.o +brcmfmac-$(CONFIG_ACPI) += \ + acpi.o ifeq ($(CONFIG_BRCMFMAC),m) obj-m += wcc/ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c new file mode 100644 index 00000000000000..dec6a83d13b135 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/acpi.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright The Asahi Linux Contributors + */ + +#include +#include "debug.h" +#include "core.h" +#include "common.h" + +void brcmf_acpi_probe(struct device *dev, enum brcmf_bus_type bus_type, + struct brcmf_mp_device *settings) +{ + acpi_status status; + const union acpi_object *o; + struct acpi_buffer buf = {ACPI_ALLOCATE_BUFFER, NULL}; + struct acpi_device *adev = ACPI_COMPANION(dev); + + if (!adev) + return; + + if (!ACPI_FAILURE(acpi_dev_get_property(adev, "module-instance", + ACPI_TYPE_STRING, &o))) { + brcmf_dbg(INFO, "ACPI module-instance=%s\n", o->string.pointer); + settings->board_type = devm_kasprintf(dev, GFP_KERNEL, + "apple,%s", + o->string.pointer); + } else { + brcmf_dbg(INFO, "No ACPI module-instance\n"); + } + + status = acpi_evaluate_object(adev->handle, "RWCV", NULL, &buf); + o = buf.pointer; + if (!ACPI_FAILURE(status) && o && o->type == ACPI_TYPE_BUFFER && + o->buffer.length >= 2) { + char *antenna_sku = devm_kzalloc(dev, 3, GFP_KERNEL); + + if (!antenna_sku) { + brcmf_err("Failed to allocate antenna-sku"); + } else { + memcpy(antenna_sku, o->buffer.pointer, 2); + brcmf_dbg(INFO, "ACPI RWCV data=%*phN antenna-sku=%s\n", + (int)o->buffer.length, o->buffer.pointer, + antenna_sku); + settings->antenna_sku = antenna_sku; + } + + kfree(buf.pointer); + } else { + brcmf_dbg(INFO, "No ACPI antenna-sku\n"); + } +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index f235beaddddba2..4fea6f0ea8c888 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -487,6 +487,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, /* No platform data for this device, try OF and DMI data */ brcmf_dmi_probe(settings, chip, chiprev); brcmf_of_probe(dev, bus_type, settings); + brcmf_acpi_probe(dev, bus_type, settings); } return settings; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index aa25abffcc7dbc..7167fd4f8c639b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -77,6 +77,15 @@ static inline void brcmf_dmi_probe(struct brcmf_mp_device *settings, u32 chip, u32 chiprev) {} #endif +#ifdef CONFIG_ACPI +void brcmf_acpi_probe(struct device *dev, enum brcmf_bus_type bus_type, + struct brcmf_mp_device *settings); +#else +static inline void brcmf_acpi_probe(struct device *dev, + enum brcmf_bus_type bus_type, + struct brcmf_mp_device *settings) {} +#endif + u8 brcmf_map_prio_to_prec(void *cfg, u8 prio); u8 brcmf_map_prio_to_aci(void *cfg, u8 prio); From 9587fcd02f3ed021ff3b81f3b18db5f54ce8f663 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 23 Dec 2021 19:30:17 +0900 Subject: [PATCH 099/635] brcmfmac: pcie: Provide a buffer of random bytes to the device Newer Apple firmwares on chipsets without a hardware RNG require the host to provide a buffer of 256 random bytes to the device on initialization. This buffer is present immediately before NVRAM, suffixed by a footer containing a magic number and the buffer length. This won't affect chips/firmwares that do not use this feature, so do it unconditionally. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index a9b9b2dc62d4fc..9b1778030ffe60 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -1653,6 +1654,13 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, return 0; } +struct brcmf_random_seed_footer { + __le32 length; + __le32 magic; +}; + +#define BRCMF_RANDOM_SEED_MAGIC 0xfeedc0de +#define BRCMF_RANDOM_SEED_LENGTH 0x100 static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, const struct firmware *fw, void *nvram, @@ -1684,11 +1692,32 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, brcmf_pcie_write_ram32(devinfo, devinfo->ci->ramsize - 4, 0); if (nvram) { + size_t rand_len = BRCMF_RANDOM_SEED_LENGTH; + struct brcmf_random_seed_footer footer = { + .length = cpu_to_le32(rand_len), + .magic = cpu_to_le32(BRCMF_RANDOM_SEED_MAGIC), + }; + void *randbuf; + brcmf_dbg(PCIE, "Download NVRAM %s\n", devinfo->nvram_name); address = devinfo->ci->rambase + devinfo->ci->ramsize - nvram_len; memcpy_toio(devinfo->tcm + address, nvram, nvram_len); brcmf_fw_nvram_free(nvram); + + /* Some Apple chips/firmwares expect a buffer of random data + * to be present before NVRAM + */ + brcmf_dbg(PCIE, "Download random seed\n"); + + address -= sizeof(footer); + memcpy_toio(devinfo->tcm + address, &footer, sizeof(footer)); + + address -= rand_len; + randbuf = kzalloc(rand_len, GFP_KERNEL); + get_random_bytes(randbuf, rand_len); + memcpy_toio(devinfo->tcm + address, randbuf, rand_len); + kfree(randbuf); } else { brcmf_dbg(PCIE, "No matching NVRAM file found %s\n", devinfo->nvram_name); From 2bd5d69c2b66f9ae26f3cfba331a2d806c99e305 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Dec 2021 20:00:57 +0900 Subject: [PATCH 100/635] brcmfmac: chip: Only disable D11 cores; handle an arbitrary number At least on BCM4387, the D11 cores are held in reset on cold startup and firmware expects to release reset itself. Just assert reset here and let firmware deassert it. Premature deassertion results in the firmware failing to initialize properly some of the time, with strange AXI bus errors. Also, BCM4387 has 3 cores, up from 2. The logic for handling that is in brcmf_chip_ai_resetcore(), but since we aren't using that any more, just handle it here. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 8073f31be27d95..a6239051404bfd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1292,15 +1292,18 @@ static bool brcmf_chip_cm3_set_active(struct brcmf_chip_priv *chip) static inline void brcmf_chip_cr4_set_passive(struct brcmf_chip_priv *chip) { + int i; struct brcmf_core *core; brcmf_chip_disable_arm(chip, BCMA_CORE_ARM_CR4); - core = brcmf_chip_get_core(&chip->pub, BCMA_CORE_80211); - brcmf_chip_resetcore(core, D11_BCMA_IOCTL_PHYRESET | - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN); + /* Disable the cores only and let the firmware enable them. + * Releasing reset ourselves breaks BCM4387 in weird ways. + */ + for (i = 0; (core = brcmf_chip_get_d11core(&chip->pub, i)); i++) + brcmf_chip_coredisable(core, D11_BCMA_IOCTL_PHYRESET | + D11_BCMA_IOCTL_PHYCLOCKEN, + D11_BCMA_IOCTL_PHYCLOCKEN); } static bool brcmf_chip_cr4_set_active(struct brcmf_chip_priv *chip, u32 rstvec) From 37e537d992c31587d4feaa25070b05f43e43566a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 21 Dec 2021 17:14:59 +0900 Subject: [PATCH 101/635] brcmfmac: chip: Handle 1024-unit sizes for TCM blocks BCM4387 has trailing odd-sized blocks as part of TCM which have their size described as a multiple of 1024 instead of 8192. Handle this so we can compute the TCM size properly. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../wireless/broadcom/brcm80211/brcmfmac/chip.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index a6239051404bfd..0c6833711a4d6e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -212,8 +212,8 @@ struct sbsocramregs { #define ARMCR4_TCBANB_MASK 0xf #define ARMCR4_TCBANB_SHIFT 0 -#define ARMCR4_BSZ_MASK 0x3f -#define ARMCR4_BSZ_MULT 8192 +#define ARMCR4_BSZ_MASK 0x7f +#define ARMCR4_BLK_1K_MASK 0x200 struct brcmf_core_priv { struct brcmf_core pub; @@ -676,7 +676,8 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) } /** Return the TCM-RAM size of the ARMCR4 core. */ -static u32 brcmf_chip_tcm_ramsize(struct brcmf_core_priv *cr4) +static u32 brcmf_chip_tcm_ramsize(struct brcmf_chip_priv *ci, + struct brcmf_core_priv *cr4) { u32 corecap; u32 memsize = 0; @@ -684,6 +685,7 @@ static u32 brcmf_chip_tcm_ramsize(struct brcmf_core_priv *cr4) u32 nbb; u32 totb; u32 bxinfo; + u32 blksize; u32 idx; corecap = brcmf_chip_core_read32(cr4, ARMCR4_CAP); @@ -695,7 +697,12 @@ static u32 brcmf_chip_tcm_ramsize(struct brcmf_core_priv *cr4) for (idx = 0; idx < totb; idx++) { brcmf_chip_core_write32(cr4, ARMCR4_BANKIDX, idx); bxinfo = brcmf_chip_core_read32(cr4, ARMCR4_BANKINFO); - memsize += ((bxinfo & ARMCR4_BSZ_MASK) + 1) * ARMCR4_BSZ_MULT; + if (bxinfo & ARMCR4_BLK_1K_MASK) + blksize = 1024; + else + blksize = 8192; + + memsize += ((bxinfo & ARMCR4_BSZ_MASK) + 1) * blksize; } return memsize; @@ -754,7 +761,7 @@ int brcmf_chip_get_raminfo(struct brcmf_chip *pub) mem = brcmf_chip_get_core(&ci->pub, BCMA_CORE_ARM_CR4); if (mem) { mem_core = container_of(mem, struct brcmf_core_priv, pub); - ci->pub.ramsize = brcmf_chip_tcm_ramsize(mem_core); + ci->pub.ramsize = brcmf_chip_tcm_ramsize(ci, mem_core); ci->pub.rambase = brcmf_chip_tcm_rambase(ci); if (ci->pub.rambase == INVALID_RAMBASE) { brcmf_err("RAM base not provided with ARM CR4 core\n"); From 467a35d83ac6b6ba8ef6818907f3ec7e6d17f88d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Dec 2021 03:39:44 +0900 Subject: [PATCH 102/635] brcmfmac: cfg80211: Add support for scan params v2 This new API version is required for at least the BCM4387 firmware. Add support for it, with a fallback to the v1 API. Acked-by: Linus Walleij Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 115 ++++++++++++++---- .../broadcom/brcm80211/brcmfmac/feature.c | 1 + .../broadcom/brcm80211/brcmfmac/feature.h | 4 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 49 +++++++- 4 files changed, 146 insertions(+), 23 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index a9690ec4c850c3..27b2ad9ca7547e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1039,12 +1039,50 @@ void brcmf_set_mpc(struct brcmf_if *ifp, int mpc) } } +static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, + struct brcmf_scan_params_v2_le *params_le, + struct cfg80211_scan_request *request); + +static void brcmf_scan_params_v2_to_v1(struct brcmf_scan_params_v2_le *params_v2_le, + struct brcmf_scan_params_le *params_le) +{ + size_t params_size; + u32 ch; + int n_channels, n_ssids; + + memcpy(¶ms_le->ssid_le, ¶ms_v2_le->ssid_le, + sizeof(params_le->ssid_le)); + memcpy(¶ms_le->bssid, ¶ms_v2_le->bssid, + sizeof(params_le->bssid)); + + params_le->bss_type = params_v2_le->bss_type; + params_le->scan_type = le32_to_cpu(params_v2_le->scan_type); + params_le->nprobes = params_v2_le->nprobes; + params_le->active_time = params_v2_le->active_time; + params_le->passive_time = params_v2_le->passive_time; + params_le->home_time = params_v2_le->home_time; + params_le->channel_num = params_v2_le->channel_num; + + ch = le32_to_cpu(params_v2_le->channel_num); + n_channels = ch & BRCMF_SCAN_PARAMS_COUNT_MASK; + n_ssids = ch >> BRCMF_SCAN_PARAMS_NSSID_SHIFT; + + params_size = sizeof(u16) * n_channels; + if (n_ssids > 0) { + params_size = roundup(params_size, sizeof(u32)); + params_size += sizeof(struct brcmf_ssid_le) * n_ssids; + } + + memcpy(¶ms_le->channel_list[0], + ¶ms_v2_le->channel_list[0], params_size); +} + s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bool aborted, bool fw_abort) { struct brcmf_pub *drvr = cfg->pub; - struct brcmf_scan_params_le params_le; + struct brcmf_scan_params_v2_le params_v2_le; struct cfg80211_scan_request *scan_request; u64 reqid; u32 bucket; @@ -1063,20 +1101,23 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, if (fw_abort) { /* Do a scan abort to stop the driver's scan engine */ brcmf_dbg(SCAN, "ABORT scan in firmware\n"); - memset(¶ms_le, 0, sizeof(params_le)); - eth_broadcast_addr(params_le.bssid); - params_le.bss_type = DOT11_BSSTYPE_ANY; - params_le.scan_type = 0; - params_le.channel_num = cpu_to_le32(1); - params_le.nprobes = cpu_to_le32(1); - params_le.active_time = cpu_to_le32(-1); - params_le.passive_time = cpu_to_le32(-1); - params_le.home_time = cpu_to_le32(-1); - /* Scan is aborted by setting channel_list[0] to -1 */ - params_le.channel_list[0] = cpu_to_le16(-1); + + brcmf_escan_prep(cfg, ¶ms_v2_le, NULL); + /* E-Scan (or anyother type) can be aborted by SCAN */ - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_le, sizeof(params_le)); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, + ¶ms_v2_le, + sizeof(params_v2_le)); + } else { + struct brcmf_scan_params_le params_le; + + brcmf_scan_params_v2_to_v1(¶ms_v2_le, ¶ms_le); + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, + ¶ms_le, + sizeof(params_le)); + } + if (err) bphy_err(drvr, "Scan abort failed\n"); } @@ -1296,7 +1337,7 @@ brcmf_cfg80211_change_iface(struct wiphy *wiphy, struct net_device *ndev, } static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, - struct brcmf_scan_params_le *params_le, + struct brcmf_scan_params_v2_le *params_le, struct cfg80211_scan_request *request) { u32 n_ssids; @@ -1305,9 +1346,14 @@ static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, s32 offset; u16 chanspec; char *ptr; + int length; struct brcmf_ssid_le ssid_le; eth_broadcast_addr(params_le->bssid); + + length = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; + + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); params_le->bss_type = DOT11_BSSTYPE_ANY; params_le->scan_type = BRCMF_SCANTYPE_ACTIVE; params_le->channel_num = 0; @@ -1317,6 +1363,15 @@ static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, params_le->home_time = cpu_to_le32(-1); memset(¶ms_le->ssid_le, 0, sizeof(params_le->ssid_le)); + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + return; + } + n_ssids = request->n_ssids; n_channels = request->n_channels; @@ -1324,6 +1379,7 @@ static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", n_channels); if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); for (i = 0; i < n_channels; i++) { chanspec = channel_to_chanspec(&cfg->d11inf, request->channels[i]); @@ -1334,12 +1390,14 @@ static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, } else { brcmf_dbg(SCAN, "Scanning all channels\n"); } + /* Copy ssid array if applicable */ brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); if (n_ssids > 0) { - offset = offsetof(struct brcmf_scan_params_le, channel_list) + + offset = offsetof(struct brcmf_scan_params_v2_le, channel_list) + n_channels * sizeof(u16); offset = roundup(offset, sizeof(u32)); + length += sizeof(ssid_le) * n_ssids, ptr = (char *)params_le + offset; for (i = 0; i < n_ssids; i++) { memset(&ssid_le, 0, sizeof(ssid_le)); @@ -1357,8 +1415,9 @@ static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, } } else { brcmf_dbg(SCAN, "Performing passive scan\n"); - params_le->scan_type = BRCMF_SCANTYPE_PASSIVE; + params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_PASSIVE); } + params_le->length = cpu_to_le16(length); /* Adding mask to channel numbers */ params_le->channel_num = cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | @@ -1370,8 +1429,8 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, struct cfg80211_scan_request *request) { struct brcmf_pub *drvr = cfg->pub; - s32 params_size = BRCMF_SCAN_PARAMS_FIXED_SIZE + - offsetof(struct brcmf_escan_params_le, params_le); + s32 params_size = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE + + offsetof(struct brcmf_escan_params_le, params_v2_le); struct brcmf_escan_params_le *params; s32 err = 0; @@ -1391,8 +1450,22 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, goto exit; } BUG_ON(params_size + sizeof("escan") >= BRCMF_DCMD_MEDLEN); - brcmf_escan_prep(cfg, ¶ms->params_le, request); - params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION); + brcmf_escan_prep(cfg, ¶ms->params_v2_le, request); + + params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION_V2); + + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { + struct brcmf_escan_params_le *params_v1; + + params_size -= BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; + params_size += BRCMF_SCAN_PARAMS_FIXED_SIZE; + params_v1 = kzalloc(params_size, GFP_KERNEL); + params_v1->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION); + brcmf_scan_params_v2_to_v1(¶ms->params_v2_le, ¶ms_v1->params_le); + kfree(params); + params = params_v1; + } + params->action = cpu_to_le16(WL_ESCAN_ACTION_START); params->sync_id = cpu_to_le16(0x1234); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 10bac865d72435..b6797f800e55ac 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -290,6 +290,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) ifp->drvr->feat_flags |= BIT(BRCMF_FEAT_SCAN_RANDOM_MAC); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); + brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_SCAN_V2, "scan_ver"); if (drvr->settings->feature_disable) { brcmf_dbg(INFO, "Features: 0x%02x, disable: 0x%02x\n", diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index f1b086a69d735c..549298c55b5587 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -30,6 +30,7 @@ * SAE: simultaneous authentication of equals * FWAUTH: Firmware authenticator * DUMP_OBSS: Firmware has capable to dump obss info to support ACS + * SCAN_V2: Version 2 scan params */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -53,7 +54,8 @@ BRCMF_FEAT_DEF(DOT11H) \ BRCMF_FEAT_DEF(SAE) \ BRCMF_FEAT_DEF(FWAUTH) \ - BRCMF_FEAT_DEF(DUMP_OBSS) + BRCMF_FEAT_DEF(DUMP_OBSS) \ + BRCMF_FEAT_DEF(SCAN_V2) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 04e1beedfd81f4..b3844d0d1adbc3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -48,6 +48,10 @@ /* size of brcmf_scan_params not including variable length array */ #define BRCMF_SCAN_PARAMS_FIXED_SIZE 64 +#define BRCMF_SCAN_PARAMS_V2_FIXED_SIZE 72 + +/* version of brcmf_scan_params structure */ +#define BRCMF_SCAN_PARAMS_VERSION_V2 2 /* masks for channel and ssid count */ #define BRCMF_SCAN_PARAMS_COUNT_MASK 0x0000ffff @@ -67,6 +71,7 @@ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 #define BRCMF_ESCAN_REQ_VERSION 1 +#define BRCMF_ESCAN_REQ_VERSION_V2 2 #define BRCMF_MAXRATES_IN_SET 16 /* max # of rates in rateset */ @@ -386,6 +391,45 @@ struct brcmf_scan_params_le { __le16 channel_list[1]; /* list of chanspecs */ }; +struct brcmf_scan_params_v2_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 pad; + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[1]; /* list of chanspecs */ +}; + struct brcmf_scan_results { u32 buflen; u32 version; @@ -397,7 +441,10 @@ struct brcmf_escan_params_le { __le32 version; __le16 action; __le16 sync_id; - struct brcmf_scan_params_le params_le; + union { + struct brcmf_scan_params_le params_le; + struct brcmf_scan_params_v2_le params_v2_le; + }; }; struct brcmf_escan_result_le { From 9c95ca388a18dd59376d0a206eb34a6ae26e7fcc Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Dec 2021 18:15:10 +0900 Subject: [PATCH 103/635] brcmfmac: feature: Add support for setting feats based on WLC version The "wlc_ver" iovar returns information on the WLC and EPI versions. This can be used to determine whether the PMKID_V2 and _V3 features are supported. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/feature.c | 48 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/feature.h | 4 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 25 ++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index b6797f800e55ac..6d10c9efbe93d8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -126,6 +126,53 @@ static void brcmf_feat_firmware_overrides(struct brcmf_pub *drv) drv->feat_flags |= feat_flags; } +struct brcmf_feat_wlcfeat { + u16 min_ver_major; + u16 min_ver_minor; + u32 feat_flags; +}; + +static const struct brcmf_feat_wlcfeat brcmf_feat_wlcfeat_map[] = { + { 12, 0, BIT(BRCMF_FEAT_PMKID_V2) }, + { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) }, +}; + +static void brcmf_feat_wlc_version_overrides(struct brcmf_pub *drv) +{ + struct brcmf_if *ifp = brcmf_get_ifp(drv, 0); + const struct brcmf_feat_wlcfeat *e; + struct brcmf_wlc_version_le ver; + u32 feat_flags = 0; + int i, err, major, minor; + + err = brcmf_fil_iovar_data_get(ifp, "wlc_ver", &ver, sizeof(ver)); + if (err) + return; + + major = le16_to_cpu(ver.wlc_ver_major); + minor = le16_to_cpu(ver.wlc_ver_minor); + + brcmf_dbg(INFO, "WLC version: %d.%d\n", major, minor); + + for (i = 0; i < ARRAY_SIZE(brcmf_feat_wlcfeat_map); i++) { + e = &brcmf_feat_wlcfeat_map[i]; + if (major > e->min_ver_major || + (major == e->min_ver_major && + minor >= e->min_ver_minor)) { + feat_flags |= e->feat_flags; + } + } + + if (!feat_flags) + return; + + for (i = 0; i < BRCMF_FEAT_LAST; i++) + if (feat_flags & BIT(i)) + brcmf_dbg(INFO, "enabling firmware feature: %s\n", + brcmf_feat_names[i]); + drv->feat_flags |= feat_flags; +} + /** * brcmf_feat_iovar_int_get() - determine feature through iovar query. * @@ -299,6 +346,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) ifp->drvr->feat_flags &= ~drvr->settings->feature_disable; } + brcmf_feat_wlc_version_overrides(drvr); brcmf_feat_firmware_overrides(drvr); /* set chip related quirks */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 549298c55b5587..7f4f0b3e4a7b4a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -55,7 +55,9 @@ BRCMF_FEAT_DEF(SAE) \ BRCMF_FEAT_DEF(FWAUTH) \ BRCMF_FEAT_DEF(DUMP_OBSS) \ - BRCMF_FEAT_DEF(SCAN_V2) + BRCMF_FEAT_DEF(SCAN_V2) \ + BRCMF_FEAT_DEF(PMKID_V2) \ + BRCMF_FEAT_DEF(PMKID_V3) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index b3844d0d1adbc3..801709c26b7bf4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -788,6 +788,31 @@ struct brcmf_rev_info_le { __le32 nvramrev; }; +/** + * struct brcmf_wlc_version_le - firmware revision info. + * + * @version: structure version. + * @length: structure length. + * @epi_ver_major: EPI major version + * @epi_ver_minor: EPI minor version + * @epi_ver_rc: EPI rc version + * @epi_ver_incr: EPI increment version + * @wlc_ver_major: WLC major version + * @wlc_ver_minor: WLC minor version + */ +struct brcmf_wlc_version_le { + __le16 version; + __le16 length; + + __le16 epi_ver_major; + __le16 epi_ver_minor; + __le16 epi_ver_rc; + __le16 epi_ver_incr; + + __le16 wlc_ver_major; + __le16 wlc_ver_minor; +}; + /** * struct brcmf_assoclist_le - request assoc list. * From 289c1cd14c0f6ab6f2998b3548baa8de9a2be42d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Dec 2021 18:16:33 +0900 Subject: [PATCH 104/635] brcmfmac: cfg80211: Add support for PMKID_V3 operations Add support for the new PMKID_V3 API, which allows performing PMKID mutations individually, instead of requiring the driver to keep track of the full list. This new API is required by at least BCM4387. Note that PMKID_V2 is not implemented yet. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 52 +++++++++++- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 83 +++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 27b2ad9ca7547e..42da78096c8910 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4310,6 +4310,37 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, return 0; } +static s32 +brcmf_pmksa_v3_op(struct brcmf_if *ifp, struct cfg80211_pmksa *pmksa, + bool alive) +{ + struct brcmf_pmk_op_v3_le *pmk_op; + int length = offsetof(struct brcmf_pmk_op_v3_le, pmk); + int ret; + + pmk_op = kzalloc(sizeof(*pmk_op), GFP_KERNEL); + pmk_op->version = cpu_to_le16(BRCMF_PMKSA_VER_3); + + if (!pmksa) { + /* Flush operation, operate on entire list */ + pmk_op->count = cpu_to_le16(0); + } else { + /* Single PMK operation */ + pmk_op->count = cpu_to_le16(1); + length += sizeof(struct brcmf_pmksa_v3); + memcpy(pmk_op->pmk[0].bssid, pmksa->bssid, ETH_ALEN); + memcpy(pmk_op->pmk[0].pmkid, pmksa->pmkid, WLAN_PMKID_LEN); + pmk_op->pmk[0].pmkid_len = WLAN_PMKID_LEN; + pmk_op->pmk[0].time_left = alive ? BRCMF_PMKSA_NO_EXPIRY : 0; + } + + pmk_op->length = cpu_to_le16(length); + + ret = brcmf_fil_iovar_data_set(ifp, "pmkid_info", pmk_op, sizeof(*pmk_op)); + kfree(pmk_op); + return ret; +} + static __used s32 brcmf_update_pmklist(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) { @@ -4343,6 +4374,14 @@ brcmf_cfg80211_set_pmksa(struct wiphy *wiphy, struct net_device *ndev, if (!check_vif_up(ifp->vif)) return -EIO; + brcmf_dbg(CONN, "set_pmksa - PMK bssid: %pM =\n", pmksa->bssid); + brcmf_dbg(CONN, "%*ph\n", WLAN_PMKID_LEN, pmksa->pmkid); + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PMKID_V3)) + return brcmf_pmksa_v3_op(ifp, pmksa, true); + + /* TODO: implement PMKID_V2 */ + npmk = le32_to_cpu(cfg->pmk_list.npmk); for (i = 0; i < npmk; i++) if (!memcmp(pmksa->bssid, pmk[i].bssid, ETH_ALEN)) @@ -4359,9 +4398,6 @@ brcmf_cfg80211_set_pmksa(struct wiphy *wiphy, struct net_device *ndev, return -EINVAL; } - brcmf_dbg(CONN, "set_pmksa - PMK bssid: %pM =\n", pmk[npmk].bssid); - brcmf_dbg(CONN, "%*ph\n", WLAN_PMKID_LEN, pmk[npmk].pmkid); - err = brcmf_update_pmklist(cfg, ifp); brcmf_dbg(TRACE, "Exit\n"); @@ -4385,6 +4421,11 @@ brcmf_cfg80211_del_pmksa(struct wiphy *wiphy, struct net_device *ndev, brcmf_dbg(CONN, "del_pmksa - PMK bssid = %pM\n", pmksa->bssid); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PMKID_V3)) + return brcmf_pmksa_v3_op(ifp, pmksa, false); + + /* TODO: implement PMKID_V2 */ + npmk = le32_to_cpu(cfg->pmk_list.npmk); for (i = 0; i < npmk; i++) if (!memcmp(pmksa->bssid, pmk[i].bssid, ETH_ALEN)) @@ -4421,6 +4462,11 @@ brcmf_cfg80211_flush_pmksa(struct wiphy *wiphy, struct net_device *ndev) if (!check_vif_up(ifp->vif)) return -EIO; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PMKID_V3)) + return brcmf_pmksa_v3_op(ifp, NULL, false); + + /* TODO: implement PMKID_V2 */ + memset(&cfg->pmk_list, 0, sizeof(cfg->pmk_list)); err = brcmf_update_pmklist(cfg, ifp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 801709c26b7bf4..792adaf880b444 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -174,6 +174,10 @@ #define BRCMF_HE_CAP_MCS_MAP_NSS_MAX 8 +#define BRCMF_PMKSA_VER_2 2 +#define BRCMF_PMKSA_VER_3 3 +#define BRCMF_PMKSA_NO_EXPIRY 0xffffffff + /* MAX_CHUNK_LEN is the maximum length for data passing to firmware in each * ioctl. It is relatively small because firmware has small maximum size input * playload restriction for ioctls. @@ -355,6 +359,12 @@ struct brcmf_ssid_le { unsigned char SSID[IEEE80211_MAX_SSID_LEN]; }; +/* Alternate SSID structure used in some places... */ +struct brcmf_ssid8_le { + u8 SSID_len; + unsigned char SSID[IEEE80211_MAX_SSID_LEN]; +}; + struct brcmf_scan_params_le { struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ u8 bssid[ETH_ALEN]; /* default: bcast */ @@ -875,6 +885,51 @@ struct brcmf_pmksa { u8 pmkid[WLAN_PMKID_LEN]; }; +/** + * struct brcmf_pmksa_v2 - PMK Security Association + * + * @length: Length of the structure. + * @bssid: The AP's BSSID. + * @pmkid: The PMK ID. + * @pmk: PMK material for FILS key derivation. + * @pmk_len: Length of PMK data. + * @ssid: The AP's SSID. + * @fils_cache_id: FILS cache identifier + */ +struct brcmf_pmksa_v2 { + __le16 length; + u8 bssid[ETH_ALEN]; + u8 pmkid[WLAN_PMKID_LEN]; + u8 pmk[WLAN_PMK_LEN_SUITE_B_192]; + __le16 pmk_len; + struct brcmf_ssid8_le ssid; + u16 fils_cache_id; +}; + +/** + * struct brcmf_pmksa_v3 - PMK Security Association + * + * @bssid: The AP's BSSID. + * @pmkid: The PMK ID. + * @pmkid_len: The length of the PMK ID. + * @pmk: PMK material for FILS key derivation. + * @pmk_len: Length of PMK data. + * @fils_cache_id: FILS cache identifier + * @ssid: The AP's SSID. + * @time_left: Remaining time until expiry. 0 = expired, ~0 = no expiry. + */ +struct brcmf_pmksa_v3 { + u8 bssid[ETH_ALEN]; + u8 pmkid[WLAN_PMKID_LEN]; + u8 pmkid_len; + u8 pmk[WLAN_PMK_LEN_SUITE_B_192]; + u8 pmk_len; + __le16 fils_cache_id; + u8 pad; + struct brcmf_ssid8_le ssid; + __le32 time_left; +}; + /** * struct brcmf_pmk_list_le - List of pmksa's. * @@ -886,6 +941,34 @@ struct brcmf_pmk_list_le { struct brcmf_pmksa pmk[BRCMF_MAXPMKID]; }; +/** + * struct brcmf_pmk_list_v2_le - List of pmksa's. + * + * @version: Request version. + * @length: Length of this structure. + * @pmk: PMK SA information. + */ +struct brcmf_pmk_list_v2_le { + __le16 version; + __le16 length; + struct brcmf_pmksa_v2 pmk[BRCMF_MAXPMKID]; +}; + +/** + * struct brcmf_pmk_op_v3_le - Operation on PMKSA list. + * + * @version: Request version. + * @length: Length of this structure. + * @pmk: PMK SA information. + */ +struct brcmf_pmk_op_v3_le { + __le16 version; + __le16 length; + __le16 count; + __le16 pad; + struct brcmf_pmksa_v3 pmk[BRCMF_MAXPMKID]; +}; + /** * struct brcmf_pno_param_le - PNO scan configuration parameters * From e9f41e3067a16895a8544a4195e30ec448ab7b30 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Dec 2021 19:15:58 +0900 Subject: [PATCH 105/635] brcmfmac: cfg80211: Pass the PMK in binary instead of hex Apparently the hex passphrase mechanism does not work on newer chips/firmware (e.g. BCM4387). It seems there was a simple way of passing it in binary all along, so use that and avoid the hexification. OpenBSD has been doing it like this from the beginning, so this should work on all chips. Also clear the structure before setting the PMK. This was leaking uninitialized stack contents to the device. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 42da78096c8910..96353ae279700a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1690,13 +1690,14 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_wsec_pmk_le pmk; - int i, err; + int err; + + memset(&pmk, 0, sizeof(pmk)); - /* convert to firmware key format */ - pmk.key_len = cpu_to_le16(pmk_len << 1); - pmk.flags = cpu_to_le16(BRCMF_WSEC_PASSPHRASE); - for (i = 0; i < pmk_len; i++) - snprintf(&pmk.key[2 * i], 3, "%02x", pmk_data[i]); + /* pass pmk directly */ + pmk.key_len = cpu_to_le16(pmk_len); + pmk.flags = cpu_to_le16(0); + memcpy(pmk.key, pmk_data, pmk_len); /* store psk in firmware */ err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_WSEC_PMK, From 87f73cbe186c47c1eff28c2cfbc16de18e0d8f73 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 6 Jan 2022 19:45:15 +0900 Subject: [PATCH 106/635] brcmflac: cfg80211: Use WSEC to set SAE password Using the WSEC command instead of sae_password seems to be the supported mechanism on newer firmware, and also how the brcmdhd driver does it. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 46 ++++++++----------- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 2 +- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 96353ae279700a..45ded28737e6f1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1686,52 +1686,44 @@ static u16 brcmf_map_fw_linkdown_reason(const struct brcmf_event_msg *e) return reason; } -static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) +static int brcmf_set_wsec(struct brcmf_if *ifp, const u8 *key, u16 key_len, u16 flags) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_wsec_pmk_le pmk; int err; + if (key_len > sizeof(pmk.key)) { + bphy_err(drvr, "key must be less than %zu bytes\n", + sizeof(pmk.key)); + return -EINVAL; + } + memset(&pmk, 0, sizeof(pmk)); - /* pass pmk directly */ - pmk.key_len = cpu_to_le16(pmk_len); - pmk.flags = cpu_to_le16(0); - memcpy(pmk.key, pmk_data, pmk_len); + /* pass key material directly */ + pmk.key_len = cpu_to_le16(key_len); + pmk.flags = cpu_to_le16(flags); + memcpy(pmk.key, key, key_len); - /* store psk in firmware */ + /* store key material in firmware */ err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_WSEC_PMK, &pmk, sizeof(pmk)); if (err < 0) bphy_err(drvr, "failed to change PSK in firmware (len=%u)\n", - pmk_len); + key_len); return err; } +static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) +{ + return brcmf_set_wsec(ifp, pmk_data, pmk_len, 0); +} + static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, u16 pwd_len) { - struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_wsec_sae_pwd_le sae_pwd; - int err; - - if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { - bphy_err(drvr, "sae_password must be less than %d\n", - BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); - return -EINVAL; - } - - sae_pwd.key_len = cpu_to_le16(pwd_len); - memcpy(sae_pwd.key, pwd_data, pwd_len); - - err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd, - sizeof(sae_pwd)); - if (err < 0) - bphy_err(drvr, "failed to set SAE password in firmware (len=%u)\n", - pwd_len); - - return err; + return brcmf_set_wsec(ifp, pwd_data, pwd_len, BRCMF_WSEC_PASSPHRASE); } static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 792adaf880b444..3ba90878c47da6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -574,7 +574,7 @@ struct brcmf_wsec_key_le { struct brcmf_wsec_pmk_le { __le16 key_len; __le16 flags; - u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1]; + u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; }; /** From c8a97ffbcd27e9884fa06d3dc96b87636d5f130b Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 21 Dec 2021 17:52:03 +0900 Subject: [PATCH 107/635] brcmfmac: pcie: Add IDs/properties for BCM4387 This chip is present on Apple M1 Pro/Max (t600x) platforms: * maldives (apple,j314s): MacBook Pro (14-inch, M1 Pro, 2021) * maldives (apple,j314c): MacBook Pro (14-inch, M1 Max, 2021) * madagascar (apple,j316s): MacBook Pro (16-inch, M1 Pro, 2021) * madagascar (apple,j316c): MacBook Pro (16-inch, M1 Max, 2021) Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 2 ++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 8 ++++++++ .../net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 2 ++ 3 files changed, 12 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 0c6833711a4d6e..1803cc1c8fcbc0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -744,6 +744,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) return 0x170000; case BRCM_CC_4378_CHIP_ID: return 0x352000; + case BRCM_CC_4387_CHIP_ID: + return 0x740000; default: brcmf_err("unknown chip: %s\n", ci->pub.name); break; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 9b1778030ffe60..d78ec33697b125 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -67,6 +67,7 @@ BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie"); BRCMF_FW_DEF(4371, "brcmfmac4371-pcie"); BRCMF_FW_CLM_DEF(4377B3, "brcmfmac4377b3-pcie"); BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie"); +BRCMF_FW_CLM_DEF(4387C2, "brcmfmac4387c2-pcie"); /* firmware config files */ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.txt"); @@ -101,6 +102,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), BRCMF_FW_ENTRY(BRCM_CC_4377_CHIP_ID, 0xFFFFFFFF, 4377B3), /* revision ID 4 */ BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFFF, 4378B1), /* revision ID 3 */ + BRCMF_FW_ENTRY(BRCM_CC_4387_CHIP_ID, 0xFFFFFFFF, 4387C2), /* revision ID 7 */ }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -2045,6 +2047,11 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) base = 0x1120; words = 0x170; break; + case BRCM_CC_4387_CHIP_ID: + coreid = BCMA_CORE_GCI; + base = 0x113c; + words = 0x170; + break; default: /* OTP not supported on this chip */ return 0; @@ -2659,6 +2666,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_43596_DEVICE_ID, CYW), BRCMF_PCIE_DEVICE(BRCM_PCIE_4377_DEVICE_ID, WCC), BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID, WCC), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4387_DEVICE_ID, WCC), { /* end: all zeroes */ } }; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 896615f5795227..44684bf1b9acc6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -54,6 +54,7 @@ #define BRCM_CC_4371_CHIP_ID 0x4371 #define BRCM_CC_4377_CHIP_ID 0x4377 #define BRCM_CC_4378_CHIP_ID 0x4378 +#define BRCM_CC_4387_CHIP_ID 0x4387 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 #define CY_CC_43439_CHIP_ID 43439 @@ -95,6 +96,7 @@ #define BRCM_PCIE_43596_DEVICE_ID 0x4415 #define BRCM_PCIE_4377_DEVICE_ID 0x4488 #define BRCM_PCIE_4378_DEVICE_ID 0x4425 +#define BRCM_PCIE_4387_DEVICE_ID 0x4433 /* brcmsmac IDs */ #define BCM4313_D11N2G_ID 0x4727 /* 4313 802.11n 2.4G device */ From c9a300b040ead799834bff906b6bc0291738e6fe Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 26 Dec 2021 00:25:00 +0900 Subject: [PATCH 108/635] brcmfmac: common: Add support for downloading TxCap blobs The TxCap blobs are additional data blobs used on Apple devices, and are uploaded analogously to CLM blobs. Add core support for doing this. Acked-by: Linus Walleij Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/bus.h | 1 + .../broadcom/brcm80211/brcmfmac/common.c | 93 ++++++++++++++----- 2 files changed, 70 insertions(+), 24 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 501136e011b552..fe31051a9e11b1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -55,6 +55,7 @@ enum brcmf_bus_protocol_type { /* Firmware blobs that may be available */ enum brcmf_blob_type { BRCMF_BLOB_CLM, + BRCMF_BLOB_TXCAP, }; struct brcmf_mp_device; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 4fea6f0ea8c888..88e81b83af9d68 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -101,7 +101,7 @@ void brcmf_c_set_joinpref_default(struct brcmf_if *ifp) static int brcmf_c_download(struct brcmf_if *ifp, u16 flag, struct brcmf_dload_data_le *dload_buf, - u32 len) + u32 len, const char *var) { s32 err; @@ -111,18 +111,18 @@ static int brcmf_c_download(struct brcmf_if *ifp, u16 flag, dload_buf->len = cpu_to_le32(len); dload_buf->crc = cpu_to_le32(0); - err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf, + err = brcmf_fil_iovar_data_set(ifp, var, dload_buf, struct_size(dload_buf, data, len)); return err; } -static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) +static int brcmf_c_download_blob(struct brcmf_if *ifp, + const void *data, size_t size, + const char *loadvar, const char *statvar) { struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_bus *bus = drvr->bus_if; struct brcmf_dload_data_le *chunk_buf; - const struct firmware *clm = NULL; u32 chunk_len; u32 datalen; u32 cumulative_len; @@ -132,21 +132,14 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) brcmf_dbg(TRACE, "Enter\n"); - err = brcmf_bus_get_blob(bus, &clm, BRCMF_BLOB_CLM); - if (err || !clm) { - brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n", - err); - return 0; - } - chunk_buf = kzalloc(struct_size(chunk_buf, data, MAX_CHUNK_LEN), GFP_KERNEL); if (!chunk_buf) { err = -ENOMEM; - goto done; + return -ENOMEM; } - datalen = clm->size; + datalen = size; cumulative_len = 0; do { if (datalen > MAX_CHUNK_LEN) { @@ -155,9 +148,10 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) chunk_len = datalen; dl_flag |= DL_END; } - memcpy(chunk_buf->data, clm->data + cumulative_len, chunk_len); + memcpy(chunk_buf->data, data + cumulative_len, chunk_len); - err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len); + err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len, + loadvar); dl_flag &= ~DL_BEGIN; @@ -166,20 +160,64 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) } while ((datalen > 0) && (err == 0)); if (err) { - bphy_err(drvr, "clmload (%zu byte file) failed (%d)\n", - clm->size, err); - /* Retrieve clmload_status and print */ - err = brcmf_fil_iovar_int_get(ifp, "clmload_status", &status); + bphy_err(drvr, "%s (%zu byte file) failed (%d)\n", + loadvar, size, err); + /* Retrieve status and print */ + err = brcmf_fil_iovar_int_get(ifp, statvar, &status); if (err) - bphy_err(drvr, "get clmload_status failed (%d)\n", err); + bphy_err(drvr, "get %s failed (%d)\n", statvar, err); else - brcmf_dbg(INFO, "clmload_status=%d\n", status); + brcmf_dbg(INFO, "%s=%d\n", statvar, status); err = -EIO; } kfree(chunk_buf); -done: - release_firmware(clm); + return err; +} + +static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus = drvr->bus_if; + const struct firmware *fw = NULL; + s32 err; + + brcmf_dbg(TRACE, "Enter\n"); + + err = brcmf_bus_get_blob(bus, &fw, BRCMF_BLOB_CLM); + if (err || !fw) { + brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n", + err); + return 0; + } + + err = brcmf_c_download_blob(ifp, fw->data, fw->size, + "clmload", "clmload_status"); + + release_firmware(fw); + return err; +} + +static int brcmf_c_process_txcap_blob(struct brcmf_if *ifp) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus = drvr->bus_if; + const struct firmware *fw = NULL; + s32 err; + + brcmf_dbg(TRACE, "Enter\n"); + + err = brcmf_bus_get_blob(bus, &fw, BRCMF_BLOB_TXCAP); + if (err || !fw) { + brcmf_info("no txcap_blob available (err=%d)\n", err); + return 0; + } + + brcmf_info("TxCap blob found, loading\n"); + err = brcmf_c_download_blob(ifp, fw->data, fw->size, + "txcapload", "txcapload_status"); + + release_firmware(fw); return err; } @@ -291,6 +329,13 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) goto done; } + /* Do TxCap downloading, if needed */ + err = brcmf_c_process_txcap_blob(ifp); + if (err < 0) { + bphy_err(drvr, "download TxCap blob file failed, %d\n", err); + goto done; + } + /* query for 'ver' to get version info from firmware */ memset(buf, 0, sizeof(buf)); err = brcmf_fil_iovar_data_get(ifp, "ver", buf, sizeof(buf)); From 25db749b7ad41e50bb23891550f0df3eb9966e4d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 26 Dec 2021 00:26:10 +0900 Subject: [PATCH 109/635] brcmfmac: pcie: Load and provide TxCap blobs These blobs are named .txcap_blob, and exist alongside the existing .clm_blob files. Use the existing firmware machinery to provide them to the core. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index d78ec33697b125..6834b5fcfbb2b7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -76,6 +76,7 @@ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txt"); /* per-board firmware binaries */ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.bin"); MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.clm_blob"); +MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txcap_blob"); static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), @@ -329,7 +330,9 @@ struct brcmf_pciedev_info { char fw_name[BRCMF_FW_NAME_LEN]; char nvram_name[BRCMF_FW_NAME_LEN]; char clm_name[BRCMF_FW_NAME_LEN]; + char txcap_name[BRCMF_FW_NAME_LEN]; const struct firmware *clm_fw; + const struct firmware *txcap_fw; const struct brcmf_pcie_reginfo *reginfo; void __iomem *regs; void __iomem *tcm; @@ -1520,6 +1523,10 @@ static int brcmf_pcie_get_blob(struct device *dev, const struct firmware **fw, *fw = devinfo->clm_fw; devinfo->clm_fw = NULL; break; + case BRCMF_BLOB_TXCAP: + *fw = devinfo->txcap_fw; + devinfo->txcap_fw = NULL; + break; default: return -ENOENT; } @@ -2109,6 +2116,7 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) #define BRCMF_PCIE_FW_CODE 0 #define BRCMF_PCIE_FW_NVRAM 1 #define BRCMF_PCIE_FW_CLM 2 +#define BRCMF_PCIE_FW_TXCAP 3 static void brcmf_pcie_setup(struct device *dev, int ret, struct brcmf_fw_request *fwreq) @@ -2135,6 +2143,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data; nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len; devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary; + devinfo->txcap_fw = fwreq->items[BRCMF_PCIE_FW_TXCAP].binary; kfree(fwreq); ret = brcmf_chip_get_raminfo(devinfo->ci); @@ -2216,6 +2225,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) { ".bin", devinfo->fw_name }, { ".txt", devinfo->nvram_name }, { ".clm_blob", devinfo->clm_name }, + { ".txcap_blob", devinfo->txcap_name }, }; fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, @@ -2230,6 +2240,8 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY; fwreq->items[BRCMF_PCIE_FW_CLM].flags = BRCMF_FW_REQF_OPTIONAL; + fwreq->items[BRCMF_PCIE_FW_TXCAP].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_TXCAP].flags = BRCMF_FW_REQF_OPTIONAL; /* NVRAM reserves PCI domain 0 for Broadcom's SDK faked bus */ fwreq->domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1; fwreq->bus_nr = devinfo->pdev->bus->number; @@ -2527,6 +2539,7 @@ brcmf_pcie_remove(struct pci_dev *pdev) brcmf_pcie_reset_device(devinfo); brcmf_pcie_release_resource(devinfo); release_firmware(devinfo->clm_fw); + release_firmware(devinfo->txcap_fw); if (devinfo->ci) brcmf_chip_detach(devinfo->ci); From 85abc589d20251d6dd84cf9181b17a69085a4312 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 26 Dec 2021 00:53:37 +0900 Subject: [PATCH 110/635] brcmfmac: common: Add support for external calibration blobs The calibration blob for a chip is normally stored in SROM and loaded internally by the firmware. However, Apple ARM64 platforms instead store it as part of platform configuration data, and provide it via the Apple Device Tree. We forward this into the Linux DT in the bootloader. Add support for taking this blob from the DT and loading it into the dongle. The loading mechanism is the same as used for the CLM and TxCap blobs. Reviewed-by: Linus Walleij Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/common.c | 24 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/common.h | 2 ++ .../wireless/broadcom/brcm80211/brcmfmac/of.c | 7 ++++++ 3 files changed, 33 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 88e81b83af9d68..a194b0e68eb53a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -246,6 +246,23 @@ static const u8 brcmf_default_mac_address[ETH_ALEN] = { 0x00, 0x90, 0x4c, 0xc5, 0x12, 0x38 }; +static int brcmf_c_process_cal_blob(struct brcmf_if *ifp) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_mp_device *settings = drvr->settings; + s32 err; + + brcmf_dbg(TRACE, "Enter\n"); + + if (!settings->cal_blob || !settings->cal_size) + return 0; + + brcmf_info("Calibration blob provided by platform, loading\n"); + err = brcmf_c_download_blob(ifp, settings->cal_blob, settings->cal_size, + "calload", "calload_status"); + return err; +} + int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; @@ -336,6 +353,13 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) goto done; } + /* Download external calibration blob, if available */ + err = brcmf_c_process_cal_blob(ifp); + if (err < 0) { + bphy_err(drvr, "download calibration blob file failed, %d\n", err); + goto done; + } + /* query for 'ver' to get version info from firmware */ memset(buf, 0, sizeof(buf)); err = brcmf_fil_iovar_data_get(ifp, "ver", buf, sizeof(buf)); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 7167fd4f8c639b..2be2986d2110a2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -54,6 +54,8 @@ struct brcmf_mp_device { const char *board_type; unsigned char mac[ETH_ALEN]; const char *antenna_sku; + const void *cal_blob; + int cal_size; union { struct brcmfmac_sdio_pd sdio; } bus; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c index fdd0c9abc1a10d..52527b61341edd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c @@ -86,6 +86,13 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, if (!of_property_read_string(np, "apple,antenna-sku", &prop)) settings->antenna_sku = prop; + /* The WLAN calibration blob is normally stored in SROM, but Apple + * ARM64 platforms pass it via the DT instead. + */ + prop = of_get_property(np, "brcm,cal-blob", &settings->cal_size); + if (prop && settings->cal_size) + settings->cal_blob = prop; + /* Set board-type to the first string of the machine compatible prop */ root = of_find_node_by_path("/"); if (root && err) { From 7c83b5e236b2d2bd45459b8a1d1f7030987b7dac Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 1 Jul 2022 23:51:23 +0900 Subject: [PATCH 111/635] brcmfmac: pcie: Add BCM4378B3 support BCM4378B3 is a new silicon revision of BCM4378 present on the Apple M2 13" MacBook Pro "kyushu". Its PCI revision number is 5. Signed-off-by: Hector Martin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 6834b5fcfbb2b7..99d3adebd6e784 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -67,6 +67,7 @@ BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie"); BRCMF_FW_DEF(4371, "brcmfmac4371-pcie"); BRCMF_FW_CLM_DEF(4377B3, "brcmfmac4377b3-pcie"); BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie"); +BRCMF_FW_CLM_DEF(4378B3, "brcmfmac4378b3-pcie"); BRCMF_FW_CLM_DEF(4387C2, "brcmfmac4387c2-pcie"); /* firmware config files */ @@ -102,7 +103,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43666_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), BRCMF_FW_ENTRY(BRCM_CC_4377_CHIP_ID, 0xFFFFFFFF, 4377B3), /* revision ID 4 */ - BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFFF, 4378B1), /* revision ID 3 */ + BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0x0000000F, 4378B1), /* revision ID 3 */ + BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFE0, 4378B3), /* revision ID 5 */ BRCMF_FW_ENTRY(BRCM_CC_4387_CHIP_ID, 0xFFFFFFFF, 4387C2), /* revision ID 7 */ }; From 2e095bb994fb1e037d285b6799334cd986b6346c Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 21:59:24 +0900 Subject: [PATCH 112/635] wifi: brcmfmac: Add missing shared area defines to pcie.c There are many newer flags and extended shared area fields used by newer firmwares that are not yet defined. Add them for future usage. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 99d3adebd6e784..28766a5fe7cd95 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -213,11 +213,64 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 +#define BRCMF_PCIE_SHARED_USE_MAILBOX 0x2000000 +#define BRCMF_PCIE_SHARED_TIMESTAMP_DB0 0x8000000 #define BRCMF_PCIE_SHARED_HOSTRDY_DB1 0x10000000 +#define BRCMF_PCIE_SHARED_NO_OOB_DW 0x20000000 +#define BRCMF_PCIE_SHARED_INBAND_DS 0x40000000 +#define BRCMF_PCIE_SHARED_DAR 0x80000000 + +#define BRCMF_PCIE_SHARED2_EXTENDED_TRAP_DATA 0x1 +#define BRCMF_PCIE_SHARED2_TXSTATUS_METADATA 0x2 +#define BRCMF_PCIE_SHARED2_BT_LOGGING 0x4 +#define BRCMF_PCIE_SHARED2_SNAPSHOT_UPLOAD 0x8 +#define BRCMF_PCIE_SHARED2_SUBMIT_COUNT_WAR 0x10 +#define BRCMF_PCIE_SHARED2_FAST_DELETE_RING 0x20 +#define BRCMF_PCIE_SHARED2_EVTBUF_MAX_MASK 0xC0 +#define BRCMF_PCIE_SHARED2_PKT_TX_STATUS 0x100 +#define BRCMF_PCIE_SHARED2_FW_SMALL_MEMDUMP 0x200 +#define BRCMF_PCIE_SHARED2_FW_HC_ON_TRAP 0x400 +#define BRCMF_PCIE_SHARED2_HSCB 0x800 +#define BRCMF_PCIE_SHARED2_EDL_RING 0x1000 +#define BRCMF_PCIE_SHARED2_DEBUG_BUF_DEST 0x2000 +#define BRCMF_PCIE_SHARED2_PCIE_ENUM_RESET_FLR 0x4000 +#define BRCMF_PCIE_SHARED2_PKT_TIMESTAMP 0x8000 +#define BRCMF_PCIE_SHARED2_HP2P 0x10000 +#define BRCMF_PCIE_SHARED2_HWA 0x20000 +#define BRCMF_PCIE_SHARED2_TRAP_ON_HOST_DB7 0x40000 +#define BRCMF_PCIE_SHARED2_DURATION_SCALE 0x100000 +#define BRCMF_PCIE_SHARED2_D2H_D11_TX_STATUS 0x40000000 +#define BRCMF_PCIE_SHARED2_H2D_D11_TX_STATUS 0x80000000 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT 0x4000 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT 0x8000 +#define BRCMF_HOSTCAP_PCIEAPI_VERSION_MASK 0x000000FF +#define BRCMF_HOSTCAP_H2D_VALID_PHASE 0x00000100 +#define BRCMF_HOSTCAP_H2D_ENABLE_TRAP_ON_BADPHASE 0x00000200 +#define BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY 0x400 +#define BRCMF_HOSTCAP_DB0_TIMESTAMP 0x800 +#define BRCMF_HOSTCAP_DS_NO_OOB_DW 0x1000 +#define BRCMF_HOSTCAP_DS_INBAND_DW 0x2000 +#define BRCMF_HOSTCAP_H2D_IDMA 0x4000 +#define BRCMF_HOSTCAP_H2D_IFRM 0x8000 +#define BRCMF_HOSTCAP_H2D_DAR 0x10000 +#define BRCMF_HOSTCAP_EXTENDED_TRAP_DATA 0x20000 +#define BRCMF_HOSTCAP_TXSTATUS_METADATA 0x40000 +#define BRCMF_HOSTCAP_BT_LOGGING 0x80000 +#define BRCMF_HOSTCAP_SNAPSHOT_UPLOAD 0x100000 +#define BRCMF_HOSTCAP_FAST_DELETE_RING 0x200000 +#define BRCMF_HOSTCAP_PKT_TXSTATUS 0x400000 +#define BRCMF_HOSTCAP_UR_FW_NO_TRAP 0x800000 +#define BRCMF_HOSTCAP_HSCB 0x2000000 +#define BRCMF_HOSTCAP_EXT_TRAP_DBGBUF 0x4000000 +#define BRCMF_HOSTCAP_EDL_RING 0x10000000 +#define BRCMF_HOSTCAP_PKT_TIMESTAMP 0x20000000 +#define BRCMF_HOSTCAP_PKT_HP2P 0x40000000 +#define BRCMF_HOSTCAP_HWA 0x80000000 +#define BRCMF_HOSTCAP2_DURATION_SCALE_MASK 0x3F + +#define BRCMF_SHARED_FLAGS_OFFSET 0 #define BRCMF_SHARED_MAX_RXBUFPOST_OFFSET 34 #define BRCMF_SHARED_RING_BASE_OFFSET 52 #define BRCMF_SHARED_RX_DATAOFFSET_OFFSET 36 @@ -229,6 +282,11 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET 56 #define BRCMF_SHARED_DMA_RINGUPD_LEN_OFFSET 64 #define BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET 68 +#define BRCMF_SHARED_FLAGS2_OFFSET 80 +#define BRCMF_SHARED_HOST_CAP_OFFSET 84 +#define BRCMF_SHARED_FLAGS3_OFFSET 108 +#define BRCMF_SHARED_HOST_CAP2_OFFSET 112 +#define BRCMF_SHARED_HOST_CAP3_OFFSET 116 #define BRCMF_RING_H2D_RING_COUNT_OFFSET 0 #define BRCMF_RING_D2H_RING_COUNT_OFFSET 1 From 4daa9f1369780f539722028409a0a23703017f72 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 22:02:10 +0900 Subject: [PATCH 113/635] wifi: brcmfmac: Handle PCIe MSI properly On newer firmwares under at least certain conditions, MSI mode does not leave interrupt flags set (they are cleared by the firmware). Handle this by always checking for ring data when we get an MSI, regardless of whether any IRQ flags were set. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 28766a5fe7cd95..a724d3a94cd3b2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -404,6 +404,7 @@ struct brcmf_pciedev_info { wait_queue_head_t mbdata_resp_wait; bool mbdata_completed; bool irq_allocated; + bool have_msi; bool wowl_enabled; u8 dma_idx_sz; void *idxbuf; @@ -988,6 +989,11 @@ static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) brcmf_dbg(PCIE, "Enter\n"); return IRQ_WAKE_THREAD; } + + /* mailboxint is cleared by the firmware in MSI mode */ + if (devinfo->have_msi) + return IRQ_WAKE_THREAD; + return IRQ_NONE; } @@ -1005,12 +1011,12 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) status); if (status & devinfo->reginfo->int_fn0) brcmf_pcie_handle_mb_data(devinfo); - if (status & devinfo->reginfo->int_d2h_db) { - if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) - brcmf_proto_msgbuf_rx_trigger( - &devinfo->pdev->dev); - } } + if (devinfo->have_msi || status & devinfo->reginfo->int_d2h_db) { + if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) + brcmf_proto_msgbuf_rx_trigger(&devinfo->pdev->dev); + } + brcmf_pcie_bus_console_read(devinfo, false); if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) brcmf_pcie_intr_enable(devinfo); @@ -1028,7 +1034,10 @@ static int brcmf_pcie_request_irq(struct brcmf_pciedev_info *devinfo) brcmf_dbg(PCIE, "Enter\n"); - pci_enable_msi(pdev); + devinfo->have_msi = pci_enable_msi(pdev) >= 0; + if (devinfo->have_msi) + brcmf_dbg(PCIE, "MSI enabled\n"); + if (request_threaded_irq(pdev->irq, brcmf_pcie_quick_check_isr, brcmf_pcie_isr_thread, IRQF_SHARED, "brcmf_pcie_intr", devinfo)) { From 500f017283fb42a1e5aa6a4eedb6ce8e79d50918 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 22:06:40 +0900 Subject: [PATCH 114/635] wifi: brcmfmac: Fix logic for deciding which doorbell registers to use While the other >PCIe r64 registers (which are apparently called DAR registers) are always used on newer revisions, which doorbell registers should be used depends only on flags set by firmware. Take them out of the reginfo struct and check the flag to decide instead. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index a724d3a94cd3b2..c181b96c3493c6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -490,8 +490,6 @@ struct brcmf_pcie_reginfo { u32 intmask; u32 mailboxint; u32 mailboxmask; - u32 h2d_mailbox_0; - u32 h2d_mailbox_1; u32 int_d2h_db; u32 int_fn0; }; @@ -500,8 +498,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_default = { .intmask = BRCMF_PCIE_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_MB_INT_D2H_DB, .int_fn0 = BRCMF_PCIE_MB_INT_FN0, }; @@ -510,8 +506,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_64 = { .intmask = BRCMF_PCIE_64_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_64_MB_INT_D2H_DB, .int_fn0 = 0, }; @@ -975,9 +969,12 @@ static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) { - if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) - brcmf_pcie_write_reg32(devinfo, - devinfo->reginfo->h2d_mailbox_1, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) { + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, 1); + else + brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1); + } } static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) @@ -1126,7 +1123,10 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx) brcmf_dbg(PCIE, "RING !\n"); /* Any arbitrary value will do, lets use 1 */ - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->h2d_mailbox_0, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, 1); + else + brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1); return 0; } From 26b705b1fa341a29a57d02bb1a1b235915341221 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 22:10:08 +0900 Subject: [PATCH 115/635] wifi: brcmfmac: Support v6+ flags and set host_cap properly Interface versions 6 and above support having the host tell the dongle about what it supports via a host_cap field (it seems that if it is set to zero, some kind of unknown defaults are used). Explicitly support and set this. This also disables OOB deep sleep support; it doesn't look like deep sleep is properly supported yet at all (it needs more logic than merely acking requests, which is all pcie.c does right now). Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index c181b96c3493c6..db30e2053558f3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -351,6 +351,8 @@ struct brcmf_pcie_console { struct brcmf_pcie_shared_info { u32 tcm_base_address; u32 flags; + u32 flags2; + u32 flags3; struct brcmf_pcie_ringbuf *commonrings[BRCMF_NROF_COMMON_MSGRINGS]; struct brcmf_pcie_ringbuf *flowrings; u16 max_rxbufpost; @@ -1683,12 +1685,16 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); struct brcmf_pcie_shared_info *shared; + u32 host_cap; + u32 host_cap2; u32 addr; shared = &devinfo->shared; shared->tcm_base_address = sharedram_addr; - shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr); + shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS_OFFSET); + shared->version = (u8)(shared->flags & BRCMF_PCIE_SHARED_VERSION_MASK); brcmf_dbg(PCIE, "PCIe protocol version %d\n", shared->version); if ((shared->version > BRCMF_PCIE_MAX_SHARED_VERSION) || @@ -1729,6 +1735,33 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, brcmf_pcie_bus_console_init(devinfo); brcmf_pcie_bus_console_read(devinfo, false); + /* Features added in revision 6 follow */ + if (shared->version < 6) + return 0; + + shared->flags2 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS2_OFFSET); + shared->flags3 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS3_OFFSET); + + /* Update host support flags */ + host_cap = shared->version; + host_cap2 = 0; + + if (shared->flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) + host_cap |= BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY; + + if (shared->flags & BRCMF_PCIE_SHARED_DAR) + host_cap |= BRCMF_HOSTCAP_H2D_DAR; + + /* Disable DS: this is not currently properly supported */ + host_cap |= BRCMF_HOSTCAP_DS_NO_OOB_DW; + + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP_OFFSET, host_cap); + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP2_OFFSET, host_cap2); + return 0; } From 78037d4844078887ddb28861b2b54734e4ea0520 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 21:55:52 +0900 Subject: [PATCH 116/635] wifi: brcmfmac: Add newer msgbuf packet types up to 0x2e There are many newer msgbuf packet types that are not yet listed in the defines in msgbuf.c. Add them for future use. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 45fbcbdc7d9e4b..4405451b0c59a4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -47,6 +47,32 @@ #define MSGBUF_TYPE_RX_CMPLT 0x12 #define MSGBUF_TYPE_LPBK_DMAXFER 0x13 #define MSGBUF_TYPE_LPBK_DMAXFER_CMPLT 0x14 +#define MSGBUF_TYPE_FLOW_RING_RESUME 0x15 +#define MSGBUF_TYPE_FLOW_RING_RESUME_CMPLT 0x16 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND 0x17 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND_CMPLT 0x18 +#define MSGBUF_TYPE_INFO_BUF_POST 0x19 +#define MSGBUF_TYPE_INFO_BUF_CMPLT 0x1A +#define MSGBUF_TYPE_H2D_RING_CREATE 0x1B +#define MSGBUF_TYPE_D2H_RING_CREATE 0x1C +#define MSGBUF_TYPE_H2D_RING_CREATE_CMPLT 0x1D +#define MSGBUF_TYPE_D2H_RING_CREATE_CMPLT 0x1E +#define MSGBUF_TYPE_H2D_RING_CONFIG 0x1F +#define MSGBUF_TYPE_D2H_RING_CONFIG 0x20 +#define MSGBUF_TYPE_H2D_RING_CONFIG_CMPLT 0x21 +#define MSGBUF_TYPE_D2H_RING_CONFIG_CMPLT 0x22 +#define MSGBUF_TYPE_H2D_MAILBOX_DATA 0x23 +#define MSGBUF_TYPE_D2H_MAILBOX_DATA 0x24 +#define MSGBUF_TYPE_TIMSTAMP_BUFPOST 0x25 +#define MSGBUF_TYPE_HOSTTIMSTAMP 0x26 +#define MSGBUF_TYPE_HOSTTIMSTAMP_CMPLT 0x27 +#define MSGBUF_TYPE_FIRMWARE_TIMESTAMP 0x28 +#define MSGBUF_TYPE_SNAPSHOT_UPLOAD 0x29 +#define MSGBUF_TYPE_SNAPSHOT_CMPLT 0x2A +#define MSGBUF_TYPE_H2D_RING_DELETE 0x2B +#define MSGBUF_TYPE_D2H_RING_DELETE 0x2C +#define MSGBUF_TYPE_H2D_RING_DELETE_CMPLT 0x2D +#define MSGBUF_TYPE_D2H_RING_DELETE_CMPLT 0x2E #define NR_TX_PKTIDS 2048 #define NR_RX_PKTIDS 1024 From 9b704da9cba34cc000e7d95a5ca07c7730f1cb61 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 21:56:53 +0900 Subject: [PATCH 117/635] wifi: brcmfmac: Add a new bus op for D2H mailbox message handling Newer firmware versions use the common ring for sending mailbox messages between the dongle and host, instead of the hardware mailboxes. This needs the protocol driver to call back into the bus driver, so add a callback for this to bus.h. Signed-off-by: Hector Martin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index fe31051a9e11b1..5efd7f6d757a4c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -107,6 +107,7 @@ struct brcmf_bus_ops { void (*debugfs_create)(struct device *dev); int (*reset)(struct device *dev); void (*remove)(struct device *dev); + void (*d2h_mb_rx)(struct device *dev, u32 data); }; @@ -286,6 +287,15 @@ static inline void brcmf_bus_remove(struct brcmf_bus *bus) bus->ops->remove(bus->dev); } +static inline +void brcmf_bus_d2h_mb_rx(struct brcmf_bus *bus, u32 data) +{ + if (!bus->ops->d2h_mb_rx) + return; + + return bus->ops->d2h_mb_rx(bus->dev, data); +} + /* * interface functions from common layer */ From ab927fb01555a26c24e48604dab68b4698007e8c Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 21:58:21 +0900 Subject: [PATCH 118/635] wifi: brcmfmac: Implement the H2D/D2H mailbox data commonring messages Newer firmware versions use these to exchange mailbox data, instead of the hardware mailbox registers. Add handling for them to msgbuf.c. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 59 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/msgbuf.h | 1 + 2 files changed, 60 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 4405451b0c59a4..93206850373300 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -244,6 +244,19 @@ struct msgbuf_flowring_flush_resp { __le32 rsvd0[3]; }; +struct msgbuf_h2d_mailbox_data { + struct msgbuf_common_hdr msg; + __le32 data; + __le32 rsvd0[7]; +}; + +struct msgbuf_d2h_mailbox_data { + struct msgbuf_common_hdr msg; + struct msgbuf_completion_hdr compl_hdr; + __le32 data; + __le32 rsvd0[2]; +}; + struct brcmf_msgbuf_work_item { struct list_head queue; u32 flowid; @@ -1311,6 +1324,16 @@ brcmf_msgbuf_process_flow_ring_delete_response(struct brcmf_msgbuf *msgbuf, } +static void brcmf_msgbuf_process_d2h_mailbox_data(struct brcmf_msgbuf *msgbuf, + void *buf) +{ + struct msgbuf_d2h_mailbox_data *d2h_mb_data = buf; + struct brcmf_pub *drvr = msgbuf->drvr; + + brcmf_bus_d2h_mb_rx(drvr->bus_if, le32_to_cpu(d2h_mb_data->data)); +} + + static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_pub *drvr = msgbuf->drvr; @@ -1353,6 +1376,10 @@ static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) brcmf_dbg(MSGBUF, "MSGBUF_TYPE_RX_CMPLT\n"); brcmf_msgbuf_process_rx_complete(msgbuf, buf); break; + case MSGBUF_TYPE_D2H_MAILBOX_DATA: + brcmf_dbg(MSGBUF, "MSGBUF_TYPE_D2H_MAILBOX_DATA\n"); + brcmf_msgbuf_process_d2h_mailbox_data(msgbuf, buf); + break; default: bphy_err(drvr, "Unsupported msgtype %d\n", msg->msgtype); break; @@ -1491,6 +1518,38 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) } } + +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data) +{ + struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; + struct brcmf_commonring *commonring; + struct msgbuf_h2d_mailbox_data *request; + void *ret_ptr; + int err; + + commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT]; + brcmf_commonring_lock(commonring); + ret_ptr = brcmf_commonring_reserve_for_write(commonring); + if (!ret_ptr) { + bphy_err(drvr, "Failed to reserve space in commonring\n"); + brcmf_commonring_unlock(commonring); + return -ENOMEM; + } + + request = (struct msgbuf_h2d_mailbox_data *)ret_ptr; + request->msg.msgtype = MSGBUF_TYPE_H2D_MAILBOX_DATA; + request->msg.ifidx = -1; + request->msg.flags = 0; + request->msg.request_id = 0; + request->data = data; + + err = brcmf_commonring_write_complete(commonring); + brcmf_commonring_unlock(commonring); + + return err; +} + + #ifdef DEBUG static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index 6a849f4a94dd7f..89b6b7f9ddb748 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -32,6 +32,7 @@ int brcmf_proto_msgbuf_rx_trigger(struct device *dev); void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid); int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr); void brcmf_proto_msgbuf_detach(struct brcmf_pub *drvr); +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data); #else static inline int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) { From 5768c99742fac3ad67a5ba116b6eb0a0625e492a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Oct 2022 22:12:15 +0900 Subject: [PATCH 119/635] wifi: brcmfmac: Support exchanging power mailbox messages via commonring Newer firmwares have switched from using the hardware mailbox to commonring messages for power mailbox data. Implement this, which makes D3 work on WiFi chipsets in Apple devices. This is only enabled on v6 or newer, iff BRCMF_PCIE_SHARED_USE_MAILBOX is not set in the flags. Signed-off-by: Hector Martin --- .../broadcom/brcm80211/brcmfmac/pcie.c | 75 ++++++++++++++----- 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index db30e2053558f3..1dca969c608dea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -369,6 +369,7 @@ struct brcmf_pcie_shared_info { void *ringupd; dma_addr_t ringupd_dmahandle; u8 version; + bool mb_via_ctl; }; struct brcmf_pcie_core_info { @@ -820,6 +821,19 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) u32 i; shared = &devinfo->shared; + + if (shared->mb_via_ctl) { + struct pci_dev *pdev = devinfo->pdev; + struct brcmf_bus *bus = dev_get_drvdata(&pdev->dev); + int ret; + + ret = brcmf_msgbuf_h2d_mb_write(bus->drvr, htod_mb_data); + if (ret < 0) + brcmf_err(bus, "Failed to send H2D mailbox data (%d)\n", + ret); + return ret; + } + addr = shared->htod_mb_data_addr; cur_htod_mb_data = brcmf_pcie_read_tcm32(devinfo, addr); @@ -847,8 +861,29 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) return 0; } +static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo, u32 data) +{ + brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", data); + if (data & BRCMF_D2H_DEV_DS_ENTER_REQ) { + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); + brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); + brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); + } + if (data & BRCMF_D2H_DEV_DS_EXIT_NOTE) + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); + if (data & BRCMF_D2H_DEV_D3_ACK) { + brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); + devinfo->mbdata_completed = true; + wake_up(&devinfo->mbdata_resp_wait); + } + if (data & BRCMF_D2H_DEV_FWHALT) { + brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); + brcmf_fw_crashed(&devinfo->pdev->dev); + } +} + -static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) +static void brcmf_pcie_poll_mb_data(struct brcmf_pciedev_info *devinfo) { struct brcmf_pcie_shared_info *shared; u32 addr; @@ -863,23 +898,16 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_tcm32(devinfo, addr, 0); - brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", dtoh_mb_data); - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_ENTER_REQ) { - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); - brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); - brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_EXIT_NOTE) - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); - if (dtoh_mb_data & BRCMF_D2H_DEV_D3_ACK) { - brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); - devinfo->mbdata_completed = true; - wake_up(&devinfo->mbdata_resp_wait); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { - brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); - brcmf_fw_crashed(&devinfo->pdev->dev); - } + brcmf_pcie_handle_mb_data(devinfo, dtoh_mb_data); +} + + +static void brcmf_pcie_d2h_mb_rx(struct device *dev, u32 data) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + + brcmf_pcie_handle_mb_data(buspub->devinfo, data); } @@ -1009,7 +1037,7 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status); if (status & devinfo->reginfo->int_fn0) - brcmf_pcie_handle_mb_data(devinfo); + brcmf_pcie_poll_mb_data(devinfo); } if (devinfo->have_msi || status & devinfo->reginfo->int_d2h_db) { if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) @@ -1654,6 +1682,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .get_blob = brcmf_pcie_get_blob, .reset = brcmf_pcie_reset, .debugfs_create = brcmf_pcie_debugfs_create, + .d2h_mb_rx = brcmf_pcie_d2h_mb_rx, }; @@ -1744,6 +1773,10 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, shared->flags3 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + BRCMF_SHARED_FLAGS3_OFFSET); + /* Check which mailbox mechanism to use */ + if (!(shared->flags & BRCMF_PCIE_SHARED_USE_MAILBOX)) + shared->mb_via_ctl = true; + /* Update host support flags */ host_cap = shared->version; host_cap2 = 0; @@ -2702,10 +2735,11 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) /* Check if device is still up and running, if so we are ready */ if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { brcmf_dbg(PCIE, "Try to wakeup device....\n"); + /* Set the device up, so we can write the MB data message in ring mode */ + devinfo->state = BRCMFMAC_PCIE_STATE_UP; if (brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_D0_INFORM)) goto cleanup; brcmf_dbg(PCIE, "Hot resume, continue....\n"); - devinfo->state = BRCMFMAC_PCIE_STATE_UP; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); brcmf_bus_change_state(bus, BRCMF_BUS_UP); brcmf_pcie_intr_enable(devinfo); @@ -2715,6 +2749,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) } cleanup: + devinfo->state = BRCMFMAC_PCIE_STATE_DOWN; brcmf_chip_detach(devinfo->ci); devinfo->ci = NULL; pdev = devinfo->pdev; From d46831b4418829fa43e91619947cbed3371048c6 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 25 Mar 2023 15:23:04 +0900 Subject: [PATCH 120/635] brcmfmac: Shut up p2p unknown frame error People keep complaining about this and think their wifi is broken for some reason... Signed-off-by: Hector Martin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index d4492d02e4ea12..071b0706d1372c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1793,8 +1793,8 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, /* do not configure anything. it will be */ /* sent with a default configuration */ } else { - bphy_err(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", - category, action); + bphy_info_once(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", + category, action); return false; } From 8a96dfb06cc870730ecf78195187daec961f3e2c Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 16 Apr 2023 21:29:03 +0900 Subject: [PATCH 121/635] wifi: brcmfmac: Demote vendor-specific attach/detach messages to info People are getting spooked by brcmfmac errors on their boot console. There's no reason for these messages to be errors. Cc: stable@vger.kernel.org Fixes: d6a5c562214f ("wifi: brcmfmac: add support for vendor-specific firmware api") Signed-off-by: Hector Martin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/core.c | 4 ++-- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c | 4 ++-- drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/core.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/core.c index ac3a36fa3640ce..c83bc435b25737 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bca/core.c @@ -12,13 +12,13 @@ static int brcmf_bca_attach(struct brcmf_pub *drvr) { - pr_err("%s: executing\n", __func__); + pr_info("%s: executing\n", __func__); return 0; } static void brcmf_bca_detach(struct brcmf_pub *drvr) { - pr_err("%s: executing\n", __func__); + pr_info("%s: executing\n", __func__); } const struct brcmf_fwvid_ops brcmf_bca_ops = { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c index b75652ba9359f2..e39d66b07831c4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c @@ -12,13 +12,13 @@ static int brcmf_cyw_attach(struct brcmf_pub *drvr) { - pr_err("%s: executing\n", __func__); + pr_info("%s: executing\n", __func__); return 0; } static void brcmf_cyw_detach(struct brcmf_pub *drvr) { - pr_err("%s: executing\n", __func__); + pr_info("%s: executing\n", __func__); } const struct brcmf_fwvid_ops brcmf_cyw_ops = { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/core.c index 02de99818efa70..133d274b40255b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/wcc/core.c @@ -12,13 +12,13 @@ static int brcmf_wcc_attach(struct brcmf_pub *drvr) { - pr_err("%s: executing\n", __func__); + pr_info("%s: executing\n", __func__); return 0; } static void brcmf_wcc_detach(struct brcmf_pub *drvr) { - pr_err("%s: executing\n", __func__); + pr_info("%s: executing\n", __func__); } const struct brcmf_fwvid_ops brcmf_wcc_ops = { From 37a915312e9207e36d490c2981a04740aebd8f93 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 12 Dec 2021 20:40:04 +0100 Subject: [PATCH 122/635] HID: add device IDs for Apple SPI HID devices Apple Silicon based laptop use SPI as transport for HID. Add support for SPI-based HID devices and and Apple keyboard and trackpad devices. Intel based laptops using the keyboard input driver applespi use the same HID over SPI protocol and can be supported later. This requires SPI keyboard/mouse HID types since Apple's intenal keyboards/trackpads use the same product id. Signed-off-by: Janne Grunau --- drivers/hid/hid-core.c | 3 +++ drivers/hid/hid-ids.h | 5 +++++ include/linux/hid.h | 6 +++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 22623eb4f72f29..4db0a301df5d8d 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -2242,6 +2242,9 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_I2C: bus = "I2C"; break; + case BUS_SPI: + bus = "SPI"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index c2e9b6d1fd7d3e..b8ac02219885b7 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -89,6 +89,7 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -187,6 +188,10 @@ #define USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021 0x029f #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 #define USB_VENDOR_ID_ASUS 0x0486 #define USB_DEVICE_ID_ASUS_T91MT 0x0185 diff --git a/include/linux/hid.h b/include/linux/hid.h index 1ea8c7a3570b28..0b1bc61a989cd4 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -562,7 +562,9 @@ struct hid_input { enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -716,6 +718,8 @@ struct hid_descriptor { .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) From dd5ea76fe4554233d05e1c8a142b86789ea9884f Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 16 Dec 2021 21:15:31 +0100 Subject: [PATCH 123/635] HID: apple: add support for internal keyboards Apple MacBook keyboards started using HID over SPI in 2015. With the addition of the SPI HID transport they can be supported by this driver. Support all product ids over with the Apple SPI vendor id for now. Individual product ids will have to be added for a correct Fn/function key mapping. Enable by default on the Apple Arm platform. Signed-off-by: Janne Grunau --- drivers/hid/Kconfig | 4 ++-- drivers/hid/hid-apple.c | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 4ce012f83253ec..4d2055e2d3a62d 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -129,7 +129,7 @@ config HID_APPLE tristate "Apple {i,Power,Mac}Books" depends on LEDS_CLASS depends on NEW_LEDS - default !EXPERT + default !EXPERT || SPI_HID_APPLE help Support for some Apple devices which less or more break HID specification. @@ -1019,7 +1019,7 @@ config HID_SONY * Guitar Hero PS3 and PC guitar dongles config SONY_FF - bool "Sony PS2/3/4 accessories force feedback support" + bool "Sony PS2/3/4 accessories force feedback support" depends on HID_SONY select INPUT_FF_MEMLESS help diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 1ccab8aa326cd3..2e1e68eb897b95 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -815,6 +815,10 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; + if (id->bus == BUS_SPI && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -1056,6 +1060,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, { } }; From d31aa43ec1175e2de6f669b842cca12afd2f5358 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 19 Dec 2021 18:08:15 +0100 Subject: [PATCH 124/635] HID: apple: add Fn key mapping for Apple silicon MacBooks Signed-off-by: Janne Grunau --- drivers/hid/hid-apple.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 2e1e68eb897b95..6f763d9bcdc323 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -262,6 +262,28 @@ static const struct apple_key_translation apple_fn_keys[] = { { } }; +static const struct apple_key_translation apple_fn_keys_spi[] = { + { KEY_BACKSPACE, KEY_DELETE }, + { KEY_ENTER, KEY_INSERT }, + { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, + { KEY_F2, KEY_BRIGHTNESSUP, APPLE_FLAG_FKEY }, + { KEY_F3, KEY_SCALE, APPLE_FLAG_FKEY }, + { KEY_F4, KEY_SEARCH, APPLE_FLAG_FKEY }, + { KEY_F5, KEY_RECORD, APPLE_FLAG_FKEY }, + { KEY_F6, KEY_SLEEP, APPLE_FLAG_FKEY }, + { KEY_F7, KEY_PREVIOUSSONG, APPLE_FLAG_FKEY }, + { KEY_F8, KEY_PLAYPAUSE, APPLE_FLAG_FKEY }, + { KEY_F9, KEY_NEXTSONG, APPLE_FLAG_FKEY }, + { KEY_F10, KEY_MUTE, APPLE_FLAG_FKEY }, + { KEY_F11, KEY_VOLUMEDOWN, APPLE_FLAG_FKEY }, + { KEY_F12, KEY_VOLUMEUP, APPLE_FLAG_FKEY }, + { KEY_UP, KEY_PAGEUP }, + { KEY_DOWN, KEY_PAGEDOWN }, + { KEY_LEFT, KEY_HOME }, + { KEY_RIGHT, KEY_END }, + { } +}; + static const struct apple_key_translation powerbook_fn_keys[] = { { KEY_BACKSPACE, KEY_DELETE }, { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, @@ -463,6 +485,8 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else if (hid->product >= USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI && hid->product <= USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) table = macbookair_fn_keys; + else if (hid->vendor == SPI_VENDOR_ID_APPLE) + table = apple_fn_keys_spi; else if (hid->product < 0x21d || hid->product >= 0x300) table = powerbook_fn_keys; else @@ -642,6 +666,7 @@ static void apple_setup_input(struct input_dev *input) /* Enable all needed keys */ apple_setup_key_translation(input, apple_fn_keys); + apple_setup_key_translation(input, apple_fn_keys_spi); apple_setup_key_translation(input, powerbook_fn_keys); apple_setup_key_translation(input, powerbook_numlock_keys); apple_setup_key_translation(input, apple_iso_keyboard); From 7e1be656dd0f616533bd58951958140b5157c087 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 5 Jan 2022 23:27:34 +0100 Subject: [PATCH 125/635] HID: apple: add Fn key mapping for Macbook Pro with touchbar Signed-off-by: Janne Grunau --- drivers/hid/hid-apple.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index 6f763d9bcdc323..cbd9cb3c82a529 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -284,6 +284,28 @@ static const struct apple_key_translation apple_fn_keys_spi[] = { { } }; +static const struct apple_key_translation apple_fn_keys_mbp13[] = { + { KEY_BACKSPACE, KEY_DELETE }, + { KEY_ENTER, KEY_INSERT }, + { KEY_UP, KEY_PAGEUP }, + { KEY_DOWN, KEY_PAGEDOWN }, + { KEY_LEFT, KEY_HOME }, + { KEY_RIGHT, KEY_END }, + { KEY_1, KEY_F1 }, + { KEY_2, KEY_F2 }, + { KEY_3, KEY_F3 }, + { KEY_4, KEY_F4 }, + { KEY_5, KEY_F5 }, + { KEY_6, KEY_F6 }, + { KEY_7, KEY_F7 }, + { KEY_8, KEY_F8 }, + { KEY_9, KEY_F9 }, + { KEY_0, KEY_F10 }, + { KEY_MINUS, KEY_F11 }, + { KEY_EQUAL, KEY_F12 }, + { } +}; + static const struct apple_key_translation powerbook_fn_keys[] = { { KEY_BACKSPACE, KEY_DELETE }, { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, @@ -485,6 +507,9 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else if (hid->product >= USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI && hid->product <= USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) table = macbookair_fn_keys; + else if (hid->vendor == SPI_VENDOR_ID_APPLE && + hid->product == SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020) + table = apple_fn_keys_mbp13; else if (hid->vendor == SPI_VENDOR_ID_APPLE) table = apple_fn_keys_spi; else if (hid->product < 0x21d || hid->product >= 0x300) @@ -667,6 +692,7 @@ static void apple_setup_input(struct input_dev *input) /* Enable all needed keys */ apple_setup_key_translation(input, apple_fn_keys); apple_setup_key_translation(input, apple_fn_keys_spi); + apple_setup_key_translation(input, apple_fn_keys_mbp13); apple_setup_key_translation(input, powerbook_fn_keys); apple_setup_key_translation(input, powerbook_numlock_keys); apple_setup_key_translation(input, apple_iso_keyboard); From b55bf214d2f52669780ba23039c3d020b73a968c Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 16 Dec 2021 00:10:51 +0100 Subject: [PATCH 126/635] HID: magicmouse: use a define of the max number of touch contacts Signed-off-by: Janne Grunau --- drivers/hid/hid-magicmouse.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index c9c968d4b36a39..529efb317df08f 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -61,6 +61,8 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define DOUBLE_REPORT_ID 0xf7 #define USB_BATTERY_TIMEOUT_MS 60000 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -139,8 +141,8 @@ struct magicmouse_sc { u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; @@ -592,7 +594,7 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, From a707089a290c680ae050a3694556fb783ea2cf82 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 16 Dec 2021 00:12:35 +0100 Subject: [PATCH 127/635] HID: magicmouse: use struct input_mt_pos for X/Y Signed-off-by: Janne Grunau --- drivers/hid/hid-magicmouse.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 529efb317df08f..3eeded6764dcc5 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -131,9 +131,8 @@ struct magicmouse_sc { int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -190,7 +189,7 @@ static void magicmouse_emit_buttons(struct magicmouse_sc *msc, int state) } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -251,8 +250,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small From 4076acd2596646724a2444a2e4a78fc0715efacd Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 16 Dec 2021 00:15:30 +0100 Subject: [PATCH 128/635] HID: magicmouse: use ops function pointers for input functionality Will be used for supporting MacBook trackpads connected via SPI. Signed-off-by: Janne Grunau --- drivers/hid/hid-magicmouse.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 3eeded6764dcc5..052fb1d4808c13 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -113,6 +113,13 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. @@ -146,6 +153,7 @@ struct magicmouse_sc { struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; static int magicmouse_firm_touch(struct magicmouse_sc *msc) @@ -375,6 +383,14 @@ static int magicmouse_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; @@ -520,7 +536,17 @@ static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -807,6 +833,9 @@ static int magicmouse_probe(struct hid_device *hdev, return -ENOMEM; } + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); From 046f4388f3df3e1eb6065691ffe972fd14ab0622 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 16 Dec 2021 01:17:48 +0100 Subject: [PATCH 129/635] HID: magicmouse: add support for Macbook trackpads The trackpads in Macbooks beginning in 2015 are HID devices connected over SPI. On Intel Macbooks they are currently supported by applespi.c. This chang adds support for the trackpads on Apple Silicon Macbooks starting in late 2020. They use a new HID over SPI transport driver. The touch report format differs from USB/BT Magic Trackpads. It is the same format as the type 4 format supported by bcm5974.c. Signed-off-by: Janne Grunau --- drivers/hid/Kconfig | 4 +- drivers/hid/hid-magicmouse.c | 259 ++++++++++++++++++++++++++++++++++- 2 files changed, 260 insertions(+), 3 deletions(-) diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 4d2055e2d3a62d..aae98f41e45778 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -662,11 +662,13 @@ config LOGIWHEELS_FF config HID_MAGICMOUSE tristate "Apple Magic Mouse/Trackpad multi-touch support" + default SPI_HID_APPLE help Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 052fb1d4808c13..03e9e04d81dfaa 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -113,6 +113,18 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define J314_TP_MAX_FINGER_ORIENTATION 16384 struct magicmouse_input_ops { int (*raw_event)(struct hid_device *hdev, @@ -519,6 +531,157 @@ static int magicmouse_raw_event_usb(struct hid_device *hdev, return 1; } +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + * @crc16: on last finger: crc over the whole message struct + * (i.e. message header + this struct) minus the last + * @crc16 field; unknown on all other fingers. + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * struct trackpad report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + * @num_fingers: the number of fingers being reported in @fingers + * @clicked: same as @buttons + */ +struct tp_header { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 clicked; + u8 unknown3[14]; +}; + +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + if (data[0] != TRACKPAD2_USB_REPORT_ID) + return 0; + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, data[1] & 1); + + input_sync(input); + return 1; +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -698,6 +861,79 @@ static int magicmouse_setup_input_usb(struct input_dev *input, return 0; } +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int error; + int mt_flags = 0; + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -J314_TP_MAX_FINGER_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, J314_TP_MIN_X, J314_TP_MAX_X, + 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -J314_TP_MAX_Y, -J314_TP_MIN_Y, + 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, J314_TP_RES_X); + input_abs_set_res(input, ABS_MT_POSITION_Y, J314_TP_RES_Y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + return 0; +} + static int magicmouse_input_mapping(struct hid_device *hdev, struct hid_input *hi, struct hid_field *field, struct hid_usage *usage, unsigned long **bit, int *max) @@ -753,6 +989,9 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) feature_size = sizeof(feature_mt_trackpad2_usb); feature = feature_mt_trackpad2_usb; } + } else if (hdev->vendor == SPI_VENDOR_ID_APPLE) { + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; } else if (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { feature_size = sizeof(feature_mt_mouse2); feature = feature_mt_mouse2; @@ -827,14 +1066,26 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; + if (id->bus == BUS_SPI && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } - msc->input_ops.raw_event = magicmouse_raw_event_usb; - msc->input_ops.setup_input = magicmouse_setup_input_usb; + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + if (id->vendor == SPI_VENDOR_ID_APPLE) { + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + + } else { + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; @@ -884,6 +1135,8 @@ static int magicmouse_probe(struct hid_device *hdev, else /* USB_VENDOR_ID_APPLE */ report = hid_register_report(hdev, HID_INPUT_REPORT, TRACKPAD2_USB_REPORT_ID, 0); + } else if (id->vendor == SPI_VENDOR_ID_APPLE) { + report = hid_register_report(hdev, HID_INPUT_REPORT, 2, 0); } else { /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ report = hid_register_report(hdev, HID_INPUT_REPORT, TRACKPAD_REPORT_ID, 0); @@ -978,6 +1231,8 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); From 63e31154c04f828c861a6b148f323d56b7acc0f9 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 10 Dec 2021 19:38:43 +0100 Subject: [PATCH 130/635] WIP: HID: transport: spi: add Apple SPI transport Keyboard and trackpad of Apple Sillicon SoCs (M1, M1 Pro/Max) laptops are are HID devices connected via SPI. This is the same protocol as implemented by applespi.c. It was not noticed that protocol is a transport for HID. Adding support for ACPI based Intel MacBooks will be done in a separate commit. How HID is mapped in this protocol is not yet fully understood. Microsoft has a specification for HID over SPI [1] incompatible with the transport protocol used by Apple. [1] https://docs.microsoft.com/en-us/windows-hardware/drivers/hid/hid-over-spi Signed-off-by: Janne Grunau --- drivers/hid/Kconfig | 2 + drivers/hid/Makefile | 2 + drivers/hid/spi-hid/Kconfig | 26 + drivers/hid/spi-hid/Makefile | 10 + drivers/hid/spi-hid/spi-hid-apple-core.c | 1030 ++++++++++++++++++++++ drivers/hid/spi-hid/spi-hid-apple-of.c | 136 +++ drivers/hid/spi-hid/spi-hid-apple.h | 31 + 7 files changed, 1237 insertions(+) create mode 100644 drivers/hid/spi-hid/Kconfig create mode 100644 drivers/hid/spi-hid/Makefile create mode 100644 drivers/hid/spi-hid/spi-hid-apple-core.c create mode 100644 drivers/hid/spi-hid/spi-hid-apple-of.c create mode 100644 drivers/hid/spi-hid/spi-hid-apple.h diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index aae98f41e45778..b611db62e75897 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -1318,4 +1318,6 @@ source "drivers/hid/amd-sfh-hid/Kconfig" source "drivers/hid/surface-hid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 5d37cacbde33df..6214a3631b5119 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -167,3 +167,5 @@ obj-$(INTEL_ISH_FIRMWARE_DOWNLOADER) += intel-ish-hid/ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ diff --git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig new file mode 100644 index 00000000000000..8e37f0fec28ac9 --- /dev/null +++ b/drivers/hid/spi-hid/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + default ARCH_APPLE + depends on SPI && INPUT && OF + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + default y if SPI_HID_APPLE_OF=y + default m if SPI_HID_APPLE_OF=m + select HID + select CRC16 diff --git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile new file mode 100644 index 00000000000000..f276ee12cb94fc --- /dev/null +++ b/drivers/hid/spi-hid/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c new file mode 100644 index 00000000000000..cd018df4f38715 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -0,0 +1,1030 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @msgid: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + return -EINVAL; // spihid_get_raw_report(); + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + + if (hdr->length == 0) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, le16_to_cpu(hdr->length)); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, hdr, payload, + le16_to_cpu(hdr->length)); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + strscpy(hid->name, spihid->product, sizeof(hid->name)); + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* allocate SPI buffers */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c new file mode 100644 index 00000000000000..f1380bfc52672e --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + //.pm = &spi_hid_apple_of_pm, + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h new file mode 100644 index 00000000000000..2d9554e8a5f819 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +#endif /* SPI_HID_APPLE_H */ From f6146b2273828d1ad7d0f529ed22159219a83a67 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 8 Jul 2022 00:29:43 +0900 Subject: [PATCH 131/635] HID: add HOST vendor/device IDs for Apple MTP devices Apple M2 chips have an embedded MTP processor that handles all HID functions, and does not go over a traditional bus like SPI. The devices still have real IDs, so add them here. Signed-off-by: Hector Martin --- drivers/hid/hid-ids.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index b8ac02219885b7..d89c19d852372d 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -90,6 +90,7 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c #define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -192,6 +193,8 @@ #define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 #define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 #define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASUS 0x0486 #define USB_DEVICE_ID_ASUS_T91MT 0x0185 From d35a4663ed80b72fa69e52e1a783099afbecbfa0 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 8 Jul 2022 02:06:15 +0900 Subject: [PATCH 132/635] HID: core: Handle HOST bus type when announcing devices Signed-off-by: Hector Martin --- drivers/hid/hid-core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 4db0a301df5d8d..368bb290443fbe 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -2245,6 +2245,9 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_SPI: bus = "SPI"; break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; From da8fb0b0e9fb2808ca823c9a2ff9538971d5e863 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 8 Jul 2022 02:12:24 +0900 Subject: [PATCH 133/635] hid: apple: Bind to HOST devices for MTP We use BUS_HOST for MTP HID subdevices Signed-off-by: Hector Martin --- drivers/hid/hid-apple.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index cbd9cb3c82a529..a97367efa923b5 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -507,11 +507,16 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else if (hid->product >= USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI && hid->product <= USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) table = macbookair_fn_keys; - else if (hid->vendor == SPI_VENDOR_ID_APPLE && - hid->product == SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020) - table = apple_fn_keys_mbp13; - else if (hid->vendor == SPI_VENDOR_ID_APPLE) - table = apple_fn_keys_spi; + else if (hid->bus == BUS_HOST || hid->bus == BUS_SPI) + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = apple_fn_keys_mbp13; + break; + default: + table = apple_fn_keys_spi; + break; + } else if (hid->product < 0x21d || hid->product >= 0x300) table = powerbook_fn_keys; else @@ -866,7 +871,7 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; - if (id->bus == BUS_SPI && id->vendor == SPI_VENDOR_ID_APPLE && + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && hdev->type != HID_TYPE_SPI_KEYBOARD) return -ENODEV; @@ -1113,6 +1118,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, { } }; From 3e6ee2f89a7c41d443f9aa2b7794f95f04c435a5 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 8 Jul 2022 02:12:57 +0900 Subject: [PATCH 134/635] hid: magicmouse: Add MTP multi-touch device support Apple M2 devices expose the multi-touch device over the HID over DockChannel transport, which we represent as the HOST bus type. The report format is the same, except the legacy mouse header is gone and there is no enable request needed. Signed-off-by: Hector Martin --- drivers/hid/hid-magicmouse.c | 67 ++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 03e9e04d81dfaa..b22ab0f320c05d 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -59,6 +59,8 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define MTP_REPORT_ID 0x75 #define USB_BATTERY_TIMEOUT_MS 60000 #define MAX_CONTACTS 16 @@ -570,25 +572,32 @@ struct tp_finger { } __attribute__((packed, aligned(2))); /** - * struct trackpad report + * vendor trackpad report * - * @report_id: reportid - * @buttons: HID Usage Buttons 3 1-bit reports * @num_fingers: the number of fingers being reported in @fingers - * @clicked: same as @buttons + * @buttons: same as HID buttons */ struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { // HID mouse report u8 report_id; u8 buttons; u8 rel_x; u8 rel_y; u8 padding[4]; - // HID vendor part, up to 1751 bytes - u8 unknown[22]; - u8 num_fingers; - u8 clicked; - u8 unknown3[14]; }; static inline int le16_to_int(__le16 x) @@ -618,7 +627,7 @@ static void report_finger_data(struct input_dev *input, int slot, input_report_abs(input, ABS_MT_POSITION_Y, pos->y); } -static int magicmouse_raw_event_spi(struct hid_device *hdev, +static int magicmouse_raw_event_mtp(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); @@ -635,9 +644,6 @@ static int magicmouse_raw_event_spi(struct hid_device *hdev, // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, // size, false); - if (data[0] != TRACKPAD2_USB_REPORT_ID) - return 0; - /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) return 0; @@ -676,12 +682,26 @@ static int magicmouse_raw_event_spi(struct hid_device *hdev, } input_mt_sync_frame(input); - input_report_key(input, BTN_MOUSE, data[1] & 1); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); input_sync(input); return 1; } +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (size < hdr_sz) + return 0; + + if (data[0] != TRACKPAD2_USB_REPORT_ID) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -1066,7 +1086,7 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; - if (id->bus == BUS_SPI && id->vendor == SPI_VENDOR_ID_APPLE && + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && hdev->type != HID_TYPE_SPI_MOUSE) return -ENODEV; @@ -1078,7 +1098,10 @@ static int magicmouse_probe(struct hid_device *hdev, // internal trackpad use a data format use input ops to avoid // conflicts with the report ID. - if (id->vendor == SPI_VENDOR_ID_APPLE) { + if (id->bus == BUS_HOST) { + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + } else if (id->bus == BUS_SPI) { msc->input_ops.raw_event = magicmouse_raw_event_spi; msc->input_ops.setup_input = magicmouse_setup_input_spi; @@ -1135,8 +1158,10 @@ static int magicmouse_probe(struct hid_device *hdev, else /* USB_VENDOR_ID_APPLE */ report = hid_register_report(hdev, HID_INPUT_REPORT, TRACKPAD2_USB_REPORT_ID, 0); - } else if (id->vendor == SPI_VENDOR_ID_APPLE) { - report = hid_register_report(hdev, HID_INPUT_REPORT, 2, 0); + } else if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + } else if (id->bus == BUS_HOST) { + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); } else { /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ report = hid_register_report(hdev, HID_INPUT_REPORT, TRACKPAD_REPORT_ID, 0); @@ -1151,6 +1176,10 @@ static int magicmouse_probe(struct hid_device *hdev, } report->size = 6; + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + /* * Some devices repond with 'invalid report id' when feature * report switching it into multitouch mode is sent to it. @@ -1233,6 +1262,8 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2), .driver_data = 0 }, { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); From 4901728badc4f334c1f09f516025020f9c5c319e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 8 Jul 2022 02:09:24 +0900 Subject: [PATCH 135/635] soc: apple: Add DockChannel driver DockChannel is a simple FIFO interface used to communicate between SoC blocks. Add a driver that represents the shared interrupt controller for the DockChannel block, and then exposes probe and data transfer functions that child device drivers can use to instantiate individual FIFOs. Signed-off-by: Hector Martin --- drivers/soc/apple/Kconfig | 10 + drivers/soc/apple/Makefile | 3 + drivers/soc/apple/dockchannel.c | 407 ++++++++++++++++++++++++++ include/linux/soc/apple/dockchannel.h | 26 ++ 4 files changed, 446 insertions(+) create mode 100644 drivers/soc/apple/dockchannel.c create mode 100644 include/linux/soc/apple/dockchannel.h diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index a1596fefacffe1..9e93df0ac82582 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -41,6 +41,16 @@ config APPLE_SART Say 'y' here if you have an Apple SoC. +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + endmenu endif diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index e293770cf66d73..aa4203e0246288 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -6,3 +6,6 @@ apple-rtkit-y = rtkit.o rtkit-crashlog.o obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o + +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c new file mode 100644 index 00000000000000..b4d793bf210266 --- /dev/null +++ b/drivers/soc/apple/dockchannel.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq <= 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + struct device_node *child; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq <= 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + for_each_child_of_node(dev->of_node, child) + of_platform_device_create(child, NULL, dev); + + return 0; +} + +static int dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + return 0; +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h new file mode 100644 index 00000000000000..0b7093935ddf47 --- /dev/null +++ b/include/linux/soc/apple/dockchannel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif From da07643e905c39f65fe38be65f7b3940a0681081 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 1 Feb 2022 00:40:51 +0900 Subject: [PATCH 136/635] lib/vsprintf: Add support for generic FOURCCs by extending %p4cc %p4cc is designed for DRM/V4L2 FOURCCs with their specific quirks, but it's useful to be able to print generic 4-character codes formatted as an integer. Extend it to add format specifiers for printing generic 32-bit FOURCCs with various endian semantics: %p4ch Host-endian %p4cl Little-endian %p4cb Big-endian %p4cr Reverse-endian The endianness determines how bytes are interpreted as a u32, and the FOURCC is then always printed MSByte-first (this is the opposite of V4L/DRM FOURCCs). This covers most practical cases, e.g. %p4cr would allow printing LSByte-first FOURCCs stored in host endian order (other than the hex form being in character order, not the integer value). Signed-off-by: Hector Martin --- Documentation/core-api/printk-formats.rst | 32 +++++++++++++++++++++ lib/vsprintf.c | 35 +++++++++++++++++++---- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst index dbe1aacc79d0fc..7d72e4f70a1f9c 100644 --- a/Documentation/core-api/printk-formats.rst +++ b/Documentation/core-api/printk-formats.rst @@ -625,6 +625,38 @@ Examples:: %p4cc Y10 little-endian (0x20303159) %p4cc NV12 big-endian (0xb231564e) +Generic FourCC code +------------------- + +:: + %p4c[hnbl] gP00 (0x67503030) + +Print a generic FourCC code, as both ASCII characters and its numerical +value as hexadecimal. + +The additional ``h``, ``r``, ``b``, and ``l`` specifiers are used to specify +host, reversed, big or little endian order data respectively. Host endian +order means the data is interpreted as a 32-bit integer and the most +significant byte is printed first; that is, the character code as printed +matches the byte order stored in memory on big-endian systems, and is reversed +on little-endian systems. + +Passed by reference. + +Examples for a little-endian machine, given &(u32)0x67503030:: + + %p4ch gP00 (0x67503030) + %p4cl gP00 (0x67503030) + %p4cb 00Pg (0x30305067) + %p4cr 00Pg (0x30305067) + +Examples for a big-endian machine, given &(u32)0x67503030:: + + %p4ch gP00 (0x67503030) + %p4cl 00Pg (0x30305067) + %p4cb gP00 (0x67503030) + %p4cr 00Pg (0x30305067) + Rust ---- diff --git a/lib/vsprintf.c b/lib/vsprintf.c index be71a03c936a31..f367af3dc4beb4 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -1758,27 +1758,50 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc, char output[sizeof("0123 little-endian (0x01234567)")]; char *p = output; unsigned int i; + bool pix_fmt = false; u32 orig, val; - if (fmt[1] != 'c' || fmt[2] != 'c') + if (fmt[1] != 'c') return error_string(buf, end, "(%p4?)", spec); if (check_pointer(&buf, end, fourcc, spec)) return buf; orig = get_unaligned(fourcc); - val = orig & ~BIT(31); + switch (fmt[2]) { + case 'h': + val = orig; + break; + case 'r': + val = orig = swab32(orig); + break; + case 'l': + val = orig = le32_to_cpu(orig); + break; + case 'b': + val = orig = be32_to_cpu(orig); + break; + case 'c': + /* Pixel formats are printed LSB-first */ + val = swab32(orig & ~BIT(31)); + pix_fmt = true; + break; + default: + return error_string(buf, end, "(%p4?)", spec); + } for (i = 0; i < sizeof(u32); i++) { - unsigned char c = val >> (i * 8); + unsigned char c = val >> ((3 - i) * 8); /* Print non-control ASCII characters as-is, dot otherwise */ *p++ = isascii(c) && isprint(c) ? c : '.'; } - *p++ = ' '; - strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); - p += strlen(p); + if (pix_fmt) { + *p++ = ' '; + strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); + p += strlen(p); + } *p++ = ' '; *p++ = '('; From 32051cb996204b0ae891cd22e0f02702afa6ab94 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 5 May 2022 01:25:16 +0900 Subject: [PATCH 137/635] soc: apple: rtkit: Get rid of apple_rtkit_send_message_wait It is fundamentally broken and has no users. Just remove it. Signed-off-by: Hector Martin --- drivers/soc/apple/rtkit.c | 32 -------------------------------- include/linux/soc/apple/rtkit.h | 18 ------------------ 2 files changed, 50 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 35ec35aa500d67..fd7c1df24217ca 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -629,38 +629,6 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, } EXPORT_SYMBOL_GPL(apple_rtkit_send_message); -int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message, - unsigned long timeout, bool atomic) -{ - DECLARE_COMPLETION_ONSTACK(completion); - int ret; - long t; - - ret = apple_rtkit_send_message(rtk, ep, message, &completion, atomic); - if (ret < 0) - return ret; - - if (atomic) { - ret = mbox_flush(rtk->mbox_chan, timeout); - if (ret < 0) - return ret; - - if (try_wait_for_completion(&completion)) - return 0; - - return -ETIME; - } else { - t = wait_for_completion_interruptible_timeout( - &completion, msecs_to_jiffies(timeout)); - if (t < 0) - return t; - else if (t == 0) - return -ETIME; - return 0; - } -} -EXPORT_SYMBOL_GPL(apple_rtkit_send_message_wait); - int apple_rtkit_poll(struct apple_rtkit *rtk) { return mbox_client_peek_data(rtk->mbox_chan); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index fc456f75c13192..8c9ca857ccf6a3 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -160,24 +160,6 @@ int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint); int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, struct completion *completion, bool atomic); -/* - * Send a message to the given endpoint and wait until it has been submitted - * to the hardware FIFO. - * Will return zero on success and a negative error code on failure - * (e.g. -ETIME when the message couldn't be written within the given - * timeout) - * - * @rtk: RTKit reference - * @ep: target endpoint - * @message: message to be sent - * @timeout: timeout in milliseconds to allow the message transmission - * to be completed - * @atomic: if set to true this function can be called from atomic - * context. - */ -int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message, - unsigned long timeout, bool atomic); - /* * Process incoming messages in atomic context. * This only guarantees that messages arrive as far as the recv_message_early From d55f4cc3a82c44e82a3fe6fc02403752ea0e2053 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Feb 2022 12:47:07 +0900 Subject: [PATCH 138/635] platform/apple: Add new Apple Mac SMC driver This driver implements support for the SMC (System Management Controller) in Apple Macs. In contrast to the existing applesmc driver, it uses pluggable backends that allow it to support different SMC implementations, and uses the MFD subsystem to expose the core SMC functionality so that specific features (gpio, hwmon, battery, etc.) can be implemented by separate drivers in their respective downstream subsystems. The initial RTKit backend adds support for Apple Silicon Macs (M1 et al). We hope a backend for T2 Macs will be written in the future (since those are not supported by applesmc), and eventually an x86 backend would allow us to fully deprecate applesmc in favor of this driver. Signed-off-by: Hector Martin --- drivers/platform/Kconfig | 2 + drivers/platform/Makefile | 1 + drivers/platform/apple/Kconfig | 49 ++++ drivers/platform/apple/Makefile | 11 + drivers/platform/apple/smc.h | 28 ++ drivers/platform/apple/smc_core.c | 249 ++++++++++++++++ drivers/platform/apple/smc_rtkit.c | 453 +++++++++++++++++++++++++++++ include/linux/mfd/macsmc.h | 86 ++++++ 8 files changed, 879 insertions(+) create mode 100644 drivers/platform/apple/Kconfig create mode 100644 drivers/platform/apple/Makefile create mode 100644 drivers/platform/apple/smc.h create mode 100644 drivers/platform/apple/smc_core.c create mode 100644 drivers/platform/apple/smc_rtkit.c create mode 100644 include/linux/mfd/macsmc.h diff --git a/drivers/platform/Kconfig b/drivers/platform/Kconfig index 868b20361769c3..8bfd8a13802391 100644 --- a/drivers/platform/Kconfig +++ b/drivers/platform/Kconfig @@ -14,3 +14,5 @@ source "drivers/platform/olpc/Kconfig" source "drivers/platform/surface/Kconfig" source "drivers/platform/x86/Kconfig" + +source "drivers/platform/apple/Kconfig" diff --git a/drivers/platform/Makefile b/drivers/platform/Makefile index 41640172975a79..d2baa4eb4f133e 100644 --- a/drivers/platform/Makefile +++ b/drivers/platform/Makefile @@ -11,3 +11,4 @@ obj-$(CONFIG_OLPC_EC) += olpc/ obj-$(CONFIG_GOLDFISH) += goldfish/ obj-$(CONFIG_CHROME_PLATFORMS) += chrome/ obj-$(CONFIG_SURFACE_PLATFORMS) += surface/ +obj-$(CONFIG_APPLE_PLATFORMS) += apple/ diff --git a/drivers/platform/apple/Kconfig b/drivers/platform/apple/Kconfig new file mode 100644 index 00000000000000..5bcadd349493ac --- /dev/null +++ b/drivers/platform/apple/Kconfig @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Apple Platform-Specific Drivers +# + +menuconfig APPLE_PLATFORMS + bool "Apple Mac Platform-Specific Device Drivers" + default y + help + Say Y here to get to see options for platform-specific device drivers + for Apple devices. This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if APPLE_PLATFORMS + +config APPLE_SMC + tristate "Apple SMC Driver" + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) + default ARCH_APPLE + select MFD_CORE + help + Build support for the Apple System Management Controller present in + Apple Macs. This driver currently supports the SMC in Apple Silicon + Macs. For x86 Macs, see the applesmc driver (SENSORS_APPLESMC). + + Say Y here if you have an Apple Silicon Mac. + + To compile this driver as a module, choose M here: the module will + be called macsmc. + +if APPLE_SMC + +config APPLE_SMC_RTKIT + tristate "RTKit (Apple Silicon) backend" + depends on ARCH_APPLE || (COMPILE_TEST && 64BIT) + depends on APPLE_RTKIT + default ARCH_APPLE + help + Build support for SMC communications via the RTKit backend. This is + required for Apple Silicon Macs. + + Say Y here if you have an Apple Silicon Mac. + + To compile this driver as a module, choose M here: the module will + be called macsmc-rtkit. + +endif +endif diff --git a/drivers/platform/apple/Makefile b/drivers/platform/apple/Makefile new file mode 100644 index 00000000000000..79fac195398b0c --- /dev/null +++ b/drivers/platform/apple/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for linux/drivers/platform/apple +# Apple Platform-Specific Drivers +# + +macsmc-y += smc_core.o +macsmc-rtkit-y += smc_rtkit.o + +obj-$(CONFIG_APPLE_SMC) += macsmc.o +obj-$(CONFIG_APPLE_SMC_RTKIT) += macsmc-rtkit.o diff --git a/drivers/platform/apple/smc.h b/drivers/platform/apple/smc.h new file mode 100644 index 00000000000000..8ae51887b2c553 --- /dev/null +++ b/drivers/platform/apple/smc.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC internal core definitions + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _SMC_H +#define _SMC_H + +#include + +struct apple_smc_backend_ops { + int (*read_key)(void *cookie, smc_key key, void *buf, size_t size); + int (*write_key)(void *cookie, smc_key key, void *buf, size_t size); + int (*write_key_atomic)(void *cookie, smc_key key, void *buf, size_t size); + int (*rw_key)(void *cookie, smc_key key, void *wbuf, size_t wsize, + void *rbuf, size_t rsize); + int (*get_key_by_index)(void *cookie, int index, smc_key *key); + int (*get_key_info)(void *cookie, smc_key key, struct apple_smc_key_info *info); +}; + +struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_backend_ops *ops, + void *cookie); +void *apple_smc_get_cookie(struct apple_smc *smc); +int apple_smc_remove(struct apple_smc *smc); +void apple_smc_event_received(struct apple_smc *smc, uint32_t event); + +#endif diff --git a/drivers/platform/apple/smc_core.c b/drivers/platform/apple/smc_core.c new file mode 100644 index 00000000000000..daf029cd072f52 --- /dev/null +++ b/drivers/platform/apple/smc_core.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC core framework + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include "smc.h" + +struct apple_smc { + struct device *dev; + + void *be_cookie; + const struct apple_smc_backend_ops *be; + + struct mutex mutex; + + u32 key_count; + smc_key first_key; + smc_key last_key; + + struct blocking_notifier_head event_handlers; +}; + +static const struct mfd_cell apple_smc_devs[] = { + { + .name = "macsmc-gpio", + }, + { + .name = "macsmc-hid", + }, + { + .name = "macsmc-power", + }, + { + .name = "macsmc-reboot", + }, + { + .name = "macsmc-rtc", + }, +}; + +int apple_smc_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + int ret; + + mutex_lock(&smc->mutex); + ret = smc->be->read_key(smc->be_cookie, key, buf, size); + mutex_unlock(&smc->mutex); + + return ret; +} +EXPORT_SYMBOL(apple_smc_read); + +int apple_smc_write(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + int ret; + + mutex_lock(&smc->mutex); + ret = smc->be->write_key(smc->be_cookie, key, buf, size); + mutex_unlock(&smc->mutex); + + return ret; +} +EXPORT_SYMBOL(apple_smc_write); + +int apple_smc_write_atomic(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + int ret; + + /* + * Will fail if SMC is busy. This is only used by SMC reboot/poweroff + * final calls, so it doesn't really matter at that point. + */ + if (!mutex_trylock(&smc->mutex)) + return -EBUSY; + + ret = smc->be->write_key_atomic(smc->be_cookie, key, buf, size); + mutex_unlock(&smc->mutex); + + return ret; +} +EXPORT_SYMBOL(apple_smc_write_atomic); + +int apple_smc_rw(struct apple_smc *smc, smc_key key, void *wbuf, size_t wsize, + void *rbuf, size_t rsize) +{ + int ret; + + mutex_lock(&smc->mutex); + ret = smc->be->rw_key(smc->be_cookie, key, wbuf, wsize, rbuf, rsize); + mutex_unlock(&smc->mutex); + + return ret; +} +EXPORT_SYMBOL(apple_smc_rw); + +int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key) +{ + int ret; + + mutex_lock(&smc->mutex); + ret = smc->be->get_key_by_index(smc->be_cookie, index, key); + mutex_unlock(&smc->mutex); + + return ret; +} +EXPORT_SYMBOL(apple_smc_get_key_by_index); + +int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_key_info *info) +{ + int ret; + + mutex_lock(&smc->mutex); + ret = smc->be->get_key_info(smc->be_cookie, key, info); + mutex_unlock(&smc->mutex); + + return ret; +} +EXPORT_SYMBOL(apple_smc_get_key_info); + +int apple_smc_find_first_key_index(struct apple_smc *smc, smc_key key) +{ + int start = 0, count = smc->key_count; + int ret; + + if (key <= smc->first_key) + return 0; + if (key > smc->last_key) + return smc->key_count; + + while (count > 1) { + int pivot = start + ((count - 1) >> 1); + smc_key pkey; + + ret = apple_smc_get_key_by_index(smc, pivot, &pkey); + if (ret < 0) + return ret; + + if (pkey == key) + return pivot; + + pivot++; + + if (pkey < key) { + count -= pivot - start; + start = pivot; + } else { + count = pivot - start; + } + } + + return start; +} +EXPORT_SYMBOL(apple_smc_find_first_key_index); + +int apple_smc_get_key_count(struct apple_smc *smc) +{ + return smc->key_count; +} +EXPORT_SYMBOL(apple_smc_get_key_count); + +void apple_smc_event_received(struct apple_smc *smc, uint32_t event) +{ + dev_dbg(smc->dev, "Event: 0x%08x\n", event); + blocking_notifier_call_chain(&smc->event_handlers, event, NULL); +} +EXPORT_SYMBOL(apple_smc_event_received); + +int apple_smc_register_notifier(struct apple_smc *smc, struct notifier_block *n) +{ + return blocking_notifier_chain_register(&smc->event_handlers, n); +} +EXPORT_SYMBOL(apple_smc_register_notifier); + +int apple_smc_unregister_notifier(struct apple_smc *smc, struct notifier_block *n) +{ + return blocking_notifier_chain_unregister(&smc->event_handlers, n); +} +EXPORT_SYMBOL(apple_smc_unregister_notifier); + +void *apple_smc_get_cookie(struct apple_smc *smc) +{ + return smc->be_cookie; +} +EXPORT_SYMBOL(apple_smc_get_cookie); + +struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_backend_ops *ops, void *cookie) +{ + struct apple_smc *smc; + u32 count; + int ret; + + smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); + if (!smc) + return ERR_PTR(-ENOMEM); + + smc->dev = dev; + smc->be_cookie = cookie; + smc->be = ops; + mutex_init(&smc->mutex); + BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); + + ret = apple_smc_read_u32(smc, SMC_KEY(#KEY), &count); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to get key count")); + smc->key_count = be32_to_cpu(count); + + ret = apple_smc_get_key_by_index(smc, 0, &smc->first_key); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to get first key")); + + ret = apple_smc_get_key_by_index(smc, smc->key_count - 1, &smc->last_key); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to get last key")); + + /* Enable notifications */ + apple_smc_write_flag(smc, SMC_KEY(NTAP), 1); + + dev_info(dev, "Initialized (%d keys %p4ch..%p4ch)\n", + smc->key_count, &smc->first_key, &smc->last_key); + + dev_set_drvdata(dev, smc); + + ret = mfd_add_devices(dev, -1, apple_smc_devs, ARRAY_SIZE(apple_smc_devs), NULL, 0, NULL); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Subdevice initialization failed")); + + return smc; +} +EXPORT_SYMBOL(apple_smc_probe); + +int apple_smc_remove(struct apple_smc *smc) +{ + mfd_remove_devices(smc->dev); + + /* Disable notifications */ + apple_smc_write_flag(smc, SMC_KEY(NTAP), 1); + + return 0; +} +EXPORT_SYMBOL(apple_smc_remove); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC core"); diff --git a/drivers/platform/apple/smc_rtkit.c b/drivers/platform/apple/smc_rtkit.c new file mode 100644 index 00000000000000..583d65681c3a25 --- /dev/null +++ b/drivers/platform/apple/smc_rtkit.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC RTKit backend + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "smc.h" + +#define SMC_ENDPOINT 0x20 + +/* Guess */ +#define SMC_SHMEM_SIZE 0x1000 + +#define SMC_MSG_READ_KEY 0x10 +#define SMC_MSG_WRITE_KEY 0x11 +#define SMC_MSG_GET_KEY_BY_INDEX 0x12 +#define SMC_MSG_GET_KEY_INFO 0x13 +#define SMC_MSG_INITIALIZE 0x17 +#define SMC_MSG_NOTIFICATION 0x18 +#define SMC_MSG_RW_KEY 0x20 + +#define SMC_DATA GENMASK(63, 32) +#define SMC_WSIZE GENMASK(31, 24) +#define SMC_SIZE GENMASK(23, 16) +#define SMC_ID GENMASK(15, 12) +#define SMC_MSG GENMASK(7, 0) +#define SMC_RESULT SMC_MSG + +#define SMC_RECV_TIMEOUT 500 + +struct apple_smc_rtkit { + struct device *dev; + struct apple_smc *core; + struct apple_rtkit *rtk; + + struct completion init_done; + bool initialized; + bool alive; + + struct resource *sram; + void __iomem *sram_base; + struct apple_rtkit_shmem shmem; + + unsigned int msg_id; + + bool atomic_pending; + struct completion cmd_done; + u64 cmd_ret; +}; + +static int apple_smc_rtkit_write_key_atomic(void *cookie, smc_key key, void *buf, size_t size) +{ + struct apple_smc_rtkit *smc = cookie; + int ret; + u64 msg; + u8 result; + + if (size > SMC_SHMEM_SIZE || size == 0) + return -EINVAL; + + if (!smc->alive) + return -EIO; + + memcpy_toio(smc->shmem.iomem, buf, size); + smc->msg_id = (smc->msg_id + 1) & 0xf; + msg = (FIELD_PREP(SMC_MSG, SMC_MSG_WRITE_KEY) | + FIELD_PREP(SMC_SIZE, size) | + FIELD_PREP(SMC_ID, smc->msg_id) | + FIELD_PREP(SMC_DATA, key)); + smc->atomic_pending = true; + + ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, msg, NULL, true); + if (ret < 0) { + dev_err(smc->dev, "Failed to send command (%d)\n", ret); + return ret; + } + + while (smc->atomic_pending) { + ret = apple_rtkit_poll(smc->rtk); + if (ret < 0) { + dev_err(smc->dev, "RTKit poll failed (%llx)", msg); + return ret; + } + udelay(100); + } + + if (FIELD_GET(SMC_ID, smc->cmd_ret) != smc->msg_id) { + dev_err(smc->dev, "Command sequence mismatch (expected %d, got %d)\n", + smc->msg_id, (unsigned int)FIELD_GET(SMC_ID, smc->cmd_ret)); + return -EIO; + } + + result = FIELD_GET(SMC_RESULT, smc->cmd_ret); + if (result != 0) + return -result; + + return FIELD_GET(SMC_SIZE, smc->cmd_ret); +} + +static int apple_smc_cmd(struct apple_smc_rtkit *smc, u64 cmd, u64 arg, + u64 size, u64 wsize, u32 *ret_data) +{ + int ret; + u64 msg; + u8 result; + + if (!smc->alive) + return -EIO; + + reinit_completion(&smc->cmd_done); + + smc->msg_id = (smc->msg_id + 1) & 0xf; + msg = (FIELD_PREP(SMC_MSG, cmd) | + FIELD_PREP(SMC_SIZE, size) | + FIELD_PREP(SMC_WSIZE, wsize) | + FIELD_PREP(SMC_ID, smc->msg_id) | + FIELD_PREP(SMC_DATA, arg)); + + ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, msg, NULL, false); + if (ret < 0) { + dev_err(smc->dev, "Failed to send command\n"); + return ret; + } + + do { + if (wait_for_completion_timeout(&smc->cmd_done, + msecs_to_jiffies(SMC_RECV_TIMEOUT)) == 0) { + dev_err(smc->dev, "Command timed out (%llx)", msg); + return -ETIMEDOUT; + } + if (FIELD_GET(SMC_ID, smc->cmd_ret) == smc->msg_id) + break; + dev_err(smc->dev, "Command sequence mismatch (expected %d, got %d)\n", + smc->msg_id, (unsigned int)FIELD_GET(SMC_ID, smc->cmd_ret)); + } while(1); + + result = FIELD_GET(SMC_RESULT, smc->cmd_ret); + if (result != 0) + return -result; + + if (ret_data) + *ret_data = FIELD_GET(SMC_DATA, smc->cmd_ret); + + return FIELD_GET(SMC_SIZE, smc->cmd_ret); +} + +static int _apple_smc_rtkit_read_key(struct apple_smc_rtkit *smc, smc_key key, + void *buf, size_t size, size_t wsize) +{ + int ret; + u32 rdata; + u64 cmd; + + if (size > SMC_SHMEM_SIZE || size == 0) + return -EINVAL; + + cmd = wsize ? SMC_MSG_RW_KEY : SMC_MSG_READ_KEY; + + ret = apple_smc_cmd(smc, cmd, key, size, wsize, &rdata); + if (ret < 0) + return ret; + + if (size <= 4) + memcpy(buf, &rdata, size); + else + memcpy_fromio(buf, smc->shmem.iomem, size); + + return ret; +} + +static int apple_smc_rtkit_read_key(void *cookie, smc_key key, void *buf, size_t size) +{ + return _apple_smc_rtkit_read_key(cookie, key, buf, size, 0); +} + +static int apple_smc_rtkit_write_key(void *cookie, smc_key key, void *buf, size_t size) +{ + struct apple_smc_rtkit *smc = cookie; + + if (size > SMC_SHMEM_SIZE || size == 0) + return -EINVAL; + + memcpy_toio(smc->shmem.iomem, buf, size); + return apple_smc_cmd(smc, SMC_MSG_WRITE_KEY, key, size, 0, NULL); +} + +static int apple_smc_rtkit_rw_key(void *cookie, smc_key key, + void *wbuf, size_t wsize, void *rbuf, size_t rsize) +{ + struct apple_smc_rtkit *smc = cookie; + + if (wsize > SMC_SHMEM_SIZE || wsize == 0) + return -EINVAL; + + memcpy_toio(smc->shmem.iomem, wbuf, wsize); + return _apple_smc_rtkit_read_key(smc, key, rbuf, rsize, wsize); +} + +static int apple_smc_rtkit_get_key_by_index(void *cookie, int index, smc_key *key) +{ + struct apple_smc_rtkit *smc = cookie; + int ret; + + ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_BY_INDEX, index, 0, 0, key); + + *key = swab32(*key); + return ret; +} + +static int apple_smc_rtkit_get_key_info(void *cookie, smc_key key, struct apple_smc_key_info *info) +{ + struct apple_smc_rtkit *smc = cookie; + u8 key_info[6]; + int ret; + + ret = apple_smc_cmd(smc, SMC_MSG_GET_KEY_INFO, key, 0, 0, NULL); + if (ret >= 0 && info) { + memcpy_fromio(key_info, smc->shmem.iomem, sizeof(key_info)); + info->size = key_info[0]; + info->type_code = get_unaligned_be32(&key_info[1]); + info->flags = key_info[5]; + } + return ret; +} + +static const struct apple_smc_backend_ops apple_smc_rtkit_be_ops = { + .read_key = apple_smc_rtkit_read_key, + .write_key = apple_smc_rtkit_write_key, + .write_key_atomic = apple_smc_rtkit_write_key_atomic, + .rw_key = apple_smc_rtkit_rw_key, + .get_key_by_index = apple_smc_rtkit_get_key_by_index, + .get_key_info = apple_smc_rtkit_get_key_info, +}; + +static void apple_smc_rtkit_crashed(void *cookie) +{ + struct apple_smc_rtkit *smc = cookie; + + dev_err(smc->dev, "SMC crashed! Your system will reboot in a few seconds...\n"); + smc->alive = false; +} + +static int apple_smc_rtkit_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_smc_rtkit *smc = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + .flags = smc->sram->flags, + }; + + if (!bfr->iova) { + dev_err(smc->dev, "RTKit wants a RAM buffer\n"); + return -EIO; + } + + if (res.end < res.start || !resource_contains(smc->sram, &res)) { + dev_err(smc->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = smc->sram_base + (res.start - smc->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_smc_rtkit_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static bool apple_smc_rtkit_recv_early(void *cookie, u8 endpoint, u64 message) +{ + struct apple_smc_rtkit *smc = cookie; + + if (endpoint != SMC_ENDPOINT) { + dev_err(smc->dev, "Received message for unknown endpoint 0x%x\n", endpoint); + return false; + } + + if (!smc->initialized) { + int ret; + + smc->shmem.iova = message; + smc->shmem.size = SMC_SHMEM_SIZE; + ret = apple_smc_rtkit_shmem_setup(smc, &smc->shmem); + if (ret < 0) + dev_err(smc->dev, "Failed to initialize shared memory\n"); + else + smc->alive = true; + smc->initialized = true; + complete(&smc->init_done); + } else if (FIELD_GET(SMC_MSG, message) == SMC_MSG_NOTIFICATION) { + /* Handle these in the RTKit worker thread */ + return false; + } else { + smc->cmd_ret = message; + if (smc->atomic_pending) { + smc->atomic_pending = false; + } else { + complete(&smc->cmd_done); + } + } + + return true; +} + +static void apple_smc_rtkit_recv(void *cookie, u8 endpoint, u64 message) +{ + struct apple_smc_rtkit *smc = cookie; + + if (endpoint != SMC_ENDPOINT) { + dev_err(smc->dev, "Received message for unknown endpoint 0x%x\n", endpoint); + return; + } + + if (FIELD_GET(SMC_MSG, message) != SMC_MSG_NOTIFICATION) { + dev_err(smc->dev, "Received unknown message from worker: 0x%llx\n", message); + return; + } + + apple_smc_event_received(smc->core, FIELD_GET(SMC_DATA, message)); +} + +static const struct apple_rtkit_ops apple_smc_rtkit_ops = { + .crashed = apple_smc_rtkit_crashed, + .recv_message = apple_smc_rtkit_recv, + .recv_message_early = apple_smc_rtkit_recv_early, + .shmem_setup = apple_smc_rtkit_shmem_setup, + .shmem_destroy = apple_smc_rtkit_shmem_destroy, +}; + +static int apple_smc_rtkit_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_smc_rtkit *smc; + int ret; + + smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); + if (!smc) + return -ENOMEM; + + smc->dev = dev; + + smc->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (!smc->sram) + return dev_err_probe(dev, EIO, + "No SRAM region"); + + smc->sram_base = devm_ioremap_resource(dev, smc->sram); + if (IS_ERR(smc->sram_base)) + return dev_err_probe(dev, PTR_ERR(smc->sram_base), + "Failed to map SRAM region"); + + smc->rtk = + devm_apple_rtkit_init(dev, smc, NULL, 0, &apple_smc_rtkit_ops); + if (IS_ERR(smc->rtk)) + return dev_err_probe(dev, PTR_ERR(smc->rtk), + "Failed to intialize RTKit"); + + ret = apple_rtkit_wake(smc->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, + "Failed to wake up SMC"); + + ret = apple_rtkit_start_ep(smc->rtk, SMC_ENDPOINT); + if (ret != 0) { + dev_err(dev, "Failed to start endpoint"); + goto cleanup; + } + + init_completion(&smc->init_done); + init_completion(&smc->cmd_done); + + ret = apple_rtkit_send_message(smc->rtk, SMC_ENDPOINT, + FIELD_PREP(SMC_MSG, SMC_MSG_INITIALIZE), NULL, false); + if (ret < 0) + return dev_err_probe(dev, ret, + "Failed to send init message"); + + if (wait_for_completion_timeout(&smc->init_done, + msecs_to_jiffies(SMC_RECV_TIMEOUT)) == 0) { + ret = -ETIMEDOUT; + dev_err(dev, "Timed out initializing SMC"); + goto cleanup; + } + + if (!smc->alive) { + ret = -EIO; + goto cleanup; + } + + smc->core = apple_smc_probe(dev, &apple_smc_rtkit_be_ops, smc); + if (IS_ERR(smc->core)) { + ret = PTR_ERR(smc->core); + goto cleanup; + } + + return 0; + +cleanup: + /* Try to shut down RTKit, if it's not completely wedged */ + if (apple_rtkit_is_running(smc->rtk)) + apple_rtkit_quiesce(smc->rtk); + + return ret; +} + +static int apple_smc_rtkit_remove(struct platform_device *pdev) +{ + struct apple_smc *core = platform_get_drvdata(pdev); + struct apple_smc_rtkit *smc = apple_smc_get_cookie(core); + + apple_smc_remove(core); + + if (apple_rtkit_is_running(smc->rtk)) + apple_rtkit_quiesce(smc->rtk); + + return 0; +} + +static const struct of_device_id apple_smc_rtkit_of_match[] = { + { .compatible = "apple,smc" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_smc_rtkit_of_match); + +static struct platform_driver apple_smc_rtkit_driver = { + .driver = { + .name = "macsmc-rtkit", + .owner = THIS_MODULE, + .of_match_table = apple_smc_rtkit_of_match, + }, + .probe = apple_smc_rtkit_probe, + .remove = apple_smc_rtkit_remove, +}; +module_platform_driver(apple_smc_rtkit_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC RTKit backend driver"); diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h new file mode 100644 index 00000000000000..39b4dc4ca88106 --- /dev/null +++ b/include/linux/mfd/macsmc.h @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC core definitions + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef _LINUX_MFD_MACSMC_H +#define _LINUX_MFD_MACSMC_H + +struct apple_smc; + +typedef u32 smc_key; + +#define SMC_KEY(s) (smc_key)(_SMC_KEY(#s)) +#define _SMC_KEY(s) (((s)[0] << 24) | ((s)[1] << 16) | ((s)[2] << 8) | (s)[3]) + +#define APPLE_SMC_READABLE BIT(7) +#define APPLE_SMC_WRITABLE BIT(6) +#define APPLE_SMC_FUNCTION BIT(4) + +struct apple_smc_key_info { + u8 size; + u32 type_code; + u8 flags; +}; + +int apple_smc_read(struct apple_smc *smc, smc_key key, void *buf, size_t size); +int apple_smc_write(struct apple_smc *smc, smc_key key, void *buf, size_t size); +int apple_smc_write_atomic(struct apple_smc *smc, smc_key key, void *buf, size_t size); +int apple_smc_rw(struct apple_smc *smc, smc_key key, void *wbuf, size_t wsize, + void *rbuf, size_t rsize); + +int apple_smc_get_key_count(struct apple_smc *smc); +int apple_smc_find_first_key_index(struct apple_smc *smc, smc_key key); +int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key); +int apple_smc_get_key_info(struct apple_smc *smc, smc_key key, struct apple_smc_key_info *info); + +static inline bool apple_smc_key_exists(struct apple_smc *smc, smc_key key) +{ + return apple_smc_get_key_info(smc, key, NULL) >= 0; +} + +#define APPLE_SMC_TYPE_OPS(type) \ + static inline int apple_smc_read_##type(struct apple_smc *smc, smc_key key, type *p) \ + { \ + int ret = apple_smc_read(smc, key, p, sizeof(*p)); \ + return (ret < 0) ? ret : ((ret != sizeof(*p)) ? -EINVAL : 0); \ + } \ + static inline int apple_smc_write_##type(struct apple_smc *smc, smc_key key, type p) \ + { \ + return apple_smc_write(smc, key, &p, sizeof(p)); \ + } \ + static inline int apple_smc_write_##type##_atomic(struct apple_smc *smc, smc_key key, type p) \ + { \ + return apple_smc_write_atomic(smc, key, &p, sizeof(p)); \ + } \ + static inline int apple_smc_rw_##type(struct apple_smc *smc, smc_key key, \ + type w, type *r) \ + { \ + int ret = apple_smc_rw(smc, key, &w, sizeof(w), r, sizeof(*r)); \ + return (ret < 0) ? ret : ((ret != sizeof(*r)) ? -EINVAL : 0); \ + } + +APPLE_SMC_TYPE_OPS(u64) +APPLE_SMC_TYPE_OPS(u32) +APPLE_SMC_TYPE_OPS(u16) +APPLE_SMC_TYPE_OPS(u8) +APPLE_SMC_TYPE_OPS(s64) +APPLE_SMC_TYPE_OPS(s32) +APPLE_SMC_TYPE_OPS(s16) +APPLE_SMC_TYPE_OPS(s8) + +static inline int apple_smc_read_flag(struct apple_smc *smc, smc_key key) +{ + u8 val; + int ret = apple_smc_read_u8(smc, key, &val); + if (ret < 0) + return ret; + return val ? 1 : 0; +} +#define apple_smc_write_flag apple_smc_write_u8 + +int apple_smc_register_notifier(struct apple_smc *smc, struct notifier_block *n); +int apple_smc_unregister_notifier(struct apple_smc *smc, struct notifier_block *n); + +#endif From 205e98574d6b645964f44e731f6f093a6350c664 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Feb 2022 12:52:52 +0900 Subject: [PATCH 139/635] gpio: Add new gpio-macsmc driver for Apple Macs This driver implements the GPIO service on top of the SMC framework on Apple Mac machines. In particular, these are the GPIOs present in the PMU IC which are used to control power to certain on-board devices. Although the underlying hardware supports various pin config settings (input/output, open drain, etc.), this driver does not implement that functionality and leaves it up to the firmware to configure things properly. We also don't yet support interrupts/events. This is sufficient for device power control, which is the only thing we need to support at this point. More features will be implemented when needed. To our knowledge, only Apple Silicon Macs implement this SMC feature. Signed-off-by: Hector Martin --- drivers/gpio/Kconfig | 11 ++ drivers/gpio/Makefile | 1 + drivers/gpio/gpio-macsmc.c | 238 +++++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+) create mode 100644 drivers/gpio/gpio-macsmc.c diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index badbe058231805..41aad53ecbe5ae 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -1281,6 +1281,17 @@ config GPIO_LP87565 This driver can also be built as a module. If so, the module will be called gpio-lp87565. +config GPIO_MACSMC + tristate "Apple Mac SMC GPIO" + depends on APPLE_SMC + default ARCH_APPLE + help + Support for GPIOs controlled by the SMC microcontroller on Apple Mac + systems. + + This driver can also be built as a module. If so, the module will be + called gpio-macsmc. + config GPIO_MADERA tristate "Cirrus Logic Madera class codecs" depends on PINCTRL_MADERA diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index c048ba0033672f..cd74c7f9d514a3 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -85,6 +85,7 @@ obj-$(CONFIG_GPIO_LP873X) += gpio-lp873x.o obj-$(CONFIG_GPIO_LP87565) += gpio-lp87565.o obj-$(CONFIG_GPIO_LPC18XX) += gpio-lpc18xx.o obj-$(CONFIG_GPIO_LPC32XX) += gpio-lpc32xx.o +obj-$(CONFIG_GPIO_MACSMC) += gpio-macsmc.o obj-$(CONFIG_GPIO_MADERA) += gpio-madera.o obj-$(CONFIG_GPIO_MAX3191X) += gpio-max3191x.o obj-$(CONFIG_GPIO_MAX7300) += gpio-max7300.o diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c new file mode 100644 index 00000000000000..ff9950afb69af0 --- /dev/null +++ b/drivers/gpio/gpio-macsmc.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC GPIO driver + * Copyright The Asahi Linux Contributors + * + * This driver implements basic SMC PMU GPIO support that can read inputs + * and write outputs. Mode changes and IRQ config are not yet implemented. + */ + +#include +#include +#include +#include +#include + +#define MAX_GPIO 64 + +/* + * Commands 0-6 are, presumably, the intended API. + * Command 0xff lets you get/set the pin configuration in detail directly, + * but the bit meanings seem not to be stable between devices/PMU hardware + * versions. + * + * We're going to try to make do with the low commands for now. + * We don't implement pin mode changes at this time. + */ + +#define CMD_ACTION (0 << 24) +#define CMD_OUTPUT (1 << 24) +#define CMD_INPUT (2 << 24) +#define CMD_PINMODE (3 << 24) +#define CMD_IRQ_ENABLE (4 << 24) +#define CMD_IRQ_ACK (5 << 24) +#define CMD_IRQ_MODE (6 << 24) +#define CMD_CONFIG (0xff << 24) + +#define MODE_INPUT 0 +#define MODE_OUTPUT 1 +#define MODE_VALUE_0 0 +#define MODE_VALUE_1 2 + +#define IRQ_MODE_HIGH 0 +#define IRQ_MODE_LOW 1 +#define IRQ_MODE_RISING 2 +#define IRQ_MODE_FALLING 3 +#define IRQ_MODE_BOTH 4 + +#define CONFIG_MASK GENMASK(23, 16) +#define CONFIG_VAL GENMASK(7, 0) + +#define CONFIG_OUTMODE GENMASK(7, 6) +#define CONFIG_IRQMODE GENMASK(5, 3) +#define CONFIG_PULLDOWN BIT(2) +#define CONFIG_PULLUP BIT(1) +#define CONFIG_OUTVAL BIT(0) + +/* + * output modes seem to differ depending on the PMU in use... ? + * j274 / M1 (Sera PMU): + * 0 = input + * 1 = output + * 2 = open drain + * 3 = disable + * j314 / M1Pro (Maverick PMU): + * 0 = input + * 1 = open drain + * 2 = output + * 3 = ? + */ + +struct macsmc_gpio { + struct device *dev; + struct apple_smc *smc; + struct gpio_chip gc; + + int first_index; +}; + +static int macsmc_gpio_nr(smc_key key) +{ + int low = hex_to_bin(key & 0xff); + int high = hex_to_bin((key >> 8) & 0xff); + + if (low < 0 || high < 0) + return -1; + + return low | (high << 4); +} + +static int macsmc_gpio_key(unsigned int offset) +{ + return _SMC_KEY("gP\0\0") | (hex_asc_hi(offset) << 8) | hex_asc_lo(offset); +} + +static int macsmc_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + smc_key key = macsmc_gpio_key(offset); + u32 val; + int ret; + + /* First try reading the explicit pin mode register */ + ret = apple_smc_rw_u32(smcgp->smc, key, CMD_PINMODE, &val); + if (!ret) + return (val & MODE_OUTPUT) ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; + + /* + * Less common IRQ configs cause CMD_PINMODE to fail, and so does open drain mode. + * Fall back to reading IRQ mode, which will only succeed for inputs. + */ + ret = apple_smc_rw_u32(smcgp->smc, key, CMD_IRQ_MODE, &val); + return (!ret) ? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT; +} + +static int macsmc_gpio_get(struct gpio_chip *gc, unsigned int offset) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + smc_key key = macsmc_gpio_key(offset); + u32 val; + int ret; + + ret = macsmc_gpio_get_direction(gc, offset); + if (ret < 0) + return ret; + + if (ret == GPIO_LINE_DIRECTION_OUT) + ret = apple_smc_rw_u32(smcgp->smc, key, CMD_OUTPUT, &val); + else + ret = apple_smc_rw_u32(smcgp->smc, key, CMD_INPUT, &val); + + if (ret < 0) + return ret; + + return val ? 1 : 0; +} + +static void macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + smc_key key = macsmc_gpio_key(offset); + int ret; + + value |= CMD_OUTPUT; + ret = apple_smc_write_u32(smcgp->smc, key, CMD_OUTPUT | value); + if (ret < 0) + dev_err(smcgp->dev, "GPIO set failed %p4ch = 0x%x\n", &key, value); +} + +static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, + unsigned long *valid_mask, unsigned int ngpios) +{ + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + int count = apple_smc_get_key_count(smcgp->smc) - smcgp->first_index; + int i; + + if (count > MAX_GPIO) + count = MAX_GPIO; + + bitmap_zero(valid_mask, ngpios); + + for (i = 0; i < count; i++) { + smc_key key; + int gpio_nr; + int ret = apple_smc_get_key_by_index(smcgp->smc, smcgp->first_index + i, &key); + + if (ret < 0) + return ret; + + if (key > SMC_KEY(gPff)) + break; + + gpio_nr = macsmc_gpio_nr(key); + if (gpio_nr < 0 || gpio_nr > MAX_GPIO) { + dev_err(smcgp->dev, "Bad GPIO key %p4ch\n", &key); + continue; + } + + set_bit(gpio_nr, valid_mask); + } + + return 0; +} + +static int macsmc_gpio_probe(struct platform_device *pdev) +{ + struct macsmc_gpio *smcgp; + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + smc_key key; + int ret; + + smcgp = devm_kzalloc(&pdev->dev, sizeof(*smcgp), GFP_KERNEL); + if (!smcgp) + return -ENOMEM; + + pdev->dev.of_node = of_get_child_by_name(pdev->dev.parent->of_node, "gpio"); + + smcgp->dev = &pdev->dev; + smcgp->smc = smc; + smcgp->first_index = apple_smc_find_first_key_index(smc, SMC_KEY(gP00)); + + if (smcgp->first_index >= apple_smc_get_key_count(smc)) + return -ENODEV; + + ret = apple_smc_get_key_by_index(smc, smcgp->first_index, &key); + if (ret < 0) + return ret; + + if (key > macsmc_gpio_key(MAX_GPIO - 1)) + return -ENODEV; + + dev_info(smcgp->dev, "First GPIO key: %p4ch\n", &key); + + smcgp->gc.label = "macsmc-pmu-gpio"; + smcgp->gc.owner = THIS_MODULE; + smcgp->gc.get = macsmc_gpio_get; + smcgp->gc.set = macsmc_gpio_set; + smcgp->gc.get_direction = macsmc_gpio_get_direction; + smcgp->gc.init_valid_mask = macsmc_gpio_init_valid_mask; + smcgp->gc.can_sleep = true; + smcgp->gc.ngpio = MAX_GPIO; + smcgp->gc.base = -1; + smcgp->gc.parent = &pdev->dev; + + return devm_gpiochip_add_data(&pdev->dev, &smcgp->gc, smcgp); +} + +static struct platform_driver macsmc_gpio_driver = { + .driver = { + .name = "macsmc-gpio", + }, + .probe = macsmc_gpio_probe, +}; +module_platform_driver(macsmc_gpio_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC GPIO driver"); +MODULE_ALIAS("platform:macsmc-gpio"); From 4d782bfdf7f1a61172869ae789d478e64981e3f9 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 8 Feb 2022 02:30:16 +0900 Subject: [PATCH 140/635] power: supply: macsmc_power: Driver for Apple SMC power/battery stats This driver implements support for battery stats on top of the macsmc framework, to support Apple M1 Mac machines. Co-authored-by: Joey Gouly Signed-off-by: Hector Martin --- drivers/power/supply/Kconfig | 7 + drivers/power/supply/Makefile | 1 + drivers/power/supply/macsmc_power.c | 259 ++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 drivers/power/supply/macsmc_power.c diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index c78be9f322e6c6..193bea179271b8 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -934,4 +934,11 @@ config BATTERY_UG3105 device is off or suspended, the functionality of this driver is limited to reporting capacity only. +config CHARGER_MACSMC + tristate "Apple SMC Charger / Battery support" + depends on APPLE_SMC + help + Say Y here to enable support for the charger and battery controls on + Apple SMC controllers, as used on Apple Silicon Macs. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile index 4adbfba02d054a..cafbccaa97bade 100644 --- a/drivers/power/supply/Makefile +++ b/drivers/power/supply/Makefile @@ -109,3 +109,4 @@ obj-$(CONFIG_BATTERY_ACER_A500) += acer_a500_battery.o obj-$(CONFIG_BATTERY_SURFACE) += surface_battery.o obj-$(CONFIG_CHARGER_SURFACE) += surface_charger.o obj-$(CONFIG_BATTERY_UG3105) += ug3105_battery.o +obj-$(CONFIG_CHARGER_MACSMC) += macsmc_power.o diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c new file mode 100644 index 00000000000000..140526a963b63e --- /dev/null +++ b/drivers/power/supply/macsmc_power.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC Power/Battery Management + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#define MAX_STRING_LENGTH 256 + +struct macsmc_power { + struct device *dev; + struct apple_smc *smc; + struct power_supply *psy; + char model_name[MAX_STRING_LENGTH]; + char serial_number[MAX_STRING_LENGTH]; + + struct notifier_block nb; +}; + +static int macsmc_battery_get_status(struct macsmc_power *power) +{ + u8 val; + int ret; + + ret = apple_smc_read_u8(power->smc, SMC_KEY(BSFC), &val); + if (ret) + return ret; + if (val == 1) + return POWER_SUPPLY_STATUS_FULL; + + ret = apple_smc_read_u8(power->smc, SMC_KEY(CHSC), &val); + if (ret) + return ret; + if (val == 1) + return POWER_SUPPLY_STATUS_CHARGING; + + ret = apple_smc_read_u8(power->smc, SMC_KEY(CHCC), &val); + if (ret) + return ret; + if (val == 0) + return POWER_SUPPLY_STATUS_DISCHARGING; + + ret = apple_smc_read_u8(power->smc, SMC_KEY(CHCE), &val); + if (ret) + return ret; + if (val == 0) + return POWER_SUPPLY_STATUS_DISCHARGING; + else + return POWER_SUPPLY_STATUS_NOT_CHARGING; + + +} + +static int macsmc_battery_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u16 vu16; + u32 vu32; + s16 vs16; + s32 vs32; + s64 vs64; + + switch (psp) { + case POWER_SUPPLY_PROP_STATUS: + val->intval = macsmc_battery_get_status(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_CAPACITY: + ret = apple_smc_read_u16(power->smc, SMC_KEY(BRSC), &vu16); + val->intval = vu16; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = apple_smc_read_s16(power->smc, SMC_KEY(B0AC), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_POWER_NOW: + ret = apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &vs32); + val->intval = vs32 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(BITV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + ret = apple_smc_read_u32(power->smc, SMC_KEY(CSIL), &vu32); + val->intval = vu32 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RI), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * 1000; + break; + case POWER_SUPPLY_PROP_TEMP: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AT), &vu16); + val->intval = vu16 - 2732; + break; + case POWER_SUPPLY_PROP_CHARGE_COUNTER: + ret = apple_smc_read_s64(power->smc, SMC_KEY(BAAC), &vs64); + val->intval = vs64; + break; + case POWER_SUPPLY_PROP_MODEL_NAME: + val->strval = power->model_name; + break; + case POWER_SUPPLY_PROP_SERIAL_NUMBER: + val->strval = power->serial_number; + break; + default: + return -EINVAL; + } + + return ret; +} + +static enum power_supply_property macsmc_battery_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_POWER_NOW, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_NOW, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_CHARGE_COUNTER, + POWER_SUPPLY_PROP_MODEL_NAME, + POWER_SUPPLY_PROP_SERIAL_NUMBER, +}; + +static const struct power_supply_desc macsmc_battery_desc = { + .name = "macsmc-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .get_property = macsmc_battery_get_property, + .properties = macsmc_battery_props, + .num_properties = ARRAY_SIZE(macsmc_battery_props), +}; + +static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_power *power = container_of(nb, struct macsmc_power, nb); + + if ((event & 0xffffff00) == 0x71010100) { + bool charging = (event & 0xff) != 0; + + dev_info(power->dev, "Charging: %d\n", charging); + power_supply_changed(power->psy); + + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + +static int macsmc_power_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct power_supply_config psy_cfg = {}; + struct macsmc_power *power; + int ret; + + power = devm_kzalloc(&pdev->dev, sizeof(*power), GFP_KERNEL); + if (!power) + return -ENOMEM; + + power->dev = &pdev->dev; + power->smc = smc; + dev_set_drvdata(&pdev->dev, power); + + /* Ignore devices without a charger/battery */ + if (macsmc_battery_get_status(power) <= POWER_SUPPLY_STATUS_UNKNOWN) + return -ENODEV; + + /* Fetch string properties */ + apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, sizeof(power->model_name) - 1); + apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, sizeof(power->serial_number) - 1); + + psy_cfg.drv_data = power; + power->psy = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); + if (IS_ERR(power->psy)) { + dev_err(&pdev->dev, "Failed to register power supply\n"); + ret = PTR_ERR(power->psy); + return ret; + } + + power->nb.notifier_call = macsmc_power_event; + apple_smc_register_notifier(power->smc, &power->nb); + + return 0; +} + +static int macsmc_power_remove(struct platform_device *pdev) +{ + struct macsmc_power *power = dev_get_drvdata(&pdev->dev); + + apple_smc_unregister_notifier(power->smc, &power->nb); + + return 0; +} + +static struct platform_driver macsmc_power_driver = { + .driver = { + .name = "macsmc-power", + .owner = THIS_MODULE, + }, + .probe = macsmc_power_probe, + .remove = macsmc_power_remove, +}; +module_platform_driver(macsmc_power_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC battery and power management driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_ALIAS("platform:macsmc-power"); From 5e08536c41610f8a0c86535f05d04989ddc53869 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 8 Feb 2022 02:51:35 +0900 Subject: [PATCH 141/635] power: supply: macsmc_power: Add cycle count and health props Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index 140526a963b63e..6c939371912add 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -138,6 +138,15 @@ static int macsmc_battery_get_property(struct power_supply *psy, ret = apple_smc_read_s64(power->smc, SMC_KEY(BAAC), &vs64); val->intval = vs64; break; + case POWER_SUPPLY_PROP_CYCLE_COUNT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0CT), &vu16); + val->intval = vu16; + break; + case POWER_SUPPLY_PROP_HEALTH: + ret = apple_smc_read_flag(power->smc, SMC_KEY(BBAD)); + val->intval = ret == 1 ? POWER_SUPPLY_HEALTH_DEAD : POWER_SUPPLY_HEALTH_GOOD; + ret = ret < 0 ? ret : 0; + break; case POWER_SUPPLY_PROP_MODEL_NAME: val->strval = power->model_name; break; @@ -169,6 +178,8 @@ static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_CHARGE_NOW, POWER_SUPPLY_PROP_TEMP, POWER_SUPPLY_PROP_CHARGE_COUNTER, + POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_MODEL_NAME, POWER_SUPPLY_PROP_SERIAL_NUMBER, }; From 215762bddf10d9cc4157cb811b09cef56a3ac2d6 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 8 Feb 2022 11:01:17 +0900 Subject: [PATCH 142/635] power: supply: macsmc_power: Add present prop Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index 6c939371912add..aa67762144d8c2 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -74,6 +74,9 @@ static int macsmc_battery_get_property(struct power_supply *psy, val->intval = macsmc_battery_get_status(power); ret = val->intval < 0 ? val->intval : 0; break; + case POWER_SUPPLY_PROP_PRESENT: + val->intval = 1; + break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); val->intval = vu16 == 0xffff ? 0 : vu16 * 60; @@ -162,6 +165,7 @@ static int macsmc_battery_get_property(struct power_supply *psy, static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, POWER_SUPPLY_PROP_CAPACITY, From 97a68e4486c85108227948b7c8ac7c88c885538f Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 02:20:20 +0900 Subject: [PATCH 143/635] power: supply: macsmc_power: Add more props, rework others Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 224 +++++++++++++++++++++++++--- 1 file changed, 202 insertions(+), 22 deletions(-) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index aa67762144d8c2..2cb1644055f4e5 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -4,6 +4,7 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include #include @@ -19,50 +20,178 @@ struct macsmc_power { struct power_supply *psy; char model_name[MAX_STRING_LENGTH]; char serial_number[MAX_STRING_LENGTH]; + char mfg_date[MAX_STRING_LENGTH]; struct notifier_block nb; }; +#define CHNC_BATTERY_FULL BIT(0) +#define CHNC_NO_CHARGER BIT(7) +#define CHNC_NOCHG_CH0C BIT(14) +#define CHNC_NOCHG_CH0B_CH0K BIT(15) +#define CHNC_BATTERY_FULL_2 BIT(18) +#define CHNC_BMS_BUSY BIT(23) +#define CHNC_NOAC_CH0J BIT(53) +#define CHNC_NOAC_CH0I BIT(54) + +#define CH0R_LOWER_FLAGS GENMASK(15, 0) +#define CH0R_NOAC_CH0I BIT(0) +#define CH0R_NOAC_CH0J BIT(5) +#define CH0R_BMS_BUSY BIT(8) +#define CH0R_NOAC_CH0K BIT(9) + +#define CH0X_CH0C BIT(0) +#define CH0X_CH0B BIT(1) + static int macsmc_battery_get_status(struct macsmc_power *power) { - u8 val; + u64 nocharge_flags; + u32 nopower_flags; + u16 ac_current; int ret; - ret = apple_smc_read_u8(power->smc, SMC_KEY(BSFC), &val); - if (ret) + /* + * Note: there are fallbacks in case some of these SMC keys disappear in the future + * or are not present on some machines. We treat the absence of the CHCE/CHCC/BSFC/CHSC + * flags as an error, since they are quite fundamental and simple booleans. + */ + + /* + * If power input is inhibited, we are definitely discharging. + * However, if the only reason is the BMS is doing a balancing cycle, + * go ahead and ignore that one to avoid spooking users. + */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(CH0R), &nopower_flags); + if (!ret && (nopower_flags & CH0R_LOWER_FLAGS & ~CH0R_BMS_BUSY)) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If no charger is present, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCE)); + if (ret < 0) + return ret; + else if (!ret) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If AC is not charge capable, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCC)); + if (ret < 0) return ret; - if (val == 1) + else if (!ret) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* + * If the AC input current limit is tiny or 0, we are discharging no matter + * how much the BMS believes it can charge. + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &ac_current); + if (!ret && ac_current < 100) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If the battery is full, report it as such. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC)); + if (ret < 0) + return ret; + else if (ret) return POWER_SUPPLY_STATUS_FULL; - ret = apple_smc_read_u8(power->smc, SMC_KEY(CHSC), &val); - if (ret) + /* If there are reasons we aren't charging... */ + ret = apple_smc_read_u64(power->smc, SMC_KEY(CHNC), &nocharge_flags); + if (!ret) { + /* Perhaps the battery is full after all */ + if (nocharge_flags & CHNC_BATTERY_FULL) + return POWER_SUPPLY_STATUS_FULL; + /* Or maybe the BMS is just busy doing something, if so call it charging anyway */ + else if (nocharge_flags == CHNC_BMS_BUSY) + return POWER_SUPPLY_STATUS_CHARGING; + /* If we have other reasons we aren't charging, say we aren't */ + else if (nocharge_flags) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + /* Else we're either charging or about to charge */ + else + return POWER_SUPPLY_STATUS_CHARGING; + } + + /* As a fallback, use the system charging flag. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHSC)); + if (ret < 0) return ret; - if (val == 1) + if (!ret) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + else return POWER_SUPPLY_STATUS_CHARGING; +} - ret = apple_smc_read_u8(power->smc, SMC_KEY(CHCC), &val); +static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) +{ + int ret; + u8 val; + + /* CH0I returns a bitmask like the low byte of CH0R */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val); if (ret) return ret; - if (val == 0) - return POWER_SUPPLY_STATUS_DISCHARGING; + if (val & CH0R_NOAC_CH0I) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE; - ret = apple_smc_read_u8(power->smc, SMC_KEY(CHCE), &val); + /* CH0C returns a bitmask containing CH0B/CH0C flags */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val); if (ret) return ret; - if (val == 0) - return POWER_SUPPLY_STATUS_DISCHARGING; + if (val & CH0X_CH0C) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; else - return POWER_SUPPLY_STATUS_NOT_CHARGING; + return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; +} - +static int macsmc_battery_set_charge_behaviour(struct macsmc_power *power, int val) +{ + u8 ch0i, ch0c; + int ret; + + /* + * CH0I/CH0C are "hard" controls that will allow the battery to run down to 0. + * CH0K/CH0B are "soft" controls that are reset to 0 when SOC drops below 50%; + * we don't expose these yet. + */ + + switch (val) { + case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO: + ch0i = ch0c = 0; + break; + case POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE: + ch0i = 0; + ch0c = 1; + break; + case POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE: + ch0i = 1; + ch0c = 0; + break; + default: + return -EINVAL; + } + + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), ch0i); + if (ret) + return ret; + return apple_smc_write_u8(power->smc, SMC_KEY(CH0C), ch0c); +} + +static int macsmc_battery_get_date(const char *s, int *out) +{ + if (!isdigit(s[0]) || !isdigit(s[1])) + return -ENOTSUPP; + + *out = (s[0] - '0') * 10 + s[1] - '0'; + return 0; } static int macsmc_battery_get_property(struct power_supply *psy, - enum power_supply_property psp, - union power_supply_propval *val) + enum power_supply_property psp, + union power_supply_propval *val) { struct macsmc_power *power = power_supply_get_drvdata(psy); int ret = 0; + u8 vu8; u16 vu16; u32 vu32; s16 vs16; @@ -77,6 +206,10 @@ static int macsmc_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_PRESENT: val->intval = 1; break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + val->intval = macsmc_battery_get_charge_behaviour(power); + ret = val->intval < 0 ? val->intval : 0; + break; case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); val->intval = vu16 == 0xffff ? 0 : vu16 * 60; @@ -145,6 +278,9 @@ static int macsmc_battery_get_property(struct power_supply *psy, ret = apple_smc_read_u16(power->smc, SMC_KEY(B0CT), &vu16); val->intval = vu16; break; + case POWER_SUPPLY_PROP_SCOPE: + val->intval = POWER_SUPPLY_SCOPE_SYSTEM; + break; case POWER_SUPPLY_PROP_HEALTH: ret = apple_smc_read_flag(power->smc, SMC_KEY(BBAD)); val->intval = ret == 1 ? POWER_SUPPLY_HEALTH_DEAD : POWER_SUPPLY_HEALTH_GOOD; @@ -156,6 +292,16 @@ static int macsmc_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_SERIAL_NUMBER: val->strval = power->serial_number; break; + case POWER_SUPPLY_PROP_MANUFACTURE_YEAR: + ret = macsmc_battery_get_date(&power->mfg_date[0], &val->intval); + val->intval += 2000 - 8; /* -8 is a fixup for a firmware bug... */ + break; + case POWER_SUPPLY_PROP_MANUFACTURE_MONTH: + ret = macsmc_battery_get_date(&power->mfg_date[2], &val->intval); + break; + case POWER_SUPPLY_PROP_MANUFACTURE_DAY: + ret = macsmc_battery_get_date(&power->mfg_date[4], &val->intval); + break; default: return -EINVAL; } @@ -163,9 +309,36 @@ static int macsmc_battery_get_property(struct power_supply *psy, return ret; } +static int macsmc_battery_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return macsmc_battery_set_charge_behaviour(power, val->intval); + default: + return -EINVAL; + } +} + +static int macsmc_battery_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return true; + default: + return false; + } +} + + static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, POWER_SUPPLY_PROP_CAPACITY, @@ -183,17 +356,23 @@ static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_TEMP, POWER_SUPPLY_PROP_CHARGE_COUNTER, POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_SCOPE, POWER_SUPPLY_PROP_HEALTH, POWER_SUPPLY_PROP_MODEL_NAME, POWER_SUPPLY_PROP_SERIAL_NUMBER, + POWER_SUPPLY_PROP_MANUFACTURE_YEAR, + POWER_SUPPLY_PROP_MANUFACTURE_MONTH, + POWER_SUPPLY_PROP_MANUFACTURE_DAY, }; static const struct power_supply_desc macsmc_battery_desc = { - .name = "macsmc-battery", - .type = POWER_SUPPLY_TYPE_BATTERY, - .get_property = macsmc_battery_get_property, - .properties = macsmc_battery_props, - .num_properties = ARRAY_SIZE(macsmc_battery_props), + .name = "macsmc-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .get_property = macsmc_battery_get_property, + .set_property = macsmc_battery_set_property, + .property_is_writeable = macsmc_battery_property_is_writeable, + .properties = macsmc_battery_props, + .num_properties = ARRAY_SIZE(macsmc_battery_props), }; static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) @@ -234,6 +413,7 @@ static int macsmc_power_probe(struct platform_device *pdev) /* Fetch string properties */ apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, sizeof(power->model_name) - 1); apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, sizeof(power->serial_number) - 1); + apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, sizeof(power->mfg_date) - 1); psy_cfg.drv_data = power; power->psy = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); From 025f7dcc7c713920bddf1f78dd177884db2e5757 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 02:23:33 +0900 Subject: [PATCH 144/635] power: supply: macsmc_power: Use BUIC instead of BRSC for charge Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index 2cb1644055f4e5..b9aa6af3210436 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -219,8 +219,8 @@ static int macsmc_battery_get_property(struct power_supply *psy, val->intval = vu16 == 0xffff ? 0 : vu16 * 60; break; case POWER_SUPPLY_PROP_CAPACITY: - ret = apple_smc_read_u16(power->smc, SMC_KEY(BRSC), &vu16); - val->intval = vu16; + ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); + val->intval = vu8; break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16); From f569361c0314ddbebb4dfb39a5c7cc4e7848ea20 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 02:24:13 +0900 Subject: [PATCH 145/635] power: supply: macsmc_power: Turn off OBC flags if macOS left them on Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index b9aa6af3210436..4a8a90b775dd4c 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -415,6 +415,10 @@ static int macsmc_power_probe(struct platform_device *pdev) apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, sizeof(power->serial_number) - 1); apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, sizeof(power->mfg_date) - 1); + /* Turn off the "optimized battery charging" flags, in case macOS left them on */ + apple_smc_write_u8(power->smc, SMC_KEY(CH0K), 0); + apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); + psy_cfg.drv_data = power; power->psy = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); if (IS_ERR(power->psy)) { From ce75747ae7c5de7473d5d3bd8799747cd62bfadd Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 02:24:42 +0900 Subject: [PATCH 146/635] power: supply: macsmc_power: Add AC power supply Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 74 +++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index 4a8a90b775dd4c..dee48ee8b5a455 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -17,11 +17,14 @@ struct macsmc_power { struct device *dev; struct apple_smc *smc; - struct power_supply *psy; + + struct power_supply *batt; char model_name[MAX_STRING_LENGTH]; char serial_number[MAX_STRING_LENGTH]; char mfg_date[MAX_STRING_LENGTH]; + struct power_supply *ac; + struct notifier_block nb; }; @@ -49,7 +52,7 @@ static int macsmc_battery_get_status(struct macsmc_power *power) u32 nopower_flags; u16 ac_current; int ret; - + /* * Note: there are fallbacks in case some of these SMC keys disappear in the future * or are not present on some machines. We treat the absence of the CHCE/CHCC/BSFC/CHSC @@ -334,7 +337,6 @@ static int macsmc_battery_property_is_writeable(struct power_supply *psy, } } - static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_PRESENT, @@ -375,6 +377,54 @@ static const struct power_supply_desc macsmc_battery_desc = { .num_properties = ARRAY_SIZE(macsmc_battery_props), }; +static int macsmc_ac_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u16 vu16; + u32 vu32; + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + ret = apple_smc_read_u32(power->smc, SMC_KEY(CHIS), &vu32); + val->intval = !!vu32; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_POWER_LIMIT: + ret = apple_smc_read_u32(power->smc, SMC_KEY(ACPW), &vu32); + val->intval = vu32 * 1000; + break; + default: + return -EINVAL; + } + + return ret; +} + +static enum power_supply_property macsmc_ac_props[] = { + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, + POWER_SUPPLY_PROP_INPUT_POWER_LIMIT, +}; + +static const struct power_supply_desc macsmc_ac_desc = { + .name = "macsmc-ac", + .type = POWER_SUPPLY_TYPE_MAINS, + .get_property = macsmc_ac_get_property, + .properties = macsmc_ac_props, + .num_properties = ARRAY_SIZE(macsmc_ac_props), +}; + static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) { struct macsmc_power *power = container_of(nb, struct macsmc_power, nb); @@ -383,7 +433,8 @@ static int macsmc_power_event(struct notifier_block *nb, unsigned long event, vo bool charging = (event & 0xff) != 0; dev_info(power->dev, "Charging: %d\n", charging); - power_supply_changed(power->psy); + power_supply_changed(power->batt); + power_supply_changed(power->ac); return NOTIFY_OK; } @@ -420,10 +471,17 @@ static int macsmc_power_probe(struct platform_device *pdev) apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); psy_cfg.drv_data = power; - power->psy = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); - if (IS_ERR(power->psy)) { - dev_err(&pdev->dev, "Failed to register power supply\n"); - ret = PTR_ERR(power->psy); + power->batt = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); + if (IS_ERR(power->batt)) { + dev_err(&pdev->dev, "Failed to register battery\n"); + ret = PTR_ERR(power->batt); + return ret; + } + + power->ac = devm_power_supply_register(&pdev->dev, &macsmc_ac_desc, &psy_cfg); + if (IS_ERR(power->ac)) { + dev_err(&pdev->dev, "Failed to register AC adapter\n"); + ret = PTR_ERR(power->ac); return ret; } From d287491022d024778cdb14dd9bcf1bdc24f46812 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 8 Feb 2022 19:17:40 +0900 Subject: [PATCH 147/635] power: reset: macsmc-reboot: Add driver for rebooting via Apple SMC This driver implements the reboot/shutdown support exposed by the SMC on Apple Silicon machines, such as Apple M1 Macs. Signed-off-by: Hector Martin --- drivers/power/reset/Kconfig | 12 + drivers/power/reset/Makefile | 1 + drivers/power/reset/macsmc-reboot.c | 336 ++++++++++++++++++++++++++++ 3 files changed, 349 insertions(+) create mode 100644 drivers/power/reset/macsmc-reboot.c diff --git a/drivers/power/reset/Kconfig b/drivers/power/reset/Kconfig index 8c87eeda0feca0..3e167dff0a473a 100644 --- a/drivers/power/reset/Kconfig +++ b/drivers/power/reset/Kconfig @@ -117,6 +117,18 @@ config POWER_RESET_LINKSTATION Say Y here if you have a Buffalo LinkStation LS421D/E. +config POWER_RESET_MACSMC + tristate "Apple SMC reset/power-off driver" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_SMC + depends on OF + default ARCH_APPLE + help + This driver supports reset and power-off on Apple Mac machines + that implement this functionality via the SMC. + + Say Y here if you have an Apple Silicon Mac. + config POWER_RESET_MSM bool "Qualcomm MSM power-off driver" depends on ARCH_QCOM diff --git a/drivers/power/reset/Makefile b/drivers/power/reset/Makefile index d763e6735ee391..30b3f0005aa323 100644 --- a/drivers/power/reset/Makefile +++ b/drivers/power/reset/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_POWER_RESET_GPIO) += gpio-poweroff.o obj-$(CONFIG_POWER_RESET_GPIO_RESTART) += gpio-restart.o obj-$(CONFIG_POWER_RESET_HISI) += hisi-reboot.o obj-$(CONFIG_POWER_RESET_LINKSTATION) += linkstation-poweroff.o +obj-$(CONFIG_POWER_RESET_MACSMC) += macsmc-reboot.o obj-$(CONFIG_POWER_RESET_MSM) += msm-poweroff.o obj-$(CONFIG_POWER_RESET_MT6323) += mt6323-poweroff.o obj-$(CONFIG_POWER_RESET_OXNAS) += oxnas-restart.o diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c new file mode 100644 index 00000000000000..c33ba2a7852d48 --- /dev/null +++ b/drivers/power/reset/macsmc-reboot.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC Reboot/Poweroff Handler + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct macsmc_reboot_nvmem { + struct nvmem_cell *shutdown_flag; + struct nvmem_cell *pm_setting; + struct nvmem_cell *boot_stage; + struct nvmem_cell *boot_error_count; + struct nvmem_cell *panic_count; +}; + +static const char *nvmem_names[] = { + "shutdown_flag", + "pm_setting", + "boot_stage", + "boot_error_count", + "panic_count", +}; + +enum boot_stage { + BOOT_STAGE_SHUTDOWN = 0x00, /* Clean shutdown */ + BOOT_STAGE_IBOOT_DONE = 0x2f, /* Last stage of bootloader */ + BOOT_STAGE_KERNEL_STARTED = 0x30, /* Normal OS booting */ +}; + +enum pm_setting { + PM_SETTING_AC_POWER_RESTORE = 0x02, + PM_SETTING_AC_POWER_OFF = 0x03, +}; + +static const char *ac_power_modes[] = { "off", "restore" }; + +static int ac_power_mode_map[] = { + PM_SETTING_AC_POWER_OFF, + PM_SETTING_AC_POWER_RESTORE, +}; + +struct macsmc_reboot { + struct device *dev; + struct apple_smc *smc; + struct notifier_block reboot_notify; + + union { + struct macsmc_reboot_nvmem nvm; + struct nvmem_cell *nvm_cells[ARRAY_SIZE(nvmem_names)]; + }; +}; + +/* Helpers to read/write a u8 given a struct nvmem_cell */ +static int nvmem_cell_get_u8(struct nvmem_cell *cell) +{ + size_t len; + u8 val; + void *ret = nvmem_cell_read(cell, &len); + + if (IS_ERR(ret)) + return PTR_ERR(ret); + + if (len < 1) { + kfree(ret); + return -EINVAL; + } + + val = *(u8 *)ret; + kfree(ret); + return val; +} + +static int nvmem_cell_set_u8(struct nvmem_cell *cell, u8 val) +{ + return nvmem_cell_write(cell, &val, sizeof(val)); +} + +static ssize_t macsmc_ac_power_mode_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t n) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(dev); + int mode; + int ret; + + mode = sysfs_match_string(ac_power_modes, buf); + if (mode < 0) + return mode; + + ret = nvmem_cell_set_u8(reboot->nvm.pm_setting, ac_power_mode_map[mode]); + if (ret < 0) + return ret; + + return n; +} + +static ssize_t macsmc_ac_power_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct macsmc_reboot *reboot = dev_get_drvdata(dev); + int len = 0; + int i; + int mode = nvmem_cell_get_u8(reboot->nvm.pm_setting); + + if (mode < 0) + return mode; + + for (i = 0; i < ARRAY_SIZE(ac_power_mode_map); i++) + if (mode == ac_power_mode_map[i]) + len += scnprintf(buf+len, PAGE_SIZE-len, + "[%s] ", ac_power_modes[i]); + else + len += scnprintf(buf+len, PAGE_SIZE-len, + "%s ", ac_power_modes[i]); + buf[len-1] = '\n'; + return len; +} +static DEVICE_ATTR(ac_power_mode, 0644, macsmc_ac_power_mode_show, + macsmc_ac_power_mode_store); + +/* + * SMC 'MBSE' key actions: + * + * 'offw' - shutdown warning + * 'slpw' - sleep warning + * 'rest' - restart warning + * 'off1' - shutdown (needs PMU bit set to stay on) + * 'susp' - suspend + * 'phra' - restart ("PE Halt Restart Action"?) + * 'panb' - panic beginning + * 'pane' - panic end + */ + +static int macsmc_power_off(struct sys_off_data *data) +{ + struct macsmc_reboot *reboot = data->cb_data; + + dev_info(reboot->dev, "Issuing power off (off1)\n"); + + if (apple_smc_write_u32_atomic(reboot->smc, SMC_KEY(MBSE), SMC_KEY(off1)) < 0) { + dev_err(reboot->dev, "Failed to issue MBSE = off1 (power_off)\n"); + } else { + mdelay(100); + WARN_ON(1); + } + + return NOTIFY_OK; +} + +static int macsmc_restart(struct sys_off_data *data) +{ + struct macsmc_reboot *reboot = data->cb_data; + + dev_info(reboot->dev, "Issuing restart (phra)\n"); + + if (apple_smc_write_u32_atomic(reboot->smc, SMC_KEY(MBSE), SMC_KEY(phra)) < 0) { + dev_err(reboot->dev, "Failed to issue MBSE = phra (restart)\n"); + } else { + mdelay(100); + WARN_ON(1); + } + + return NOTIFY_OK; +} + +static int macsmc_reboot_notify(struct notifier_block *this, unsigned long action, void *data) +{ + struct macsmc_reboot *reboot = container_of(this, struct macsmc_reboot, reboot_notify); + u32 val; + u8 shutdown_flag; + + switch (action) { + case SYS_RESTART: + val = SMC_KEY(rest); + shutdown_flag = 0; + break; + case SYS_POWER_OFF: + val = SMC_KEY(offw); + shutdown_flag = 1; + break; + default: + return NOTIFY_DONE; + } + + dev_info(reboot->dev, "Preparing for reboot (%p4ch)\n", &val); + + /* On the Mac Mini, this will turn off the LED for power off */ + if (apple_smc_write_u32(reboot->smc, SMC_KEY(MBSE), val) < 0) + dev_err(reboot->dev, "Failed to issue MBSE = %p4ch (reboot_prepare)\n", &val); + + /* Set the boot_stage to 0, which means we're doing a clean shutdown/reboot. */ + if (reboot->nvm.boot_stage && + nvmem_cell_set_u8(reboot->nvm.boot_stage, BOOT_STAGE_SHUTDOWN) < 0) + dev_err(reboot->dev, "Failed to write boot_stage\n"); + + /* + * Set the PMU flag to actually reboot into the off state. + * Without this, the device will just reboot. We make it optional in case it is no longer + * necessary on newer hardware. + */ + if (reboot->nvm.shutdown_flag && + nvmem_cell_set_u8(reboot->nvm.shutdown_flag, shutdown_flag) < 0) + dev_err(reboot->dev, "Failed to write shutdown_flag\n"); + + return NOTIFY_OK; +} + +static void macsmc_power_init_error_counts(struct macsmc_reboot *reboot) +{ + int boot_error_count, panic_count; + + if (!reboot->nvm.boot_error_count || !reboot->nvm.panic_count) + return; + + boot_error_count = nvmem_cell_get_u8(reboot->nvm.boot_error_count); + if (boot_error_count < 0) { + dev_err(reboot->dev, "Failed to read boot_error_count (%d)\n", boot_error_count); + return; + } + + panic_count = nvmem_cell_get_u8(reboot->nvm.panic_count); + if (panic_count < 0) { + dev_err(reboot->dev, "Failed to read panic_count (%d)\n", panic_count); + return; + } + + if (!boot_error_count && !panic_count) + return; + + dev_warn(reboot->dev, "PMU logged %d boot error(s) and %d panic(s)\n", + boot_error_count, panic_count); + + if (nvmem_cell_set_u8(reboot->nvm.panic_count, 0) < 0) + dev_err(reboot->dev, "Failed to reset panic_count\n"); + if (nvmem_cell_set_u8(reboot->nvm.boot_error_count, 0) < 0) + dev_err(reboot->dev, "Failed to reset boot_error_count\n"); +} + +static int macsmc_reboot_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct macsmc_reboot *reboot; + int ret, i; + + /* Ignore devices without this functionality */ + if (!apple_smc_key_exists(smc, SMC_KEY(MBSE))) + return -ENODEV; + + reboot = devm_kzalloc(&pdev->dev, sizeof(*reboot), GFP_KERNEL); + if (!reboot) + return -ENOMEM; + + reboot->dev = &pdev->dev; + reboot->smc = smc; + + platform_set_drvdata(pdev, reboot); + + pdev->dev.of_node = of_get_child_by_name(pdev->dev.parent->of_node, "reboot"); + + for (i = 0; i < ARRAY_SIZE(nvmem_names); i++) { + struct nvmem_cell *cell; + cell = devm_nvmem_cell_get(&pdev->dev, + nvmem_names[i]); + if (IS_ERR(cell)) { + if (PTR_ERR(cell) == -EPROBE_DEFER) + return -EPROBE_DEFER; + dev_warn(&pdev->dev, "Missing NVMEM cell %s (%ld)\n", + nvmem_names[i], PTR_ERR(cell)); + /* Non fatal, we'll deal with it */ + cell = NULL; + } + reboot->nvm_cells[i] = cell; + } + + /* Set the boot_stage to indicate we're running the OS kernel */ + if (reboot->nvm.boot_stage && + nvmem_cell_set_u8(reboot->nvm.boot_stage, BOOT_STAGE_KERNEL_STARTED) < 0) + dev_err(reboot->dev, "Failed to write boot_stage\n"); + + /* Display and clear the error counts */ + macsmc_power_init_error_counts(reboot); + + reboot->reboot_notify.notifier_call = macsmc_reboot_notify; + + ret = devm_register_sys_off_handler(&pdev->dev, SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_HIGH, + macsmc_power_off, reboot); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to register power-off handler\n"); + + ret = devm_register_sys_off_handler(&pdev->dev, SYS_OFF_MODE_RESTART, SYS_OFF_PRIO_HIGH, + macsmc_restart, reboot); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to register restart handler\n"); + + ret = devm_register_reboot_notifier(&pdev->dev, &reboot->reboot_notify); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to register reboot notifier\n"); + + dev_info(&pdev->dev, "Handling reboot and poweroff requests via SMC\n"); + + if (device_create_file(&pdev->dev, &dev_attr_ac_power_mode)) + dev_warn(&pdev->dev, "could not create sysfs file\n"); + + return 0; +} + +static int macsmc_reboot_remove(struct platform_device *pdev) +{ + device_remove_file(&pdev->dev, &dev_attr_ac_power_mode); + + return 0; +} + + +static struct platform_driver macsmc_reboot_driver = { + .driver = { + .name = "macsmc-reboot", + .owner = THIS_MODULE, + }, + .probe = macsmc_reboot_probe, + .remove = macsmc_reboot_remove, +}; +module_platform_driver(macsmc_reboot_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC reboot/poweroff driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_ALIAS("platform:macsmc-reboot"); From 43300801974680618f88968bc8be4b6d2134e7c3 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 18:47:13 +0900 Subject: [PATCH 148/635] rtc: Add new rtc-macsmc driver for Apple Silicon Macs Apple Silicon Macs (M1, etc.) have an RTC that is part of the PMU IC, but most of the PMU functionality is abstracted out by the SMC. On T600x machines, the RTC counter must be accessed via the SMC to get full functionality, and it seems likely that future machines will move towards making SMC handle all RTC functionality. The SMC RTC counter access is implemented on all current machines as of the time of this writing, on firmware 12.x. However, the RTC offset (needed to set the time) is still only accessible via direct PMU access. To handle this, we expose the RTC offset as an NVMEM cell from the SPMI PMU device node, and this driver consumes that cell and uses it to compute/set the current time. Alarm functionality is not yet implemented. This would also go via the PMU today, but could change in the future. Signed-off-by: Hector Martin --- drivers/rtc/Kconfig | 13 ++++ drivers/rtc/Makefile | 1 + drivers/rtc/rtc-macsmc.c | 130 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 drivers/rtc/rtc-macsmc.c diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 5a71579af0a14e..341fa656faaa5a 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -1978,4 +1978,17 @@ config RTC_DRV_POLARFIRE_SOC This driver can also be built as a module, if so, the module will be called "rtc-mpfs". +config RTC_DRV_MACSMC + tristate "Apple Mac SMC RTC" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_SMC + depends on OF + default ARCH_APPLE + help + If you say yes here you get support for RTC functions + inside Apple SPMI PMUs. + + To compile this driver as a module, choose M here: the + module will be called rtc-macsmc. + endif # RTC_CLASS diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index ea445d1ebb1728..40a4c32e42eeb6 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -88,6 +88,7 @@ obj-$(CONFIG_RTC_DRV_M41T94) += rtc-m41t94.o obj-$(CONFIG_RTC_DRV_M48T35) += rtc-m48t35.o obj-$(CONFIG_RTC_DRV_M48T59) += rtc-m48t59.o obj-$(CONFIG_RTC_DRV_M48T86) += rtc-m48t86.o +obj-$(CONFIG_RTC_DRV_MACSMC) += rtc-macsmc.o obj-$(CONFIG_RTC_DRV_MAX6900) += rtc-max6900.o obj-$(CONFIG_RTC_DRV_MAX6902) += rtc-max6902.o obj-$(CONFIG_RTC_DRV_MAX6916) += rtc-max6916.o diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c new file mode 100644 index 00000000000000..34730c92524883 --- /dev/null +++ b/drivers/rtc/rtc-macsmc.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC RTC driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* 48-bit RTC */ +#define RTC_BYTES 6 +#define RTC_BITS (8 * RTC_BYTES) + +/* 32768 Hz clock */ +#define RTC_SEC_SHIFT 15 + +struct macsmc_rtc { + struct device *dev; + struct apple_smc *smc; + struct rtc_device *rtc_dev; + struct nvmem_cell *rtc_offset; +}; + +static int macsmc_rtc_get_time(struct device *dev, struct rtc_time *tm) +{ + struct macsmc_rtc *rtc = dev_get_drvdata(dev); + u64 ctr = 0, off = 0; + time64_t now; + void *p_off; + size_t len; + int ret; + + ret = apple_smc_read(rtc->smc, SMC_KEY(CLKM), &ctr, RTC_BYTES); + if (ret != RTC_BYTES) + return ret < 0 ? ret : -EIO; + + p_off = nvmem_cell_read(rtc->rtc_offset, &len); + if (IS_ERR(p_off)) + return PTR_ERR(p_off); + if (len < RTC_BYTES) { + kfree(p_off); + return -EIO; + } + + memcpy(&off, p_off, RTC_BYTES); + kfree(p_off); + + /* Sign extend from 48 to 64 bits, then arithmetic shift right 15 bits to get seconds */ + now = sign_extend64(ctr + off, RTC_BITS - 1) >> RTC_SEC_SHIFT; + rtc_time64_to_tm(now, tm); + + return ret; +} + +static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) +{ + struct macsmc_rtc *rtc = dev_get_drvdata(dev); + u64 ctr = 0, off = 0; + int ret; + + ret = apple_smc_read(rtc->smc, SMC_KEY(CLKM), &ctr, RTC_BYTES); + if (ret != RTC_BYTES) + return ret < 0 ? ret : -EIO; + + /* This sets the offset such that the set second begins now */ + off = (rtc_tm_to_time64(tm) << RTC_SEC_SHIFT) - ctr; + return nvmem_cell_write(rtc->rtc_offset, &off, RTC_BYTES); +} + +static const struct rtc_class_ops macsmc_rtc_ops = { + .read_time = macsmc_rtc_get_time, + .set_time = macsmc_rtc_set_time, +}; + +static int macsmc_rtc_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct macsmc_rtc *rtc; + + /* Ignore devices without this functionality */ + if (!apple_smc_key_exists(smc, SMC_KEY(CLKM))) + return -ENODEV; + + rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL); + if (!rtc) + return -ENOMEM; + + rtc->dev = &pdev->dev; + rtc->smc = smc; + + pdev->dev.of_node = of_get_child_by_name(pdev->dev.parent->of_node, "rtc"); + + rtc->rtc_offset = devm_nvmem_cell_get(&pdev->dev, "rtc_offset"); + if (IS_ERR(rtc->rtc_offset)) + return dev_err_probe(&pdev->dev, PTR_ERR(rtc->rtc_offset), + "Failed to get rtc_offset NVMEM cell\n"); + + rtc->rtc_dev = devm_rtc_allocate_device(&pdev->dev); + if (IS_ERR(rtc->rtc_dev)) + return PTR_ERR(rtc->rtc_dev); + + rtc->rtc_dev->ops = &macsmc_rtc_ops; + rtc->rtc_dev->range_min = S64_MIN >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + rtc->rtc_dev->range_max = S64_MAX >> (RTC_SEC_SHIFT + (64 - RTC_BITS)); + + platform_set_drvdata(pdev, rtc); + + return devm_rtc_register_device(rtc->rtc_dev); +} + +static struct platform_driver macsmc_rtc_driver = { + .driver = { + .name = "macsmc-rtc", + .owner = THIS_MODULE, + }, + .probe = macsmc_rtc_probe, +}; +module_platform_driver(macsmc_rtc_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC RTC driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_ALIAS("platform:macsmc-rtc"); From 6f14ae41f80fbef4aba56d1dfa3242d57280f456 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Mar 2022 19:21:19 +0900 Subject: [PATCH 149/635] Input: macsmc-hid: New driver to handle the Apple Mac SMC buttons/lid This driver implements power button and lid switch support for Apple Mac devices using SMC controllers driven by the macsmc driver. In addition to basic input support, this also responds to the final shutdown warning (when the power button is held down long enough) by doing an emergency kernel poweroff. This allows the NVMe controller to be cleanly shut down, which prevents data loss for in-cache data. Signed-off-by: Hector Martin --- drivers/input/misc/Kconfig | 12 +++ drivers/input/misc/Makefile | 1 + drivers/input/misc/macsmc-hid.c | 157 ++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 drivers/input/misc/macsmc-hid.c diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig index 5c2d0c06d2a531..cce36b75a7330c 100644 --- a/drivers/input/misc/Kconfig +++ b/drivers/input/misc/Kconfig @@ -928,4 +928,16 @@ config INPUT_STPMIC1_ONKEY To compile this driver as a module, choose M here: the module will be called stpmic1_onkey. +config INPUT_MACSMC_HID + tristate "Apple Mac SMC lid/buttons" + depends on APPLE_SMC + default ARCH_APPLE + help + Say Y here if you want to use the input events delivered via the + SMC controller on Apple Mac machines using the macsmc driver. + This includes lid open/close and the power button. + + To compile this driver as a module, choose M here: the + module will be called macsmc-hid. + endif diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile index 61949263300d5a..be9a0afed83f82 100644 --- a/drivers/input/misc/Makefile +++ b/drivers/input/misc/Makefile @@ -48,6 +48,7 @@ obj-$(CONFIG_INPUT_IQS7222) += iqs7222.o obj-$(CONFIG_INPUT_KEYSPAN_REMOTE) += keyspan_remote.o obj-$(CONFIG_INPUT_KXTJ9) += kxtj9.o obj-$(CONFIG_INPUT_M68K_BEEP) += m68kspkr.o +obj-$(CONFIG_INPUT_MACSMC_HID) += macsmc-hid.o obj-$(CONFIG_INPUT_MAX77650_ONKEY) += max77650-onkey.o obj-$(CONFIG_INPUT_MAX77693_HAPTIC) += max77693-haptic.o obj-$(CONFIG_INPUT_MAX8925_ONKEY) += max8925_onkey.o diff --git a/drivers/input/misc/macsmc-hid.c b/drivers/input/misc/macsmc-hid.c new file mode 100644 index 00000000000000..1c0f7476081f30 --- /dev/null +++ b/drivers/input/misc/macsmc-hid.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC input event driver + * Copyright The Asahi Linux Contributors + * + * This driver exposes HID events from the SMC as an input device. + * This includes the lid open/close and power button notifications. + */ + +#include +#include +#include +#include +#include + +struct macsmc_hid { + struct device *dev; + struct apple_smc *smc; + struct input_dev *input; + struct notifier_block nb; +}; + +#define SMC_EV_BTN 0x7201 +#define SMC_EV_LID 0x7203 + +#define BTN_POWER 0x06 +#define BTN_POWER_HELD1 0xfe +#define BTN_POWER_HELD2 0x00 + +static int macsmc_hid_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_hid *smchid = container_of(nb, struct macsmc_hid, nb); + u16 type = event >> 16; + u8 d1 = (event >> 8) & 0xff; + u8 d2 = event & 0xff; + + switch (type) { + case SMC_EV_BTN: + switch (d1) { + case BTN_POWER: + input_report_key(smchid->input, KEY_POWER, d2); + input_sync(smchid->input); + break; + case BTN_POWER_HELD1: + /* + * TODO: is this pre-warning useful? + */ + if (d2) + dev_warn(smchid->dev, "Power button held down\n"); + break; + case BTN_POWER_HELD2: + /* + * If we get here, we have about 4 seconds before forced shutdown. + * Try to do an emergency shutdown to make sure the NVMe cache is + * flushed. macOS actually does this by panicing (!)... + */ + if (d2) { + dev_crit(smchid->dev, "Triggering forced shutdown!\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? */ + kernel_restart("SMC power button triggered restart"); + } + break; + default: + dev_info(smchid->dev, "Unknown SMC button event: %02x %02x\n", d1, d2); + break; + } + return NOTIFY_OK; + case SMC_EV_LID: + input_report_switch(smchid->input, SW_LID, d1); + input_sync(smchid->input); + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + +static int macsmc_hid_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct macsmc_hid *smchid; + bool have_lid, have_power; + int ret; + + have_lid = apple_smc_key_exists(smc, SMC_KEY(MSLD)); + have_power = apple_smc_key_exists(smc, SMC_KEY(bHLD)); + + if (!have_lid && !have_power) + return -ENODEV; + + smchid = devm_kzalloc(&pdev->dev, sizeof(*smchid), GFP_KERNEL); + if (!smchid) + return -ENOMEM; + + smchid->dev = &pdev->dev; + smchid->smc = smc; + + smchid->input = devm_input_allocate_device(&pdev->dev); + if (!smchid->input) + return -ENOMEM; + + smchid->input->phys = "macsmc-hid (0)"; + smchid->input->name = "Apple SMC power/lid events"; + + if (have_lid) + input_set_capability(smchid->input, EV_SW, SW_LID); + if (have_power) + input_set_capability(smchid->input, EV_KEY, KEY_POWER); + + ret = input_register_device(smchid->input); + if (ret) { + dev_err(&pdev->dev, "Failed to register input device: %d\n", ret); + return ret; + } + + if (have_lid) { + u8 val; + + ret = apple_smc_read_u8(smc, SMC_KEY(MSLD), &val); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to read initial lid state\n"); + } else { + input_report_switch(smchid->input, SW_LID, val); + } + } + if (have_power) { + u32 val; + + ret = apple_smc_read_u32(smc, SMC_KEY(bHLD), &val); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to read initial power button state\n"); + } else { + input_report_key(smchid->input, KEY_POWER, val & 1); + } + } + + input_sync(smchid->input); + + smchid->nb.notifier_call = macsmc_hid_event; + apple_smc_register_notifier(smc, &smchid->nb); + + return 0; +} + +static struct platform_driver macsmc_hid_driver = { + .driver = { + .name = "macsmc-hid", + }, + .probe = macsmc_hid_probe, +}; +module_platform_driver(macsmc_hid_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC GPIO driver"); +MODULE_ALIAS("platform:macsmc-hid"); From e6d8296ee79feddd42ae704c1c74579b83562b0d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 5 Mar 2022 01:08:04 +0900 Subject: [PATCH 150/635] Input: macsmc-hid: Support button/lid wakeups Signed-off-by: Hector Martin --- drivers/input/misc/macsmc-hid.c | 41 +++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/drivers/input/misc/macsmc-hid.c b/drivers/input/misc/macsmc-hid.c index 1c0f7476081f30..c4c7440d4ee68c 100644 --- a/drivers/input/misc/macsmc-hid.c +++ b/drivers/input/misc/macsmc-hid.c @@ -18,6 +18,7 @@ struct macsmc_hid { struct apple_smc *smc; struct input_dev *input; struct notifier_block nb; + bool wakeup_mode; }; #define SMC_EV_BTN 0x7201 @@ -38,8 +39,15 @@ static int macsmc_hid_event(struct notifier_block *nb, unsigned long event, void case SMC_EV_BTN: switch (d1) { case BTN_POWER: - input_report_key(smchid->input, KEY_POWER, d2); - input_sync(smchid->input); + if (smchid->wakeup_mode) { + if (d2) { + dev_info(smchid->dev, "Button wakeup\n"); + pm_wakeup_hard_event(smchid->dev); + } + } else { + input_report_key(smchid->input, KEY_POWER, d2); + input_sync(smchid->input); + } break; case BTN_POWER_HELD1: /* @@ -68,6 +76,10 @@ static int macsmc_hid_event(struct notifier_block *nb, unsigned long event, void } return NOTIFY_OK; case SMC_EV_LID: + if (smchid->wakeup_mode && !d1) { + dev_info(smchid->dev, "Lid wakeup\n"); + pm_wakeup_hard_event(smchid->dev); + } input_report_switch(smchid->input, SW_LID, d1); input_sync(smchid->input); return NOTIFY_OK; @@ -95,6 +107,7 @@ static int macsmc_hid_probe(struct platform_device *pdev) smchid->dev = &pdev->dev; smchid->smc = smc; + platform_set_drvdata(pdev, smchid); smchid->input = devm_input_allocate_device(&pdev->dev); if (!smchid->input) @@ -140,12 +153,36 @@ static int macsmc_hid_probe(struct platform_device *pdev) smchid->nb.notifier_call = macsmc_hid_event; apple_smc_register_notifier(smc, &smchid->nb); + device_init_wakeup(&pdev->dev, 1); + + return 0; +} + +static int macsmc_hid_pm_prepare(struct device *dev) +{ + struct macsmc_hid *smchid = dev_get_drvdata(dev); + + smchid->wakeup_mode = true; return 0; } +static void macsmc_hid_pm_complete(struct device *dev) +{ + struct macsmc_hid *smchid = dev_get_drvdata(dev); + + smchid->wakeup_mode = false; +} + +static const struct dev_pm_ops macsmc_hid_pm_ops = { + .prepare = macsmc_hid_pm_prepare, + .complete = macsmc_hid_pm_complete, +}; + static struct platform_driver macsmc_hid_driver = { .driver = { .name = "macsmc-hid", + .owner = THIS_MODULE, + .pm = &macsmc_hid_pm_ops, }, .probe = macsmc_hid_probe, }; From 68777a146973db40e13fc4fb129a832a828b3b64 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 5 May 2022 01:38:19 +0900 Subject: [PATCH 151/635] gpio: macsmc: Add IRQ support Signed-off-by: Hector Martin --- drivers/gpio/gpio-macsmc.c | 150 +++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index ff9950afb69af0..98fc74af69d4c1 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -68,10 +69,21 @@ * 3 = ? */ +#define SMC_EV_GPIO 0x7202 + struct macsmc_gpio { struct device *dev; struct apple_smc *smc; struct gpio_chip gc; + struct irq_chip ic; + struct notifier_block nb; + + struct mutex irq_mutex; + DECLARE_BITMAP(irq_supported, MAX_GPIO); + DECLARE_BITMAP(irq_enable_shadow, MAX_GPIO); + DECLARE_BITMAP(irq_enable, MAX_GPIO); + u32 irq_mode_shadow[MAX_GPIO]; + u32 irq_mode[MAX_GPIO]; int first_index; }; @@ -161,6 +173,7 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, for (i = 0; i < count; i++) { smc_key key; int gpio_nr; + u32 val; int ret = apple_smc_get_key_by_index(smcgp->smc, smcgp->first_index + i, &key); if (ret < 0) @@ -176,11 +189,127 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, } set_bit(gpio_nr, valid_mask); + + /* Check for IRQ support */ + ret = apple_smc_rw_u32(smcgp->smc, key, CMD_IRQ_MODE, &val); + if (!ret) + set_bit(gpio_nr, smcgp->irq_supported); + } + + return 0; +} + +static int macsmc_gpio_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_gpio *smcgp = container_of(nb, struct macsmc_gpio, nb); + u16 type = event >> 16; + u8 offset = (event >> 8) & 0xff; + smc_key key = macsmc_gpio_key(offset); + unsigned long flags; + + if (type != SMC_EV_GPIO) + return NOTIFY_DONE; + + if (offset > MAX_GPIO) { + dev_err(smcgp->dev, "GPIO event index %d out of range\n", offset); + return NOTIFY_BAD; + } + + local_irq_save(flags); + generic_handle_irq_desc(irq_resolve_mapping(smcgp->gc.irq.domain, offset)); + local_irq_restore(flags); + + if (apple_smc_write_u32(smcgp->smc, key, CMD_IRQ_ACK | 1) < 0) + dev_err(smcgp->dev, "GPIO IRQ ack failed for %p4ch\n", &key); + + return NOTIFY_OK; +} + +static void macsmc_gpio_irq_enable(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + + set_bit(irqd_to_hwirq(d), smcgp->irq_enable_shadow); +} + +static void macsmc_gpio_irq_disable(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + + clear_bit(irqd_to_hwirq(d), smcgp->irq_enable_shadow); +} + +static int macsmc_gpio_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + int offset = irqd_to_hwirq(d); + u32 mode; + + if (!test_bit(offset, smcgp->irq_supported)) + return -EINVAL; + + switch (type & IRQ_TYPE_SENSE_MASK) { + case IRQ_TYPE_LEVEL_HIGH: + mode = IRQ_MODE_HIGH; + break; + case IRQ_TYPE_LEVEL_LOW: + mode = IRQ_MODE_LOW; + break; + case IRQ_TYPE_EDGE_RISING: + mode = IRQ_MODE_RISING; + break; + case IRQ_TYPE_EDGE_FALLING: + mode = IRQ_MODE_FALLING; + break; + case IRQ_TYPE_EDGE_BOTH: + mode = IRQ_MODE_BOTH; + break; + default: + return -EINVAL; } + smcgp->irq_mode_shadow[offset] = mode; return 0; } +static void macsmc_gpio_irq_bus_lock(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + + mutex_lock(&smcgp->irq_mutex); +} + +static void macsmc_gpio_irq_bus_sync_unlock(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + smc_key key = macsmc_gpio_key(irqd_to_hwirq(d)); + int offset = irqd_to_hwirq(d); + bool val; + + if (smcgp->irq_mode_shadow[offset] != smcgp->irq_mode[offset]) { + u32 cmd = CMD_IRQ_MODE | smcgp->irq_mode_shadow[offset]; + if (apple_smc_write_u32(smcgp->smc, key, cmd) < 0) + dev_err(smcgp->dev, "GPIO IRQ config failed for %p4ch = 0x%x\n", &key, cmd); + else + smcgp->irq_mode_shadow[offset] = smcgp->irq_mode[offset]; + } + + val = test_bit(offset, smcgp->irq_enable_shadow); + if (test_bit(offset, smcgp->irq_enable) != val) { + if (apple_smc_write_u32(smcgp->smc, key, CMD_IRQ_ENABLE | val) < 0) + dev_err(smcgp->dev, "GPIO IRQ en/disable failed for %p4ch\n", &key); + else + change_bit(offset, smcgp->irq_enable); + } + + mutex_unlock(&smcgp->irq_mutex); +} + static int macsmc_gpio_probe(struct platform_device *pdev) { struct macsmc_gpio *smcgp; @@ -221,6 +350,27 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->gc.base = -1; smcgp->gc.parent = &pdev->dev; + smcgp->ic.name = "macsmc-pmu-gpio"; + smcgp->ic.irq_mask = macsmc_gpio_irq_disable; + smcgp->ic.irq_unmask = macsmc_gpio_irq_enable; + smcgp->ic.irq_set_type = macsmc_gpio_irq_set_type; + smcgp->ic.irq_bus_lock = macsmc_gpio_irq_bus_lock; + smcgp->ic.irq_bus_sync_unlock = macsmc_gpio_irq_bus_sync_unlock; + smcgp->ic.irq_set_type = macsmc_gpio_irq_set_type; + smcgp->ic.flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND; + + smcgp->gc.irq.chip = &smcgp->ic; + smcgp->gc.irq.parent_handler = NULL; + smcgp->gc.irq.num_parents = 0; + smcgp->gc.irq.parents = NULL; + smcgp->gc.irq.default_type = IRQ_TYPE_NONE; + smcgp->gc.irq.handler = handle_simple_irq; + + mutex_init(&smcgp->irq_mutex); + + smcgp->nb.notifier_call = macsmc_gpio_event; + apple_smc_register_notifier(smc, &smcgp->nb); + return devm_gpiochip_add_data(&pdev->dev, &smcgp->gc, smcgp); } From 0cd0a3c8333ec8b2de5d0077e2e7aa55d37b3a06 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 18 Sep 2022 23:06:02 +0900 Subject: [PATCH 152/635] Input: macsmc-hid: Support the power button on desktops Signed-off-by: Hector Martin --- drivers/input/misc/macsmc-hid.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/macsmc-hid.c b/drivers/input/misc/macsmc-hid.c index c4c7440d4ee68c..49296cbb70cc67 100644 --- a/drivers/input/misc/macsmc-hid.c +++ b/drivers/input/misc/macsmc-hid.c @@ -24,7 +24,8 @@ struct macsmc_hid { #define SMC_EV_BTN 0x7201 #define SMC_EV_LID 0x7203 -#define BTN_POWER 0x06 +#define BTN_POWER 0x01 +#define BTN_TOUCHID 0x06 #define BTN_POWER_HELD1 0xfe #define BTN_POWER_HELD2 0x00 @@ -39,6 +40,7 @@ static int macsmc_hid_event(struct notifier_block *nb, unsigned long event, void case SMC_EV_BTN: switch (d1) { case BTN_POWER: + case BTN_TOUCHID: if (smchid->wakeup_mode) { if (d2) { dev_info(smchid->dev, "Button wakeup\n"); From 5226027209097d8148aa7e0483e4f83c92ee9307 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 4 Dec 2022 21:51:01 +0900 Subject: [PATCH 153/635] power: supply: macsmc_power: Add critical level shutdown & misc events Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 123 ++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index dee48ee8b5a455..804e989a3a8c06 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #define MAX_STRING_LENGTH 256 @@ -26,6 +29,9 @@ struct macsmc_power { struct power_supply *ac; struct notifier_block nb; + + struct work_struct critical_work; + bool shutdown_started; }; #define CHNC_BATTERY_FULL BIT(0) @@ -46,6 +52,9 @@ struct macsmc_power { #define CH0X_CH0C BIT(0) #define CH0X_CH0B BIT(1) +#define ACSt_CAN_BOOT_AP BIT(2) +#define ACSt_CAN_BOOT_IBOOT BIT(1) + static int macsmc_battery_get_status(struct macsmc_power *power) { u64 nocharge_flags; @@ -188,6 +197,34 @@ static int macsmc_battery_get_date(const char *s, int *out) return 0; } +static int macsmc_battery_get_capacity_level(struct macsmc_power *power) +{ + u32 val; + int ret; + + /* Check for emergency shutdown condition */ + if (apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val) >= 0 && val) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + /* Check AC status for whether we could boot in this state */ + if (apple_smc_read_u32(power->smc, SMC_KEY(ACSt), &val) >= 0) { + if (!(val & ACSt_CAN_BOOT_IBOOT)) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + if (!(val & ACSt_CAN_BOOT_AP)) + return POWER_SUPPLY_CAPACITY_LEVEL_LOW; + } + + /* Check battery full flag */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC)); + if (ret > 0) + return POWER_SUPPLY_CAPACITY_LEVEL_FULL; + else if (ret == 0) + return POWER_SUPPLY_CAPACITY_LEVEL_NORMAL; + else + return POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN; +} + static int macsmc_battery_get_property(struct power_supply *psy, enum power_supply_property psp, union power_supply_propval *val) @@ -225,6 +262,10 @@ static int macsmc_battery_get_property(struct power_supply *psy, ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); val->intval = vu8; break; + case POWER_SUPPLY_PROP_CAPACITY_LEVEL: + val->intval = macsmc_battery_get_capacity_level(power); + ret = val->intval < 0 ? val->intval : 0; + break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16); val->intval = vu16 * 1000; @@ -344,6 +385,7 @@ static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_CAPACITY_LEVEL, POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_CURRENT_NOW, POWER_SUPPLY_PROP_POWER_NOW, @@ -425,6 +467,59 @@ static const struct power_supply_desc macsmc_ac_desc = { .num_properties = ARRAY_SIZE(macsmc_ac_props), }; +static void macsmc_power_critical_work(struct work_struct *wrk) { + struct macsmc_power *power = container_of(wrk, struct macsmc_power, critical_work); + int ret; + u32 bcf0; + u16 bitv, b0av; + + /* + * Check if the battery voltage is below the design voltage. If it is, + * we have a few seconds until the machine dies. Explicitly shut down, + * which at least gets the NVMe controller to flush its cache. + */ + if (apple_smc_read_u16(power->smc, SMC_KEY(BITV), &bitv) >= 0 && + apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &b0av) >= 0 && + b0av < bitv) { + dev_crit(power->dev, "Emergency notification: Battery is critical\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? In this state, this will not boot anyway. */ + kernel_restart("Battery is critical"); + } + + /* This spams once per second, so make sure we only trigger shutdown once. */ + if (power->shutdown_started) + return; + + /* Check for battery empty condition */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &bcf0); + if (ret < 0) { + dev_err(power->dev, + "Emergency notification: Failed to read battery status\n"); + } else if (bcf0 == 0) { + dev_warn(power->dev, "Emergency notification: Battery status is OK?\n"); + return; + } else { + dev_warn(power->dev, "Emergency notification: Battery is empty\n"); + } + + power->shutdown_started = true; + + /* + * Attempt to trigger an orderly shutdown. At this point, we should have a few + * minutes of reserve capacity left, enough to do a clean shutdown. + */ + dev_warn(power->dev, "Shutting down in 10 seconds\n"); + ssleep(10); + + /* + * Don't force it; if this stalls or fails, the last-resort check above will + * trigger a hard shutdown when shutdown is truly imminent. + */ + orderly_poweroff(false); +} + static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) { struct macsmc_power *power = container_of(nb, struct macsmc_power, nb); @@ -436,6 +531,28 @@ static int macsmc_power_event(struct notifier_block *nb, unsigned long event, vo power_supply_changed(power->batt); power_supply_changed(power->ac); + return NOTIFY_OK; + } else if (event == 0x71020000) { + schedule_work(&power->critical_work); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71060000) { + u8 changed_port = event >> 8; + u8 cur_port; + + /* Port charging state change? */ + if (apple_smc_read_u8(power->smc, SMC_KEY(AC-W), &cur_port) >= 0) { + dev_info(power->dev, "Port %d state change (charge port: %d)\n", + changed_port + 1, cur_port); + } + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xff000000) == 0x71000000) { + dev_info(power->dev, "Unknown charger event 0x%lx\n", event); + return NOTIFY_OK; } @@ -447,6 +564,7 @@ static int macsmc_power_probe(struct platform_device *pdev) struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); struct power_supply_config psy_cfg = {}; struct macsmc_power *power; + u32 val; int ret; power = devm_kzalloc(&pdev->dev, sizeof(*power), GFP_KERNEL); @@ -470,6 +588,9 @@ static int macsmc_power_probe(struct platform_device *pdev) apple_smc_write_u8(power->smc, SMC_KEY(CH0K), 0); apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); + /* Doing one read of this flag enables critical shutdown notifications */ + apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val); + psy_cfg.drv_data = power; power->batt = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); if (IS_ERR(power->batt)) { @@ -488,6 +609,8 @@ static int macsmc_power_probe(struct platform_device *pdev) power->nb.notifier_call = macsmc_power_event; apple_smc_register_notifier(power->smc, &power->nb); + INIT_WORK(&power->critical_work, macsmc_power_critical_work); + return 0; } From 77250320fdd6b5252951dcdc219487c1107a8090 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 12 Dec 2022 23:51:41 +0900 Subject: [PATCH 154/635] platform/apple: smc: Add apple_smc_read_f32_scaled Signed-off-by: Hector Martin --- drivers/platform/apple/smc_core.c | 48 +++++++++++++++++++++++++++++++ include/linux/mfd/macsmc.h | 2 ++ 2 files changed, 50 insertions(+) diff --git a/drivers/platform/apple/smc_core.c b/drivers/platform/apple/smc_core.c index daf029cd072f52..27605398be3b24 100644 --- a/drivers/platform/apple/smc_core.c +++ b/drivers/platform/apple/smc_core.c @@ -4,6 +4,7 @@ * Copyright The Asahi Linux Contributors */ +#include #include #include #include @@ -98,6 +99,53 @@ int apple_smc_rw(struct apple_smc *smc, smc_key key, void *wbuf, size_t wsize, } EXPORT_SYMBOL(apple_smc_rw); +int apple_smc_read_f32_scaled(struct apple_smc *smc, smc_key key, int *p, int scale) +{ + u32 fval; + u64 val; + int ret, exp; + + ret = apple_smc_read_u32(smc, key, &fval); + if (ret < 0) + return ret; + + val = ((u64)((fval & GENMASK(22, 0)) | BIT(23))); + exp = ((fval >> 23) & 0xff) - 127 - 23; + if (scale < 0) { + val <<= 32; + exp -= 32; + val /= -scale; + } else { + val *= scale; + } + + if (exp > 63) + val = U64_MAX; + else if (exp < -63) + val = 0; + else if (exp < 0) + val >>= -exp; + else if (exp != 0 && (val & ~((1UL << (64 - exp)) - 1))) /* overflow */ + val = U64_MAX; + else + val <<= exp; + + if (fval & BIT(31)) { + if (val > (-(s64)INT_MIN)) + *p = INT_MIN; + else + *p = -val; + } else { + if (val > INT_MAX) + *p = INT_MAX; + else + *p = val; + } + + return ret; +} +EXPORT_SYMBOL(apple_smc_read_f32_scaled); + int apple_smc_get_key_by_index(struct apple_smc *smc, int index, smc_key *key) { int ret; diff --git a/include/linux/mfd/macsmc.h b/include/linux/mfd/macsmc.h index 39b4dc4ca88106..30bcac19818c81 100644 --- a/include/linux/mfd/macsmc.h +++ b/include/linux/mfd/macsmc.h @@ -80,6 +80,8 @@ static inline int apple_smc_read_flag(struct apple_smc *smc, smc_key key) } #define apple_smc_write_flag apple_smc_write_u8 +int apple_smc_read_f32_scaled(struct apple_smc *smc, smc_key key, int *p, int scale); + int apple_smc_register_notifier(struct apple_smc *smc, struct notifier_block *n); int apple_smc_unregister_notifier(struct apple_smc *smc, struct notifier_block *n); From 29116cdf239340853b0cc3392564088257d1f4f6 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 12 Dec 2022 23:36:17 +0900 Subject: [PATCH 155/635] power: supply: macsmc_power: Add a debug mode to print power usage Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 81 ++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index 804e989a3a8c06..3b30384c47f7d7 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -32,8 +32,25 @@ struct macsmc_power { struct work_struct critical_work; bool shutdown_started; + + struct delayed_work dbg_log_work; +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp); + +static const struct kernel_param_ops macsmc_log_power_ops = { + .set = macsmc_log_power_set, + .get = param_get_bool, }; +static bool log_power = false; +module_param_cb(log_power, &macsmc_log_power_ops, &log_power, 0644); +MODULE_PARM_DESC(log_power, "Periodically log power consumption for debugging"); + +#define POWER_LOG_INTERVAL (HZ) + +static struct macsmc_power *g_power; + #define CHNC_BATTERY_FULL BIT(0) #define CHNC_NO_CHARGER BIT(7) #define CHNC_NOCHG_CH0C BIT(14) @@ -467,7 +484,58 @@ static const struct power_supply_desc macsmc_ac_desc = { .num_properties = ARRAY_SIZE(macsmc_ac_props), }; -static void macsmc_power_critical_work(struct work_struct *wrk) { +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_bool(val, kp); + + if (ret < 0) + return ret; + + if (log_power && g_power) + schedule_delayed_work(&g_power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_dbg_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(to_delayed_work(wrk), + struct macsmc_power, dbg_log_work); + int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; + s32 p_bat = 0; + s16 t_full = 0, t_empty = 0; + u8 charge = 0; + + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PDTR), &p_in, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSTR), &p_sys, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PMVR), &p_3v8, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PHPC), &p_cpu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSVR), &p_clvr, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPMC), &p_mpmu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPSC), &p_spmu, 1000); + apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &p_bat); + apple_smc_read_s16(power->smc, SMC_KEY(B0TE), &t_empty); + apple_smc_read_s16(power->smc, SMC_KEY(B0TF), &t_full); + apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &charge); + +#define FD3(x) ((x) / 1000), abs((x) % 1000) + + dev_info(power->dev, + "In %2d.%03dW Sys %2d.%03dW 3V8 %2d.%03dW MPMU %2d.%03dW SPMU %2d.%03dW " + "CLVR %2d.%03dW CPU %2d.%03dW Batt %2d.%03dW %d%% T%s %dm\n", + FD3(p_in), FD3(p_sys), FD3(p_3v8), FD3(p_mpmu), FD3(p_spmu), FD3(p_clvr), + FD3(p_cpu), FD3(p_bat), charge, + t_full >= 0 ? "full" : "empty", + t_full >= 0 ? t_full : t_empty); + +#undef FD3 + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, POWER_LOG_INTERVAL); +} + +static void macsmc_power_critical_work(struct work_struct *wrk) +{ struct macsmc_power *power = container_of(wrk, struct macsmc_power, critical_work); int ret; u32 bcf0; @@ -610,6 +678,12 @@ static int macsmc_power_probe(struct platform_device *pdev) apple_smc_register_notifier(power->smc, &power->nb); INIT_WORK(&power->critical_work, macsmc_power_critical_work); + INIT_DELAYED_WORK(&power->dbg_log_work, macsmc_dbg_work); + + g_power = power; + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, 0); return 0; } @@ -618,6 +692,11 @@ static int macsmc_power_remove(struct platform_device *pdev) { struct macsmc_power *power = dev_get_drvdata(&pdev->dev); + cancel_work(&power->critical_work); + cancel_delayed_work(&power->dbg_log_work); + + g_power = NULL; + apple_smc_unregister_notifier(power->smc, &power->nb); return 0; From 5a9520ff6b0e3c937c2c8642e0aa691ca1c12a12 Mon Sep 17 00:00:00 2001 From: Jean-Francois Bortolotti Date: Fri, 4 Feb 2022 00:06:13 +0100 Subject: [PATCH 156/635] spmi: add a first basic spmi driver for Apple SoC Signed-off-by: Jean-Francois Bortolotti --- drivers/spmi/Kconfig | 8 + drivers/spmi/Makefile | 1 + drivers/spmi/spmi-apple-controller.c | 223 +++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 drivers/spmi/spmi-apple-controller.c diff --git a/drivers/spmi/Kconfig b/drivers/spmi/Kconfig index 73780204631463..96c73c5b572022 100644 --- a/drivers/spmi/Kconfig +++ b/drivers/spmi/Kconfig @@ -45,4 +45,12 @@ config SPMI_MTK_PMIF This is required for communicating with Mediatek PMICs and other devices that have the SPMI interface. +config SPMI_APPLE + tristate "Apple SoC SPMI Controller platform driver" + depends on ARCH_APPLE || COMPILE_TEST + help + This enables basic support for the SPMI controller present on + many Apple SoCs, including the t8103 (M1) and t600x + (M1 Pro/Max). + endif diff --git a/drivers/spmi/Makefile b/drivers/spmi/Makefile index 9d974424c8c183..989b84bbca604f 100644 --- a/drivers/spmi/Makefile +++ b/drivers/spmi/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_SPMI) += spmi.o obj-$(CONFIG_SPMI_HISI3670) += hisi-spmi-controller.o obj-$(CONFIG_SPMI_MSM_PMIC_ARB) += spmi-pmic-arb.o obj-$(CONFIG_SPMI_MTK_PMIF) += spmi-mtk-pmif.o +obj-$(CONFIG_SPMI_APPLE) += spmi-apple-controller.o diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c new file mode 100644 index 00000000000000..c14c4874654d15 --- /dev/null +++ b/drivers/spmi/spmi-apple-controller.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Apple SoC SPMI device driver + * + * Copyright The Asahi Linux Contributors + * + * Inspired by: + * OpenBSD support Copyright (c) 2021 Mark Kettenis + * Correllium support Copyright (C) 2021 Corellium LLC + * hisi-spmi-controller.c + * spmi-pmic-ard.c Copyright (c) 2021, The Linux Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* SPMI Controller Registers */ +#define SPMI_STATUS_REG 0 +#define SPMI_CMD_REG 0x4 +#define SPMI_RSP_REG 0x8 + +#define SPMI_RX_FIFO_EMPTY BIT(24) +#define SPMI_TX_FIFO_EMPTY BIT(8) + +/* Apple SPMI controler */ +struct apple_spmi { + void __iomem *regs; + struct spmi_controller *ctrl; +}; + +static inline u32 read_reg(struct apple_spmi *spmi, int offset) +{ + return (readl_relaxed(spmi->regs + offset)); +} + +static inline void write_reg(u32 value, struct apple_spmi *spmi, int offset) +{ + writel_relaxed(value, spmi->regs + offset); +} + +static int spmi_read_cmd(struct spmi_controller *ctrl, u8 opc, u8 slave_id, + u16 slave_addr, u8 *__buf, size_t bc) +{ + struct apple_spmi *spmi; + u32 spmi_cmd = opc | slave_id << 8 | slave_addr << 16 | (bc - 1) | + (1 << 15); + u32 rsp; + volatile u32 status; + size_t len_to_read; + u8 i; + + spmi = spmi_controller_get_drvdata(ctrl); + + write_reg(spmi_cmd, spmi, SPMI_CMD_REG); + + /* Wait for Rx FIFO to have something */ + /* Quite ugly msleep, need to find a better way to do it */ + i = 0; + do { + status = read_reg(spmi, SPMI_STATUS_REG); + msleep(10); + i += 1; + } while ((status & SPMI_RX_FIFO_EMPTY) && i < 5); + + if (i >= 5) { + dev_err(&ctrl->dev, + "spmi_read_cmd:took to long to get the status"); + return -1; + } + + /* Read SPMI reply status */ + rsp = read_reg(spmi, SPMI_RSP_REG); + + len_to_read = 0; + /* Read SPMI data reply */ + while (!(status & SPMI_RX_FIFO_EMPTY) && (len_to_read < bc)) { + rsp = read_reg(spmi, SPMI_RSP_REG); + i = 0; + while ((len_to_read < bc) && (i < 4)) { + __buf[len_to_read++] = ((0xff << (8 * i)) & rsp) >> + (8 * i); + i += 1; + } + } + + return 0; +} + +static int spmi_write_cmd(struct spmi_controller *ctrl, u8 opc, u8 slave_id, + u16 slave_addr, const u8 *__buf, size_t bc) +{ + struct apple_spmi *spmi; + u32 spmi_cmd = opc | slave_id << 8 | slave_addr << 16 | (bc - 1) | + (1 << 15); + volatile u32 rsp; + volatile u32 status; + size_t i = 0, j; + + spmi = spmi_controller_get_drvdata(ctrl); + + write_reg(spmi_cmd, spmi, SPMI_CMD_REG); + + while (i < bc) { + j = 0; + spmi_cmd = 0; + while ((j < 4) & (i < bc)) { + spmi_cmd |= __buf[i++] << (j++ * 8); + } + write_reg(spmi_cmd, spmi, SPMI_CMD_REG); + } + + /* Wait for Rx FIFO to have something */ + /* Quite ugly msleep, need to find a better way to do it */ + i = 0; + do { + status = read_reg(spmi, SPMI_STATUS_REG); + msleep(10); + i += 1; + } while ((status & SPMI_RX_FIFO_EMPTY) && i < 5); + + if (i >= 5) { + dev_err(&ctrl->dev, + "spmi_write_cmd:took to long to get the status"); + return -1; + } + + rsp = read_reg(spmi, SPMI_RSP_REG); + (void)rsp; // TODO: check stuff here + + return 0; +} + +static int spmi_controller_probe(struct platform_device *pdev) +{ + struct apple_spmi *spmi; + struct spmi_controller *ctrl; + int ret; + + ctrl = spmi_controller_alloc(&pdev->dev, sizeof(struct apple_spmi)); + if (IS_ERR(ctrl)) { + dev_err_probe(&pdev->dev, PTR_ERR(ctrl), + "Can't allocate spmi_controller data\n"); + return -ENOMEM; + } + + spmi = spmi_controller_get_drvdata(ctrl); + spmi->ctrl = ctrl; + platform_set_drvdata(pdev, ctrl); + + spmi->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(spmi->regs)) { + dev_err_probe(&pdev->dev, PTR_ERR(spmi->regs), + "Can't get ioremap regs.\n"); + return PTR_ERR(spmi->regs); + } + + ctrl->dev.of_node = of_node_get(pdev->dev.of_node); + + /* Callbacks */ + ctrl->read_cmd = spmi_read_cmd; + ctrl->write_cmd = spmi_write_cmd; + + ret = spmi_controller_add(ctrl); + if (ret) { + dev_err(&pdev->dev, + "spmi_controller_add failed with error %d!\n", ret); + goto err_put_controller; + } + + /* Let's look for other nodes in device tree like the rtc */ + ret = devm_of_platform_populate(&pdev->dev); + if (ret) { + dev_err(&pdev->dev, + "spmi_controller_probe: devm_of_platform_populate failed with error %d!\n", + ret); + goto err_devm_of_platform_populate; + } + + return 0; + +err_put_controller: + spmi_controller_put(ctrl); +err_devm_of_platform_populate: + return ret; +} + +static int spmi_del_controller(struct platform_device *pdev) +{ + struct spmi_controller *ctrl = platform_get_drvdata(pdev); + + spmi_controller_remove(ctrl); + spmi_controller_put(ctrl); + return 0; +} + +static const struct of_device_id spmi_controller_match_table[] = { + { + .compatible = "apple,spmi", + }, + {} +}; +MODULE_DEVICE_TABLE(of, spmi_controller_match_table); + +static struct platform_driver spmi_controller_driver = { + .probe = spmi_controller_probe, + .remove = spmi_del_controller, + .driver = { + .name = "apple-spmi", + .owner = THIS_MODULE, + .of_match_table = spmi_controller_match_table, + }, +}; +module_platform_driver(spmi_controller_driver); + +MODULE_AUTHOR("Jean-Francois Bortolotti "); +MODULE_DESCRIPTION("Apple SoC SPMI driver"); +MODULE_LICENSE("GPL"); From 5bfbed1babeb1c5a450845d50c88fc02c20eb6ff Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 18:43:17 +0900 Subject: [PATCH 157/635] mfd: Add a simple-mfd-spmi driver This is the SPMI counterpart to simple-mfd-i2c. It merely exposes the SPMI register address space as an MFD device, such that different aspects of a device can be managed by separate drivers. Signed-off-by: Hector Martin --- drivers/mfd/Kconfig | 28 ++++++++++++++++++++ drivers/mfd/Makefile | 1 + drivers/mfd/simple-mfd-spmi.c | 49 +++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 drivers/mfd/simple-mfd-spmi.c diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index fcc141e067b9b8..bcf569ca3100a8 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -52,6 +52,21 @@ config MFD_ACT8945A linear regulators, along with a complete ActivePath battery charger. +config MFD_APPLE_SPMI_PMU + tristate "Apple SPMI PMUs" + depends on SPMI + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + select MFD_SIMPLE_MFD_SPMI + help + Say yes here to enable support for Apple PMUs attached via the + SPMI bus. These can be found on Apple devices such as Apple + Silicon Macs. + + This driver itself only attaches to the core device, and relies + on subsystem drivers for individual device functions. You must + enable those for it to be useful. + config MFD_SUN4I_GPADC tristate "Allwinner sunxi platforms' GPADC MFD driver" select MFD_CORE @@ -1249,6 +1264,19 @@ config MFD_SIMPLE_MFD_I2C sub-devices represented by child nodes in Device Tree will be subsequently registered. +config MFD_SIMPLE_MFD_SPMI + tristate + depends on SPMI + select MFD_CORE + select REGMAP_SPMI + help + This driver creates a single register map with the intention for it + to be shared by all sub-devices. + + Once the register map has been successfully initialised, any + sub-devices represented by child nodes in Device Tree will be + subsequently registered. + config MFD_SL28CPLD tristate "Kontron sl28cpld Board Management Controller" depends on I2C diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 2f6c89d1e2777f..3a5d0c4a80c37f 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -261,6 +261,7 @@ obj-$(CONFIG_MFD_QCOM_PM8008) += qcom-pm8008.o obj-$(CONFIG_SGI_MFD_IOC3) += ioc3.o obj-$(CONFIG_MFD_SIMPLE_MFD_I2C) += simple-mfd-i2c.o +obj-$(CONFIG_MFD_SIMPLE_MFD_SPMI) += simple-mfd-spmi.o obj-$(CONFIG_MFD_SMPRO) += smpro-core.o obj-$(CONFIG_MFD_INTEL_M10_BMC_CORE) += intel-m10-bmc-core.o diff --git a/drivers/mfd/simple-mfd-spmi.c b/drivers/mfd/simple-mfd-spmi.c new file mode 100644 index 00000000000000..99f25751000a3b --- /dev/null +++ b/drivers/mfd/simple-mfd-spmi.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Simple MFD - SPMI + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include + +static const struct regmap_config spmi_regmap_config = { + .reg_bits = 16, + .val_bits = 8, + .max_register = 0xffff, +}; + +static int simple_spmi_probe(struct spmi_device *sdev) +{ + struct regmap *regmap; + + regmap = devm_regmap_init_spmi_ext(sdev, &spmi_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + return devm_of_platform_populate(&sdev->dev); +} + +static const struct of_device_id simple_spmi_id_table[] = { + { .compatible = "apple,spmi-pmu" }, + {} +}; +MODULE_DEVICE_TABLE(of, simple_spmi_id_table); + +static struct spmi_driver pmic_spmi_driver = { + .probe = simple_spmi_probe, + .driver = { + .name = "simple-mfd-spmi", + .owner = THIS_MODULE, + .of_match_table = simple_spmi_id_table, + }, +}; +module_spmi_driver(pmic_spmi_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Simple MFD - SPMI driver"); +MODULE_AUTHOR("Hector Martin "); From e587981bdf5ce8fa7ab32e16169249ad324e581d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 18:45:25 +0900 Subject: [PATCH 158/635] nvmem: Add spmi-mfd-nvmem driver This driver exposes part of an SPMI MFD device as an NVMEM device. It is intended to be used with e.g. PMUs/PMICs that are used to hold power-management configuration, such as used on Apple Silicon Macs. Signed-off-by: Hector Martin --- drivers/nvmem/Kconfig | 13 +++++ drivers/nvmem/Makefile | 2 + drivers/nvmem/spmi-mfd-nvmem.c | 98 ++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 drivers/nvmem/spmi-mfd-nvmem.c diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index 6dec38805041bd..978989cc427cb0 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -271,6 +271,19 @@ config NVMEM_SNVS_LPGPR This driver can also be built as a module. If so, the module will be called nvmem-snvs-lpgpr. +config NVMEM_SPMI_MFD + tristate "Generic SPMI MFD NVMEM" + depends on MFD_SIMPLE_MFD_SPMI || COMPILE_TEST + default ARCH_APPLE + help + Say y here to build a generic driver to expose an SPMI MFD device + as a NVMEM provider. This can be used for PMIC/PMU devices which + are used to store power and RTC-related settings on certain + platforms, such as Apple Silicon Macs. + + This driver can also be built as a module. If so, the module + will be called nvmem-spmi-mfd. + config NVMEM_SPMI_SDAM tristate "SPMI SDAM Support" depends on SPMI diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile index 6a1efffa88f074..6c44d28a70bbfa 100644 --- a/drivers/nvmem/Makefile +++ b/drivers/nvmem/Makefile @@ -55,6 +55,8 @@ obj-$(CONFIG_NVMEM_SC27XX_EFUSE) += nvmem-sc27xx-efuse.o nvmem-sc27xx-efuse-y := sc27xx-efuse.o obj-$(CONFIG_NVMEM_SNVS_LPGPR) += nvmem_snvs_lpgpr.o nvmem_snvs_lpgpr-y := snvs_lpgpr.o +obj-$(CONFIG_NVMEM_SPMI_MFD) += nvmem_spmi_mfd.o +nvmem_spmi_mfd-y := spmi-mfd-nvmem.o obj-$(CONFIG_NVMEM_SPMI_SDAM) += nvmem_qcom-spmi-sdam.o nvmem_qcom-spmi-sdam-y += qcom-spmi-sdam.o obj-$(CONFIG_NVMEM_SPRD_EFUSE) += nvmem_sprd_efuse.o diff --git a/drivers/nvmem/spmi-mfd-nvmem.c b/drivers/nvmem/spmi-mfd-nvmem.c new file mode 100644 index 00000000000000..e74ced47e68d85 --- /dev/null +++ b/drivers/nvmem/spmi-mfd-nvmem.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Generic SPMI MFD NVMEM driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +struct spmi_mfd_nvmem { + struct regmap *regmap; + unsigned int base; +}; + +static int spmi_mfd_nvmem_read(void *priv, unsigned int offset, + void *val, size_t bytes) +{ + struct spmi_mfd_nvmem *nvmem = priv; + + return regmap_bulk_read(nvmem->regmap, nvmem->base + offset, val, bytes); +} + +static int spmi_mfd_nvmem_write(void *priv, unsigned int offset, + void *val, size_t bytes) +{ + struct spmi_mfd_nvmem *nvmem = priv; + + return regmap_bulk_write(nvmem->regmap, nvmem->base + offset, val, bytes); +} + +static int spmi_mfd_nvmem_probe(struct platform_device *pdev) +{ + struct spmi_mfd_nvmem *nvmem; + const __be32 *addr; + int len; + struct nvmem_config nvmem_cfg = { + .dev = &pdev->dev, + .name = "spmi_mfd_nvmem", + .id = NVMEM_DEVID_AUTO, + .word_size = 1, + .stride = 1, + .reg_read = spmi_mfd_nvmem_read, + .reg_write = spmi_mfd_nvmem_write, + }; + + nvmem = devm_kzalloc(&pdev->dev, sizeof(*nvmem), GFP_KERNEL); + if (!nvmem) + return -ENOMEM; + + nvmem_cfg.priv = nvmem; + + nvmem->regmap = dev_get_regmap(pdev->dev.parent, NULL); + if (!nvmem->regmap) { + dev_err(&pdev->dev, "Parent regmap unavailable.\n"); + return -ENXIO; + } + + addr = of_get_property(pdev->dev.of_node, "reg", &len); + if (!addr) { + dev_err(&pdev->dev, "no reg property\n"); + return -EINVAL; + } + if (len != 2 * sizeof(u32)) { + dev_err(&pdev->dev, "invalid reg property\n"); + return -EINVAL; + } + + nvmem->base = be32_to_cpup(&addr[0]); + nvmem_cfg.size = be32_to_cpup(&addr[1]); + + return PTR_ERR_OR_ZERO(devm_nvmem_register(&pdev->dev, &nvmem_cfg)); +} + +static const struct of_device_id spmi_mfd_nvmem_id_table[] = { + { .compatible = "apple,spmi-pmu-nvmem" }, + { .compatible = "spmi-mfd-nvmem" }, + { }, +}; +MODULE_DEVICE_TABLE(of, spmi_mfd_nvmem_id_table); + +static struct platform_driver spmi_mfd_nvmem_driver = { + .probe = spmi_mfd_nvmem_probe, + .driver = { + .name = "spmi-mfd-nvmem", + .of_match_table = spmi_mfd_nvmem_id_table, + }, +}; + +module_platform_driver(spmi_mfd_nvmem_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_AUTHOR("Hector Martin "); +MODULE_DESCRIPTION("SPMI MFD NVMEM driver"); From 7a18695c1c29ed61602234fa6ef7a1041e7aedce Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 2 May 2022 21:17:41 +0900 Subject: [PATCH 159/635] dt-bindings: pci: apple,pcie: Add subnode binding, pwren-gpios property We weren't properly validating root port subnodes, so let's do that. Then, also add the new `pwren-gpios` property there to handle device power-up. Signed-off-by: Hector Martin --- .../devicetree/bindings/pci/apple,pcie.yaml | 51 +++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index aa38680aaaca0c..156f1777fc9674 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -71,6 +71,27 @@ properties: power-domains: maxItems: 1 +patternProperties: + "^pci@": + $ref: /schemas/pci/pci-bus.yaml# + type: object + description: A single PCI root port + + properties: + reg: + maxItems: 1 + + pwren-gpios: + description: Optional GPIO to power on the device + maxItems: 1 + + required: + - reset-gpios + - interrupt-controller + - "#interrupt-cells" + - interrupt-map-mask + - interrupt-map + required: - compatible - reg @@ -141,7 +162,7 @@ examples: pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; - pci@0,0 { + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 152 0>; @@ -149,9 +170,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; }; - pci@1,0 { + port01: pci@1,0 { device_type = "pci"; reg = <0x800 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 153 0>; @@ -159,9 +188,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; }; - pci@2,0 { + port02: pci@2,0 { device_type = "pci"; reg = <0x1000 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 33 0>; @@ -169,6 +206,14 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; }; }; }; From 919643e38b5143a492021ac78496d35e4449c748 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 2 May 2022 21:22:46 +0900 Subject: [PATCH 160/635] PCI: apple: Use gpiod_set_value_cansleep in probe flow We're allowed to sleep here, so tell the GPIO core by using gpiod_set_value_cansleep instead of gpiod_set_value. Fixes: 1e33888fbe44 ("PCI: apple: Add initial hardware bring-up") Acked-by: Marc Zyngier Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 66f37e403a09c3..32988fb4c20497 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -541,7 +541,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ - gpiod_set_value(reset, 1); + gpiod_set_value_cansleep(reset, 1); ret = apple_pcie_setup_refclk(pcie, port); if (ret < 0) @@ -552,7 +552,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, /* Deassert PERST# */ rmw_set(PORT_PERST_OFF, port->base + PORT_PERST); - gpiod_set_value(reset, 0); + gpiod_set_value_cansleep(reset, 0); /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ msleep(100); From 1d5c1ce089fe9fd4fa594fc3d96960a41f6f6a29 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 6 Feb 2022 21:15:39 +0900 Subject: [PATCH 161/635] PCI: apple: Probe all GPIOs for availability first If we're probing the PCI controller and some GPIOs are not available and cause a probe defer, we can end up leaving some ports initialized and not others and making a mess. Check for PERST# GPIOs for all ports first, and just return -EPROBE_DEFER if any are not ready yet, without bringing anything up. Fixes: 1e33888fbe44 ("PCI: apple: Add initial hardware bring-up") Cc: stable@vger.kernel.org Acked-by: Marc Zyngier Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 32988fb4c20497..a9cd29a752015d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -507,6 +507,20 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return readl_relaxed(port->base + PORT_RID2SID(idx)); } +static int apple_pcie_probe_port(struct device_node *np) +{ + struct gpio_desc *gd; + + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "reset", 0, + GPIOD_OUT_LOW, "PERST#"); + if (IS_ERR(gd)) { + return PTR_ERR(gd); + } + + gpiod_put(gd); + return 0; +} + static int apple_pcie_setup_port(struct apple_pcie *pcie, struct device_node *np) { @@ -797,8 +811,19 @@ static int apple_pcie_init(struct pci_config_window *cfg) static int apple_pcie_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; + struct device_node *of_port; int ret; + /* Check for probe dependencies for all ports first */ + for_each_available_child_of_node(dev->of_node, of_port) { + ret = apple_pcie_probe_port(of_port); + if (ret) { + of_node_put(of_port); + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", of_port); + } + } + ret = bus_register_notifier(&pci_bus_type, &apple_pcie_nb); if (ret) return ret; From 7ebb9e8fe91741db33ffb083418769dcf397b962 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 6 Feb 2022 21:18:18 +0900 Subject: [PATCH 162/635] PCI: apple: Add support for optional PWREN GPIO WiFi and SD card devices on M1 Macs have a separate power enable GPIO. Add support for this to the PCIe controller. This is modeled after how pcie-fu740 does it. Acked-by: Marc Zyngier Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 34 ++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a9cd29a752015d..45c0327a3345a0 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -518,6 +518,16 @@ static int apple_pcie_probe_port(struct device_node *np) } gpiod_put(gd); + + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "pwren", 0, + GPIOD_OUT_LOW, "PWREN"); + if (IS_ERR(gd)) { + if (PTR_ERR(gd) != -ENOENT) + return PTR_ERR(gd); + } else { + gpiod_put(gd); + } + return 0; } @@ -526,7 +536,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, { struct platform_device *platform = to_platform_device(pcie->dev); struct apple_pcie_port *port; - struct gpio_desc *reset; + struct gpio_desc *reset, *pwren = NULL; u32 stat, idx; int ret, i; @@ -535,6 +545,15 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (IS_ERR(reset)) return PTR_ERR(reset); + pwren = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "pwren", + GPIOD_OUT_LOW, "PWREN"); + if (IS_ERR(pwren)) { + if (PTR_ERR(pwren) == -ENOENT) + pwren = NULL; + else + return PTR_ERR(pwren); + } + port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; @@ -557,12 +576,21 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, /* Assert PERST# before setting up the clock */ gpiod_set_value_cansleep(reset, 1); + /* Power on the device if required */ + gpiod_set_value_cansleep(pwren, 1); + ret = apple_pcie_setup_refclk(pcie, port); if (ret < 0) return ret; - /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ - usleep_range(100, 200); + /* + * The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) + * If powering up, the minimal Tpvperl is 100ms + */ + if (pwren) + msleep(100); + else + usleep_range(100, 200); /* Deassert PERST# */ rmw_set(PORT_PERST_OFF, port->base + PORT_PERST); From b61b03966eb5685e1b280796c2f1bdf0c017186e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 5 Dec 2022 18:13:40 +0900 Subject: [PATCH 163/635] PCI: apple: Fix missing OF node reference in apple_pcie_setup_port In the success path we hang onto a reference to the node, so make sure to grab one. The caller iterator puts our borrowed reference when we return. Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 45c0327a3345a0..84fe85d5c79c4a 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -627,6 +627,9 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, list_add_tail(&port->entry, &pcie->ports); init_completion(&pcie->event); + /* In the success path, we keep a reference to np around */ + of_node_get(np); + ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); From 15f4442897ec9b0d240d447d6fb6a5de511ce5b2 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 13 Feb 2023 21:48:15 +0100 Subject: [PATCH 164/635] PCI: apple: Set only available ports up Fixes "interrupt-map" parsing in of_irq_parse_raw() which takes the node's availability into account. This became apparent after disabling unused PCIe ports in the Apple silicon device trees instead of disabling them. Link: https://lore.kernel.org/asahi/20230214-apple_dts_pcie_disable_unused-v1-0-5ea0d3ddcde3@jannau.net/ Link: https://lore.kernel.org/asahi/1ea2107a-bb86-8c22-0bbc-82c453ab08ce@linaro.org/ Fixes: 1e33888fbe44 ("PCI: apple: Add initial hardware bring-up") Cc: stable@vger.kernel.org Signed-off-by: Janne Grunau --- drivers/pci/controller/pcie-apple.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 84fe85d5c79c4a..eb4ad854fbdf85 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -828,7 +828,7 @@ static int apple_pcie_init(struct pci_config_window *cfg) cfg->priv = pcie; INIT_LIST_HEAD(&pcie->ports); - for_each_child_of_node(dev->of_node, of_port) { + for_each_available_child_of_node(dev->of_node, of_port) { ret = apple_pcie_setup_port(pcie, of_port); if (ret) { dev_err(pcie->dev, "Port %pOF setup fail: %d\n", of_port, ret); From a53c4dcf6228fd950a3046abfe7722ab6b8ea348 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Sat, 11 Mar 2023 14:34:53 +0100 Subject: [PATCH 165/635] PCI: apple: Initialize pcie->nvecs before using it apple_pcie_setup_port computes ilog2(pcie->nvecs) to setup the number of MSIs available for each port. It is however called before apple_msi_init which actually initializes pcie->nvecs. Luckily, pcie->nvecs is part of kzalloc-ed structure and thus initialized as zero. ilog2(0) happens to be 0xffffffff which then just configures more MSIs in hardware than we actually have. This doesn't break anything because we never hand out those vectors. Let's swap the order of the two calls so that we use the correctly initialized value. Fixes: 476c41ed4597 ("PCI: apple: Implement MSI support") Signed-off-by: Sven Peter Reviewed-by: Marc Zyngier Reviewed-by: Eric Curtin Reviewed-by: Alyssa Rosenzweig --- drivers/pci/controller/pcie-apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index eb4ad854fbdf85..7dae19e25712ea 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -828,6 +828,10 @@ static int apple_pcie_init(struct pci_config_window *cfg) cfg->priv = pcie; INIT_LIST_HEAD(&pcie->ports); + ret = apple_msi_init(pcie); + if (ret) + return ret; + for_each_available_child_of_node(dev->of_node, of_port) { ret = apple_pcie_setup_port(pcie, of_port); if (ret) { @@ -837,7 +841,7 @@ static int apple_pcie_init(struct pci_config_window *cfg) } } - return apple_msi_init(pcie); + return 0; } static int apple_pcie_probe(struct platform_device *pdev) From ba88b6475f475aaa34d42b41c7b8943d4bd51e95 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 19:35:27 +0900 Subject: [PATCH 166/635] PCI: apple: Move port PHY registers to their own reg items T602x PCIe cores move these registers around. Instead of hardcoding in another offset, let's move them into their own reg entries. This matches what Apple does on macOS device trees too. Maintains backwards compatibility with old DTs by using the old offsets. Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 51 +++++++++++++++++++---------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 7dae19e25712ea..ca929b91e26dfb 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -40,14 +40,18 @@ #define CORE_RC_STAT_READY BIT(0) #define CORE_FABRIC_STAT 0x04000 #define CORE_FABRIC_STAT_MASK 0x001F001F -#define CORE_LANE_CFG(port) (0x84000 + 0x4000 * (port)) -#define CORE_LANE_CFG_REFCLK0REQ BIT(0) -#define CORE_LANE_CFG_REFCLK1REQ BIT(1) -#define CORE_LANE_CFG_REFCLK0ACK BIT(2) -#define CORE_LANE_CFG_REFCLK1ACK BIT(3) -#define CORE_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) -#define CORE_LANE_CTL(port) (0x84004 + 0x4000 * (port)) -#define CORE_LANE_CTL_CFGACC BIT(15) + +#define CORE_PHY_DEFAULT_BASE(port) (0x84000 + 0x4000 * (port)) + +#define PHY_LANE_CFG 0x00000 +#define PHY_LANE_CFG_REFCLK0REQ BIT(0) +#define PHY_LANE_CFG_REFCLK1REQ BIT(1) +#define PHY_LANE_CFG_REFCLK0ACK BIT(2) +#define PHY_LANE_CFG_REFCLK1ACK BIT(3) +#define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) +#define PHY_LANE_CFG_REFCLKCGEN (BIT(30) | BIT(31)) +#define PHY_LANE_CTL 0x00004 +#define PHY_LANE_CTL_CFGACC BIT(15) #define PORT_LTSSMCTL 0x00080 #define PORT_LTSSMCTL_START BIT(0) @@ -146,6 +150,7 @@ struct apple_pcie_port { struct apple_pcie *pcie; struct device_node *np; void __iomem *base; + void __iomem *phy; struct irq_domain *domain; struct list_head entry; DECLARE_BITMAP(sid_map, MAX_RID2SID); @@ -474,26 +479,26 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, if (res < 0) return res; - rmw_set(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx)); - rmw_set(CORE_LANE_CFG_REFCLK0REQ, pcie->base + CORE_LANE_CFG(port->idx)); + rmw_set(PHY_LANE_CTL_CFGACC, port->phy + PHY_LANE_CTL); + rmw_set(PHY_LANE_CFG_REFCLK0REQ, port->phy + PHY_LANE_CFG); - res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx), - stat, stat & CORE_LANE_CFG_REFCLK0ACK, + res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, + stat, stat & PHY_LANE_CFG_REFCLK0ACK, 100, 50000); if (res < 0) return res; - rmw_set(CORE_LANE_CFG_REFCLK1REQ, pcie->base + CORE_LANE_CFG(port->idx)); - res = readl_relaxed_poll_timeout(pcie->base + CORE_LANE_CFG(port->idx), - stat, stat & CORE_LANE_CFG_REFCLK1ACK, + rmw_set(PHY_LANE_CFG_REFCLK1REQ, port->phy + PHY_LANE_CFG); + res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, + stat, stat & PHY_LANE_CFG_REFCLK1ACK, 100, 50000); if (res < 0) return res; - rmw_clear(CORE_LANE_CTL_CFGACC, pcie->base + CORE_LANE_CTL(port->idx)); + rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + PHY_LANE_CTL); - rmw_set(CORE_LANE_CFG_REFCLKEN, pcie->base + CORE_LANE_CFG(port->idx)); + rmw_set(PHY_LANE_CFG_REFCLKEN, port->phy + PHY_LANE_CFG); rmw_set(PORT_REFCLK_EN, port->base + PORT_REFCLK); return 0; @@ -539,6 +544,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, struct gpio_desc *reset, *pwren = NULL; u32 stat, idx; int ret, i; + char name[16]; reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", GPIOD_OUT_LOW, "PERST#"); @@ -567,9 +573,18 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, port->pcie = pcie; port->np = np; - port->base = devm_platform_ioremap_resource(platform, port->idx + 2); + snprintf(name, sizeof(name), "port%d", port->idx); + port->base = devm_platform_ioremap_resource_byname(platform, name); if (IS_ERR(port->base)) + port->base = devm_platform_ioremap_resource(platform, port->idx + 2); + if (IS_ERR(port->base)) { return PTR_ERR(port->base); + } + + snprintf(name, sizeof(name), "phy%d", port->idx); + port->phy = devm_platform_ioremap_resource_byname(platform, name); + if (IS_ERR(port->phy)) + port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); From 40ffa5b885c74b68997c75ca6fed5424d5221fa8 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 19:37:52 +0900 Subject: [PATCH 167/635] PCI: apple: Drop poll for CORE_RC_PHYIF_STAT_REFCLK This is checking a core refclk in per-port setup which doesn't make a lot of sense, and the bootloader needs to have gone through this anyway. It doesn't work on T602x, so just drop it across the board. --- drivers/pci/controller/pcie-apple.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index ca929b91e26dfb..bb78e3aa2abc60 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -473,12 +473,6 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, u32 stat; int res; - res = readl_relaxed_poll_timeout(pcie->base + CORE_RC_PHYIF_STAT, stat, - stat & CORE_RC_PHYIF_STAT_REFCLK, - 100, 50000); - if (res < 0) - return res; - rmw_set(PHY_LANE_CTL_CFGACC, port->phy + PHY_LANE_CTL); rmw_set(PHY_LANE_CFG_REFCLK0REQ, port->phy + PHY_LANE_CFG); From f5cbf03cdd67d8c68765cde5f8de62feac0230de Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 19:38:37 +0900 Subject: [PATCH 168/635] PCIE: apple: Add T602x PCIe support This version of the hardware moved around a bunch of registers, so we drop the old compatible for these and introduce register offset structures to handle the differences. Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 127 +++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 21 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index bb78e3aa2abc60..d2fa591f3a7f9a 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -105,7 +106,7 @@ #define PORT_REFCLK_CGDIS BIT(8) #define PORT_PERST 0x00814 #define PORT_PERST_OFF BIT(0) -#define PORT_RID2SID(i16) (0x00828 + 4 * (i16)) +#define PORT_RID2SID 0x00828 #define PORT_RID2SID_VALID BIT(31) #define PORT_RID2SID_SID_SHIFT 16 #define PORT_RID2SID_BUS_SHIFT 8 @@ -123,7 +124,7 @@ #define PORT_TUNSTAT_PERST_ACK_PEND BIT(1) #define PORT_PREFMEM_ENABLE 0x00994 -#define MAX_RID2SID 64 +#define MAX_RID2SID 512 /* * The doorbell address is set to 0xfffff000, which by convention @@ -134,6 +135,57 @@ */ #define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR +struct reg_info { + u32 phy_lane_ctl; + u32 port_msiaddr; + u32 port_msiaddr_hi; + u32 port_refclk; + u32 port_perst; + u32 port_rid2sid; + u32 port_msimap; + u32 max_rid2sid; + u32 max_msimap; +}; + +const struct reg_info t8103_hw = { + .phy_lane_ctl = PHY_LANE_CTL, + .port_msiaddr = PORT_MSIADDR, + .port_msiaddr_hi = 0, + .port_refclk = PORT_REFCLK, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .port_msimap = 0, + .max_rid2sid = 64, + .max_msimap = 0, +}; + +#define PORT_T602X_MSIADDR 0x016c +#define PORT_T602X_MSIADDR_HI 0x0170 +#define PORT_T602X_PERST 0x082c +#define PORT_T602X_RID2SID 0x3000 +#define PORT_T602X_MSIMAP 0x3800 + +#define PORT_MSIMAP_ENABLE BIT(31) +#define PORT_MSIMAP_TARGET GENMASK(7, 0) + +const struct reg_info t602x_hw = { + .phy_lane_ctl = 0, + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_refclk = 0, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 512, /* 16 on t602x, guess for autodetect on future HW */ + .max_msimap = 512, /* 96 on t602x, guess for autodetect on future HW */ +}; + +static const struct of_device_id apple_pcie_of_match_hw[] = { + { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, + { .compatible = "apple,pcie", .data = &t8103_hw }, + { } +}; + struct apple_pcie { struct mutex lock; struct device *dev; @@ -144,6 +196,7 @@ struct apple_pcie { struct completion event; struct irq_fwspec fwspec; u32 nvecs; + const struct reg_info *hw; }; struct apple_pcie_port { @@ -267,14 +320,14 @@ static void apple_port_irq_mask(struct irq_data *data) { struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); - writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKSET); + rmw_set(BIT(data->hwirq), port->base + PORT_INTMSK); } static void apple_port_irq_unmask(struct irq_data *data) { struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); - writel_relaxed(BIT(data->hwirq), port->base + PORT_INTMSKCLR); + rmw_clear(BIT(data->hwirq), port->base + PORT_INTMSK); } static bool hwirq_is_intx(unsigned int hwirq) @@ -378,6 +431,7 @@ static void apple_port_irq_handler(struct irq_desc *desc) static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) { struct fwnode_handle *fwnode = &port->np->fwnode; + struct apple_pcie *pcie = port->pcie; unsigned int irq; /* FIXME: consider moving each interrupt under each port */ @@ -393,19 +447,35 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) return -ENOMEM; /* Disable all interrupts */ - writel_relaxed(~0, port->base + PORT_INTMSKSET); + writel_relaxed(~0, port->base + PORT_INTMSK); writel_relaxed(~0, port->base + PORT_INTSTAT); + writel_relaxed(~0, port->base + PORT_LINKCMDSTS); irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port); /* Configure MSI base address */ - BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); - writel_relaxed(lower_32_bits(DOORBELL_ADDR), port->base + PORT_MSIADDR); + BUG_ON(upper_32_bits(DOORBELL_ADDR)); + writel_relaxed(lower_32_bits(DOORBELL_ADDR), + port->base + pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + writel_relaxed(0, port->base + pcie->hw->port_msiaddr_hi); /* Enable MSIs, shared between all ports */ - writel_relaxed(0, port->base + PORT_MSIBASE); - writel_relaxed((ilog2(port->pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT) | - PORT_MSICFG_EN, port->base + PORT_MSICFG); + if (pcie->hw->port_msimap) { + int i; + + for (i = 0; i < pcie->nvecs; i++) { + writel_relaxed(FIELD_PREP(PORT_MSIMAP_TARGET, i) | + PORT_MSIMAP_ENABLE, + port->base + pcie->hw->port_msimap + 4 * i); + } + + writel_relaxed(PORT_MSICFG_EN, port->base + PORT_MSICFG); + } else { + writel_relaxed(0, port->base + PORT_MSIBASE); + writel_relaxed((ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT) | + PORT_MSICFG_EN, port->base + PORT_MSICFG); + } return 0; } @@ -473,7 +543,9 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, u32 stat; int res; - rmw_set(PHY_LANE_CTL_CFGACC, port->phy + PHY_LANE_CTL); + if (pcie->hw->phy_lane_ctl) + rmw_set(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); + rmw_set(PHY_LANE_CFG_REFCLK0REQ, port->phy + PHY_LANE_CFG); res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, @@ -490,10 +562,13 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, if (res < 0) return res; - rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + PHY_LANE_CTL); + if (pcie->hw->phy_lane_ctl) + rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); rmw_set(PHY_LANE_CFG_REFCLKEN, port->phy + PHY_LANE_CFG); - rmw_set(PORT_REFCLK_EN, port->base + PORT_REFCLK); + + if (pcie->hw->port_refclk) + rmw_set(PORT_REFCLK_EN, port->base + pcie->hw->port_refclk); return 0; } @@ -501,9 +576,9 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, int idx, u32 val) { - writel_relaxed(val, port->base + PORT_RID2SID(idx)); + writel_relaxed(val, port->base + port->pcie->hw->port_rid2sid + 4 * idx); /* Read back to ensure completion of the write */ - return readl_relaxed(port->base + PORT_RID2SID(idx)); + return readl_relaxed(port->base + port->pcie->hw->port_rid2sid + 4 * idx); } static int apple_pcie_probe_port(struct device_node *np) @@ -602,7 +677,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, usleep_range(100, 200); /* Deassert PERST# */ - rmw_set(PORT_PERST_OFF, port->base + PORT_PERST); + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); gpiod_set_value_cansleep(reset, 0); /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ @@ -615,15 +690,12 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, return ret; } - rmw_clear(PORT_REFCLK_CGDIS, port->base + PORT_REFCLK); - rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); - ret = apple_pcie_port_setup_irq(port); if (ret) return ret; /* Reset all RID/SID mappings, and check for RAZ/WI registers */ - for (i = 0; i < MAX_RID2SID; i++) { + for (i = 0; i < pcie->hw->max_rid2sid; i++) { if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d) break; apple_pcie_rid2sid_write(port, i, 0); @@ -647,6 +719,12 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + if (pcie->hw->port_refclk) + rmw_clear(PORT_REFCLK_CGDIS, port->base + PORT_REFCLK); + else + rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + return 0; } @@ -763,7 +841,7 @@ static void apple_pcie_release_device(struct apple_pcie_port *port, for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { u32 val; - val = readl_relaxed(port->base + PORT_RID2SID(idx)); + val = readl_relaxed(port->base + port->pcie->hw->port_rid2sid + 4 * idx); if ((val & 0xffff) == rid) { apple_pcie_rid2sid_write(port, idx, 0); bitmap_release_region(port->sid_map, idx, 0); @@ -820,13 +898,19 @@ static int apple_pcie_init(struct pci_config_window *cfg) struct platform_device *platform = to_platform_device(dev); struct device_node *of_port; struct apple_pcie *pcie; + const struct of_device_id *match; int ret; + match = of_match_device(apple_pcie_of_match_hw, dev); + if (!match) + return -ENODEV; + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); if (!pcie) return -ENOMEM; pcie->dev = dev; + pcie->hw = match->data; mutex_init(&pcie->lock); @@ -889,6 +973,7 @@ static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { }; static const struct of_device_id apple_pcie_of_match[] = { + { .compatible = "apple,t6020-pcie", .data = &apple_pcie_cfg_ecam_ops }, { .compatible = "apple,pcie", .data = &apple_pcie_cfg_ecam_ops }, { } }; From 20e07dd06239891ed8dd510b2eddb61d523ecd2c Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 30 Aug 2022 02:09:14 +0900 Subject: [PATCH 169/635] usb: renesas-xhci: Build loader into the main xhci-pci module This doesn't make any sense as a module, since it becomes a symbol dependency of the main driver if enabled, and therefore would always be loaded regardless of the user's hardware. Just compile it in, if enabled. Signed-off-by: Hector Martin --- drivers/usb/host/Kconfig | 4 ++-- drivers/usb/host/Makefile | 3 ++- drivers/usb/host/{xhci-pci.c => xhci-pci-core.c} | 0 drivers/usb/host/xhci-pci-renesas.c | 4 +--- drivers/usb/host/xhci-pci.h | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) rename drivers/usb/host/{xhci-pci.c => xhci-pci-core.c} (100%) diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index eacb603ad1b214..539149f13c9ea2 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -40,11 +40,11 @@ config USB_XHCI_DBGCAP config USB_XHCI_PCI tristate depends on USB_PCI - depends on USB_XHCI_PCI_RENESAS || !USB_XHCI_PCI_RENESAS default y config USB_XHCI_PCI_RENESAS - tristate "Support for additional Renesas xHCI controller with firmware" + bool "Support for Renesas xHCI controllers with firmware" + depends on USB_XHCI_PCI help Say 'Y' to enable the support for the Renesas xHCI controller with firmware. Make sure you have the firmware for the device and diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index 5a13712f367da4..5dada163018962 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -68,7 +68,8 @@ obj-$(CONFIG_USB_UHCI_HCD) += uhci-hcd.o obj-$(CONFIG_USB_FHCI_HCD) += fhci.o obj-$(CONFIG_USB_XHCI_HCD) += xhci-hcd.o obj-$(CONFIG_USB_XHCI_PCI) += xhci-pci.o -obj-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o +xhci-pci-y += xhci-pci-core.o +xhci-pci-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o obj-$(CONFIG_USB_XHCI_RCAR) += xhci-rcar-hcd.o diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci-core.c similarity index 100% rename from drivers/usb/host/xhci-pci.c rename to drivers/usb/host/xhci-pci-core.c diff --git a/drivers/usb/host/xhci-pci-renesas.c b/drivers/usb/host/xhci-pci-renesas.c index 93f8b355bc706e..dddd35e50a2c54 100644 --- a/drivers/usb/host/xhci-pci-renesas.c +++ b/drivers/usb/host/xhci-pci-renesas.c @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -625,6 +624,5 @@ int renesas_xhci_check_request_fw(struct pci_dev *pdev, release_firmware(fw); return err; } -EXPORT_SYMBOL_GPL(renesas_xhci_check_request_fw); -MODULE_LICENSE("GPL v2"); +MODULE_FIRMWARE("renesas_usb_fw.mem"); diff --git a/drivers/usb/host/xhci-pci.h b/drivers/usb/host/xhci-pci.h index cb9a8f331a4463..dff1b99564cc69 100644 --- a/drivers/usb/host/xhci-pci.h +++ b/drivers/usb/host/xhci-pci.h @@ -9,7 +9,7 @@ int renesas_xhci_check_request_fw(struct pci_dev *dev, const struct pci_device_id *id); #else -static int renesas_xhci_check_request_fw(struct pci_dev *dev, +static inline int renesas_xhci_check_request_fw(struct pci_dev *dev, const struct pci_device_id *id) { return 0; From b0363a0319cae11019dddebb74588ba3f1f394fb Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 30 Aug 2022 02:11:48 +0900 Subject: [PATCH 170/635] xhci-pci: asmedia: Add a firmware loader for ASM2214a chips Apple ships ASM2214a ICs in some Apple Silicon hardware (notably, the 2021 iMac and the 2022 Mac Studio) without a flash ROM, and relies on the OS to load firmware on startup. Add support for this to the generic xhci-pci driver. The loader code first checks the firmware version, and only attempts to load firmware if the version isn't the known ROM version. Since this arrangement only exists on Apple machines so far, and Apple are the only source of the (non-redistributable) firmware intended for use on these machines, the firmware is named asmedia/asm2214a-apple.bin. If this style of firmware loading ever becomes necessary on non-Apple machines, we should add a generic firmware name at the time (if it can be part of linux-firmware) or another vendor-specific firmware name. Signed-off-by: Hector Martin --- drivers/usb/host/Kconfig | 9 + drivers/usb/host/Makefile | 1 + drivers/usb/host/xhci-pci-asmedia.c | 312 ++++++++++++++++++++++++++++ drivers/usb/host/xhci-pci-core.c | 24 +++ drivers/usb/host/xhci-pci.h | 13 ++ drivers/usb/host/xhci.h | 1 + 6 files changed, 360 insertions(+) create mode 100644 drivers/usb/host/xhci-pci-asmedia.c diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index 539149f13c9ea2..fe7cba789733b5 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -51,6 +51,15 @@ config USB_XHCI_PCI_RENESAS installed on your system for this device to work. If unsure, say 'N'. +config USB_XHCI_PCI_ASMEDIA + bool "Support for ASMedia xHCI controller with firmware" + default ARCH_APPLE + depends on USB_XHCI_PCI + help + Say 'Y' to enable support for ASMedia xHCI controllers with + host-supplied firmware. These are usually present on Apple devices. + If unsure, say 'N'. + config USB_XHCI_PLATFORM tristate "Generic xHCI driver for a platform device" help diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index 5dada163018962..b3de7f6c68e2c2 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -70,6 +70,7 @@ obj-$(CONFIG_USB_XHCI_HCD) += xhci-hcd.o obj-$(CONFIG_USB_XHCI_PCI) += xhci-pci.o xhci-pci-y += xhci-pci-core.o xhci-pci-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o +xhci-pci-$(CONFIG_USB_XHCI_PCI_ASMEDIA) += xhci-pci-asmedia.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o obj-$(CONFIG_USB_XHCI_RCAR) += xhci-rcar-hcd.o diff --git a/drivers/usb/host/xhci-pci-asmedia.c b/drivers/usb/host/xhci-pci-asmedia.c new file mode 100644 index 00000000000000..b52412e46bfcf4 --- /dev/null +++ b/drivers/usb/host/xhci-pci-asmedia.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * ASMedia xHCI firmware loader + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "xhci.h" +#include "xhci-trace.h" +#include "xhci-pci.h" + +/* Configuration space registers */ +#define ASMT_CFG_CONTROL 0xe0 +#define ASMT_CFG_CONTROL_WRITE BIT(1) +#define ASMT_CFG_CONTROL_READ BIT(0) + +#define ASMT_CFG_SRAM_ADDR 0xe2 + +#define ASMT_CFG_SRAM_ACCESS 0xef +#define ASMT_CFG_SRAM_ACCESS_READ BIT(6) +#define ASMT_CFG_SRAM_ACCESS_ENABLE BIT(7) + +#define ASMT_CFG_DATA_READ0 0xf0 +#define ASMT_CFG_DATA_READ1 0xf4 + +#define ASMT_CFG_DATA_WRITE0 0xf8 +#define ASMT_CFG_DATA_WRITE1 0xfc + +#define ASMT_CMD_GET_FWVER 0x8000060840 +#define ASMT_FWVER_ROM 0x010250090816 + +/* BAR0 registers */ +#define ASMT_REG_ADDR 0x3000 + +#define ASMT_REG_DATA 0x3004 + +#define ASMT_REG_STATUS 0x3009 +#define ASMT_REG_STATUS_BUSY BIT(7) + +#define ASMT_REG_WDATA 0x3010 +#define ASMT_REG_RDATA 0x3018 + +#define TIMEOUT_USEC 10000 +#define RESET_TIMEOUT_USEC 500000 + +static int asmedia_mbox_tx(struct pci_dev *pdev, u64 data) +{ + u8 op; + int i; + + for (i = 0; i < TIMEOUT_USEC; i++) { + pci_read_config_byte(pdev, ASMT_CFG_CONTROL, &op); + if (!(op & ASMT_CFG_CONTROL_WRITE)) + break; + udelay(1); + } + + if (op & ASMT_CFG_CONTROL_WRITE) { + dev_err(&pdev->dev, + "Timed out on mailbox tx: 0x%llx\n", + data); + return -ETIMEDOUT; + } + + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE0, data); + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE1, data >> 32); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, + ASMT_CFG_CONTROL_WRITE); + + return 0; +} + +static int asmedia_mbox_rx(struct pci_dev *pdev, u64 *data) +{ + u8 op; + u32 low, high; + int i; + + for (i = 0; i < TIMEOUT_USEC; i++) { + pci_read_config_byte(pdev, ASMT_CFG_CONTROL, &op); + if (op & ASMT_CFG_CONTROL_READ) + break; + udelay(1); + } + + if (!(op & ASMT_CFG_CONTROL_READ)) { + dev_err(&pdev->dev, "Timed out on mailbox rx\n"); + return -ETIMEDOUT; + } + + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ0, &low); + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ1, &high); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, + ASMT_CFG_CONTROL_READ); + + *data = ((u64)high << 32) | low; + return 0; +} + +static int asmedia_get_fw_version(struct pci_dev *pdev, u64 *version) +{ + int err = 0; + u64 cmd; + + err = asmedia_mbox_tx(pdev, ASMT_CMD_GET_FWVER); + if (err) + return err; + err = asmedia_mbox_tx(pdev, 0); + if (err) + return err; + + err = asmedia_mbox_rx(pdev, &cmd); + if (err) + return err; + err = asmedia_mbox_rx(pdev, version); + if (err) + return err; + + if (cmd != ASMT_CMD_GET_FWVER) { + dev_err(&pdev->dev, "Unexpected reply command 0x%llx\n", cmd); + return -EIO; + } + + return 0; +} + +static bool asmedia_check_firmware(struct pci_dev *pdev) +{ + u64 fwver; + int ret; + + ret = asmedia_get_fw_version(pdev, &fwver); + if (ret) + return ret; + + dev_info(&pdev->dev, "Firmware version: 0x%llx\n", fwver); + + return fwver != ASMT_FWVER_ROM; +} + +static int asmedia_wait_reset(struct pci_dev *pdev) +{ + struct usb_hcd *hcd = dev_get_drvdata(&pdev->dev); + struct xhci_cap_regs __iomem *cap = hcd->regs; + struct xhci_op_regs __iomem *op; + u32 val; + int ret; + + op = hcd->regs + HC_LENGTH(readl(&cap->hc_capbase)); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (!ret) + return 0; + + dev_err(hcd->self.controller, "Reset timed out, trying to kick it\n"); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (ret) + dev_err(hcd->self.controller, "Reset timed out, giving up\n"); + + return ret; +} + +static void asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data) { + void __iomem *regs = hcd->regs; + u8 status; + int ret; + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, + status, !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) + dev_err(hcd->self.controller, + "Write addr timed out ([%04x] = %02x)\n", + addr, data); + + writeb_relaxed(data, regs + ASMT_REG_DATA); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, + status, !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) + dev_err(hcd->self.controller, + "Write data timed out ([%04x] = %02x)\n", + addr, data); +} + +static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) +{ + struct usb_hcd *hcd; + void __iomem *regs; + const u16 *fw_data = (const u16 *)fw->data; + u32 data; + size_t index = 0, addr = 0; + size_t words = fw->size >> 1; + int ret; + + hcd = dev_get_drvdata(&pdev->dev); + regs = hcd->regs; + + asmedia_write_reg(hcd, 0x5040, 2); + asmedia_write_reg(hcd, 0x5042, 1); + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed pre-upload reset\n"); + return ret; + } + + asmedia_write_reg(hcd, 0x500e, 1); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + /* The firmware upload is interleaved in 0x4000 word blocks */ + addr = index = 0; + while (index < words) { + data = fw_data[index]; + if ((index | 0x4000) < words) + data |= fw_data[index | 0x4000] << 16; + + pci_write_config_word(pdev, ASMT_CFG_SRAM_ADDR, + addr); + + writel_relaxed(data, regs + ASMT_REG_WDATA); + + if (++index & 0x4000) + index += 0x4000; + addr += 2; + } + + asmedia_write_reg(hcd, 0x5040, 3); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + asmedia_write_reg(hcd, 0x500e, 0); + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed post-upload reset\n"); + return ret; + } + + return 0; +} + +int asmedia_xhci_check_request_fw(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct xhci_driver_data *driver_data = + (struct xhci_driver_data *)id->driver_data; + const char *fw_name = driver_data->firmware; + const struct firmware *fw; + int ret; + + /* Check if device has firmware, if so skip everything */ + ret = asmedia_check_firmware(pdev); + if (ret < 0) + return ret; + else if (ret == 1) + return 0; + + pci_dev_get(pdev); + ret = request_firmware(&fw, fw_name, &pdev->dev); + pci_dev_put(pdev); + if (ret) { + dev_err(&pdev->dev, "Could not load firmware %s: %d\n", + fw_name, ret); + return ret; + } + + ret = asmedia_load_fw(pdev, fw); + if (ret) { + dev_err(&pdev->dev, "Firmware upload failed: %d\n", ret); + goto err; + } + + ret = asmedia_check_firmware(pdev); + if (ret < 0) { + goto err; + } else if (ret != 1) { + dev_err(&pdev->dev, "Firmware version is too old after upload\n"); + ret = -EIO; + } else { + ret = 0; + } + +err: + release_firmware(fw); + return ret; +} diff --git a/drivers/usb/host/xhci-pci-core.c b/drivers/usb/host/xhci-pci-core.c index 6db07ca419c317..a1cb26b42abded 100644 --- a/drivers/usb/host/xhci-pci-core.c +++ b/drivers/usb/host/xhci-pci-core.c @@ -395,6 +395,18 @@ static int xhci_pci_setup(struct usb_hcd *hcd) struct xhci_hcd *xhci; struct pci_dev *pdev = to_pci_dev(hcd->self.controller); int retval; + struct xhci_driver_data *driver_data; + const struct pci_device_id *id; + + id = pci_match_id(to_pci_driver(pdev->dev.driver)->id_table, pdev); + if (id && id->driver_data && usb_hcd_is_primary_hcd(hcd)) { + driver_data = (struct xhci_driver_data *)id->driver_data; + if (driver_data->quirks & XHCI_ASMEDIA_FW_QUIRK) { + retval = asmedia_xhci_check_request_fw(pdev, id); + if (retval < 0) + return retval; + } + } xhci = hcd_to_xhci(hcd); if (!xhci->sbrn) @@ -738,6 +750,11 @@ static const struct xhci_driver_data reneses_data = { .firmware = "renesas_usb_fw.mem", }; +static const struct xhci_driver_data asmedia_data = { + .quirks = XHCI_ASMEDIA_FW_QUIRK, + .firmware = "asmedia/asm2214a-apple.bin", +}; + /* PCI driver selection metadata; PCI hotplugging uses this */ static const struct pci_device_id pci_ids[] = { { PCI_DEVICE(0x1912, 0x0014), @@ -746,6 +763,9 @@ static const struct pci_device_id pci_ids[] = { { PCI_DEVICE(0x1912, 0x0015), .driver_data = (unsigned long)&reneses_data, }, + { PCI_DEVICE(0x1b21, 0x2142), + .driver_data = (unsigned long)&asmedia_data, + }, /* handle any USB 3.0 xHCI controller */ { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0), }, @@ -761,6 +781,10 @@ MODULE_DEVICE_TABLE(pci, pci_ids); MODULE_FIRMWARE("renesas_usb_fw.mem"); #endif +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +MODULE_FIRMWARE("asmedia/asm2214a-apple.bin"); +#endif + /* pci driver glue; this is a "new style" PCI driver module */ static struct pci_driver xhci_pci_driver = { .name = hcd_name, diff --git a/drivers/usb/host/xhci-pci.h b/drivers/usb/host/xhci-pci.h index dff1b99564cc69..279c95acc43f6d 100644 --- a/drivers/usb/host/xhci-pci.h +++ b/drivers/usb/host/xhci-pci.h @@ -17,6 +17,19 @@ static inline int renesas_xhci_check_request_fw(struct pci_dev *dev, #endif +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id); + +#else +static inline int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id) +{ + return 0; +} + +#endif + struct xhci_driver_data { u64 quirks; const char *firmware; diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 786002bb35db0f..221c325e187322 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1905,6 +1905,7 @@ struct xhci_hcd { #define XHCI_EP_CTX_BROKEN_DCS BIT_ULL(42) #define XHCI_SUSPEND_RESUME_CLKS BIT_ULL(43) #define XHCI_RESET_TO_DEFAULT BIT_ULL(44) +#define XHCI_ASMEDIA_FW_QUIRK BIT_ULL(45) unsigned int num_active_eps; unsigned int limit_active_eps; From 63d03032561d227d1f994a4c77bb2905fac74139 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 29 Oct 2022 00:16:53 +0300 Subject: [PATCH 171/635] dt-bindings: pwm: Add Apple PWM controller Apple SoCs such as the M1 contain a PWM controller used among other things to control the keyboard backlight. Signed-off-by: Sasha Finkelstein --- .../bindings/pwm/apple,s5l-fpwm.yaml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml new file mode 100644 index 00000000000000..142157bff0cd85 --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pwm/apple,s5l-fpwm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple FPWM controller + +maintainers: + - asahi@lists.linux.dev + - Sasha Finkelstein + +description: PWM controller used for keyboard backlight on ARM Macs + +properties: + compatible: + items: + - enum: + - apple,t8103-fpwm + - apple,t6000-fpwm + - apple,t8112-fpwm + - const: apple,s5l-fpwm + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + power-domains: + maxItems: 1 + + "#pwm-cells": + const: 2 + +required: + - compatible + - reg + - clocks + +additionalProperties: false + +examples: + - | + pwm@235044000 { + compatible = "apple,t8103-fpwm", "apple,s5l-fpwm"; + reg = <0x35044000 0x4000>; + power-domains = <&ps_fpwm1>; + clocks = <&clkref>; + #pwm-cells = <2>; + }; From 8f2c06bffb0881543b3e3390d430ae0e0b5f11fa Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 29 Oct 2022 00:16:53 +0300 Subject: [PATCH 172/635] pwm: Add Apple PWM controller Adds the Apple PWM controller driver. Signed-off-by: Sasha Finkelstein --- drivers/pwm/Kconfig | 12 ++++ drivers/pwm/Makefile | 1 + drivers/pwm/pwm-apple.c | 128 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 drivers/pwm/pwm-apple.c diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig index dae023d783a225..8df861b1f4a30b 100644 --- a/drivers/pwm/Kconfig +++ b/drivers/pwm/Kconfig @@ -51,6 +51,18 @@ config PWM_AB8500 To compile this driver as a module, choose M here: the module will be called pwm-ab8500. +config PWM_APPLE + tristate "Apple SoC PWM support" + depends on ARCH_APPLE || COMPILE_TEST + help + Generic PWM framework driver for PWM controller present on + Apple SoCs + + Say Y here if you have an ARM Apple laptop, otherwise say N + + To compile this driver as a module, choose M here: the module + will be called pwm-apple. + config PWM_ATMEL tristate "Atmel PWM support" depends on ARCH_AT91 || COMPILE_TEST diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile index 7bf1a29f02b843..19899b912e0096 100644 --- a/drivers/pwm/Makefile +++ b/drivers/pwm/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_PWM) += core.o obj-$(CONFIG_PWM_SYSFS) += sysfs.o obj-$(CONFIG_PWM_AB8500) += pwm-ab8500.o +obj-$(CONFIG_PWM_APPLE) += pwm-apple.o obj-$(CONFIG_PWM_ATMEL) += pwm-atmel.o obj-$(CONFIG_PWM_ATMEL_HLCDC_PWM) += pwm-atmel-hlcdc.o obj-$(CONFIG_PWM_ATMEL_TCB) += pwm-atmel-tcb.o diff --git a/drivers/pwm/pwm-apple.c b/drivers/pwm/pwm-apple.c new file mode 100644 index 00000000000000..ac4367123a8596 --- /dev/null +++ b/drivers/pwm/pwm-apple.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Driver for the Apple SoC PWM controller + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PWM_CONTROL 0x00 +#define PWM_ON_CYCLES 0x1c +#define PWM_OFF_CYCLES 0x18 + +#define CTRL_ENABLE BIT(0) +#define CTRL_MODE BIT(2) +#define CTRL_UPDATE BIT(5) +#define CTRL_TRIGGER BIT(9) +#define CTRL_INVERT BIT(10) +#define CTRL_OUTPUT_ENABLE BIT(14) + +struct apple_pwm { + struct pwm_chip chip; + void __iomem *base; + u64 clkrate; +}; + +static int apple_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, + const struct pwm_state *state) +{ + struct apple_pwm *fpwm; + u64 on_cycles, off_cycles; + + fpwm = container_of(chip, struct apple_pwm, chip); + if (state->enabled) { + on_cycles = mul_u64_u64_div_u64(fpwm->clkrate, + state->duty_cycle, NSEC_PER_SEC); + off_cycles = mul_u64_u64_div_u64(fpwm->clkrate, + state->period, NSEC_PER_SEC) - on_cycles; + writel(on_cycles, fpwm->base + PWM_ON_CYCLES); + writel(off_cycles, fpwm->base + PWM_OFF_CYCLES); + writel(CTRL_ENABLE | CTRL_OUTPUT_ENABLE | CTRL_UPDATE, + fpwm->base + PWM_CONTROL); + } else { + writel(0, fpwm->base + PWM_CONTROL); + } + return 0; +} + +static int apple_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, + struct pwm_state *state) +{ + struct apple_pwm *fpwm; + u32 on_cycles, off_cycles, ctrl; + + fpwm = container_of(chip, struct apple_pwm, chip); + + ctrl = readl(fpwm->base + PWM_CONTROL); + on_cycles = readl(fpwm->base + PWM_ON_CYCLES); + off_cycles = readl(fpwm->base + PWM_OFF_CYCLES); + + state->enabled = (ctrl & CTRL_ENABLE) && (ctrl & CTRL_OUTPUT_ENABLE); + state->polarity = PWM_POLARITY_NORMAL; + state->duty_cycle = div_u64(on_cycles, fpwm->clkrate) * NSEC_PER_SEC; + state->period = div_u64(off_cycles + on_cycles, fpwm->clkrate) * NSEC_PER_SEC; + + return 0; +} + +static const struct pwm_ops apple_pwm_ops = { + .apply = apple_pwm_apply, + .get_state = apple_pwm_get_state, + .owner = THIS_MODULE, +}; + +static int apple_pwm_probe(struct platform_device *pdev) +{ + struct apple_pwm *pwm; + struct clk *clk; + int ret; + + pwm = devm_kzalloc(&pdev->dev, sizeof(*pwm), GFP_KERNEL); + if (!pwm) + return -ENOMEM; + + pwm->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pwm->base)) + return PTR_ERR(pwm->base); + + platform_set_drvdata(pdev, pwm); + + clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + pwm->clkrate = clk_get_rate(clk); + pwm->chip.dev = &pdev->dev; + pwm->chip.npwm = 1; + pwm->chip.ops = &apple_pwm_ops; + + ret = devm_pwmchip_add(&pdev->dev, &pwm->chip); + return ret; +} + +static const struct of_device_id apple_pwm_of_match[] = { + { .compatible = "apple,s5l-fpwm" }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_pwm_of_match); + +static struct platform_driver apple_pwm_driver = { + .probe = apple_pwm_probe, + .driver = { + .name = "apple-pwm", + .of_match_table = apple_pwm_of_match, + }, +}; +module_platform_driver(apple_pwm_driver); + +MODULE_DESCRIPTION("Apple SoC PWM driver"); +MODULE_LICENSE("Dual MIT/GPL"); From e527518cf3e0d069c0701fe3c6ae86eb8e2b0046 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 30 Nov 2022 22:11:02 +0100 Subject: [PATCH 173/635] nvmem: allow bit offset > 8 Signed-off-by: Sven Peter --- drivers/nvmem/core.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 22024b830788f8..453922a19c7df6 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -1382,15 +1382,23 @@ EXPORT_SYMBOL_GPL(nvmem_cell_put); static void nvmem_shift_read_buffer_in_place(struct nvmem_cell_entry *cell, void *buf) { u8 *p, *b; - int i, extra, bit_offset = cell->bit_offset; + int i, padding, extra, bit_offset = cell->bit_offset; + int bytes = cell->bytes; p = b = buf; if (bit_offset) { + padding = bit_offset/8; + if (padding) { + memmove(buf, buf + padding, bytes - padding); + bit_offset -= BITS_PER_BYTE * padding; + bytes -= padding; + } + /* First shift */ *b++ >>= bit_offset; /* setup rest of the bytes if any */ - for (i = 1; i < cell->bytes; i++) { + for (i = 1; i < bytes; i++) { /* Get bits from next byte and shift them towards msb */ *p |= *b << (BITS_PER_BYTE - bit_offset); @@ -1403,7 +1411,7 @@ static void nvmem_shift_read_buffer_in_place(struct nvmem_cell_entry *cell, void } /* result fits in less bytes */ - extra = cell->bytes - DIV_ROUND_UP(cell->nbits, BITS_PER_BYTE); + extra = bytes - DIV_ROUND_UP(cell->nbits, BITS_PER_BYTE); while (--extra >= 0) *p-- = 0; From 6215beaa44271a51f73787ad5b2bd7a05e11e0c1 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 30 Nov 2022 22:11:04 +0100 Subject: [PATCH 174/635] nvmem: round up to word_size Signed-off-by: Sven Peter --- drivers/nvmem/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 453922a19c7df6..1cfa440db633a9 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -471,8 +471,8 @@ static int nvmem_cell_info_to_nvmem_cell_entry_nodup(struct nvmem_device *nvmem, cell->np = info->np; if (cell->nbits) - cell->bytes = DIV_ROUND_UP(cell->nbits + cell->bit_offset, - BITS_PER_BYTE); + cell->bytes = round_up(DIV_ROUND_UP(cell->nbits + cell->bit_offset, + BITS_PER_BYTE), nvmem->word_size); if (!IS_ALIGNED(cell->offset, nvmem->stride)) { dev_err(&nvmem->dev, From 42ffc8978cdb02ee0131e1315bd7548f92f09690 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 30 Nov 2022 22:11:05 +0100 Subject: [PATCH 175/635] WIP: phy: apple: Add Apple Type-C PHY Signed-off-by: Sven Peter --- drivers/phy/Kconfig | 1 + drivers/phy/Makefile | 1 + drivers/phy/apple/Kconfig | 11 + drivers/phy/apple/Makefile | 6 + drivers/phy/apple/atc.c | 2404 ++++++++++++++++++++++++++++++++++++ drivers/phy/apple/atc.h | 139 +++ drivers/phy/apple/trace.c | 4 + drivers/phy/apple/trace.h | 147 +++ 8 files changed, 2713 insertions(+) create mode 100644 drivers/phy/apple/Kconfig create mode 100644 drivers/phy/apple/Makefile create mode 100644 drivers/phy/apple/atc.c create mode 100644 drivers/phy/apple/atc.h create mode 100644 drivers/phy/apple/trace.c create mode 100644 drivers/phy/apple/trace.h diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig index 7bd00a11d074b4..512d49cf476674 100644 --- a/drivers/phy/Kconfig +++ b/drivers/phy/Kconfig @@ -73,6 +73,7 @@ config PHY_CAN_TRANSCEIVER source "drivers/phy/allwinner/Kconfig" source "drivers/phy/amlogic/Kconfig" +source "drivers/phy/apple/Kconfig" source "drivers/phy/broadcom/Kconfig" source "drivers/phy/cadence/Kconfig" source "drivers/phy/freescale/Kconfig" diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile index 54f312c10a408a..a1ba5a1675b9bf 100644 --- a/drivers/phy/Makefile +++ b/drivers/phy/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_PHY_PISTACHIO_USB) += phy-pistachio-usb.o obj-$(CONFIG_USB_LGM_PHY) += phy-lgm-usb.o obj-y += allwinner/ \ amlogic/ \ + apple/ \ broadcom/ \ cadence/ \ freescale/ \ diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig new file mode 100644 index 00000000000000..090d8542651f86 --- /dev/null +++ b/drivers/phy/apple/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +config PHY_APPLE_ATC + tristate "Apple Type-C PHY" + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + select GENERIC_PHY + select TYPEC + help + Enable this to add support for the Apple Type-C PHY, switch + and mux found in Apple SoCs such as the M1. + This driver currently provides support for USB2 and USB3. diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile new file mode 100644 index 00000000000000..af863fa299dc5f --- /dev/null +++ b/drivers/phy/apple/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +CFLAGS_trace.o := -I$(src) + +obj-$(CONFIG_PHY_APPLE_ATC) += phy-apple-atc.o +phy-apple-atc-y := atc.o +phy-apple-atc-$(CONFIG_TRACING) += trace.o diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c new file mode 100644 index 00000000000000..8397227ae13251 --- /dev/null +++ b/drivers/phy/apple/atc.c @@ -0,0 +1,2404 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple Type-C PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#include "atc.h" +#include "trace.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define rcdev_to_apple_atcphy(_rcdev) \ + container_of(_rcdev, struct apple_atcphy, rcdev) + +#define AUSPLL_APB_CMD_OVERRIDE 0x2000 +#define AUSPLL_APB_CMD_OVERRIDE_REQ BIT(0) +#define AUSPLL_APB_CMD_OVERRIDE_ACK BIT(1) +#define AUSPLL_APB_CMD_OVERRIDE_UNK28 BIT(28) +#define AUSPLL_APB_CMD_OVERRIDE_CMD GENMASK(27, 3) + +#define AUSPLL_FREQ_DESC_A 0x2080 +#define AUSPLL_FD_FREQ_COUNT_TARGET GENMASK(9, 0) +#define AUSPLL_FD_FBDIVN_HALF BIT(10) +#define AUSPLL_FD_REV_DIVN GENMASK(13, 11) +#define AUSPLL_FD_KI_MAN GENMASK(17, 14) +#define AUSPLL_FD_KI_EXP GENMASK(21, 18) +#define AUSPLL_FD_KP_MAN GENMASK(25, 22) +#define AUSPLL_FD_KP_EXP GENMASK(29, 26) +#define AUSPLL_FD_KPKI_SCALE_HBW GENMASK(31, 30) + +#define AUSPLL_FREQ_DESC_B 0x2084 +#define AUSPLL_FD_FBDIVN_FRAC_DEN GENMASK(13, 0) +#define AUSPLL_FD_FBDIVN_FRAC_NUM GENMASK(27, 14) + +#define AUSPLL_FREQ_DESC_C 0x2088 +#define AUSPLL_FD_SDM_SSC_STEP GENMASK(7, 0) +#define AUSPLL_FD_SDM_SSC_EN BIT(8) +#define AUSPLL_FD_PCLK_DIV_SEL GENMASK(13, 9) +#define AUSPLL_FD_LFSDM_DIV GENMASK(15, 14) +#define AUSPLL_FD_LFCLK_CTRL GENMASK(19, 16) +#define AUSPLL_FD_VCLK_OP_DIVN GENMASK(21, 20) +#define AUSPLL_FD_VCLK_PRE_DIVN BIT(22) + +#define AUSPLL_DCO_EFUSE_SPARE 0x222c +#define AUSPLL_RODCO_ENCAP_EFUSE GENMASK(10, 9) +#define AUSPLL_RODCO_BIAS_ADJUST_EFUSE GENMASK(14, 12) + +#define AUSPLL_FRACN_CAN 0x22a4 +#define AUSPLL_DLL_START_CAPCODE GENMASK(18, 17) + +#define AUSPLL_CLKOUT_MASTER 0x2200 +#define AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN BIT(2) +#define AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN BIT(4) +#define AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN BIT(6) + +#define AUSPLL_CLKOUT_DIV 0x2208 +#define AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI GENMASK(20, 16) + +#define AUSPLL_BGR 0x2214 +#define AUSPLL_BGR_CTRL_AVAIL BIT(0) + +#define AUSPLL_CLKOUT_DTC_VREG 0x2220 +#define AUSPLL_DTC_VREG_ADJUST GENMASK(16, 14) +#define AUSPLL_DTC_VREG_BYPASS BIT(7) + +#define AUSPLL_FREQ_CFG 0x2224 +#define AUSPLL_FREQ_REFCLK GENMASK(1, 0) + +#define AUS_COMMON_SHIM_BLK_VREG 0x0a04 +#define AUS_VREG_TRIM GENMASK(6, 2) + +#define CIO3PLL_CLK_CTRL 0x2a00 +#define CIO3PLL_CLK_PCLK_EN BIT(1) +#define CIO3PLL_CLK_REFCLK_EN BIT(5) + +#define CIO3PLL_DCO_NCTRL 0x2a38 +#define CIO3PLL_DCO_COARSEBIN_EFUSE0 GENMASK(6, 0) +#define CIO3PLL_DCO_COARSEBIN_EFUSE1 GENMASK(23, 17) + +#define CIO3PLL_FRACN_CAN 0x2aa4 +#define CIO3PLL_DLL_CAL_START_CAPCODE GENMASK(18, 17) + +#define CIO3PLL_DTC_VREG 0x2a20 +#define CIO3PLL_DTC_VREG_ADJUST GENMASK(16, 14) + +#define ACIOPHY_CROSSBAR 0x4c +#define ACIOPHY_CROSSBAR_PROTOCOL GENMASK(4, 0) +#define ACIOPHY_CROSSBAR_PROTOCOL_USB4 0x0 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED 0x1 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3 0xa +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED 0xb +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP 0x10 +#define ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED 0x11 +#define ACIOPHY_CROSSBAR_PROTOCOL_DP 0x14 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA GENMASK(16, 5) +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE 0x0000 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK100 0x100 +#define ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008 0x008 +#define ACIOPHY_CROSSBAR_DP_BOTH_PMA BIT(17) + +#define ACIOPHY_LANE_MODE 0x48 +#define ACIOPHY_LANE_MODE_RX0 GENMASK(2, 0) +#define ACIOPHY_LANE_MODE_TX0 GENMASK(5, 3) +#define ACIOPHY_LANE_MODE_RX1 GENMASK(8, 6) +#define ACIOPHY_LANE_MODE_TX1 GENMASK(11, 9) +#define ACIOPHY_LANE_MODE_USB4 0 +#define ACIOPHY_LANE_MODE_USB3 1 +#define ACIOPHY_LANE_MODE_DP 2 +#define ACIOPHY_LANE_MODE_OFF 3 + +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1 0x84 +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN BIT(27) +#define ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN BIT(28) + +#define ACIOPHY_TOP_BIST_OV_CFG 0x8c +#define ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV BIT(13) +#define ACIOPHY_TOP_BIST_OV_CFG_LN0_PWR_DOWN_OV BIT(25) + +#define ACIOPHY_TOP_BIST_READ_CTRL 0x90 +#define ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE BIT(2) + +#define ACIOPHY_TOP_PHY_STAT 0x9c +#define ACIOPHY_TOP_PHY_STAT_LN0_UNK0 BIT(0) +#define ACIOPHY_TOP_PHY_STAT_LN0_UNK23 BIT(23) + +#define ACIOPHY_TOP_BIST_PHY_CFG0 0xa8 +#define ACIOPHY_TOP_BIST_PHY_CFG0_LN0_RESET_N BIT(0) + +#define ACIOPHY_TOP_BIST_PHY_CFG1 0xac +#define ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN GENMASK(13, 10) + +#define ACIOPHY_PLL_COMMON_CTRL 0x1028 +#define ACIOPHY_PLL_WAIT_FOR_CMN_READY_BEFORE_RESET_EXIT BIT(24) + +#define ATCPHY_POWER_CTRL 0x20000 +#define ATCPHY_POWER_STAT 0x20004 +#define ATCPHY_POWER_SLEEP_SMALL BIT(0) +#define ATCPHY_POWER_SLEEP_BIG BIT(1) +#define ATCPHY_POWER_CLAMP_EN BIT(2) +#define ATCPHY_POWER_APB_RESET_N BIT(3) +#define ATCPHY_POWER_PHY_RESET_N BIT(4) + +#define ATCPHY_MISC 0x20008 +#define ATCPHY_MISC_RESET_N BIT(0) +#define ATCPHY_MISC_LANE_SWAP BIT(2) + +#define ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0 0x7000 +#define DP_PMA_BYTECLK_RESET BIT(0) +#define DP_MAC_DIV20_CLK_SEL BIT(1) +#define DPTXPHY_PMA_LANE_RESET_N BIT(2) +#define DPTXPHY_PMA_LANE_RESET_N_OV BIT(3) +#define DPTX_PCLK1_SELECT GENMASK(6, 4) +#define DPTX_PCLK2_SELECT GENMASK(9, 7) +#define DPRX_PCLK_SELECT GENMASK(12, 10) +#define DPTX_PCLK1_ENABLE BIT(13) +#define DPTX_PCLK2_ENABLE BIT(14) +#define DPRX_PCLK_ENABLE BIT(15) + +#define ACIOPHY_DP_PCLK_STAT 0x7044 +#define ACIOPHY_AUSPLL_LOCK BIT(3) + +#define LN0_AUSPMA_RX_TOP 0x9000 +#define LN0_AUSPMA_RX_EQ 0xA000 +#define LN0_AUSPMA_RX_SHM 0xB000 +#define LN0_AUSPMA_TX_TOP 0xC000 +#define LN0_AUSPMA_TX_SHM 0xD000 + +#define LN1_AUSPMA_RX_TOP 0x10000 +#define LN1_AUSPMA_RX_EQ 0x11000 +#define LN1_AUSPMA_RX_SHM 0x12000 +#define LN1_AUSPMA_TX_TOP 0x13000 +#define LN1_AUSPMA_TX_SHM 0x14000 + +#define LN_AUSPMA_RX_TOP_PMAFSM 0x0010 +#define LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV BIT(0) +#define LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ BIT(9) + +#define LN_AUSPMA_RX_TOP_TJ_CFG_RX_TXMODE 0x00F0 +#define LN_RX_TXMODE BIT(0) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0 0x00 +#define LN_TX_CLK_EN BIT(20) +#define LN_TX_CLK_EN_OV BIT(21) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1 0x04 +#define LN_RX_DIV20_RESET_N_OV BIT(29) +#define LN_RX_DIV20_RESET_N BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL2 0x08 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL3 0x0C +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL4 0x10 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL5 0x14 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL6 0x18 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL7 0x1C +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL8 0x20 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL9 0x24 +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10 0x28 +#define LN_DTVREG_ADJUST GENMASK(31, 27) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11 0x2C +#define LN_DTVREG_BIG_EN BIT(23) +#define LN_DTVREG_BIG_EN_OV BIT(24) +#define LN_DTVREG_SML_EN BIT(25) +#define LN_DTVREG_SML_EN_OV BIT(26) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12 0x30 +#define LN_TX_BYTECLK_RESET_SYNC_CLR BIT(22) +#define LN_TX_BYTECLK_RESET_SYNC_CLR_OV BIT(23) +#define LN_TX_BYTECLK_RESET_SYNC_EN BIT(24) +#define LN_TX_BYTECLK_RESET_SYNC_EN_OV BIT(25) +#define LN_TX_HRCLK_SEL BIT(28) +#define LN_TX_HRCLK_SEL_OV BIT(29) +#define LN_TX_PBIAS_EN BIT(30) +#define LN_TX_PBIAS_EN_OV BIT(31) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13 0x34 +#define LN_TX_PRE_EN BIT(0) +#define LN_TX_PRE_EN_OV BIT(1) +#define LN_TX_PST1_EN BIT(2) +#define LN_TX_PST1_EN_OV BIT(3) +#define LN_DTVREG_ADJUST_OV BIT(15) + +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL14A 0x38 +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL14B 0x3C +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL15A 0x40 +#define LN_AUSPMA_RX_SHM_TJ_UNK_CTRL15B 0x44 +#define LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16 0x48 +#define LN_RXTERM_EN BIT(21) +#define LN_RXTERM_EN_OV BIT(22) +#define LN_RXTERM_PULLUP_LEAK_EN BIT(23) +#define LN_RXTERM_PULLUP_LEAK_EN_OV BIT(24) +#define LN_TX_CAL_CODE GENMASK(29, 25) +#define LN_TX_CAL_CODE_OV BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17 0x4C +#define LN_TX_MARGIN GENMASK(19, 15) +#define LN_TX_MARGIN_OV BIT(20) +#define LN_TX_MARGIN_LSB BIT(21) +#define LN_TX_MARGIN_LSB_OV BIT(22) +#define LN_TX_MARGIN_P1 GENMASK(26, 23) +#define LN_TX_MARGIN_P1_OV BIT(27) +#define LN_TX_MARGIN_P1_LSB GENMASK(29, 28) +#define LN_TX_MARGIN_P1_LSB_OV BIT(30) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18 0x50 +#define LN_TX_P1_CODE GENMASK(3, 0) +#define LN_TX_P1_CODE_OV BIT(4) +#define LN_TX_P1_LSB_CODE GENMASK(6, 5) +#define LN_TX_P1_LSB_CODE_OV BIT(7) +#define LN_TX_MARGIN_PRE GENMASK(10, 8) +#define LN_TX_MARGIN_PRE_OV BIT(11) +#define LN_TX_MARGIN_PRE_LSB GENMASK(13, 12) +#define LN_TX_MARGIN_PRE_LSB_OV BIT(14) +#define LN_TX_PRE_LSB_CODE GENMASK(16, 15) +#define LN_TX_PRE_LSB_CODE_OV BIT(17) +#define LN_TX_PRE_CODE GENMASK(21, 18) +#define LN_TX_PRE_CODE_OV BIT(22) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19 0x54 +#define LN_TX_TEST_EN BIT(21) +#define LN_TX_TEST_EN_OV BIT(22) +#define LN_TX_EN BIT(23) +#define LN_TX_EN_OV BIT(24) +#define LN_TX_CLK_DLY_CTRL_TAPGEN GENMASK(27, 25) +#define LN_TX_CLK_DIV2_EN BIT(28) +#define LN_TX_CLK_DIV2_EN_OV BIT(29) +#define LN_TX_CLK_DIV2_RST BIT(30) +#define LN_TX_CLK_DIV2_RST_OV BIT(31) + +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL20 0x58 +#define LN_AUSPMA_RX_SHM_TJ_RXA_UNK_CTRL21 0x5C +#define LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22 0x60 +#define LN_VREF_ADJUST_GRAY GENMASK(11, 7) +#define LN_VREF_ADJUST_GRAY_OV BIT(12) +#define LN_VREF_BIAS_SEL GENMASK(14, 13) +#define LN_VREF_BIAS_SEL_OV BIT(15) +#define LN_VREF_BOOST_EN BIT(16) +#define LN_VREF_BOOST_EN_OV BIT(17) +#define LN_VREF_EN BIT(18) +#define LN_VREF_EN_OV BIT(19) +#define LN_VREF_LPBKIN_DATA GENMASK(29, 28) +#define LN_VREF_TEST_RXLPBKDT_EN BIT(30) +#define LN_VREF_TEST_RXLPBKDT_EN_OV BIT(31) + +#define LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0 0x00 +#define LN_BYTECLK_RESET_SYNC_EN_OV BIT(2) +#define LN_BYTECLK_RESET_SYNC_EN BIT(3) +#define LN_BYTECLK_RESET_SYNC_CLR_OV BIT(4) +#define LN_BYTECLK_RESET_SYNC_CLR BIT(5) +#define LN_BYTECLK_RESET_SYNC_SEL_OV BIT(6) + +#define LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1 0x04 +#define LN_TXA_DIV2_EN_OV BIT(8) +#define LN_TXA_DIV2_EN BIT(9) +#define LN_TXA_DIV2_RESET_OV BIT(10) +#define LN_TXA_DIV2_RESET BIT(11) +#define LN_TXA_CLK_EN_OV BIT(22) +#define LN_TXA_CLK_EN BIT(23) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG0 0x08 +#define LN_TXA_CAL_CTRL_OV BIT(0) +#define LN_TXA_CAL_CTRL GENMASK(18, 1) +#define LN_TXA_CAL_CTRL_BASE_OV BIT(19) +#define LN_TXA_CAL_CTRL_BASE GENMASK(23, 20) +#define LN_TXA_HIZ_OV BIT(29) +#define LN_TXA_HIZ BIT(30) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG1 0x0C +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG2 0x10 +#define LN_TXA_MARGIN_OV BIT(0) +#define LN_TXA_MARGIN GENMASK(18, 1) +#define LN_TXA_MARGIN_2R_OV BIT(19) +#define LN_TXA_MARGIN_2R BIT(20) + +#define LN_AUSPMA_TX_SHM_TXA_IMP_REG3 0x14 +#define LN_TXA_MARGIN_POST_OV BIT(0) +#define LN_TXA_MARGIN_POST GENMASK(10, 1) +#define LN_TXA_MARGIN_POST_2R_OV BIT(11) +#define LN_TXA_MARGIN_POST_2R BIT(12) +#define LN_TXA_MARGIN_POST_4R_OV BIT(13) +#define LN_TXA_MARGIN_POST_4R BIT(14) +#define LN_TXA_MARGIN_PRE_OV BIT(15) +#define LN_TXA_MARGIN_PRE GENMASK(21, 16) +#define LN_TXA_MARGIN_PRE_2R_OV BIT(22) +#define LN_TXA_MARGIN_PRE_2R BIT(23) +#define LN_TXA_MARGIN_PRE_4R_OV BIT(24) +#define LN_TXA_MARGIN_PRE_4R BIT(25) + +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG0 0x18 +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG1 0x1C +#define LN_AUSPMA_TX_SHM_TXA_UNK_REG2 0x20 + +#define LN_AUSPMA_TX_SHM_TXA_LDOCLK 0x24 +#define LN_LDOCLK_BYPASS_SML_OV BIT(8) +#define LN_LDOCLK_BYPASS_SML BIT(9) +#define LN_LDOCLK_BYPASS_BIG_OV BIT(10) +#define LN_LDOCLK_BYPASS_BIG BIT(11) +#define LN_LDOCLK_EN_SML_OV BIT(12) +#define LN_LDOCLK_EN_SML BIT(13) +#define LN_LDOCLK_EN_BIG_OV BIT(14) +#define LN_LDOCLK_EN_BIG BIT(15) + +/* LPDPTX registers */ +#define LPDPTX_AUX_CFG_BLK_AUX_CTRL 0x0000 +#define LPDPTX_BLK_AUX_CTRL_PWRDN BIT(4) +#define LPDPTX_BLK_AUX_RXOFFSET GENMASK(25, 22) + +#define LPDPTX_AUX_CFG_BLK_AUX_LDO_CTRL 0x0008 + +#define LPDPTX_AUX_CFG_BLK_AUX_MARGIN 0x000c +#define LPDPTX_MARGIN_RCAL_RXOFFSET_EN BIT(5) +#define LPDPTX_AUX_MARGIN_RCAL_TXSWING GENMASK(10, 6) + +#define LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG0 0x0204 +#define LPDPTX_CFG_PMA_AUX_SEL_LF_DATA BIT(15) + +#define LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1 0x0208 +#define LPDPTX_CFG_PMA_PHYS_ADJ GENMASK(22, 20) +#define LPDPTX_CFG_PMA_PHYS_ADJ_OV BIT(19) + +#define LPDPTX_AUX_CONTROL 0x4000 +#define LPDPTX_AUX_PWN_DOWN 0x10 +#define LPDPTX_AUX_CLAMP_EN 0x04 +#define LPDPTX_SLEEP_B_BIG_IN 0x02 +#define LPDPTX_SLEEP_B_SML_IN 0x01 +#define LPDPTX_TXTERM_CODEMSB 0x400 +#define LPDPTX_TXTERM_CODE GENMASK(9, 5) + +/* pipehandler registers */ +#define PIPEHANDLER_OVERRIDE 0x00 +#define PIPEHANDLER_OVERRIDE_RXVALID BIT(0) +#define PIPEHANDLER_OVERRIDE_RXDETECT BIT(2) + +#define PIPEHANDLER_OVERRIDE_VALUES 0x04 + +#define PIPEHANDLER_MUX_CTRL 0x0c +#define PIPEHANDLER_MUX_MODE GENMASK(1, 0) +#define PIPEHANDLER_MUX_MODE_USB3PHY 0 +#define PIPEHANDLER_MUX_MODE_DUMMY_PHY 2 +#define PIPEHANDLER_CLK_SELECT GENMASK(5, 3) +#define PIPEHANDLER_CLK_USB3PHY 1 +#define PIPEHANDLER_CLK_DUMMY_PHY 4 +#define PIPEHANDLER_LOCK_REQ 0x10 +#define PIPEHANDLER_LOCK_ACK 0x14 +#define PIPEHANDLER_LOCK_EN BIT(0) + +#define PIPEHANDLER_AON_GEN 0x1C +#define PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN BIT(4) +#define PIPEHANDLER_AON_GEN_DWC3_RESET_N BIT(0) + +#define PIPEHANDLER_NONSELECTED_OVERRIDE 0x20 +#define PIPEHANDLER_NONSELECTED_NATIVE_RESET BIT(12) +#define PIPEHANDLER_DUMMY_PHY_EN BIT(15) +#define PIPEHANDLER_NONSELECTED_NATIVE_POWER_DOWN GENMASK(3, 0) + +/* USB2 PHY regs */ +#define USB2PHY_USBCTL 0x00 +#define USB2PHY_USBCTL_HOST_EN BIT(1) + +#define USB2PHY_CTL 0x04 +#define USB2PHY_CTL_RESET BIT(0) +#define USB2PHY_CTL_PORT_RESET BIT(1) +#define USB2PHY_CTL_APB_RESET_N BIT(2) +#define USB2PHY_CTL_SIDDQ BIT(3) + +#define USB2PHY_SIG 0x08 +#define USB2PHY_SIG_VBUSDET_FORCE_VAL BIT(0) +#define USB2PHY_SIG_VBUSDET_FORCE_EN BIT(1) +#define USB2PHY_SIG_VBUSVLDEXT_FORCE_VAL BIT(2) +#define USB2PHY_SIG_VBUSVLDEXT_FORCE_EN BIT(3) +#define USB2PHY_SIG_HOST (7 << 12) + +static const struct { + const struct atcphy_mode_configuration normal; + const struct atcphy_mode_configuration swapped; + bool enable_dp_aux; + enum atcphy_pipehandler_state pipehandler_state; +} atcphy_modes[] = { + [APPLE_ATCPHY_MODE_OFF] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, /* doesn't matter since the SS lanes are off */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB2, + }, + [APPLE_ATCPHY_MODE_USB2] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, /* doesn't matter since the SS lanes are off */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB2, + }, + [APPLE_ATCPHY_MODE_USB3] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB3, ACIOPHY_LANE_MODE_OFF}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_OFF, ACIOPHY_LANE_MODE_USB3}, + .dp_lane = {false, false}, + .set_swap = true, + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB3, + }, + [APPLE_ATCPHY_MODE_USB3_DP] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB3, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {false, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB3_DP_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_USB3}, + .dp_lane = {true, false}, + .set_swap = true, + }, + .enable_dp_aux = true, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB3, + }, + [APPLE_ATCPHY_MODE_USB4] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_USB4_SWAPPED, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_NONE, + .crossbar_dp_both_pma = false, + .lane_mode = {ACIOPHY_LANE_MODE_USB4, ACIOPHY_LANE_MODE_USB4}, + .dp_lane = {false, false}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = false, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB2, + }, + [APPLE_ATCPHY_MODE_DP] = { + .normal = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK100, + .crossbar_dp_both_pma = true, + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {true, true}, + .set_swap = false, + }, + .swapped = { + .crossbar = ACIOPHY_CROSSBAR_PROTOCOL_DP, + .crossbar_dp_single_pma = ACIOPHY_CROSSBAR_DP_SINGLE_PMA_UNK008, + .crossbar_dp_both_pma = false, /* intentionally false */ + .lane_mode = {ACIOPHY_LANE_MODE_DP, ACIOPHY_LANE_MODE_DP}, + .dp_lane = {true, true}, + .set_swap = false, /* intentionally false */ + }, + .enable_dp_aux = true, + .pipehandler_state = ATCPHY_PIPEHANDLER_STATE_USB2, + }, +}; + +static const struct atcphy_dp_link_rate_configuration dp_lr_config[] = { + [ATCPHY_DP_LINK_RATE_RBR] = { + .freqinit_count_target = 0x21c, + .fbdivn_frac_den = 0x0, + .fbdivn_frac_num = 0x0, + .pclk_div_sel = 0x13, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x2, + .plla_clkout_vreg_bypass = true, + .bypass_txa_ldoclk = true, + .txa_div2_en = true, + }, + [ATCPHY_DP_LINK_RATE_HBR] = { + .freqinit_count_target = 0x1c2, + .fbdivn_frac_den = 0x3ffe, + .fbdivn_frac_num = 0x1fff, + .pclk_div_sel = 0x9, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x2, + .plla_clkout_vreg_bypass = true, + .bypass_txa_ldoclk = true, + .txa_div2_en = false, + }, + [ATCPHY_DP_LINK_RATE_HBR2] = { + .freqinit_count_target = 0x1c2, + .fbdivn_frac_den = 0x3ffe, + .fbdivn_frac_num = 0x1fff, + .pclk_div_sel = 0x4, + .lfclk_ctrl = 0x5, + .vclk_op_divn = 0x0, + .plla_clkout_vreg_bypass = true, + .bypass_txa_ldoclk = true, + .txa_div2_en = false, + }, + [ATCPHY_DP_LINK_RATE_HBR3] = { + .freqinit_count_target = 0x2a3, + .fbdivn_frac_den = 0x3ffc, + .fbdivn_frac_num = 0x2ffd, + .pclk_div_sel = 0x4, + .lfclk_ctrl = 0x6, + .vclk_op_divn = 0x0, + .plla_clkout_vreg_bypass = false, + .bypass_txa_ldoclk = false, + .txa_div2_en = false, + }, +}; + +static inline void mask32(void __iomem *reg, u32 mask, u32 set) +{ + u32 value = readl(reg); + value &= ~mask; + value |= set; + writel(value, reg); +} + +static inline void core_mask32(struct apple_atcphy *atcphy, u32 reg, u32 mask, + u32 set) +{ + mask32(atcphy->regs.core + reg, mask, set); +} + +static inline void set32(void __iomem *reg, u32 set) +{ + mask32(reg, 0, set); +} + +static inline void core_set32(struct apple_atcphy *atcphy, u32 reg, u32 set) +{ + core_mask32(atcphy, reg, 0, set); +} + +static inline void clear32(void __iomem *reg, u32 clear) +{ + mask32(reg, clear, 0); +} + +static inline void core_clear32(struct apple_atcphy *atcphy, u32 reg, u32 clear) +{ + core_mask32(atcphy, reg, clear, 0); +} + +static void atcphy_apply_tunable(struct apple_atcphy *atcphy, + void __iomem *regs, + struct atcphy_tunable *tunable) +{ + size_t i; + + for (i = 0; i < tunable->sz; ++i) + mask32(regs + tunable->values[i].offset, + tunable->values[i].mask, tunable->values[i].value); +} + +static void atcphy_apply_tunables(struct apple_atcphy *atcphy, + enum atcphy_mode mode) +{ + int lane0 = atcphy->swap_lanes ? 1 : 0; + int lane1 = atcphy->swap_lanes ? 0 : 1; + + atcphy_apply_tunable(atcphy, atcphy->regs.axi2af, + &atcphy->tunables.axi2af); + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.common); + + switch (mode) { + case APPLE_ATCPHY_MODE_USB3: + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_usb3[lane0]); + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_usb3[lane1]); + break; + + case APPLE_ATCPHY_MODE_USB3_DP: + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_usb3[lane0]); + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_displayport[lane1]); + break; + + case APPLE_ATCPHY_MODE_DP: + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_displayport[lane0]); + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_displayport[lane1]); + break; + + case APPLE_ATCPHY_MODE_USB4: + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_usb4[lane0]); + atcphy_apply_tunable(atcphy, atcphy->regs.core, + &atcphy->tunables.lane_usb4[lane1]); + break; + + default: + dev_warn(atcphy->dev, + "Unknown mode %d in atcphy_apply_tunables\n", mode); + fallthrough; + case APPLE_ATCPHY_MODE_OFF: + case APPLE_ATCPHY_MODE_USB2: + break; + } +} + +static void atcphy_setup_pll_fuses(struct apple_atcphy *atcphy) +{ + void __iomem *regs = atcphy->regs.core; + + if (!atcphy->fuses.present) + return; + + /* CIO3PLL fuses */ + mask32(regs + CIO3PLL_DCO_NCTRL, CIO3PLL_DCO_COARSEBIN_EFUSE0, + FIELD_PREP(CIO3PLL_DCO_COARSEBIN_EFUSE0, + atcphy->fuses.cio3pll_dco_coarsebin[0])); + mask32(regs + CIO3PLL_DCO_NCTRL, CIO3PLL_DCO_COARSEBIN_EFUSE1, + FIELD_PREP(CIO3PLL_DCO_COARSEBIN_EFUSE1, + atcphy->fuses.cio3pll_dco_coarsebin[1])); + mask32(regs + CIO3PLL_FRACN_CAN, CIO3PLL_DLL_CAL_START_CAPCODE, + FIELD_PREP(CIO3PLL_DLL_CAL_START_CAPCODE, + atcphy->fuses.cio3pll_dll_start_capcode[0])); + + if (atcphy->quirks.t8103_cio3pll_workaround) { + mask32(regs + AUS_COMMON_SHIM_BLK_VREG, AUS_VREG_TRIM, + FIELD_PREP(AUS_VREG_TRIM, + atcphy->fuses.aus_cmn_shm_vreg_trim)); + mask32(regs + CIO3PLL_FRACN_CAN, CIO3PLL_DLL_CAL_START_CAPCODE, + FIELD_PREP(CIO3PLL_DLL_CAL_START_CAPCODE, + atcphy->fuses.cio3pll_dll_start_capcode[1])); + mask32(regs + CIO3PLL_DTC_VREG, CIO3PLL_DTC_VREG_ADJUST, + FIELD_PREP(CIO3PLL_DTC_VREG_ADJUST, + atcphy->fuses.cio3pll_dtc_vreg_adjust)); + } else { + mask32(regs + CIO3PLL_DTC_VREG, CIO3PLL_DTC_VREG_ADJUST, + FIELD_PREP(CIO3PLL_DTC_VREG_ADJUST, + atcphy->fuses.cio3pll_dtc_vreg_adjust)); + mask32(regs + AUS_COMMON_SHIM_BLK_VREG, AUS_VREG_TRIM, + FIELD_PREP(AUS_VREG_TRIM, + atcphy->fuses.aus_cmn_shm_vreg_trim)); + } + + /* AUSPLL fuses */ + mask32(regs + AUSPLL_DCO_EFUSE_SPARE, AUSPLL_RODCO_ENCAP_EFUSE, + FIELD_PREP(AUSPLL_RODCO_ENCAP_EFUSE, + atcphy->fuses.auspll_rodco_encap)); + mask32(regs + AUSPLL_DCO_EFUSE_SPARE, AUSPLL_RODCO_BIAS_ADJUST_EFUSE, + FIELD_PREP(AUSPLL_RODCO_BIAS_ADJUST_EFUSE, + atcphy->fuses.auspll_rodco_bias_adjust)); + mask32(regs + AUSPLL_FRACN_CAN, AUSPLL_DLL_START_CAPCODE, + FIELD_PREP(AUSPLL_DLL_START_CAPCODE, + atcphy->fuses.auspll_fracn_dll_start_capcode)); + mask32(regs + AUSPLL_CLKOUT_DTC_VREG, AUSPLL_DTC_VREG_ADJUST, + FIELD_PREP(AUSPLL_DTC_VREG_ADJUST, + atcphy->fuses.auspll_dtc_vreg_adjust)); + + /* TODO: is this actually required again? */ + mask32(regs + AUS_COMMON_SHIM_BLK_VREG, AUS_VREG_TRIM, + FIELD_PREP(AUS_VREG_TRIM, atcphy->fuses.aus_cmn_shm_vreg_trim)); +} + +static int atcphy_cio_power_off(struct apple_atcphy *atcphy) +{ + u32 reg; + int ret; + + /* enable all reset lines */ + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_APB_RESET_N); + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_CLAMP_EN); + core_clear32(atcphy, ATCPHY_MISC, ATCPHY_MISC_RESET_N); + + // TODO: why clear? is this SLEEP_N? or do we enable some power management here? + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_BIG); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + !(reg & ATCPHY_POWER_SLEEP_BIG), 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to sleep atcphy \"big\"\n"); + return ret; + } + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_SMALL); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + !(reg & ATCPHY_POWER_SLEEP_SMALL), 100, + 100000); + if (ret) { + dev_err(atcphy->dev, "failed to sleep atcphy \"small\"\n"); + return ret; + } + + return 0; +} + +static int atcphy_cio_power_on(struct apple_atcphy *atcphy) +{ + u32 reg; + int ret; + + core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_RESET_N); + + // TODO: why set?! see above + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_SMALL); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + reg & ATCPHY_POWER_SLEEP_SMALL, 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to wakeup atcphy \"small\"\n"); + return ret; + } + + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_SLEEP_BIG); + ret = readl_poll_timeout(atcphy->regs.core + ATCPHY_POWER_STAT, reg, + reg & ATCPHY_POWER_SLEEP_BIG, 100, 100000); + if (ret) { + dev_err(atcphy->dev, "failed to wakeup atcphy \"big\"\n"); + return ret; + } + + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_CLAMP_EN); + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_APB_RESET_N); + + return 0; +} + +static void atcphy_configure_lanes(struct apple_atcphy *atcphy, + enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *mode_cfg; + + if (atcphy->swap_lanes) + mode_cfg = &atcphy_modes[mode].swapped; + else + mode_cfg = &atcphy_modes[mode].normal; + + trace_atcphy_configure_lanes(mode, mode_cfg); + + if (mode_cfg->set_swap) + core_set32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); + else + core_clear32(atcphy, ATCPHY_MISC, ATCPHY_MISC_LANE_SWAP); + + if (mode_cfg->dp_lane[0]) { + core_set32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + core_clear32(atcphy, + LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ); + } + if (mode_cfg->dp_lane[1]) { + core_set32(atcphy, LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); + core_clear32(atcphy, + LN1_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, + LN_AUSPMA_RX_TOP_PMAFSM_PCS_REQ); + } + + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_RX0, + FIELD_PREP(ACIOPHY_LANE_MODE_RX0, mode_cfg->lane_mode[0])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_TX0, + FIELD_PREP(ACIOPHY_LANE_MODE_TX0, mode_cfg->lane_mode[0])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_RX1, + FIELD_PREP(ACIOPHY_LANE_MODE_RX1, mode_cfg->lane_mode[1])); + core_mask32(atcphy, ACIOPHY_LANE_MODE, ACIOPHY_LANE_MODE_TX1, + FIELD_PREP(ACIOPHY_LANE_MODE_TX1, mode_cfg->lane_mode[1])); + core_mask32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_PROTOCOL, + FIELD_PREP(ACIOPHY_CROSSBAR_PROTOCOL, mode_cfg->crossbar)); + + core_mask32(atcphy, ACIOPHY_CROSSBAR, ACIOPHY_CROSSBAR_DP_SINGLE_PMA, + FIELD_PREP(ACIOPHY_CROSSBAR_DP_SINGLE_PMA, + mode_cfg->crossbar_dp_single_pma)); + if (mode_cfg->crossbar_dp_both_pma) + core_set32(atcphy, ACIOPHY_CROSSBAR, + ACIOPHY_CROSSBAR_DP_BOTH_PMA); + else + core_clear32(atcphy, ACIOPHY_CROSSBAR, + ACIOPHY_CROSSBAR_DP_BOTH_PMA); +} + +static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) +{ + int ret; + u32 reg; + + if (readl_relaxed(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ) & + PIPEHANDLER_LOCK_EN) + dev_warn(atcphy->dev, "pipehandler already locked\n"); + + set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, + PIPEHANDLER_LOCK_EN); + + ret = readl_poll_timeout(atcphy->regs.pipehandler + + PIPEHANDLER_LOCK_ACK, + reg, reg & PIPEHANDLER_LOCK_EN, 1000, 1000000); + if (ret) { + clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, 1); + dev_err(atcphy->dev, + "pipehandler lock not acked, this type-c port is probably dead until the next reboot.\n"); + } + + return ret; +} + +static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) +{ + int ret; + u32 reg; + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, + PIPEHANDLER_LOCK_EN); + ret = readl_poll_timeout( + atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, + !(reg & PIPEHANDLER_LOCK_EN), 1000, 1000000); + if (ret) + dev_err(atcphy->dev, + "pipehandler lock release not acked, this type-c port is probably dead until the next reboot.\n"); + + return ret; +} + +static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, + enum atcphy_pipehandler_state state) +{ + int ret; + u32 reg; + + if (atcphy->pipehandler_state == state) + return 0; + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, + 14); // TODO: why 14? + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID | PIPEHANDLER_OVERRIDE_RXDETECT); + + ret = atcphy_pipehandler_lock(atcphy); + if (ret) + return ret; + + switch (state) { + case ATCPHY_PIPEHANDLER_STATE_USB3: + core_set32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG0, + ACIOPHY_TOP_BIST_PHY_CFG0_LN0_RESET_N); + core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, + ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV); + ret = readl_poll_timeout( + atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 100, 100000); + if (ret) + dev_warn( + atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + + // TODO: macOS does this but this breaks waiting for + // ACIOPHY_TOP_PHY_STAT_LN0_UNK0 then for some reason :/ + // this is probably status reset which clears the ln0 + // ready status but then the ready status never comes + // up again +#if 0 + core_set32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, + ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); + core_clear32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, + ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); +#endif + core_mask32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG1, + ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN, + FIELD_PREP(ACIOPHY_TOP_BIST_PHY_CFG1_LN0_PWR_DOWN, + 3)); + core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, + ACIOPHY_TOP_BIST_OV_CFG_LN0_PWR_DOWN_OV); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + writel(0, atcphy->regs.core + ACIOPHY_TOP_BIST_CIOPHY_CFG1); + + ret = readl_poll_timeout( + atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + (reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK0), 100, 100000); + if (ret) + dev_warn( + atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK0\n"); + + ret = readl_poll_timeout( + atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, + !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 100, 100000); + if (ret) + dev_warn( + atcphy->dev, + "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + + writel(0, atcphy->regs.core + ACIOPHY_TOP_BIST_OV_CFG); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); + core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, + ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + + /* switch dwc3's superspeed PHY to the real physical PHY */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_CLK_SELECT); + clear32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_MUX_MODE); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_CLK_SELECT, + FIELD_PREP(PIPEHANDLER_CLK_SELECT, + PIPEHANDLER_CLK_USB3PHY)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_MUX_MODE, + FIELD_PREP(PIPEHANDLER_MUX_MODE, + PIPEHANDLER_MUX_MODE_USB3PHY)); + + /* use real rx detect/valid values again */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID | + PIPEHANDLER_OVERRIDE_RXDETECT); + break; + default: + dev_warn( + atcphy->dev, + "unknown mode in pipehandler_configure: %d, switching to safe state\n", + state); + fallthrough; + case ATCPHY_PIPEHANDLER_STATE_USB2: + /* switch dwc3's superspeed PHY back to the dummy (and also USB4 PHY?) */ + clear32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_CLK_SELECT); + clear32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_MUX_MODE); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_CLK_SELECT, + FIELD_PREP(PIPEHANDLER_CLK_SELECT, + PIPEHANDLER_CLK_DUMMY_PHY)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, + PIPEHANDLER_MUX_MODE, + FIELD_PREP(PIPEHANDLER_MUX_MODE, + PIPEHANDLER_MUX_MODE_DUMMY_PHY)); + + /* keep ignoring rx detect and valid values from the USB3/4 PHY? */ + set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID | + PIPEHANDLER_OVERRIDE_RXDETECT); + break; + } + + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) + return ret; + + // TODO: macos seems to always clear it for USB3 - what about USB2/4? + clear32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NONSELECTED_NATIVE_RESET); + + // TODO: why? without this superspeed devices sometimes come up as highspeed + msleep(500); + + atcphy->pipehandler_state = state; + + return 0; +} + +static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) +{ + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTXPHY_PMA_LANE_RESET_N); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTXPHY_PMA_LANE_RESET_N_OV); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPRX_PCLK_SELECT, FIELD_PREP(DPRX_PCLK_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPRX_PCLK_ENABLE); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, FIELD_PREP(DPTX_PCLK1_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_SELECT, FIELD_PREP(DPTX_PCLK2_SELECT, 1)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + + core_set32(atcphy, ACIOPHY_PLL_COMMON_CTRL, + ACIOPHY_PLL_WAIT_FOR_CMN_READY_BEFORE_RESET_EXIT); + + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_SML_IN); + udelay(2); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_SLEEP_B_BIG_IN); + udelay(2); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, + LPDPTX_TXTERM_CODEMSB); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_TXTERM_CODE, + FIELD_PREP(LPDPTX_TXTERM_CODE, 0x16)); + + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_LDO_CTRL, 0x1c00); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1, + LPDPTX_CFG_PMA_PHYS_ADJ, FIELD_PREP(LPDPTX_CFG_PMA_PHYS_ADJ, 5)); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG1, + LPDPTX_CFG_PMA_PHYS_ADJ_OV); + + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_MARGIN, + LPDPTX_MARGIN_RCAL_RXOFFSET_EN); + + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, + LPDPTX_BLK_AUX_CTRL_PWRDN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_SHM_CFG_BLK_AUX_CTRL_REG0, + LPDPTX_CFG_PMA_AUX_SEL_LF_DATA); + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, + LPDPTX_BLK_AUX_RXOFFSET, FIELD_PREP(LPDPTX_BLK_AUX_RXOFFSET, 3)); + + mask32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_MARGIN, + LPDPTX_AUX_MARGIN_RCAL_TXSWING, + FIELD_PREP(LPDPTX_AUX_MARGIN_RCAL_TXSWING, 12)); + + atcphy->dp_link_rate = -1; +} + +static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) +{ + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CFG_BLK_AUX_CTRL, + LPDPTX_BLK_AUX_CTRL_PWRDN); + set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_CLAMP_EN); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, + LPDPTX_SLEEP_B_SML_IN); + udelay(2); + clear32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, + LPDPTX_SLEEP_B_BIG_IN); + udelay(2); + + // TODO: maybe? + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTXPHY_PMA_LANE_RESET_N); + // _OV? + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPRX_PCLK_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + + // clear 0x1000000 / BIT(24) maybe + // writel(0x1830630, atcphy->regs.core + 0x1028); +} + +static int +atcphy_dp_configure_lane(struct apple_atcphy *atcphy, unsigned int lane, + const struct atcphy_dp_link_rate_configuration *cfg) +{ + void __iomem *tx_shm, *rx_shm, *rx_top; + + switch (lane) { + case 0: + tx_shm = atcphy->regs.core + LN0_AUSPMA_TX_SHM; + rx_shm = atcphy->regs.core + LN0_AUSPMA_RX_SHM; + rx_top = atcphy->regs.core + LN0_AUSPMA_RX_TOP; + break; + case 1: + tx_shm = atcphy->regs.core + LN1_AUSPMA_TX_SHM; + rx_shm = atcphy->regs.core + LN1_AUSPMA_RX_SHM; + rx_top = atcphy->regs.core + LN1_AUSPMA_RX_TOP; + break; + default: + return -EINVAL; + } + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_SML); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_SML_OV); + udelay(2); + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_BIG); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, LN_LDOCLK_EN_BIG_OV); + udelay(2); + + if (cfg->bypass_txa_ldoclk) { + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_SML); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_SML_OV); + udelay(2); + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_BIG); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_BIG_OV); + udelay(2); + } else { + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_SML); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_SML_OV); + udelay(2); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_BIG); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_LDOCLK, + LN_LDOCLK_BYPASS_BIG_OV); + udelay(2); + } + + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, + LN_BYTECLK_RESET_SYNC_SEL_OV); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, + LN_BYTECLK_RESET_SYNC_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, + LN_BYTECLK_RESET_SYNC_EN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, + LN_BYTECLK_RESET_SYNC_CLR); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG0, + LN_BYTECLK_RESET_SYNC_CLR_OV); + + if (cfg->txa_div2_en) + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, + LN_TXA_DIV2_EN); + else + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, + LN_TXA_DIV2_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_EN_OV); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_CLK_EN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_CLK_EN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, LN_TXA_DIV2_RESET); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_CFG_MAIN_REG1, + LN_TXA_DIV2_RESET_OV); + + mask32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_BASE, + FIELD_PREP(LN_TXA_CAL_CTRL_BASE, 0xf)); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_BASE_OV); + mask32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL, + FIELD_PREP(LN_TXA_CAL_CTRL, 0x3f)); // TODO: 3f? + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_CAL_CTRL_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG2, LN_TXA_MARGIN_2R_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_2R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_4R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_POST_4R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_2R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_2R_OV); + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_4R); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG3, LN_TXA_MARGIN_PRE_4R_OV); + + clear32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_HIZ); + set32(tx_shm + LN_AUSPMA_TX_SHM_TXA_IMP_REG0, LN_TXA_HIZ_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, + LN_RX_DIV20_RESET_N); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, + LN_RX_DIV20_RESET_N_OV); + udelay(2); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_AFE_CTRL1, LN_RX_DIV20_RESET_N); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, + LN_TX_BYTECLK_RESET_SYNC_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, + LN_TX_BYTECLK_RESET_SYNC_EN_OV); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_TX_CAL_CODE, + FIELD_PREP(LN_TX_CAL_CODE, 6)); // TODO 6? + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_TX_CAL_CODE_OV); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, + LN_TX_CLK_DLY_CTRL_TAPGEN, + FIELD_PREP(LN_TX_CLK_DLY_CTRL_TAPGEN, 3)); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10, LN_DTVREG_ADJUST); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_DTVREG_ADJUST_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, LN_RXTERM_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_TEST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_TEST_EN_OV); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_TEST_RXLPBKDT_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_TEST_RXLPBKDT_EN_OV); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_LPBKIN_DATA, FIELD_PREP(LN_VREF_LPBKIN_DATA, 3)); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BIAS_SEL, + FIELD_PREP(LN_VREF_BIAS_SEL, 2)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_BIAS_SEL_OV); + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_ADJUST_GRAY, FIELD_PREP(LN_VREF_ADJUST_GRAY, 0x18)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_ADJUST_GRAY_OV); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_EN_OV); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_BOOST_EN_OV); + udelay(2); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, LN_VREF_BOOST_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_VREF_CTRL22, + LN_VREF_BOOST_EN_OV); + udelay(2); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PRE_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PRE_EN_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PST1_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_TX_PST1_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_PBIAS_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_PBIAS_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, + LN_RXTERM_PULLUP_LEAK_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_SAVOS_CTRL16, + LN_RXTERM_PULLUP_LEAK_EN_OV); + + set32(rx_top + LN_AUSPMA_RX_TOP_TJ_CFG_RX_TXMODE, LN_RX_TXMODE); + + if (cfg->txa_div2_en) + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, + LN_TX_CLK_DIV2_EN); + else + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, + LN_TX_CLK_DIV2_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, + LN_TX_CLK_DIV2_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, + LN_TX_CLK_DIV2_RST); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, + LN_TX_CLK_DIV2_RST_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_HRCLK_SEL); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, LN_TX_HRCLK_SEL_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_LSB_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, LN_TX_MARGIN_P1_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, + LN_TX_MARGIN_P1_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL17, + LN_TX_MARGIN_P1_LSB_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_LSB_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_P1_LSB_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_MARGIN_PRE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, + LN_TX_MARGIN_PRE_LSB); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, + LN_TX_MARGIN_PRE_LSB_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_LSB_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, + LN_TX_PRE_LSB_CODE_OV); + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_CODE); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TX_CTRL18, LN_TX_PRE_CODE_OV); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_SML_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_SML_EN_OV); + udelay(2); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_BIG_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL11, LN_DTVREG_BIG_EN_OV); + udelay(2); + + mask32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL10, LN_DTVREG_ADJUST, + FIELD_PREP(LN_DTVREG_ADJUST, 0xa)); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL13, LN_DTVREG_ADJUST_OV); + udelay(2); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_TERM_CTRL19, LN_TX_EN_OV); + udelay(2); + + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0, LN_TX_CLK_EN); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_CTLE_CTRL0, LN_TX_CLK_EN_OV); + + clear32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, + LN_TX_BYTECLK_RESET_SYNC_CLR); + set32(rx_shm + LN_AUSPMA_RX_SHM_TJ_RXA_DFE_CTRL12, + LN_TX_BYTECLK_RESET_SYNC_CLR_OV); + + return 0; +} + +static int atcphy_auspll_apb_command(struct apple_atcphy *atcphy, u32 command) +{ + int ret; + u32 reg; + + reg = readl(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + reg &= ~AUSPLL_APB_CMD_OVERRIDE_CMD; + reg |= FIELD_PREP(AUSPLL_APB_CMD_OVERRIDE_CMD, command); + reg |= AUSPLL_APB_CMD_OVERRIDE_REQ; + reg |= AUSPLL_APB_CMD_OVERRIDE_UNK28; + writel(reg, atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE); + + ret = readl_poll_timeout(atcphy->regs.core + AUSPLL_APB_CMD_OVERRIDE, + reg, (reg & AUSPLL_APB_CMD_OVERRIDE_ACK), 100, + 100000); + if (ret) { + dev_err(atcphy->dev, "AUSPLL APB command was not acked.\n"); + return ret; + } + + core_clear32(atcphy, AUSPLL_APB_CMD_OVERRIDE, + AUSPLL_APB_CMD_OVERRIDE_REQ); + + return 0; +} + +static int atcphy_dp_configure(struct apple_atcphy *atcphy, + enum atcphy_dp_link_rate lr) +{ + const struct atcphy_dp_link_rate_configuration *cfg = &dp_lr_config[lr]; + const struct atcphy_mode_configuration *mode_cfg; + int ret; + u32 reg; + + trace_atcphy_dp_configure(atcphy, lr); + + if (atcphy->dp_link_rate == lr) + return 0; + + if (atcphy->swap_lanes) + mode_cfg = &atcphy_modes[atcphy->mode].swapped; + else + mode_cfg = &atcphy_modes[atcphy->mode].normal; + + core_clear32(atcphy, AUSPLL_FREQ_CFG, AUSPLL_FREQ_REFCLK); + + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FREQ_COUNT_TARGET, + FIELD_PREP(AUSPLL_FD_FREQ_COUNT_TARGET, + cfg->freqinit_count_target)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_FBDIVN_HALF); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_REV_DIVN); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KI_MAN, + FIELD_PREP(AUSPLL_FD_KI_MAN, 8)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KI_EXP, + FIELD_PREP(AUSPLL_FD_KI_EXP, 3)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KP_MAN, + FIELD_PREP(AUSPLL_FD_KP_MAN, 8)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KP_EXP, + FIELD_PREP(AUSPLL_FD_KP_EXP, 7)); + core_clear32(atcphy, AUSPLL_FREQ_DESC_A, AUSPLL_FD_KPKI_SCALE_HBW); + + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, AUSPLL_FD_FBDIVN_FRAC_DEN, + FIELD_PREP(AUSPLL_FD_FBDIVN_FRAC_DEN, + cfg->fbdivn_frac_den)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_B, AUSPLL_FD_FBDIVN_FRAC_NUM, + FIELD_PREP(AUSPLL_FD_FBDIVN_FRAC_NUM, + cfg->fbdivn_frac_num)); + + core_clear32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_SDM_SSC_STEP); + core_clear32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_SDM_SSC_EN); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_PCLK_DIV_SEL, + FIELD_PREP(AUSPLL_FD_PCLK_DIV_SEL, cfg->pclk_div_sel)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_LFSDM_DIV, + FIELD_PREP(AUSPLL_FD_LFSDM_DIV, 1)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_LFCLK_CTRL, + FIELD_PREP(AUSPLL_FD_LFCLK_CTRL, cfg->lfclk_ctrl)); + core_mask32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_VCLK_OP_DIVN, + FIELD_PREP(AUSPLL_FD_VCLK_OP_DIVN, cfg->vclk_op_divn)); + core_set32(atcphy, AUSPLL_FREQ_DESC_C, AUSPLL_FD_VCLK_PRE_DIVN); + + core_mask32(atcphy, AUSPLL_CLKOUT_DIV, AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, + FIELD_PREP(AUSPLL_CLKOUT_PLLA_REFBUFCLK_DI, 7)); + + if (cfg->plla_clkout_vreg_bypass) + core_set32(atcphy, AUSPLL_CLKOUT_DTC_VREG, + AUSPLL_DTC_VREG_BYPASS); + else + core_clear32(atcphy, AUSPLL_CLKOUT_DTC_VREG, + AUSPLL_DTC_VREG_BYPASS); + + core_set32(atcphy, AUSPLL_BGR, AUSPLL_BGR_CTRL_AVAIL); + + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, + AUSPLL_CLKOUT_MASTER_PCLK_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, + AUSPLL_CLKOUT_MASTER_PCLK2_DRVR_EN); + core_set32(atcphy, AUSPLL_CLKOUT_MASTER, + AUSPLL_CLKOUT_MASTER_REFBUFCLK_DRVR_EN); + + ret = atcphy_auspll_apb_command(atcphy, 0); + if (ret) + return ret; + + ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_DP_PCLK_STAT, reg, + (reg & ACIOPHY_AUSPLL_LOCK), 100, 100000); + if (ret) { + dev_err(atcphy->dev, "ACIOPHY_DP_PCLK did not lock.\n"); + return ret; + } + + ret = atcphy_auspll_apb_command(atcphy, 0x2800); + if (ret) + return ret; + + if (mode_cfg->dp_lane[0]) { + ret = atcphy_dp_configure_lane(atcphy, 0, cfg); + if (ret) + return ret; + } + + if (mode_cfg->dp_lane[1]) { + ret = atcphy_dp_configure_lane(atcphy, 1, cfg); + if (ret) + return ret; + } + + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DP_PMA_BYTECLK_RESET); + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DP_MAC_DIV20_CLK_SEL); + + atcphy->dp_link_rate = lr; + return 0; +} + +static int atcphy_cio_configure(struct apple_atcphy *atcphy, + enum atcphy_mode mode) +{ + int ret; + + BUG_ON(!mutex_is_locked(&atcphy->lock)); + + ret = atcphy_cio_power_on(atcphy); + if (ret) + return ret; + + atcphy_setup_pll_fuses(atcphy); + atcphy_apply_tunables(atcphy, mode); + + // TODO: without this sometimes device aren't recognized but no idea what it does + // ACIOPHY_PLL_TOP_BLK_AUSPLL_PCTL_FSM_CTRL1.APB_REQ_OV_SEL = 255 + core_set32(atcphy, 0x1014, 255 << 13); + core_set32(atcphy, AUSPLL_APB_CMD_OVERRIDE, + AUSPLL_APB_CMD_OVERRIDE_UNK28); + + writel(0x10000cef, atcphy->regs.core + 0x8); // ACIOPHY_CFG0 + writel(0x15570cff, atcphy->regs.core + 0x1b0); // ACIOPHY_SLEEP_CTRL + writel(0x11833fef, atcphy->regs.core + 0x8); // ACIOPHY_CFG0 + + /* enable clocks and configure lanes */ + core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_PCLK_EN); + core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_REFCLK_EN); + atcphy_configure_lanes(atcphy, mode); + + /* take the USB3 PHY out of reset */ + core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + + /* setup AUX channel if DP altmode is requested */ + if (atcphy_modes[mode].enable_dp_aux) + atcphy_enable_dp_aux(atcphy); + + atcphy->mode = mode; + return 0; +} + +static int atcphy_usb3_power_on(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + enum atcphy_pipehandler_state state; + int ret = 0; + + /* + * Both usb role switch and mux set work will be running concurrently. + * Make sure atcphy_mux_set_work is done bringing up ATCPHY before + * trying to switch dwc3 to the correct PHY. + */ + mutex_lock(&atcphy->lock); + if (atcphy->mode != atcphy->target_mode) { + reinit_completion(&atcphy->atcphy_online_event); + mutex_unlock(&atcphy->lock); + wait_for_completion_timeout(&atcphy->atcphy_online_event, + msecs_to_jiffies(1000)); + mutex_lock(&atcphy->lock); + } + + if (atcphy->mode != atcphy->target_mode) { + dev_err(atcphy->dev, "ATCPHY did not come up; won't allow dwc3 to come up.\n"); + return -EINVAL; + } + + atcphy->dwc3_online = true; + state = atcphy_modes[atcphy->mode].pipehandler_state; + switch (state) { + case ATCPHY_PIPEHANDLER_STATE_USB2: + case ATCPHY_PIPEHANDLER_STATE_USB3: + ret = atcphy_configure_pipehandler(atcphy, state); + break; + + case ATCPHY_PIPEHANDLER_STATE_INVALID: + default: + dev_warn(atcphy->dev, "Invalid state %d in usb3_set_phy\n", + state); + ret = -EINVAL; + } + + mutex_unlock(&atcphy->lock); + + return 0; +} + +static int atcphy_usb3_power_off(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + mutex_lock(&atcphy->lock); + + atcphy_configure_pipehandler(atcphy, ATCPHY_PIPEHANDLER_STATE_USB2); + + atcphy->dwc3_online = false; + complete(&atcphy->dwc3_shutdown_event); + + mutex_unlock(&atcphy->lock); + + return 0; +} + +static const struct phy_ops apple_atc_usb3_phy_ops = { + .owner = THIS_MODULE, + .power_on = atcphy_usb3_power_on, + .power_off = atcphy_usb3_power_off, +}; + +static int atcphy_usb2_power_on(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + mutex_lock(&atcphy->lock); + + /* take the PHY out of its low power state */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_SIDDQ); + udelay(10); + + /* reset the PHY for good measure */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, + USB2PHY_CTL_RESET | USB2PHY_CTL_PORT_RESET); + udelay(10); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, + USB2PHY_CTL_RESET | USB2PHY_CTL_PORT_RESET); + + set32(atcphy->regs.usb2phy + USB2PHY_SIG, + USB2PHY_SIG_VBUSDET_FORCE_VAL | USB2PHY_SIG_VBUSDET_FORCE_EN | + USB2PHY_SIG_VBUSVLDEXT_FORCE_VAL | + USB2PHY_SIG_VBUSVLDEXT_FORCE_EN); + + /* enable the dummy PHY for the SS lanes */ + set32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_DUMMY_PHY_EN); + + mutex_unlock(&atcphy->lock); + + return 0; +} + +static int atcphy_usb2_power_off(struct phy *phy) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + mutex_lock(&atcphy->lock); + + /* reset the PHY before transitioning to low power mode */ + clear32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_APB_RESET_N); + set32(atcphy->regs.usb2phy + USB2PHY_CTL, + USB2PHY_CTL_RESET | USB2PHY_CTL_PORT_RESET); + + /* switch the PHY to low power mode */ + set32(atcphy->regs.usb2phy + USB2PHY_CTL, USB2PHY_CTL_SIDDQ); + + mutex_unlock(&atcphy->lock); + + return 0; +} + +static int atcphy_usb2_set_mode(struct phy *phy, enum phy_mode mode, + int submode) +{ + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; + + mutex_lock(&atcphy->lock); + + switch (mode) { + case PHY_MODE_USB_HOST: + case PHY_MODE_USB_HOST_LS: + case PHY_MODE_USB_HOST_FS: + case PHY_MODE_USB_HOST_HS: + case PHY_MODE_USB_HOST_SS: + set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + set32(atcphy->regs.usb2phy + USB2PHY_USBCTL, + USB2PHY_USBCTL_HOST_EN); + ret = 0; + break; + + case PHY_MODE_USB_DEVICE: + case PHY_MODE_USB_DEVICE_LS: + case PHY_MODE_USB_DEVICE_FS: + case PHY_MODE_USB_DEVICE_HS: + case PHY_MODE_USB_DEVICE_SS: + clear32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); + clear32(atcphy->regs.usb2phy + USB2PHY_USBCTL, + USB2PHY_USBCTL_HOST_EN); + ret = 0; + break; + + default: + dev_err(atcphy->dev, "Unknown mode for usb2 phy: %d\n", mode); + ret = -EINVAL; + } + + mutex_unlock(&atcphy->lock); + return ret; +} + +static const struct phy_ops apple_atc_usb2_phy_ops = { + .owner = THIS_MODULE, + .set_mode = atcphy_usb2_set_mode, + /* + * This PHY is always matched with a dwc3 controller. Currently, + * first dwc3 initializes the PHY and then soft-resets itself and + * then finally powers on the PHY. This should be reasonable. + * Annoyingly, the dwc3 soft reset is never completed when the USB2 PHY + * is powered off so we have to pretend that these two are actually + * init/exit here to ensure the PHY is powered on and out of reset + * early enough. + */ + .init = atcphy_usb2_power_on, + .exit = atcphy_usb2_power_off, +}; + +static int atcphy_dpphy_set_mode(struct phy *phy, enum phy_mode mode, + int submode) +{ + /* nothing to do here since the setup already happened in mux_set */ + if (mode == PHY_MODE_DP && submode == 0) + return 0; + return -EINVAL; +} + +static int atcphy_dpphy_validate(struct phy *phy, enum phy_mode mode, + int submode, union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + + if (mode != PHY_MODE_DP) + return -EINVAL; + if (submode != 0) + return -EINVAL; + + switch (atcphy->mode) { + case APPLE_ATCPHY_MODE_USB3_DP: + opts->lanes = 2; + break; + case APPLE_ATCPHY_MODE_DP: + opts->lanes = 4; + break; + default: + opts->lanes = 0; + } + + opts->link_rate = 8100; + + for (int i = 0; i < 4; ++i) { + opts->voltage[i] = 3; + opts->pre[i] = 3; + } + + return 0; +} + +static int atcphy_dpphy_configure(struct phy *phy, + union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_atcphy *atcphy = phy_get_drvdata(phy); + enum atcphy_dp_link_rate link_rate; + int ret = 0; + + /* might be possibly but we don't know how */ + if (opts->set_voltages) + return -EINVAL; + + /* TODO? or maybe just ack since this mux_set should've done this? */ + if (opts->set_lanes) + return -EINVAL; + + if (opts->set_rate) { + switch (opts->link_rate) { + case 1620: + link_rate = ATCPHY_DP_LINK_RATE_RBR; + break; + case 2700: + link_rate = ATCPHY_DP_LINK_RATE_HBR; + break; + case 5400: + link_rate = ATCPHY_DP_LINK_RATE_HBR2; + break; + case 8100: + link_rate = ATCPHY_DP_LINK_RATE_HBR3; + break; + case 0: + // TODO: disable! + return 0; + break; + default: + dev_err(atcphy->dev, "Unsupported link rate: %d\n", + opts->link_rate); + return -EINVAL; + } + + mutex_lock(&atcphy->lock); + ret = atcphy_dp_configure(atcphy, link_rate); + mutex_unlock(&atcphy->lock); + } + + return ret; +} + +static const struct phy_ops apple_atc_dp_phy_ops = { + .owner = THIS_MODULE, + .configure = atcphy_dpphy_configure, + .validate = atcphy_dpphy_validate, + .set_mode = atcphy_dpphy_set_mode, +}; + +static struct phy *atcphy_xlate(struct device *dev, + struct of_phandle_args *args) +{ + struct apple_atcphy *atcphy = dev_get_drvdata(dev); + + switch (args->args[0]) { + case PHY_TYPE_USB2: + return atcphy->phy_usb2; + case PHY_TYPE_USB3: + return atcphy->phy_usb3; + case PHY_TYPE_DP: + return atcphy->phy_dp; + } + return ERR_PTR(-ENODEV); +} + +static int atcphy_probe_phy(struct apple_atcphy *atcphy) +{ + atcphy->phy_usb2 = + devm_phy_create(atcphy->dev, NULL, &apple_atc_usb2_phy_ops); + if (IS_ERR(atcphy->phy_usb2)) + return PTR_ERR(atcphy->phy_usb2); + phy_set_drvdata(atcphy->phy_usb2, atcphy); + + atcphy->phy_usb3 = + devm_phy_create(atcphy->dev, NULL, &apple_atc_usb3_phy_ops); + if (IS_ERR(atcphy->phy_usb3)) + return PTR_ERR(atcphy->phy_usb3); + phy_set_drvdata(atcphy->phy_usb3, atcphy); + + atcphy->phy_dp = + devm_phy_create(atcphy->dev, NULL, &apple_atc_dp_phy_ops); + if (IS_ERR(atcphy->phy_dp)) + return PTR_ERR(atcphy->phy_dp); + phy_set_drvdata(atcphy->phy_dp, atcphy); + + atcphy->phy_provider = + devm_of_phy_provider_register(atcphy->dev, atcphy_xlate); + if (IS_ERR(atcphy->phy_provider)) + return PTR_ERR(atcphy->phy_provider); + + return 0; +} + +static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct apple_atcphy *atcphy = rcdev_to_apple_atcphy(rcdev); + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_RESET_N); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + + return 0; +} + +static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct apple_atcphy *atcphy = rcdev_to_apple_atcphy(rcdev); + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + set32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, + PIPEHANDLER_AON_GEN_DWC3_RESET_N); + + return 0; +} + +const struct reset_control_ops atcphy_dwc3_reset_ops = { + .assert = atcphy_dwc3_reset_assert, + .deassert = atcphy_dwc3_reset_deassert, +}; + +static int atcphy_reset_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + return 0; +} + +static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) +{ + atcphy->rcdev.owner = THIS_MODULE; + atcphy->rcdev.nr_resets = 1; + atcphy->rcdev.ops = &atcphy_dwc3_reset_ops; + atcphy->rcdev.of_node = atcphy->dev->of_node; + atcphy->rcdev.of_reset_n_cells = 0; + atcphy->rcdev.of_xlate = atcphy_reset_xlate; + + return devm_reset_controller_register(atcphy->dev, &atcphy->rcdev); +} + +static int atcphy_sw_set(struct typec_switch_dev *sw, + enum typec_orientation orientation) +{ + struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); + + trace_atcphy_sw_set(orientation); + + mutex_lock(&atcphy->lock); + switch (orientation) { + case TYPEC_ORIENTATION_NONE: + break; + case TYPEC_ORIENTATION_NORMAL: + atcphy->swap_lanes = false; + break; + case TYPEC_ORIENTATION_REVERSE: + atcphy->swap_lanes = true; + break; + } + mutex_unlock(&atcphy->lock); + + return 0; +} + +static int atcphy_probe_switch(struct apple_atcphy *atcphy) +{ + struct typec_switch_desc sw_desc = { + .drvdata = atcphy, + .fwnode = atcphy->dev->fwnode, + .set = atcphy_sw_set, + }; + + return PTR_ERR_OR_ZERO(typec_switch_register(atcphy->dev, &sw_desc)); +} + +static void atcphy_mux_set_work(struct work_struct *work) +{ + struct apple_atcphy *atcphy = container_of(work, struct apple_atcphy, mux_set_work); + + mutex_lock(&atcphy->lock); + /* + * If we're transitiong to TYPEC_STATE_SAFE dwc3 will have gotten + * a usb-role-switch event to ROLE_NONE which is deferred to a work + * queue. dwc3 will try to switch the pipehandler mux to USB2 and + * we have to make sure that has happened before we disable ATCPHY. + * If we instead disable ATCPHY first dwc3 will get stuck and the + * port won't work anymore until a full SoC reset. + * We're guaranteed that no other role switch event will be generated + * before we return because the mux_set callback runs in the same + * thread that generates these. We can thus unlock the mutex, wait + * for dwc3_shutdown_event from the usb3 phy's power_off callback after + * it has taken the mutex and the lock again. + */ + if (atcphy->dwc3_online && atcphy->target_mode == APPLE_ATCPHY_MODE_OFF) { + reinit_completion(&atcphy->dwc3_shutdown_event); + mutex_unlock(&atcphy->lock); + wait_for_completion_timeout(&atcphy->dwc3_shutdown_event, + msecs_to_jiffies(1000)); + mutex_lock(&atcphy->lock); + WARN_ON(atcphy->dwc3_online); + } + + switch (atcphy->target_mode) { + case APPLE_ATCPHY_MODE_DP: + case APPLE_ATCPHY_MODE_USB3_DP: + case APPLE_ATCPHY_MODE_USB3: + case APPLE_ATCPHY_MODE_USB4: + atcphy_cio_configure(atcphy, atcphy->target_mode); + break; + default: + dev_warn(atcphy->dev, "Unknown mode %d in atcphy_mux_set\n", + atcphy->target_mode); + fallthrough; + case APPLE_ATCPHY_MODE_USB2: + case APPLE_ATCPHY_MODE_OFF: + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy_disable_dp_aux(atcphy); + atcphy_cio_power_off(atcphy); + } + + complete(&atcphy->atcphy_online_event); + mutex_unlock(&atcphy->lock); +} + +static int atcphy_mux_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + + // TODO: + flush_work(&atcphy->mux_set_work); + + mutex_lock(&atcphy->lock); + trace_atcphy_mux_set(state); + + if (state->mode == TYPEC_STATE_SAFE) { + atcphy->target_mode = APPLE_ATCPHY_MODE_OFF; + } else if (state->mode == TYPEC_STATE_USB) { + atcphy->target_mode = APPLE_ATCPHY_MODE_USB3; + } else if (state->alt && state->alt->svid == USB_TYPEC_DP_SID) { + switch (state->mode) { + case TYPEC_DP_STATE_C: + case TYPEC_DP_STATE_E: + atcphy->target_mode = APPLE_ATCPHY_MODE_DP; + break; + case TYPEC_DP_STATE_D: + atcphy->target_mode = APPLE_ATCPHY_MODE_USB3_DP; + break; + default: + dev_err(atcphy->dev, + "Unsupported DP pin assignment: 0x%lx.\n", + state->mode); + atcphy->target_mode = APPLE_ATCPHY_MODE_OFF; + } + } else if (state->alt && state->alt->svid == USB_TYPEC_TBT_SID) { + dev_err(atcphy->dev, "USB4/TBT mode is not supported yet.\n"); + atcphy->target_mode = APPLE_ATCPHY_MODE_OFF; + } else if (state->alt) { + dev_err(atcphy->dev, "Unknown alternate mode SVID: 0x%x\n", + state->alt->svid); + atcphy->target_mode = APPLE_ATCPHY_MODE_OFF; + } else { + dev_err(atcphy->dev, "Unknown mode: 0x%lx\n", state->mode); + atcphy->target_mode = APPLE_ATCPHY_MODE_OFF; + } + + if (atcphy->mode != atcphy->target_mode) + WARN_ON(!schedule_work(&atcphy->mux_set_work)); + + mutex_unlock(&atcphy->lock); + + return 0; +} + +static int atcphy_probe_mux(struct apple_atcphy *atcphy) +{ + struct typec_mux_desc mux_desc = { + .drvdata = atcphy, + .fwnode = atcphy->dev->fwnode, + .set = atcphy_mux_set, + }; + + return PTR_ERR_OR_ZERO(typec_mux_register(atcphy->dev, &mux_desc)); +} + +static int atcphy_parse_legacy_tunable(struct apple_atcphy *atcphy, + struct atcphy_tunable *tunable, + const char *name) +{ + struct property *prop; + const __le32 *p = NULL; + int i; + +#if 0 + WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND, + "parsing legacy tunable; please update m1n1"); +#endif + + prop = of_find_property(atcphy->np, name, NULL); + if (!prop) { + dev_err(atcphy->dev, "tunable %s not found\n", name); + return -ENOENT; + } + + if (prop->length % (3 * sizeof(u32))) + return -EINVAL; + + tunable->sz = prop->length / (3 * sizeof(u32)); + tunable->values = devm_kcalloc(atcphy->dev, tunable->sz, + sizeof(*tunable->values), GFP_KERNEL); + if (!tunable->values) + return -ENOMEM; + + for (i = 0; i < tunable->sz; ++i) { + p = of_prop_next_u32(prop, p, &tunable->values[i].offset); + p = of_prop_next_u32(prop, p, &tunable->values[i].mask); + p = of_prop_next_u32(prop, p, &tunable->values[i].value); + } + + trace_atcphy_parsed_tunable(name, tunable); + + return 0; +} + +static int atcphy_parse_new_tunable(struct apple_atcphy *atcphy, + struct atcphy_tunable *tunable, + const char *name) +{ + struct property *prop; + u64 *fdt_tunable; + int ret, i; + + prop = of_find_property(atcphy->np, name, NULL); + if (!prop) { + dev_err(atcphy->dev, "tunable %s not found\n", name); + return -ENOENT; + } + + if (prop->length % (4 * sizeof(u64))) + return -EINVAL; + + fdt_tunable = kzalloc(prop->length, GFP_KERNEL); + if (!fdt_tunable) + return -ENOMEM; + + tunable->sz = prop->length / (4 * sizeof(u64)); + ret = of_property_read_variable_u64_array(atcphy->np, name, fdt_tunable, + tunable->sz, tunable->sz); + if (ret < 0) + goto err_free_fdt; + + tunable->values = devm_kcalloc(atcphy->dev, tunable->sz, + sizeof(*tunable->values), GFP_KERNEL); + if (!tunable->values) { + ret = -ENOMEM; + goto err_free_fdt; + } + + for (i = 0; i < tunable->sz; ++i) { + u32 offset, size, mask, value; + + offset = fdt_tunable[4 * i]; + size = fdt_tunable[4 * i + 1]; + mask = fdt_tunable[4 * i + 2]; + value = fdt_tunable[4 * i + 3]; + + if (offset > U32_MAX || size != 4 || mask > U32_MAX || + value > U32_MAX) { + ret = -EINVAL; + goto err_free_values; + } + + tunable->values[i].offset = offset; + tunable->values[i].mask = mask; + tunable->values[i].value = value; + } + + trace_atcphy_parsed_tunable(name, tunable); + kfree(fdt_tunable); + + BUG_ON(1); + return 0; + +err_free_values: + devm_kfree(atcphy->dev, tunable->values); +err_free_fdt: + kfree(fdt_tunable); + return ret; +} + +static int atcphy_parse_tunable(struct apple_atcphy *atcphy, + struct atcphy_tunable *tunable, + const char *name) +{ + int ret; + + if (!of_find_property(atcphy->np, name, NULL)) { + dev_err(atcphy->dev, "tunable %s not found\n", name); + return -ENOENT; + } + + ret = atcphy_parse_new_tunable(atcphy, tunable, name); + if (ret) + ret = atcphy_parse_legacy_tunable(atcphy, tunable, name); + + return ret; +} + +static int atcphy_load_tunables(struct apple_atcphy *atcphy) +{ + int ret; + + ret = atcphy_parse_tunable(atcphy, &atcphy->tunables.axi2af, + "apple,tunable-axi2af"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, &atcphy->tunables.common, + "apple,tunable-common"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, &atcphy->tunables.lane_usb3[0], + "apple,tunable-lane0-usb"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, &atcphy->tunables.lane_usb3[1], + "apple,tunable-lane1-usb"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, &atcphy->tunables.lane_usb4[0], + "apple,tunable-lane0-cio"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, &atcphy->tunables.lane_usb4[1], + "apple,tunable-lane1-cio"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, + &atcphy->tunables.lane_displayport[0], + "apple,tunable-lane0-dp"); + if (ret) + return ret; + ret = atcphy_parse_tunable(atcphy, + &atcphy->tunables.lane_displayport[1], + "apple,tunable-lane1-dp"); + if (ret) + return ret; + + return 0; +} + +static int atcphy_load_fuses(struct apple_atcphy *atcphy) +{ + int ret; + + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "aus_cmn_shm_vreg_trim", + &atcphy->fuses.aus_cmn_shm_vreg_trim); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "auspll_rodco_encap", + &atcphy->fuses.auspll_rodco_encap); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "auspll_rodco_bias_adjust", + &atcphy->fuses.auspll_rodco_bias_adjust); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "auspll_fracn_dll_start_capcode", + &atcphy->fuses.auspll_fracn_dll_start_capcode); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "auspll_dtc_vreg_adjust", + &atcphy->fuses.auspll_dtc_vreg_adjust); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "cio3pll_dco_coarsebin0", + &atcphy->fuses.cio3pll_dco_coarsebin[0]); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "cio3pll_dco_coarsebin1", + &atcphy->fuses.cio3pll_dco_coarsebin[1]); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "cio3pll_dll_start_capcode", + &atcphy->fuses.cio3pll_dll_start_capcode[0]); + if (ret) + return ret; + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "cio3pll_dtc_vreg_adjust", + &atcphy->fuses.cio3pll_dtc_vreg_adjust); + if (ret) + return ret; + + /* + * Only one of the two t8103 PHYs requires the following additional fuse + * and a slighly different configuration sequence if it's present. + * The other t8103 instance and all t6000 instances don't which means + * we must not fail here in case the fuse isn't present. + */ + ret = nvmem_cell_read_variable_le_u32( + atcphy->dev, "cio3pll_dll_start_capcode_workaround", + &atcphy->fuses.cio3pll_dll_start_capcode[1]); + switch (ret) { + case 0: + atcphy->quirks.t8103_cio3pll_workaround = true; + break; + case -ENOENT: + atcphy->quirks.t8103_cio3pll_workaround = false; + break; + default: + return ret; + } + + atcphy->fuses.present = true; + + trace_atcphy_fuses(atcphy); + return 0; +} + +static int atcphy_probe(struct platform_device *pdev) +{ + struct apple_atcphy *atcphy; + struct device *dev = &pdev->dev; + int ret; + + atcphy = devm_kzalloc(&pdev->dev, sizeof(*atcphy), GFP_KERNEL); + if (!atcphy) + return -ENOMEM; + + atcphy->dev = dev; + atcphy->np = dev->of_node; + platform_set_drvdata(pdev, atcphy); + + mutex_init(&atcphy->lock); + init_completion(&atcphy->dwc3_shutdown_event); + init_completion(&atcphy->atcphy_online_event); + INIT_WORK(&atcphy->mux_set_work, atcphy_mux_set_work); + + atcphy->regs.core = devm_platform_ioremap_resource_byname(pdev, "core"); + if (IS_ERR(atcphy->regs.core)) + return PTR_ERR(atcphy->regs.core); + atcphy->regs.lpdptx = + devm_platform_ioremap_resource_byname(pdev, "lpdptx"); + if (IS_ERR(atcphy->regs.lpdptx)) + return PTR_ERR(atcphy->regs.lpdptx); + atcphy->regs.axi2af = + devm_platform_ioremap_resource_byname(pdev, "axi2af"); + if (IS_ERR(atcphy->regs.axi2af)) + return PTR_ERR(atcphy->regs.axi2af); + atcphy->regs.usb2phy = + devm_platform_ioremap_resource_byname(pdev, "usb2phy"); + if (IS_ERR(atcphy->regs.usb2phy)) + return PTR_ERR(atcphy->regs.usb2phy); + atcphy->regs.pipehandler = + devm_platform_ioremap_resource_byname(pdev, "pipehandler"); + if (IS_ERR(atcphy->regs.pipehandler)) + return PTR_ERR(atcphy->regs.pipehandler); + + if (of_property_read_bool(dev->of_node, "nvmem-cells")) { + ret = atcphy_load_fuses(atcphy); + if (ret) + return ret; + } + + ret = atcphy_load_tunables(atcphy); + if (ret) + return ret; + + atcphy->mode = APPLE_ATCPHY_MODE_OFF; + atcphy->pipehandler_state = ATCPHY_PIPEHANDLER_STATE_INVALID; + + ret = atcphy_probe_rcdev(atcphy); + if (ret) + return ret; + ret = atcphy_probe_mux(atcphy); + if (ret) + return ret; + ret = atcphy_probe_switch(atcphy); + if (ret) + return ret; + return atcphy_probe_phy(atcphy); +} + +static const struct of_device_id atcphy_match[] = { + { + .compatible = "apple,t8103-atcphy", + }, + { + .compatible = "apple,t6000-atcphy", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, atcphy_match); + +static struct platform_driver atcphy_driver = { + .driver = { + .name = "phy-apple-atc", + .of_match_table = atcphy_match, + }, + .probe = atcphy_probe, +}; + +module_platform_driver(atcphy_driver); + +MODULE_AUTHOR("Sven Peter "); +MODULE_DESCRIPTION("Apple Type-C PHY driver"); + +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/apple/atc.h b/drivers/phy/apple/atc.h new file mode 100644 index 00000000000000..6b020953965fa5 --- /dev/null +++ b/drivers/phy/apple/atc.h @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple Type-C PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#ifndef APPLE_PHY_ATC_H +#define APPLE_PHY_ATC_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum atcphy_dp_link_rate { + ATCPHY_DP_LINK_RATE_RBR, + ATCPHY_DP_LINK_RATE_HBR, + ATCPHY_DP_LINK_RATE_HBR2, + ATCPHY_DP_LINK_RATE_HBR3, +}; + +enum atcphy_pipehandler_state { + ATCPHY_PIPEHANDLER_STATE_INVALID, + ATCPHY_PIPEHANDLER_STATE_USB2, + ATCPHY_PIPEHANDLER_STATE_USB3, +}; + +enum atcphy_mode { + APPLE_ATCPHY_MODE_OFF, + APPLE_ATCPHY_MODE_USB2, + APPLE_ATCPHY_MODE_USB3, + APPLE_ATCPHY_MODE_USB3_DP, + APPLE_ATCPHY_MODE_USB4, + APPLE_ATCPHY_MODE_DP, +}; + +struct atcphy_dp_link_rate_configuration { + u16 freqinit_count_target; + u16 fbdivn_frac_den; + u16 fbdivn_frac_num; + u16 pclk_div_sel; + u8 lfclk_ctrl; + u8 vclk_op_divn; + bool plla_clkout_vreg_bypass; + bool bypass_txa_ldoclk; + bool txa_div2_en; +}; + +struct atcphy_mode_configuration { + u32 crossbar; + u32 crossbar_dp_single_pma; + bool crossbar_dp_both_pma; + u32 lane_mode[2]; + bool dp_lane[2]; + bool set_swap; +}; + +struct atcphy_tunable { + size_t sz; + struct { + u32 offset; + u32 mask; + u32 value; + } * values; +}; + +struct apple_atcphy { + struct device_node *np; + struct device *dev; + + struct { + unsigned int t8103_cio3pll_workaround : 1; + } quirks; + + /* calibration fuse values */ + struct { + bool present; + u32 aus_cmn_shm_vreg_trim; + u32 auspll_rodco_encap; + u32 auspll_rodco_bias_adjust; + u32 auspll_fracn_dll_start_capcode; + u32 auspll_dtc_vreg_adjust; + u32 cio3pll_dco_coarsebin[2]; + u32 cio3pll_dll_start_capcode[2]; + u32 cio3pll_dtc_vreg_adjust; + } fuses; + + /* tunables provided by firmware through the device tree */ + struct { + struct atcphy_tunable axi2af; + struct atcphy_tunable common; + struct atcphy_tunable lane_usb3[2]; + struct atcphy_tunable lane_displayport[2]; + struct atcphy_tunable lane_usb4[2]; + } tunables; + + bool usb3_power_on; + bool swap_lanes; + + enum atcphy_mode mode; + int dp_link_rate; + + struct { + void __iomem *core; + void __iomem *axi2af; + void __iomem *usb2phy; + void __iomem *pipehandler; + void __iomem *lpdptx; + } regs; + + struct phy *phy_usb2; + struct phy *phy_usb3; + struct phy *phy_dp; + struct phy_provider *phy_provider; + struct reset_controller_dev rcdev; + struct typec_switch *sw; + struct typec_mux *mux; + + bool dwc3_online; + struct completion dwc3_shutdown_event; + struct completion atcphy_online_event; + + enum atcphy_pipehandler_state pipehandler_state; + + struct mutex lock; + + struct work_struct mux_set_work; + enum atcphy_mode target_mode; +}; + +#endif diff --git a/drivers/phy/apple/trace.c b/drivers/phy/apple/trace.c new file mode 100644 index 00000000000000..a82dc089f6caa8 --- /dev/null +++ b/drivers/phy/apple/trace.c @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 +#define CREATE_TRACE_POINTS +#include "trace.h" + diff --git a/drivers/phy/apple/trace.h b/drivers/phy/apple/trace.h new file mode 100644 index 00000000000000..c4c21c84e8917c --- /dev/null +++ b/drivers/phy/apple/trace.h @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple Type-C PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM appletypecphy + +#if !defined(_APPLETYPECPHY_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _APPLETYPECPHY_TRACE_H_ + +#include +#include +#include +#include "atc.h" + +#define show_dp_lr(lr) \ + __print_symbolic(lr, { ATCPHY_DP_LINK_RATE_RBR, "RBR" }, \ + { ATCPHY_DP_LINK_RATE_HBR, "HBR" }, \ + { ATCPHY_DP_LINK_RATE_HBR2, "HBR2" }, \ + { ATCPHY_DP_LINK_RATE_HBR3, "HBR3" }) + +#define show_sw_orientation(orientation) \ + __print_symbolic(orientation, { TYPEC_ORIENTATION_NONE, "none" }, \ + { TYPEC_ORIENTATION_NORMAL, "normal" }, \ + { TYPEC_ORIENTATION_REVERSE, "reverse" }) + +TRACE_EVENT(atcphy_sw_set, TP_PROTO(enum typec_orientation orientation), + TP_ARGS(orientation), + + TP_STRUCT__entry(__field(enum typec_orientation, orientation)), + + TP_fast_assign(__entry->orientation = orientation;), + + TP_printk("orientation: %s", + show_sw_orientation(__entry->orientation))); + +#define show_mux_state(state) \ + __print_symbolic(state.mode, { TYPEC_STATE_SAFE, "USB Safe State" }, \ + { TYPEC_STATE_USB, "USB" }) + +#define show_atcphy_mode(mode) \ + __print_symbolic(mode, { APPLE_ATCPHY_MODE_OFF, "off" }, \ + { APPLE_ATCPHY_MODE_USB2, "USB2" }, \ + { APPLE_ATCPHY_MODE_USB3, "USB3" }, \ + { APPLE_ATCPHY_MODE_USB3_DP, "DP + USB" }, \ + { APPLE_ATCPHY_MODE_USB4, "USB4" }, \ + { APPLE_ATCPHY_MODE_DP, "DP-only" }) + +TRACE_EVENT(atcphy_usb3_set_mode, + TP_PROTO(struct apple_atcphy *atcphy, enum phy_mode mode, + int submode), + TP_ARGS(atcphy, mode, submode), + + TP_STRUCT__entry(__field(enum atcphy_mode, mode) + __field(enum phy_mode, phy_mode) + __field(int, submode)), + + TP_fast_assign(__entry->mode = atcphy->mode; + __entry->phy_mode = mode; + __entry->submode = submode;), + + TP_printk("mode: %s, phy_mode: %d, submode: %d", + show_atcphy_mode(__entry->mode), __entry->phy_mode, + __entry->submode)); + +TRACE_EVENT( + atcphy_configure_lanes, + TP_PROTO(enum atcphy_mode mode, + const struct atcphy_mode_configuration *cfg), + TP_ARGS(mode, cfg), + + TP_STRUCT__entry(__field(enum atcphy_mode, mode) __field_struct( + struct atcphy_mode_configuration, cfg)), + + TP_fast_assign(__entry->mode = mode; __entry->cfg = *cfg;), + + TP_printk( + "mode: %s, crossbar: 0x%02x, lanes: {0x%02x, 0x%02x}, swap: %d", + show_atcphy_mode(__entry->mode), __entry->cfg.crossbar, + __entry->cfg.lane_mode[0], __entry->cfg.lane_mode[1], + __entry->cfg.set_swap)); + +TRACE_EVENT(atcphy_mux_set, TP_PROTO(struct typec_mux_state *state), + TP_ARGS(state), + + TP_STRUCT__entry(__field_struct(struct typec_mux_state, state)), + + TP_fast_assign(__entry->state = *state;), + + TP_printk("state: %s", show_mux_state(__entry->state))); + +TRACE_EVENT(atcphy_parsed_tunable, + TP_PROTO(const char *name, struct atcphy_tunable *tunable), + TP_ARGS(name, tunable), + + TP_STRUCT__entry(__field(const char *, name) + __field(size_t, sz)), + + TP_fast_assign(__entry->name = name; __entry->sz = tunable->sz;), + + TP_printk("%s with %zu entries", __entry->name, + __entry->sz)); + +TRACE_EVENT( + atcphy_fuses, TP_PROTO(struct apple_atcphy *atcphy), TP_ARGS(atcphy), + TP_STRUCT__entry(__field(struct apple_atcphy *, atcphy)), + TP_fast_assign(__entry->atcphy = atcphy;), + TP_printk( + "aus_cmn_shm_vreg_trim: 0x%02x; auspll_rodco_encap: 0x%02x; auspll_rodco_bias_adjust: 0x%02x; auspll_fracn_dll_start_capcode: 0x%02x; auspll_dtc_vreg_adjust: 0x%02x; cio3pll_dco_coarsebin: 0x%02x, 0x%02x; cio3pll_dll_start_capcode: 0x%02x, 0x%02x; cio3pll_dtc_vreg_adjust: 0x%02x", + __entry->atcphy->fuses.aus_cmn_shm_vreg_trim, + __entry->atcphy->fuses.auspll_rodco_encap, + __entry->atcphy->fuses.auspll_rodco_bias_adjust, + __entry->atcphy->fuses.auspll_fracn_dll_start_capcode, + __entry->atcphy->fuses.auspll_dtc_vreg_adjust, + __entry->atcphy->fuses.cio3pll_dco_coarsebin[0], + __entry->atcphy->fuses.cio3pll_dco_coarsebin[1], + __entry->atcphy->fuses.cio3pll_dll_start_capcode[0], + __entry->atcphy->fuses.cio3pll_dll_start_capcode[1], + __entry->atcphy->fuses.cio3pll_dtc_vreg_adjust)); + + + +TRACE_EVENT(atcphy_dp_configure, + TP_PROTO(struct apple_atcphy *atcphy, enum atcphy_dp_link_rate lr), + TP_ARGS(atcphy, lr), + + TP_STRUCT__entry(__string(devname, dev_name(atcphy->dev)) + __field(enum atcphy_dp_link_rate, lr)), + + TP_fast_assign(__assign_str(devname, dev_name(atcphy->dev)); + __entry->lr = lr;), + + TP_printk("%s: link rate: %s", __get_str(devname), + show_dp_lr(__entry->lr))); + +#endif /* _APPLETYPECPHY_TRACE_H_ */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#include From e3ed55ca947e67fac029ec3917d1f2ca6371d360 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 30 Nov 2022 22:11:14 +0100 Subject: [PATCH 176/635] usb: typec: tipd: Clear interrupts first Right now the interrupt handler first reads all updated status registers and only then clears the interrupts. It's possible that a duplicate interrupt for a changed register or plug state comes in after the interrupts have been processed but before they have been cleared: * plug is inserted, TPS_REG_INT_PLUG_EVENT is set * TPS_REG_INT_EVENT1 is read * tps6598x_handle_plug_event() has run and registered the plug * plug is removed again, TPS_REG_INT_PLUG_EVENT is set (again) * TPS_REG_INT_CLEAR1 is written, TPS_REG_INT_PLUG_EVENT is cleared We then have no plug connected and no pending interrupt but the tipd core still thinks there is a plug. It's possible to trigger this with e.g. a slightly broken Type-C to USB A converter. Fix this by first clearing the interrupts and only then reading the updated registers. Fixes: 45188f27b3d0 ("usb: typec: tipd: Add support for Apple CD321X") Fixes: 0a4c005bd171 ("usb: typec: driver for TI TPS6598x USB Power Delivery controllers") Cc: stable Signed-off-by: Sven Peter --- drivers/usb/typec/tipd/core.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 485b90c13078a9..a2fb609e8ebcd3 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -494,24 +494,23 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) if (!event) goto err_unlock; + tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); + if (!tps6598x_read_status(tps, &status)) - goto err_clear_ints; + goto err_unlock; if (event & APPLE_CD_REG_INT_POWER_STATUS_UPDATE) if (!tps6598x_read_power_status(tps)) - goto err_clear_ints; + goto err_unlock; if (event & APPLE_CD_REG_INT_DATA_STATUS_UPDATE) if (!tps6598x_read_data_status(tps)) - goto err_clear_ints; + goto err_unlock; /* Handle plug insert or removal */ if (event & APPLE_CD_REG_INT_PLUG_EVENT) tps6598x_handle_plug_event(tps, status); -err_clear_ints: - tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event); - err_unlock: mutex_unlock(&tps->lock); @@ -541,25 +540,24 @@ static irqreturn_t tps6598x_interrupt(int irq, void *data) if (!(event1 | event2)) goto err_unlock; + tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event1); + tps6598x_write64(tps, TPS_REG_INT_CLEAR2, event2); + if (!tps6598x_read_status(tps, &status)) - goto err_clear_ints; + goto err_unlock; if ((event1 | event2) & TPS_REG_INT_POWER_STATUS_UPDATE) if (!tps6598x_read_power_status(tps)) - goto err_clear_ints; + goto err_unlock; if ((event1 | event2) & TPS_REG_INT_DATA_STATUS_UPDATE) if (!tps6598x_read_data_status(tps)) - goto err_clear_ints; + goto err_unlock; /* Handle plug insert or removal */ if ((event1 | event2) & TPS_REG_INT_PLUG_EVENT) tps6598x_handle_plug_event(tps, status); -err_clear_ints: - tps6598x_write64(tps, TPS_REG_INT_CLEAR1, event1); - tps6598x_write64(tps, TPS_REG_INT_CLEAR2, event2); - err_unlock: mutex_unlock(&tps->lock); From d3f1b337225a4b25ac51b815df59fd8eb15ce84e Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Fri, 27 Aug 2021 11:19:51 -0400 Subject: [PATCH 177/635] WIP: drm/apple: Add DCP display driver Add a DRM/KMS driver for Apple system on chips using the DCP coprocessor, namely the Apple M1. The DCP was added in Apple A14; this driver does not apply to older iDevices. This driver targets the DCP firmware API shipped by macOS 12.1. Currently no incompatibilities with macOS 12.0.1 or 12.2.1 are known. Signed-off-by: Alyssa Rosenzweig Co-developed-by: Janne Grunau Signed-off-by: Janne Grunau --- MAINTAINERS | 7 + drivers/gpu/drm/Kconfig | 2 + drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/apple/Kconfig | 11 + drivers/gpu/drm/apple/Makefile | 9 + drivers/gpu/drm/apple/apple_drv.c | 440 ++++++++ drivers/gpu/drm/apple/dcp.c | 1394 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 44 + drivers/gpu/drm/apple/dcpep.h | 406 ++++++++ drivers/gpu/drm/apple/dummy-piodma.c | 31 + drivers/gpu/drm/apple/parser.c | 451 +++++++++ drivers/gpu/drm/apple/parser.h | 32 + 12 files changed, 2828 insertions(+) create mode 100644 drivers/gpu/drm/apple/Kconfig create mode 100644 drivers/gpu/drm/apple/Makefile create mode 100644 drivers/gpu/drm/apple/apple_drv.c create mode 100644 drivers/gpu/drm/apple/dcp.c create mode 100644 drivers/gpu/drm/apple/dcp.h create mode 100644 drivers/gpu/drm/apple/dcpep.h create mode 100644 drivers/gpu/drm/apple/dummy-piodma.c create mode 100644 drivers/gpu/drm/apple/parser.c create mode 100644 drivers/gpu/drm/apple/parser.h diff --git a/MAINTAINERS b/MAINTAINERS index c6545eb541047e..d690d8d16092d6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1500,6 +1500,13 @@ L: linux-input@vger.kernel.org S: Odd fixes F: drivers/input/mouse/bcm5974.c +APPLE DRM DISPLAY DRIVER +M: Alyssa Rosenzweig +L: dri-devel@lists.freedesktop.org +S: Maintained +T: git git://anongit.freedesktop.org/drm/drm-misc +F: drivers/gpu/drm/apple/ + APPLE PCIE CONTROLLER DRIVER M: Alyssa Rosenzweig M: Marc Zyngier diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index dc0f94f02a82ed..a26c6010eea9ea 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -367,6 +367,8 @@ source "drivers/gpu/drm/solomon/Kconfig" source "drivers/gpu/drm/sprd/Kconfig" +source "drivers/gpu/drm/apple/Kconfig" + config DRM_HYPERV tristate "DRM Support for Hyper-V synthetic video device" depends on DRM && PCI && MMU && HYPERV diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index ab4460fcd63f6b..d57e1fbddbe1cc 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -182,6 +182,7 @@ obj-$(CONFIG_DRM_XEN) += xen/ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ obj-$(CONFIG_DRM_LIMA) += lima/ obj-$(CONFIG_DRM_PANFROST) += panfrost/ +obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig new file mode 100644 index 00000000000000..53c7d4edfd017e --- /dev/null +++ b/drivers/gpu/drm/apple/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DRM_APPLE + tristate "DRM Support for Apple display controllers" + depends on DRM && OF && ARM64 + depends on ARCH_APPLE || COMPILE_TEST + select DRM_KMS_HELPER + select DRM_KMS_DMA_HELPER + select DRM_GEM_DMA_HELPER + select VIDEOMODE_HELPERS + help + Say Y if you have an Apple Silicon chipset. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile new file mode 100644 index 00000000000000..db7ef359987d3d --- /dev/null +++ b/drivers/gpu/drm/apple/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only + +appledrm-y := apple_drv.o +apple_dcp-y := dcp.o parser.o +apple_piodma-y := dummy-piodma.o + +obj-$(CONFIG_DRM_APPLE) += appledrm.o +obj-$(CONFIG_DRM_APPLE) += apple_dcp.o +obj-$(CONFIG_DRM_APPLE) += apple_piodma.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c new file mode 100644 index 00000000000000..c0ad2e3d426909 --- /dev/null +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2021 Alyssa Rosenzweig */ +/* Based on meson driver which is + * Copyright (C) 2016 BayLibre, SAS + * Author: Neil Armstrong + * Copyright (C) 2015 Amlogic, Inc. All rights reserved. + * Copyright (C) 2014 Endless Mobile + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" + +#define DRIVER_NAME "apple" +#define DRIVER_DESC "Apple display controller DRM driver" + +#define FRAC_16_16(mult, div) (((mult) << 16) / (div)) + +#define MAX_COPROCESSORS 2 + +struct apple_drm_private { + struct drm_device drm; +}; + +DEFINE_DRM_GEM_DMA_FOPS(apple_fops); + +static const struct drm_driver apple_drm_driver = { + DRM_GEM_DMA_DRIVER_OPS, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = "20210901", + .major = 1, + .minor = 0, + .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC, + .fops = &apple_fops, +}; + +static int apple_plane_atomic_check(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *new_plane_state; + struct drm_crtc_state *crtc_state; + + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + + if (!new_plane_state->crtc) + return 0; + + crtc_state = drm_atomic_get_crtc_state(state, new_plane_state->crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + /* + * DCP limits downscaling to 2x and upscaling to 4x. Attempting to + * scale outside these bounds errors out when swapping. + * + * This function also takes care of clipping the src/dest rectangles, + * which is required for correct operation. Partially off-screen + * surfaces may appear corrupted. + * + * DCP does not distinguish plane types in the hardware, so we set + * can_position. If the primary plane does not fill the screen, the + * hardware will fill in zeroes (black). + */ + return drm_atomic_helper_check_plane_state(new_plane_state, + crtc_state, + FRAC_16_16(1, 4), + FRAC_16_16(2, 1), + true, true); +} + +static void apple_plane_atomic_update(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + /* Handled in atomic_flush */ +} + +static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { + .atomic_check = apple_plane_atomic_check, + .atomic_update = apple_plane_atomic_update, +}; + +static const struct drm_plane_funcs apple_plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .destroy = drm_plane_cleanup, + .reset = drm_atomic_helper_plane_reset, + .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, +}; + +/* + * Table of supported formats, mapping from DRM fourccs to DCP fourccs. + * + * For future work, DCP supports more formats not listed, including YUV + * formats, an extra RGBA format, and a biplanar RGB10_A8 format (fourcc b3a8) + * used for HDR. + * + * Note: we don't have non-alpha formats but userspace breaks without XRGB. It + * doesn't matter for the primary plane, but cursors/overlays must not + * advertise formats without alpha. + */ +static const u32 dcp_formats[] = { + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, +}; + +u64 apple_format_modifiers[] = { + DRM_FORMAT_MOD_LINEAR, + DRM_FORMAT_MOD_INVALID +}; + +static struct drm_plane *apple_plane_init(struct drm_device *dev, + enum drm_plane_type type) +{ + int ret; + struct drm_plane *plane; + + plane = devm_kzalloc(dev->dev, sizeof(*plane), GFP_KERNEL); + + ret = drm_universal_plane_init(dev, plane, 0x1, &apple_plane_funcs, + dcp_formats, ARRAY_SIZE(dcp_formats), + apple_format_modifiers, type, NULL); + if (ret) + return ERR_PTR(ret); + + drm_plane_helper_add(plane, &apple_plane_helper_funcs); + + return plane; +} + +static int apple_enable_vblank(struct drm_crtc *crtc) +{ + to_apple_crtc(crtc)->vsync_disabled = false; + + return 0; +} + +static void apple_disable_vblank(struct drm_crtc *crtc) +{ + to_apple_crtc(crtc)->vsync_disabled = true; +} + +static enum drm_connector_status +apple_connector_detect(struct drm_connector *connector, bool force) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + + return apple_connector->connected ? connector_status_connected : + connector_status_disconnected; +} + +static void apple_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + drm_crtc_vblank_on(crtc); +} + +static void apple_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + drm_crtc_vblank_off(crtc); + + if (crtc->state->event && !crtc->state->active) { + spin_lock_irq(&crtc->dev->event_lock); + drm_crtc_send_vblank_event(crtc, crtc->state->event); + spin_unlock_irq(&crtc->dev->event_lock); + + crtc->state->event = NULL; + } +} + +static void apple_crtc_atomic_begin(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + unsigned long flags; + + if (crtc->state->event) { + WARN_ON(drm_crtc_vblank_get(crtc) != 0); + + spin_lock_irqsave(&crtc->dev->event_lock, flags); + apple_crtc->event = crtc->state->event; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + crtc->state->event = NULL; + } +} + +void apple_crtc_vblank(struct apple_crtc *crtc) +{ + unsigned long flags; + + if (crtc->vsync_disabled) + return; + + drm_crtc_handle_vblank(&crtc->base); + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + drm_crtc_vblank_put(&crtc->base); + crtc->event = NULL; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + +static const struct drm_crtc_funcs apple_crtc_funcs = { + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .destroy = drm_crtc_cleanup, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .set_config = drm_atomic_helper_set_config, + .enable_vblank = apple_enable_vblank, + .disable_vblank = apple_disable_vblank, +}; + +static const struct drm_mode_config_funcs apple_mode_config_funcs = { + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, + .fb_create = drm_gem_fb_create, +}; + +static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { + .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, +}; + +static const struct drm_connector_funcs apple_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = drm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .detect = apple_connector_detect, +}; + +static const struct drm_connector_helper_funcs apple_connector_helper_funcs = { + .get_modes = dcp_get_modes, + .mode_valid = dcp_mode_valid, +}; + +static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { + .atomic_begin = apple_crtc_atomic_begin, + .atomic_flush = dcp_flush, + .atomic_enable = apple_crtc_atomic_enable, + .atomic_disable = apple_crtc_atomic_disable, +}; + +static int apple_probe_per_dcp(struct device *dev, + struct drm_device *drm, + struct platform_device *dcp) +{ + struct apple_crtc *crtc; + struct apple_connector *connector; + struct drm_encoder *encoder; + struct drm_plane *primary; + int ret; + + primary = apple_plane_init(drm, DRM_PLANE_TYPE_PRIMARY); + + if (IS_ERR(primary)) + return PTR_ERR(primary); + + crtc = devm_kzalloc(dev, sizeof(*crtc), GFP_KERNEL); + ret = drm_crtc_init_with_planes(drm, &crtc->base, primary, NULL, + &apple_crtc_funcs, NULL); + if (ret) + return ret; + + drm_crtc_helper_add(&crtc->base, &apple_crtc_helper_funcs); + + encoder = devm_kzalloc(dev, sizeof(*encoder), GFP_KERNEL); + encoder->possible_crtcs = drm_crtc_mask(&crtc->base); + ret = drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS); + if (ret) + return ret; + + connector = devm_kzalloc(dev, sizeof(*connector), GFP_KERNEL); + drm_connector_helper_add(&connector->base, + &apple_connector_helper_funcs); + + ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, + DRM_MODE_CONNECTOR_HDMIA); + if (ret) + return ret; + + connector->base.polled = DRM_CONNECTOR_POLL_HPD; + connector->connected = true; /* XXX */ + connector->dcp = dcp; + + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); + + crtc->dcp = dcp; + dcp_link(dcp, crtc, connector); + + return drm_connector_attach_encoder(&connector->base, encoder); +} + +static int apple_platform_probe(struct platform_device *pdev) +{ + struct apple_drm_private *apple; + struct platform_device *dcp[MAX_COPROCESSORS]; + int ret, nr_dcp, i; + + for (nr_dcp = 0; nr_dcp < MAX_COPROCESSORS; ++nr_dcp) { + struct device_node *np; + + np = of_parse_phandle(pdev->dev.of_node, "apple,coprocessors", + nr_dcp); + + if (!np) + break; + + dcp[nr_dcp] = of_find_device_by_node(np); + + if (!dcp[nr_dcp]) + return -ENODEV; + + /* DCP needs to be initialized before KMS can come online */ + if (!platform_get_drvdata(dcp[nr_dcp])) + return -EPROBE_DEFER; + + if (!dcp_is_initialized(dcp[nr_dcp])) + return -EPROBE_DEFER; + } + + /* Need at least 1 DCP for a display subsystem */ + if (nr_dcp < 1) + return -ENODEV; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + + apple = devm_drm_dev_alloc(&pdev->dev, &apple_drm_driver, + struct apple_drm_private, drm); + if (IS_ERR(apple)) + return PTR_ERR(apple); + + ret = drm_vblank_init(&apple->drm, 1); + if (ret) + return ret; + + ret = drmm_mode_config_init(&apple->drm); + if (ret) + goto err_unload; + + /* + * IOMFB::UPPipeDCP_H13P::verify_surfaces produces the error "plane + * requires a minimum of 32x32 for the source buffer" if smaller + */ + apple->drm.mode_config.min_width = 32; + apple->drm.mode_config.min_height = 32; + + /* Unknown maximum, use the iMac (24-inch, 2021) display resolution as + * maximum. + */ + apple->drm.mode_config.max_width = 4480; + apple->drm.mode_config.max_height = 2520; + + apple->drm.mode_config.funcs = &apple_mode_config_funcs; + apple->drm.mode_config.helper_private = &apple_mode_config_helpers; + + for (i = 0; i < nr_dcp; ++i) { + ret = apple_probe_per_dcp(&pdev->dev, &apple->drm, dcp[i]); + + if (ret) + goto err_unload; + } + + drm_mode_config_reset(&apple->drm); + + ret = drm_aperture_remove_framebuffers(false, &apple_drm_driver); + if (ret) + return ret; + + ret = drm_dev_register(&apple->drm, 0); + if (ret) + goto err_unload; + + drm_fbdev_generic_setup(&apple->drm, 32); + + return 0; + +err_unload: + drm_dev_put(&apple->drm); + return ret; +} + +static int apple_platform_remove(struct platform_device *pdev) +{ + struct drm_device *drm = platform_get_drvdata(pdev); + + drm_dev_unregister(drm); + + return 0; +} + +static const struct of_device_id of_match[] = { + { .compatible = "apple,display-subsystem" }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +static struct platform_driver apple_platform_driver = { + .driver = { + .name = "apple-drm", + .of_match_table = of_match, + }, + .probe = apple_platform_probe, + .remove = apple_platform_remove, +}; + +module_platform_driver(apple_platform_driver); + +MODULE_AUTHOR("Alyssa Rosenzweig "); +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c new file mode 100644 index 00000000000000..f635740f195a3a --- /dev/null +++ b/drivers/gpu/drm/apple/dcp.c @@ -0,0 +1,1394 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "dcpep.h" +#include "dcp.h" +#include "parser.h" + +struct apple_dcp; + +/* Register defines used in bandwidth setup structure */ +#define REG_SCRATCH (0x14) +#define REG_DOORBELL (0x0) +#define REG_DOORBELL_BIT (2) + +#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) + +/* Limit on call stack depth (arbitrary). Some nesting is required */ +#define DCP_MAX_CALL_DEPTH 8 + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + +struct dcp_call_channel { + dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; + void *cookies[DCP_MAX_CALL_DEPTH]; + void *output[DCP_MAX_CALL_DEPTH]; + u16 end[DCP_MAX_CALL_DEPTH]; + + /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ + u8 depth; +}; + +struct dcp_cb_channel { + u8 depth; + void *output[DCP_MAX_CALL_DEPTH]; +}; + +/* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ +struct dcp_chunks { + size_t length; + void *data; +}; + +#define DCP_MAX_MAPPINGS (128) /* should be enough */ +#define MAX_DISP_REGISTERS (7) + +struct apple_dcp { + struct device *dev; + struct platform_device *piodma; + struct apple_rtkit *rtk; + struct apple_crtc *crtc; + struct apple_connector *connector; + + /* DCP has crashed */ + bool crashed; + + /* DCP shared memory */ + void *shmem; + + /* Display registers mappable to the DCP */ + struct resource *disp_registers[MAX_DISP_REGISTERS]; + unsigned int nr_disp_registers; + + /* Number of memory mappings made by the DCP, used as an ID */ + u32 nr_mappings; + + /* Indexed table of mappings */ + struct sg_table mappings[DCP_MAX_MAPPINGS]; + + struct dcp_call_channel ch_cmd, ch_oobcmd; + struct dcp_cb_channel ch_cb, ch_oobcb, ch_async; + + /* Active chunked transfer. There can only be one at a time. */ + struct dcp_chunks chunks; + + /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ + struct dcp_swap_submit_req swap; + + /* Current display mode */ + bool valid_mode; + struct dcp_set_digital_out_mode_req mode; + + /* Is the DCP booted? */ + bool active; + + /* Modes valid for the connected display */ + struct dcp_display_mode *modes; + unsigned int nr_modes; + + /* Attributes of the connected display */ + int width_mm, height_mm; +}; + +/* + * A channel is busy if we have sent a message that has yet to be + * acked. The driver must not sent a message to a busy channel. + */ +static bool dcp_channel_busy(struct dcp_call_channel *ch) +{ + return (ch->depth != 0); +} + +/* Get a call channel for a context */ +static struct dcp_call_channel * +dcp_get_call_channel(struct apple_dcp *dcp, enum dcp_context_id context) +{ + switch (context) { + case DCP_CONTEXT_CMD: + case DCP_CONTEXT_CB: + return &dcp->ch_cmd; + case DCP_CONTEXT_OOBCMD: + case DCP_CONTEXT_OOBCB: + return &dcp->ch_oobcmd; + default: + return NULL; + } +} + +/* + * Get the context ID passed to the DCP for a command we push. The rule is + * simple: callback contexts are used when replying to the DCP, command + * contexts are used otherwise. That corresponds to a non/zero call stack + * depth. This rule frees the caller from tracking the call context manually. + */ +static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) +{ + u8 depth = oob ? dcp->ch_oobcmd.depth : dcp->ch_cmd.depth; + + if (depth) + return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; + else + return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; +} + +/* Get a callback channel for a context */ +static struct dcp_cb_channel *dcp_get_cb_channel(struct apple_dcp *dcp, + enum dcp_context_id context) +{ + switch (context) { + case DCP_CONTEXT_CB: + return &dcp->ch_cb; + case DCP_CONTEXT_OOBCB: + return &dcp->ch_oobcb; + case DCP_CONTEXT_ASYNC: + return &dcp->ch_async; + default: + return NULL; + } +} + +/* Get the start of a packet: after the end of the previous packet */ +static u16 dcp_packet_start(struct dcp_call_channel *ch, u8 depth) +{ + if (depth > 0) + return ch->end[depth - 1]; + else + return 0; +} + +/* Pushes and pops the depth of the call stack with safety checks */ +static u8 dcp_push_depth(u8 *depth) +{ + u8 ret = (*depth)++; + + WARN_ON(ret >= DCP_MAX_CALL_DEPTH); + return ret; +} + +static u8 dcp_pop_depth(u8 *depth) +{ + WARN_ON((*depth) == 0); + + return --(*depth); +} + +#define DCP_METHOD(tag, name) [name] = { #name, tag } + +const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + DCP_METHOD("A000", dcpep_late_init_signal), + DCP_METHOD("A029", dcpep_setup_video_limits), + DCP_METHOD("A034", dcpep_update_notify_clients_dcp), + DCP_METHOD("A357", dcpep_set_create_dfb), + DCP_METHOD("A401", dcpep_start_signal), + DCP_METHOD("A407", dcpep_swap_start), + DCP_METHOD("A408", dcpep_swap_submit), + DCP_METHOD("A410", dcpep_set_display_device), + DCP_METHOD("A412", dcpep_set_digital_out_mode), + DCP_METHOD("A443", dcpep_create_default_fb), + DCP_METHOD("A454", dcpep_first_client_open), + DCP_METHOD("A460", dcpep_set_display_refresh_properties), + DCP_METHOD("A463", dcpep_flush_supports_power), + DCP_METHOD("A468", dcpep_set_power_state), +}; + +/* Call a DCP function given by a tag */ +static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie) +{ + struct dcp_call_channel *ch = oob ? &dcp->ch_oobcmd : &dcp->ch_cmd; + enum dcp_context_id context = dcp_call_context(dcp, oob); + + struct dcp_packet_header header = { + .in_len = in_len, + .out_len = out_len, + + /* Tag is reversed due to endianness of the fourcc */ + .tag[0] = dcp_methods[method].tag[3], + .tag[1] = dcp_methods[method].tag[2], + .tag[2] = dcp_methods[method].tag[1], + .tag[3] = dcp_methods[method].tag[0], + }; + + u8 depth = dcp_push_depth(&ch->depth); + u16 offset = dcp_packet_start(ch, depth); + + void *out = dcp->shmem + dcp_tx_offset(context) + offset; + void *out_data = out + sizeof(header); + size_t data_len = sizeof(header) + in_len + out_len; + + memcpy(out, &header, sizeof(header)); + + if (in_len > 0) + memcpy(out_data, data, in_len); + + dev_dbg(dcp->dev, "---> %s: context %u, offset %u, depth %u\n", + dcp_methods[method].name, context, offset, depth); + + ch->callbacks[depth] = cb; + ch->cookies[depth] = cookie; + ch->output[depth] = out + sizeof(header) + in_len; + ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); + + apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, + dcpep_msg(context, data_len, offset), + NULL, false); +} + +#define DCP_THUNK_VOID(func, handle) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, handle, 0, 0, NULL, cb, cookie); \ + } + +#define DCP_THUNK_OUT(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, handle, 0, sizeof(T), NULL, cb, cookie); \ + } + +#define DCP_THUNK_IN(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, T *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, handle, sizeof(T), 0, data, cb, cookie); \ + } + +#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ + static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, handle, sizeof(T_in), sizeof(T_out), data, \ + cb, cookie); \ + } + +DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, + struct dcp_swap_submit_resp); + +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, + struct dcp_swap_start_resp); + +DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, + struct dcp_set_power_state_req, + struct dcp_set_power_state_resp); + +DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, + struct dcp_set_digital_out_mode_req, u32); + +DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); + +DCP_THUNK_OUT(dcp_set_display_refresh_properties, + dcpep_set_display_refresh_properties, u32); + +DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); +DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); +DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); +DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); +DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); + +__attribute__((unused)) +DCP_THUNK_IN(dcp_update_notify_clients_dcp, dcpep_update_notify_clients_dcp, + struct dcp_update_notify_clients_dcp); + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +static int dcp_parse_tag(char tag[4]) +{ + u32 d[3]; + int i; + + if (tag[3] != 'D') + return -EINVAL; + + for (i = 0; i < 3; ++i) { + d[i] = (u32)(tag[i] - '0'); + + if (d[i] > 9) + return -EINVAL; + } + + return d[0] + (d[1] * 10) + (d[2] * 100); +} + +/* Ack a callback from the DCP */ +static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) +{ + struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); + + dcp_pop_depth(&ch->depth); + apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, dcpep_ack(context), + NULL, false); +} + +/* DCP callback handlers */ +static void dcpep_cb_nop(struct apple_dcp *dcp) +{ + /* No operation */ +} + +static u8 dcpep_cb_true(struct apple_dcp *dcp) +{ + return true; +} + +static u8 dcpep_cb_false(struct apple_dcp *dcp) +{ + return false; +} + +static u32 dcpep_cb_zero(struct apple_dcp *dcp) +{ + return 0; +} + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp) +{ + apple_crtc_vblank(dcp->crtc); +} + +static struct dcp_get_uint_prop_resp +dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) +{ + /* unimplemented for now */ + return (struct dcp_get_uint_prop_resp) { + .value = 0 + }; +} + +/* + * Callback to map a buffer allocated with allocate_buf for PIODMA usage. + * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated + * stream of the display DART, rather than the expected DCP DART. + * + * XXX: This relies on dma_get_sgtable in concert with dma_map_sgtable, which + * is a "fundamentally unsafe" operation according to the docs. And yet + * everyone does it... + */ +static struct dcp_map_buf_resp +dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) +{ + struct sg_table *map; + int ret; + + if (req->buffer >= ARRAY_SIZE(dcp->mappings)) + goto reject; + + map = &dcp->mappings[req->buffer]; + + if (!map->sgl) + goto reject; + + /* Use PIODMA device instead of DCP to map against the right IOMMU. */ + ret = dma_map_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); + + if (ret) + goto reject; + + return (struct dcp_map_buf_resp) { + .dva = sg_dma_address(map->sgl) + }; + +reject: + dev_err(dcp->dev, "denying map of invalid buffer %llx for pidoma\n", + req->buffer); + return (struct dcp_map_buf_resp) { + .ret = EINVAL + }; +} + +/* + * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be + * physically contigiuous, however we should save the sgtable in case the + * buffer needs to be later mapped for PIODMA. + */ +static struct dcp_allocate_buffer_resp +dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_allocate_buffer_req *req) +{ + struct dcp_allocate_buffer_resp resp = { 0 }; + void *buf; + + resp.dva_size = ALIGN(req->size, 4096); + resp.mem_desc_id = ++dcp->nr_mappings; + + if (resp.mem_desc_id >= ARRAY_SIZE(dcp->mappings)) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring"); + return resp; + } + + buf = dma_alloc_coherent(dcp->dev, resp.dva_size, &resp.dva, + GFP_KERNEL); + + dma_get_sgtable(dcp->dev, &dcp->mappings[resp.mem_desc_id], buf, + resp.dva, resp.dva_size); + return resp; +} + +/* Validate that the specified region is a display register */ +static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +{ + int i; + + for (i = 0; i < dcp->nr_disp_registers; ++i) { + struct resource *r = dcp->disp_registers[i]; + + if ((start >= r->start) && (end <= r->end)) + return true; + } + + return false; +} + +/* + * Map contiguous physical memory into the DCP's address space. The firmware + * uses this to map the display registers we advertise in + * sr_map_device_memory_with_index, so we bounds check against that to guard + * safe against malicious coprocessors. + */ +static struct dcp_map_physical_resp +dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) +{ + int size = ALIGN(req->size, 4096); + + if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + dev_err(dcp->dev, "refusing to map phys address %llx size %llx", + req->paddr, req->size); + return (struct dcp_map_physical_resp) { }; + } + + return (struct dcp_map_physical_resp) { + .dva_size = size, + .mem_desc_id = ++dcp->nr_mappings, + .dva = dma_map_resource(dcp->dev, req->paddr, size, + DMA_BIDIRECTIONAL, 0), + }; +} + +static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) +{ + /* Pixel clock frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + return 533333328; +} + +static struct dcp_map_reg_resp +dcpep_cb_map_reg(struct apple_dcp *dcp, struct dcp_map_reg_req *req) +{ + if (req->index >= dcp->nr_disp_registers) { + dev_warn(dcp->dev, "attempted to read invalid reg index %u", + req->index); + + return (struct dcp_map_reg_resp) { + .ret = 1 + }; + } else { + struct resource *rsrc = dcp->disp_registers[req->index]; + + return (struct dcp_map_reg_resp) { + .addr = rsrc->start, + .length = resource_size(rsrc) + }; + } +} + +/* Chunked data transfer for property dictionaries */ +static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) +{ + if (dcp->chunks.data != NULL) { + dev_warn(dcp->dev, "ignoring spurious transfer start\n"); + return false; + } + + dcp->chunks.length = *length; + dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "failed to allocate chunks\n"); + return false; + } + + return true; +} + +static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_chunk_req *req) +{ + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious chunk\n"); + return false; + } + + if (req->offset + req->length > dcp->chunks.length) { + dev_warn(dcp->dev, "ignoring overflowing chunk\n"); + return false; + } + + memcpy(dcp->chunks.data + req->offset, req->data, req->length); + return true; +} + +static void dcp_set_dimensions(struct apple_dcp *dcp) +{ + int i; + + /* Set the connector info */ + if (dcp->connector) { + struct drm_connector *connector = &dcp->connector->base; + + mutex_lock(&connector->dev->mode_config.mutex); + connector->display_info.width_mm = dcp->width_mm; + connector->display_info.height_mm = dcp->height_mm; + mutex_unlock(&connector->dev->mode_config.mutex); + } + + /* + * Fix up any probed modes. Modes are created when parsing + * TimingElements, dimensions are calculated when parsing + * DisplayAttributes, and TimingElements may be sent first + */ + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = dcp->width_mm; + dcp->modes[i].mode.height_mm = dcp->height_mm; + } +} + +static bool dcpep_process_chunks(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + struct dcp_parse_ctx ctx; + int ret; + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious end\n"); + return false; + } + + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); + + if (ret) { + dev_warn(dcp->dev, "bad header on dcpav props\n"); + return false; + } + + if (!strcmp(req->key, "TimingElements")) { + dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, + dcp->width_mm, dcp->height_mm); + + if (IS_ERR(dcp->modes)) { + dev_warn(dcp->dev, "failed to parse modes\n"); + dcp->modes = NULL; + dcp->nr_modes = 0; + return false; + } + } else if (!strcmp(req->key, "DisplayAttributes")) { + ret = parse_display_attributes(&ctx, &dcp->width_mm, + &dcp->height_mm); + + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } + + dcp_set_dimensions(dcp); + } + + return true; +} + +static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + u8 resp = dcpep_process_chunks(dcp, req); + + /* Reset for the next transfer */ + devm_kfree(dcp->dev, dcp->chunks.data); + dcp->chunks.data = NULL; + + return resp; +} + +/* Boot sequence */ +static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_cb_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); +} + +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_late_init_signal(dcp, false, boot_5, NULL); +} + +static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 v_true = true; + + dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); +} + +static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_setup_video_limits(dcp, false, boot_3, NULL); +} + +static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_create_default_fb(dcp, false, boot_2, NULL); +} + +/* Use special function signature to defer the ACK */ +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, void *out, void *in) +{ + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); + return false; +} + +static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) +{ + return (struct dcp_rt_bandwidth) { + .reg_scratch = dcp->disp_registers[5]->start + REG_SCRATCH, + .reg_doorbell = dcp->disp_registers[6]->start + REG_DOORBELL, + .doorbell_bit = REG_DOORBELL_BIT, + + .padding[3] = 0x4, // XXX: required by 11.x firmware + }; +} + +/* Callback to get the current time as milliseconds since the UNIX epoch */ +static u64 dcpep_cb_get_time(struct apple_dcp *dcp) +{ + return ktime_to_ms(ktime_get_real()); +} + +/* + * Helper to send a DRM hotplug event. The DCP is accessed from a single + * (RTKit) thread. To handle hotplug callbacks, we need to call + * drm_kms_helper_hotplug_event, which does an atomic commit (via DCP) and + * waits for vblank (a DCP callback). That means we deadlock if we call from + * the RTKit thread! Instead, move the call to another thread via a workqueue. + */ +void dcp_hotplug(struct work_struct *work) +{ + struct apple_connector *connector; + struct drm_device *dev; + + connector = container_of(work, struct apple_connector, hotplug_wq); + dev = connector->base.dev; + + /* + * DCP defers link training until we set a display mode. But we set + * display modes from atomic_flush, so userspace needs to trigger a + * flush, or the CRTC gets no signal. + */ + if (connector->connected) { + drm_connector_set_link_status_property( + &connector->base, DRM_MODE_LINK_STATUS_BAD); + } + + if (dev && dev->registered) + drm_kms_helper_hotplug_event(dev); +} +EXPORT_SYMBOL_GPL(dcp_hotplug); + +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + struct apple_connector *connector = dcp->connector; + + /* Hotplug invalidates mode. DRM doesn't always handle this. */ + dcp->valid_mode = false; + + if (connector) { + connector->connected = !!(*connected); + schedule_work(&connector->hotplug_wq); + } +} + +#define DCPEP_MAX_CB (1000) + +/* + * Define type-safe trampolines. Define typedefs to enforce type-safety on the + * input data (so if the types don't match, gcc errors out). + */ + +#define TRAMPOLINE_VOID(func, handler) \ + static bool func(struct apple_dcp *dcp, void *out, void *in) \ + { \ + dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + handler(dcp); \ + return true; \ + } + +#define TRAMPOLINE_IN(func, handler, T_in) \ + typedef void (*callback_##name)(struct apple_dcp *, T_in *); \ + \ + static bool func(struct apple_dcp *dcp, void *out, void *in) \ + { \ + callback_##name cb = handler; \ + \ + dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ + typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool func(struct apple_dcp *dcp, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + callback_##handler cb = handler; \ + \ + dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + *typed_out = cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_OUT(func, handler, T_out) \ + static bool func(struct apple_dcp *dcp, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + \ + dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + *typed_out = handler(dcp); \ + return true; \ + } + +TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); +TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); +TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); +TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); +TRAMPOLINE_VOID(trampoline_swap_complete, dcpep_cb_swap_complete); +TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, + struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, + struct dcp_map_buf_req, struct dcp_map_buf_resp); +TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, + struct dcp_allocate_buffer_req, + struct dcp_allocate_buffer_resp); +TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, + struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct dcp_map_reg_req, + struct dcp_map_reg_resp); +TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); +TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, + struct dcp_set_dcpav_prop_chunk_req, u8); +TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, + struct dcp_set_dcpav_prop_end_req, u8); +TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, + struct dcp_rt_bandwidth); +TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); +TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); + +bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, void *, void *) = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [100] = trampoline_nop, /* match_pmu_service */ + [101] = trampoline_zero, /* get_display_default_stride */ + [103] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_false, /* create_backlight_service */ + [116] = dcpep_cb_boot_1, + [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [120] = trampoline_false, /* read_edt_data */ + [122] = trampoline_prop_start, + [123] = trampoline_prop_chunk, + [124] = trampoline_prop_end, + [201] = trampoline_map_piodma, + [206] = trampoline_true, /* match_pmu_service_2 */ + [207] = trampoline_true, /* match_backlight_service */ + [208] = trampoline_get_time, + [211] = trampoline_nop, /* update_backlight_factor_prop */ + [300] = trampoline_nop, /* pr_publish */ + [401] = trampoline_get_uint_prop, + [406] = trampoline_nop, /* set_fx_prop */ + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_true, /* sr_set_property_int */ + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [589] = trampoline_swap_complete, + [591] = trampoline_nop, /* swap_complete_intent_gated */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; + +static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct device *dev = dcp->dev; + struct dcp_packet_header *hdr = data; + void *in, *out; + int tag = dcp_parse_tag(hdr->tag); + struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); + u8 depth; + + if (tag < 0 || tag >= DCPEP_MAX_CB || !dcpep_cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + return; + } + + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + + if (dcpep_cb_handlers[tag](dcp, out, in)) + dcp_ack(dcp, context); +} + +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct dcp_packet_header *header = data; + struct dcp_call_channel *ch = dcp_get_call_channel(dcp, context); + void *cookie; + dcp_callback_t cb; + + if (!ch) { + dev_warn(dcp->dev, "ignoring ack on context %X\n", context); + return; + } + + dcp_pop_depth(&ch->depth); + + cb = ch->callbacks[ch->depth]; + cookie = ch->cookies[ch->depth]; + + if (cb) + cb(dcp, data + sizeof(*header) + header->in_len, cookie); +} + +static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcp_context_id ctx_id; + u16 offset; + u32 length; + int channel_offset; + void *data; + + ctx_id = (message & DCPEP_CONTEXT_MASK) >> DCPEP_CONTEXT_SHIFT; + offset = (message & DCPEP_OFFSET_MASK) >> DCPEP_OFFSET_SHIFT; + length = (message >> DCPEP_LENGTH_SHIFT); + + channel_offset = dcp_channel_offset(ctx_id); + + if (channel_offset < 0) { + dev_warn(dcp->dev, "invalid context received %u", ctx_id); + return; + } + + data = dcp->shmem + channel_offset + offset; + + if (message & DCPEP_ACK) + dcpep_handle_ack(dcp, ctx_id, data, length); + else + dcpep_handle_cb(dcp, ctx_id, data, length); +} + +/* + * Callback for swap requests. If a swap failed, we'll never get a swap + * complete event so we need to fake a vblank event early to avoid a hang. + */ + +static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_submit_resp *resp = data; + + if (resp->ret) { + dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + apple_crtc_vblank(dcp->crtc); + } +} + +static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + + dcp->swap.swap.swap_id = resp->swap_id; + + dcp_swap_submit(dcp, false, &dcp->swap, dcp_swapped, NULL); +} + +/* + * DRM specifies rectangles as start and end coordinates. DCP specifies + * rectangles as a start coordinate and a width/height. Convert a DRM rectangle + * to a DCP rectangle. + */ +static struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect) +{ + return (struct dcp_rect) { + .x = rect->x1, + .y = rect->y1, + .w = drm_rect_width(rect), + .h = drm_rect_height(rect) + }; +} + +static u32 drm_format_to_dcp(u32 drm) +{ + switch (drm) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + return fourcc_code('A', 'R', 'G', 'B'); + + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ABGR8888: + return fourcc_code('A', 'B', 'G', 'R'); + } + + pr_warn("DRM format %X not supported in DCP\n", drm); + return 0; +} + +int dcp_get_modes(struct drm_connector *connector) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode; + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); + + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + return 0; + } + + drm_mode_probed_add(connector, mode); + } + + return dcp->nr_modes; +} +EXPORT_SYMBOL_GPL(dcp_get_modes); + +/* The user may own drm_display_mode, so we need to search for our copy */ +static struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + struct drm_display_mode *mode) +{ + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + if (drm_mode_match(mode, &dcp->modes[i].mode, + DRM_MODE_MATCH_TIMINGS | + DRM_MODE_MATCH_CLOCK)) + return &dcp->modes[i]; + } + + return NULL; +} + +int dcp_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; +} +EXPORT_SYMBOL_GPL(dcp_mode_valid); + +/* Helpers to modeset and swap, used to flush */ +static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_req start_req = { 0 }; + + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); +} + +static void dcp_modeset(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_digital_out_mode(dcp, false, &dcp->mode, do_swap, NULL); +} + +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_plane *plane; + struct drm_plane_state *new_state, *old_state; + struct drm_crtc_state *crtc_state; + struct dcp_swap_submit_req *req = &dcp->swap; + int l; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + if (WARN(dcp_channel_busy(&dcp->ch_cmd), "unexpected busy channel") || + WARN(!dcp->connector->connected, "can't flush if disconnected")) { + apple_crtc_vblank(dcp->crtc); + return; + } + + /* Reset to defaults */ + memset(req, 0, sizeof(*req)); + for (l = 0; l < SWAP_SURFACES; l++) + req->surf_null[l] = true; + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, l) { + struct drm_framebuffer *fb = new_state->fb; + struct drm_rect src_rect; + + WARN_ON(l >= SWAP_SURFACES); + + req->swap.swap_enabled |= BIT(l); + + if (!new_state->fb) { + if (old_state->fb) + req->swap.swap_enabled |= DCP_REMOVE_LAYERS; + + continue; + } + req->surf_null[l] = false; + + // XXX: awful hack! race condition between a framebuffer unbind + // getting swapped out and GEM unreferencing a framebuffer. If + // we lose the race, the display gets IOVA faults and the DCP + // crashes. We need to extend the lifetime of the + // drm_framebuffer (and hence the GEM object) until after we + // get a swap complete for the swap unbinding it. + drm_framebuffer_get(fb); + + drm_rect_fp_to_int(&src_rect, &new_state->src); + + req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); + req->swap.dst_rect[l] = drm_to_dcp_rect(&new_state->dst); + + req->surf_iova[l] = drm_fb_dma_get_gem_addr(fb, new_state, 0); + + req->surf[l] = (struct dcp_surface) { + .format = drm_format_to_dcp(fb->format->format), + .xfer_func = 13, + .colorspace = 1, + .stride = fb->pitches[0], + .width = fb->width, + .height = fb->height, + .buf_size = fb->height * fb->pitches[0], + .surface_id = req->swap.surf_ids[l], + + /* Only used for compressed or multiplanar surfaces */ + .pix_size = 1, + .pel_w = 1, + .pel_h = 1, + .has_comp = 1, + .has_planes = 1, + }; + } + + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + + if (drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode) { + struct dcp_display_mode *mode; + u32 handle = 2; + + mode = lookup_mode(dcp, &crtc_state->mode); + if (!mode) { + dev_warn(dcp->dev, "no match for " DRM_MODE_FMT, + DRM_MODE_ARG(&crtc_state->mode)); + schedule_work(&dcp->vblank_wq); + return; + } + + dcp->mode = (struct dcp_set_digital_out_mode_req) { + .color_mode_id = mode->color_mode_id, + .timing_mode_id = mode->timing_mode_id + }; + + dcp->valid_mode = true; + + dcp_set_display_device(dcp, false, &handle, dcp_modeset, NULL); + } else + do_swap(dcp, NULL, NULL); +} +EXPORT_SYMBOL_GPL(dcp_flush); + +bool dcp_is_initialized(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->active; +} +EXPORT_SYMBOL_GPL(dcp_is_initialized); + +static void init_done(struct apple_dcp *dcp, void *out, void *cookie) +{ +} + +static void init_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + dcp_set_power_state(dcp, false, &req, init_done, NULL); +} + +static void init_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_first_client_open(dcp, false, init_3, NULL); +} + +static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + dev_info(dcp->dev, "DCP booted\n"); + + init_2(dcp, data, cookie); + + dcp->active = true; +} + +static void dcp_got_msg(void *cookie, u8 endpoint, u64 message) +{ + struct apple_dcp *dcp = cookie; + enum dcpep_type type = (message >> DCPEP_TYPE_SHIFT) & DCPEP_TYPE_MASK; + + WARN_ON(endpoint != DCP_ENDPOINT); + + if (type == DCPEP_TYPE_INITIALIZED) + dcp_start_signal(dcp, false, dcp_started, NULL); + else if (type == DCPEP_TYPE_MESSAGE) + dcpep_got_msg(dcp, message); + else + dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); +} + +static void dcp_rtk_crashed(void *cookie) +{ + struct apple_dcp *dcp = cookie; + + dcp->crashed = true; + dev_err(dcp->dev, "DCP has crashed"); +} + +static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp *dcp = cookie; + + if (bfr->iova) { + struct iommu_domain *domain = iommu_get_domain_for_dev(dcp->dev); + phys_addr_t phy_addr; + + if (!domain) + return -ENOMEM; + + // TODO: get map from device-tree + phy_addr = iommu_iova_to_phys(domain, bfr->iova & 0xFFFFFFFF); + if (!phy_addr) + return -ENOMEM; + + // TODO: verify phy_addr, cache attribute + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + + bfr->is_mapped = true; + dev_info(dcp->dev, "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx", + (uintptr_t)bfr->iova, (uintptr_t)phy_addr, (uintptr_t)bfr->buffer); + } else { + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + + dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx", + (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); + } + + return 0; +} + +static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp *dcp = cookie; + + if (bfr->is_mapped) + memunmap(bfr->buffer); + else + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static struct apple_rtkit_ops rtkit_ops = { + .crashed = dcp_rtk_crashed, + .recv_message = dcp_got_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + +void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + dcp->crtc = crtc; + dcp->connector = connector; + + /* Dimensions might already be parsed */ + dcp_set_dimensions(dcp); +} +EXPORT_SYMBOL_GPL(dcp_link); + +static struct platform_device *dcp_get_dev(struct device *dev, const char *name) +{ + struct device_node *node = of_get_child_by_name(dev->of_node, name); + + if (!node) + return NULL; + + return of_find_device_by_node(node); +} + +static int dcp_get_disp_regs(struct apple_dcp *dcp) +{ + struct platform_device *pdev = to_platform_device(dcp->dev); + int count = pdev->num_resources - 1; + int i; + + if (count <= 0 || count > MAX_DISP_REGISTERS) + return -EINVAL; + + for (i = 0; i < count; ++i) { + dcp->disp_registers[i] = + platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + } + + dcp->nr_disp_registers = count; + return 0; +} + +static int dcp_platform_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct apple_dcp *dcp; + dma_addr_t shmem_iova; + int ret; + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + + platform_set_drvdata(pdev, dcp); + dcp->dev = dev; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); + if (ret) + return ret; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "coproc"); + if (!res) + return -EINVAL; + + of_platform_default_populate(dev->of_node, NULL, dev); + + dcp->piodma = dcp_get_dev(dev, "piodma"); + if (!dcp->piodma) { + dev_err(dev, "failed to find piodma\n"); + return -ENODEV; + } + + ret = dcp_get_disp_regs(dcp); + if (ret) { + dev_err(dev, "failed to find display registers\n"); + return ret; + } + + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + if (IS_ERR(dcp->rtk)) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to intialize RTKit"); + + ret = apple_rtkit_wake(dcp->rtk); + if (ret) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to boot RTKit: %d", ret); + + apple_rtkit_start_ep(dcp->rtk, DCP_ENDPOINT); + + dcp->shmem = dma_alloc_coherent(dev, DCP_SHMEM_SIZE, &shmem_iova, + GFP_KERNEL); + + apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, + dcpep_set_shmem(shmem_iova), NULL, false); + + return ret; +} + +/* + * We need to shutdown DCP before tearing down the display subsystem. Otherwise + * the DCP will crash and briefly flash a green screen of death. + */ +static void dcp_platform_shutdown(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + struct dcp_set_power_state_req req = { + /* defaults are ok */ + }; + + dcp_set_power_state(dcp, false, &req, NULL, NULL); +} + +static const struct of_device_id of_match[] = { + { .compatible = "apple,dcp" }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +static struct platform_driver apple_platform_driver = { + .probe = dcp_platform_probe, + .shutdown = dcp_platform_shutdown, + .driver = { + .name = "apple-dcp", + .of_match_table = of_match, + }, +}; + +module_platform_driver(apple_platform_driver); + +MODULE_AUTHOR("Alyssa Rosenzweig "); +MODULE_DESCRIPTION("Apple Display Controller DRM driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h new file mode 100644 index 00000000000000..4582fe984c8aa5 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_H__ +#define __APPLE_DCP_H__ + +#include +#include "parser.h" + +struct apple_crtc { + struct drm_crtc base; + struct drm_pending_vblank_event *event; + bool vsync_disabled; + + /* Reference to the DCP device owning this CRTC */ + struct platform_device *dcp; +}; + +#define to_apple_crtc(x) container_of(x, struct apple_crtc, base) + +void dcp_hotplug(struct work_struct *work); + +struct apple_connector { + struct drm_connector base; + bool connected; + + struct platform_device *dcp; + + /* Workqueue for sending hotplug events to the associated device */ + struct work_struct hotplug_wq; +}; + +#define to_apple_connector(x) container_of(x, struct apple_connector, base) + +void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, + struct apple_connector *connector); +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); +bool dcp_is_initialized(struct platform_device *pdev); +void apple_crtc_vblank(struct apple_crtc *apple); +int dcp_get_modes(struct drm_connector *connector); +int dcp_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode); + +#endif diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/dcpep.h new file mode 100644 index 00000000000000..6301bf8f8c21d1 --- /dev/null +++ b/drivers/gpu/drm/apple/dcpep.h @@ -0,0 +1,406 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCPEP_H__ +#define __APPLE_DCPEP_H__ + +/* Endpoint for general DCP traffic (dcpep in macOS) */ +#define DCP_ENDPOINT 0x37 + +/* Fixed size of shared memory between DCP and AP */ +#define DCP_SHMEM_SIZE 0x100000 + +/* DCP message contexts */ +enum dcp_context_id { + /* Callback */ + DCP_CONTEXT_CB = 0, + + /* Command */ + DCP_CONTEXT_CMD = 2, + + /* Asynchronous */ + DCP_CONTEXT_ASYNC = 3, + + /* Out-of-band callback */ + DCP_CONTEXT_OOBCB = 4, + + /* Out-of-band command */ + DCP_CONTEXT_OOBCMD = 6, + + DCP_NUM_CONTEXTS +}; + +static int dcp_tx_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_CMD: return 0x00000; + case DCP_CONTEXT_OOBCB: + case DCP_CONTEXT_OOBCMD: return 0x08000; + default: return -EINVAL; + } +} + +static int dcp_channel_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_ASYNC: return 0x40000; + case DCP_CONTEXT_CB: return 0x60000; + case DCP_CONTEXT_OOBCB: return 0x68000; + default: return dcp_tx_offset(id); + } +} + +/* RTKit endpoint message types */ +enum dcpep_type { + /* Set shared memory */ + DCPEP_TYPE_SET_SHMEM = 0, + + /* DCP is initialized */ + DCPEP_TYPE_INITIALIZED = 1, + + /* Remote procedure call */ + DCPEP_TYPE_MESSAGE = 2, +}; + +/* Message */ +#define DCPEP_TYPE_SHIFT (0) +#define DCPEP_TYPE_MASK GENMASK(1, 0) +#define DCPEP_ACK BIT_ULL(6) +#define DCPEP_CONTEXT_SHIFT (8) +#define DCPEP_CONTEXT_MASK GENMASK(11, 8) +#define DCPEP_OFFSET_SHIFT (16) +#define DCPEP_OFFSET_MASK GENMASK(31, 16) +#define DCPEP_LENGTH_SHIFT (32) + +/* Set shmem */ +#define DCPEP_DVA_SHIFT (16) +#define DCPEP_FLAG_SHIFT (4) +#define DCPEP_FLAG_VALUE (4) + +struct dcp_packet_header { + char tag[4]; + u32 in_len; + u32 out_len; +} __packed; + +#define DCP_IS_NULL(ptr) ((ptr) ? 1 : 0) +#define DCP_PACKET_ALIGNMENT (0x40) + +static inline u64 +dcpep_set_shmem(u64 dart_va) +{ + return (DCPEP_TYPE_SET_SHMEM << DCPEP_TYPE_SHIFT) | + (DCPEP_FLAG_VALUE << DCPEP_FLAG_SHIFT) | + (dart_va << DCPEP_DVA_SHIFT); +} + +static inline u64 +dcpep_msg(enum dcp_context_id id, u32 length, u16 offset) +{ + return (DCPEP_TYPE_MESSAGE << DCPEP_TYPE_SHIFT) | + ((u64) id << DCPEP_CONTEXT_SHIFT) | + ((u64) offset << DCPEP_OFFSET_SHIFT) | + ((u64) length << DCPEP_LENGTH_SHIFT); +} + +static inline u64 +dcpep_ack(enum dcp_context_id id) +{ + return dcpep_msg(id, 0, 0) | DCPEP_ACK; +} + +/* Structures used in v12.0 firmware */ + +#define SWAP_SURFACES 4 +#define MAX_PLANES 3 + +struct dcp_iouserclient { + /* Handle for the IOUserClient. macOS sets this to a kernel VA. */ + u64 handle; + u32 unk; + u8 flag1; + u8 flag2; + u8 padding[2]; +} __packed; + +struct dcp_rect { + u32 x; + u32 y; + u32 w; + u32 h; +} __packed; + +/* + * Set in the swap_{enabled,completed} field to remove missing + * layers. Without this flag, the DCP will assume missing layers have + * not changed since the previous frame and will preserve their + * content. + */ +#define DCP_REMOVE_LAYERS BIT(31) + +struct dcp_swap { + u64 ts1; + u64 ts2; + u64 unk_10[6]; + u64 flags1; + u64 flags2; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 unk_10c; + u8 unk_110[0x1b8]; + u32 unk_2c8; + u8 unk_2cc[0x14]; + u32 unk_2e0; + u8 unk_2e4[0x3c]; +} __packed; + +/* Information describing a plane of a planar compressed surface */ +struct dcp_plane_info { + u32 width; + u32 height; + u32 base; + u32 offset; + u32 stride; + u32 size; + u16 tile_size; + u8 tile_w; + u8 tile_h; + u32 unk[13]; +} __packed; + +struct dcp_component_types { + u8 count; + u8 types[7]; +} __packed; + +/* Information describing a surface */ +struct dcp_surface { + u8 is_tiled; + u8 unk_1; + u8 unk_2; + u32 plane_cnt; + u32 plane_cnt2; + u32 format; /* DCP fourcc */ + u32 unk_f; + u8 xfer_func; + u8 colorspace; + u32 stride; + u16 pix_size; + u8 pel_w; + u8 pel_h; + u32 offset; + u32 width; + u32 height; + u32 buf_size; + u32 unk_2d; + u32 unk_31; + u32 surface_id; + struct dcp_component_types comp_types[MAX_PLANES]; + u64 has_comp; + struct dcp_plane_info planes[MAX_PLANES]; + u64 has_planes; + u32 compression_info[MAX_PLANES][13]; + u64 has_compr_info; + u64 unk_1f5; + u8 padding[7]; +} __packed; + +struct dcp_rt_bandwidth { + u64 unk1; + u64 reg_scratch; + u64 reg_doorbell; + u32 unk2; + u32 doorbell_bit; + u32 padding[7]; +} __packed; + +/* Method calls */ + +enum dcpep_method { + dcpep_late_init_signal, + dcpep_setup_video_limits, + dcpep_set_create_dfb, + dcpep_start_signal, + dcpep_swap_start, + dcpep_swap_submit, + dcpep_set_display_device, + dcpep_set_digital_out_mode, + dcpep_create_default_fb, + dcpep_set_display_refresh_properties, + dcpep_flush_supports_power, + dcpep_set_power_state, + dcpep_first_client_open, + dcpep_update_notify_clients_dcp, + dcpep_num_methods +}; + +struct dcp_method_entry { + const char *name; + char tag[4]; +}; + +/* Prototypes */ + +struct dcp_set_digital_out_mode_req { + u32 color_mode_id; + u32 timing_mode_id; +} __packed; + +struct dcp_map_buf_req { + u64 buffer; + u8 unk; + u8 buf_null; + u8 vaddr_null; + u8 dva_null; +} __packed; + +struct dcp_map_buf_resp { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; + +struct dcp_allocate_buffer_req { + u32 unk0; + u64 size; + u32 unk2; + u8 paddr_null; + u8 dva_null; + u8 dva_size_null; + u8 padding; +} __packed; + +struct dcp_allocate_buffer_resp { + u64 paddr; + u64 dva; + u64 dva_size; + u32 mem_desc_id; +} __packed; + +struct dcp_map_physical_req { + u64 paddr; + u64 size; + u32 flags; + u8 dva_null; + u8 dva_size_null; + u8 padding[2]; +} __packed; + +struct dcp_map_physical_resp { + u64 dva; + u64 dva_size; + u32 mem_desc_id; +} __packed; + +struct dcp_map_reg_req { + char obj[4]; + u32 index; + u32 flags; + u8 addr_null; + u8 length_null; + u8 padding[2]; +} __packed; + +struct dcp_map_reg_resp { + u64 addr; + u64 length; + u32 ret; +} __packed; + +struct dcp_swap_start_req { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct dcp_swap_start_resp { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct dcp_swap_submit_req { + struct dcp_swap swap; + struct dcp_surface surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; + u8 unkbool; + u64 unkdouble; + u32 unkint; + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; + u8 unkoutbool_null; + u8 padding[1]; +} __packed; + +struct dcp_swap_submit_resp { + u8 unkoutbool; + u32 ret; + u8 padding[3]; +} __packed; + +struct dcp_get_uint_prop_req { + char obj[4]; + char key[0x40]; + u64 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_get_uint_prop_resp { + u64 value; + u8 ret; + u8 padding[3]; +} __packed; + +struct dcp_set_power_state_req { + u64 unklong; + u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; + +struct dcp_set_power_state_resp { + u32 unkint; + u32 ret; +} __packed; + +struct dcp_set_dcpav_prop_chunk_req { + char data[0x1000]; + u32 offset; + u32 length; +} __packed; + +struct dcp_set_dcpav_prop_end_req { + char key[0x40]; +} __packed; + +struct dcp_update_notify_clients_dcp { + u32 client_0; + u32 client_1; + u32 client_2; + u32 client_3; + u32 client_4; + u32 client_5; + u32 client_6; + u32 client_7; + u32 client_8; + u32 client_9; + u32 client_a; + u32 client_b; + u32 client_c; + u32 client_d; +} __packed; + +#endif diff --git a/drivers/gpu/drm/apple/dummy-piodma.c b/drivers/gpu/drm/apple/dummy-piodma.c new file mode 100644 index 00000000000000..a84a28b4dc2ae0 --- /dev/null +++ b/drivers/gpu/drm/apple/dummy-piodma.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include + +static int dcp_piodma_probe(struct platform_device *pdev) +{ + return dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); +} + +static const struct of_device_id of_match[] = { + { .compatible = "apple,dcp-piodma" }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +static struct platform_driver dcp_piodma_platform_driver = { + .probe = dcp_piodma_probe, + .driver = { + .name = "apple,dcp-piodma", + .of_match_table = of_match, + }, +}; + +module_platform_driver(dcp_piodma_platform_driver); + +MODULE_AUTHOR("Alyssa Rosenzweig "); +MODULE_DESCRIPTION("[HACK] Apple DCP PIODMA shim"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c new file mode 100644 index 00000000000000..7205179e7785aa --- /dev/null +++ b/drivers/gpu/drm/apple/parser.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include "parser.h" + +#define DCP_PARSE_HEADER 0xd3 + +enum dcp_parse_type { + DCP_TYPE_DICTIONARY = 1, + DCP_TYPE_ARRAY = 2, + DCP_TYPE_INT64 = 4, + DCP_TYPE_STRING = 9, + DCP_TYPE_BLOB = 10, + DCP_TYPE_BOOL = 11 +}; + +struct dcp_parse_tag { + unsigned int size : 24; + enum dcp_parse_type type : 5; + unsigned int padding : 2; + bool last : 1; +} __packed; + +static void *parse_bytes(struct dcp_parse_ctx *ctx, size_t count) +{ + void *ptr = ctx->blob + ctx->pos; + + if (ctx->pos + count > ctx->len) + return ERR_PTR(-EINVAL); + + ctx->pos += count; + return ptr; +} + +static u32 *parse_u32(struct dcp_parse_ctx *ctx) +{ + return parse_bytes(ctx, sizeof(u32)); +} + +static struct dcp_parse_tag *parse_tag(struct dcp_parse_ctx *ctx) +{ + struct dcp_parse_tag *tag; + + /* Align to 32-bits */ + ctx->pos = round_up(ctx->pos, 4); + + tag = parse_bytes(ctx, sizeof(struct dcp_parse_tag)); + + if (IS_ERR(tag)) + return tag; + + if (tag->padding) + return ERR_PTR(-EINVAL); + + return tag; +} + +static struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, + enum dcp_parse_type type) +{ + struct dcp_parse_tag *tag = parse_tag(ctx); + + if (IS_ERR(tag)) + return tag; + + if (tag->type != type) + return ERR_PTR(-EINVAL); + + return tag; +} + +static int skip(struct dcp_parse_ctx *handle) +{ + struct dcp_parse_tag *tag = parse_tag(handle); + int ret = 0; + int i; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + switch (tag->type) { + case DCP_TYPE_DICTIONARY: + for (i = 0; i < tag->size; ++i) { + ret |= skip(handle); /* key */ + ret |= skip(handle); /* value */ + } + + return ret; + + case DCP_TYPE_ARRAY: + for (i = 0; i < tag->size; ++i) + ret |= skip(handle); + + return ret; + + case DCP_TYPE_INT64: + handle->pos += sizeof(s64); + return 0; + + case DCP_TYPE_STRING: + case DCP_TYPE_BLOB: + handle->pos += tag->size; + return 0; + + case DCP_TYPE_BOOL: + return 0; + + default: + return -EINVAL; + } +} + +/* Caller must free the result */ +static char *parse_string(struct dcp_parse_ctx *handle) +{ + struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_STRING); + const char *in; + char *out; + + if (IS_ERR(tag)) + return (void *)tag; + + in = parse_bytes(handle, tag->size); + if (IS_ERR(in)) + return (void *)in; + + out = kmalloc(tag->size + 1, GFP_KERNEL); + + memcpy(out, in, tag->size); + out[tag->size] = '\0'; + return out; +} + +static int parse_int(struct dcp_parse_ctx *handle, s64 *value) +{ + void *tag = parse_tag_of_type(handle, DCP_TYPE_INT64); + s64 *in; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + in = parse_bytes(handle, sizeof(s64)); + + if (IS_ERR(in)) + return PTR_ERR(in); + + memcpy(value, in, sizeof(*value)); + return 0; +} + +static int parse_bool(struct dcp_parse_ctx *handle, bool *b) +{ + struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_BOOL); + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + *b = !!tag->size; + return 0; +} + +struct iterator { + struct dcp_parse_ctx *handle; + u32 idx, len; +}; + +static int iterator_begin(struct dcp_parse_ctx *handle, struct iterator *it, + bool dict) +{ + struct dcp_parse_tag *tag; + enum dcp_parse_type type = dict ? DCP_TYPE_DICTIONARY : DCP_TYPE_ARRAY; + + *it = (struct iterator) { + .handle = handle, + .idx = 0 + }; + + tag = parse_tag_of_type(it->handle, type); + if (IS_ERR(tag)) + return PTR_ERR(tag); + + it->len = tag->size; + return 0; +} + +#define dcp_parse_foreach_in_array(handle, it) \ + for (iterator_begin(handle, &it, false); it.idx < it.len; ++it.idx) +#define dcp_parse_foreach_in_dict(handle, it) \ + for (iterator_begin(handle, &it, true); it.idx < it.len; ++it.idx) + +int parse(void *blob, size_t size, struct dcp_parse_ctx *ctx) +{ + u32 *header; + + *ctx = (struct dcp_parse_ctx) { + .blob = blob, + .len = size, + .pos = 0, + }; + + header = parse_u32(ctx); + if (IS_ERR(header)) + return PTR_ERR(header); + + if (*header != DCP_PARSE_HEADER) + return -EINVAL; + + return 0; +} + +struct dimension { + s64 total, front_porch, sync_width, active; + s64 precise_sync_rate; +}; + +static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) +{ + struct iterator it; + int ret = 0; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(handle); + else if (!strcmp(key, "Active")) + ret = parse_int(it.handle, &dim->active); + else if (!strcmp(key, "Total")) + ret = parse_int(it.handle, &dim->total); + else if (!strcmp(key, "FrontPorch")) + ret = parse_int(it.handle, &dim->front_porch); + else if (!strcmp(key, "SyncWidth")) + ret = parse_int(it.handle, &dim->sync_width); + else if (!strcmp(key, "PreciseSyncRate")) + ret = parse_int(it.handle, &dim->precise_sync_rate); + else + skip(it.handle); + + if (ret) + return ret; + } + + return 0; +} + +static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) +{ + struct iterator outer_it; + int ret = 0; + s64 best_score = -1; + + *best_id = -1; + + dcp_parse_foreach_in_array(handle, outer_it) { + struct iterator it; + s64 score = -1, id = -1; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, &score); + else if (!strcmp(key, "ID")) + ret = parse_int(it.handle, &id); + else + skip(it.handle); + + if (ret) + return ret; + } + + /* Skip partial entries */ + if (score < 0 || id < 0) + continue; + + if (score > best_score) { + best_score = score; + *best_id = id; + } + } + + return 0; +} + +/* + * Calculate the pixel clock for a mode given the 16:16 fixed-point refresh + * rate. The pixel clock is the refresh rate times the pixel count. DRM + * specifies the clock in kHz. The intermediate result may overflow a u32, so + * use a u64 where required. + */ +static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) +{ + u32 pixels = horiz->total * vert->total; + u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); + + return DIV_ROUND_CLOSEST_ULL(clock >> 16, 1000); +} + +static int parse_mode(struct dcp_parse_ctx *handle, + struct dcp_display_mode *out, s64 *score, int width_mm, + int height_mm) +{ + int ret = 0; + struct iterator it; + struct dimension horiz, vert; + s64 id = -1; + s64 best_color_mode = -1; + bool is_virtual = false; + struct drm_display_mode *mode = &out->mode; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "HorizontalAttributes")) + ret = parse_dimension(it.handle, &horiz); + else if (!strcmp(key, "VerticalAttributes")) + ret = parse_dimension(it.handle, &vert); + else if (!strcmp(key, "ColorModes")) + ret = parse_color_modes(it.handle, &best_color_mode); + else if (!strcmp(key, "ID")) + ret = parse_int(it.handle, &id); + else if (!strcmp(key, "IsVirtual")) + ret = parse_bool(it.handle, &is_virtual); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, score); + else + skip(it.handle); + + if (ret) + return ret; + } + + /* + * We need to skip virtual modes. In some cases, virtual modes are "too + * big" for the monitor and can cause breakage. It is unclear why the + * DCP reports these modes at all. Treat as a recoverable error. + */ + if (is_virtual) + return -EINVAL; + + /* From here we must succeed. Start filling out the mode. */ + *mode = (struct drm_display_mode) { + .type = DRM_MODE_TYPE_DRIVER, + .clock = calculate_clock(&horiz, &vert), + + .vdisplay = vert.active, + .vsync_start = vert.active + vert.front_porch, + .vsync_end = vert.active + vert.front_porch + vert.sync_width, + .vtotal = vert.total, + + .hdisplay = horiz.active, + .hsync_start = horiz.active + horiz.front_porch, + .hsync_end = horiz.active + horiz.front_porch + + horiz.sync_width, + .htotal = horiz.total, + + .width_mm = width_mm, + .height_mm = height_mm, + }; + + drm_mode_set_name(mode); + + out->timing_mode_id = id; + out->color_mode_id = best_color_mode; + + return 0; +} + +struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, + unsigned int *count, int width_mm, + int height_mm) +{ + struct iterator it; + int ret; + struct dcp_display_mode *mode, *modes; + struct dcp_display_mode *best_mode = NULL; + s64 score, best_score = -1; + + ret = iterator_begin(handle, &it, false); + + if (ret) + return ERR_PTR(ret); + + /* Start with a worst case allocation */ + modes = kmalloc_array(it.len, sizeof(*modes), GFP_KERNEL); + *count = 0; + + if (!modes) + return ERR_PTR(-ENOMEM); + + for (; it.idx < it.len; ++it.idx) { + mode = &modes[*count]; + ret = parse_mode(it.handle, mode, &score, width_mm, height_mm); + + /* Errors for a single mode are recoverable -- just skip it. */ + if (ret) + continue; + + /* Process a successful mode */ + (*count)++; + + if (score > best_score) { + best_score = score; + best_mode = mode; + } + } + + if (best_mode != NULL) + best_mode->mode.type |= DRM_MODE_TYPE_PREFERRED; + + return modes; +} + +int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm) +{ + int ret = 0; + struct iterator it; + s64 width_cm = 0, height_cm = 0; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "MaxHorizontalImageSize")) + ret = parse_int(it.handle, &width_cm); + else if (!strcmp(key, "MaxVerticalImageSize")) + ret = parse_int(it.handle, &height_cm); + else + skip(it.handle); + + if (ret) + return ret; + } + + /* 1cm = 10mm */ + *width_mm = 10 * width_cm; + *height_mm = 10 * height_cm; + + return 0; +} diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h new file mode 100644 index 00000000000000..afe3ed4700add7 --- /dev/null +++ b/drivers/gpu/drm/apple/parser.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_PARSER_H__ +#define __APPLE_DCP_PARSER_H__ + +/* For mode parsing */ +#include + +struct dcp_parse_ctx { + void *blob; + u32 pos, len; +}; + +/* + * Represents a single display mode. These mode objects are populated at + * runtime based on the TimingElements dictionary sent by the DCP. + */ +struct dcp_display_mode { + struct drm_display_mode mode; + u32 color_mode_id; + u32 timing_mode_id; +}; + +int parse(void *blob, size_t size, struct dcp_parse_ctx *ctx); +struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, + unsigned int *count, int width_mm, + int height_mm); +int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm); + +#endif From 83cd1b8be57c4970043ef0db22a03774619b956d Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 20 Mar 2022 18:44:00 +0100 Subject: [PATCH 178/635] drm: apple: Relicense DCP driver as dual MIT / GPL v2.0 Link: https://oftc.irclog.whitequark.org/asahi-dev/2022-03-20#30747564 Link: https://oftc.irclog.whitequark.org/asahi-dev/2022-03-20#30747570 Signed-off-by: Alyssa Rosenzweig Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/Kconfig | 2 +- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/apple_drv.c | 4 ++-- drivers/gpu/drm/apple/dcp.c | 4 ++-- drivers/gpu/drm/apple/dcp.h | 2 +- drivers/gpu/drm/apple/dcpep.h | 2 +- drivers/gpu/drm/apple/dummy-piodma.c | 4 ++-- drivers/gpu/drm/apple/parser.c | 2 +- drivers/gpu/drm/apple/parser.h | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 53c7d4edfd017e..9b9bcb7b5433e0 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: GPL-2.0-only OR MIT config DRM_APPLE tristate "DRM Support for Apple display controllers" depends on DRM && OF && ARM64 diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index db7ef359987d3d..8c758d5720b642 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: GPL-2.0-only OR MIT appledrm-y := apple_drv.o apple_dcp-y := dcp.o parser.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c0ad2e3d426909..356cf39cc1e7ae 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ /* Based on meson driver which is * Copyright (C) 2016 BayLibre, SAS @@ -437,4 +437,4 @@ module_platform_driver(apple_platform_driver); MODULE_AUTHOR("Alyssa Rosenzweig "); MODULE_DESCRIPTION(DRIVER_DESC); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f635740f195a3a..31078d32540f9e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ #include @@ -1391,4 +1391,4 @@ module_platform_driver(apple_platform_driver); MODULE_AUTHOR("Alyssa Rosenzweig "); MODULE_DESCRIPTION("Apple Display Controller DRM driver"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 4582fe984c8aa5..794544456df963 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ #ifndef __APPLE_DCP_H__ diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/dcpep.h index 6301bf8f8c21d1..f3d62d1d5f0328 100644 --- a/drivers/gpu/drm/apple/dcpep.h +++ b/drivers/gpu/drm/apple/dcpep.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ #ifndef __APPLE_DCPEP_H__ diff --git a/drivers/gpu/drm/apple/dummy-piodma.c b/drivers/gpu/drm/apple/dummy-piodma.c index a84a28b4dc2ae0..3d4454df4a25da 100644 --- a/drivers/gpu/drm/apple/dummy-piodma.c +++ b/drivers/gpu/drm/apple/dummy-piodma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ #include @@ -28,4 +28,4 @@ module_platform_driver(dcp_piodma_platform_driver); MODULE_AUTHOR("Alyssa Rosenzweig "); MODULE_DESCRIPTION("[HACK] Apple DCP PIODMA shim"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 7205179e7785aa..f0cd38c2a81048 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ #include diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index afe3ed4700add7..66a675079dc164 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ #ifndef __APPLE_DCP_PARSER_H__ From 0e487e3e17461e40959f1c8b94ebd46d3e5e54ca Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 25 Jul 2022 21:36:38 +0200 Subject: [PATCH 179/635] drm/apple: Start coprocessor on probe Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 31078d32540f9e..ed4f89149f6c69 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -21,6 +21,9 @@ struct apple_dcp; +#define APPLE_DCP_COPROC_CPU_CONTROL 0x44 +#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) + /* Register defines used in bandwidth setup structure */ #define REG_SCRATCH (0x14) #define REG_DOORBELL (0x0) @@ -70,6 +73,9 @@ struct apple_dcp { /* DCP shared memory */ void *shmem; + /* Coprocessor control register */ + void __iomem *coproc_reg; + /* Display registers mappable to the DCP */ struct resource *disp_registers[MAX_DISP_REGISTERS]; unsigned int nr_disp_registers; @@ -1302,9 +1308,9 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) static int dcp_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct resource *res; struct apple_dcp *dcp; dma_addr_t shmem_iova; + u32 cpu_ctrl; int ret; dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); @@ -1318,9 +1324,9 @@ static int dcp_platform_probe(struct platform_device *pdev) if (ret) return ret; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "coproc"); - if (!res) - return -EINVAL; + dcp->coproc_reg = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(dcp->coproc_reg)) + return PTR_ERR(dcp->coproc_reg); of_platform_default_populate(dev->of_node, NULL, dev); @@ -1336,6 +1342,10 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; } + cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) return dev_err_probe(dev, PTR_ERR(dcp->rtk), From 0bc43ae79496eabc7533bd061ecc461893eeccaa Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 15 Jan 2022 18:29:55 +0100 Subject: [PATCH 180/635] HACK: drm/apple: avoid DCP swaps without attached surfaces Xorg startup with modesetting driver triggers this. Move vblank signalling to dcp to avoid a circular dependency between apple_drv and dcp. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 18 ---------- drivers/gpu/drm/apple/dcp.c | 60 +++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 356cf39cc1e7ae..660355261d04d2 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -208,24 +208,6 @@ static void apple_crtc_atomic_begin(struct drm_crtc *crtc, } } -void apple_crtc_vblank(struct apple_crtc *crtc) -{ - unsigned long flags; - - if (crtc->vsync_disabled) - return; - - drm_crtc_handle_vblank(&crtc->base); - - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); - if (crtc->event) { - drm_crtc_send_vblank_event(&crtc->base, crtc->event); - drm_crtc_vblank_put(&crtc->base); - crtc->event = NULL; - } - spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); -} - static const struct drm_crtc_funcs apple_crtc_funcs = { .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ed4f89149f6c69..17ed6845c71cd5 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include "dcpep.h" #include "dcp.h" @@ -108,6 +110,9 @@ struct apple_dcp { /* Attributes of the connected display */ int width_mm, height_mm; + + /* Workqueue for sending vblank events when a dcp swap is not possible */ + struct work_struct vblank_wq; }; /* @@ -364,9 +369,28 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } +/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ +void dcp_drm_crtc_vblank(struct apple_crtc *crtc) +{ + unsigned long flags; + + if (crtc->vsync_disabled) + return; + + drm_crtc_handle_vblank(&crtc->base); + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + drm_crtc_vblank_put(&crtc->base); + crtc->event = NULL; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + static void dcpep_cb_swap_complete(struct apple_dcp *dcp) { - apple_crtc_vblank(dcp->crtc); + dcp_drm_crtc_vblank(dcp->crtc); } static struct dcp_get_uint_prop_resp @@ -732,6 +756,21 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) } } +/* + * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp + * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks + * send a vblank event via a workqueue. + */ +static void dcp_delayed_vblank(struct work_struct *work) +{ + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, vblank_wq); + mdelay(5); + dcp_drm_crtc_vblank(dcp->crtc); +} + + #define DCPEP_MAX_CB (1000) /* @@ -944,7 +983,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); - apple_crtc_vblank(dcp->crtc); + dcp_drm_crtc_vblank(dcp->crtc); } } @@ -1062,12 +1101,16 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) struct drm_crtc_state *crtc_state; struct dcp_swap_submit_req *req = &dcp->swap; int l; + int has_surface = 0; crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (WARN(dcp_channel_busy(&dcp->ch_cmd), "unexpected busy channel") || WARN(!dcp->connector->connected, "can't flush if disconnected")) { - apple_crtc_vblank(dcp->crtc); + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). + */ + schedule_work(&dcp->vblank_wq); return; } @@ -1091,6 +1134,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) continue; } req->surf_null[l] = false; + has_surface = 1; // XXX: awful hack! race condition between a framebuffer unbind // getting swapped out and GEM unreferencing a framebuffer. If @@ -1149,6 +1193,14 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) dcp->valid_mode = true; dcp_set_display_device(dcp, false, &handle, dcp_modeset, NULL); + } + else if (!has_surface) { + dev_warn(dcp->dev, "can't flush without surfaces, vsync:%d", dcp->crtc->vsync_disabled); + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). It's currently unkown + * if and how DCP supports swaps without attached surfaces. + */ + schedule_work(&dcp->vblank_wq); } else do_swap(dcp, NULL, NULL); } @@ -1342,6 +1394,8 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; } + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); From 5f908e30814cb72112f0376d74bc4127cfcfb0f6 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 18:34:58 +0200 Subject: [PATCH 181/635] drm/apple: Use a device tree defined clock for dcpep_cb_get_frequency Frequency differs between M1 and M1 Pro/Max/Ultra. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 17ed6845c71cd5..fbfa4c89f56aa1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -72,6 +73,9 @@ struct apple_dcp { /* DCP has crashed */ bool crashed; + /* clock rate request by dcp in */ + struct clk *clk; + /* DCP shared memory */ void *shmem; @@ -512,8 +516,7 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) { - /* Pixel clock frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ - return 533333328; + return clk_get_rate(dcp->clk); } static struct dcp_map_reg_resp @@ -1394,6 +1397,10 @@ static int dcp_platform_probe(struct platform_device *pdev) return ret; } + dcp->clk = devm_clk_get(dev, NULL); + if (IS_ERR(dcp->clk)) + return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); From bc9a66c96f8d16d070287d09fefa8cf476213d2b Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 12 Mar 2022 11:40:32 +0100 Subject: [PATCH 182/635] drm/apple: Fix rt_bandwidth for t600x Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fbfa4c89f56aa1..8738f5fca73265 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -29,6 +29,7 @@ struct apple_dcp; /* Register defines used in bandwidth setup structure */ #define REG_SCRATCH (0x14) +#define REG_SCRATCH_T600X (0x988) #define REG_DOORBELL (0x0) #define REG_DOORBELL_BIT (2) @@ -701,13 +702,26 @@ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, void *out, void *in) static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) { - return (struct dcp_rt_bandwidth) { - .reg_scratch = dcp->disp_registers[5]->start + REG_SCRATCH, - .reg_doorbell = dcp->disp_registers[6]->start + REG_DOORBELL, - .doorbell_bit = REG_DOORBELL_BIT, + if (dcp->disp_registers[5] && dcp->disp_registers[6]) + return (struct dcp_rt_bandwidth) { + .reg_scratch = dcp->disp_registers[5]->start + REG_SCRATCH, + .reg_doorbell = dcp->disp_registers[6]->start + REG_DOORBELL, + .doorbell_bit = REG_DOORBELL_BIT, - .padding[3] = 0x4, // XXX: required by 11.x firmware - }; + .padding[3] = 0x4, // XXX: required by 11.x firmware + }; + else if (dcp->disp_registers[4]) + return (struct dcp_rt_bandwidth) { + .reg_scratch = dcp->disp_registers[4]->start + REG_SCRATCH_T600X, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; + else + return (struct dcp_rt_bandwidth) { + .reg_scratch = 0, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; } /* Callback to get the current time as milliseconds since the UNIX epoch */ From 5a8bf41e4ca4fbe0a36721384e64eee9b18e5992 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 12 Mar 2022 11:43:04 +0100 Subject: [PATCH 183/635] drm/apple: Add nop sr_set_uint_prop callback for t600x-dcp Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8738f5fca73265..2d0b96fba692e8 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -892,6 +892,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, void *, void * [211] = trampoline_nop, /* update_backlight_factor_prop */ [300] = trampoline_nop, /* pr_publish */ [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ [406] = trampoline_nop, /* set_fx_prop */ [408] = trampoline_get_frequency, [411] = trampoline_map_reg, From bb88b53bcd3da92f90fb318120ab41dd75d282ab Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 12 Mar 2022 11:46:11 +0100 Subject: [PATCH 184/635] drm/apple: Reference only swapped out framebuffers The framebuffer can be unreferenced by GEM while the display controller is still using it for scanout resulting in IOVA faults and crashed dcp. dcp has to hold a reference until the swap is complete. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 48 +++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2d0b96fba692e8..3a1b7e6bc4dec0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -118,6 +118,16 @@ struct apple_dcp { /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; + + /* List of referenced drm_framebuffers which can be unreferenced + * on the next successfully completed swap. + */ + struct list_head swapped_out_fbs; +}; + +struct dcp_fb_reference { + struct list_head head; + struct drm_framebuffer *fb; }; /* @@ -1002,6 +1012,17 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); } } @@ -1145,6 +1166,24 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->swap.swap_enabled |= BIT(l); + if (old_state->fb && fb != old_state->fb) { + /* + * Race condition between a framebuffer unbind getting + * swapped out and GEM unreferencing a framebuffer. If + * we lose the race, the display gets IOVA faults and + * the DCP crashes. We need to extend the lifetime of + * the drm_framebuffer (and hence the GEM object) until + * after we get a swap complete for the swap unbinding + * it. + */ + struct dcp_fb_reference *entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (entry) { + entry->fb = old_state->fb; + list_add_tail(&entry->head, &dcp->swapped_out_fbs); + } + drm_framebuffer_get(old_state->fb); + } + if (!new_state->fb) { if (old_state->fb) req->swap.swap_enabled |= DCP_REMOVE_LAYERS; @@ -1154,13 +1193,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->surf_null[l] = false; has_surface = 1; - // XXX: awful hack! race condition between a framebuffer unbind - // getting swapped out and GEM unreferencing a framebuffer. If - // we lose the race, the display gets IOVA faults and the DCP - // crashes. We need to extend the lifetime of the - // drm_framebuffer (and hence the GEM object) until after we - // get a swap complete for the swap unbinding it. - drm_framebuffer_get(fb); drm_rect_fp_to_int(&src_rect, &new_state->src); @@ -1418,6 +1450,8 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); From 67a3a82d641e86b9520aa3d9f09a1c7ec72ee819 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 12 Mar 2022 12:48:34 +0100 Subject: [PATCH 185/635] drm/apple: Use "apple,asc-dram-mask" for rtkit iovas Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3a1b7e6bc4dec0..251913becf80b2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -77,6 +77,9 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; + /* mask for DCP IO virtual addresses shared over rtkit */ + u64 asc_dram_mask; + /* DCP shared memory */ void *shmem; @@ -482,6 +485,7 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_allocate_buffer_req * dma_get_sgtable(dcp->dev, &dcp->mappings[resp.mem_desc_id], buf, resp.dva, resp.dva_size); + resp.dva |= dcp->asc_dram_mask; return resp; } @@ -521,7 +525,7 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) .dva_size = size, .mem_desc_id = ++dcp->nr_mappings, .dva = dma_map_resource(dcp->dev, req->paddr, size, - DMA_BIDIRECTIONAL, 0), + DMA_BIDIRECTIONAL, 0) | dcp->asc_dram_mask, }; } @@ -1325,7 +1329,7 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) return -ENOMEM; // TODO: get map from device-tree - phy_addr = iommu_iova_to_phys(domain, bfr->iova & 0xFFFFFFFF); + phy_addr = iommu_iova_to_phys(domain, bfr->iova & ~dcp->asc_dram_mask); if (!phy_addr) return -ENOMEM; @@ -1342,6 +1346,8 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) if (!bfr->buffer) return -ENOMEM; + bfr->iova |= dcp->asc_dram_mask; + dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx", (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); } @@ -1356,7 +1362,7 @@ static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) if (bfr->is_mapped) memunmap(bfr->buffer); else - dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova & ~dcp->asc_dram_mask); } static struct apple_rtkit_ops rtkit_ops = { @@ -1448,6 +1454,12 @@ static int dcp_platform_probe(struct platform_device *pdev) if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + ret = of_property_read_u64(dev->of_node, "apple,asc-dram-mask", + &dcp->asc_dram_mask); + if (ret) + dev_warn(dev, "failed read 'apple,asc-dram-mask': %d\n", ret); + dev_dbg(dev, "'apple,asc-dram-mask': 0x%011llx\n", dcp->asc_dram_mask); + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); @@ -1471,6 +1483,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->shmem = dma_alloc_coherent(dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); + shmem_iova |= dcp->asc_dram_mask; apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, dcpep_set_shmem(shmem_iova), NULL, false); From 968f90e40ad95a71c2a0369a0c572235059d8783 Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Tue, 22 Mar 2022 16:24:53 -0400 Subject: [PATCH 186/635] drm/apple: Implement suspend/resume for DCP Use the firmware setPowerState callback. Signed-off-by: Alyssa Rosenzweig --- drivers/gpu/drm/apple/dcp.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 251913becf80b2..58507f018421b0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1502,6 +1502,10 @@ static void dcp_platform_shutdown(struct platform_device *pdev) /* defaults are ok */ }; + /* We're going down */ + dcp->active = false; + dcp->valid_mode = false; + dcp_set_power_state(dcp, false, &req, NULL, NULL); } @@ -1511,12 +1515,41 @@ static const struct of_device_id of_match[] = { }; MODULE_DEVICE_TABLE(of, of_match); +#ifdef CONFIG_PM_SLEEP +/* + * We don't hold any useful persistent state, so for suspend/resume it suffices + * to power off/on the entire DCP. The firmware will sort out the details for + * us. + */ +static int dcp_suspend(struct device *dev) +{ + dcp_platform_shutdown(to_platform_device(dev)); + return 0; +} + +static int dcp_resume(struct device *dev) +{ + struct apple_dcp *dcp = platform_get_drvdata(to_platform_device(dev)); + + dcp_start_signal(dcp, false, dcp_started, NULL); + return 0; +} + +static const struct dev_pm_ops dcp_pm_ops = { + .suspend = dcp_suspend, + .resume = dcp_resume, +}; +#endif + static struct platform_driver apple_platform_driver = { .probe = dcp_platform_probe, .shutdown = dcp_platform_shutdown, .driver = { .name = "apple-dcp", .of_match_table = of_match, +#ifdef CONFIG_PM_SLEEP + .pm = &dcp_pm_ops, +#endif }, }; From 9fb66abed08468738279177fa2bdecdcd7b5313b Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 14 Apr 2022 18:57:20 +0200 Subject: [PATCH 187/635] drm/apple: dcp: fix TRAMPOLINE_IN macro fixup! WIP: drm/apple: Add DCP display driver Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 58507f018421b0..8ba01120ca39e2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -818,11 +818,11 @@ static void dcp_delayed_vblank(struct work_struct *work) } #define TRAMPOLINE_IN(func, handler, T_in) \ - typedef void (*callback_##name)(struct apple_dcp *, T_in *); \ + typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ \ static bool func(struct apple_dcp *dcp, void *out, void *in) \ { \ - callback_##name cb = handler; \ + callback_##handler cb = handler; \ \ dev_dbg(dcp->dev, "received callback %s\n", #handler); \ cb(dcp, in); \ From 07f5de38af2ae956606fd85de2949c744ff273f5 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 18:02:23 +0200 Subject: [PATCH 188/635] drm/apple: Switch to nonblocking commit handling The swap completes only after the async reply from DCP. Uses drm_atomic_helper_wait_for_flip_done instead of drm_atomic_helper_wait_for_vblanks. This should allow ius to get rid of the scheduled fake vblanks. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 660355261d04d2..f6773c8eb0c5af 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -208,6 +208,27 @@ static void apple_crtc_atomic_begin(struct drm_crtc *crtc, } } +static void dcp_atomic_commit_tail(struct drm_atomic_state *old_state) +{ + struct drm_device *dev = old_state->dev; + + drm_atomic_helper_commit_modeset_disables(dev, old_state); + + drm_atomic_helper_commit_modeset_enables(dev, old_state); + + drm_atomic_helper_commit_planes(dev, old_state, + DRM_PLANE_COMMIT_ACTIVE_ONLY); + + drm_atomic_helper_fake_vblank(old_state); + + drm_atomic_helper_commit_hw_done(old_state); + + drm_atomic_helper_wait_for_flip_done(dev, old_state); + + drm_atomic_helper_cleanup_planes(dev, old_state); +} + + static const struct drm_crtc_funcs apple_crtc_funcs = { .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, @@ -226,7 +247,7 @@ static const struct drm_mode_config_funcs apple_mode_config_funcs = { }; static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { - .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, + .atomic_commit_tail = dcp_atomic_commit_tail, }; static const struct drm_connector_funcs apple_connector_funcs = { From b540d956b407aa632530f80c33c4039b3b86178a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 18:15:30 +0200 Subject: [PATCH 189/635] drm/apple: Log callbacks with their tag as debug output Mostly for the generic callbacks nop, true, false. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8ba01120ca39e2..51ec8e529cf8ab 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -708,8 +708,9 @@ static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) } /* Use special function signature to defer the ACK */ -static bool dcpep_cb_boot_1(struct apple_dcp *dcp, void *out, void *in) +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, __func__); dcp_set_create_dfb(dcp, false, boot_1_5, NULL); return false; } @@ -810,9 +811,9 @@ static void dcp_delayed_vblank(struct work_struct *work) */ #define TRAMPOLINE_VOID(func, handler) \ - static bool func(struct apple_dcp *dcp, void *out, void *in) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ - dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ handler(dcp); \ return true; \ } @@ -820,11 +821,11 @@ static void dcp_delayed_vblank(struct work_struct *work) #define TRAMPOLINE_IN(func, handler, T_in) \ typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ \ - static bool func(struct apple_dcp *dcp, void *out, void *in) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ callback_##handler cb = handler; \ \ - dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ cb(dcp, in); \ return true; \ } @@ -832,22 +833,22 @@ static void dcp_delayed_vblank(struct work_struct *work) #define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ \ - static bool func(struct apple_dcp *dcp, void *out, void *in) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ T_out *typed_out = out; \ callback_##handler cb = handler; \ \ - dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ *typed_out = cb(dcp, in); \ return true; \ } #define TRAMPOLINE_OUT(func, handler, T_out) \ - static bool func(struct apple_dcp *dcp, void *out, void *in) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ T_out *typed_out = out; \ \ - dev_dbg(dcp->dev, "received callback %s\n", #handler); \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ *typed_out = handler(dcp); \ return true; \ } @@ -879,7 +880,7 @@ TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); -bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, void *, void *) = { +bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, void *) = { [0] = trampoline_true, /* did_boot_signal */ [1] = trampoline_true, /* did_power_on_signal */ [2] = trampoline_nop, /* will_power_off_signal */ @@ -951,7 +952,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, depth = dcp_push_depth(&ch->depth); ch->output[depth] = out; - if (dcpep_cb_handlers[tag](dcp, out, in)) + if (dcpep_cb_handlers[tag](dcp, tag, out, in)) dcp_ack(dcp, context); } From 9d56a1af4d6ba65bf22917e8316af81354844ec9 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 18:19:28 +0200 Subject: [PATCH 190/635] drm/apple: Add DCP interface definitions used on t600x Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 12 ++++++++++++ drivers/gpu/drm/apple/dcpep.h | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 51ec8e529cf8ab..0ace01f17c700a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -226,8 +226,11 @@ const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { DCP_METHOD("A407", dcpep_swap_start), DCP_METHOD("A408", dcpep_swap_submit), DCP_METHOD("A410", dcpep_set_display_device), + DCP_METHOD("A411", dcpep_is_main_display), DCP_METHOD("A412", dcpep_set_digital_out_mode), + DCP_METHOD("A439", dcpep_set_parameter_dcp), DCP_METHOD("A443", dcpep_create_default_fb), + DCP_METHOD("A447", dcpep_enable_disable_video_power_savings), DCP_METHOD("A454", dcpep_first_client_open), DCP_METHOD("A460", dcpep_set_display_refresh_properties), DCP_METHOD("A463", dcpep_flush_supports_power), @@ -337,6 +340,15 @@ __attribute__((unused)) DCP_THUNK_IN(dcp_update_notify_clients_dcp, dcpep_update_notify_clients_dcp, struct dcp_update_notify_clients_dcp); +DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, + struct dcp_set_parameter_dcp, u32); + +DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, + dcpep_enable_disable_video_power_savings, + u32, int); + +DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); + /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ static int dcp_parse_tag(char tag[4]) { diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/dcpep.h index f3d62d1d5f0328..d04a1b9ca9c55f 100644 --- a/drivers/gpu/drm/apple/dcpep.h +++ b/drivers/gpu/drm/apple/dcpep.h @@ -241,6 +241,9 @@ enum dcpep_method { dcpep_set_power_state, dcpep_first_client_open, dcpep_update_notify_clients_dcp, + dcpep_set_parameter_dcp, + dcpep_enable_disable_video_power_savings, + dcpep_is_main_display, dcpep_num_methods }; @@ -350,6 +353,15 @@ struct dcp_swap_submit_resp { u8 padding[3]; } __packed; +struct dc_swap_complete_resp { + u32 swap_id; + u8 unkbool; + u64 swap_data; + u8 swap_info[0x6c4]; + u32 unkint; + u8 swap_info_null; +} __packed; + struct dcp_get_uint_prop_req { char obj[4]; char key[0x40]; @@ -403,4 +415,18 @@ struct dcp_update_notify_clients_dcp { u32 client_d; } __packed; +struct dcp_set_parameter_dcp { + u32 param; + u32 value[8]; + u32 count; +} __packed; + +struct dcp_swap_complete_intent_gated { + u32 swap_id; + u8 unkBool; + u32 unkInt; + u32 width; + u32 height; +} __packed; + #endif From a091653a6ac77214532c3f90ad93787bda6778d6 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 18:43:01 +0200 Subject: [PATCH 191/635] drm/apple: Clear used callback/cookie on dcp_ack Avoids unexpected callbacks on nesting state errors. Encountered when making DCP calls from a second thread for the backlight. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0ace01f17c700a..a74ff56fa49cce 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -986,6 +986,9 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, cb = ch->callbacks[ch->depth]; cookie = ch->cookies[ch->depth]; + ch->callbacks[ch->depth] = NULL; + ch->cookies[ch->depth] = NULL; + if (cb) cb(dcp, data + sizeof(*header) + header->in_len, cookie); } From aef23e10107f933d744025b0228b2dff87bea925 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 19:31:16 +0200 Subject: [PATCH 192/635] drm/apple: Add t600x support Call power-on/-off handling explicitly from apple_crtc_atomic_enable / apple_crtc_atomic_disable. This makes DPMS work. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 4 + drivers/gpu/drm/apple/dcp.c | 302 +++++++++++++++++++++++++++--- drivers/gpu/drm/apple/dcp.h | 2 + 3 files changed, 280 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index f6773c8eb0c5af..cff24df268c6a5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -175,13 +175,17 @@ apple_connector_detect(struct drm_connector *connector, bool force) static void apple_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dcp_poweron(apple_crtc->dcp); drm_crtc_vblank_on(crtc); } static void apple_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); drm_crtc_vblank_off(crtc); + dcp_poweroff(apple_crtc->dcp); if (crtc->state->event && !crtc->state->active) { spin_lock_irq(&crtc->dev->event_lock); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a74ff56fa49cce..8c9f120395a8de 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -112,6 +113,11 @@ struct apple_dcp { /* Is the DCP booted? */ bool active; + /* eDP display without DP-HDMI conversion */ + bool main_display; + + bool ignore_swap_complete; + /* Modes valid for the connected display */ struct dcp_display_mode *modes; unsigned int nr_modes; @@ -133,6 +139,11 @@ struct dcp_fb_reference { struct drm_framebuffer *fb; }; +struct dcp_wait_cookie { + struct completion done; + atomic_t refcount; +}; + /* * A channel is busy if we have sent a message that has yet to be * acked. The driver must not sent a message to a busy channel. @@ -418,9 +429,11 @@ void dcp_drm_crtc_vblank(struct apple_crtc *crtc) spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } -static void dcpep_cb_swap_complete(struct apple_dcp *dcp) +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct dc_swap_complete_resp *resp) { - dcp_drm_crtc_vblank(dcp->crtc); + if (!dcp->ignore_swap_complete) + dcp_drm_crtc_vblank(dcp->crtc); } static struct dcp_get_uint_prop_resp @@ -757,6 +770,182 @@ static u64 dcpep_cb_get_time(struct apple_dcp *dcp) return ktime_to_ms(ktime_get_real()); } +struct dcp_swap_cookie { + struct completion done; + atomic_t refcount; + u32 swap_id; +}; + +static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_submit_resp *resp = data; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + complete(&info->done); + if (atomic_dec_and_test(&info->refcount)) + kfree(info); + } + + if (resp->ret) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + dcp->swap.swap.swap_id = resp->swap_id; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + info->swap_id = resp->swap_id; + } + + dcp_swap_submit(dcp, false, &dcp->swap, dcp_swap_cleared, cookie); +} + +static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + if (atomic_dec_and_test(&wait->refcount)) + kfree(wait); + } +} + +static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = 14, + .value = { 0 }, + .count = 1, + }; + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_final, cookie); +} + +void dcp_poweron(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct dcp_wait_cookie * cookie; + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + int ret; + u32 handle; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + + init_completion(&cookie->done); + atomic_set(&cookie->refcount, 2); + + if (dcp->main_display) { + handle = 0; + dcp_set_display_device(dcp, false, &handle, dcp_on_final, cookie); + } else { + handle = 2; + dcp_set_display_device(dcp, false, &handle, dcp_on_set_parameter, cookie); + } + dcp_set_power_state(dcp, true, &req, NULL, NULL); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); + + if (ret == 0) + dev_warn(dcp->dev, "wait for power timed out"); + + if (atomic_dec_and_test(&cookie->refcount)) + kfree(cookie); +} +EXPORT_SYMBOL(dcp_poweron); + +static void complete_set_powerstate(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + if (atomic_dec_and_test(&wait->refcount)) + kfree(wait); + } +} + +void dcp_poweroff(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret, swap_id; + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + struct dcp_swap_cookie *cookie; + struct dcp_wait_cookie *poff_cookie; + struct dcp_swap_start_req swap_req= { 0 }; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + atomic_set(&cookie->refcount, 2); + + // clear surfaces + memset(&dcp->swap, 0, sizeof(dcp->swap)); + + dcp->swap.swap.swap_enabled = DCP_REMOVE_LAYERS | 0x7; + dcp->swap.swap.swap_completed = DCP_REMOVE_LAYERS | 0x7; + dcp->swap.swap.unk_10c = 0xFF000000; + + for (int l = 0; l < SWAP_SURFACES; l++) + dcp->swap.surf_null[l] = true; + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + swap_id = cookie->swap_id; + if (atomic_dec_and_test(&cookie->refcount)) + kfree(cookie); + if (ret <= 0) { + dcp->crashed = true; + return; + } + + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); + if (!poff_cookie) + return; + init_completion(&poff_cookie->done); + atomic_set(&poff_cookie->refcount, 2); + + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, poff_cookie); + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setPowerState(0) timeout %u ms", 1000); + else if (ret > 0) + dev_dbg(dcp->dev, "setPowerState(0) finished with %d ms to spare", + jiffies_to_msecs(ret)); + + if (atomic_dec_and_test(&poff_cookie->refcount)) + kfree(poff_cookie); + dev_dbg(dcp->dev, "%s: setPowerState(0) done", __func__); +} +EXPORT_SYMBOL(dcp_poweroff); + /* * Helper to send a DRM hotplug event. The DCP is accessed from a single * (RTKit) thread. To handle hotplug callbacks, we need to call @@ -768,16 +957,19 @@ void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; struct drm_device *dev; + struct apple_dcp *dcp; connector = container_of(work, struct apple_connector, hotplug_wq); dev = connector->base.dev; + dcp = platform_get_drvdata(connector->dcp); + /* * DCP defers link training until we set a display mode. But we set * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (connector->connected) { + if (!dcp->valid_mode && connector->connected) { drm_connector_set_link_status_property( &connector->base, DRM_MODE_LINK_STATUS_BAD); } @@ -792,10 +984,16 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) struct apple_connector *connector = dcp->connector; /* Hotplug invalidates mode. DRM doesn't always handle this. */ - dcp->valid_mode = false; + if (!(*connected)) { + dcp->valid_mode = false; + /* after unplug swap will not complete until the next + * set_digital_out_mode */ + schedule_work(&dcp->vblank_wq); + } - if (connector) { + if (connector && connector->connected != !!(*connected)) { connector->connected = !!(*connected); + dcp->valid_mode = false; schedule_work(&connector->hotplug_wq); } } @@ -869,7 +1067,8 @@ TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); -TRAMPOLINE_VOID(trampoline_swap_complete, dcpep_cb_swap_complete); +TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, + struct dc_swap_complete_resp); TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, @@ -907,6 +1106,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, v [110] = trampoline_true, /* create_iomfb_service */ [111] = trampoline_false, /* create_backlight_service */ [116] = dcpep_cb_boot_1, + [117] = trampoline_false, /* is_dark_boot */ [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ [120] = trampoline_false, /* read_edt_data */ [122] = trampoline_prop_start, @@ -939,6 +1139,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, v [582] = trampoline_true, /* create_default_fb_surface */ [589] = trampoline_swap_complete, [591] = trampoline_nop, /* swap_complete_intent_gated */ + [593] = trampoline_nop, /* enable_backlight_message_ap_gated */ [598] = trampoline_nop, /* find_swap_function_gated */ }; @@ -1143,12 +1344,24 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_req start_req = { 0 }; - dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + if (dcp->connector && dcp->connector->connected) + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + else + dcp_drm_crtc_vblank(dcp->crtc); } -static void dcp_modeset(struct apple_dcp *dcp, void *out, void *cookie) +static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, + void *cookie) { - dcp_set_digital_out_mode(dcp, false, &dcp->mode, do_swap, NULL); + struct dcp_wait_cookie *wait = cookie; + + dcp->ignore_swap_complete = false; + + if (wait) { + complete(&wait->done); + if (atomic_dec_and_test(&wait->refcount)) + kfree(wait); + } } void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) @@ -1245,7 +1458,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) if (drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode) { struct dcp_display_mode *mode; - u32 handle = 2; + struct dcp_wait_cookie *cookie; + int ret; mode = lookup_mode(dcp, &crtc_state->mode); if (!mode) { @@ -1260,19 +1474,43 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) .timing_mode_id = mode->timing_mode_id }; - dcp->valid_mode = true; + cookie = kzalloc(sizeof(cookie), GFP_KERNEL); + if (!cookie) { + schedule_work(&dcp->vblank_wq); + return; + } + + init_completion(&cookie->done); + atomic_set(&cookie->refcount, 2); + + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); - dcp_set_display_device(dcp, false, &handle, dcp_modeset, NULL); + if (atomic_dec_and_test(&cookie->refcount)) + kfree(cookie); + + if (ret == 0) { + dev_dbg(dcp->dev, "set_digital_out_mode 200 ms"); + schedule_work(&dcp->vblank_wq); + return; + } + else if (ret > 0) { + dev_dbg(dcp->dev, "set_digital_out_mode finished with %d to spare", + jiffies_to_msecs(ret)); + } + + dcp->valid_mode = true; } - else if (!has_surface) { - dev_warn(dcp->dev, "can't flush without surfaces, vsync:%d", dcp->crtc->vsync_disabled); - /* HACK: issue a delayed vblank event to avoid timeouts in - * drm_atomic_helper_wait_for_vblanks(). It's currently unkown - * if and how DCP supports swaps without attached surfaces. - */ + + if (!has_surface) { + dev_warn(dcp->dev, "flush without surfaces, vsync:%d", + dcp->crtc->vsync_disabled); schedule_work(&dcp->vblank_wq); - } else - do_swap(dcp, NULL, NULL); + return; + } + do_swap(dcp, NULL, NULL); } EXPORT_SYMBOL_GPL(dcp_flush); @@ -1284,16 +1522,20 @@ bool dcp_is_initialized(struct platform_device *pdev) } EXPORT_SYMBOL_GPL(dcp_is_initialized); -static void init_done(struct apple_dcp *dcp, void *out, void *cookie) + +static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) { + int result = *(int *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); + + dcp->main_display = result != 0; + + dcp->active = true; } static void init_3(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_set_power_state_req req = { - .unklong = 1, - }; - dcp_set_power_state(dcp, false, &req, init_done, NULL); + dcp_is_main_display(dcp, false, res_is_main_display, NULL); } static void init_2(struct apple_dcp *dcp, void *out, void *cookie) @@ -1301,13 +1543,17 @@ static void init_2(struct apple_dcp *dcp, void *out, void *cookie) dcp_first_client_open(dcp, false, init_3, NULL); } +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 val = 0; + dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); +} + static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { dev_info(dcp->dev, "DCP booted\n"); - init_2(dcp, data, cookie); - - dcp->active = true; + init_1(dcp, data, cookie); } static void dcp_got_msg(void *cookie, u8 endpoint, u64 message) diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 794544456df963..9e3e3738a39377 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -32,6 +32,8 @@ struct apple_connector { #define to_apple_connector(x) container_of(x, struct apple_connector, base) +void dcp_poweroff(struct platform_device *pdev); +void dcp_poweron(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); From cea7799c50d0bd20f8b295eafd66c05122de5ea7 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 6 Jun 2022 22:14:27 +0200 Subject: [PATCH 193/635] drm/apple: toggle power only when active state changes squash! WIP: GPU/apple: t600x work in progress Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index cff24df268c6a5..bcac8bb9f476a7 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -175,17 +175,32 @@ apple_connector_detect(struct drm_connector *connector, bool force) static void apple_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); - dcp_poweron(apple_crtc->dcp); + struct drm_crtc_state *crtc_state; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + if (crtc_state->active_changed && crtc_state->active) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dev_dbg(&apple_crtc->dcp->dev, "%s", __func__); + dcp_poweron(apple_crtc->dcp); + dev_dbg(&apple_crtc->dcp->dev, "%s finished", __func__); + } drm_crtc_vblank_on(crtc); } static void apple_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct drm_crtc_state *crtc_state; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + drm_crtc_vblank_off(crtc); - dcp_poweroff(apple_crtc->dcp); + + if (crtc_state->active_changed && !crtc_state->active) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dev_dbg(&apple_crtc->dcp->dev, "%s", __func__); + dcp_poweroff(apple_crtc->dcp); + dev_dbg(&apple_crtc->dcp->dev, "%s finished", __func__); + } if (crtc->state->event && !crtc->state->active) { spin_lock_irq(&crtc->dev->event_lock); From 905f098d1b9684c41c94792c8ffb9346d37dbf46 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 19:39:10 +0200 Subject: [PATCH 194/635] drm/apple: Add somewhat useful debug prints Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8c9f120395a8de..40ea791b0b8b01 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -432,6 +432,9 @@ void dcp_drm_crtc_vblank(struct apple_crtc *crtc) static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct dc_swap_complete_resp *resp) { + dev_dbg(dcp->dev, "swap complete for swap_id: %u vblank: %u", + resp->swap_id, dcp->ignore_swap_complete); + if (!dcp->ignore_swap_complete) dcp_drm_crtc_vblank(dcp->crtc); } @@ -700,6 +703,7 @@ static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_cb_channel *ch = &dcp->ch_cb; u8 *succ = ch->output[ch->depth - 1]; + dev_dbg(dcp->dev, "boot done"); *succ = true; dcp_ack(dcp, DCP_CONTEXT_CB); @@ -808,6 +812,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_resp *resp = data; + dev_dbg(dcp->dev, "%s swap_id: %u", __func__, resp->swap_id); dcp->swap.swap.swap_id = resp->swap_id; if (cookie) { @@ -925,6 +930,8 @@ void dcp_poweroff(struct platform_device *pdev) return; } + dev_dbg(dcp->dev, "%s: clear swap submitted: %u", __func__, swap_id); + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) return; @@ -963,6 +970,7 @@ void dcp_hotplug(struct work_struct *work) dev = connector->base.dev; dcp = platform_get_drvdata(connector->dcp); + dev_info(dcp->dev, "%s: connected: %d", __func__, connector->connected); /* * DCP defers link training until we set a display mode. But we set @@ -998,6 +1006,14 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) } } +static void +dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, + struct dcp_swap_complete_intent_gated *info) +{ + dev_dbg(dcp->dev, "swap_id:%u width:%u height:%u", info->swap_id, + info->width, info->height); +} + /* * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks @@ -1090,6 +1106,9 @@ TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, + dcpep_cb_swap_complete_intent_gated, + struct dcp_swap_complete_intent_gated); bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, void *) = { [0] = trampoline_true, /* did_boot_signal */ @@ -1138,7 +1157,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, v [577] = trampoline_nop, /* powerstate_notify */ [582] = trampoline_true, /* create_default_fb_surface */ [589] = trampoline_swap_complete, - [591] = trampoline_nop, /* swap_complete_intent_gated */ + [591] = trampoline_swap_complete_intent_gated, [593] = trampoline_nop, /* enable_backlight_message_ap_gated */ [598] = trampoline_nop, /* find_swap_function_gated */ }; @@ -1469,6 +1488,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) return; } + dev_info(dcp->dev, "set_digital_out_mode(color:%d timing:%d)", + mode->color_mode_id, mode->timing_mode_id); dcp->mode = (struct dcp_set_digital_out_mode_req) { .color_mode_id = mode->color_mode_id, .timing_mode_id = mode->timing_mode_id @@ -1486,6 +1507,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); + dev_dbg(dcp->dev, "%s - wait for modeset", __func__); ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); if (atomic_dec_and_test(&cookie->refcount)) From e7a0f84d0d7a0d42d6ce4fad0c16e227a3bb9d13 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 31 Jul 2022 19:40:50 +0200 Subject: [PATCH 195/635] drm/apple: Add less tons of questionable debug prints Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 40ea791b0b8b01..97014561cd8639 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -783,6 +783,7 @@ struct dcp_swap_cookie { static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_submit_resp *resp = data; + dev_dbg(dcp->dev, "%s", __func__); if (cookie) { struct dcp_swap_cookie *info = cookie; @@ -826,6 +827,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_wait_cookie *wait = cookie; + dev_dbg(dcp->dev, "%s", __func__); if (wait) { complete(&wait->done); @@ -841,6 +843,7 @@ static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) .value = { 0 }, .count = 1, }; + dev_dbg(dcp->dev, "%s", __func__); dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_final, cookie); } @@ -854,6 +857,7 @@ void dcp_poweron(struct platform_device *pdev) }; int ret; u32 handle; + dev_dbg(dcp->dev, "%s", __func__); cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) @@ -903,6 +907,8 @@ void dcp_poweroff(struct platform_device *pdev) struct dcp_wait_cookie *poff_cookie; struct dcp_swap_start_req swap_req= { 0 }; + dev_dbg(dcp->dev, "%s", __func__); + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1362,6 +1368,7 @@ EXPORT_SYMBOL_GPL(dcp_mode_valid); static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_req start_req = { 0 }; + dev_dbg(dcp->dev, "%s", __func__); if (dcp->connector && dcp->connector->connected) dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); @@ -1373,6 +1380,7 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_wait_cookie *wait = cookie; + dev_dbg(dcp->dev, "%s", __func__); dcp->ignore_swap_complete = false; @@ -1393,6 +1401,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) struct dcp_swap_submit_req *req = &dcp->swap; int l; int has_surface = 0; + dev_dbg(dcp->dev, "%s", __func__); crtc_state = drm_atomic_get_new_crtc_state(state, crtc); From 0af808f33883095c885029e0e7e631b4de09355c Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 5 Jun 2022 19:32:01 +0200 Subject: [PATCH 196/635] drm/apple: implement read_edt_data Handling it with trampoline_false can result in errors due to uninitialized data. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 13 ++++++++++++- drivers/gpu/drm/apple/dcpep.h | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 97014561cd8639..fcebaf16bec140 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -582,6 +582,15 @@ dcpep_cb_map_reg(struct apple_dcp *dcp, struct dcp_map_reg_req *req) } } +static struct dcp_read_edt_data_resp +dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) +{ + return (struct dcp_read_edt_data_resp) { + .value[0] = req->value[0], + .ret = 0, + }; +} + /* Chunked data transfer for property dictionaries */ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) { @@ -1102,6 +1111,8 @@ TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, struct dcp_map_physical_req, struct dcp_map_physical_resp); TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct dcp_map_reg_req, struct dcp_map_reg_resp); +TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, + struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, struct dcp_set_dcpav_prop_chunk_req, u8); @@ -1133,7 +1144,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, v [116] = dcpep_cb_boot_1, [117] = trampoline_false, /* is_dark_boot */ [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [120] = trampoline_false, /* read_edt_data */ + [120] = trampoline_read_edt_data, [122] = trampoline_prop_start, [123] = trampoline_prop_chunk, [124] = trampoline_prop_end, diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/dcpep.h index d04a1b9ca9c55f..12d81c7b4e2734 100644 --- a/drivers/gpu/drm/apple/dcpep.h +++ b/drivers/gpu/drm/apple/dcpep.h @@ -429,4 +429,15 @@ struct dcp_swap_complete_intent_gated { u32 height; } __packed; +struct dcp_read_edt_data_req { + char key[0x40]; + u32 count; + u32 value[8]; +} __packed; + +struct dcp_read_edt_data_resp { + u32 value[8]; + u8 ret; +} __packed; + #endif From 884a6400b4c094f77f8bc5931a4b944ff715c4b0 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 5 Jun 2022 19:47:25 +0200 Subject: [PATCH 197/635] drm/apple: clear callback's output data If there is a mismatch in the output size this way we have at leas consistant results. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fcebaf16bec140..4671b0ef7e0ce0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1198,6 +1198,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, in = data + sizeof(*hdr); out = in + hdr->in_len; + // TODO: verify that in_len and out_len match our prototypes + // for now just clear the out data to have at least consistant results + if (hdr->out_len) + memset(out, 0, hdr->out_len); + depth = dcp_push_depth(&ch->depth); ch->output[depth] = out; From 91bc7047d23acaa74fb69a6418d14e05556e3e28 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 5 Jun 2022 20:00:13 +0200 Subject: [PATCH 198/635] drm/apple: Support memory unmapping/freeing Used by dcp on mode changes on the Macbook Pro 14" (M1 Max). Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 129 ++++++++++++++++++++++++++++++---- drivers/gpu/drm/apple/dcpep.h | 8 +++ 2 files changed, 122 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4671b0ef7e0ce0..b8d367af9bb6b2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include #include @@ -65,6 +66,14 @@ struct dcp_chunks { #define DCP_MAX_MAPPINGS (128) /* should be enough */ #define MAX_DISP_REGISTERS (7) +struct dcp_mem_descriptor { + size_t size; + void *buf; + dma_addr_t dva; + struct sg_table map; + u64 reg; +}; + struct apple_dcp { struct device *dev; struct platform_device *piodma; @@ -91,11 +100,11 @@ struct apple_dcp { struct resource *disp_registers[MAX_DISP_REGISTERS]; unsigned int nr_disp_registers; - /* Number of memory mappings made by the DCP, used as an ID */ - u32 nr_mappings; + /* Bitmap of memory descriptors used for mappings made by the DCP */ + DECLARE_BITMAP(memdesc_map, DCP_MAX_MAPPINGS); - /* Indexed table of mappings */ - struct sg_table mappings[DCP_MAX_MAPPINGS]; + /* Indexed table of memory descriptors */ + struct dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; struct dcp_call_channel ch_cmd, ch_oobcmd; struct dcp_cb_channel ch_cb, ch_oobcb, ch_async; @@ -463,10 +472,10 @@ dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) struct sg_table *map; int ret; - if (req->buffer >= ARRAY_SIZE(dcp->mappings)) + if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) goto reject; - map = &dcp->mappings[req->buffer]; + map = &dcp->memdesc[req->buffer].map; if (!map->sgl) goto reject; @@ -489,6 +498,37 @@ dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) }; } +static void +dcpep_cb_unmap_piodma(struct apple_dcp *dcp, struct dcp_unmap_buf_resp *resp) +{ + struct sg_table *map; + dma_addr_t dma_addr; + + if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu", + resp->buffer); + return; + } + + map = &dcp->memdesc[resp->buffer].map; + + if (!map->sgl) { + dev_warn(dcp->dev, "unmap for non-mapped buffer %llu iova:0x%08llx", + resp->buffer, resp->dva); + return; + } + + dma_addr = sg_dma_address(map->sgl); + if (dma_addr != resp->dva) { + dev_warn(dcp->dev, "unmap buffer %llu address mismatch dma_addr:%llx dva:%llx", + resp->buffer, dma_addr, resp->dva); + return; + } + + /* Use PIODMA device instead of DCP to unmap from the right IOMMU. */ + dma_unmap_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); +} + /* * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be * physically contigiuous, however we should save the sgtable in case the @@ -498,25 +538,68 @@ static struct dcp_allocate_buffer_resp dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_allocate_buffer_req *req) { struct dcp_allocate_buffer_resp resp = { 0 }; - void *buf; + struct dcp_mem_descriptor *memdesc; + u32 id; resp.dva_size = ALIGN(req->size, 4096); - resp.mem_desc_id = ++dcp->nr_mappings; + resp.mem_desc_id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - if (resp.mem_desc_id >= ARRAY_SIZE(dcp->mappings)) { + if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring"); + resp.dva_size = 0; + resp.mem_desc_id = 0; return resp; } + id = resp.mem_desc_id; + set_bit(id, dcp->memdesc_map); - buf = dma_alloc_coherent(dcp->dev, resp.dva_size, &resp.dva, + memdesc = &dcp->memdesc[id]; + + memdesc->size = resp.dva_size; + memdesc->buf = dma_alloc_coherent(dcp->dev, memdesc->size, &memdesc->dva, GFP_KERNEL); - dma_get_sgtable(dcp->dev, &dcp->mappings[resp.mem_desc_id], buf, - resp.dva, resp.dva_size); - resp.dva |= dcp->asc_dram_mask; + dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, + memdesc->dva, memdesc->size); + resp.dva = memdesc->dva; + return resp; } +static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) +{ + struct dcp_mem_descriptor *memdesc; + u32 id = *mem_desc_id; + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "unmap request for out of range mem_desc_id %u", + id); + return 0; + } + + if (!test_and_clear_bit(id, dcp->memdesc_map)) { + dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u", + id); + return 0; + } + + memdesc = &dcp->memdesc[id]; + if (memdesc->buf) { + + dma_free_coherent(dcp->dev, memdesc->size, memdesc->buf, + memdesc->dva); + + memdesc->buf = NULL; + memset(&memdesc->map, 0, sizeof(memdesc->map)); + } else { + memdesc->reg = 0; + } + + memdesc->size = 0; + + return 1; +} + /* Validate that the specified region is a display register */ static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) { @@ -542,6 +625,7 @@ static struct dcp_map_physical_resp dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) { int size = ALIGN(req->size, 4096); + u32 id; if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { dev_err(dcp->dev, "refusing to map phys address %llx size %llx", @@ -549,11 +633,16 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) return (struct dcp_map_physical_resp) { }; } + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + set_bit(id, dcp->memdesc_map); + dcp->memdesc[id].size = size; + dcp->memdesc[id].reg = req->paddr; + return (struct dcp_map_physical_resp) { .dva_size = size, - .mem_desc_id = ++dcp->nr_mappings, + .mem_desc_id = id, .dva = dma_map_resource(dcp->dev, req->paddr, size, - DMA_BIDIRECTIONAL, 0) | dcp->asc_dram_mask, + DMA_BIDIRECTIONAL, 0), }; } @@ -1104,11 +1193,15 @@ TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, struct dcp_map_buf_req, struct dcp_map_buf_resp); +TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, + struct dcp_unmap_buf_resp); TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, struct dcp_allocate_buffer_req, struct dcp_allocate_buffer_resp); TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, + u32, u8); TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct dcp_map_reg_req, struct dcp_map_reg_resp); TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, @@ -1149,6 +1242,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, v [123] = trampoline_prop_chunk, [124] = trampoline_prop_end, [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, [206] = trampoline_true, /* match_pmu_service_2 */ [207] = trampoline_true, /* match_backlight_service */ [208] = trampoline_get_time, @@ -1164,6 +1258,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, v [415] = trampoline_true, /* sr_set_property_bool */ [451] = trampoline_allocate_buffer, [452] = trampoline_map_physical, + [456] = trampoline_release_mem_desc, [552] = trampoline_true, /* set_property_dict_0 */ [561] = trampoline_true, /* set_property_dict */ [563] = trampoline_true, /* set_property_int */ @@ -1769,6 +1864,10 @@ static int dcp_platform_probe(struct platform_device *pdev) dev_warn(dev, "failed read 'apple,asc-dram-mask': %d\n", ret); dev_dbg(dev, "'apple,asc-dram-mask': 0x%011llx\n", dcp->asc_dram_mask); + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); + // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry + set_bit(0, dcp->memdesc_map); + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/dcpep.h index 12d81c7b4e2734..7c4fd97c45e54e 100644 --- a/drivers/gpu/drm/apple/dcpep.h +++ b/drivers/gpu/drm/apple/dcpep.h @@ -273,6 +273,14 @@ struct dcp_map_buf_resp { u32 ret; } __packed; +struct dcp_unmap_buf_resp { + u64 buffer; + u64 vaddr; + u64 dva; + u8 unk; + u8 buf_null; +} __packed; + struct dcp_allocate_buffer_req { u32 unk0; u64 size; From 13a173bacd8c990514fab62f81f2f7956836ca93 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 6 Jun 2022 12:53:18 +0200 Subject: [PATCH 199/635] WIP: drm/apple: Change the way to clear unused surfaces This seems to be incorrect but the flag seems to be related to clearing. Allows unmapping surfaces after the swap finished. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 10 ++++++---- drivers/gpu/drm/apple/dcpep.h | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index b8d367af9bb6b2..347ee92f7ecc5d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1647,10 +1647,12 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) } if (!has_surface) { - dev_warn(dcp->dev, "flush without surfaces, vsync:%d", - dcp->crtc->vsync_disabled); - schedule_work(&dcp->vblank_wq); - return; + if (crtc_state->enable && crtc_state->active && !crtc_state->planes_changed) { + schedule_work(&dcp->vblank_wq); + return; + } + + req->clear = 1; } do_swap(dcp, NULL, NULL); } diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/dcpep.h index 7c4fd97c45e54e..a796b16bd55ad7 100644 --- a/drivers/gpu/drm/apple/dcpep.h +++ b/drivers/gpu/drm/apple/dcpep.h @@ -348,7 +348,7 @@ struct dcp_swap_submit_req { u64 surf_iova[SWAP_SURFACES]; u8 unkbool; u64 unkdouble; - u32 unkint; + u32 clear; // or maybe switch to default fb? u8 swap_null; u8 surf_null[SWAP_SURFACES]; u8 unkoutbool_null; From c3f0ae4cc0ea3d0fd2c2591f9aca89562f983196 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 27 Sep 2022 00:02:20 +0200 Subject: [PATCH 200/635] drm/apple: laod piodma dev via explicit phandle Use a device_link to ensure piodma has is bound to its DART. fixup! WIP: drm/apple: Add DCP display driver Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 347ee92f7ecc5d..9b695e52bd9a69 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -77,6 +77,7 @@ struct dcp_mem_descriptor { struct apple_dcp { struct device *dev; struct platform_device *piodma; + struct device_link *piodma_link; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; @@ -1793,12 +1794,15 @@ EXPORT_SYMBOL_GPL(dcp_link); static struct platform_device *dcp_get_dev(struct device *dev, const char *name) { - struct device_node *node = of_get_child_by_name(dev->of_node, name); + struct platform_device *pdev; + struct device_node *node = of_parse_phandle(dev->of_node, name, 0); if (!node) return NULL; - return of_find_device_by_node(node); + pdev = of_find_device_by_node(node); + of_node_put(node); + return pdev; } static int dcp_get_disp_regs(struct apple_dcp *dcp) @@ -1844,12 +1848,22 @@ static int dcp_platform_probe(struct platform_device *pdev) of_platform_default_populate(dev->of_node, NULL, dev); - dcp->piodma = dcp_get_dev(dev, "piodma"); + dcp->piodma = dcp_get_dev(dev, "apple,piodma-mapper"); if (!dcp->piodma) { dev_err(dev, "failed to find piodma\n"); return -ENODEV; } + dcp->piodma_link = device_link_add(dev, &dcp->piodma->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!dcp->piodma_link) { + dev_err(dev, "Failed to link to piodma device"); + return -EINVAL; + } + + if (dcp->piodma_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + ret = dcp_get_disp_regs(dcp); if (ret) { dev_err(dev, "failed to find display registers\n"); From f0a633185d965033125ea93e33e9db842f1ff519 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 28 Sep 2022 17:47:01 +0900 Subject: [PATCH 201/635] drm/apple: Fix kzalloc in dcp_flush() Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/dcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9b695e52bd9a69..fb27446ab95106 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1616,7 +1616,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) .timing_mode_id = mode->timing_mode_id }; - cookie = kzalloc(sizeof(cookie), GFP_KERNEL); + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) { schedule_work(&dcp->vblank_wq); return; From 2babad7335e9bbbf313c9237b198fc9a5ef9b1e6 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 1 Oct 2022 09:48:08 +0200 Subject: [PATCH 202/635] drm/apple: Allow modesets even when disconnected Fixes a display wakeup issue seen with Plasma/X11. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fb27446ab95106..7c426ef7261097 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1513,12 +1513,15 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) struct dcp_swap_submit_req *req = &dcp->swap; int l; int has_surface = 0; + bool modeset; dev_dbg(dcp->dev, "%s", __func__); crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + if (WARN(dcp_channel_busy(&dcp->ch_cmd), "unexpected busy channel") || - WARN(!dcp->connector->connected, "can't flush if disconnected")) { + WARN(!modeset && !dcp->connector->connected, "can't flush if disconnected")) { /* HACK: issue a delayed vblank event to avoid timeouts in * drm_atomic_helper_wait_for_vblanks(). */ @@ -1596,7 +1599,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) /* These fields should be set together */ req->swap.swap_completed = req->swap.swap_enabled; - if (drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode) { + if (modeset) { struct dcp_display_mode *mode; struct dcp_wait_cookie *cookie; int ret; From 8a18621959917e4ced512ce803540e5694918dc3 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 1 Oct 2022 11:05:56 +0200 Subject: [PATCH 203/635] drm/apple: Mark the connecter on init only with modes as connected Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 2 +- drivers/gpu/drm/apple/dcp.c | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index bcac8bb9f476a7..378be6237d9db6 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -329,7 +329,7 @@ static int apple_probe_per_dcp(struct device *dev, return ret; connector->base.polled = DRM_CONNECTOR_POLL_HPD; - connector->connected = true; /* XXX */ + connector->connected = false; connector->dcp = dcp; INIT_WORK(&connector->hotplug_wq, dcp_hotplug); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 7c426ef7261097..4a157e2c7ad578 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1673,12 +1673,19 @@ EXPORT_SYMBOL_GPL(dcp_is_initialized); static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) { + struct apple_connector *connector; int result = *(int *)out; dev_info(dcp->dev, "DCP is_main_display: %d\n", result); dcp->main_display = result != 0; dcp->active = true; + + connector = dcp->connector; + if (connector) { + connector->connected = dcp->nr_modes > 0; + schedule_work(&connector->hotplug_wq); + } } static void init_3(struct apple_dcp *dcp, void *out, void *cookie) @@ -1790,6 +1797,9 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, dcp->crtc = crtc; dcp->connector = connector; + /* init connector status by modes offered by dcp */ + connector->connected = dcp->nr_modes > 0; + /* Dimensions might already be parsed */ dcp_set_dimensions(dcp); } From c4794c012796aa897c7d0dfb61a289685fcf8c10 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 1 Oct 2022 14:46:11 +0200 Subject: [PATCH 204/635] drm/apple: make note about drm.mode_config.max_width/height Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 378be6237d9db6..f576cefd8c3353 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -398,6 +398,8 @@ static int apple_platform_probe(struct platform_device *pdev) /* Unknown maximum, use the iMac (24-inch, 2021) display resolution as * maximum. + * TODO: this is the max framebuffer size not the maximal supported output + * resolution. DCP reports the maximal framebuffer size take it from there. */ apple->drm.mode_config.max_width = 4480; apple->drm.mode_config.max_height = 2520; From 8b23e9d9f469c09a78dd19bf730086073079d850 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 2 Oct 2022 18:22:32 +0200 Subject: [PATCH 205/635] drm/apple: Split dcpep/iomfb out of dcp.c For external display support DCP will use more endpoints. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/dcp-internal.h | 133 ++ drivers/gpu/drm/apple/dcp.c | 1758 +------------------- drivers/gpu/drm/apple/dcp.h | 11 + drivers/gpu/drm/apple/iomfb.c | 1626 ++++++++++++++++++ drivers/gpu/drm/apple/{dcpep.h => iomfb.h} | 48 +- 6 files changed, 1837 insertions(+), 1741 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-internal.h create mode 100644 drivers/gpu/drm/apple/iomfb.c rename drivers/gpu/drm/apple/{dcpep.h => iomfb.h} (86%) diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 8c758d5720b642..d1f909792229e5 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT appledrm-y := apple_drv.o -apple_dcp-y := dcp.o parser.o +apple_dcp-y := dcp.o iomfb.o parser.o apple_piodma-y := dummy-piodma.o obj-$(CONFIG_DRM_APPLE) += appledrm.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h new file mode 100644 index 00000000000000..648d543b9cd91a --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_INTERNAL_H__ +#define __APPLE_DCP_INTERNAL_H__ + +#include +#include +#include + +#include "iomfb.h" + +struct apple_dcp; + +/* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ +struct dcp_chunks { + size_t length; + void *data; +}; + +#define DCP_MAX_MAPPINGS (128) /* should be enough */ +#define MAX_DISP_REGISTERS (7) + +struct dcp_mem_descriptor { + size_t size; + void *buf; + dma_addr_t dva; + struct sg_table map; + u64 reg; +}; + +/* Limit on call stack depth (arbitrary). Some nesting is required */ +#define DCP_MAX_CALL_DEPTH 8 + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + +struct dcp_call_channel { + dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; + void *cookies[DCP_MAX_CALL_DEPTH]; + void *output[DCP_MAX_CALL_DEPTH]; + u16 end[DCP_MAX_CALL_DEPTH]; + + /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ + u8 depth; +}; + +struct dcp_cb_channel { + u8 depth; + void *output[DCP_MAX_CALL_DEPTH]; +}; + +struct dcp_fb_reference { + struct list_head head; + struct drm_framebuffer *fb; +}; + +/* TODO: move IOMFB members to its own struct */ +struct apple_dcp { + struct device *dev; + struct platform_device *piodma; + struct device_link *piodma_link; + struct apple_rtkit *rtk; + struct apple_crtc *crtc; + struct apple_connector *connector; + + /* Coprocessor control register */ + void __iomem *coproc_reg; + + /* mask for DCP IO virtual addresses shared over rtkit */ + u64 asc_dram_mask; + + /* DCP has crashed */ + bool crashed; + + /************* IOMFB ************************************************** + * everything below is mostly used inside IOMFB but it could make * + * sense keep some of the the members in apple_dcp. * + **********************************************************************/ + + /* clock rate request by dcp in */ + struct clk *clk; + + /* DCP shared memory */ + void *shmem; + + /* Display registers mappable to the DCP */ + struct resource *disp_registers[MAX_DISP_REGISTERS]; + unsigned int nr_disp_registers; + + /* Bitmap of memory descriptors used for mappings made by the DCP */ + DECLARE_BITMAP(memdesc_map, DCP_MAX_MAPPINGS); + + /* Indexed table of memory descriptors */ + struct dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; + + struct dcp_call_channel ch_cmd, ch_oobcmd; + struct dcp_cb_channel ch_cb, ch_oobcb, ch_async; + + /* Active chunked transfer. There can only be one at a time. */ + struct dcp_chunks chunks; + + /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ + struct dcp_swap_submit_req swap; + + /* Current display mode */ + bool valid_mode; + struct dcp_set_digital_out_mode_req mode; + + /* Is the DCP booted? */ + bool active; + + /* eDP display without DP-HDMI conversion */ + bool main_display; + + bool ignore_swap_complete; + + /* Modes valid for the connected display */ + struct dcp_display_mode *modes; + unsigned int nr_modes; + + /* Attributes of the connected display */ + int width_mm, height_mm; + + /* Workqueue for sending vblank events when a dcp swap is not possible */ + struct work_struct vblank_wq; + + /* List of referenced drm_framebuffers which can be unreferenced + * on the next successfully completed swap. + */ + struct list_head swapped_out_fbs; +}; + +#endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4a157e2c7ad578..0b3597507739c0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -20,1105 +20,59 @@ #include #include -#include "dcpep.h" #include "dcp.h" +#include "dcp-internal.h" #include "parser.h" -struct apple_dcp; - -#define APPLE_DCP_COPROC_CPU_CONTROL 0x44 -#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) - -/* Register defines used in bandwidth setup structure */ -#define REG_SCRATCH (0x14) -#define REG_SCRATCH_T600X (0x988) -#define REG_DOORBELL (0x0) -#define REG_DOORBELL_BIT (2) - -#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) - -/* Limit on call stack depth (arbitrary). Some nesting is required */ -#define DCP_MAX_CALL_DEPTH 8 - -typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); - -struct dcp_call_channel { - dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; - void *cookies[DCP_MAX_CALL_DEPTH]; - void *output[DCP_MAX_CALL_DEPTH]; - u16 end[DCP_MAX_CALL_DEPTH]; - - /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ - u8 depth; -}; - -struct dcp_cb_channel { - u8 depth; - void *output[DCP_MAX_CALL_DEPTH]; -}; - -/* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ -struct dcp_chunks { - size_t length; - void *data; -}; - -#define DCP_MAX_MAPPINGS (128) /* should be enough */ -#define MAX_DISP_REGISTERS (7) - -struct dcp_mem_descriptor { - size_t size; - void *buf; - dma_addr_t dva; - struct sg_table map; - u64 reg; -}; - -struct apple_dcp { - struct device *dev; - struct platform_device *piodma; - struct device_link *piodma_link; - struct apple_rtkit *rtk; - struct apple_crtc *crtc; - struct apple_connector *connector; - - /* DCP has crashed */ - bool crashed; - - /* clock rate request by dcp in */ - struct clk *clk; - - /* mask for DCP IO virtual addresses shared over rtkit */ - u64 asc_dram_mask; - - /* DCP shared memory */ - void *shmem; - - /* Coprocessor control register */ - void __iomem *coproc_reg; - - /* Display registers mappable to the DCP */ - struct resource *disp_registers[MAX_DISP_REGISTERS]; - unsigned int nr_disp_registers; - - /* Bitmap of memory descriptors used for mappings made by the DCP */ - DECLARE_BITMAP(memdesc_map, DCP_MAX_MAPPINGS); - - /* Indexed table of memory descriptors */ - struct dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; - - struct dcp_call_channel ch_cmd, ch_oobcmd; - struct dcp_cb_channel ch_cb, ch_oobcb, ch_async; - - /* Active chunked transfer. There can only be one at a time. */ - struct dcp_chunks chunks; - - /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ - struct dcp_swap_submit_req swap; - - /* Current display mode */ - bool valid_mode; - struct dcp_set_digital_out_mode_req mode; - - /* Is the DCP booted? */ - bool active; - - /* eDP display without DP-HDMI conversion */ - bool main_display; - - bool ignore_swap_complete; - - /* Modes valid for the connected display */ - struct dcp_display_mode *modes; - unsigned int nr_modes; - - /* Attributes of the connected display */ - int width_mm, height_mm; - - /* Workqueue for sending vblank events when a dcp swap is not possible */ - struct work_struct vblank_wq; - - /* List of referenced drm_framebuffers which can be unreferenced - * on the next successfully completed swap. - */ - struct list_head swapped_out_fbs; -}; - -struct dcp_fb_reference { - struct list_head head; - struct drm_framebuffer *fb; -}; - -struct dcp_wait_cookie { - struct completion done; - atomic_t refcount; -}; - -/* - * A channel is busy if we have sent a message that has yet to be - * acked. The driver must not sent a message to a busy channel. - */ -static bool dcp_channel_busy(struct dcp_call_channel *ch) -{ - return (ch->depth != 0); -} - -/* Get a call channel for a context */ -static struct dcp_call_channel * -dcp_get_call_channel(struct apple_dcp *dcp, enum dcp_context_id context) -{ - switch (context) { - case DCP_CONTEXT_CMD: - case DCP_CONTEXT_CB: - return &dcp->ch_cmd; - case DCP_CONTEXT_OOBCMD: - case DCP_CONTEXT_OOBCB: - return &dcp->ch_oobcmd; - default: - return NULL; - } -} - -/* - * Get the context ID passed to the DCP for a command we push. The rule is - * simple: callback contexts are used when replying to the DCP, command - * contexts are used otherwise. That corresponds to a non/zero call stack - * depth. This rule frees the caller from tracking the call context manually. - */ -static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) -{ - u8 depth = oob ? dcp->ch_oobcmd.depth : dcp->ch_cmd.depth; - - if (depth) - return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; - else - return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; -} - -/* Get a callback channel for a context */ -static struct dcp_cb_channel *dcp_get_cb_channel(struct apple_dcp *dcp, - enum dcp_context_id context) -{ - switch (context) { - case DCP_CONTEXT_CB: - return &dcp->ch_cb; - case DCP_CONTEXT_OOBCB: - return &dcp->ch_oobcb; - case DCP_CONTEXT_ASYNC: - return &dcp->ch_async; - default: - return NULL; - } -} - -/* Get the start of a packet: after the end of the previous packet */ -static u16 dcp_packet_start(struct dcp_call_channel *ch, u8 depth) -{ - if (depth > 0) - return ch->end[depth - 1]; - else - return 0; -} - -/* Pushes and pops the depth of the call stack with safety checks */ -static u8 dcp_push_depth(u8 *depth) -{ - u8 ret = (*depth)++; - - WARN_ON(ret >= DCP_MAX_CALL_DEPTH); - return ret; -} - -static u8 dcp_pop_depth(u8 *depth) -{ - WARN_ON((*depth) == 0); - - return --(*depth); -} - -#define DCP_METHOD(tag, name) [name] = { #name, tag } - -const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { - DCP_METHOD("A000", dcpep_late_init_signal), - DCP_METHOD("A029", dcpep_setup_video_limits), - DCP_METHOD("A034", dcpep_update_notify_clients_dcp), - DCP_METHOD("A357", dcpep_set_create_dfb), - DCP_METHOD("A401", dcpep_start_signal), - DCP_METHOD("A407", dcpep_swap_start), - DCP_METHOD("A408", dcpep_swap_submit), - DCP_METHOD("A410", dcpep_set_display_device), - DCP_METHOD("A411", dcpep_is_main_display), - DCP_METHOD("A412", dcpep_set_digital_out_mode), - DCP_METHOD("A439", dcpep_set_parameter_dcp), - DCP_METHOD("A443", dcpep_create_default_fb), - DCP_METHOD("A447", dcpep_enable_disable_video_power_savings), - DCP_METHOD("A454", dcpep_first_client_open), - DCP_METHOD("A460", dcpep_set_display_refresh_properties), - DCP_METHOD("A463", dcpep_flush_supports_power), - DCP_METHOD("A468", dcpep_set_power_state), -}; - -/* Call a DCP function given by a tag */ -static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, - u32 in_len, u32 out_len, void *data, dcp_callback_t cb, - void *cookie) -{ - struct dcp_call_channel *ch = oob ? &dcp->ch_oobcmd : &dcp->ch_cmd; - enum dcp_context_id context = dcp_call_context(dcp, oob); - - struct dcp_packet_header header = { - .in_len = in_len, - .out_len = out_len, - - /* Tag is reversed due to endianness of the fourcc */ - .tag[0] = dcp_methods[method].tag[3], - .tag[1] = dcp_methods[method].tag[2], - .tag[2] = dcp_methods[method].tag[1], - .tag[3] = dcp_methods[method].tag[0], - }; - - u8 depth = dcp_push_depth(&ch->depth); - u16 offset = dcp_packet_start(ch, depth); - - void *out = dcp->shmem + dcp_tx_offset(context) + offset; - void *out_data = out + sizeof(header); - size_t data_len = sizeof(header) + in_len + out_len; - - memcpy(out, &header, sizeof(header)); - - if (in_len > 0) - memcpy(out_data, data, in_len); - - dev_dbg(dcp->dev, "---> %s: context %u, offset %u, depth %u\n", - dcp_methods[method].name, context, offset, depth); - - ch->callbacks[depth] = cb; - ch->cookies[depth] = cookie; - ch->output[depth] = out + sizeof(header) + in_len; - ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); - - apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, - dcpep_msg(context, data_len, offset), - NULL, false); -} - -#define DCP_THUNK_VOID(func, handle) \ - static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ - void *cookie) \ - { \ - dcp_push(dcp, oob, handle, 0, 0, NULL, cb, cookie); \ - } - -#define DCP_THUNK_OUT(func, handle, T) \ - static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ - void *cookie) \ - { \ - dcp_push(dcp, oob, handle, 0, sizeof(T), NULL, cb, cookie); \ - } - -#define DCP_THUNK_IN(func, handle, T) \ - static void func(struct apple_dcp *dcp, bool oob, T *data, \ - dcp_callback_t cb, void *cookie) \ - { \ - dcp_push(dcp, oob, handle, sizeof(T), 0, data, cb, cookie); \ - } - -#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ - static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ - dcp_callback_t cb, void *cookie) \ - { \ - dcp_push(dcp, oob, handle, sizeof(T_in), sizeof(T_out), data, \ - cb, cookie); \ - } - -DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, - struct dcp_swap_submit_resp); - -DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, - struct dcp_swap_start_resp); - -DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, - struct dcp_set_power_state_req, - struct dcp_set_power_state_resp); - -DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, - struct dcp_set_digital_out_mode_req, u32); - -DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); - -DCP_THUNK_OUT(dcp_set_display_refresh_properties, - dcpep_set_display_refresh_properties, u32); - -DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); -DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); -DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); -DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); -DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); -DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); -DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); - -__attribute__((unused)) -DCP_THUNK_IN(dcp_update_notify_clients_dcp, dcpep_update_notify_clients_dcp, - struct dcp_update_notify_clients_dcp); - -DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, - struct dcp_set_parameter_dcp, u32); - -DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, - dcpep_enable_disable_video_power_savings, - u32, int); - -DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); - -/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ -static int dcp_parse_tag(char tag[4]) -{ - u32 d[3]; - int i; - - if (tag[3] != 'D') - return -EINVAL; - - for (i = 0; i < 3; ++i) { - d[i] = (u32)(tag[i] - '0'); - - if (d[i] > 9) - return -EINVAL; - } - - return d[0] + (d[1] * 10) + (d[2] * 100); -} - -/* Ack a callback from the DCP */ -static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) -{ - struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); - - dcp_pop_depth(&ch->depth); - apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, dcpep_ack(context), - NULL, false); -} - -/* DCP callback handlers */ -static void dcpep_cb_nop(struct apple_dcp *dcp) -{ - /* No operation */ -} - -static u8 dcpep_cb_true(struct apple_dcp *dcp) -{ - return true; -} - -static u8 dcpep_cb_false(struct apple_dcp *dcp) -{ - return false; -} - -static u32 dcpep_cb_zero(struct apple_dcp *dcp) -{ - return 0; -} - -/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ -void dcp_drm_crtc_vblank(struct apple_crtc *crtc) -{ - unsigned long flags; - - if (crtc->vsync_disabled) - return; - - drm_crtc_handle_vblank(&crtc->base); - - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); - if (crtc->event) { - drm_crtc_send_vblank_event(&crtc->base, crtc->event); - drm_crtc_vblank_put(&crtc->base); - crtc->event = NULL; - } - spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); -} - -static void dcpep_cb_swap_complete(struct apple_dcp *dcp, - struct dc_swap_complete_resp *resp) -{ - dev_dbg(dcp->dev, "swap complete for swap_id: %u vblank: %u", - resp->swap_id, dcp->ignore_swap_complete); - - if (!dcp->ignore_swap_complete) - dcp_drm_crtc_vblank(dcp->crtc); -} - -static struct dcp_get_uint_prop_resp -dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) -{ - /* unimplemented for now */ - return (struct dcp_get_uint_prop_resp) { - .value = 0 - }; -} - -/* - * Callback to map a buffer allocated with allocate_buf for PIODMA usage. - * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated - * stream of the display DART, rather than the expected DCP DART. - * - * XXX: This relies on dma_get_sgtable in concert with dma_map_sgtable, which - * is a "fundamentally unsafe" operation according to the docs. And yet - * everyone does it... - */ -static struct dcp_map_buf_resp -dcpep_cb_map_piodma(struct apple_dcp *dcp, struct dcp_map_buf_req *req) -{ - struct sg_table *map; - int ret; - - if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) - goto reject; - - map = &dcp->memdesc[req->buffer].map; - - if (!map->sgl) - goto reject; - - /* Use PIODMA device instead of DCP to map against the right IOMMU. */ - ret = dma_map_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); - - if (ret) - goto reject; - - return (struct dcp_map_buf_resp) { - .dva = sg_dma_address(map->sgl) - }; - -reject: - dev_err(dcp->dev, "denying map of invalid buffer %llx for pidoma\n", - req->buffer); - return (struct dcp_map_buf_resp) { - .ret = EINVAL - }; -} - -static void -dcpep_cb_unmap_piodma(struct apple_dcp *dcp, struct dcp_unmap_buf_resp *resp) -{ - struct sg_table *map; - dma_addr_t dma_addr; - - if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { - dev_warn(dcp->dev, "unmap request for out of range buffer %llu", - resp->buffer); - return; - } - - map = &dcp->memdesc[resp->buffer].map; - - if (!map->sgl) { - dev_warn(dcp->dev, "unmap for non-mapped buffer %llu iova:0x%08llx", - resp->buffer, resp->dva); - return; - } - - dma_addr = sg_dma_address(map->sgl); - if (dma_addr != resp->dva) { - dev_warn(dcp->dev, "unmap buffer %llu address mismatch dma_addr:%llx dva:%llx", - resp->buffer, dma_addr, resp->dva); - return; - } - - /* Use PIODMA device instead of DCP to unmap from the right IOMMU. */ - dma_unmap_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); -} - -/* - * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be - * physically contigiuous, however we should save the sgtable in case the - * buffer needs to be later mapped for PIODMA. - */ -static struct dcp_allocate_buffer_resp -dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_allocate_buffer_req *req) -{ - struct dcp_allocate_buffer_resp resp = { 0 }; - struct dcp_mem_descriptor *memdesc; - u32 id; - - resp.dva_size = ALIGN(req->size, 4096); - resp.mem_desc_id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - - if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { - dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring"); - resp.dva_size = 0; - resp.mem_desc_id = 0; - return resp; - } - id = resp.mem_desc_id; - set_bit(id, dcp->memdesc_map); - - memdesc = &dcp->memdesc[id]; - - memdesc->size = resp.dva_size; - memdesc->buf = dma_alloc_coherent(dcp->dev, memdesc->size, &memdesc->dva, - GFP_KERNEL); - - dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, - memdesc->dva, memdesc->size); - resp.dva = memdesc->dva; - - return resp; -} - -static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) -{ - struct dcp_mem_descriptor *memdesc; - u32 id = *mem_desc_id; - - if (id >= DCP_MAX_MAPPINGS) { - dev_warn(dcp->dev, "unmap request for out of range mem_desc_id %u", - id); - return 0; - } - - if (!test_and_clear_bit(id, dcp->memdesc_map)) { - dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u", - id); - return 0; - } - - memdesc = &dcp->memdesc[id]; - if (memdesc->buf) { - - dma_free_coherent(dcp->dev, memdesc->size, memdesc->buf, - memdesc->dva); - - memdesc->buf = NULL; - memset(&memdesc->map, 0, sizeof(memdesc->map)); - } else { - memdesc->reg = 0; - } - - memdesc->size = 0; - - return 1; -} - -/* Validate that the specified region is a display register */ -static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) -{ - int i; - - for (i = 0; i < dcp->nr_disp_registers; ++i) { - struct resource *r = dcp->disp_registers[i]; - - if ((start >= r->start) && (end <= r->end)) - return true; - } - - return false; -} - -/* - * Map contiguous physical memory into the DCP's address space. The firmware - * uses this to map the display registers we advertise in - * sr_map_device_memory_with_index, so we bounds check against that to guard - * safe against malicious coprocessors. - */ -static struct dcp_map_physical_resp -dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) -{ - int size = ALIGN(req->size, 4096); - u32 id; - - if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { - dev_err(dcp->dev, "refusing to map phys address %llx size %llx", - req->paddr, req->size); - return (struct dcp_map_physical_resp) { }; - } - - id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - set_bit(id, dcp->memdesc_map); - dcp->memdesc[id].size = size; - dcp->memdesc[id].reg = req->paddr; - - return (struct dcp_map_physical_resp) { - .dva_size = size, - .mem_desc_id = id, - .dva = dma_map_resource(dcp->dev, req->paddr, size, - DMA_BIDIRECTIONAL, 0), - }; -} - -static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) -{ - return clk_get_rate(dcp->clk); -} - -static struct dcp_map_reg_resp -dcpep_cb_map_reg(struct apple_dcp *dcp, struct dcp_map_reg_req *req) -{ - if (req->index >= dcp->nr_disp_registers) { - dev_warn(dcp->dev, "attempted to read invalid reg index %u", - req->index); - - return (struct dcp_map_reg_resp) { - .ret = 1 - }; - } else { - struct resource *rsrc = dcp->disp_registers[req->index]; - - return (struct dcp_map_reg_resp) { - .addr = rsrc->start, - .length = resource_size(rsrc) - }; - } -} - -static struct dcp_read_edt_data_resp -dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) -{ - return (struct dcp_read_edt_data_resp) { - .value[0] = req->value[0], - .ret = 0, - }; -} - -/* Chunked data transfer for property dictionaries */ -static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) -{ - if (dcp->chunks.data != NULL) { - dev_warn(dcp->dev, "ignoring spurious transfer start\n"); - return false; - } - - dcp->chunks.length = *length; - dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); - - if (!dcp->chunks.data) { - dev_warn(dcp->dev, "failed to allocate chunks\n"); - return false; - } - - return true; -} - -static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, - struct dcp_set_dcpav_prop_chunk_req *req) -{ - if (!dcp->chunks.data) { - dev_warn(dcp->dev, "ignoring spurious chunk\n"); - return false; - } - - if (req->offset + req->length > dcp->chunks.length) { - dev_warn(dcp->dev, "ignoring overflowing chunk\n"); - return false; - } - - memcpy(dcp->chunks.data + req->offset, req->data, req->length); - return true; -} - -static void dcp_set_dimensions(struct apple_dcp *dcp) -{ - int i; - - /* Set the connector info */ - if (dcp->connector) { - struct drm_connector *connector = &dcp->connector->base; - - mutex_lock(&connector->dev->mode_config.mutex); - connector->display_info.width_mm = dcp->width_mm; - connector->display_info.height_mm = dcp->height_mm; - mutex_unlock(&connector->dev->mode_config.mutex); - } - - /* - * Fix up any probed modes. Modes are created when parsing - * TimingElements, dimensions are calculated when parsing - * DisplayAttributes, and TimingElements may be sent first - */ - for (i = 0; i < dcp->nr_modes; ++i) { - dcp->modes[i].mode.width_mm = dcp->width_mm; - dcp->modes[i].mode.height_mm = dcp->height_mm; - } -} - -static bool dcpep_process_chunks(struct apple_dcp *dcp, - struct dcp_set_dcpav_prop_end_req *req) -{ - struct dcp_parse_ctx ctx; - int ret; - - if (!dcp->chunks.data) { - dev_warn(dcp->dev, "ignoring spurious end\n"); - return false; - } - - ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); - - if (ret) { - dev_warn(dcp->dev, "bad header on dcpav props\n"); - return false; - } - - if (!strcmp(req->key, "TimingElements")) { - dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, - dcp->width_mm, dcp->height_mm); - - if (IS_ERR(dcp->modes)) { - dev_warn(dcp->dev, "failed to parse modes\n"); - dcp->modes = NULL; - dcp->nr_modes = 0; - return false; - } - } else if (!strcmp(req->key, "DisplayAttributes")) { - ret = parse_display_attributes(&ctx, &dcp->width_mm, - &dcp->height_mm); - - if (ret) { - dev_warn(dcp->dev, "failed to parse display attribs\n"); - return false; - } - - dcp_set_dimensions(dcp); - } - - return true; -} - -static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, - struct dcp_set_dcpav_prop_end_req *req) -{ - u8 resp = dcpep_process_chunks(dcp, req); - - /* Reset for the next transfer */ - devm_kfree(dcp->dev, dcp->chunks.data); - dcp->chunks.data = NULL; - - return resp; -} - -/* Boot sequence */ -static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_cb_channel *ch = &dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - dev_dbg(dcp->dev, "boot done"); - - *succ = true; - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); -} - -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_late_init_signal(dcp, false, boot_5, NULL); -} - -static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) -{ - u32 v_true = true; - - dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); -} - -static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_setup_video_limits(dcp, false, boot_3, NULL); -} - -static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_create_default_fb(dcp, false, boot_2, NULL); -} - -/* Use special function signature to defer the ACK */ -static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) -{ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, __func__); - dcp_set_create_dfb(dcp, false, boot_1_5, NULL); - return false; -} - -static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) -{ - if (dcp->disp_registers[5] && dcp->disp_registers[6]) - return (struct dcp_rt_bandwidth) { - .reg_scratch = dcp->disp_registers[5]->start + REG_SCRATCH, - .reg_doorbell = dcp->disp_registers[6]->start + REG_DOORBELL, - .doorbell_bit = REG_DOORBELL_BIT, - - .padding[3] = 0x4, // XXX: required by 11.x firmware - }; - else if (dcp->disp_registers[4]) - return (struct dcp_rt_bandwidth) { - .reg_scratch = dcp->disp_registers[4]->start + REG_SCRATCH_T600X, - .reg_doorbell = 0, - .doorbell_bit = 0, - }; - else - return (struct dcp_rt_bandwidth) { - .reg_scratch = 0, - .reg_doorbell = 0, - .doorbell_bit = 0, - }; -} - -/* Callback to get the current time as milliseconds since the UNIX epoch */ -static u64 dcpep_cb_get_time(struct apple_dcp *dcp) -{ - return ktime_to_ms(ktime_get_real()); -} - -struct dcp_swap_cookie { - struct completion done; - atomic_t refcount; - u32 swap_id; -}; - -static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_submit_resp *resp = data; - dev_dbg(dcp->dev, "%s", __func__); - - if (cookie) { - struct dcp_swap_cookie *info = cookie; - complete(&info->done); - if (atomic_dec_and_test(&info->refcount)) - kfree(info); - } - - if (resp->ret) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} - -static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, - void *cookie) -{ - struct dcp_swap_start_resp *resp = data; - dev_dbg(dcp->dev, "%s swap_id: %u", __func__, resp->swap_id); - dcp->swap.swap.swap_id = resp->swap_id; - - if (cookie) { - struct dcp_swap_cookie *info = cookie; - info->swap_id = resp->swap_id; - } - - dcp_swap_submit(dcp, false, &dcp->swap, dcp_swap_cleared, cookie); -} - -static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_wait_cookie *wait = cookie; - dev_dbg(dcp->dev, "%s", __func__); - - if (wait) { - complete(&wait->done); - if (atomic_dec_and_test(&wait->refcount)) - kfree(wait); - } -} +#define APPLE_DCP_COPROC_CPU_CONTROL 0x44 +#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) -static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_set_parameter_dcp param = { - .param = 14, - .value = { 0 }, - .count = 1, - }; - dev_dbg(dcp->dev, "%s", __func__); - - dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_final, cookie); -} +#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -void dcp_poweron(struct platform_device *pdev) +/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ +void dcp_drm_crtc_vblank(struct apple_crtc *crtc) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); - struct dcp_wait_cookie * cookie; - struct dcp_set_power_state_req req = { - .unklong = 1, - }; - int ret; - u32 handle; - dev_dbg(dcp->dev, "%s", __func__); + unsigned long flags; - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) + if (crtc->vsync_disabled) return; - init_completion(&cookie->done); - atomic_set(&cookie->refcount, 2); - - if (dcp->main_display) { - handle = 0; - dcp_set_display_device(dcp, false, &handle, dcp_on_final, cookie); - } else { - handle = 2; - dcp_set_display_device(dcp, false, &handle, dcp_on_set_parameter, cookie); - } - dcp_set_power_state(dcp, true, &req, NULL, NULL); - - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); - - if (ret == 0) - dev_warn(dcp->dev, "wait for power timed out"); - - if (atomic_dec_and_test(&cookie->refcount)) - kfree(cookie); -} -EXPORT_SYMBOL(dcp_poweron); - -static void complete_set_powerstate(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_wait_cookie *wait = cookie; + drm_crtc_handle_vblank(&crtc->base); - if (wait) { - complete(&wait->done); - if (atomic_dec_and_test(&wait->refcount)) - kfree(wait); + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + drm_crtc_vblank_put(&crtc->base); + crtc->event = NULL; } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } -void dcp_poweroff(struct platform_device *pdev) +void dcp_set_dimensions(struct apple_dcp *dcp) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); - int ret, swap_id; - struct dcp_set_power_state_req power_req = { - .unklong = 0, - }; - struct dcp_swap_cookie *cookie; - struct dcp_wait_cookie *poff_cookie; - struct dcp_swap_start_req swap_req= { 0 }; - - dev_dbg(dcp->dev, "%s", __func__); - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) - return; - init_completion(&cookie->done); - atomic_set(&cookie->refcount, 2); - - // clear surfaces - memset(&dcp->swap, 0, sizeof(dcp->swap)); - - dcp->swap.swap.swap_enabled = DCP_REMOVE_LAYERS | 0x7; - dcp->swap.swap.swap_completed = DCP_REMOVE_LAYERS | 0x7; - dcp->swap.swap.unk_10c = 0xFF000000; - - for (int l = 0; l < SWAP_SURFACES; l++) - dcp->swap.surf_null[l] = true; + int i; - dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + /* Set the connector info */ + if (dcp->connector) { + struct drm_connector *connector = &dcp->connector->base; - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); - swap_id = cookie->swap_id; - if (atomic_dec_and_test(&cookie->refcount)) - kfree(cookie); - if (ret <= 0) { - dcp->crashed = true; - return; + mutex_lock(&connector->dev->mode_config.mutex); + connector->display_info.width_mm = dcp->width_mm; + connector->display_info.height_mm = dcp->height_mm; + mutex_unlock(&connector->dev->mode_config.mutex); } - dev_dbg(dcp->dev, "%s: clear swap submitted: %u", __func__, swap_id); - - poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); - if (!poff_cookie) - return; - init_completion(&poff_cookie->done); - atomic_set(&poff_cookie->refcount, 2); - - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, poff_cookie); - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); - - if (ret == 0) - dev_warn(dcp->dev, "setPowerState(0) timeout %u ms", 1000); - else if (ret > 0) - dev_dbg(dcp->dev, "setPowerState(0) finished with %d ms to spare", - jiffies_to_msecs(ret)); - - if (atomic_dec_and_test(&poff_cookie->refcount)) - kfree(poff_cookie); - dev_dbg(dcp->dev, "%s: setPowerState(0) done", __func__); -} -EXPORT_SYMBOL(dcp_poweroff); - -/* - * Helper to send a DRM hotplug event. The DCP is accessed from a single - * (RTKit) thread. To handle hotplug callbacks, we need to call - * drm_kms_helper_hotplug_event, which does an atomic commit (via DCP) and - * waits for vblank (a DCP callback). That means we deadlock if we call from - * the RTKit thread! Instead, move the call to another thread via a workqueue. - */ -void dcp_hotplug(struct work_struct *work) -{ - struct apple_connector *connector; - struct drm_device *dev; - struct apple_dcp *dcp; - - connector = container_of(work, struct apple_connector, hotplug_wq); - dev = connector->base.dev; - - dcp = platform_get_drvdata(connector->dcp); - dev_info(dcp->dev, "%s: connected: %d", __func__, connector->connected); - /* - * DCP defers link training until we set a display mode. But we set - * display modes from atomic_flush, so userspace needs to trigger a - * flush, or the CRTC gets no signal. + * Fix up any probed modes. Modes are created when parsing + * TimingElements, dimensions are calculated when parsing + * DisplayAttributes, and TimingElements may be sent first */ - if (!dcp->valid_mode && connector->connected) { - drm_connector_set_link_status_property( - &connector->base, DRM_MODE_LINK_STATUS_BAD); - } - - if (dev && dev->registered) - drm_kms_helper_hotplug_event(dev); -} -EXPORT_SYMBOL_GPL(dcp_hotplug); - -static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) -{ - struct apple_connector *connector = dcp->connector; - - /* Hotplug invalidates mode. DRM doesn't always handle this. */ - if (!(*connected)) { - dcp->valid_mode = false; - /* after unplug swap will not complete until the next - * set_digital_out_mode */ - schedule_work(&dcp->vblank_wq); - } - - if (connector && connector->connected != !!(*connected)) { - connector->connected = !!(*connected); - dcp->valid_mode = false; - schedule_work(&connector->hotplug_wq); + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = dcp->width_mm; + dcp->modes[i].mode.height_mm = dcp->height_mm; } } -static void -dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, - struct dcp_swap_complete_intent_gated *info) -{ - dev_dbg(dcp->dev, "swap_id:%u width:%u height:%u", info->swap_id, - info->width, info->height); -} - /* * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks @@ -1133,597 +87,16 @@ static void dcp_delayed_vblank(struct work_struct *work) dcp_drm_crtc_vblank(dcp->crtc); } - -#define DCPEP_MAX_CB (1000) - -/* - * Define type-safe trampolines. Define typedefs to enforce type-safety on the - * input data (so if the types don't match, gcc errors out). - */ - -#define TRAMPOLINE_VOID(func, handler) \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ - handler(dcp); \ - return true; \ - } - -#define TRAMPOLINE_IN(func, handler, T_in) \ - typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ - \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - callback_##handler cb = handler; \ - \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ - cb(dcp, in); \ - return true; \ - } - -#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ - typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ - \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - T_out *typed_out = out; \ - callback_##handler cb = handler; \ - \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ - *typed_out = cb(dcp, in); \ - return true; \ - } - -#define TRAMPOLINE_OUT(func, handler, T_out) \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - T_out *typed_out = out; \ - \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ - *typed_out = handler(dcp); \ - return true; \ - } - -TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); -TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); -TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); -TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); -TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, - struct dc_swap_complete_resp); -TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, - struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); -TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct dcp_map_buf_req, struct dcp_map_buf_resp); -TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, - struct dcp_unmap_buf_resp); -TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, - struct dcp_allocate_buffer_req, - struct dcp_allocate_buffer_resp); -TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, - struct dcp_map_physical_req, struct dcp_map_physical_resp); -TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, - u32, u8); -TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct dcp_map_reg_req, - struct dcp_map_reg_resp); -TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, - struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); -TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); -TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, - struct dcp_set_dcpav_prop_chunk_req, u8); -TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, - struct dcp_set_dcpav_prop_end_req, u8); -TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, - struct dcp_rt_bandwidth); -TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); -TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); -TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); -TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, - dcpep_cb_swap_complete_intent_gated, - struct dcp_swap_complete_intent_gated); - -bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, void *) = { - [0] = trampoline_true, /* did_boot_signal */ - [1] = trampoline_true, /* did_power_on_signal */ - [2] = trampoline_nop, /* will_power_off_signal */ - [3] = trampoline_rt_bandwidth, - [100] = trampoline_nop, /* match_pmu_service */ - [101] = trampoline_zero, /* get_display_default_stride */ - [103] = trampoline_nop, /* set_boolean_property */ - [106] = trampoline_nop, /* remove_property */ - [107] = trampoline_true, /* create_provider_service */ - [108] = trampoline_true, /* create_product_service */ - [109] = trampoline_true, /* create_pmu_service */ - [110] = trampoline_true, /* create_iomfb_service */ - [111] = trampoline_false, /* create_backlight_service */ - [116] = dcpep_cb_boot_1, - [117] = trampoline_false, /* is_dark_boot */ - [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [120] = trampoline_read_edt_data, - [122] = trampoline_prop_start, - [123] = trampoline_prop_chunk, - [124] = trampoline_prop_end, - [201] = trampoline_map_piodma, - [202] = trampoline_unmap_piodma, - [206] = trampoline_true, /* match_pmu_service_2 */ - [207] = trampoline_true, /* match_backlight_service */ - [208] = trampoline_get_time, - [211] = trampoline_nop, /* update_backlight_factor_prop */ - [300] = trampoline_nop, /* pr_publish */ - [401] = trampoline_get_uint_prop, - [404] = trampoline_nop, /* sr_set_uint_prop */ - [406] = trampoline_nop, /* set_fx_prop */ - [408] = trampoline_get_frequency, - [411] = trampoline_map_reg, - [413] = trampoline_true, /* sr_set_property_dict */ - [414] = trampoline_true, /* sr_set_property_int */ - [415] = trampoline_true, /* sr_set_property_bool */ - [451] = trampoline_allocate_buffer, - [452] = trampoline_map_physical, - [456] = trampoline_release_mem_desc, - [552] = trampoline_true, /* set_property_dict_0 */ - [561] = trampoline_true, /* set_property_dict */ - [563] = trampoline_true, /* set_property_int */ - [565] = trampoline_true, /* set_property_bool */ - [567] = trampoline_true, /* set_property_str */ - [574] = trampoline_zero, /* power_up_dart */ - [576] = trampoline_hotplug, - [577] = trampoline_nop, /* powerstate_notify */ - [582] = trampoline_true, /* create_default_fb_surface */ - [589] = trampoline_swap_complete, - [591] = trampoline_swap_complete_intent_gated, - [593] = trampoline_nop, /* enable_backlight_message_ap_gated */ - [598] = trampoline_nop, /* find_swap_function_gated */ -}; - -static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) -{ - struct device *dev = dcp->dev; - struct dcp_packet_header *hdr = data; - void *in, *out; - int tag = dcp_parse_tag(hdr->tag); - struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); - u8 depth; - - if (tag < 0 || tag >= DCPEP_MAX_CB || !dcpep_cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - return; - } - - in = data + sizeof(*hdr); - out = in + hdr->in_len; - - // TODO: verify that in_len and out_len match our prototypes - // for now just clear the out data to have at least consistant results - if (hdr->out_len) - memset(out, 0, hdr->out_len); - - depth = dcp_push_depth(&ch->depth); - ch->output[depth] = out; - - if (dcpep_cb_handlers[tag](dcp, tag, out, in)) - dcp_ack(dcp, context); -} - -static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) -{ - struct dcp_packet_header *header = data; - struct dcp_call_channel *ch = dcp_get_call_channel(dcp, context); - void *cookie; - dcp_callback_t cb; - - if (!ch) { - dev_warn(dcp->dev, "ignoring ack on context %X\n", context); - return; - } - - dcp_pop_depth(&ch->depth); - - cb = ch->callbacks[ch->depth]; - cookie = ch->cookies[ch->depth]; - - ch->callbacks[ch->depth] = NULL; - ch->cookies[ch->depth] = NULL; - - if (cb) - cb(dcp, data + sizeof(*header) + header->in_len, cookie); -} - -static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) -{ - enum dcp_context_id ctx_id; - u16 offset; - u32 length; - int channel_offset; - void *data; - - ctx_id = (message & DCPEP_CONTEXT_MASK) >> DCPEP_CONTEXT_SHIFT; - offset = (message & DCPEP_OFFSET_MASK) >> DCPEP_OFFSET_SHIFT; - length = (message >> DCPEP_LENGTH_SHIFT); - - channel_offset = dcp_channel_offset(ctx_id); - - if (channel_offset < 0) { - dev_warn(dcp->dev, "invalid context received %u", ctx_id); - return; - } - - data = dcp->shmem + channel_offset + offset; - - if (message & DCPEP_ACK) - dcpep_handle_ack(dcp, ctx_id, data, length); - else - dcpep_handle_cb(dcp, ctx_id, data, length); -} - -/* - * Callback for swap requests. If a swap failed, we'll never get a swap - * complete event so we need to fake a vblank event early to avoid a hang. - */ - -static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_submit_resp *resp = data; - - if (resp->ret) { - dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} - -static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_start_resp *resp = data; - - dcp->swap.swap.swap_id = resp->swap_id; - - dcp_swap_submit(dcp, false, &dcp->swap, dcp_swapped, NULL); -} - -/* - * DRM specifies rectangles as start and end coordinates. DCP specifies - * rectangles as a start coordinate and a width/height. Convert a DRM rectangle - * to a DCP rectangle. - */ -static struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect) -{ - return (struct dcp_rect) { - .x = rect->x1, - .y = rect->y1, - .w = drm_rect_width(rect), - .h = drm_rect_height(rect) - }; -} - -static u32 drm_format_to_dcp(u32 drm) -{ - switch (drm) { - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - return fourcc_code('A', 'R', 'G', 'B'); - - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ABGR8888: - return fourcc_code('A', 'B', 'G', 'R'); - } - - pr_warn("DRM format %X not supported in DCP\n", drm); - return 0; -} - -int dcp_get_modes(struct drm_connector *connector) -{ - struct apple_connector *apple_connector = to_apple_connector(connector); - struct platform_device *pdev = apple_connector->dcp; - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - struct drm_device *dev = connector->dev; - struct drm_display_mode *mode; - int i; - - for (i = 0; i < dcp->nr_modes; ++i) { - mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); - - if (!mode) { - dev_err(dev->dev, "Failed to duplicate display mode\n"); - return 0; - } - - drm_mode_probed_add(connector, mode); - } - - return dcp->nr_modes; -} -EXPORT_SYMBOL_GPL(dcp_get_modes); - -/* The user may own drm_display_mode, so we need to search for our copy */ -static struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, - struct drm_display_mode *mode) -{ - int i; - - for (i = 0; i < dcp->nr_modes; ++i) { - if (drm_mode_match(mode, &dcp->modes[i].mode, - DRM_MODE_MATCH_TIMINGS | - DRM_MODE_MATCH_CLOCK)) - return &dcp->modes[i]; - } - - return NULL; -} - -int dcp_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - struct apple_connector *apple_connector = to_apple_connector(connector); - struct platform_device *pdev = apple_connector->dcp; - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; -} -EXPORT_SYMBOL_GPL(dcp_mode_valid); - -/* Helpers to modeset and swap, used to flush */ -static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_start_req start_req = { 0 }; - dev_dbg(dcp->dev, "%s", __func__); - - if (dcp->connector && dcp->connector->connected) - dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); - else - dcp_drm_crtc_vblank(dcp->crtc); -} - -static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, - void *cookie) -{ - struct dcp_wait_cookie *wait = cookie; - dev_dbg(dcp->dev, "%s", __func__); - - dcp->ignore_swap_complete = false; - - if (wait) { - complete(&wait->done); - if (atomic_dec_and_test(&wait->refcount)) - kfree(wait); - } -} - -void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - struct platform_device *pdev = to_apple_crtc(crtc)->dcp; - struct apple_dcp *dcp = platform_get_drvdata(pdev); - struct drm_plane *plane; - struct drm_plane_state *new_state, *old_state; - struct drm_crtc_state *crtc_state; - struct dcp_swap_submit_req *req = &dcp->swap; - int l; - int has_surface = 0; - bool modeset; - dev_dbg(dcp->dev, "%s", __func__); - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; - - if (WARN(dcp_channel_busy(&dcp->ch_cmd), "unexpected busy channel") || - WARN(!modeset && !dcp->connector->connected, "can't flush if disconnected")) { - /* HACK: issue a delayed vblank event to avoid timeouts in - * drm_atomic_helper_wait_for_vblanks(). - */ - schedule_work(&dcp->vblank_wq); - return; - } - - /* Reset to defaults */ - memset(req, 0, sizeof(*req)); - for (l = 0; l < SWAP_SURFACES; l++) - req->surf_null[l] = true; - - for_each_oldnew_plane_in_state(state, plane, old_state, new_state, l) { - struct drm_framebuffer *fb = new_state->fb; - struct drm_rect src_rect; - - WARN_ON(l >= SWAP_SURFACES); - - req->swap.swap_enabled |= BIT(l); - - if (old_state->fb && fb != old_state->fb) { - /* - * Race condition between a framebuffer unbind getting - * swapped out and GEM unreferencing a framebuffer. If - * we lose the race, the display gets IOVA faults and - * the DCP crashes. We need to extend the lifetime of - * the drm_framebuffer (and hence the GEM object) until - * after we get a swap complete for the swap unbinding - * it. - */ - struct dcp_fb_reference *entry = kzalloc(sizeof(*entry), GFP_KERNEL); - if (entry) { - entry->fb = old_state->fb; - list_add_tail(&entry->head, &dcp->swapped_out_fbs); - } - drm_framebuffer_get(old_state->fb); - } - - if (!new_state->fb) { - if (old_state->fb) - req->swap.swap_enabled |= DCP_REMOVE_LAYERS; - - continue; - } - req->surf_null[l] = false; - has_surface = 1; - - - drm_rect_fp_to_int(&src_rect, &new_state->src); - - req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); - req->swap.dst_rect[l] = drm_to_dcp_rect(&new_state->dst); - - req->surf_iova[l] = drm_fb_dma_get_gem_addr(fb, new_state, 0); - - req->surf[l] = (struct dcp_surface) { - .format = drm_format_to_dcp(fb->format->format), - .xfer_func = 13, - .colorspace = 1, - .stride = fb->pitches[0], - .width = fb->width, - .height = fb->height, - .buf_size = fb->height * fb->pitches[0], - .surface_id = req->swap.surf_ids[l], - - /* Only used for compressed or multiplanar surfaces */ - .pix_size = 1, - .pel_w = 1, - .pel_h = 1, - .has_comp = 1, - .has_planes = 1, - }; - } - - /* These fields should be set together */ - req->swap.swap_completed = req->swap.swap_enabled; - - if (modeset) { - struct dcp_display_mode *mode; - struct dcp_wait_cookie *cookie; - int ret; - - mode = lookup_mode(dcp, &crtc_state->mode); - if (!mode) { - dev_warn(dcp->dev, "no match for " DRM_MODE_FMT, - DRM_MODE_ARG(&crtc_state->mode)); - schedule_work(&dcp->vblank_wq); - return; - } - - dev_info(dcp->dev, "set_digital_out_mode(color:%d timing:%d)", - mode->color_mode_id, mode->timing_mode_id); - dcp->mode = (struct dcp_set_digital_out_mode_req) { - .color_mode_id = mode->color_mode_id, - .timing_mode_id = mode->timing_mode_id - }; - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) { - schedule_work(&dcp->vblank_wq); - return; - } - - init_completion(&cookie->done); - atomic_set(&cookie->refcount, 2); - - dcp_set_digital_out_mode(dcp, false, &dcp->mode, - complete_set_digital_out_mode, cookie); - - dev_dbg(dcp->dev, "%s - wait for modeset", __func__); - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); - - if (atomic_dec_and_test(&cookie->refcount)) - kfree(cookie); - - if (ret == 0) { - dev_dbg(dcp->dev, "set_digital_out_mode 200 ms"); - schedule_work(&dcp->vblank_wq); - return; - } - else if (ret > 0) { - dev_dbg(dcp->dev, "set_digital_out_mode finished with %d to spare", - jiffies_to_msecs(ret)); - } - - dcp->valid_mode = true; - } - - if (!has_surface) { - if (crtc_state->enable && crtc_state->active && !crtc_state->planes_changed) { - schedule_work(&dcp->vblank_wq); - return; - } - - req->clear = 1; - } - do_swap(dcp, NULL, NULL); -} -EXPORT_SYMBOL_GPL(dcp_flush); - -bool dcp_is_initialized(struct platform_device *pdev) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - return dcp->active; -} -EXPORT_SYMBOL_GPL(dcp_is_initialized); - - -static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct apple_connector *connector; - int result = *(int *)out; - dev_info(dcp->dev, "DCP is_main_display: %d\n", result); - - dcp->main_display = result != 0; - - dcp->active = true; - - connector = dcp->connector; - if (connector) { - connector->connected = dcp->nr_modes > 0; - schedule_work(&connector->hotplug_wq); - } -} - -static void init_3(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_is_main_display(dcp, false, res_is_main_display, NULL); -} - -static void init_2(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_first_client_open(dcp, false, init_3, NULL); -} - -static void init_1(struct apple_dcp *dcp, void *out, void *cookie) -{ - u32 val = 0; - dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); -} - -static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) -{ - dev_info(dcp->dev, "DCP booted\n"); - - init_1(dcp, data, cookie); -} - -static void dcp_got_msg(void *cookie, u8 endpoint, u64 message) +static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; - enum dcpep_type type = (message >> DCPEP_TYPE_SHIFT) & DCPEP_TYPE_MASK; - WARN_ON(endpoint != DCP_ENDPOINT); - - if (type == DCPEP_TYPE_INITIALIZED) - dcp_start_signal(dcp, false, dcp_started, NULL); - else if (type == DCPEP_TYPE_MESSAGE) - dcpep_got_msg(dcp, message); - else - dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); + switch (endpoint) { + case IOMFB_ENDPOINT: + return iomfb_recv_msg(dcp, message); + default: + WARN(endpoint, "unknown DCP endpoint %hhu", endpoint); + } } static void dcp_rtk_crashed(void *cookie) @@ -1739,14 +112,16 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) struct apple_dcp *dcp = cookie; if (bfr->iova) { - struct iommu_domain *domain = iommu_get_domain_for_dev(dcp->dev); + struct iommu_domain *domain = + iommu_get_domain_for_dev(dcp->dev); phys_addr_t phy_addr; if (!domain) return -ENOMEM; // TODO: get map from device-tree - phy_addr = iommu_iova_to_phys(domain, bfr->iova & ~dcp->asc_dram_mask); + phy_addr = iommu_iova_to_phys(domain, + bfr->iova & ~dcp->asc_dram_mask); if (!phy_addr) return -ENOMEM; @@ -1756,10 +131,13 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) return -ENOMEM; bfr->is_mapped = true; - dev_info(dcp->dev, "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx", - (uintptr_t)bfr->iova, (uintptr_t)phy_addr, (uintptr_t)bfr->buffer); + dev_info(dcp->dev, + "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx", + (uintptr_t)bfr->iova, (uintptr_t)phy_addr, + (uintptr_t)bfr->buffer); } else { - bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, &bfr->iova, GFP_KERNEL); + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, + &bfr->iova, GFP_KERNEL); if (!bfr->buffer) return -ENOMEM; @@ -1779,12 +157,13 @@ static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) if (bfr->is_mapped) memunmap(bfr->buffer); else - dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova & ~dcp->asc_dram_mask); + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, + bfr->iova & ~dcp->asc_dram_mask); } static struct apple_rtkit_ops rtkit_ops = { .crashed = dcp_rtk_crashed, - .recv_message = dcp_got_msg, + .recv_message = dcp_recv_msg, .shmem_setup = dcp_rtk_shmem_setup, .shmem_destroy = dcp_rtk_shmem_destroy, }; @@ -1840,7 +219,6 @@ static int dcp_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_dcp *dcp; - dma_addr_t shmem_iova; u32 cpu_ctrl; int ret; @@ -1885,7 +263,8 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->clk = devm_clk_get(dev, NULL); if (IS_ERR(dcp->clk)) - return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + return dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); ret = of_property_read_u64(dev->of_node, "apple,asc-dram-mask", &dcp->asc_dram_mask); @@ -1899,9 +278,11 @@ static int dcp_platform_probe(struct platform_device *pdev) INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + dcp->swapped_out_fbs = + (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); - cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -1915,14 +296,11 @@ static int dcp_platform_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to boot RTKit: %d", ret); - apple_rtkit_start_ep(dcp->rtk, DCP_ENDPOINT); - - dcp->shmem = dma_alloc_coherent(dev, DCP_SHMEM_SIZE, &shmem_iova, - GFP_KERNEL); - - shmem_iova |= dcp->asc_dram_mask; - apple_rtkit_send_message(dcp->rtk, DCP_ENDPOINT, - dcpep_set_shmem(shmem_iova), NULL, false); + /* start RTKit endpoints */ + ret = iomfb_start_rtkit(dcp); + if (ret) + return dev_err_probe(dev, ret, + "Failed to start IOMFB endpoint: %d", ret); return ret; } @@ -1935,15 +313,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - struct dcp_set_power_state_req req = { - /* defaults are ok */ - }; - - /* We're going down */ - dcp->active = false; - dcp->valid_mode = false; - - dcp_set_power_state(dcp, false, &req, NULL, NULL); + iomfb_shutdown(dcp); } static const struct of_device_id of_match[] = { @@ -1966,9 +336,7 @@ static int dcp_suspend(struct device *dev) static int dcp_resume(struct device *dev) { - struct apple_dcp *dcp = platform_get_drvdata(to_platform_device(dev)); - - dcp_start_signal(dcp, false, dcp_started, NULL); + dcp_poweron(to_platform_device(dev)); return 0; } diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 9e3e3738a39377..18d71afaf6b72e 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -5,6 +5,9 @@ #define __APPLE_DCP_H__ #include +#include + +#include "dcp-internal.h" #include "parser.h" struct apple_crtc { @@ -39,8 +42,16 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); bool dcp_is_initialized(struct platform_device *pdev); void apple_crtc_vblank(struct apple_crtc *apple); +void dcp_drm_crtc_vblank(struct apple_crtc *crtc); int dcp_get_modes(struct drm_connector *connector); int dcp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode); +void dcp_set_dimensions(struct apple_dcp *dcp); + + +int iomfb_start_rtkit(struct apple_dcp *dcp); +void iomfb_shutdown(struct apple_dcp *dcp); +/* rtkit message handler for IOMFB messages */ +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); #endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c new file mode 100644 index 00000000000000..0da3de1aa27e78 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb.c @@ -0,0 +1,1626 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "parser.h" + +/* Register defines used in bandwidth setup structure */ +#define REG_SCRATCH (0x14) +#define REG_SCRATCH_T600X (0x988) +#define REG_DOORBELL (0x0) +#define REG_DOORBELL_BIT (2) + +struct dcp_wait_cookie { + struct completion done; + atomic_t refcount; +}; + +static int dcp_tx_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_CMD: + return 0x00000; + case DCP_CONTEXT_OOBCB: + case DCP_CONTEXT_OOBCMD: + return 0x08000; + default: + return -EINVAL; + } +} + +static int dcp_channel_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_ASYNC: + return 0x40000; + case DCP_CONTEXT_CB: + return 0x60000; + case DCP_CONTEXT_OOBCB: + return 0x68000; + default: + return dcp_tx_offset(id); + } +} + +static inline u64 dcpep_set_shmem(u64 dart_va) +{ + return (DCPEP_TYPE_SET_SHMEM << DCPEP_TYPE_SHIFT) | + (DCPEP_FLAG_VALUE << DCPEP_FLAG_SHIFT) | + (dart_va << DCPEP_DVA_SHIFT); +} + +static inline u64 dcpep_msg(enum dcp_context_id id, u32 length, u16 offset) +{ + return (DCPEP_TYPE_MESSAGE << DCPEP_TYPE_SHIFT) | + ((u64)id << DCPEP_CONTEXT_SHIFT) | + ((u64)offset << DCPEP_OFFSET_SHIFT) | + ((u64)length << DCPEP_LENGTH_SHIFT); +} + +static inline u64 dcpep_ack(enum dcp_context_id id) +{ + return dcpep_msg(id, 0, 0) | DCPEP_ACK; +} + +/* + * A channel is busy if we have sent a message that has yet to be + * acked. The driver must not sent a message to a busy channel. + */ +static bool dcp_channel_busy(struct dcp_call_channel *ch) +{ + return (ch->depth != 0); +} + +/* Get a call channel for a context */ +static struct dcp_call_channel * +dcp_get_call_channel(struct apple_dcp *dcp, enum dcp_context_id context) +{ + switch (context) { + case DCP_CONTEXT_CMD: + case DCP_CONTEXT_CB: + return &dcp->ch_cmd; + case DCP_CONTEXT_OOBCMD: + case DCP_CONTEXT_OOBCB: + return &dcp->ch_oobcmd; + default: + return NULL; + } +} + +/* + * Get the context ID passed to the DCP for a command we push. The rule is + * simple: callback contexts are used when replying to the DCP, command + * contexts are used otherwise. That corresponds to a non/zero call stack + * depth. This rule frees the caller from tracking the call context manually. + */ +static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) +{ + u8 depth = oob ? dcp->ch_oobcmd.depth : dcp->ch_cmd.depth; + + if (depth) + return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; + else + return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; +} + +/* Get a callback channel for a context */ +static struct dcp_cb_channel *dcp_get_cb_channel(struct apple_dcp *dcp, + enum dcp_context_id context) +{ + switch (context) { + case DCP_CONTEXT_CB: + return &dcp->ch_cb; + case DCP_CONTEXT_OOBCB: + return &dcp->ch_oobcb; + case DCP_CONTEXT_ASYNC: + return &dcp->ch_async; + default: + return NULL; + } +} + +/* Get the start of a packet: after the end of the previous packet */ +static u16 dcp_packet_start(struct dcp_call_channel *ch, u8 depth) +{ + if (depth > 0) + return ch->end[depth - 1]; + else + return 0; +} + +/* Pushes and pops the depth of the call stack with safety checks */ +static u8 dcp_push_depth(u8 *depth) +{ + u8 ret = (*depth)++; + + WARN_ON(ret >= DCP_MAX_CALL_DEPTH); + return ret; +} + +static u8 dcp_pop_depth(u8 *depth) +{ + WARN_ON((*depth) == 0); + + return --(*depth); +} + +#define DCP_METHOD(tag, name) [name] = { #name, tag } + +const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + DCP_METHOD("A000", dcpep_late_init_signal), + DCP_METHOD("A029", dcpep_setup_video_limits), + DCP_METHOD("A034", dcpep_update_notify_clients_dcp), + DCP_METHOD("A357", dcpep_set_create_dfb), + DCP_METHOD("A401", dcpep_start_signal), + DCP_METHOD("A407", dcpep_swap_start), + DCP_METHOD("A408", dcpep_swap_submit), + DCP_METHOD("A410", dcpep_set_display_device), + DCP_METHOD("A411", dcpep_is_main_display), + DCP_METHOD("A412", dcpep_set_digital_out_mode), + DCP_METHOD("A439", dcpep_set_parameter_dcp), + DCP_METHOD("A443", dcpep_create_default_fb), + DCP_METHOD("A447", dcpep_enable_disable_video_power_savings), + DCP_METHOD("A454", dcpep_first_client_open), + DCP_METHOD("A460", dcpep_set_display_refresh_properties), + DCP_METHOD("A463", dcpep_flush_supports_power), + DCP_METHOD("A468", dcpep_set_power_state), +}; + +/* Call a DCP function given by a tag */ +static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie) +{ + struct dcp_call_channel *ch = oob ? &dcp->ch_oobcmd : &dcp->ch_cmd; + enum dcp_context_id context = dcp_call_context(dcp, oob); + + struct dcp_packet_header header = { + .in_len = in_len, + .out_len = out_len, + + /* Tag is reversed due to endianness of the fourcc */ + .tag[0] = dcp_methods[method].tag[3], + .tag[1] = dcp_methods[method].tag[2], + .tag[2] = dcp_methods[method].tag[1], + .tag[3] = dcp_methods[method].tag[0], + }; + + u8 depth = dcp_push_depth(&ch->depth); + u16 offset = dcp_packet_start(ch, depth); + + void *out = dcp->shmem + dcp_tx_offset(context) + offset; + void *out_data = out + sizeof(header); + size_t data_len = sizeof(header) + in_len + out_len; + + memcpy(out, &header, sizeof(header)); + + if (in_len > 0) + memcpy(out_data, data, in_len); + + dev_dbg(dcp->dev, "---> %s: context %u, offset %u, depth %u\n", + dcp_methods[method].name, context, offset, depth); + + ch->callbacks[depth] = cb; + ch->cookies[depth] = cookie; + ch->output[depth] = out + sizeof(header) + in_len; + ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); + + apple_rtkit_send_message(dcp->rtk, IOMFB_ENDPOINT, + dcpep_msg(context, data_len, offset), NULL, + false); +} + +#define DCP_THUNK_VOID(func, handle) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, handle, 0, 0, NULL, cb, cookie); \ + } + +#define DCP_THUNK_OUT(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, handle, 0, sizeof(T), NULL, cb, cookie); \ + } + +#define DCP_THUNK_IN(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, T *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, handle, sizeof(T), 0, data, cb, cookie); \ + } + +#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ + static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, handle, sizeof(T_in), sizeof(T_out), data, \ + cb, cookie); \ + } + +DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, + struct dcp_swap_submit_resp); + +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, + struct dcp_swap_start_resp); + +DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, + struct dcp_set_power_state_req, + struct dcp_set_power_state_resp); + +DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, + struct dcp_set_digital_out_mode_req, u32); + +DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); + +DCP_THUNK_OUT(dcp_set_display_refresh_properties, + dcpep_set_display_refresh_properties, u32); + +DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); +DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); +DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); +DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); +DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); + +__attribute__((unused)) +DCP_THUNK_IN(dcp_update_notify_clients_dcp, dcpep_update_notify_clients_dcp, + struct dcp_update_notify_clients_dcp); + +DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, + struct dcp_set_parameter_dcp, u32); + +DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, + dcpep_enable_disable_video_power_savings, u32, int); + +DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +static int dcp_parse_tag(char tag[4]) +{ + u32 d[3]; + int i; + + if (tag[3] != 'D') + return -EINVAL; + + for (i = 0; i < 3; ++i) { + d[i] = (u32)(tag[i] - '0'); + + if (d[i] > 9) + return -EINVAL; + } + + return d[0] + (d[1] * 10) + (d[2] * 100); +} + +/* Ack a callback from the DCP */ +static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) +{ + struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); + + dcp_pop_depth(&ch->depth); + apple_rtkit_send_message(dcp->rtk, IOMFB_ENDPOINT, dcpep_ack(context), + NULL, false); +} + +/* DCP callback handlers */ +static void dcpep_cb_nop(struct apple_dcp *dcp) +{ + /* No operation */ +} + +static u8 dcpep_cb_true(struct apple_dcp *dcp) +{ + return true; +} + +static u8 dcpep_cb_false(struct apple_dcp *dcp) +{ + return false; +} + +static u32 dcpep_cb_zero(struct apple_dcp *dcp) +{ + return 0; +} + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct dc_swap_complete_resp *resp) +{ + dev_dbg(dcp->dev, "swap complete for swap_id: %u vblank: %u", + resp->swap_id, dcp->ignore_swap_complete); + + if (!dcp->ignore_swap_complete) + dcp_drm_crtc_vblank(dcp->crtc); +} + +static struct dcp_get_uint_prop_resp +dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) +{ + /* unimplemented for now */ + return (struct dcp_get_uint_prop_resp){ .value = 0 }; +} + +/* + * Callback to map a buffer allocated with allocate_buf for PIODMA usage. + * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated + * stream of the display DART, rather than the expected DCP DART. + * + * XXX: This relies on dma_get_sgtable in concert with dma_map_sgtable, which + * is a "fundamentally unsafe" operation according to the docs. And yet + * everyone does it... + */ +static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct sg_table *map; + int ret; + + if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) + goto reject; + + map = &dcp->memdesc[req->buffer].map; + + if (!map->sgl) + goto reject; + + /* Use PIODMA device instead of DCP to map against the right IOMMU. */ + ret = dma_map_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); + + if (ret) + goto reject; + + return (struct dcp_map_buf_resp){ .dva = sg_dma_address(map->sgl) }; + +reject: + dev_err(dcp->dev, "denying map of invalid buffer %llx for pidoma\n", + req->buffer); + return (struct dcp_map_buf_resp){ .ret = EINVAL }; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *resp) +{ + struct sg_table *map; + dma_addr_t dma_addr; + + if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu", + resp->buffer); + return; + } + + map = &dcp->memdesc[resp->buffer].map; + + if (!map->sgl) { + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx", + resp->buffer, resp->dva); + return; + } + + dma_addr = sg_dma_address(map->sgl); + if (dma_addr != resp->dva) { + dev_warn(dcp->dev, "unmap buffer %llu address mismatch dma_addr:%llx dva:%llx", + resp->buffer, dma_addr, resp->dva); + return; + } + + /* Use PIODMA device instead of DCP to unmap from the right IOMMU. */ + dma_unmap_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); +} + +/* + * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be + * physically contigiuous, however we should save the sgtable in case the + * buffer needs to be later mapped for PIODMA. + */ +static struct dcp_allocate_buffer_resp +dcpep_cb_allocate_buffer(struct apple_dcp *dcp, + struct dcp_allocate_buffer_req *req) +{ + struct dcp_allocate_buffer_resp resp = { 0 }; + struct dcp_mem_descriptor *memdesc; + u32 id; + + resp.dva_size = ALIGN(req->size, 4096); + resp.mem_desc_id = + find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + + if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring"); + resp.dva_size = 0; + resp.mem_desc_id = 0; + return resp; + } + id = resp.mem_desc_id; + set_bit(id, dcp->memdesc_map); + + memdesc = &dcp->memdesc[id]; + + memdesc->size = resp.dva_size; + memdesc->buf = dma_alloc_coherent(dcp->dev, memdesc->size, + &memdesc->dva, GFP_KERNEL); + + dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, + memdesc->size); + resp.dva = memdesc->dva; + + return resp; +} + +static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) +{ + struct dcp_mem_descriptor *memdesc; + u32 id = *mem_desc_id; + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, + "unmap request for out of range mem_desc_id %u", id); + return 0; + } + + if (!test_and_clear_bit(id, dcp->memdesc_map)) { + dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u", + id); + return 0; + } + + memdesc = &dcp->memdesc[id]; + if (memdesc->buf) { + dma_free_coherent(dcp->dev, memdesc->size, memdesc->buf, + memdesc->dva); + + memdesc->buf = NULL; + memset(&memdesc->map, 0, sizeof(memdesc->map)); + } else { + memdesc->reg = 0; + } + + memdesc->size = 0; + + return 1; +} + +/* Validate that the specified region is a display register */ +static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +{ + int i; + + for (i = 0; i < dcp->nr_disp_registers; ++i) { + struct resource *r = dcp->disp_registers[i]; + + if ((start >= r->start) && (end <= r->end)) + return true; + } + + return false; +} + +/* + * Map contiguous physical memory into the DCP's address space. The firmware + * uses this to map the display registers we advertise in + * sr_map_device_memory_with_index, so we bounds check against that to guard + * safe against malicious coprocessors. + */ +static struct dcp_map_physical_resp +dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) +{ + int size = ALIGN(req->size, 4096); + u32 id; + + if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + dev_err(dcp->dev, "refusing to map phys address %llx size %llx", + req->paddr, req->size); + return (struct dcp_map_physical_resp){}; + } + + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + set_bit(id, dcp->memdesc_map); + dcp->memdesc[id].size = size; + dcp->memdesc[id].reg = req->paddr; + + return (struct dcp_map_physical_resp){ + .dva_size = size, + .mem_desc_id = id, + .dva = dma_map_resource(dcp->dev, req->paddr, size, + DMA_BIDIRECTIONAL, 0), + }; +} + +static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) +{ + return clk_get_rate(dcp->clk); +} + +static struct dcp_map_reg_resp dcpep_cb_map_reg(struct apple_dcp *dcp, + struct dcp_map_reg_req *req) +{ + if (req->index >= dcp->nr_disp_registers) { + dev_warn(dcp->dev, "attempted to read invalid reg index %u", + req->index); + + return (struct dcp_map_reg_resp){ .ret = 1 }; + } else { + struct resource *rsrc = dcp->disp_registers[req->index]; + + return (struct dcp_map_reg_resp){ + .addr = rsrc->start, .length = resource_size(rsrc) + }; + } +} + +static struct dcp_read_edt_data_resp +dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) +{ + return (struct dcp_read_edt_data_resp){ + .value[0] = req->value[0], + .ret = 0, + }; +} + +/* Chunked data transfer for property dictionaries */ +static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) +{ + if (dcp->chunks.data != NULL) { + dev_warn(dcp->dev, "ignoring spurious transfer start\n"); + return false; + } + + dcp->chunks.length = *length; + dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "failed to allocate chunks\n"); + return false; + } + + return true; +} + +static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_chunk_req *req) +{ + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious chunk\n"); + return false; + } + + if (req->offset + req->length > dcp->chunks.length) { + dev_warn(dcp->dev, "ignoring overflowing chunk\n"); + return false; + } + + memcpy(dcp->chunks.data + req->offset, req->data, req->length); + return true; +} + +static bool dcpep_process_chunks(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + struct dcp_parse_ctx ctx; + int ret; + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious end\n"); + return false; + } + + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); + + if (ret) { + dev_warn(dcp->dev, "bad header on dcpav props\n"); + return false; + } + + if (!strcmp(req->key, "TimingElements")) { + dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, + dcp->width_mm, dcp->height_mm); + + if (IS_ERR(dcp->modes)) { + dev_warn(dcp->dev, "failed to parse modes\n"); + dcp->modes = NULL; + dcp->nr_modes = 0; + return false; + } + } else if (!strcmp(req->key, "DisplayAttributes")) { + ret = parse_display_attributes(&ctx, &dcp->width_mm, + &dcp->height_mm); + + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } + + dcp_set_dimensions(dcp); + } + + return true; +} + +static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + u8 resp = dcpep_process_chunks(dcp, req); + + /* Reset for the next transfer */ + devm_kfree(dcp->dev, dcp->chunks.data); + dcp->chunks.data = NULL; + + return resp; +} + +/* Boot sequence */ +static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_cb_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + dev_dbg(dcp->dev, "boot done"); + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); +} + +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_late_init_signal(dcp, false, boot_5, NULL); +} + +static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 v_true = true; + + dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); +} + +static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_setup_video_limits(dcp, false, boot_3, NULL); +} + +static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_create_default_fb(dcp, false, boot_2, NULL); +} + +/* Use special function signature to defer the ACK */ +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, __func__); + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); + return false; +} + +static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) +{ + if (dcp->disp_registers[5] && dcp->disp_registers[6]) + return (struct dcp_rt_bandwidth){ + .reg_scratch = + dcp->disp_registers[5]->start + REG_SCRATCH, + .reg_doorbell = + dcp->disp_registers[6]->start + REG_DOORBELL, + .doorbell_bit = REG_DOORBELL_BIT, + + .padding[3] = 0x4, // XXX: required by 11.x firmware + }; + else if (dcp->disp_registers[4]) + return (struct dcp_rt_bandwidth){ + .reg_scratch = dcp->disp_registers[4]->start + + REG_SCRATCH_T600X, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; + else + return (struct dcp_rt_bandwidth){ + .reg_scratch = 0, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; +} + +/* Callback to get the current time as milliseconds since the UNIX epoch */ +static u64 dcpep_cb_get_time(struct apple_dcp *dcp) +{ + return ktime_to_ms(ktime_get_real()); +} + +struct dcp_swap_cookie { + struct completion done; + atomic_t refcount; + u32 swap_id; +}; + +static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_submit_resp *resp = data; + dev_dbg(dcp->dev, "%s", __func__); + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + complete(&info->done); + if (atomic_dec_and_test(&info->refcount)) + kfree(info); + } + + if (resp->ret) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + dev_dbg(dcp->dev, "%s swap_id: %u", __func__, resp->swap_id); + dcp->swap.swap.swap_id = resp->swap_id; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + info->swap_id = resp->swap_id; + } + + dcp_swap_submit(dcp, false, &dcp->swap, dcp_swap_cleared, cookie); +} + +static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + dev_dbg(dcp->dev, "%s", __func__); + + if (wait) { + complete(&wait->done); + if (atomic_dec_and_test(&wait->refcount)) + kfree(wait); + } +} + +static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = 14, + .value = { 0 }, + .count = 1, + }; + dev_dbg(dcp->dev, "%s", __func__); + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_final, cookie); +} + +void dcp_poweron(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct dcp_wait_cookie *cookie; + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + int ret; + u32 handle; + dev_dbg(dcp->dev, "%s", __func__); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + + init_completion(&cookie->done); + atomic_set(&cookie->refcount, 2); + + if (dcp->main_display) { + handle = 0; + dcp_set_display_device(dcp, false, &handle, dcp_on_final, + cookie); + } else { + handle = 2; + dcp_set_display_device(dcp, false, &handle, + dcp_on_set_parameter, cookie); + } + dcp_set_power_state(dcp, true, &req, NULL, NULL); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); + + if (ret == 0) + dev_warn(dcp->dev, "wait for power timed out"); + + if (atomic_dec_and_test(&cookie->refcount)) + kfree(cookie); +} +EXPORT_SYMBOL(dcp_poweron); + +static void complete_set_powerstate(struct apple_dcp *dcp, void *out, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + if (atomic_dec_and_test(&wait->refcount)) + kfree(wait); + } +} + +void dcp_poweroff(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret, swap_id; + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + struct dcp_swap_cookie *cookie; + struct dcp_wait_cookie *poff_cookie; + struct dcp_swap_start_req swap_req = { 0 }; + + dev_dbg(dcp->dev, "%s", __func__); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + atomic_set(&cookie->refcount, 2); + + // clear surfaces + memset(&dcp->swap, 0, sizeof(dcp->swap)); + + dcp->swap.swap.swap_enabled = DCP_REMOVE_LAYERS | 0x7; + dcp->swap.swap.swap_completed = DCP_REMOVE_LAYERS | 0x7; + dcp->swap.swap.unk_10c = 0xFF000000; + + for (int l = 0; l < SWAP_SURFACES; l++) + dcp->swap.surf_null[l] = true; + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + swap_id = cookie->swap_id; + if (atomic_dec_and_test(&cookie->refcount)) + kfree(cookie); + if (ret <= 0) { + dcp->crashed = true; + return; + } + + dev_dbg(dcp->dev, "%s: clear swap submitted: %u", __func__, swap_id); + + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); + if (!poff_cookie) + return; + init_completion(&poff_cookie->done); + atomic_set(&poff_cookie->refcount, 2); + + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + poff_cookie); + ret = wait_for_completion_timeout(&poff_cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setPowerState(0) timeout %u ms", 1000); + else if (ret > 0) + dev_dbg(dcp->dev, + "setPowerState(0) finished with %d ms to spare", + jiffies_to_msecs(ret)); + + if (atomic_dec_and_test(&poff_cookie->refcount)) + kfree(poff_cookie); + dev_dbg(dcp->dev, "%s: setPowerState(0) done", __func__); +} +EXPORT_SYMBOL(dcp_poweroff); + +/* + * Helper to send a DRM hotplug event. The DCP is accessed from a single + * (RTKit) thread. To handle hotplug callbacks, we need to call + * drm_kms_helper_hotplug_event, which does an atomic commit (via DCP) and + * waits for vblank (a DCP callback). That means we deadlock if we call from + * the RTKit thread! Instead, move the call to another thread via a workqueue. + */ +void dcp_hotplug(struct work_struct *work) +{ + struct apple_connector *connector; + struct drm_device *dev; + struct apple_dcp *dcp; + + connector = container_of(work, struct apple_connector, hotplug_wq); + dev = connector->base.dev; + + dcp = platform_get_drvdata(connector->dcp); + dev_info(dcp->dev, "%s: connected: %d", __func__, connector->connected); + + /* + * DCP defers link training until we set a display mode. But we set + * display modes from atomic_flush, so userspace needs to trigger a + * flush, or the CRTC gets no signal. + */ + if (!dcp->valid_mode && connector->connected) { + drm_connector_set_link_status_property( + &connector->base, DRM_MODE_LINK_STATUS_BAD); + } + + if (dev && dev->registered) + drm_kms_helper_hotplug_event(dev); +} +EXPORT_SYMBOL_GPL(dcp_hotplug); + +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + struct apple_connector *connector = dcp->connector; + + /* Hotplug invalidates mode. DRM doesn't always handle this. */ + if (!(*connected)) { + dcp->valid_mode = false; + /* after unplug swap will not complete until the next + * set_digital_out_mode */ + schedule_work(&dcp->vblank_wq); + } + + if (connector && connector->connected != !!(*connected)) { + connector->connected = !!(*connected); + dcp->valid_mode = false; + schedule_work(&connector->hotplug_wq); + } +} + +static void +dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, + struct dcp_swap_complete_intent_gated *info) +{ + dev_dbg(dcp->dev, "swap_id:%u width:%u height:%u", info->swap_id, + info->width, info->height); +} + +#define DCPEP_MAX_CB (1000) + +/* + * Define type-safe trampolines. Define typedefs to enforce type-safety on the + * input data (so if the types don't match, gcc errors out). + */ + +#define TRAMPOLINE_VOID(func, handler) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + handler(dcp); \ + return true; \ + } + +#define TRAMPOLINE_IN(func, handler, T_in) \ + typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + callback_##handler cb = handler; \ + \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ + typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + callback_##handler cb = handler; \ + \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + *typed_out = cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_OUT(func, handler, T_out) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + \ + dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + *typed_out = handler(dcp); \ + return true; \ + } + +TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); +TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); +TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); +TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); +TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, + struct dc_swap_complete_resp); +TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, + struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, + struct dcp_map_buf_req, struct dcp_map_buf_resp); +TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, + struct dcp_unmap_buf_resp); +TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, + struct dcp_allocate_buffer_req, + struct dcp_allocate_buffer_resp); +TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, + struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, + u8); +TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct dcp_map_reg_req, + struct dcp_map_reg_resp); +TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, + struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); +TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); +TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, + struct dcp_set_dcpav_prop_chunk_req, u8); +TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, + struct dcp_set_dcpav_prop_end_req, u8); +TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, + struct dcp_rt_bandwidth); +TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); +TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, + dcpep_cb_swap_complete_intent_gated, + struct dcp_swap_complete_intent_gated); + +bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, + void *) = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [100] = trampoline_nop, /* match_pmu_service */ + [101] = trampoline_zero, /* get_display_default_stride */ + [103] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_false, /* create_backlight_service */ + [116] = dcpep_cb_boot_1, + [117] = trampoline_false, /* is_dark_boot */ + [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [120] = trampoline_read_edt_data, + [122] = trampoline_prop_start, + [123] = trampoline_prop_chunk, + [124] = trampoline_prop_end, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = trampoline_true, /* match_pmu_service_2 */ + [207] = trampoline_true, /* match_backlight_service */ + [208] = trampoline_get_time, + [211] = trampoline_nop, /* update_backlight_factor_prop */ + [300] = trampoline_nop, /* pr_publish */ + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_nop, /* set_fx_prop */ + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_true, /* sr_set_property_int */ + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [456] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [593] = trampoline_nop, /* enable_backlight_message_ap_gated */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; + +static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct device *dev = dcp->dev; + struct dcp_packet_header *hdr = data; + void *in, *out; + int tag = dcp_parse_tag(hdr->tag); + struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); + u8 depth; + + if (tag < 0 || tag >= DCPEP_MAX_CB || !dcpep_cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + return; + } + + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + // TODO: verify that in_len and out_len match our prototypes + // for now just clear the out data to have at least consistant results + if (hdr->out_len) + memset(out, 0, hdr->out_len); + + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + + if (dcpep_cb_handlers[tag](dcp, tag, out, in)) + dcp_ack(dcp, context); +} + +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct dcp_packet_header *header = data; + struct dcp_call_channel *ch = dcp_get_call_channel(dcp, context); + void *cookie; + dcp_callback_t cb; + + if (!ch) { + dev_warn(dcp->dev, "ignoring ack on context %X\n", context); + return; + } + + dcp_pop_depth(&ch->depth); + + cb = ch->callbacks[ch->depth]; + cookie = ch->cookies[ch->depth]; + + ch->callbacks[ch->depth] = NULL; + ch->cookies[ch->depth] = NULL; + + if (cb) + cb(dcp, data + sizeof(*header) + header->in_len, cookie); +} + +static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcp_context_id ctx_id; + u16 offset; + u32 length; + int channel_offset; + void *data; + + ctx_id = (message & DCPEP_CONTEXT_MASK) >> DCPEP_CONTEXT_SHIFT; + offset = (message & DCPEP_OFFSET_MASK) >> DCPEP_OFFSET_SHIFT; + length = (message >> DCPEP_LENGTH_SHIFT); + + channel_offset = dcp_channel_offset(ctx_id); + + if (channel_offset < 0) { + dev_warn(dcp->dev, "invalid context received %u", ctx_id); + return; + } + + data = dcp->shmem + channel_offset + offset; + + if (message & DCPEP_ACK) + dcpep_handle_ack(dcp, ctx_id, data, length); + else + dcpep_handle_cb(dcp, ctx_id, data, length); +} + +/* + * Callback for swap requests. If a swap failed, we'll never get a swap + * complete event so we need to fake a vblank event early to avoid a hang. + */ + +static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_submit_resp *resp = data; + + if (resp->ret) { + dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + + dcp->swap.swap.swap_id = resp->swap_id; + + dcp_swap_submit(dcp, false, &dcp->swap, dcp_swapped, NULL); +} + +/* + * DRM specifies rectangles as start and end coordinates. DCP specifies + * rectangles as a start coordinate and a width/height. Convert a DRM rectangle + * to a DCP rectangle. + */ +static struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect) +{ + return (struct dcp_rect){ .x = rect->x1, + .y = rect->y1, + .w = drm_rect_width(rect), + .h = drm_rect_height(rect) }; +} + +static u32 drm_format_to_dcp(u32 drm) +{ + switch (drm) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + return fourcc_code('A', 'R', 'G', 'B'); + + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ABGR8888: + return fourcc_code('A', 'B', 'G', 'R'); + } + + pr_warn("DRM format %X not supported in DCP\n", drm); + return 0; +} + +int dcp_get_modes(struct drm_connector *connector) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode; + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); + + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + return 0; + } + + drm_mode_probed_add(connector, mode); + } + + return dcp->nr_modes; +} +EXPORT_SYMBOL_GPL(dcp_get_modes); + +/* The user may own drm_display_mode, so we need to search for our copy */ +static struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + struct drm_display_mode *mode) +{ + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + if (drm_mode_match(mode, &dcp->modes[i].mode, + DRM_MODE_MATCH_TIMINGS | + DRM_MODE_MATCH_CLOCK)) + return &dcp->modes[i]; + } + + return NULL; +} + +int dcp_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; +} +EXPORT_SYMBOL_GPL(dcp_mode_valid); + +/* Helpers to modeset and swap, used to flush */ +static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_req start_req = { 0 }; + dev_dbg(dcp->dev, "%s", __func__); + + if (dcp->connector && dcp->connector->connected) + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + else + dcp_drm_crtc_vblank(dcp->crtc); +} + +static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + dev_dbg(dcp->dev, "%s", __func__); + + dcp->ignore_swap_complete = false; + + if (wait) { + complete(&wait->done); + if (atomic_dec_and_test(&wait->refcount)) + kfree(wait); + } +} + +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_plane *plane; + struct drm_plane_state *new_state, *old_state; + struct drm_crtc_state *crtc_state; + struct dcp_swap_submit_req *req = &dcp->swap; + int l; + int has_surface = 0; + bool modeset; + dev_dbg(dcp->dev, "%s", __func__); + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + + if (WARN(dcp_channel_busy(&dcp->ch_cmd), "unexpected busy channel") || + WARN(!modeset && !dcp->connector->connected, + "can't flush if disconnected")) { + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). + */ + schedule_work(&dcp->vblank_wq); + return; + } + + /* Reset to defaults */ + memset(req, 0, sizeof(*req)); + for (l = 0; l < SWAP_SURFACES; l++) + req->surf_null[l] = true; + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, l) { + struct drm_framebuffer *fb = new_state->fb; + struct drm_rect src_rect; + + WARN_ON(l >= SWAP_SURFACES); + + req->swap.swap_enabled |= BIT(l); + + if (old_state->fb && fb != old_state->fb) { + /* + * Race condition between a framebuffer unbind getting + * swapped out and GEM unreferencing a framebuffer. If + * we lose the race, the display gets IOVA faults and + * the DCP crashes. We need to extend the lifetime of + * the drm_framebuffer (and hence the GEM object) until + * after we get a swap complete for the swap unbinding + * it. + */ + struct dcp_fb_reference *entry = + kzalloc(sizeof(*entry), GFP_KERNEL); + if (entry) { + entry->fb = old_state->fb; + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + } + drm_framebuffer_get(old_state->fb); + } + + if (!new_state->fb) { + if (old_state->fb) + req->swap.swap_enabled |= DCP_REMOVE_LAYERS; + + continue; + } + req->surf_null[l] = false; + has_surface = 1; + + drm_rect_fp_to_int(&src_rect, &new_state->src); + + req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); + req->swap.dst_rect[l] = drm_to_dcp_rect(&new_state->dst); + + req->surf_iova[l] = drm_fb_dma_get_gem_addr(fb, new_state, 0); + + req->surf[l] = (struct dcp_surface){ + .format = drm_format_to_dcp(fb->format->format), + .xfer_func = 13, + .colorspace = 1, + .stride = fb->pitches[0], + .width = fb->width, + .height = fb->height, + .buf_size = fb->height * fb->pitches[0], + .surface_id = req->swap.surf_ids[l], + + /* Only used for compressed or multiplanar surfaces */ + .pix_size = 1, + .pel_w = 1, + .pel_h = 1, + .has_comp = 1, + .has_planes = 1, + }; + } + + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + + if (modeset) { + struct dcp_display_mode *mode; + struct dcp_wait_cookie *cookie; + int ret; + + mode = lookup_mode(dcp, &crtc_state->mode); + if (!mode) { + dev_warn(dcp->dev, "no match for " DRM_MODE_FMT, + DRM_MODE_ARG(&crtc_state->mode)); + schedule_work(&dcp->vblank_wq); + return; + } + + dev_info(dcp->dev, "set_digital_out_mode(color:%d timing:%d)", + mode->color_mode_id, mode->timing_mode_id); + dcp->mode = (struct dcp_set_digital_out_mode_req){ + .color_mode_id = mode->color_mode_id, + .timing_mode_id = mode->timing_mode_id + }; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) { + schedule_work(&dcp->vblank_wq); + return; + } + + init_completion(&cookie->done); + atomic_set(&cookie->refcount, 2); + + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); + + dev_dbg(dcp->dev, "%s - wait for modeset", __func__); + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(500)); + + if (atomic_dec_and_test(&cookie->refcount)) + kfree(cookie); + + if (ret == 0) { + dev_dbg(dcp->dev, "set_digital_out_mode 200 ms"); + schedule_work(&dcp->vblank_wq); + return; + } else if (ret > 0) { + dev_dbg(dcp->dev, + "set_digital_out_mode finished with %d to spare", + jiffies_to_msecs(ret)); + } + + dcp->valid_mode = true; + } + + if (!has_surface) { + if (crtc_state->enable && crtc_state->active && + !crtc_state->planes_changed) { + schedule_work(&dcp->vblank_wq); + return; + } + + req->clear = 1; + } + do_swap(dcp, NULL, NULL); +} +EXPORT_SYMBOL_GPL(dcp_flush); + +bool dcp_is_initialized(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->active; +} +EXPORT_SYMBOL_GPL(dcp_is_initialized); + +static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct apple_connector *connector; + int result = *(int *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); + + dcp->main_display = result != 0; + + dcp->active = true; + + connector = dcp->connector; + if (connector) { + connector->connected = dcp->nr_modes > 0; + schedule_work(&connector->hotplug_wq); + } +} + +static void init_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_is_main_display(dcp, false, res_is_main_display, NULL); +} + +static void init_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_first_client_open(dcp, false, init_3, NULL); +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 val = 0; + dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); +} + +static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + dev_info(dcp->dev, "DCP booted\n"); + + init_1(dcp, data, cookie); +} + +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcpep_type type = (message >> DCPEP_TYPE_SHIFT) & DCPEP_TYPE_MASK; + + if (type == DCPEP_TYPE_INITIALIZED) + dcp_start_signal(dcp, false, dcp_started, NULL); + else if (type == DCPEP_TYPE_MESSAGE) + dcpep_got_msg(dcp, message); + else + dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); +} + +int iomfb_start_rtkit(struct apple_dcp *dcp) +{ + dma_addr_t shmem_iova; + apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + + dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, + GFP_KERNEL); + + shmem_iova |= dcp->asc_dram_mask; + apple_rtkit_send_message(dcp->rtk, IOMFB_ENDPOINT, + dcpep_set_shmem(shmem_iova), NULL, false); + + return 0; +} + +void iomfb_shutdown(struct apple_dcp *dcp) +{ + struct dcp_set_power_state_req req = { + /* defaults are ok */ + }; + + /* We're going down */ + dcp->active = false; + dcp->valid_mode = false; + + dcp_set_power_state(dcp, false, &req, NULL, NULL); +} diff --git a/drivers/gpu/drm/apple/dcpep.h b/drivers/gpu/drm/apple/iomfb.h similarity index 86% rename from drivers/gpu/drm/apple/dcpep.h rename to drivers/gpu/drm/apple/iomfb.h index a796b16bd55ad7..96dfd170b8e330 100644 --- a/drivers/gpu/drm/apple/dcpep.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -4,8 +4,10 @@ #ifndef __APPLE_DCPEP_H__ #define __APPLE_DCPEP_H__ +#include + /* Endpoint for general DCP traffic (dcpep in macOS) */ -#define DCP_ENDPOINT 0x37 +#define IOMFB_ENDPOINT 0x37 /* Fixed size of shared memory between DCP and AP */ #define DCP_SHMEM_SIZE 0x100000 @@ -30,27 +32,6 @@ enum dcp_context_id { DCP_NUM_CONTEXTS }; -static int dcp_tx_offset(enum dcp_context_id id) -{ - switch (id) { - case DCP_CONTEXT_CB: - case DCP_CONTEXT_CMD: return 0x00000; - case DCP_CONTEXT_OOBCB: - case DCP_CONTEXT_OOBCMD: return 0x08000; - default: return -EINVAL; - } -} - -static int dcp_channel_offset(enum dcp_context_id id) -{ - switch (id) { - case DCP_CONTEXT_ASYNC: return 0x40000; - case DCP_CONTEXT_CB: return 0x60000; - case DCP_CONTEXT_OOBCB: return 0x68000; - default: return dcp_tx_offset(id); - } -} - /* RTKit endpoint message types */ enum dcpep_type { /* Set shared memory */ @@ -87,29 +68,6 @@ struct dcp_packet_header { #define DCP_IS_NULL(ptr) ((ptr) ? 1 : 0) #define DCP_PACKET_ALIGNMENT (0x40) -static inline u64 -dcpep_set_shmem(u64 dart_va) -{ - return (DCPEP_TYPE_SET_SHMEM << DCPEP_TYPE_SHIFT) | - (DCPEP_FLAG_VALUE << DCPEP_FLAG_SHIFT) | - (dart_va << DCPEP_DVA_SHIFT); -} - -static inline u64 -dcpep_msg(enum dcp_context_id id, u32 length, u16 offset) -{ - return (DCPEP_TYPE_MESSAGE << DCPEP_TYPE_SHIFT) | - ((u64) id << DCPEP_CONTEXT_SHIFT) | - ((u64) offset << DCPEP_OFFSET_SHIFT) | - ((u64) length << DCPEP_LENGTH_SHIFT); -} - -static inline u64 -dcpep_ack(enum dcp_context_id id) -{ - return dcpep_msg(id, 0, 0) | DCPEP_ACK; -} - /* Structures used in v12.0 firmware */ #define SWAP_SURFACES 4 From 829753717dce465b1f1c0f8e66b643eb58a5b60a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 3 Oct 2022 10:43:02 +0200 Subject: [PATCH 206/635] WIP: add header test target copied from i915 Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/.gitignore | 1 + drivers/gpu/drm/apple/Makefile | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 drivers/gpu/drm/apple/.gitignore diff --git a/drivers/gpu/drm/apple/.gitignore b/drivers/gpu/drm/apple/.gitignore new file mode 100644 index 00000000000000..d9a77f3b59b21a --- /dev/null +++ b/drivers/gpu/drm/apple/.gitignore @@ -0,0 +1 @@ +*.hdrtest diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index d1f909792229e5..288c89739af196 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -7,3 +7,18 @@ apple_piodma-y := dummy-piodma.o obj-$(CONFIG_DRM_APPLE) += appledrm.o obj-$(CONFIG_DRM_APPLE) += apple_dcp.o obj-$(CONFIG_DRM_APPLE) += apple_piodma.o + +# header test + +# exclude some broken headers from the test coverage +no-header-test := \ + +always-y += \ + $(patsubst %.h,%.hdrtest, $(filter-out $(no-header-test), \ + $(shell cd $(srctree)/$(src) && find * -name '*.h'))) + +quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@ + +$(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) From a1497acbfbd1965bc916598e287aa5a98db84020 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 22 Oct 2022 09:27:51 +0200 Subject: [PATCH 207/635] gpu: drm: apple: Use connector types from devicetree Needs to be re-done for upstream submission but the exisiting port, bridge, connector and display bindings are a bad fit since DCP manages everything. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index f576cefd8c3353..fa966dcc7e9447 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -298,6 +299,7 @@ static int apple_probe_per_dcp(struct device *dev, struct apple_connector *connector; struct drm_encoder *encoder; struct drm_plane *primary; + int con_type; int ret; primary = apple_plane_init(drm, DRM_PLANE_TYPE_PRIMARY); @@ -323,8 +325,17 @@ static int apple_probe_per_dcp(struct device *dev, drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); + if (of_property_match_string(dcp->dev.of_node, "apple,connector-type", "eDP") >= 0) + con_type = DRM_MODE_CONNECTOR_eDP; + else if (of_property_match_string(dcp->dev.of_node, "apple,connector-type", "HDMI-A") >= 0) + con_type = DRM_MODE_CONNECTOR_HDMIA; + else if (of_property_match_string(dcp->dev.of_node, "apple,connector-type", "USB-C") >= 0) + con_type = DRM_MODE_CONNECTOR_USB; + else + con_type = DRM_MODE_CONNECTOR_Unknown; + ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, - DRM_MODE_CONNECTOR_HDMIA); + con_type); if (ret) return ret; From 3a81d4a5e3841a3f8637c1e20361d8987fe2dfe0 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 5 Oct 2022 22:30:58 +0200 Subject: [PATCH 208/635] drm: apple: Fix connector state on devices with integrated display DCP issues hotplug_gated callbacks after SetPowerState() calls on devices with display (macbooks, imacs). This must not result in connector state changes on DRM side. Weston will not re-enable the CRTC after DPMS off if the connector is not in connected state. DCP provides with dcp_is_main_display() a call to query if the device has an integrated display. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 0da3de1aa27e78..3c930209382685 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -983,6 +983,16 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) { struct apple_connector *connector = dcp->connector; + /* DCP issues hotplug_gated callbacks after SetPowerState() calls on + * devices with display (macbooks, imacs). This must not result in + * connector state changes on DRM side. Some applications won't enable + * a CRTC with a connector in disconnected state. Weston after DPMS off + * is one example. dcp_is_main_display() returns true on devices with + * integrated display. Ignore the hotplug_gated() callbacks there. + */ + if (dcp->main_display) + return; + /* Hotplug invalidates mode. DRM doesn't always handle this. */ if (!(*connected)) { dcp->valid_mode = false; From 6b70879f1b53ee86ceed34b306499510a8040d88 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 5 Oct 2022 22:59:54 +0200 Subject: [PATCH 209/635] drm: apple: Replace atomic refcount with kref fixup! drm/apple: Add t600x support Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 61 ++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3c930209382685..68ca1c2aa8354e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -32,10 +33,18 @@ #define REG_DOORBELL_BIT (2) struct dcp_wait_cookie { + struct kref refcount; struct completion done; - atomic_t refcount; }; +static void release_wait_cookie(struct kref *ref) +{ + struct dcp_wait_cookie *cookie; + cookie = container_of(ref, struct dcp_wait_cookie, refcount); + + kfree(cookie); +} + static int dcp_tx_offset(enum dcp_context_id id) { switch (id) { @@ -755,11 +764,19 @@ static u64 dcpep_cb_get_time(struct apple_dcp *dcp) } struct dcp_swap_cookie { + struct kref refcount; struct completion done; - atomic_t refcount; u32 swap_id; }; +static void release_swap_cookie(struct kref *ref) +{ + struct dcp_swap_cookie *cookie; + cookie = container_of(ref, struct dcp_swap_cookie, refcount); + + kfree(cookie); +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_submit_resp *resp = data; @@ -768,8 +785,7 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) if (cookie) { struct dcp_swap_cookie *info = cookie; complete(&info->done); - if (atomic_dec_and_test(&info->refcount)) - kfree(info); + kref_put(&info->refcount, release_swap_cookie); } if (resp->ret) { @@ -811,8 +827,7 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) if (wait) { complete(&wait->done); - if (atomic_dec_and_test(&wait->refcount)) - kfree(wait); + kref_put(&wait->refcount, release_wait_cookie); } } @@ -844,7 +859,9 @@ void dcp_poweron(struct platform_device *pdev) return; init_completion(&cookie->done); - atomic_set(&cookie->refcount, 2); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); if (dcp->main_display) { handle = 0; @@ -862,8 +879,7 @@ void dcp_poweron(struct platform_device *pdev) if (ret == 0) dev_warn(dcp->dev, "wait for power timed out"); - if (atomic_dec_and_test(&cookie->refcount)) - kfree(cookie); + kref_put(&cookie->refcount, release_wait_cookie);; } EXPORT_SYMBOL(dcp_poweron); @@ -874,8 +890,7 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, if (wait) { complete(&wait->done); - if (atomic_dec_and_test(&wait->refcount)) - kfree(wait); + kref_put(&wait->refcount, release_wait_cookie); } } @@ -896,7 +911,9 @@ void dcp_poweroff(struct platform_device *pdev) if (!cookie) return; init_completion(&cookie->done); - atomic_set(&cookie->refcount, 2); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); // clear surfaces memset(&dcp->swap, 0, sizeof(dcp->swap)); @@ -912,8 +929,7 @@ void dcp_poweroff(struct platform_device *pdev) ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); swap_id = cookie->swap_id; - if (atomic_dec_and_test(&cookie->refcount)) - kfree(cookie); + kref_put(&cookie->refcount, release_swap_cookie); if (ret <= 0) { dcp->crashed = true; return; @@ -925,7 +941,9 @@ void dcp_poweroff(struct platform_device *pdev) if (!poff_cookie) return; init_completion(&poff_cookie->done); - atomic_set(&poff_cookie->refcount, 2); + kref_init(&poff_cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&poff_cookie->refcount); dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, poff_cookie); @@ -939,8 +957,7 @@ void dcp_poweroff(struct platform_device *pdev) "setPowerState(0) finished with %d ms to spare", jiffies_to_msecs(ret)); - if (atomic_dec_and_test(&poff_cookie->refcount)) - kfree(poff_cookie); + kref_put(&poff_cookie->refcount, release_wait_cookie); dev_dbg(dcp->dev, "%s: setPowerState(0) done", __func__); } EXPORT_SYMBOL(dcp_poweroff); @@ -1379,8 +1396,7 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, if (wait) { complete(&wait->done); - if (atomic_dec_and_test(&wait->refcount)) - kfree(wait); + kref_put(&wait->refcount, release_wait_cookie); } } @@ -1509,7 +1525,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) } init_completion(&cookie->done); - atomic_set(&cookie->refcount, 2); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); @@ -1518,8 +1536,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); - if (atomic_dec_and_test(&cookie->refcount)) - kfree(cookie); + kref_put(&cookie->refcount, release_wait_cookie); if (ret == 0) { dev_dbg(dcp->dev, "set_digital_out_mode 200 ms"); From 0926f5e3146a35b9e1a179f942077cbb80b74128 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 8 Oct 2022 18:30:41 +0200 Subject: [PATCH 210/635] gpu: drm: apple: Start using tracepoints Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/Makefile | 5 + drivers/gpu/drm/apple/dcp-internal.h | 11 ++ drivers/gpu/drm/apple/dcp.c | 10 ++ drivers/gpu/drm/apple/dcp.h | 2 +- drivers/gpu/drm/apple/iomfb.c | 32 +++--- drivers/gpu/drm/apple/iomfb.h | 3 - drivers/gpu/drm/apple/trace.c | 9 ++ drivers/gpu/drm/apple/trace.h | 166 +++++++++++++++++++++++++++ 8 files changed, 217 insertions(+), 21 deletions(-) create mode 100644 drivers/gpu/drm/apple/trace.c create mode 100644 drivers/gpu/drm/apple/trace.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 288c89739af196..2502f781a5dcef 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -1,7 +1,12 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT +CFLAGS_trace.o = -I$(src) + appledrm-y := apple_drv.o + apple_dcp-y := dcp.o iomfb.o parser.o +apple_dcp-$(CONFIG_TRACING) += trace.o + apple_piodma-y := dummy-piodma.o obj-$(CONFIG_DRM_APPLE) += appledrm.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 648d543b9cd91a..fd7c3aac603e35 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -12,6 +12,17 @@ struct apple_dcp; +enum { + SYSTEM_ENDPOINT = 0x20, + TEST_ENDPOINT = 0x21, + DCP_EXPERT_ENDPOINT = 0x22, + DISP0_ENDPOINT = 0x23, + DPTX_ENDPOINT = 0x2a, + HDCP_ENDPOINT = 0x2b, + REMOTE_ALLOC_ENDPOINT = 0x2d, + IOMFB_ENDPOINT = 0x37, +}; + /* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ struct dcp_chunks { size_t length; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0b3597507739c0..a23949ec73fae0 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -23,6 +23,7 @@ #include "dcp.h" #include "dcp-internal.h" #include "parser.h" +#include "trace.h" #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) @@ -91,6 +92,8 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; + trace_dcp_recv_msg(dcp, endpoint, message); + switch (endpoint) { case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); @@ -168,6 +171,13 @@ static struct apple_rtkit_ops rtkit_ops = { .shmem_destroy = dcp_rtk_shmem_destroy, }; +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) +{ + trace_dcp_send_msg(dcp, endpoint, message); + apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, + false); +} + void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, struct apple_connector *connector) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 18d71afaf6b72e..047c1b3a160457 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -47,7 +47,7 @@ int dcp_get_modes(struct drm_connector *connector); int dcp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode); void dcp_set_dimensions(struct apple_dcp *dcp); - +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int iomfb_start_rtkit(struct apple_dcp *dcp); void iomfb_shutdown(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 68ca1c2aa8354e..5bab8825df5a60 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -25,6 +25,7 @@ #include "dcp-internal.h" #include "iomfb.h" #include "parser.h" +#include "trace.h" /* Register defines used in bandwidth setup structure */ #define REG_SCRATCH (0x14) @@ -228,17 +229,15 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, if (in_len > 0) memcpy(out_data, data, in_len); - dev_dbg(dcp->dev, "---> %s: context %u, offset %u, depth %u\n", - dcp_methods[method].name, context, offset, depth); + trace_iomfb_push(dcp, &dcp_methods[method], context, offset, depth); ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); - apple_rtkit_send_message(dcp->rtk, IOMFB_ENDPOINT, - dcpep_msg(context, data_len, offset), NULL, - false); + dcp_send_message(dcp, IOMFB_ENDPOINT, + dcpep_msg(context, data_len, offset)); } #define DCP_THUNK_VOID(func, handle) \ @@ -333,8 +332,8 @@ static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); dcp_pop_depth(&ch->depth); - apple_rtkit_send_message(dcp->rtk, IOMFB_ENDPOINT, dcpep_ack(context), - NULL, false); + dcp_send_message(dcp, IOMFB_ENDPOINT, + dcpep_ack(context)); } /* DCP callback handlers */ @@ -361,8 +360,7 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct dc_swap_complete_resp *resp) { - dev_dbg(dcp->dev, "swap complete for swap_id: %u vblank: %u", - resp->swap_id, dcp->ignore_swap_complete); + trace_iomfb_swap_complete(dcp, resp->swap_id); if (!dcp->ignore_swap_complete) dcp_drm_crtc_vblank(dcp->crtc); @@ -725,7 +723,7 @@ static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) /* Use special function signature to defer the ACK */ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, __func__); + trace_iomfb_callback(dcp, tag, __func__); dcp_set_create_dfb(dcp, false, boot_1_5, NULL); return false; } @@ -1029,7 +1027,7 @@ static void dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, struct dcp_swap_complete_intent_gated *info) { - dev_dbg(dcp->dev, "swap_id:%u width:%u height:%u", info->swap_id, + trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, info->width, info->height); } @@ -1043,7 +1041,7 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, #define TRAMPOLINE_VOID(func, handler) \ static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + trace_iomfb_callback(dcp, tag, #handler); \ handler(dcp); \ return true; \ } @@ -1055,7 +1053,7 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, { \ callback_##handler cb = handler; \ \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + trace_iomfb_callback(dcp, tag, #handler); \ cb(dcp, in); \ return true; \ } @@ -1068,7 +1066,7 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, T_out *typed_out = out; \ callback_##handler cb = handler; \ \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + trace_iomfb_callback(dcp, tag, #handler); \ *typed_out = cb(dcp, in); \ return true; \ } @@ -1078,7 +1076,7 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, { \ T_out *typed_out = out; \ \ - dev_dbg(dcp->dev, "Callback D%03d %s\n", tag, #handler); \ + trace_iomfb_callback(dcp, tag, #handler); \ *typed_out = handler(dcp); \ return true; \ } @@ -1290,6 +1288,7 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp->swap.swap.swap_id = resp->swap_id; + trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_swap_submit(dcp, false, &dcp->swap, dcp_swapped, NULL); } @@ -1633,8 +1632,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) GFP_KERNEL); shmem_iova |= dcp->asc_dram_mask; - apple_rtkit_send_message(dcp->rtk, IOMFB_ENDPOINT, - dcpep_set_shmem(shmem_iova), NULL, false); + dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 96dfd170b8e330..5b1f4ba789bccf 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -6,9 +6,6 @@ #include -/* Endpoint for general DCP traffic (dcpep in macOS) */ -#define IOMFB_ENDPOINT 0x37 - /* Fixed size of shared memory between DCP and AP */ #define DCP_SHMEM_SIZE 0x100000 diff --git a/drivers/gpu/drm/apple/trace.c b/drivers/gpu/drm/apple/trace.c new file mode 100644 index 00000000000000..6f40d5a583df01 --- /dev/null +++ b/drivers/gpu/drm/apple/trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Tracepoints for Apple DCP driver + * + * Copyright (C) The Asahi Linux Contributors + */ + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h new file mode 100644 index 00000000000000..d6a4742fcf470d --- /dev/null +++ b/drivers/gpu/drm/apple/trace.h @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright (C) The Asahi Linux Contributors */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dcp + +#if !defined(_TRACE_DCP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DCP_H + +#include "dcp-internal.h" + +#include +#include +#include + +#define show_dcp_endpoint(ep) \ + __print_symbolic(ep, { SYSTEM_ENDPOINT, "system" }, \ + { TEST_ENDPOINT, "test" }, \ + { DCP_EXPERT_ENDPOINT, "dcpexpert" }, \ + { DISP0_ENDPOINT, "disp0" }, \ + { DPTX_ENDPOINT, "dptxport" }, \ + { HDCP_ENDPOINT, "hdcp" }, \ + { REMOTE_ALLOC_ENDPOINT, "remotealloc" }, \ + { IOMFB_ENDPOINT, "iomfb" }) + +TRACE_EVENT(dcp_recv_msg, + TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), + TP_ARGS(dcp, endpoint, message), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u8, endpoint) + __field(u64, message)), + + TP_fast_assign(__assign_str(devname, dev_name(dcp->dev)); + __entry->endpoint = endpoint; + __entry->message = message;), + + TP_printk("%s: endpoint 0x%x (%s): received message 0x%016llx", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->message)); + +TRACE_EVENT(dcp_send_msg, + TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), + TP_ARGS(dcp, endpoint, message), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u8, endpoint) + __field(u64, message)), + + TP_fast_assign(__assign_str(devname, dev_name(dcp->dev)); + __entry->endpoint = endpoint; + __entry->message = message;), + + TP_printk("%s: endpoint 0x%x (%s): will send message 0x%016llx", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->message)); + +TRACE_EVENT(iomfb_callback, + TP_PROTO(struct apple_dcp *dcp, int tag, const char *name), + TP_ARGS(dcp, tag, name), + + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __field(int, tag) + __field(const char *, name) + ), + + TP_fast_assign( + __assign_str(devname, dev_name(dcp->dev)); + __entry->tag = tag; __entry->name = name; + ), + + TP_printk("%s: Callback D%03d %s", __get_str(devname), __entry->tag, + __entry->name)); + +TRACE_EVENT(iomfb_push, + TP_PROTO(struct apple_dcp *dcp, + const struct dcp_method_entry *method, int context, + int offset, int depth), + TP_ARGS(dcp, method, context, offset, depth), + + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __string(name, method->name) + __field(int, context) + __field(int, offset) + __field(int, depth)), + + TP_fast_assign( + __assign_str(devname, dev_name(dcp->dev)); + __assign_str(name, method->name); + __entry->context = context; __entry->offset = offset; + __entry->depth = depth; + ), + + TP_printk("%s: Method %s: context %u, offset %u, depth %u", + __get_str(devname), __get_str(name), __entry->context, + __entry->offset, __entry->depth)); + +TRACE_EVENT(iomfb_swap_submit, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%d", + __entry->dcp, + __entry->swap_id) +); + +TRACE_EVENT(iomfb_swap_complete, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%d", + __entry->dcp, + __entry->swap_id + ) +); + +TRACE_EVENT(iomfb_swap_complete_intent_gated, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id, u32 width, u32 height), + TP_ARGS(dcp, swap_id, width, height), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + __field(u32, width) + __field(u32, height) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + __entry->height = height; + __entry->width = width; + ), + TP_printk("dcp=%llx, swap_id=%u %ux%u", + __entry->dcp, + __entry->swap_id, + __entry->width, + __entry->height + ) +); + +#endif /* _TRACE_DCP_H */ + +/* This part must be outside protection */ + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#include From 127d181b78db5ebc13509a39f2830e95e0e6d112 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 11 Oct 2022 08:34:03 +0200 Subject: [PATCH 211/635] gpu: drm: apple: Unbreak multiple DCP plane <-> crtc matching Still untested so this changes the status from definitively broken to probably broken. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 11 +++++++---- drivers/gpu/drm/apple/iomfb.c | 12 ++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index fa966dcc7e9447..ade8eeb9cfb91c 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -134,6 +134,7 @@ u64 apple_format_modifiers[] = { }; static struct drm_plane *apple_plane_init(struct drm_device *dev, + unsigned long possible_crtcs, enum drm_plane_type type) { int ret; @@ -141,7 +142,8 @@ static struct drm_plane *apple_plane_init(struct drm_device *dev, plane = devm_kzalloc(dev->dev, sizeof(*plane), GFP_KERNEL); - ret = drm_universal_plane_init(dev, plane, 0x1, &apple_plane_funcs, + ret = drm_universal_plane_init(dev, plane, possible_crtcs, + &apple_plane_funcs, dcp_formats, ARRAY_SIZE(dcp_formats), apple_format_modifiers, type, NULL); if (ret) @@ -293,7 +295,8 @@ static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { static int apple_probe_per_dcp(struct device *dev, struct drm_device *drm, - struct platform_device *dcp) + struct platform_device *dcp, + int num) { struct apple_crtc *crtc; struct apple_connector *connector; @@ -302,7 +305,7 @@ static int apple_probe_per_dcp(struct device *dev, int con_type; int ret; - primary = apple_plane_init(drm, DRM_PLANE_TYPE_PRIMARY); + primary = apple_plane_init(drm, 1U << num, DRM_PLANE_TYPE_PRIMARY); if (IS_ERR(primary)) return PTR_ERR(primary); @@ -419,7 +422,7 @@ static int apple_platform_probe(struct platform_device *pdev) apple->drm.mode_config.helper_private = &apple_mode_config_helpers; for (i = 0; i < nr_dcp; ++i) { - ret = apple_probe_per_dcp(&pdev->dev, &apple->drm, dcp[i]); + ret = apple_probe_per_dcp(&pdev->dev, &apple->drm, dcp[i], i); if (ret) goto err_unload; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5bab8825df5a60..d4b3dc8d3e6dd5 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1407,7 +1407,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) struct drm_plane_state *new_state, *old_state; struct drm_crtc_state *crtc_state; struct dcp_swap_submit_req *req = &dcp->swap; - int l; + int plane_idx, l; int has_surface = 0; bool modeset; dev_dbg(dcp->dev, "%s", __func__); @@ -1431,10 +1431,15 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; - for_each_oldnew_plane_in_state(state, plane, old_state, new_state, l) { + l = 0; + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { struct drm_framebuffer *fb = new_state->fb; struct drm_rect src_rect; + /* skip planes not for this crtc */ + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + WARN_ON(l >= SWAP_SURFACES); req->swap.swap_enabled |= BIT(l); @@ -1463,6 +1468,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) if (old_state->fb) req->swap.swap_enabled |= DCP_REMOVE_LAYERS; + l += 1; continue; } req->surf_null[l] = false; @@ -1492,6 +1498,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) .has_comp = 1, .has_planes = 1, }; + + l += 1; } /* These fields should be set together */ From 86590197a5daf4a7b6d9504fc4e3240686580d2e Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 11 Oct 2022 08:38:32 +0200 Subject: [PATCH 212/635] gpu: drm: apple: Add support for DRM_FORMAT_XRGB2101010 iboot uses DRM_FORMAT_XRGB2101010 at boot announce preference by listing it as first format. Disabled for now since it results in oversaturated colors. Might be fixed by using "IOMFB::UPPipe2::set_matrix()". Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 2 ++ drivers/gpu/drm/apple/iomfb.c | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index ade8eeb9cfb91c..b3518f47306660 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -122,6 +122,8 @@ static const struct drm_plane_funcs apple_plane_funcs = { * advertise formats without alpha. */ static const u32 dcp_formats[] = { + // DRM_FORMAT_XRGB2101010, + // DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d4b3dc8d3e6dd5..56622c9e463173 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1315,12 +1315,33 @@ static u32 drm_format_to_dcp(u32 drm) case DRM_FORMAT_XBGR8888: case DRM_FORMAT_ABGR8888: return fourcc_code('A', 'B', 'G', 'R'); + + case DRM_FORMAT_ARGB2101010: + case DRM_FORMAT_XRGB2101010: + return fourcc_code('r', '0', '3', 'w'); } pr_warn("DRM format %X not supported in DCP\n", drm); return 0; } +static u8 drm_format_to_colorspace(u32 drm) +{ + switch (drm) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ABGR8888: + return 1; + + case DRM_FORMAT_ARGB2101010: + case DRM_FORMAT_XRGB2101010: + return 2; + } + + return 1; +} + int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); @@ -1484,7 +1505,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->surf[l] = (struct dcp_surface){ .format = drm_format_to_dcp(fb->format->format), .xfer_func = 13, - .colorspace = 1, + .colorspace = drm_format_to_colorspace(fb->format->format), .stride = fb->pitches[0], .width = fb->width, .height = fb->height, From e42e40d13924d01571fbb0e1a07f3e70736a18d1 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 11 Oct 2022 08:40:46 +0200 Subject: [PATCH 213/635] gpu: drm: apple: Add apple_drm_gem_dumb_create() DCP needs a 64-byte aligned pitch, override drm_gem_dma_dumb_create() to align the pitch as necessary and use drm_gem_dma_dumb_create_internal() to allocate the GEM object. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b3518f47306660..6eb67efb1febd5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -45,8 +45,18 @@ struct apple_drm_private { DEFINE_DRM_GEM_DMA_FOPS(apple_fops); +static int apple_drm_gem_dumb_create(struct drm_file *file_priv, + struct drm_device *drm, + struct drm_mode_create_dumb *args) +{ + args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); + args->size = args->pitch * args->height; + + return drm_gem_dma_dumb_create_internal(file_priv, drm, args); +} + static const struct drm_driver apple_drm_driver = { - DRM_GEM_DMA_DRIVER_OPS, + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), .name = DRIVER_NAME, .desc = DRIVER_DESC, .date = "20210901", From 8cef6b64ba29e0ac02680ed4ea1814a48b97f067 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 12 Oct 2022 23:12:07 +0200 Subject: [PATCH 214/635] gpu: drm: apple: Reject modes without valid color mode Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/parser.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index f0cd38c2a81048..bb7d57f272ddb9 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -339,6 +339,12 @@ static int parse_mode(struct dcp_parse_ctx *handle, return ret; } + /* + * Reject modes without valid color mode. + */ + if (best_color_mode < 0) + return -EINVAL; + /* * We need to skip virtual modes. In some cases, virtual modes are "too * big" for the monitor and can cause breakage. It is unclear why the From e6627879cec1bbad3703d5a2856e8498b760cf90 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 12 Oct 2022 23:25:49 +0200 Subject: [PATCH 215/635] gpu: drm: apple: Convert 2 non-assert WARN()s to dev_err() Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 56622c9e463173..3893c0f86ef296 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1437,9 +1437,18 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; - if (WARN(dcp_channel_busy(&dcp->ch_cmd), "unexpected busy channel") || - WARN(!modeset && !dcp->connector->connected, - "can't flush if disconnected")) { + if (dcp_channel_busy(&dcp->ch_cmd)) + { + dev_err(dcp->dev, "unexpected busy command channel"); + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). + */ + schedule_work(&dcp->vblank_wq); + return; + } + if (!modeset && !dcp->connector->connected) + { + dev_err(dcp->dev, "dcp_flush while disconnected"); /* HACK: issue a delayed vblank event to avoid timeouts in * drm_atomic_helper_wait_for_vblanks(). */ From 326853e12ef202b88010fe09a77b3071c2e22040 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 20 Oct 2022 20:42:09 +0200 Subject: [PATCH 216/635] gpu: drm: apple: Send an disconnected hotplug event on ASC crash Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a23949ec73fae0..e8163fcd6c08cb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -108,6 +108,10 @@ static void dcp_rtk_crashed(void *cookie) dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed"); + if (dcp->connector) { + dcp->connector->connected = 0; + schedule_work(&dcp->connector->hotplug_wq); + } } static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) From 0b366b0e1f394f6ee124d3f8443e2845a30e81c0 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 20 Oct 2022 23:02:50 +0200 Subject: [PATCH 217/635] gpu: drm: apple: Add dcp_crtc_atomic_check Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 1 + drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 38 ++++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 9 ------- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 6eb67efb1febd5..8eb08bc4b53fc6 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -300,6 +300,7 @@ static const struct drm_connector_helper_funcs apple_connector_helper_funcs = { static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { .atomic_begin = apple_crtc_atomic_begin, + .atomic_check = dcp_crtc_atomic_check, .atomic_flush = dcp_flush, .atomic_enable = apple_crtc_atomic_enable, .atomic_disable = apple_crtc_atomic_disable, diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index fd7c3aac603e35..c7a7b9563a156f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -10,6 +10,8 @@ #include "iomfb.h" +#define DCP_MAX_PLANES 2 + struct apple_dcp; enum { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e8163fcd6c08cb..02f4c34e6274e3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -182,6 +182,44 @@ void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) false); } +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_plane_state *new_state, *old_state; + struct drm_plane *plane; + struct drm_crtc_state *crtc_state; + int plane_idx, plane_count = 0; + bool needs_modeset; + + if (dcp->crashed) + return -EINVAL; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + if (!needs_modeset && !dcp->connector->connected) { + dev_err(dcp->dev, "crtc_atomic_check: disconnected but no modeset"); + return -EINVAL; + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { + /* skip planes not for this crtc */ + if (new_state->crtc != crtc) + continue; + + plane_count += 1; + } + + if (plane_count > DCP_MAX_PLANES) { + dev_err(dcp->dev, "crtc_atomic_check: Blend supports only 2 layers!"); + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL_GPL(dcp_crtc_atomic_check); + void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, struct apple_connector *connector) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 047c1b3a160457..72ac1315372e5c 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -37,6 +37,7 @@ struct apple_connector { void dcp_poweroff(struct platform_device *pdev); void dcp_poweron(struct platform_device *pdev); +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3893c0f86ef296..da6d34b6d7d146 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1446,15 +1446,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) schedule_work(&dcp->vblank_wq); return; } - if (!modeset && !dcp->connector->connected) - { - dev_err(dcp->dev, "dcp_flush while disconnected"); - /* HACK: issue a delayed vblank event to avoid timeouts in - * drm_atomic_helper_wait_for_vblanks(). - */ - schedule_work(&dcp->vblank_wq); - return; - } /* Reset to defaults */ memset(req, 0, sizeof(*req)); From f0f8e933e5d3569698e292578cd7b2caa8971921 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 21 Oct 2022 22:31:42 +0200 Subject: [PATCH 218/635] gpu: drm: apple: Fix DCP run time PM Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 02f4c34e6274e3..c15f015df988d1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -382,7 +382,7 @@ MODULE_DEVICE_TABLE(of, of_match); */ static int dcp_suspend(struct device *dev) { - dcp_platform_shutdown(to_platform_device(dev)); + dcp_poweroff(to_platform_device(dev)); return 0; } From 6bcf663c76b0c7c9997044271ad7d81dfda4fe50 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 21 Oct 2022 22:34:23 +0200 Subject: [PATCH 219/635] gpu: drm: apple: Fix DCP initialisation Try to avoid races between DRM driver and DCP(ext) probing and initialisation: - do not start RTKit endpoints in dcp probe, iomfb excepts DRM objects to be already initialized on startup - ensure in apple_drv that all DCPs are probe via device links - initialize DRM planes, connectors, encoders - start DCP RTKit endpoints synchronously Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 26 ++++++++++++++++++-------- drivers/gpu/drm/apple/dcp.c | 24 +++++++++++++----------- drivers/gpu/drm/apple/dcp.h | 1 + 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8eb08bc4b53fc6..526d976bb82dd4 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -369,14 +369,16 @@ static int apple_probe_per_dcp(struct device *dev, static int apple_platform_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct apple_drm_private *apple; struct platform_device *dcp[MAX_COPROCESSORS]; int ret, nr_dcp, i; for (nr_dcp = 0; nr_dcp < MAX_COPROCESSORS; ++nr_dcp) { struct device_node *np; + struct device_link *dcp_link; - np = of_parse_phandle(pdev->dev.of_node, "apple,coprocessors", + np = of_parse_phandle(dev->of_node, "apple,coprocessors", nr_dcp); if (!np) @@ -387,11 +389,14 @@ static int apple_platform_probe(struct platform_device *pdev) if (!dcp[nr_dcp]) return -ENODEV; - /* DCP needs to be initialized before KMS can come online */ - if (!platform_get_drvdata(dcp[nr_dcp])) - return -EPROBE_DEFER; + dcp_link = device_link_add(dev, &dcp[nr_dcp]->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!dcp_link) { + dev_err(dev, "Failed to link to DCP %d device", nr_dcp); + return -EINVAL; + } - if (!dcp_is_initialized(dcp[nr_dcp])) + if (dcp_link->supplier->links.status != DL_DEV_DRIVER_BOUND) return -EPROBE_DEFER; } @@ -399,11 +404,11 @@ static int apple_platform_probe(struct platform_device *pdev) if (nr_dcp < 1) return -ENODEV; - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); if (ret) return ret; - apple = devm_drm_dev_alloc(&pdev->dev, &apple_drm_driver, + apple = devm_drm_dev_alloc(dev, &apple_drm_driver, struct apple_drm_private, drm); if (IS_ERR(apple)) return PTR_ERR(apple); @@ -435,7 +440,12 @@ static int apple_platform_probe(struct platform_device *pdev) apple->drm.mode_config.helper_private = &apple_mode_config_helpers; for (i = 0; i < nr_dcp; ++i) { - ret = apple_probe_per_dcp(&pdev->dev, &apple->drm, dcp[i], i); + ret = apple_probe_per_dcp(dev, &apple->drm, dcp[i], i); + + if (ret) + goto err_unload; + + ret = dcp_start(dcp[i]); if (ret) goto err_unload; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c15f015df988d1..e1fa4b58377706 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -227,14 +227,22 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, dcp->crtc = crtc; dcp->connector = connector; +} +EXPORT_SYMBOL_GPL(dcp_link); - /* init connector status by modes offered by dcp */ - connector->connected = dcp->nr_modes > 0; +int dcp_start(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; - /* Dimensions might already be parsed */ - dcp_set_dimensions(dcp); + /* start RTKit endpoints */ + ret = iomfb_start_rtkit(dcp); + if (ret) + dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d", ret); + + return ret; } -EXPORT_SYMBOL_GPL(dcp_link); +EXPORT_SYMBOL(dcp_start); static struct platform_device *dcp_get_dev(struct device *dev, const char *name) { @@ -348,12 +356,6 @@ static int dcp_platform_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to boot RTKit: %d", ret); - /* start RTKit endpoints */ - ret = iomfb_start_rtkit(dcp); - if (ret) - return dev_err_probe(dev, ret, - "Failed to start IOMFB endpoint: %d", ret); - return ret; } diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 72ac1315372e5c..60e9bcfa4714e0 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -40,6 +40,7 @@ void dcp_poweron(struct platform_device *pdev); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); +int dcp_start(struct platform_device *pdev); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); bool dcp_is_initialized(struct platform_device *pdev); void apple_crtc_vblank(struct apple_crtc *apple); From 7363a5c7445bad845810a979dec12154f5ed8171 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 21 Oct 2022 23:43:57 +0200 Subject: [PATCH 220/635] gpu: drm: apple: Specify correct number of DCP*s for drm_vblank_init Unbreaks dcpext a little further. fixup! WIP: drm/apple: Add DCP display driver Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 526d976bb82dd4..360675cb8ca438 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -413,7 +413,7 @@ static int apple_platform_probe(struct platform_device *pdev) if (IS_ERR(apple)) return PTR_ERR(apple); - ret = drm_vblank_init(&apple->drm, 1); + ret = drm_vblank_init(&apple->drm, nr_dcp); if (ret) return ret; From 533bc0e28f0deb63b2f47852e98816f8bfc751f1 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 25 Oct 2022 08:17:10 +0200 Subject: [PATCH 221/635] gpu: drm: apple: Remove other framebuffers before DRM setup Allows taking over the device node from simpledrm and appaers to be the common pattern in DRM drivers replacing boot framebuffers. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 360675cb8ca438..7e05f48d2857e0 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -453,6 +453,7 @@ static int apple_platform_probe(struct platform_device *pdev) drm_mode_config_reset(&apple->drm); + // remove before registering our DRM device ret = drm_aperture_remove_framebuffers(false, &apple_drm_driver); if (ret) return ret; From 2139f85f5b01dc5d6806d41f72a0de5c899a3205 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 27 Oct 2022 02:09:45 +0200 Subject: [PATCH 222/635] gpu: drm: apple: Support opaque pixel formats Opaque pixel formats such as XRGB8888 or YUV formats require flag in struct dcp_surface to be set to be displayed properly. Fixes display issues with fbcon after the handover from simpledrm on j314c with 16:10 modes (notch hiding). Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 5 +++++ drivers/gpu/drm/apple/iomfb.h | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index da6d34b6d7d146..2358f17ab509bf 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1456,6 +1456,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { struct drm_framebuffer *fb = new_state->fb; struct drm_rect src_rect; + bool opaque = false; /* skip planes not for this crtc */ if (old_state->crtc != crtc && new_state->crtc != crtc) @@ -1495,6 +1496,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->surf_null[l] = false; has_surface = 1; + if (!fb->format->has_alpha || + new_state->plane->type == DRM_PLANE_TYPE_PRIMARY) + opaque = true; drm_rect_fp_to_int(&src_rect, &new_state->src); req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); @@ -1503,6 +1507,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->surf_iova[l] = drm_fb_dma_get_gem_addr(fb, new_state, 0); req->surf[l] = (struct dcp_surface){ + .opaque = opaque, .format = drm_format_to_dcp(fb->format->format), .xfer_func = 13, .colorspace = drm_format_to_colorspace(fb->format->format), diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 5b1f4ba789bccf..f9ead84c21f255 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -142,7 +142,7 @@ struct dcp_component_types { struct dcp_surface { u8 is_tiled; u8 unk_1; - u8 unk_2; + u8 opaque; /** ignore alpha, also required YUV overlays */ u32 plane_cnt; u32 plane_cnt2; u32 format; /* DCP fourcc */ From 3b6d72b78ed19f74dae50cb3c22b08809e3b2b60 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 25 Oct 2022 08:24:01 +0200 Subject: [PATCH 223/635] gpu: drm: apple: Provide notch-less modes If the device tree caries a "apple,notch-height" property subtract it from all modes and render all framebuffers offsetted by it. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++++ drivers/gpu/drm/apple/dcp.c | 7 +++++++ drivers/gpu/drm/apple/iomfb.c | 6 +++++- drivers/gpu/drm/apple/parser.c | 9 ++++++--- drivers/gpu/drm/apple/parser.h | 2 +- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index c7a7b9563a156f..6624672109c33e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -67,6 +67,8 @@ struct dcp_fb_reference { struct drm_framebuffer *fb; }; +#define MAX_NOTCH_HEIGHT 160 + /* TODO: move IOMFB members to its own struct */ struct apple_dcp { struct device *dev; @@ -134,6 +136,8 @@ struct apple_dcp { /* Attributes of the connected display */ int width_mm, height_mm; + unsigned notch_height; + /* Workqueue for sending vblank events when a dcp swap is not possible */ struct work_struct vblank_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e1fa4b58377706..4e0dab31d83888 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -332,6 +332,13 @@ static int dcp_platform_probe(struct platform_device *pdev) dev_warn(dev, "failed read 'apple,asc-dram-mask': %d\n", ret); dev_dbg(dev, "'apple,asc-dram-mask': 0x%011llx\n", dcp->asc_dram_mask); + ret = of_property_read_u32(dev->of_node, "apple,notch-height", + &dcp->notch_height); + if (dcp->notch_height > MAX_NOTCH_HEIGHT) + dcp->notch_height = MAX_NOTCH_HEIGHT; + if (dcp->notch_height > 0) + dev_info(dev, "Detected display with notch of %u pixel\n", dcp->notch_height); + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2358f17ab509bf..404f7638cfb2c0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -647,7 +647,8 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, if (!strcmp(req->key, "TimingElements")) { dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, - dcp->width_mm, dcp->height_mm); + dcp->width_mm, dcp->height_mm, + dcp->notch_height); if (IS_ERR(dcp->modes)) { dev_warn(dcp->dev, "failed to parse modes\n"); @@ -1504,6 +1505,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); req->swap.dst_rect[l] = drm_to_dcp_rect(&new_state->dst); + if (dcp->notch_height > 0) + req->swap.dst_rect[l].y += dcp->notch_height; + req->surf_iova[l] = drm_fb_dma_get_gem_addr(fb, new_state, 0); req->surf[l] = (struct dcp_surface){ diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index bb7d57f272ddb9..fc52c26490ec80 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -305,7 +305,7 @@ static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm) + int height_mm, unsigned notch_height) { int ret = 0; struct iterator it; @@ -353,6 +353,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (is_virtual) return -EINVAL; + vert.active -= notch_height; + vert.sync_width += notch_height; + /* From here we must succeed. Start filling out the mode. */ *mode = (struct drm_display_mode) { .type = DRM_MODE_TYPE_DRIVER, @@ -383,7 +386,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, - int height_mm) + int height_mm, unsigned notch_height) { struct iterator it; int ret; @@ -405,7 +408,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, for (; it.idx < it.len; ++it.idx) { mode = &modes[*count]; - ret = parse_mode(it.handle, mode, &score, width_mm, height_mm); + ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height); /* Errors for a single mode are recoverable -- just skip it. */ if (ret) diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 66a675079dc164..a2d479258ed0eb 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -25,7 +25,7 @@ struct dcp_display_mode { int parse(void *blob, size_t size, struct dcp_parse_ctx *ctx); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, - int height_mm); + int height_mm, unsigned notch_height); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm); From b1ccf113cf3cfbe983164abf55a2fa3f44f5b115 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 6 Nov 2022 10:39:05 +0100 Subject: [PATCH 224/635] gpu: drm: apple: Fix shutdown of partially probed dcp No need to shut the co-processor down if it wasn't booted to begin with. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4e0dab31d83888..fb496ec1e3940a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -374,7 +374,8 @@ static void dcp_platform_shutdown(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - iomfb_shutdown(dcp); + if (dcp->shmem) + iomfb_shutdown(dcp); } static const struct of_device_id of_match[] = { From 8bd3aabdd274ff18abcd3f276f436e498d9cb824 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 6 Nov 2022 19:23:08 +0100 Subject: [PATCH 225/635] gpu: drm: apple: Set maximal framebuffer size correctly DCP reports this in the IOMFBMaxSrcPixels dictionary. Use that instead of the hardcoded values from M1 Max. Fixes multiscreen X11 setups. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 7e05f48d2857e0..bc84fd04697ef6 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -428,13 +428,15 @@ static int apple_platform_probe(struct platform_device *pdev) apple->drm.mode_config.min_width = 32; apple->drm.mode_config.min_height = 32; - /* Unknown maximum, use the iMac (24-inch, 2021) display resolution as - * maximum. - * TODO: this is the max framebuffer size not the maximal supported output - * resolution. DCP reports the maximal framebuffer size take it from there. + /* + * TODO: this is the max framebuffer size not the maximal supported + * output resolution. DCP reports the maximal framebuffer size take it + * from there. + * Hardcode it for now to the M1 Max DCP reported 'MaxSrcBufferWidth' + * and 'MaxSrcBufferHeight' of 16384. */ - apple->drm.mode_config.max_width = 4480; - apple->drm.mode_config.max_height = 2520; + apple->drm.mode_config.max_width = 16384; + apple->drm.mode_config.max_height = 16384; apple->drm.mode_config.funcs = &apple_mode_config_funcs; apple->drm.mode_config.helper_private = &apple_mode_config_helpers; From 7599d2e9022c6a118d1473feb7bb365165796bb8 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 6 Nov 2022 20:35:41 +0100 Subject: [PATCH 226/635] gpu: drm: apple: Prevent NULL pointer in dcp_hotplug A bit hackish, probably missing something in the setup or simply calling this too early. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 404f7638cfb2c0..ee00db453da00b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -985,7 +985,7 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ - if (!dcp->valid_mode && connector->connected) { + if (connector->base.state && !dcp->valid_mode && connector->connected) { drm_connector_set_link_status_property( &connector->base, DRM_MODE_LINK_STATUS_BAD); } From 4409fe086dbcf1a3f4cc49be4505e63d8bb673d5 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 6 Nov 2022 23:02:54 +0100 Subject: [PATCH 227/635] gpu: drm: apple: Update date last update Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index bc84fd04697ef6..b01f2207faeba6 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -59,7 +59,7 @@ static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), .name = DRIVER_NAME, .desc = DRIVER_DESC, - .date = "20210901", + .date = "20221106", .major = 1, .minor = 0, .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC, From d03074353f27dac6fbfaa5f65632cb83a10e875e Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 30 Oct 2022 13:11:06 +0100 Subject: [PATCH 228/635] gpu: drm: apple: iomfb: Use FIELD_{GET,PREP} Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 30 +++++++++++++++--------------- drivers/gpu/drm/apple/iomfb.h | 26 ++++++++++++-------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ee00db453da00b..e9d500eded40b8 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -76,22 +76,22 @@ static int dcp_channel_offset(enum dcp_context_id id) static inline u64 dcpep_set_shmem(u64 dart_va) { - return (DCPEP_TYPE_SET_SHMEM << DCPEP_TYPE_SHIFT) | - (DCPEP_FLAG_VALUE << DCPEP_FLAG_SHIFT) | - (dart_va << DCPEP_DVA_SHIFT); + return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_SET_SHMEM) | + FIELD_PREP(IOMFB_SHMEM_FLAG, IOMFB_SHMEM_FLAG_VALUE) | + FIELD_PREP(IOMFB_SHMEM_DVA, dart_va); } static inline u64 dcpep_msg(enum dcp_context_id id, u32 length, u16 offset) { - return (DCPEP_TYPE_MESSAGE << DCPEP_TYPE_SHIFT) | - ((u64)id << DCPEP_CONTEXT_SHIFT) | - ((u64)offset << DCPEP_OFFSET_SHIFT) | - ((u64)length << DCPEP_LENGTH_SHIFT); + return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_MSG) | + FIELD_PREP(IOMFB_MSG_CONTEXT, id) | + FIELD_PREP(IOMFB_MSG_OFFSET, offset) | + FIELD_PREP(IOMFB_MSG_LENGTH, length); } static inline u64 dcpep_ack(enum dcp_context_id id) { - return dcpep_msg(id, 0, 0) | DCPEP_ACK; + return dcpep_msg(id, 0, 0) | IOMFB_MSG_ACK; } /* @@ -1238,9 +1238,9 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) int channel_offset; void *data; - ctx_id = (message & DCPEP_CONTEXT_MASK) >> DCPEP_CONTEXT_SHIFT; - offset = (message & DCPEP_OFFSET_MASK) >> DCPEP_OFFSET_SHIFT; - length = (message >> DCPEP_LENGTH_SHIFT); + ctx_id = FIELD_GET(IOMFB_MSG_CONTEXT, message); + offset = FIELD_GET(IOMFB_MSG_OFFSET, message); + length = FIELD_GET(IOMFB_MSG_LENGTH, message); channel_offset = dcp_channel_offset(ctx_id); @@ -1251,7 +1251,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) data = dcp->shmem + channel_offset + offset; - if (message & DCPEP_ACK) + if (FIELD_GET(IOMFB_MSG_ACK, message)) dcpep_handle_ack(dcp, ctx_id, data, length); else dcpep_handle_cb(dcp, ctx_id, data, length); @@ -1651,11 +1651,11 @@ static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { - enum dcpep_type type = (message >> DCPEP_TYPE_SHIFT) & DCPEP_TYPE_MASK; + enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); - if (type == DCPEP_TYPE_INITIALIZED) + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) dcp_start_signal(dcp, false, dcp_started, NULL); - else if (type == DCPEP_TYPE_MESSAGE) + else if (type == IOMFB_MESSAGE_TYPE_MSG) dcpep_got_msg(dcp, message); else dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index f9ead84c21f255..a82d960512bfd1 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -32,29 +32,27 @@ enum dcp_context_id { /* RTKit endpoint message types */ enum dcpep_type { /* Set shared memory */ - DCPEP_TYPE_SET_SHMEM = 0, + IOMFB_MESSAGE_TYPE_SET_SHMEM = 0, /* DCP is initialized */ - DCPEP_TYPE_INITIALIZED = 1, + IOMFB_MESSAGE_TYPE_INITIALIZED = 1, /* Remote procedure call */ - DCPEP_TYPE_MESSAGE = 2, + IOMFB_MESSAGE_TYPE_MSG = 2, }; +#define IOMFB_MESSAGE_TYPE GENMASK_ULL( 3, 0) + /* Message */ -#define DCPEP_TYPE_SHIFT (0) -#define DCPEP_TYPE_MASK GENMASK(1, 0) -#define DCPEP_ACK BIT_ULL(6) -#define DCPEP_CONTEXT_SHIFT (8) -#define DCPEP_CONTEXT_MASK GENMASK(11, 8) -#define DCPEP_OFFSET_SHIFT (16) -#define DCPEP_OFFSET_MASK GENMASK(31, 16) -#define DCPEP_LENGTH_SHIFT (32) +#define IOMFB_MSG_LENGTH GENMASK_ULL(63, 32) +#define IOMFB_MSG_OFFSET GENMASK_ULL(31, 16) +#define IOMFB_MSG_CONTEXT GENMASK_ULL(11, 8) +#define IOMFB_MSG_ACK BIT_ULL(6) /* Set shmem */ -#define DCPEP_DVA_SHIFT (16) -#define DCPEP_FLAG_SHIFT (4) -#define DCPEP_FLAG_VALUE (4) +#define IOMFB_SHMEM_DVA GENMASK_ULL(63, 16) +#define IOMFB_SHMEM_FLAG GENMASK_ULL( 7, 4) +#define IOMFB_SHMEM_FLAG_VALUE 4 struct dcp_packet_header { char tag[4]; From f30f15bcbda10ce75724bebe1fb6ad3b5a200fc2 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 30 Oct 2022 13:17:28 +0100 Subject: [PATCH 229/635] gpu: drm: apple: iomfb: Unify call and callback channels AP calls initiated inside callbacks are using the callback channel and context. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 11 ++----- drivers/gpu/drm/apple/iomfb.c | 45 +++++++++++----------------- 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 6624672109c33e..8c12382e281b42 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -47,7 +47,7 @@ struct dcp_mem_descriptor { typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); -struct dcp_call_channel { +struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; @@ -57,11 +57,6 @@ struct dcp_call_channel { u8 depth; }; -struct dcp_cb_channel { - u8 depth; - void *output[DCP_MAX_CALL_DEPTH]; -}; - struct dcp_fb_reference { struct list_head head; struct drm_framebuffer *fb; @@ -108,8 +103,8 @@ struct apple_dcp { /* Indexed table of memory descriptors */ struct dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; - struct dcp_call_channel ch_cmd, ch_oobcmd; - struct dcp_cb_channel ch_cb, ch_oobcb, ch_async; + struct dcp_channel ch_cmd, ch_oobcmd; + struct dcp_channel ch_cb, ch_oobcb, ch_async; /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e9d500eded40b8..5dc5a6772e2482 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -98,27 +98,11 @@ static inline u64 dcpep_ack(enum dcp_context_id id) * A channel is busy if we have sent a message that has yet to be * acked. The driver must not sent a message to a busy channel. */ -static bool dcp_channel_busy(struct dcp_call_channel *ch) +static bool dcp_channel_busy(struct dcp_channel *ch) { return (ch->depth != 0); } -/* Get a call channel for a context */ -static struct dcp_call_channel * -dcp_get_call_channel(struct apple_dcp *dcp, enum dcp_context_id context) -{ - switch (context) { - case DCP_CONTEXT_CMD: - case DCP_CONTEXT_CB: - return &dcp->ch_cmd; - case DCP_CONTEXT_OOBCMD: - case DCP_CONTEXT_OOBCB: - return &dcp->ch_oobcmd; - default: - return NULL; - } -} - /* * Get the context ID passed to the DCP for a command we push. The rule is * simple: callback contexts are used when replying to the DCP, command @@ -135,15 +119,19 @@ static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; } -/* Get a callback channel for a context */ -static struct dcp_cb_channel *dcp_get_cb_channel(struct apple_dcp *dcp, - enum dcp_context_id context) +/* Get a channel for a context */ +static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, + enum dcp_context_id context) { switch (context) { case DCP_CONTEXT_CB: return &dcp->ch_cb; + case DCP_CONTEXT_CMD: + return &dcp->ch_cmd; case DCP_CONTEXT_OOBCB: return &dcp->ch_oobcb; + case DCP_CONTEXT_OOBCMD: + return &dcp->ch_oobcmd; case DCP_CONTEXT_ASYNC: return &dcp->ch_async; default: @@ -152,7 +140,7 @@ static struct dcp_cb_channel *dcp_get_cb_channel(struct apple_dcp *dcp, } /* Get the start of a packet: after the end of the previous packet */ -static u16 dcp_packet_start(struct dcp_call_channel *ch, u8 depth) +static u16 dcp_packet_start(struct dcp_channel *ch, u8 depth) { if (depth > 0) return ch->end[depth - 1]; @@ -203,8 +191,8 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { - struct dcp_call_channel *ch = oob ? &dcp->ch_oobcmd : &dcp->ch_cmd; enum dcp_context_id context = dcp_call_context(dcp, oob); + struct dcp_channel *ch = dcp_get_channel(dcp, context); struct dcp_packet_header header = { .in_len = in_len, @@ -329,7 +317,7 @@ static int dcp_parse_tag(char tag[4]) /* Ack a callback from the DCP */ static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) { - struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); + struct dcp_channel *ch = dcp_get_channel(dcp, context); dcp_pop_depth(&ch->depth); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -686,7 +674,7 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, /* Boot sequence */ static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) { - struct dcp_cb_channel *ch = &dcp->ch_cb; + struct dcp_channel *ch = &dcp->ch_cb; u8 *succ = ch->output[ch->depth - 1]; dev_dbg(dcp->dev, "boot done"); @@ -1175,13 +1163,13 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, }; static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) + void *data, u32 length, u16 offset) { struct device *dev = dcp->dev; struct dcp_packet_header *hdr = data; void *in, *out; int tag = dcp_parse_tag(hdr->tag); - struct dcp_cb_channel *ch = dcp_get_cb_channel(dcp, context); + struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; if (tag < 0 || tag >= DCPEP_MAX_CB || !dcpep_cb_handlers[tag]) { @@ -1200,6 +1188,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, depth = dcp_push_depth(&ch->depth); ch->output[depth] = out; + ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); if (dcpep_cb_handlers[tag](dcp, tag, out, in)) dcp_ack(dcp, context); @@ -1209,7 +1198,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, void *data, u32 length) { struct dcp_packet_header *header = data; - struct dcp_call_channel *ch = dcp_get_call_channel(dcp, context); + struct dcp_channel *ch = dcp_get_channel(dcp, context); void *cookie; dcp_callback_t cb; @@ -1254,7 +1243,7 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) if (FIELD_GET(IOMFB_MSG_ACK, message)) dcpep_handle_ack(dcp, ctx_id, data, length); else - dcpep_handle_cb(dcp, ctx_id, data, length); + dcpep_handle_cb(dcp, ctx_id, data, length, offset); } /* From 735cc90f6ff2f5dd11eae456913e2956bc0ad9d9 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 30 Oct 2022 13:29:23 +0100 Subject: [PATCH 230/635] gpu: drm: apple: "match" PMU/backlight services on init Verify that this still works on HDMI/USB-C. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 103 +++++++++++++++++++++++++++++++--- drivers/gpu/drm/apple/iomfb.h | 9 +++ 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 5dc5a6772e2482..58d1a91d054c62 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -170,7 +170,10 @@ const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { DCP_METHOD("A000", dcpep_late_init_signal), DCP_METHOD("A029", dcpep_setup_video_limits), DCP_METHOD("A034", dcpep_update_notify_clients_dcp), + DCP_METHOD("A131", iomfbep_a131_pmu_service_matched), + DCP_METHOD("A132", iomfbep_a132_backlight_service_matched), DCP_METHOD("A357", dcpep_set_create_dfb), + DCP_METHOD("A358", iomfbep_a358_vi_set_temperature_hint), DCP_METHOD("A401", dcpep_start_signal), DCP_METHOD("A407", dcpep_swap_start), DCP_METHOD("A408", dcpep_swap_submit), @@ -257,6 +260,10 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, cb, cookie); \ } +DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); +DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); +DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); + DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, struct dcp_swap_submit_resp); @@ -354,11 +361,91 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp_drm_crtc_vblank(dcp->crtc); } +/* special */ +static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) +{ + // ack D100 cb_match_pmu_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + iomfb_a358_vi_set_temperature_hint(dcp, false, + complete_vi_set_temperature_hint, + NULL); + + // return false for deferred ACK + return false; +} + +static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_pmu_service_2 + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service_2(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a131_pmu_service_matched(dcp, false, complete_pmu_service_matched, + out); + + // return false for deferred ACK + return false; +} + +static void complete_backlight_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_backlight_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a132_backlight_service_matched(dcp, false, complete_backlight_service_matched, out); + + // return false for deferred ACK + return false; +} + static struct dcp_get_uint_prop_resp dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) { - /* unimplemented for now */ - return (struct dcp_get_uint_prop_resp){ .value = 0 }; + struct dcp_get_uint_prop_resp resp = (struct dcp_get_uint_prop_resp){ + .value = 0 + }; + + if (memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ + if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { + /* + * TODO: value from j314c, find out if it is temperature in + * centigrade C and which temperature sensor reports it + */ + resp.value = 3029; + resp.ret = true; + } + } + + return resp; +} + +static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) +{ + // TODO: trace this, see if there properties which needs to used later } /* @@ -1078,6 +1165,8 @@ TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, struct dc_swap_complete_resp); TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, + struct iomfb_set_fx_prop_req) TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, struct dcp_map_buf_req, struct dcp_map_buf_resp); TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, @@ -1113,7 +1202,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [1] = trampoline_true, /* did_power_on_signal */ [2] = trampoline_nop, /* will_power_off_signal */ [3] = trampoline_rt_bandwidth, - [100] = trampoline_nop, /* match_pmu_service */ + [100] = iomfbep_cb_match_pmu_service, [101] = trampoline_zero, /* get_display_default_stride */ [103] = trampoline_nop, /* set_boolean_property */ [106] = trampoline_nop, /* remove_property */ @@ -1121,7 +1210,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [108] = trampoline_true, /* create_product_service */ [109] = trampoline_true, /* create_pmu_service */ [110] = trampoline_true, /* create_iomfb_service */ - [111] = trampoline_false, /* create_backlight_service */ + [111] = trampoline_true, /* create_backlight_service */ [116] = dcpep_cb_boot_1, [117] = trampoline_false, /* is_dark_boot */ [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ @@ -1131,14 +1220,14 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [124] = trampoline_prop_end, [201] = trampoline_map_piodma, [202] = trampoline_unmap_piodma, - [206] = trampoline_true, /* match_pmu_service_2 */ - [207] = trampoline_true, /* match_backlight_service */ + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, [208] = trampoline_get_time, [211] = trampoline_nop, /* update_backlight_factor_prop */ [300] = trampoline_nop, /* pr_publish */ [401] = trampoline_get_uint_prop, [404] = trampoline_nop, /* sr_set_uint_prop */ - [406] = trampoline_nop, /* set_fx_prop */ + [406] = trampoline_set_fx_prop, [408] = trampoline_get_frequency, [411] = trampoline_map_reg, [413] = trampoline_true, /* sr_set_property_dict */ diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index a82d960512bfd1..68fdc654d597f5 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -197,6 +197,9 @@ enum dcpep_method { dcpep_set_parameter_dcp, dcpep_enable_disable_video_power_savings, dcpep_is_main_display, + iomfbep_a131_pmu_service_matched, + iomfbep_a132_backlight_service_matched, + iomfbep_a358_vi_set_temperature_hint, dcpep_num_methods }; @@ -337,6 +340,12 @@ struct dcp_get_uint_prop_resp { u8 padding[3]; } __packed; +struct iomfb_set_fx_prop_req { + char obj[4]; + char key[0x40]; + u32 value; +} __packed; + struct dcp_set_power_state_req { u64 unklong; u8 unkbool; From 94507a09b0edaa4dbfedc5c288901665e23efee1 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 31 Oct 2022 01:11:36 +0100 Subject: [PATCH 231/635] gpu: drm: apple: Brightness control via atomic commits This abuses color_mgnt_change in drm_crtc_state and will be changed once phase 2 of the "drm/kms: control display brightness through drm_connector properties" RfC (linked below) is implemented. The lookup of DAC values from brightness (nits) is not fully understood. Since IOMFB reports te brightness back the easiest solution would be to create our own lookup table or find a approximation which works. DCP appears to report the brightness in nits by "PropRelay::pr_publish(prop_id=15, value=...)" (scaled by "Brightness_scale"). Link: https://lore.kernel.org/dri-devel/b61d3eeb-6213-afac-2e70-7b9791c86d2e@redhat.com/ Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/apple_drv.c | 12 +- drivers/gpu/drm/apple/dcp-internal.h | 15 ++ drivers/gpu/drm/apple/dcp.c | 29 ++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dcp_backlight.c | 232 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 47 +++++- drivers/gpu/drm/apple/iomfb.h | 25 ++- 8 files changed, 347 insertions(+), 16 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 2502f781a5dcef..2c02e4dcfd076d 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -4,7 +4,7 @@ CFLAGS_trace.o = -I$(src) appledrm-y := apple_drv.o -apple_dcp-y := dcp.o iomfb.o parser.o +apple_dcp-y := dcp.o dcp_backlight.o iomfb.o parser.o apple_dcp-$(CONFIG_TRACING) += trace.o apple_piodma-y := dummy-piodma.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b01f2207faeba6..d6ee078d30b935 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -315,7 +315,6 @@ static int apple_probe_per_dcp(struct device *dev, struct apple_connector *connector; struct drm_encoder *encoder; struct drm_plane *primary; - int con_type; int ret; primary = apple_plane_init(drm, 1U << num, DRM_PLANE_TYPE_PRIMARY); @@ -341,17 +340,8 @@ static int apple_probe_per_dcp(struct device *dev, drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); - if (of_property_match_string(dcp->dev.of_node, "apple,connector-type", "eDP") >= 0) - con_type = DRM_MODE_CONNECTOR_eDP; - else if (of_property_match_string(dcp->dev.of_node, "apple,connector-type", "HDMI-A") >= 0) - con_type = DRM_MODE_CONNECTOR_HDMIA; - else if (of_property_match_string(dcp->dev.of_node, "apple,connector-type", "USB-C") >= 0) - con_type = DRM_MODE_CONNECTOR_USB; - else - con_type = DRM_MODE_CONNECTOR_Unknown; - ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, - con_type); + dcp_get_connector_type(dcp)); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8c12382e281b42..c2fa002b45d5de 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -64,6 +64,14 @@ struct dcp_fb_reference { #define MAX_NOTCH_HEIGHT 160 +struct dcp_brightness { + u32 dac; + int nits; + int set; + int scale; + bool update; +}; + /* TODO: move IOMFB members to its own struct */ struct apple_dcp { struct device *dev; @@ -128,6 +136,9 @@ struct apple_dcp { struct dcp_display_mode *modes; unsigned int nr_modes; + /* Attributes of the connector */ + int connector_type; + /* Attributes of the connected display */ int width_mm, height_mm; @@ -140,6 +151,10 @@ struct apple_dcp { * on the next successfully completed swap. */ struct list_head swapped_out_fbs; + + struct dcp_brightness brightness; }; +int dcp_backlight_register(struct apple_dcp *dcp); + #endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fb496ec1e3940a..a1227b48aae2ca 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -22,6 +22,7 @@ #include "dcp.h" #include "dcp-internal.h" +#include "iomfb.h" #include "parser.h" #include "trace.h" @@ -220,6 +221,14 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) } EXPORT_SYMBOL_GPL(dcp_crtc_atomic_check); +int dcp_get_connector_type(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return (dcp->connector_type); +} +EXPORT_SYMBOL_GPL(dcp_get_connector_type); + void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, struct apple_connector *connector) { @@ -278,6 +287,7 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) static int dcp_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct device_node *panel_np; struct apple_dcp *dcp; u32 cpu_ctrl; int ret; @@ -299,6 +309,25 @@ static int dcp_platform_probe(struct platform_device *pdev) of_platform_default_populate(dev->of_node, NULL, dev); + /* intialize brightness scale to a sensible default to avoid divide by 0*/ + dcp->brightness.scale = 65536; + panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + if (panel_np) { + of_node_put(panel_np); + dcp->connector_type = DRM_MODE_CONNECTOR_eDP; + + /* try to register backlight device, */ + ret = dcp_backlight_register(dcp); + if (ret) + return dev_err_probe(dev, ret, + "Unable to register backlight device\n"); + } else if (of_property_match_string(dev->of_node, "apple,connector-type", "HDMI-A") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_HDMIA; + else if (of_property_match_string(dev->of_node, "apple,connector-type", "USB-C") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + else + dcp->connector_type = DRM_MODE_CONNECTOR_Unknown; + dcp->piodma = dcp_get_dev(dev, "apple,piodma-mapper"); if (!dcp->piodma) { dev_err(dev, "failed to find piodma\n"); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 60e9bcfa4714e0..dfe014f3f5d1da 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -38,6 +38,7 @@ struct apple_connector { void dcp_poweroff(struct platform_device *pdev); void dcp_poweron(struct platform_device *pdev); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); +int dcp_get_connector_type(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); int dcp_start(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c new file mode 100644 index 00000000000000..c695e3bad7db91 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright (C) The Asahi Linux Contributors */ + +#include +#include +#include +#include + +#include +#include +#include +#include "linux/jiffies.h" + +#include "dcp.h" +#include "dcp-internal.h" + +#define MIN_BRIGHTNESS_PART1 2U +#define MAX_BRIGHTNESS_PART1 99U +#define MIN_BRIGHTNESS_PART2 103U +#define MAX_BRIGHTNESS_PART2 510U + +/* + * lookup for display brightness 2 to 99 nits + * */ +static u32 brightness_part1[] = { + 0x0000000, 0x0810038, 0x0f000bd, 0x143011c, + 0x1850165, 0x1bc01a1, 0x1eb01d4, 0x2140200, + 0x2380227, 0x2590249, 0x2770269, 0x2930285, + 0x2ac02a0, 0x2c402b8, 0x2d902cf, 0x2ee02e4, + 0x30102f8, 0x314030b, 0x325031c, 0x335032d, + 0x345033d, 0x354034d, 0x362035b, 0x3700369, + 0x37d0377, 0x38a0384, 0x3960390, 0x3a2039c, + 0x3ad03a7, 0x3b803b3, 0x3c303bd, 0x3cd03c8, + 0x3d703d2, 0x3e103dc, 0x3ea03e5, 0x3f303ef, + 0x3fc03f8, 0x4050400, 0x40d0409, 0x4150411, + 0x41d0419, 0x4250421, 0x42d0429, 0x4340431, + 0x43c0438, 0x443043f, 0x44a0446, 0x451044d, + 0x4570454, 0x45e045b, 0x4640461, 0x46b0468, + 0x471046e, 0x4770474, 0x47d047a, 0x4830480, + 0x4890486, 0x48e048b, 0x4940491, 0x4990497, + 0x49f049c, 0x4a404a1, 0x4a904a7, 0x4ae04ac, + 0x4b304b1, 0x4b804b6, 0x4bd04bb, 0x4c204c0, + 0x4c704c5, 0x4cc04c9, 0x4d004ce, 0x4d504d3, + 0x4d904d7, 0x4de04dc, 0x4e204e0, 0x4e704e4, + 0x4eb04e9, 0x4ef04ed, 0x4f304f1, 0x4f704f5, + 0x4fb04f9, 0x4ff04fd, 0x5030501, 0x5070505, + 0x50b0509, 0x50f050d, 0x5130511, 0x5160515, + 0x51a0518, 0x51e051c, 0x5210520, 0x5250523, + 0x5290527, 0x52c052a, 0x52f052e, 0x5330531, + 0x5360535, 0x53a0538, 0x53d053b, 0x540053f, + 0x5440542, 0x5470545, 0x54a0548, 0x54d054c, + 0x550054f, 0x5530552, 0x5560555, 0x5590558, + 0x55c055b, 0x55f055e, 0x5620561, 0x5650564, + 0x5680567, 0x56b056a, 0x56e056d, 0x571056f, + 0x5740572, 0x5760575, 0x5790578, 0x57c057b, + 0x57f057d, 0x5810580, 0x5840583, 0x5870585, + 0x5890588, 0x58c058b, 0x58f058d +}; + +static u32 brightness_part12[] = { 0x58f058d, 0x59d058f }; + +/* + * lookup table for display brightness 103.3 to 510 nits + * */ +static u32 brightness_part2[] = { + 0x59d058f, 0x5b805ab, 0x5d105c5, 0x5e805dd, + 0x5fe05f3, 0x6120608, 0x625061c, 0x637062e, + 0x6480640, 0x6580650, 0x6680660, 0x677066f, + 0x685067e, 0x693068c, 0x6a00699, 0x6ac06a6, + 0x6b806b2, 0x6c406be, 0x6cf06ca, 0x6da06d5, + 0x6e506df, 0x6ef06ea, 0x6f906f4, 0x70206fe, + 0x70c0707, 0x7150710, 0x71e0719, 0x7260722, + 0x72f072a, 0x7370733, 0x73f073b, 0x7470743, + 0x74e074a, 0x7560752, 0x75d0759, 0x7640760, + 0x76b0768, 0x772076e, 0x7780775, 0x77f077c, + 0x7850782, 0x78c0789, 0x792078f, 0x7980795, + 0x79e079b, 0x7a407a1, 0x7aa07a7, 0x7af07ac, + 0x7b507b2, 0x7ba07b8, 0x7c007bd, 0x7c507c2, + 0x7ca07c8, 0x7cf07cd, 0x7d407d2, 0x7d907d7, + 0x7de07dc, 0x7e307e1, 0x7e807e5, 0x7ec07ea, + 0x7f107ef, 0x7f607f3, 0x7fa07f8, 0x7fe07fc +}; + + +static int dcp_get_brightness(struct backlight_device *bd) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + return dcp->brightness.nits; +} + +#define SCALE_FACTOR (1 << 10) + +static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) +{ + u32 frac; + u64 low, high; + u32 interpolated = (tbl_size - 1) * ((val - min) * SCALE_FACTOR) / (max - min); + + size_t index = interpolated / SCALE_FACTOR; + + if (WARN(index + 1 >= tbl_size, "invalid index %zu for brightness %u", index, val)) + return tbl[tbl_size / 2]; + + frac = interpolated & (SCALE_FACTOR - 1); + low = tbl[index]; + high = tbl[index + 1]; + + return ((frac * high) + ((SCALE_FACTOR - frac) * low)) / SCALE_FACTOR; +} + +static u32 calculate_dac(struct apple_dcp *dcp, int val) +{ + u32 dac; + + if (val <= MIN_BRIGHTNESS_PART1) + return 16 * brightness_part1[0]; + else if (val == MAX_BRIGHTNESS_PART1) + return 16 * brightness_part1[ARRAY_SIZE(brightness_part1) - 1]; + else if (val == MIN_BRIGHTNESS_PART2) + return 16 * brightness_part2[0]; + else if (val >= MAX_BRIGHTNESS_PART2) + return brightness_part2[ARRAY_SIZE(brightness_part2) - 1]; + + if (val < MAX_BRIGHTNESS_PART1) { + dac = interpolate(val, MIN_BRIGHTNESS_PART1, MAX_BRIGHTNESS_PART1, + brightness_part1, ARRAY_SIZE(brightness_part1)); + } else if (val > MIN_BRIGHTNESS_PART2) { + dac = interpolate(val, MIN_BRIGHTNESS_PART2, MAX_BRIGHTNESS_PART2, + brightness_part2, ARRAY_SIZE(brightness_part2)); + } else { + dac = interpolate(val, MAX_BRIGHTNESS_PART1, MIN_BRIGHTNESS_PART2, + brightness_part12, ARRAY_SIZE(brightness_part12)); + } + + return 16 * dac; +} + +static int drm_crtc_set_brightness(struct drm_crtc *crtc, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; + int ret = 0; + + state = drm_atomic_state_alloc(crtc->dev); + if (!state) + return -ENOMEM; + + state->acquire_ctx = ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + + crtc_state->color_mgmt_changed |= true; + + ret = drm_atomic_commit(state); + +fail: + drm_atomic_state_put(state); + return ret; +} + +static int dcp_set_brightness(struct backlight_device *bd) +{ + int ret = 0; + struct apple_dcp *dcp = bl_get_data(bd); + + bd->props.power = FB_BLANK_UNBLANK; + if (dcp->brightness.set != bd->props.brightness) { + dcp->brightness.dac = calculate_dac(dcp, bd->props.brightness); + dcp->brightness.set = bd->props.brightness; + dcp->brightness.update = true; + } + + if (dcp->brightness.update && dcp->crtc) { + struct drm_modeset_acquire_ctx ctx; + struct drm_device *drm_dev = dcp->crtc->base.dev; + + DRM_MODESET_LOCK_ALL_BEGIN(drm_dev, ctx, 0, ret); + ret = drm_crtc_set_brightness(&dcp->crtc->base, &ctx); + DRM_MODESET_LOCK_ALL_END(drm_dev, ctx, ret); + } + + return ret; +} + +static const struct backlight_ops dcp_backlight_ops = { + .get_brightness = dcp_get_brightness, + .update_status = dcp_set_brightness, +}; + +int dcp_backlight_register(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct backlight_device *bd; + struct device_node *panel_np; + struct backlight_properties props = { + .type = BACKLIGHT_PLATFORM, + .brightness = dcp->brightness.nits, + .max_brightness = 0, + .scale = BACKLIGHT_SCALE_LINEAR, + }; + u32 max_brightness; + int ret = 0; + + panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + if (!panel_np) + return 0; + + if (!of_device_is_available(panel_np)) + goto out_put; + + ret = of_property_read_u32(panel_np, "apple,max-brightness", &max_brightness); + if (ret) { + dev_err(dev, "Missing property 'apple,max-brightness'\n"); + goto out_put; + } + props.max_brightness = min(max_brightness, MAX_BRIGHTNESS_PART2 - 1); + + bd = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, + &dcp_backlight_ops, &props); + if (IS_ERR(bd)) + ret = PTR_ERR(bd); + +out_put: + of_node_put(panel_np); + + return ret; +} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 58d1a91d054c62..eff9178e060228 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -422,6 +422,21 @@ static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, v return false; } +static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) +{ + switch (prop->id) { + case IOMFB_PROPERTY_NITS: + dcp->brightness.nits = prop->value / dcp->brightness.scale; + /* temporary for user debugging during tesing */ + dev_info(dcp->dev, "Backlight updated to %u nits\n", + dcp->brightness.nits); + dcp->brightness.update = false; + break; + default: + dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); + } +} + static struct dcp_get_uint_prop_resp dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) { @@ -443,6 +458,19 @@ dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) return resp; } +static u8 iomfbep_cb_sr_set_property_int(struct apple_dcp *dcp, + struct iomfb_sr_set_property_int_req *req) +{ + if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ + if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { + if (!req->value_null) + dcp->brightness.scale = req->value; + } + } + + return 1; +} + static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) { // TODO: trace this, see if there properties which needs to used later @@ -1171,6 +1199,8 @@ TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, struct dcp_map_buf_req, struct dcp_map_buf_resp); TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, struct dcp_unmap_buf_resp); +TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, + struct iomfb_sr_set_property_int_req, u8); TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, struct dcp_allocate_buffer_req, struct dcp_allocate_buffer_resp); @@ -1195,6 +1225,8 @@ TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, dcpep_cb_swap_complete_intent_gated, struct dcp_swap_complete_intent_gated); +TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, + struct iomfb_property); bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, void *) = { @@ -1204,6 +1236,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [3] = trampoline_rt_bandwidth, [100] = iomfbep_cb_match_pmu_service, [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ [103] = trampoline_nop, /* set_boolean_property */ [106] = trampoline_nop, /* remove_property */ [107] = trampoline_true, /* create_provider_service */ @@ -1224,14 +1257,14 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [207] = iomfbep_cb_match_backlight_service, [208] = trampoline_get_time, [211] = trampoline_nop, /* update_backlight_factor_prop */ - [300] = trampoline_nop, /* pr_publish */ + [300] = trampoline_pr_publish, [401] = trampoline_get_uint_prop, [404] = trampoline_nop, /* sr_set_uint_prop */ [406] = trampoline_set_fx_prop, [408] = trampoline_get_frequency, [411] = trampoline_map_reg, [413] = trampoline_true, /* sr_set_property_dict */ - [414] = trampoline_true, /* sr_set_property_int */ + [414] = trampoline_sr_set_property_int, [415] = trampoline_true, /* sr_set_property_bool */ [451] = trampoline_allocate_buffer, [452] = trampoline_map_physical, @@ -1613,6 +1646,14 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) /* These fields should be set together */ req->swap.swap_completed = req->swap.swap_enabled; + /* update brightness if changed */ + if (dcp->brightness.update) { + req->swap.bl_unk = 1; + req->swap.bl_value = dcp->brightness.dac; + req->swap.bl_power = 0x40; + dcp->brightness.update = false; + } + if (modeset) { struct dcp_display_mode *mode; struct dcp_wait_cookie *cookie; @@ -1666,7 +1707,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) dcp->valid_mode = true; } - if (!has_surface) { + if (!has_surface && !crtc_state->color_mgmt_changed) { if (crtc_state->enable && crtc_state->active && !crtc_state->planes_changed) { schedule_work(&dcp->vblank_wq); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 68fdc654d597f5..386a84cfcc5cd1 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -63,6 +63,12 @@ struct dcp_packet_header { #define DCP_IS_NULL(ptr) ((ptr) ? 1 : 0) #define DCP_PACKET_ALIGNMENT (0x40) +enum iomfb_property_id { + IOMFB_PROPERTY_NITS = 15, // divide by Brightness_Scale +}; + +#define IOMFB_BRIGHTNESS_MIN 0x10000000 + /* Structures used in v12.0 firmware */ #define SWAP_SURFACES 4 @@ -114,7 +120,11 @@ struct dcp_swap { u32 unk_2c8; u8 unk_2cc[0x14]; u32 unk_2e0; - u8 unk_2e4[0x3c]; + u16 unk_2e2; + u64 bl_unk; + u32 bl_value; // min value is 0x10000000 + u8 bl_power; // constant 0x40 for on + u8 unk_2f3[0x2d]; } __packed; /* Information describing a plane of a planar compressed surface */ @@ -340,6 +350,14 @@ struct dcp_get_uint_prop_resp { u8 padding[3]; } __packed; +struct iomfb_sr_set_property_int_req { + char obj[4]; + char key[0x40]; + u64 value; + u8 value_null; + u8 padding[3]; +} __packed; + struct iomfb_set_fx_prop_req { char obj[4]; char key[0x40]; @@ -410,4 +428,9 @@ struct dcp_read_edt_data_resp { u8 ret; } __packed; +struct iomfb_property { + u32 id; + u32 value; +} __packed; + #endif From 73085c28ddcecd8755c504afa18215a894aa6bb0 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 16 Nov 2022 00:10:31 +0100 Subject: [PATCH 232/635] HACK: gpu: drm: apple: j314/j316: Ignore 120 Hz mode for integrated display It's currently not useful as DCP limits the swap rate to 60 Hz anyway and marcan reported pointer choppiness with 120 Hz. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/parser.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index fc52c26490ec80..31aecd4b2fc195 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -353,6 +353,19 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (is_virtual) return -EINVAL; + /* + * HACK: + * Ignore the 120 Hz mode on j314/j316 (identified by resolution). + * DCP limits normal swaps to 60 Hz anyway and the 120 Hz mode might + * cause choppiness with X11. + * Just downscoring it and thus making the 60 Hz mode the preferred mode + * seems not enough for some user space. + */ + if (vert.precise_sync_rate >> 16 == 120 && + ((horiz.active == 3024 && vert.active == 1964) || + (horiz.active == 3456 && vert.active == 2234))) + return -EINVAL; + vert.active -= notch_height; vert.sync_width += notch_height; From e2c16347efb61e648a0483bc3cdfb5e576eadc67 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 17 Nov 2022 21:51:09 +0900 Subject: [PATCH 233/635] drm/apple: Fix suspend/resume handling Use the drm_modeset helpers for suspend/resume at the subsystem level, which actually do the proper save/restore dance and work, instead of open-coding calls to dcp_poweroff/dcp_poweron which is clearly wrong and doesn't restore properly (nor would it be correct if the display was already off when suspended). Also fix apple_platform_remove while I'm here, since drvdata wasn't getting set so that would never work. Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/apple_drv.c | 31 +++++++++++++++++++++++++++++-- drivers/gpu/drm/apple/dcp.c | 27 --------------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d6ee078d30b935..a210b60484dd8f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -403,6 +403,8 @@ static int apple_platform_probe(struct platform_device *pdev) if (IS_ERR(apple)) return PTR_ERR(apple); + dev_set_drvdata(dev, apple); + ret = drm_vblank_init(&apple->drm, nr_dcp); if (ret) return ret; @@ -465,9 +467,9 @@ static int apple_platform_probe(struct platform_device *pdev) static int apple_platform_remove(struct platform_device *pdev) { - struct drm_device *drm = platform_get_drvdata(pdev); + struct apple_drm_private *apple = platform_get_drvdata(pdev); - drm_dev_unregister(drm); + drm_dev_unregister(&apple->drm); return 0; } @@ -478,10 +480,35 @@ static const struct of_device_id of_match[] = { }; MODULE_DEVICE_TABLE(of, of_match); +#ifdef CONFIG_PM_SLEEP +static int apple_platform_suspend(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + return drm_mode_config_helper_suspend(&apple->drm); +} + +static int apple_platform_resume(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + drm_mode_config_helper_resume(&apple->drm); + return 0; +} + +static const struct dev_pm_ops apple_platform_pm_ops = { + .suspend = apple_platform_suspend, + .resume = apple_platform_resume, +}; +#endif + static struct platform_driver apple_platform_driver = { .driver = { .name = "apple-drm", .of_match_table = of_match, +#ifdef CONFIG_PM_SLEEP + .pm = &apple_platform_pm_ops, +#endif }, .probe = apple_platform_probe, .remove = apple_platform_remove, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a1227b48aae2ca..a90cb030faf4df 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -413,39 +413,12 @@ static const struct of_device_id of_match[] = { }; MODULE_DEVICE_TABLE(of, of_match); -#ifdef CONFIG_PM_SLEEP -/* - * We don't hold any useful persistent state, so for suspend/resume it suffices - * to power off/on the entire DCP. The firmware will sort out the details for - * us. - */ -static int dcp_suspend(struct device *dev) -{ - dcp_poweroff(to_platform_device(dev)); - return 0; -} - -static int dcp_resume(struct device *dev) -{ - dcp_poweron(to_platform_device(dev)); - return 0; -} - -static const struct dev_pm_ops dcp_pm_ops = { - .suspend = dcp_suspend, - .resume = dcp_resume, -}; -#endif - static struct platform_driver apple_platform_driver = { .probe = dcp_platform_probe, .shutdown = dcp_platform_shutdown, .driver = { .name = "apple-dcp", .of_match_table = of_match, -#ifdef CONFIG_PM_SLEEP - .pm = &dcp_pm_ops, -#endif }, }; From 12d7ab109c95b01bbb05da245efde040d9a48ae1 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 20 Nov 2022 09:20:14 +0100 Subject: [PATCH 234/635] gpu: drm: apple: Avoid drm_fb_dma_get_gem_addr It adjust the address by the source position duplicating setting the source postion in IOMFB's swap_submit struct. Prefer the later since it is more explicit. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index eff9178e060228..cd0a6e2c8d9211 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -1567,6 +1568,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) l = 0; for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { struct drm_framebuffer *fb = new_state->fb; + struct drm_gem_dma_object *obj; struct drm_rect src_rect; bool opaque = false; @@ -1619,7 +1621,13 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) if (dcp->notch_height > 0) req->swap.dst_rect[l].y += dcp->notch_height; - req->surf_iova[l] = drm_fb_dma_get_gem_addr(fb, new_state, 0); + /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts + * the address for source x/y offsets. Since IOMFB has a direct + * support source position prefer that. + */ + obj = drm_fb_dma_get_gem_obj(fb, 0); + if (obj) + req->surf_iova[l] = obj->dma_addr + fb->offsets[0]; req->surf[l] = (struct dcp_surface){ .opaque = opaque, From 0d636afccdfdcc6ebdf12e5c3657af39061674e8 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 23 Nov 2022 00:19:11 +0100 Subject: [PATCH 235/635] drm/apple: register backlight device after IOMFB start This allows us to specify the boot display brightness as initial brightness of the baclight device. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 8 +++- drivers/gpu/drm/apple/dcp.c | 37 +++++++++++++--- drivers/gpu/drm/apple/dcp_backlight.c | 61 +++++++++------------------ drivers/gpu/drm/apple/iomfb.c | 9 ++-- 4 files changed, 64 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index c2fa002b45d5de..9f32fd9d0182ed 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -4,7 +4,9 @@ #ifndef __APPLE_DCP_INTERNAL_H__ #define __APPLE_DCP_INTERNAL_H__ +#include #include +#include #include #include @@ -65,9 +67,10 @@ struct dcp_fb_reference { #define MAX_NOTCH_HEIGHT 160 struct dcp_brightness { + struct backlight_device *bl_dev; + u32 maximum; u32 dac; int nits; - int set; int scale; bool update; }; @@ -153,6 +156,9 @@ struct apple_dcp { struct list_head swapped_out_fbs; struct dcp_brightness brightness; + /* Workqueue for updating the initial initial brightness */ + struct work_struct bl_register_wq; + struct mutex bl_register_mutex; }; int dcp_backlight_register(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a90cb030faf4df..1234617beb3ccd 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -13,6 +13,7 @@ #include #include #include +#include "linux/workqueue.h" #include #include @@ -253,6 +254,28 @@ int dcp_start(struct platform_device *pdev) } EXPORT_SYMBOL(dcp_start); +static void dcp_work_register_backlight(struct work_struct *work) +{ + int ret; + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, bl_register_wq); + + mutex_lock(&dcp->bl_register_mutex); + if (dcp->brightness.bl_dev) + goto out_unlock; + + /* try to register backlight device, */ + ret = dcp_backlight_register(dcp); + if (ret) { + dev_err(dcp->dev, "Unable to register backlight device\n"); + dcp->brightness.maximum = 0; + } + +out_unlock: + mutex_unlock(&dcp->bl_register_mutex); +} + static struct platform_device *dcp_get_dev(struct device *dev, const char *name) { struct platform_device *pdev; @@ -313,14 +336,16 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp->brightness.scale = 65536; panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); if (panel_np) { + if (of_device_is_available(panel_np)) { + ret = of_property_read_u32(panel_np, "apple,max-brightness", + &dcp->brightness.maximum); + if (ret) + dev_err(dev, "Missing property 'apple,max-brightness'\n"); + } of_node_put(panel_np); dcp->connector_type = DRM_MODE_CONNECTOR_eDP; - - /* try to register backlight device, */ - ret = dcp_backlight_register(dcp); - if (ret) - return dev_err_probe(dev, ret, - "Unable to register backlight device\n"); + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + mutex_init(&dcp->bl_register_mutex); } else if (of_property_match_string(dev->of_node, "apple,connector-type", "HDMI-A") >= 0) dcp->connector_type = DRM_MODE_CONNECTOR_HDMIA; else if (of_property_match_string(dev->of_node, "apple,connector-type", "USB-C") >= 0) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index c695e3bad7db91..5b4a41c53ca21b 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -165,29 +165,28 @@ static int drm_crtc_set_brightness(struct drm_crtc *crtc, static int dcp_set_brightness(struct backlight_device *bd) { - int ret = 0; + int ret; struct apple_dcp *dcp = bl_get_data(bd); + struct drm_modeset_acquire_ctx ctx; - bd->props.power = FB_BLANK_UNBLANK; - if (dcp->brightness.set != bd->props.brightness) { - dcp->brightness.dac = calculate_dac(dcp, bd->props.brightness); - dcp->brightness.set = bd->props.brightness; - dcp->brightness.update = true; - } + if (bd->props.state & BL_CORE_SUSPENDED) + return 0; - if (dcp->brightness.update && dcp->crtc) { - struct drm_modeset_acquire_ctx ctx; - struct drm_device *drm_dev = dcp->crtc->base.dev; + if (!dcp->crtc) + return -EAGAIN; - DRM_MODESET_LOCK_ALL_BEGIN(drm_dev, ctx, 0, ret); - ret = drm_crtc_set_brightness(&dcp->crtc->base, &ctx); - DRM_MODESET_LOCK_ALL_END(drm_dev, ctx, ret); - } + dcp->brightness.dac = calculate_dac(dcp, bd->props.brightness); + dcp->brightness.update = true; + + DRM_MODESET_LOCK_ALL_BEGIN(dcp->crtc->base.dev, ctx, 0, ret); + ret = drm_crtc_set_brightness(&dcp->crtc->base, &ctx); + DRM_MODESET_LOCK_ALL_END(dcp->crtc->base.dev, ctx, ret); return ret; } static const struct backlight_ops dcp_backlight_ops = { + .options = BL_CORE_SUSPENDRESUME, .get_brightness = dcp_get_brightness, .update_status = dcp_set_brightness, }; @@ -195,38 +194,20 @@ static const struct backlight_ops dcp_backlight_ops = { int dcp_backlight_register(struct apple_dcp *dcp) { struct device *dev = dcp->dev; - struct backlight_device *bd; - struct device_node *panel_np; + struct backlight_device *bl_dev; struct backlight_properties props = { .type = BACKLIGHT_PLATFORM, .brightness = dcp->brightness.nits, - .max_brightness = 0, .scale = BACKLIGHT_SCALE_LINEAR, }; - u32 max_brightness; - int ret = 0; + props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); - panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); - if (!panel_np) - return 0; - - if (!of_device_is_available(panel_np)) - goto out_put; - - ret = of_property_read_u32(panel_np, "apple,max-brightness", &max_brightness); - if (ret) { - dev_err(dev, "Missing property 'apple,max-brightness'\n"); - goto out_put; - } - props.max_brightness = min(max_brightness, MAX_BRIGHTNESS_PART2 - 1); - - bd = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, - &dcp_backlight_ops, &props); - if (IS_ERR(bd)) - ret = PTR_ERR(bd); + bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, + &dcp_backlight_ops, &props); + if (IS_ERR(bl_dev)) + return PTR_ERR(bl_dev); -out_put: - of_node_put(panel_np); + dcp->brightness.bl_dev = bl_dev; - return ret; + return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index cd0a6e2c8d9211..59fe986f71ba67 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -427,12 +427,13 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr { switch (prop->id) { case IOMFB_PROPERTY_NITS: + { dcp->brightness.nits = prop->value / dcp->brightness.scale; - /* temporary for user debugging during tesing */ - dev_info(dcp->dev, "Backlight updated to %u nits\n", - dcp->brightness.nits); - dcp->brightness.update = false; + /* notify backlight device of the initial brightness */ + if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); break; + } default: dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); } From 512be623b3bef328b89219e5a1028196e8c67535 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 23 Nov 2022 00:27:38 +0100 Subject: [PATCH 236/635] drm/apple: Add trace point for display brightness Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 1 + drivers/gpu/drm/apple/trace.h | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 59fe986f71ba67..ca11c0ca8df23f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -432,6 +432,7 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr /* notify backlight device of the initial brightness */ if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) schedule_work(&dcp->bl_register_wq); + trace_iomfb_brightness(dcp, prop->value); break; } default: diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d6a4742fcf470d..ddf6ff9839e4f7 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -153,6 +153,24 @@ TRACE_EVENT(iomfb_swap_complete_intent_gated, ) ); +TRACE_EVENT(iomfb_brightness, + TP_PROTO(struct apple_dcp *dcp, u32 nits), + TP_ARGS(dcp, nits), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, nits) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->nits = nits; + ), + TP_printk("dcp=%llx, nits=%u (raw=0x%05x)", + __entry->dcp, + __entry->nits >> 16, + __entry->nits + ) +); + #endif /* _TRACE_DCP_H */ /* This part must be outside protection */ From a0818931ca1aad0e653df33aca38296b137f42a4 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 23 Nov 2022 23:21:54 +0100 Subject: [PATCH 237/635] drm/apple: Implement drm_crtc_helper_funcs.mode_fixup Prevents KDE for now to set modes not reported by IOMFB. Seen on j493 with the common display resolution of 2560x1600. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 1 + drivers/gpu/drm/apple/dcp.h | 3 +++ drivers/gpu/drm/apple/iomfb.c | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index a210b60484dd8f..bce91de706eb0f 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -304,6 +304,7 @@ static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { .atomic_flush = dcp_flush, .atomic_enable = apple_crtc_atomic_enable, .atomic_disable = apple_crtc_atomic_disable, + .mode_fixup = dcp_crtc_mode_fixup, }; static int apple_probe_per_dcp(struct device *dev, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index dfe014f3f5d1da..fb4397e7b390fe 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -49,6 +49,9 @@ void dcp_drm_crtc_vblank(struct apple_crtc *crtc); int dcp_get_modes(struct drm_connector *connector); int dcp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode); +bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode); void dcp_set_dimensions(struct apple_dcp *dcp); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ca11c0ca8df23f..d13e4a02deeb5d 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1484,7 +1484,7 @@ EXPORT_SYMBOL_GPL(dcp_get_modes); /* The user may own drm_display_mode, so we need to search for our copy */ static struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, - struct drm_display_mode *mode) + const struct drm_display_mode *mode) { int i; @@ -1509,6 +1509,19 @@ int dcp_mode_valid(struct drm_connector *connector, } EXPORT_SYMBOL_GPL(dcp_mode_valid); +bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct platform_device *pdev = apple_crtc->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + /* TODO: support synthesized modes through scaling */ + return lookup_mode(dcp, mode) != NULL; +} +EXPORT_SYMBOL(dcp_crtc_mode_fixup); + /* Helpers to modeset and swap, used to flush */ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) { From a5c8424e984e4d2a471308cfa41e6d7bcb08694a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 24 Nov 2022 21:44:36 +0100 Subject: [PATCH 238/635] drm/apple: Read display dimensions from devicetree Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 22 +++++++++++++++------- drivers/gpu/drm/apple/iomfb.c | 15 ++++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1234617beb3ccd..03bb574a77bc24 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -332,16 +332,31 @@ static int dcp_platform_probe(struct platform_device *pdev) of_platform_default_populate(dev->of_node, NULL, dev); + ret = of_property_read_u32(dev->of_node, "apple,notch-height", + &dcp->notch_height); + if (dcp->notch_height > MAX_NOTCH_HEIGHT) + dcp->notch_height = MAX_NOTCH_HEIGHT; + if (dcp->notch_height > 0) + dev_info(dev, "Detected display with notch of %u pixel\n", dcp->notch_height); + /* intialize brightness scale to a sensible default to avoid divide by 0*/ dcp->brightness.scale = 65536; panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); if (panel_np) { + const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + if (of_device_is_available(panel_np)) { ret = of_property_read_u32(panel_np, "apple,max-brightness", &dcp->brightness.maximum); if (ret) dev_err(dev, "Missing property 'apple,max-brightness'\n"); } + + of_property_read_u32(panel_np, "width-mm", &dcp->width_mm); + /* use adjusted height as long as the notch is hidden */ + of_property_read_u32(panel_np, height_prop[!dcp->notch_height], + &dcp->height_mm); + of_node_put(panel_np); dcp->connector_type = DRM_MODE_CONNECTOR_eDP; INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); @@ -386,13 +401,6 @@ static int dcp_platform_probe(struct platform_device *pdev) dev_warn(dev, "failed read 'apple,asc-dram-mask': %d\n", ret); dev_dbg(dev, "'apple,asc-dram-mask': 0x%011llx\n", dcp->asc_dram_mask); - ret = of_property_read_u32(dev->of_node, "apple,notch-height", - &dcp->notch_height); - if (dcp->notch_height > MAX_NOTCH_HEIGHT) - dcp->notch_height = MAX_NOTCH_HEIGHT; - if (dcp->notch_height > 0) - dev_info(dev, "Detected display with notch of %u pixel\n", dcp->notch_height); - bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d13e4a02deeb5d..921e593129daeb 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -763,12 +763,17 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, return false; } } else if (!strcmp(req->key, "DisplayAttributes")) { - ret = parse_display_attributes(&ctx, &dcp->width_mm, - &dcp->height_mm); + /* DisplayAttributes are empty for integrated displays, use + * display dimensions read from the devicetree + */ + if (dcp->main_display) { + ret = parse_display_attributes(&ctx, &dcp->width_mm, + &dcp->height_mm); - if (ret) { - dev_warn(dcp->dev, "failed to parse display attribs\n"); - return false; + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } } dcp_set_dimensions(dcp); From 6dff165be1423f2004b528a637fef8f05cdceda4 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 28 Nov 2022 00:42:25 +0900 Subject: [PATCH 239/635] drm/apple: Wait for power on request to complete synchronously Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/iomfb.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 921e593129daeb..435999f3a354c7 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -941,6 +941,16 @@ static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) } } +static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + dev_dbg(dcp->dev, "%s", __func__); + + dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); +} + static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_parameter_dcp param = { @@ -950,16 +960,13 @@ static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) }; dev_dbg(dcp->dev, "%s", __func__); - dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_final, cookie); + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_power_state, cookie); } void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); struct dcp_wait_cookie *cookie; - struct dcp_set_power_state_req req = { - .unklong = 1, - }; int ret; u32 handle; dev_dbg(dcp->dev, "%s", __func__); @@ -975,15 +982,13 @@ void dcp_poweron(struct platform_device *pdev) if (dcp->main_display) { handle = 0; - dcp_set_display_device(dcp, false, &handle, dcp_on_final, + dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, cookie); } else { handle = 2; dcp_set_display_device(dcp, false, &handle, dcp_on_set_parameter, cookie); } - dcp_set_power_state(dcp, true, &req, NULL, NULL); - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); if (ret == 0) From e87f450d5e6e0f0d67a314c978378d541faed09b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 28 Nov 2022 00:44:41 +0900 Subject: [PATCH 240/635] drm/apple: Remove obsolete ignore_swap_complete Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/dcp-internal.h | 2 -- drivers/gpu/drm/apple/iomfb.c | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9f32fd9d0182ed..24db9f39d78e2a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -133,8 +133,6 @@ struct apple_dcp { /* eDP display without DP-HDMI conversion */ bool main_display; - bool ignore_swap_complete; - /* Modes valid for the connected display */ struct dcp_display_mode *modes; unsigned int nr_modes; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 435999f3a354c7..a07bd468189193 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -358,8 +358,7 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, { trace_iomfb_swap_complete(dcp, resp->swap_id); - if (!dcp->ignore_swap_complete) - dcp_drm_crtc_vblank(dcp->crtc); + dcp_drm_crtc_vblank(dcp->crtc); } /* special */ @@ -1550,8 +1549,6 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, struct dcp_wait_cookie *wait = cookie; dev_dbg(dcp->dev, "%s", __func__); - dcp->ignore_swap_complete = false; - if (wait) { complete(&wait->done); kref_put(&wait->refcount, release_wait_cookie); From e0091c138f88fdeaef2cda72be821eb401362114 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 28 Nov 2022 01:02:24 +0900 Subject: [PATCH 241/635] drm/asahi: Fix backlight restores on non-microLED devices Apparently what happens here is that the DCP's idea of backlight brightness is desynced with the real brightness across power cycles. This means that even if we just force an update after a power cycle, it doesn't work since it considers it unchanged. To fix this, we need to both force an update on poweron and also explicitly turn the backlight off on poweroff, which makes DCP listen to us and actually update the backlight state properly. Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/dcp_backlight.c | 1 + drivers/gpu/drm/apple/iomfb.c | 31 ++++++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 5b4a41c53ca21b..42b1097eaa0180 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -208,6 +208,7 @@ int dcp_backlight_register(struct apple_dcp *dcp) return PTR_ERR(bl_dev); dcp->brightness.bl_dev = bl_dev; + dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a07bd468189193..17d4602b1cb006 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -994,6 +994,9 @@ void dcp_poweron(struct platform_device *pdev) dev_warn(dcp->dev, "wait for power timed out"); kref_put(&cookie->refcount, release_wait_cookie);; + + /* Force a brightness update after poweron, to restore the brightness */ + dcp->brightness.update = true; } EXPORT_SYMBOL(dcp_poweron); @@ -1036,6 +1039,17 @@ void dcp_poweroff(struct platform_device *pdev) dcp->swap.swap.swap_completed = DCP_REMOVE_LAYERS | 0x7; dcp->swap.swap.unk_10c = 0xFF000000; + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + dcp->swap.swap.bl_unk = 1; + dcp->swap.swap.bl_value = 0; + dcp->swap.swap.bl_power = 0; + for (int l = 0; l < SWAP_SURFACES; l++) dcp->swap.surf_null[l] = true; @@ -1676,14 +1690,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) /* These fields should be set together */ req->swap.swap_completed = req->swap.swap_enabled; - /* update brightness if changed */ - if (dcp->brightness.update) { - req->swap.bl_unk = 1; - req->swap.bl_value = dcp->brightness.dac; - req->swap.bl_power = 0x40; - dcp->brightness.update = false; - } - if (modeset) { struct dcp_display_mode *mode; struct dcp_wait_cookie *cookie; @@ -1746,6 +1752,15 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->clear = 1; } + + /* update brightness if changed */ + if (dcp->brightness.update) { + req->swap.bl_unk = 1; + req->swap.bl_value = dcp->brightness.dac; + req->swap.bl_power = 0x40; + dcp->brightness.update = false; + } + do_swap(dcp, NULL, NULL); } EXPORT_SYMBOL_GPL(dcp_flush); From 1abf0282618d402f046fb9684b991eaafd82e305 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 27 Nov 2022 21:48:44 +0100 Subject: [PATCH 242/635] drm/apple: Schedule backlight update on enable_backlight_message_ap_gated On non mini-LED displays the backlight comes out of power-off (DPMS) with minimal backlight brightness. This seems to be a DCP firmware issue. It logs "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" to syslog although the brightness in the swap_submit call is valid. This fixes the issue only for clients using swap. For other clients an atomic backlight update has to be scheduled via a work queue. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 17d4602b1cb006..571e950ec787d4 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -696,6 +696,17 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) }; } +static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, + u8 *enabled) +{ + /* + * update backlight brightness on next swap, on non mini-LED displays + * DCP seems to set an invalid iDAC value after coming out of DPMS. + * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" + */ + dcp->brightness.update = true; +} + /* Chunked data transfer for property dictionaries */ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) { @@ -1251,6 +1262,8 @@ TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, dcpep_cb_swap_complete_intent_gated, struct dcp_swap_complete_intent_gated); +TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, + iomfbep_cb_enable_backlight_message_ap_gated, u8); TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, struct iomfb_property); @@ -1306,7 +1319,7 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [582] = trampoline_true, /* create_default_fb_surface */ [589] = trampoline_swap_complete, [591] = trampoline_swap_complete_intent_gated, - [593] = trampoline_nop, /* enable_backlight_message_ap_gated */ + [593] = trampoline_enable_backlight_message_ap_gated, [598] = trampoline_nop, /* find_swap_function_gated */ }; From b2d556890164e075d31cea2ceb85bf6a89f98dbd Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 27 Nov 2022 22:45:22 +0100 Subject: [PATCH 243/635] drm/apple: Report "PMUS.Temperature" only for mini-LED backlights Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp.c | 7 ++++++- drivers/gpu/drm/apple/iomfb.c | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 24db9f39d78e2a..7fe6490509f754 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -133,6 +133,9 @@ struct apple_dcp { /* eDP display without DP-HDMI conversion */ bool main_display; + /* panel has a mini-LED backllight */ + bool has_mini_led; + /* Modes valid for the connected display */ struct dcp_display_mode *modes; unsigned int nr_modes; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 03bb574a77bc24..015abb06ffd613 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -341,7 +341,12 @@ static int dcp_platform_probe(struct platform_device *pdev) /* intialize brightness scale to a sensible default to avoid divide by 0*/ dcp->brightness.scale = 65536; - panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + panel_np = of_get_compatible_child(dev->of_node, "apple,panel-mini-led"); + if (panel_np) + dcp->has_mini_led = true; + else + panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + if (panel_np) { const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 571e950ec787d4..c9e12d86c4930e 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -446,7 +446,8 @@ dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) .value = 0 }; - if (memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ + if (dcp->has_mini_led && + memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { /* * TODO: value from j314c, find out if it is temperature in From 4a49bc3fa54e286fefdfd1fd4fe4dc3fd339156b Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 4 Dec 2022 11:36:05 +0100 Subject: [PATCH 244/635] drm/apple: Check if DCP firmware is supported Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 8 +++ drivers/gpu/drm/apple/dcp.c | 77 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7fe6490509f754..18969ab18e8319 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -16,6 +16,11 @@ struct apple_dcp; +enum dcp_firmware_version { + DCP_FIRMWARE_UNKNOWN, + DCP_FIRMWARE_V_12_3, +}; + enum { SYSTEM_ENDPOINT = 0x20, TEST_ENDPOINT = 0x21, @@ -84,6 +89,9 @@ struct apple_dcp { struct apple_crtc *crtc; struct apple_connector *connector; + /* firmware version and compatible firmware version */ + enum dcp_firmware_version fw_compat; + /* Coprocessor control register */ void __iomem *coproc_reg; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 015abb06ffd613..8ac852619f37c3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3,8 +3,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -307,18 +309,93 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) return 0; } +#define DCP_FW_VERSION_MIN_LEN 3 +#define DCP_FW_VERSION_MAX_LEN 5 +#define DCP_FW_VERSION_STR_LEN (DCP_FW_VERSION_MAX_LEN * 4) + +static int dcp_read_fw_version(struct device *dev, const char *name, + char *version_str) +{ + u32 ver[DCP_FW_VERSION_MAX_LEN]; + int len_str; + int len; + + len = of_property_read_variable_u32_array(dev->of_node, name, ver, + DCP_FW_VERSION_MIN_LEN, + DCP_FW_VERSION_MAX_LEN); + + switch (len) { + case 3: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d", ver[0], ver[1], ver[2]); + break; + case 4: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3]); + break; + case 5: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3], ver[4]); + break; + default: + len_str = strscpy(version_str, "UNKNOWN", + DCP_FW_VERSION_STR_LEN); + if (len >= 0) + len = -EOVERFLOW; + break; + } + + if (len_str >= DCP_FW_VERSION_STR_LEN) + dev_warn(dev, "'%s' truncated: '%s'\n", name, version_str); + + return len; +} + +static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) +{ + char compat_str[DCP_FW_VERSION_STR_LEN]; + char fw_str[DCP_FW_VERSION_STR_LEN]; + int ret; + + /* firmware version is just informative */ + dcp_read_fw_version(dev, "apple,firmware-version", fw_str); + + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0) { + dev_err(dev, "Could not read 'apple,firmware-compat': %d\n", ret); + return DCP_FIRMWARE_UNKNOWN; + } + + if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_12_3; + + dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", + compat_str, fw_str); + + return DCP_FIRMWARE_UNKNOWN; +} + static int dcp_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *panel_np; struct apple_dcp *dcp; + enum dcp_firmware_version fw_compat; u32 cpu_ctrl; int ret; + fw_compat = dcp_check_firmware_version(dev); + if (fw_compat == DCP_FIRMWARE_UNKNOWN) + return -ENODEV; + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); if (!dcp) return -ENOMEM; + dcp->fw_compat = fw_compat; + platform_set_drvdata(pdev, dcp); dcp->dev = dev; From 68cf804cf602c04f70e0d15a8bdb268bc8b1952d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 27 Nov 2022 19:32:55 +0900 Subject: [PATCH 245/635] drm/apple: Disable fake vblank IRQ machinery The hardware does not have a vblank IRQ and drm already knows how to deal with that appropriately, so don't pretend it does. Fixes Xorg. Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/apple_drv.c | 23 ----------------------- drivers/gpu/drm/apple/dcp.c | 6 ------ 2 files changed, 29 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index bce91de706eb0f..0816c492519d7e 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -166,18 +166,6 @@ static struct drm_plane *apple_plane_init(struct drm_device *dev, return plane; } -static int apple_enable_vblank(struct drm_crtc *crtc) -{ - to_apple_crtc(crtc)->vsync_disabled = false; - - return 0; -} - -static void apple_disable_vblank(struct drm_crtc *crtc) -{ - to_apple_crtc(crtc)->vsync_disabled = true; -} - static enum drm_connector_status apple_connector_detect(struct drm_connector *connector, bool force) { @@ -199,7 +187,6 @@ static void apple_crtc_atomic_enable(struct drm_crtc *crtc, dcp_poweron(apple_crtc->dcp); dev_dbg(&apple_crtc->dcp->dev, "%s finished", __func__); } - drm_crtc_vblank_on(crtc); } static void apple_crtc_atomic_disable(struct drm_crtc *crtc, @@ -208,8 +195,6 @@ static void apple_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state; crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - drm_crtc_vblank_off(crtc); - if (crtc_state->active_changed && !crtc_state->active) { struct apple_crtc *apple_crtc = to_apple_crtc(crtc); dev_dbg(&apple_crtc->dcp->dev, "%s", __func__); @@ -233,8 +218,6 @@ static void apple_crtc_atomic_begin(struct drm_crtc *crtc, unsigned long flags; if (crtc->state->event) { - WARN_ON(drm_crtc_vblank_get(crtc) != 0); - spin_lock_irqsave(&crtc->dev->event_lock, flags); apple_crtc->event = crtc->state->event; spin_unlock_irqrestore(&crtc->dev->event_lock, flags); @@ -270,8 +253,6 @@ static const struct drm_crtc_funcs apple_crtc_funcs = { .page_flip = drm_atomic_helper_page_flip, .reset = drm_atomic_helper_crtc_reset, .set_config = drm_atomic_helper_set_config, - .enable_vblank = apple_enable_vblank, - .disable_vblank = apple_disable_vblank, }; static const struct drm_mode_config_funcs apple_mode_config_funcs = { @@ -406,10 +387,6 @@ static int apple_platform_probe(struct platform_device *pdev) dev_set_drvdata(dev, apple); - ret = drm_vblank_init(&apple->drm, nr_dcp); - if (ret) - return ret; - ret = drmm_mode_config_init(&apple->drm); if (ret) goto err_unload; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8ac852619f37c3..aa609ad1f508e1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -39,15 +39,9 @@ void dcp_drm_crtc_vblank(struct apple_crtc *crtc) { unsigned long flags; - if (crtc->vsync_disabled) - return; - - drm_crtc_handle_vblank(&crtc->base); - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); if (crtc->event) { drm_crtc_send_vblank_event(&crtc->base, crtc->event); - drm_crtc_vblank_put(&crtc->base); crtc->event = NULL; } spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); From 8d255857ee0443c54e44d49a2129084aff051b37 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 13:42:49 +0100 Subject: [PATCH 246/635] gpu: drm: apple: Parse color modes completely Selecting the mode with the highest score may result in HDR mode for some displays. Since HDR is not support on driver side this produces an unexpected color representation. Full parsing allows us to reject virtual color modes (should already be invalid due to missing "Score"). Preparation for color and timing mode tracing. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/parser.c | 41 ++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 31aecd4b2fc195..17060bd5e87ba6 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -248,6 +248,16 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) return 0; } +struct color_mode { + s64 colorimetry; + s64 depth; + s64 dynamic_range; + s64 eotf; + s64 id; + s64 pixel_encoding; + s64 score; +}; + static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) { struct iterator outer_it; @@ -258,17 +268,30 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) dcp_parse_foreach_in_array(handle, outer_it) { struct iterator it; - s64 score = -1, id = -1; + bool is_virtual = true; + struct color_mode cmode; dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); if (IS_ERR(key)) ret = PTR_ERR(key); - else if (!strcmp(key, "Score")) - ret = parse_int(it.handle, &score); + else if (!strcmp(key, "Colorimetry")) + ret = parse_int(it.handle, &cmode.colorimetry); + else if (!strcmp(key, "Depth")) + ret = parse_int(it.handle, &cmode.depth); + else if (!strcmp(key, "DynamicRange")) + ret = parse_int(it.handle, &cmode.dynamic_range); + else if (!strcmp(key, "EOTF")) + ret = parse_int(it.handle, &cmode.eotf); else if (!strcmp(key, "ID")) - ret = parse_int(it.handle, &id); + ret = parse_int(it.handle, &cmode.id); + else if (!strcmp(key, "IsVirtual")) + ret = parse_bool(it.handle, &is_virtual); + else if (!strcmp(key, "PixelEncoding")) + ret = parse_int(it.handle, &cmode.pixel_encoding); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, &cmode.score); else skip(it.handle); @@ -276,13 +299,13 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) return ret; } - /* Skip partial entries */ - if (score < 0 || id < 0) + /* Skip virtual or partial entries */ + if (is_virtual || cmode.score < 0 || cmode.id < 0) continue; - if (score > best_score) { - best_score = score; - *best_id = id; + if (cmode.score > best_score) { + best_score = cmode.score; + *best_id = cmode.id; } } From 5ac607f620cfe40a923fd610f21fd42871a0df52 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 14:11:30 +0100 Subject: [PATCH 247/635] gpu: drm: apple: Skip parsing elements of virtual timing modes Prevents trace points of unused color modes. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/parser.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 17060bd5e87ba6..d84c77be5fd78b 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -343,6 +343,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (IS_ERR(key)) ret = PTR_ERR(key); + else if (is_virtual) + skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) ret = parse_dimension(it.handle, &horiz); else if (!strcmp(key, "VerticalAttributes")) From 8bbace7869076a007a731dd1c08d0982490a2f6a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 13:58:53 +0100 Subject: [PATCH 248/635] gpu: drm: apple: Add tracing for color and timing modes Restrict symbol mapping for EOTF, pixel encoding and colorimetry to tracing due to low confidence that it is correct. Main concern is the colorimetry mapping. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 3 + drivers/gpu/drm/apple/parser.c | 11 +++ drivers/gpu/drm/apple/parser.h | 3 + drivers/gpu/drm/apple/trace.h | 120 +++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index c9e12d86c4930e..8be49c918e6d64 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -755,6 +755,9 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, return false; } + /* used just as opaque pointer for tracing */ + ctx.dcp = dcp; + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); if (ret) { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index d84c77be5fd78b..a253ec62640ba5 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -6,7 +6,9 @@ #include #include #include + #include "parser.h" +#include "trace.h" #define DCP_PARSE_HEADER 0xd3 @@ -303,6 +305,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) if (is_virtual || cmode.score < 0 || cmode.id < 0) continue; + trace_iomfb_color_mode(handle->dcp, cmode.id, cmode.score, + cmode.depth, cmode.colorimetry, + cmode.eotf, cmode.dynamic_range, + cmode.pixel_encoding); + if (cmode.score > best_score) { best_score = cmode.score; *best_id = cmode.id; @@ -419,6 +426,10 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->timing_mode_id = id; out->color_mode_id = best_color_mode; + trace_iomfb_timing_mode(handle->dcp, id, *score, horiz.active, + vert.active, vert.precise_sync_rate, + best_color_mode); + return 0; } diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index a2d479258ed0eb..92fe9473d56718 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -7,7 +7,10 @@ /* For mode parsing */ #include +struct apple_dcp; + struct dcp_parse_ctx { + struct apple_dcp *dcp; void *blob; u32 pos, len; }; diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index ddf6ff9839e4f7..b691fc5a472587 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -171,6 +171,126 @@ TRACE_EVENT(iomfb_brightness, ) ); +#define show_eotf(eotf) \ + __print_symbolic(eotf, { 0, "SDR gamma"}, \ + { 1, "HDR gamma"}, \ + { 2, "ST 2084 (PQ)"}, \ + { 3, "BT.2100 (HLG)"}, \ + { 4, "unexpected"}) + +#define show_encoding(enc) \ + __print_symbolic(enc, { 0, "RGB"}, \ + { 1, "YUV 4:2:0"}, \ + { 3, "YUV 4:2:2"}, \ + { 2, "YUV 4:4:4"}, \ + { 4, "DolbyVision (native)"}, \ + { 5, "DolbyVision (HDMI)"}, \ + { 6, "YCbCr 4:2:2 (DP tunnel)"}, \ + { 7, "YCbCr 4:2:2 (HDMI tunnel)"}, \ + { 8, "DolbyVision LL YCbCr 4:2:2"}, \ + { 9, "DolbyVision LL YCbCr 4:2:2 (DP)"}, \ + {10, "DolbyVision LL YCbCr 4:2:2 (HDMI)"}, \ + {11, "DolbyVision LL YCbCr 4:4:4"}, \ + {12, "DolbyVision LL RGB 4:2:2"}, \ + {13, "GRGB as YCbCr422 (Even line blue)"}, \ + {14, "GRGB as YCbCr422 (Even line red)"}, \ + {15, "unexpected"}) + +#define show_colorimetry(col) \ + __print_symbolic(col, { 0, "SMPTE 170M/BT.601"}, \ + { 1, "BT.701"}, \ + { 2, "xvYCC601"}, \ + { 3, "xvYCC709"}, \ + { 4, "sYCC601"}, \ + { 5, "AdobeYCC601"}, \ + { 6, "BT.2020 (c)"}, \ + { 7, "BT.2020 (nc)"}, \ + { 8, "DolbyVision VSVDB"}, \ + { 9, "BT.2020 (RGB)"}, \ + {10, "sRGB"}, \ + {11, "scRGB"}, \ + {12, "scRGBfixed"}, \ + {13, "AdobeRGB"}, \ + {14, "DCI-P3 (D65)"}, \ + {15, "DCI-P3 (Theater)"}, \ + {16, "Default RGB"}, \ + {17, "unexpected"}) + +#define show_range(range) \ + __print_symbolic(range, { 0, "Full"}, \ + { 1, "Limited"}, \ + { 2, "unexpected"}) + +TRACE_EVENT(iomfb_color_mode, + TP_PROTO(struct apple_dcp *dcp, u32 id, u32 score, u32 depth, + u32 colorimetry, u32 eotf, u32 range, u32 pixel_enc), + TP_ARGS(dcp, id, score, depth, colorimetry, eotf, range, pixel_enc), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, id) + __field(u32, score) + __field(u32, depth) + __field(u32, colorimetry) + __field(u32, eotf) + __field(u32, range) + __field(u32, pixel_enc) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->id = id; + __entry->score = score; + __entry->depth = depth; + __entry->colorimetry = min_t(u32, colorimetry, 17U); + __entry->eotf = min_t(u32, eotf, 4U); + __entry->range = min_t(u32, range, 2U); + __entry->pixel_enc = min_t(u32, pixel_enc, 15U); + ), + TP_printk("dcp=%llx, id=%u, score=%u, depth=%u, colorimetry=%s, eotf=%s, range=%s, pixel_enc=%s", + __entry->dcp, + __entry->id, + __entry->score, + __entry->depth, + show_colorimetry(__entry->colorimetry), + show_eotf(__entry->eotf), + show_range(__entry->range), + show_encoding(__entry->pixel_enc) + ) +); + +TRACE_EVENT(iomfb_timing_mode, + TP_PROTO(struct apple_dcp *dcp, u32 id, u32 score, u32 width, + u32 height, u32 clock, u32 color_mode), + TP_ARGS(dcp, id, score, width, height, clock, color_mode), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, id) + __field(u32, score) + __field(u32, width) + __field(u32, height) + __field(u32, clock) + __field(u32, color_mode) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->id = id; + __entry->score = score; + __entry->width = width; + __entry->height = height; + __entry->clock = clock; + __entry->color_mode = color_mode; + ), + TP_printk("dcp=%llx, id=%u, score=%u, %ux%u@%u.%u, color_mode=%u", + __entry->dcp, + __entry->id, + __entry->score, + __entry->width, + __entry->height, + __entry->clock >> 16, + ((__entry->clock & 0xffff) * 1000) >> 16, + __entry->color_mode + ) +); + #endif /* _TRACE_DCP_H */ /* This part must be outside protection */ From e679065ce59fd5bf5437599265c63c343689dc26 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 14:27:13 +0100 Subject: [PATCH 249/635] gpu: drm: apple: Prefer SDR color modes DCP best scored color mode might result in an HDR mode. As long as the driver (and DRM) is not ready for HDR try to avoid such modes. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/parser.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index a253ec62640ba5..678c0e42e10682 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -260,13 +260,14 @@ struct color_mode { s64 score; }; -static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) +static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *preferred_id) { struct iterator outer_it; int ret = 0; - s64 best_score = -1; + s64 best_score = -1, best_score_sdr = -1; + s64 best_id = -1, best_id_sdr = -1; - *best_id = -1; + *preferred_id = -1; dcp_parse_foreach_in_array(handle, outer_it) { struct iterator it; @@ -310,12 +311,25 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *best_id) cmode.eotf, cmode.dynamic_range, cmode.pixel_encoding); - if (cmode.score > best_score) { - best_score = cmode.score; - *best_id = cmode.id; + if (cmode.eotf == 0) { + if (cmode.score > best_score_sdr) { + best_score_sdr = cmode.score; + best_id_sdr = cmode.id; + } + } else { + if (cmode.score > best_score) { + best_score = cmode.score; + best_id = cmode.id; + } } } + /* prefer SDR color modes as long as HDR is not supported */ + if (best_score_sdr >= 0) + *preferred_id = best_id_sdr; + else if (best_score >= 0) + *preferred_id = best_id; + return 0; } From 5ac907ebc4c6f1fc0fa8c48f0ba715cfecc61f3a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 17:54:40 +0100 Subject: [PATCH 250/635] gpu: drm: apple: Add IOMobileFramebufferAP::get_color_remap_mode Probably not important but avoids an unnecessary difference compred to macOS. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 19 ++++++++++++++++++- drivers/gpu/drm/apple/iomfb.h | 12 ++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8be49c918e6d64..07a1179f75bc38 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -181,6 +181,7 @@ const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { DCP_METHOD("A410", dcpep_set_display_device), DCP_METHOD("A411", dcpep_is_main_display), DCP_METHOD("A412", dcpep_set_digital_out_mode), + DCP_METHOD("A426", iomfbep_get_color_remap_mode), DCP_METHOD("A439", dcpep_set_parameter_dcp), DCP_METHOD("A443", dcpep_create_default_fb), DCP_METHOD("A447", dcpep_enable_disable_video_power_savings), @@ -261,10 +262,21 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, cb, cookie); \ } +#define IOMFB_THUNK_INOUT(name, T_in, T_out) \ + static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, T_in *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, iomfbep_ ## name, sizeof(T_in), sizeof(T_out), \ + data, cb, cookie); \ + } + DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); +IOMFB_THUNK_INOUT(get_color_remap_mode, struct iomfb_get_color_remap_mode_req, + struct iomfb_get_color_remap_mode_resp); + DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, struct dcp_swap_submit_resp); @@ -1825,9 +1837,14 @@ static void init_1(struct apple_dcp *dcp, void *out, void *cookie) static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { + struct iomfb_get_color_remap_mode_req color_remap = + (struct iomfb_get_color_remap_mode_req){ + .mode = 6, + }; + dev_info(dcp->dev, "DCP booted\n"); - init_1(dcp, data, cookie); + iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); } void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 386a84cfcc5cd1..083ebd4e0be33f 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -210,6 +210,7 @@ enum dcpep_method { iomfbep_a131_pmu_service_matched, iomfbep_a132_backlight_service_matched, iomfbep_a358_vi_set_temperature_hint, + iomfbep_get_color_remap_mode, dcpep_num_methods }; @@ -433,4 +434,15 @@ struct iomfb_property { u32 value; } __packed; +struct iomfb_get_color_remap_mode_req { + u32 mode; + u8 mode_null; + u8 padding[3]; +} __packed; + +struct iomfb_get_color_remap_mode_resp { + u32 mode; + u32 ret; +} __packed; + #endif From fbc7c1c7be8c3159e8e52e1a9a8c8be0d1e64540 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 14:39:02 +0100 Subject: [PATCH 251/635] gpu: drm: apple: reenable support for {A,X}RGB2101010 Seems to work now with 'surface.colorspace = 12'. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 0816c492519d7e..8c3d54b2f117e1 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -132,8 +132,8 @@ static const struct drm_plane_funcs apple_plane_funcs = { * advertise formats without alpha. */ static const u32 dcp_formats[] = { - // DRM_FORMAT_XRGB2101010, - // DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 07a1179f75bc38..f8f202acd51a77 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1503,7 +1503,7 @@ static u8 drm_format_to_colorspace(u32 drm) case DRM_FORMAT_ARGB2101010: case DRM_FORMAT_XRGB2101010: - return 2; + return 12; } return 1; From 374eb1ae54fd89fd4d999ace941952994aa7d285 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 23:48:37 +0100 Subject: [PATCH 252/635] gpu: drm: apple: Add show_notch module parameter Can be used on devices with camera notch to use the full display height and thus show the notch. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index aa609ad1f508e1..46d52d80abc665 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,10 @@ #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) +static bool show_notch; +module_param(show_notch, bool, 0644); +MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); + /* HACK: moved here to avoid circular dependency between apple_drv and dcp */ void dcp_drm_crtc_vblank(struct apple_crtc *crtc) { @@ -403,8 +408,10 @@ static int dcp_platform_probe(struct platform_device *pdev) of_platform_default_populate(dev->of_node, NULL, dev); - ret = of_property_read_u32(dev->of_node, "apple,notch-height", - &dcp->notch_height); + if (!show_notch) + ret = of_property_read_u32(dev->of_node, "apple,notch-height", + &dcp->notch_height); + if (dcp->notch_height > MAX_NOTCH_HEIGHT) dcp->notch_height = MAX_NOTCH_HEIGHT; if (dcp->notch_height > 0) From 8895630f0f8e59eed0e28eb072e7921eac650f3d Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 13 Dec 2022 00:38:56 +0100 Subject: [PATCH 253/635] Revert "gpu: drm: apple: reenable support for {A,X}RGB2101010" This reverts commit d39542179a8f5a5d2e80eebf77bc739857a1051c. 'w30r' is a wide gammut mode. As long as the display is SDR DCP will end up displaying picture correctly but on HDR displays like the display in the Macbook Pro 14"/16" (2021) or external displays with HDR EOTF the picture has oversaturated / black colors. The non-wide gammut 10-bit per component pixelformats "l10r" and "R10k" are not supported. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8c3d54b2f117e1..0816c492519d7e 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -132,8 +132,8 @@ static const struct drm_plane_funcs apple_plane_funcs = { * advertise formats without alpha. */ static const u32 dcp_formats[] = { - DRM_FORMAT_XRGB2101010, - DRM_FORMAT_ARGB2101010, + // DRM_FORMAT_XRGB2101010, + // DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index f8f202acd51a77..07a1179f75bc38 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1503,7 +1503,7 @@ static u8 drm_format_to_colorspace(u32 drm) case DRM_FORMAT_ARGB2101010: case DRM_FORMAT_XRGB2101010: - return 12; + return 2; } return 1; From b5ea2edd5fe6bdbe65a75b12d3eaee6c2d69558b Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 26 Dec 2022 00:26:05 +0900 Subject: [PATCH 254/635] drm/apple: Enable 10-bit mode & set colorspace to native This works on both 8-bit and 10-bit modes without any weirdness, and gives us the native colorspace without any conversion. Color correction should probably be handled in software anyway. However, we need to use surface 1 (at least on t600x), since 0 seems stuck in bg-sRGB mode for some reason... Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/apple_drv.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 24 ++++-------------------- drivers/gpu/drm/apple/iomfb.h | 11 +++++++++++ 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 0816c492519d7e..8c3d54b2f117e1 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -132,8 +132,8 @@ static const struct drm_plane_funcs apple_plane_funcs = { * advertise formats without alpha. */ static const u32 dcp_formats[] = { - // DRM_FORMAT_XRGB2101010, - // DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 07a1179f75bc38..6188a37035a662 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1492,23 +1492,6 @@ static u32 drm_format_to_dcp(u32 drm) return 0; } -static u8 drm_format_to_colorspace(u32 drm) -{ - switch (drm) { - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ABGR8888: - return 1; - - case DRM_FORMAT_ARGB2101010: - case DRM_FORMAT_XRGB2101010: - return 2; - } - - return 1; -} - int dcp_get_modes(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); @@ -1630,7 +1613,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; - l = 0; + // Surface 0 has limitations at least on t600x. + l = 1; for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { struct drm_framebuffer *fb = new_state->fb; struct drm_gem_dma_object *obj; @@ -1697,8 +1681,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->surf[l] = (struct dcp_surface){ .opaque = opaque, .format = drm_format_to_dcp(fb->format->format), - .xfer_func = 13, - .colorspace = drm_format_to_colorspace(fb->format->format), + .xfer_func = DCP_XFER_FUNC_SDR, + .colorspace = DCP_COLORSPACE_NATIVE, .stride = fb->pitches[0], .width = fb->width, .height = fb->height, diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 083ebd4e0be33f..90b6668b075000 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -74,6 +74,17 @@ enum iomfb_property_id { #define SWAP_SURFACES 4 #define MAX_PLANES 3 +enum dcp_colorspace { + DCP_COLORSPACE_BG_SRGB = 0, + DCP_COLORSPACE_BG_BT2020 = 9, + DCP_COLORSPACE_NATIVE = 12, +}; + +enum dcp_xfer_func { + DCP_XFER_FUNC_SDR = 13, + DCP_XFER_FUNC_HDR = 16, +}; + struct dcp_iouserclient { /* Handle for the IOUserClient. macOS sets this to a kernel VA. */ u64 handle; From 36b2cadf70d710370bc71bfd23d837273adc6a14 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 31 Dec 2022 14:53:59 +0100 Subject: [PATCH 255/635] gpu: drm: apple: Clear all surfaces on startup With "drm/apple: Enable 10-bit mode & set colorspace to native" kernel log messages are shown in an Apple logo shaped region in the middle of the display when using BGRA. The field currently identified as "opaque" is mislabeled and has to be investigated further. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/iomfb.c | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 18969ab18e8319..8ca2324cb038c5 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -141,6 +141,9 @@ struct apple_dcp { /* eDP display without DP-HDMI conversion */ bool main_display; + /* clear all surfaces on init */ + bool surfaces_cleared; + /* panel has a mini-LED backllight */ bool has_mini_led; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 6188a37035a662..b58bd0097f4155 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1613,6 +1613,15 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) for (l = 0; l < SWAP_SURFACES; l++) req->surf_null[l] = true; + /* + * Clear all surfaces on startup. The boot framebuffer in surface 0 + * sticks around. + */ + if (!dcp->surfaces_cleared) { + req->swap.swap_enabled = DCP_REMOVE_LAYERS | 0xF; + dcp->surfaces_cleared = true; + } + // Surface 0 has limitations at least on t600x. l = 1; for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { From 397ea221fc0f39eaaab99fd08bb31378b7c85d1a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 2 Jan 2023 23:37:51 +0100 Subject: [PATCH 256/635] drm/apple: Update swap handling - opaque -> is_premultiplied - swap_enabled BIT(31) seems to be update background with dcp_swap.bg_color - add unused fields is_tearing_allowed, ycbcr_matrix, protection_opts, unk_num, unk_denom Changes: use is_premultiplied only for XRGB8/XBGR8, Update background only when necessary. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 37 +++++++++++++++++++++-------------- drivers/gpu/drm/apple/iomfb.h | 23 ++++++++++------------ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b58bd0097f4155..92d366e6be558b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1062,9 +1062,9 @@ void dcp_poweroff(struct platform_device *pdev) // clear surfaces memset(&dcp->swap, 0, sizeof(dcp->swap)); - dcp->swap.swap.swap_enabled = DCP_REMOVE_LAYERS | 0x7; - dcp->swap.swap.swap_completed = DCP_REMOVE_LAYERS | 0x7; - dcp->swap.swap.unk_10c = 0xFF000000; + dcp->swap.swap.swap_enabled = + dcp->swap.swap.swap_completed = IOMFB_SET_BACKGROUND | 0xF; + dcp->swap.swap.bg_color = 0xFF000000; /* * Turn off the backlight. This matters because the DCP's idea of @@ -1618,7 +1618,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) * sticks around. */ if (!dcp->surfaces_cleared) { - req->swap.swap_enabled = DCP_REMOVE_LAYERS | 0xF; + req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0xF; + req->swap.bg_color = 0xFF000000; dcp->surfaces_cleared = true; } @@ -1628,7 +1629,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) struct drm_framebuffer *fb = new_state->fb; struct drm_gem_dma_object *obj; struct drm_rect src_rect; - bool opaque = false; + bool is_premultiplied = false; /* skip planes not for this crtc */ if (old_state->crtc != crtc && new_state->crtc != crtc) @@ -1659,18 +1660,21 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) } if (!new_state->fb) { - if (old_state->fb) - req->swap.swap_enabled |= DCP_REMOVE_LAYERS; - l += 1; continue; } req->surf_null[l] = false; has_surface = 1; - if (!fb->format->has_alpha || - new_state->plane->type == DRM_PLANE_TYPE_PRIMARY) - opaque = true; + /* + * DCP doesn't support XBGR8 / XRGB8 natively. Blending as + * pre-multiplied alpha with a black background can be used as + * workaround for the bottommost plane. + */ + if (fb->format->format == DRM_FORMAT_XRGB8888 || + fb->format->format == DRM_FORMAT_XBGR8888) + is_premultiplied = true; + drm_rect_fp_to_int(&src_rect, &new_state->src); req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); @@ -1688,7 +1692,7 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) req->surf_iova[l] = obj->dma_addr + fb->offsets[0]; req->surf[l] = (struct dcp_surface){ - .opaque = opaque, + .is_premultiplied = is_premultiplied, .format = drm_format_to_dcp(fb->format->format), .xfer_func = DCP_XFER_FUNC_SDR, .colorspace = DCP_COLORSPACE_NATIVE, @@ -1709,9 +1713,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) l += 1; } - /* These fields should be set together */ - req->swap.swap_completed = req->swap.swap_enabled; - if (modeset) { struct dcp_display_mode *mode; struct dcp_wait_cookie *cookie; @@ -1772,9 +1773,15 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) return; } + /* Set black background */ + req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; + req->swap.bg_color = 0xFF000000; req->clear = 1; } + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + /* update brightness if changed */ if (dcp->brightness.update) { req->swap.bl_unk = 1; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 90b6668b075000..a7e9b62425b2c4 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -102,12 +102,9 @@ struct dcp_rect { } __packed; /* - * Set in the swap_{enabled,completed} field to remove missing - * layers. Without this flag, the DCP will assume missing layers have - * not changed since the previous frame and will preserve their - * content. - */ -#define DCP_REMOVE_LAYERS BIT(31) + * Update background color to struct dcp_swap.bg_color + */ +#define IOMFB_SET_BACKGROUND BIT(31) struct dcp_swap { u64 ts1; @@ -126,7 +123,7 @@ struct dcp_swap { u32 swap_enabled; u32 swap_completed; - u32 unk_10c; + u32 bg_color; u8 unk_110[0x1b8]; u32 unk_2c8; u8 unk_2cc[0x14]; @@ -160,12 +157,12 @@ struct dcp_component_types { /* Information describing a surface */ struct dcp_surface { u8 is_tiled; - u8 unk_1; - u8 opaque; /** ignore alpha, also required YUV overlays */ + u8 is_tearing_allowed; + u8 is_premultiplied; u32 plane_cnt; u32 plane_cnt2; u32 format; /* DCP fourcc */ - u32 unk_f; + u32 ycbcr_matrix; u8 xfer_func; u8 colorspace; u32 stride; @@ -176,8 +173,7 @@ struct dcp_surface { u32 width; u32 height; u32 buf_size; - u32 unk_2d; - u32 unk_31; + u64 protection_opts; u32 surface_id; struct dcp_component_types comp_types[MAX_PLANES]; u64 has_comp; @@ -185,7 +181,8 @@ struct dcp_surface { u64 has_planes; u32 compression_info[MAX_PLANES][13]; u64 has_compr_info; - u64 unk_1f5; + u32 unk_num; + u32 unk_denom; u8 padding[7]; } __packed; From 8f40958e5ec20da3719542ea6ac6782cc310f916 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 22 Dec 2022 23:58:44 +0100 Subject: [PATCH 257/635] gpu: drm: apple: Use drm_aperture_remove_conflicting_framebuffers This does not seem to be as racy as drm_aperture_remove_framebuffers() and seems to reliably takes over simpledrm's device node. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 53 ++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8c3d54b2f117e1..847aa7e63c290a 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -339,10 +340,45 @@ static int apple_probe_per_dcp(struct device *dev, return drm_connector_attach_encoder(&connector->base, encoder); } +static int apple_get_fb_resource(struct device *dev, const char *name, + struct resource *fb_r) +{ + int idx, ret = -ENODEV; + struct device_node *node; + + idx = of_property_match_string(dev->of_node, "memory-region-names", name); + + node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!node) { + dev_err(dev, "reserved-memory node '%s' not found\n", name); + return -ENODEV; + } + + if (!of_device_is_available(node)) { + dev_err(dev, "reserved-memory node '%s' is unavailable\n", name); + goto err; + } + + if (!of_device_is_compatible(node, "framebuffer")) { + dev_err(dev, "reserved-memory node '%s' is incompatible\n", + node->full_name); + goto err; + } + + ret = of_address_to_resource(node, 0, fb_r); + +err: + of_node_put(node); + return ret; +} + + static int apple_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_drm_private *apple; + struct resource fb_r; + resource_size_t fb_size; struct platform_device *dcp[MAX_COPROCESSORS]; int ret, nr_dcp, i; @@ -380,6 +416,18 @@ static int apple_platform_probe(struct platform_device *pdev) if (ret) return ret; + ret = apple_get_fb_resource(dev, "framebuffer", &fb_r); + if (ret) + return ret; + + fb_size = fb_r.end - fb_r.start + 1; + ret = drm_aperture_remove_conflicting_framebuffers(fb_r.start, fb_size, + false, &apple_drm_driver); + if (ret) { + dev_err(dev, "Failed remove fb: %d\n", ret); + return ret; + } + apple = devm_drm_dev_alloc(dev, &apple_drm_driver, struct apple_drm_private, drm); if (IS_ERR(apple)) @@ -425,11 +473,6 @@ static int apple_platform_probe(struct platform_device *pdev) drm_mode_config_reset(&apple->drm); - // remove before registering our DRM device - ret = drm_aperture_remove_framebuffers(false, &apple_drm_driver); - if (ret) - return ret; - ret = drm_dev_register(&apple->drm, 0); if (ret) goto err_unload; From c22d0b072bd3c04cdb4f8fa74ee13f34e44525ba Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 31 Dec 2022 15:27:07 +0100 Subject: [PATCH 258/635] drm/apple: Use drm_module_platform_driver This check for the "nomodeset" kernel command line parameter in its register method. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 3 ++- drivers/gpu/drm/apple/dcp.c | 3 ++- drivers/gpu/drm/apple/dummy-piodma.c | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 847aa7e63c290a..318f6896d4f427 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -535,7 +536,7 @@ static struct platform_driver apple_platform_driver = { .remove = apple_platform_remove, }; -module_platform_driver(apple_platform_driver); +drm_module_platform_driver(apple_platform_driver); MODULE_AUTHOR("Alyssa Rosenzweig "); MODULE_DESCRIPTION(DRIVER_DESC); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 46d52d80abc665..af008b3c5e0db3 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -538,7 +539,7 @@ static struct platform_driver apple_platform_driver = { }, }; -module_platform_driver(apple_platform_driver); +drm_module_platform_driver(apple_platform_driver); MODULE_AUTHOR("Alyssa Rosenzweig "); MODULE_DESCRIPTION("Apple Display Controller DRM driver"); diff --git a/drivers/gpu/drm/apple/dummy-piodma.c b/drivers/gpu/drm/apple/dummy-piodma.c index 3d4454df4a25da..daf4327592a041 100644 --- a/drivers/gpu/drm/apple/dummy-piodma.c +++ b/drivers/gpu/drm/apple/dummy-piodma.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include + #include #include #include @@ -24,7 +26,7 @@ static struct platform_driver dcp_piodma_platform_driver = { }, }; -module_platform_driver(dcp_piodma_platform_driver); +drm_module_platform_driver(dcp_piodma_platform_driver); MODULE_AUTHOR("Alyssa Rosenzweig "); MODULE_DESCRIPTION("[HACK] Apple DCP PIODMA shim"); From fefa4702295ef593dd1a7a4f5fabd83946b93523 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 2 Jan 2023 19:32:07 +0100 Subject: [PATCH 259/635] drm/apple: Allocate drm objects according to drm's expectations drm's documentaion explicitly tells us not to use devm_kzalloc(). drm device structures might out live the device when they are in use by userspace while the device vanishes. --- drivers/gpu/drm/apple/apple_drv.c | 43 +++++++++++++++++++++---------- drivers/gpu/drm/apple/dcp.h | 7 +++++ 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 318f6896d4f427..15400b5bf6e50e 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -113,10 +113,16 @@ static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { .atomic_update = apple_plane_atomic_update, }; +static void apple_plane_cleanup(struct drm_plane *plane) +{ + drm_plane_cleanup(plane); + kfree(plane); +} + static const struct drm_plane_funcs apple_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, - .destroy = drm_plane_cleanup, + .destroy = apple_plane_cleanup, .reset = drm_atomic_helper_plane_reset, .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, @@ -154,7 +160,7 @@ static struct drm_plane *apple_plane_init(struct drm_device *dev, int ret; struct drm_plane *plane; - plane = devm_kzalloc(dev->dev, sizeof(*plane), GFP_KERNEL); + plane = kzalloc(sizeof(*plane), GFP_KERNEL); ret = drm_universal_plane_init(dev, plane, possible_crtcs, &apple_plane_funcs, @@ -247,11 +253,16 @@ static void dcp_atomic_commit_tail(struct drm_atomic_state *old_state) drm_atomic_helper_cleanup_planes(dev, old_state); } +static void apple_crtc_cleanup(struct drm_crtc *crtc) +{ + drm_crtc_cleanup(crtc); + kfree(to_apple_crtc(crtc)); +} static const struct drm_crtc_funcs apple_crtc_funcs = { .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, - .destroy = drm_crtc_cleanup, + .destroy = apple_crtc_cleanup, .page_flip = drm_atomic_helper_page_flip, .reset = drm_atomic_helper_crtc_reset, .set_config = drm_atomic_helper_set_config, @@ -267,9 +278,15 @@ static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { .atomic_commit_tail = dcp_atomic_commit_tail, }; +static void appledrm_connector_cleanup(struct drm_connector *connector) +{ + drm_connector_cleanup(connector); + kfree(to_apple_connector(connector)); +} + static const struct drm_connector_funcs apple_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = drm_connector_cleanup, + .destroy = appledrm_connector_cleanup, .reset = drm_atomic_helper_connector_reset, .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, @@ -297,7 +314,7 @@ static int apple_probe_per_dcp(struct device *dev, { struct apple_crtc *crtc; struct apple_connector *connector; - struct drm_encoder *encoder; + struct apple_encoder *enc; struct drm_plane *primary; int ret; @@ -306,7 +323,7 @@ static int apple_probe_per_dcp(struct device *dev, if (IS_ERR(primary)) return PTR_ERR(primary); - crtc = devm_kzalloc(dev, sizeof(*crtc), GFP_KERNEL); + crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); ret = drm_crtc_init_with_planes(drm, &crtc->base, primary, NULL, &apple_crtc_funcs, NULL); if (ret) @@ -314,13 +331,13 @@ static int apple_probe_per_dcp(struct device *dev, drm_crtc_helper_add(&crtc->base, &apple_crtc_helper_funcs); - encoder = devm_kzalloc(dev, sizeof(*encoder), GFP_KERNEL); - encoder->possible_crtcs = drm_crtc_mask(&crtc->base); - ret = drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS); - if (ret) - return ret; + enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, + DRM_MODE_ENCODER_TMDS); + if (IS_ERR(enc)) + return PTR_ERR(enc); + enc->base.possible_crtcs = drm_crtc_mask(&crtc->base); - connector = devm_kzalloc(dev, sizeof(*connector), GFP_KERNEL); + connector = kzalloc(sizeof(*connector), GFP_KERNEL); drm_connector_helper_add(&connector->base, &apple_connector_helper_funcs); @@ -338,7 +355,7 @@ static int apple_probe_per_dcp(struct device *dev, crtc->dcp = dcp; dcp_link(dcp, crtc, connector); - return drm_connector_attach_encoder(&connector->base, encoder); + return drm_connector_attach_encoder(&connector->base, &enc->base); } static int apple_get_fb_resource(struct device *dev, const char *name, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index fb4397e7b390fe..f4476f4acaf265 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -5,6 +5,7 @@ #define __APPLE_DCP_H__ #include +#include #include #include "dcp-internal.h" @@ -35,6 +36,12 @@ struct apple_connector { #define to_apple_connector(x) container_of(x, struct apple_connector, base) +struct apple_encoder { + struct drm_encoder base; +}; + +#define to_apple_encoder(x) container_of(x, struct apple_encoder, base) + void dcp_poweroff(struct platform_device *pdev); void dcp_poweron(struct platform_device *pdev); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); From f64bbf9667909f60ee692ee9d8fb1a9c30ccbab3 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 29 Dec 2022 21:05:40 +0100 Subject: [PATCH 260/635] gpu: drm: apple: Use components to avoid deferred probing There was a report of a race between DRM device registration (and removal of the simpledrm device) and GDM startup. The component based device binding ensures that all necessary devices are bind in the probe method of the last missing component. Technically the piodma-mapper should be a component of dcp but since it is only used for its iommu it can be a component of the display subsystem. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 197 ++++++++++++++++++++------- drivers/gpu/drm/apple/dcp-internal.h | 1 - drivers/gpu/drm/apple/dcp.c | 107 +++++++++------ drivers/gpu/drm/apple/dummy-piodma.c | 39 +++++- 4 files changed, 249 insertions(+), 95 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 15400b5bf6e50e..6627b3457f4163 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -7,8 +7,9 @@ * Copyright (C) 2014 Endless Mobile */ -#include +#include #include +#include #include #include @@ -390,46 +391,55 @@ static int apple_get_fb_resource(struct device *dev, const char *name, return ret; } +static const struct of_device_id apple_dcp_id_tbl[] = { + { .compatible = "apple,dcp" }, + {}, +}; -static int apple_platform_probe(struct platform_device *pdev) +static int apple_drm_init_dcp(struct device *dev) { - struct device *dev = &pdev->dev; - struct apple_drm_private *apple; - struct resource fb_r; - resource_size_t fb_size; + struct apple_drm_private *apple = dev_get_drvdata(dev); struct platform_device *dcp[MAX_COPROCESSORS]; - int ret, nr_dcp, i; - - for (nr_dcp = 0; nr_dcp < MAX_COPROCESSORS; ++nr_dcp) { - struct device_node *np; - struct device_link *dcp_link; + struct device_node *np; + int ret, num_dcp = 0; - np = of_parse_phandle(dev->of_node, "apple,coprocessors", - nr_dcp); - - if (!np) - break; + for_each_matching_node(np, apple_dcp_id_tbl) { + if (!of_device_is_available(np)) { + of_node_put(np); + continue; + } - dcp[nr_dcp] = of_find_device_by_node(np); + dcp[num_dcp] = of_find_device_by_node(np); + of_node_put(np); + if (!dcp[num_dcp]) + continue; - if (!dcp[nr_dcp]) - return -ENODEV; + ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], + num_dcp); + if (ret) + continue; - dcp_link = device_link_add(dev, &dcp[nr_dcp]->dev, - DL_FLAG_AUTOREMOVE_CONSUMER); - if (!dcp_link) { - dev_err(dev, "Failed to link to DCP %d device", nr_dcp); - return -EINVAL; - } + ret = dcp_start(dcp[num_dcp]); + if (ret) + continue; - if (dcp_link->supplier->links.status != DL_DEV_DRIVER_BOUND) - return -EPROBE_DEFER; + num_dcp++; } - /* Need at least 1 DCP for a display subsystem */ - if (nr_dcp < 1) + if (num_dcp < 1) return -ENODEV; + + return 0; +} + +static int apple_drm_init(struct device *dev) +{ + struct apple_drm_private *apple; + struct resource fb_r; + resource_size_t fb_size; + int ret; + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); if (ret) return ret; @@ -438,14 +448,6 @@ static int apple_platform_probe(struct platform_device *pdev) if (ret) return ret; - fb_size = fb_r.end - fb_r.start + 1; - ret = drm_aperture_remove_conflicting_framebuffers(fb_r.start, fb_size, - false, &apple_drm_driver); - if (ret) { - dev_err(dev, "Failed remove fb: %d\n", ret); - return ret; - } - apple = devm_drm_dev_alloc(dev, &apple_drm_driver, struct apple_drm_private, drm); if (IS_ERR(apple)) @@ -453,6 +455,18 @@ static int apple_platform_probe(struct platform_device *pdev) dev_set_drvdata(dev, apple); + ret = component_bind_all(dev, apple); + if (ret) + return ret; + + fb_size = fb_r.end - fb_r.start + 1; + ret = drm_aperture_remove_conflicting_framebuffers(fb_r.start, fb_size, + false, &apple_drm_driver); + if (ret) { + dev_err(dev, "Failed remove fb: %d\n", ret); + goto err_unbind; + } + ret = drmm_mode_config_init(&apple->drm); if (ret) goto err_unload; @@ -477,17 +491,9 @@ static int apple_platform_probe(struct platform_device *pdev) apple->drm.mode_config.funcs = &apple_mode_config_funcs; apple->drm.mode_config.helper_private = &apple_mode_config_helpers; - for (i = 0; i < nr_dcp; ++i) { - ret = apple_probe_per_dcp(dev, &apple->drm, dcp[i], i); - - if (ret) - goto err_unload; - - ret = dcp_start(dcp[i]); - - if (ret) - goto err_unload; - } + ret = apple_drm_init_dcp(dev); + if (ret) + goto err_unload; drm_mode_config_reset(&apple->drm); @@ -501,14 +507,96 @@ static int apple_platform_probe(struct platform_device *pdev) err_unload: drm_dev_put(&apple->drm); +err_unbind: + component_unbind_all(dev, NULL); return ret; } -static int apple_platform_remove(struct platform_device *pdev) +static void apple_drm_uninit(struct device *dev) { - struct apple_drm_private *apple = platform_get_drvdata(pdev); + struct apple_drm_private *apple = dev_get_drvdata(dev); drm_dev_unregister(&apple->drm); + drm_atomic_helper_shutdown(&apple->drm); + + component_unbind_all(dev, NULL); + + dev_set_drvdata(dev, NULL); +} + +static int apple_drm_bind(struct device *dev) +{ + return apple_drm_init(dev); +} + +static void apple_drm_unbind(struct device *dev) +{ + apple_drm_uninit(dev); +} + +const struct component_master_ops apple_drm_ops = { + .bind = apple_drm_bind, + .unbind = apple_drm_unbind, +}; + +static const struct of_device_id apple_component_id_tbl[] = { + { .compatible = "apple,dcp-piodma" }, + {}, +}; + +static int add_display_components(struct device *dev, + struct component_match **matchptr) +{ + struct device_node *np; + + for_each_matching_node(np, apple_component_id_tbl) { + if (of_device_is_available(np)) + drm_of_component_match_add(dev, matchptr, + component_compare_of, np); + of_node_put(np); + } + + return 0; +} + +static int add_dcp_components(struct device *dev, + struct component_match **matchptr) +{ + struct device_node *np; + int num = 0; + + for_each_matching_node(np, apple_dcp_id_tbl) { + if (of_device_is_available(np)) { + drm_of_component_match_add(dev, matchptr, + component_compare_of, np); + num++; + } + of_node_put(np); + } + + return num; +} + +static int apple_platform_probe(struct platform_device *pdev) +{ + struct device *mdev = &pdev->dev; + struct component_match *match = NULL; + int num_dcp; + + /* add PIODMA mapper components */ + add_display_components(mdev, &match); + + /* add DCP components, handle less than 1 as probe error */ + num_dcp = add_dcp_components(mdev, &match); + if (num_dcp < 1) + return -ENODEV; + + return component_master_add_with_match(mdev, &apple_drm_ops, match); +} + +static int apple_platform_remove(struct platform_device *pdev) +{ + component_master_del(&pdev->dev, &apple_drm_ops); return 0; } @@ -524,14 +612,19 @@ static int apple_platform_suspend(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - return drm_mode_config_helper_suspend(&apple->drm); + if (apple) + return drm_mode_config_helper_suspend(&apple->drm); + + return 0; } static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - drm_mode_config_helper_resume(&apple->drm); + if (apple) + drm_mode_config_helper_resume(&apple->drm); + return 0; } diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 8ca2324cb038c5..ed11d38f80bb59 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -84,7 +84,6 @@ struct dcp_brightness { struct apple_dcp { struct device *dev; struct platform_device *piodma; - struct device_link *piodma_link; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index af008b3c5e0db3..cbf090d4d1c025 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1,22 +1,23 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include +#include #include #include +#include +#include +#include +#include +#include #include #include #include -#include -#include #include -#include -#include -#include -#include -#include +#include #include -#include -#include "linux/workqueue.h" +#include +#include #include #include @@ -377,33 +378,18 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) return DCP_FIRMWARE_UNKNOWN; } -static int dcp_platform_probe(struct platform_device *pdev) +static int dcp_comp_bind(struct device *dev, struct device *main, void *data) { - struct device *dev = &pdev->dev; struct device_node *panel_np; - struct apple_dcp *dcp; - enum dcp_firmware_version fw_compat; + struct apple_dcp *dcp = dev_get_drvdata(dev); u32 cpu_ctrl; int ret; - fw_compat = dcp_check_firmware_version(dev); - if (fw_compat == DCP_FIRMWARE_UNKNOWN) - return -ENODEV; - - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); - if (!dcp) - return -ENOMEM; - - dcp->fw_compat = fw_compat; - - platform_set_drvdata(pdev, dcp); - dcp->dev = dev; - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); if (ret) return ret; - dcp->coproc_reg = devm_platform_ioremap_resource_byname(pdev, "coproc"); + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); if (IS_ERR(dcp->coproc_reg)) return PTR_ERR(dcp->coproc_reg); @@ -452,22 +438,17 @@ static int dcp_platform_probe(struct platform_device *pdev) else dcp->connector_type = DRM_MODE_CONNECTOR_Unknown; + /* + * Components do not ensure the bind order of sub components but + * the piodma device is only used for its iommu. The iommu is fully + * initialized by the time dcp_piodma_probe() calls component_add(). + */ dcp->piodma = dcp_get_dev(dev, "apple,piodma-mapper"); if (!dcp->piodma) { dev_err(dev, "failed to find piodma\n"); return -ENODEV; } - dcp->piodma_link = device_link_add(dev, &dcp->piodma->dev, - DL_FLAG_AUTOREMOVE_CONSUMER); - if (!dcp->piodma_link) { - dev_err(dev, "Failed to link to piodma device"); - return -EINVAL; - } - - if (dcp->piodma_link->supplier->links.status != DL_DEV_DRIVER_BOUND) - return -EPROBE_DEFER; - ret = dcp_get_disp_regs(dcp); if (ret) { dev_err(dev, "failed to find display registers\n"); @@ -516,12 +497,57 @@ static int dcp_platform_probe(struct platform_device *pdev) * We need to shutdown DCP before tearing down the display subsystem. Otherwise * the DCP will crash and briefly flash a green screen of death. */ -static void dcp_platform_shutdown(struct platform_device *pdev) +static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_dcp *dcp = dev_get_drvdata(dev); - if (dcp->shmem) + if (dcp && dcp->shmem) iomfb_shutdown(dcp); + + platform_device_put(dcp->piodma); + dcp->piodma = NULL; + + devm_clk_put(dev, dcp->clk); + dcp->clk = NULL; +} + +static const struct component_ops dcp_comp_ops = { + .bind = dcp_comp_bind, + .unbind = dcp_comp_unbind, +}; + +static int dcp_platform_probe(struct platform_device *pdev) +{ + enum dcp_firmware_version fw_compat; + struct device *dev = &pdev->dev; + struct apple_dcp *dcp; + + fw_compat = dcp_check_firmware_version(dev); + if (fw_compat == DCP_FIRMWARE_UNKNOWN) + return -ENODEV; + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + + dcp->fw_compat = fw_compat; + dcp->dev = dev; + + platform_set_drvdata(pdev, dcp); + + return component_add(&pdev->dev, &dcp_comp_ops); +} + +static int dcp_platform_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_comp_ops); + + return 0; +} + +static void dcp_platform_shutdown(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_comp_ops); } static const struct of_device_id of_match[] = { @@ -532,6 +558,7 @@ MODULE_DEVICE_TABLE(of, of_match); static struct platform_driver apple_platform_driver = { .probe = dcp_platform_probe, + .remove = dcp_platform_remove, .shutdown = dcp_platform_shutdown, .driver = { .name = "apple-dcp", diff --git a/drivers/gpu/drm/apple/dummy-piodma.c b/drivers/gpu/drm/apple/dummy-piodma.c index daf4327592a041..ec5d4fecf356c0 100644 --- a/drivers/gpu/drm/apple/dummy-piodma.c +++ b/drivers/gpu/drm/apple/dummy-piodma.c @@ -3,13 +3,46 @@ #include -#include +#include #include +#include #include +static int dcp_piodma_comp_bind(struct device *dev, struct device *main, + void *data) +{ + return 0; +} + +static void dcp_piodma_comp_unbind(struct device *dev, struct device *main, + void *data) +{ + /* nothing to do */ +} + +static const struct component_ops dcp_piodma_comp_ops = { + .bind = dcp_piodma_comp_bind, + .unbind = dcp_piodma_comp_unbind, +}; static int dcp_piodma_probe(struct platform_device *pdev) { - return dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); + int ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); + if (ret) + return ret; + + return component_add(&pdev->dev, &dcp_piodma_comp_ops); +} + +static int dcp_piodma_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_piodma_comp_ops); + + return 0; +} + +static void dcp_piodma_shutdown(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_piodma_comp_ops); } static const struct of_device_id of_match[] = { @@ -20,6 +53,8 @@ MODULE_DEVICE_TABLE(of, of_match); static struct platform_driver dcp_piodma_platform_driver = { .probe = dcp_piodma_probe, + .remove = dcp_piodma_remove, + .shutdown = dcp_piodma_shutdown, .driver = { .name = "apple,dcp-piodma", .of_match_table = of_match, From 3f393ab91b759d5d187969443e4035d13f73f60c Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 29 Dec 2022 21:38:44 +0100 Subject: [PATCH 261/635] gpu: drm: apple: Wait for iomfb initialization Avoids "[drm] Cannot find any crtc or sizes" during fbdev initialization if a display is connected. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 19 ++++++++++++++++++- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp.c | 26 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 5 +++-- 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 6627b3457f4163..a8daf46b94754e 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -401,7 +402,8 @@ static int apple_drm_init_dcp(struct device *dev) struct apple_drm_private *apple = dev_get_drvdata(dev); struct platform_device *dcp[MAX_COPROCESSORS]; struct device_node *np; - int ret, num_dcp = 0; + u64 timeout; + int i, ret, num_dcp = 0; for_each_matching_node(np, apple_dcp_id_tbl) { if (!of_device_is_available(np)) { @@ -429,6 +431,21 @@ static int apple_drm_init_dcp(struct device *dev) if (num_dcp < 1) return -ENODEV; + timeout = get_jiffies_64() + msecs_to_jiffies(500); + + for (i = 0; i < num_dcp; ++i) { + u64 jiffies = get_jiffies_64(); + u64 wait = time_after_eq64(jiffies, timeout) ? + 0 : + timeout - jiffies; + ret = dcp_wait_ready(dcp[i], wait); + /* There is nothing we can do if a dcp/dcpext does not boot + * (successfully). Ignoring it should not do any harm now. + * Needs to reevaluated whenn adding dcpext support. + */ + if (ret) + dev_warn(dev, "DCP[%d] not ready: %d\n", i, ret); + } return 0; } diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ed11d38f80bb59..85e9137bbdf10e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -134,6 +134,9 @@ struct apple_dcp { bool valid_mode; struct dcp_set_digital_out_mode_req mode; + /* completion for active turning true */ + struct completion start_done; + /* Is the DCP booted? */ bool active; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index cbf090d4d1c025..4afc4ce53593bb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -117,6 +117,7 @@ static void dcp_rtk_crashed(void *cookie) dcp->connector->connected = 0; schedule_work(&dcp->connector->hotplug_wq); } + complete(&dcp->start_done); } static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) @@ -248,6 +249,8 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + init_completion(&dcp->start_done); + /* start RTKit endpoints */ ret = iomfb_start_rtkit(dcp); if (ret) @@ -257,6 +260,29 @@ int dcp_start(struct platform_device *pdev) } EXPORT_SYMBOL(dcp_start); +int dcp_wait_ready(struct platform_device *pdev, u64 timeout) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; + + if (dcp->crashed) + return -ENODEV; + if (dcp->active) + return 0; + if (timeout <= 0) + return -ETIMEDOUT; + + ret = wait_for_completion_timeout(&dcp->start_done, timeout); + if (ret < 0) + return ret; + + if (dcp->crashed) + return -ENODEV; + + return dcp->active ? 0 : -ETIMEDOUT; +} +EXPORT_SYMBOL(dcp_wait_ready); + static void dcp_work_register_backlight(struct work_struct *work) { int ret; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index f4476f4acaf265..2011d27e809d53 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -49,6 +49,7 @@ int dcp_get_connector_type(struct platform_device *pdev); void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, struct apple_connector *connector); int dcp_start(struct platform_device *pdev); +int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); bool dcp_is_initialized(struct platform_device *pdev); void apple_crtc_vblank(struct apple_crtc *apple); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 92d366e6be558b..4f1161b7da5d15 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1810,13 +1810,14 @@ static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) dcp->main_display = result != 0; - dcp->active = true; - connector = dcp->connector; if (connector) { connector->connected = dcp->nr_modes > 0; schedule_work(&connector->hotplug_wq); } + + dcp->active = true; + complete(&dcp->start_done); } static void init_3(struct apple_dcp *dcp, void *out, void *cookie) From 81af9415bde8dd69500a612c1d3722e89475bc3a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 8 Jan 2023 21:24:51 +0100 Subject: [PATCH 262/635] drm/apple: simplify IOMFB_THUNK_INOUT Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 4f1161b7da5d15..ed126b10b209c2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -262,20 +262,22 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, cb, cookie); \ } -#define IOMFB_THUNK_INOUT(name, T_in, T_out) \ - static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, T_in *data, \ - dcp_callback_t cb, void *cookie) \ - { \ - dcp_push(dcp, oob, iomfbep_ ## name, sizeof(T_in), sizeof(T_out), \ - data, cb, cookie); \ +#define IOMFB_THUNK_INOUT(name) \ + static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, \ + struct iomfb_ ## name ## _req *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, iomfbep_ ## name, \ + sizeof(struct iomfb_ ## name ## _req), \ + sizeof(struct iomfb_ ## name ## _resp), \ + data, cb, cookie); \ } DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); -IOMFB_THUNK_INOUT(get_color_remap_mode, struct iomfb_get_color_remap_mode_req, - struct iomfb_get_color_remap_mode_resp); +IOMFB_THUNK_INOUT(get_color_remap_mode); DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, struct dcp_swap_submit_resp); From f9fe72729bcb8e2de2c81518ae550e2a74cb5555 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Feb 2023 20:57:56 +0900 Subject: [PATCH 263/635] drm/apple: Fix parse_string() memory leaks Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/parser.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 678c0e42e10682..1d85d76ae0e16f 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -243,6 +243,9 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) else skip(it.handle); + if (!IS_ERR_OR_NULL(key)) + kfree(key); + if (ret) return ret; } @@ -298,6 +301,9 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, s64 *preferred_id) else skip(it.handle); + if (!IS_ERR_OR_NULL(key)) + kfree(key); + if (ret) return ret; } @@ -381,6 +387,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, else skip(it.handle); + if (!IS_ERR_OR_NULL(key)) + kfree(key); + if (ret) return ret; } @@ -511,6 +520,9 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, else skip(it.handle); + if (!IS_ERR_OR_NULL(key)) + kfree(key); + if (ret) return ret; } From 77ba8c6ad5c7ad64fd5357306f3284a5fda4a752 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Feb 2023 20:57:47 +0900 Subject: [PATCH 264/635] drm/apple: Fix bad error return Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 1d85d76ae0e16f..5665c2ba19682f 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -229,7 +229,7 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) char *key = parse_string(it.handle); if (IS_ERR(key)) - ret = PTR_ERR(handle); + ret = PTR_ERR(key); else if (!strcmp(key, "Active")) ret = parse_int(it.handle, &dim->active); else if (!strcmp(key, "Total")) From 358811f025b225dcaa0e9ed89697f62bcb94d551 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 22 Jan 2023 20:16:28 +0100 Subject: [PATCH 265/635] drm/apple: Set backlight level indirectly if no mode is set Fixes following warning when systemd-backlight restores the backlight level on boot before a mode is set: Call trace: drm_atomic_helper_crtc_duplicate_state+0x58/0x74 drm_atomic_get_crtc_state+0x84/0x120 dcp_set_brightness+0xd8/0x21c [apple_dcp] backlight_device_set_brightness+0x78/0x130 ... Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp_backlight.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 42b1097eaa0180..45827fe6041a27 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -165,21 +165,30 @@ static int drm_crtc_set_brightness(struct drm_crtc *crtc, static int dcp_set_brightness(struct backlight_device *bd) { - int ret; + int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); struct drm_modeset_acquire_ctx ctx; if (bd->props.state & BL_CORE_SUSPENDED) return 0; - if (!dcp->crtc) - return -EAGAIN; + DRM_MODESET_LOCK_ALL_BEGIN(dcp->crtc->base.dev, ctx, 0, ret); dcp->brightness.dac = calculate_dac(dcp, bd->props.brightness); dcp->brightness.update = true; - DRM_MODESET_LOCK_ALL_BEGIN(dcp->crtc->base.dev, ctx, 0, ret); + /* + * Do not actively try to change brightness if no mode is set. + * TODO: should this be reflected the in backlight's power property? + * defer this hopefully until it becomes irrelevant due to proper + * drm integrated backlight handling + */ + if (!dcp->valid_mode) + goto out; + ret = drm_crtc_set_brightness(&dcp->crtc->base, &ctx); + +out: DRM_MODESET_LOCK_ALL_END(dcp->crtc->base.dev, ctx, ret); return ret; From adee091cd9e0ff3ed9876157047bc2b58f7d543b Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 22 Jan 2023 19:43:35 +0100 Subject: [PATCH 266/635] drm/apple: Use backlight_get_brightness() Backlight drivers are expected to use this instead of accessing backlight properties. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp_backlight.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 45827fe6041a27..d063ecd7ad2068 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -168,13 +168,11 @@ static int dcp_set_brightness(struct backlight_device *bd) int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); struct drm_modeset_acquire_ctx ctx; - - if (bd->props.state & BL_CORE_SUSPENDED) - return 0; + int brightness = backlight_get_brightness(bd); DRM_MODESET_LOCK_ALL_BEGIN(dcp->crtc->base.dev, ctx, 0, ret); - dcp->brightness.dac = calculate_dac(dcp, bd->props.brightness); + dcp->brightness.dac = calculate_dac(dcp, brightness); dcp->brightness.update = true; /* From 9d700cbcbd0583b1066738b334be4f2f007f7a25 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 26 Mar 2023 15:56:03 +0200 Subject: [PATCH 267/635] drm/apple: Move panel options to its own sub-struct Fixes overwriting the panel's physical dimensions on poweroff/sleep. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/dcp-internal.h | 16 +++++++++++++--- drivers/gpu/drm/apple/dcp.c | 21 ++++++++++++++------- drivers/gpu/drm/apple/iomfb.c | 2 +- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 85e9137bbdf10e..b34294816ec1ff 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -80,6 +80,16 @@ struct dcp_brightness { bool update; }; +/** laptop/AiO integrated panel parameters from DT */ +struct dcp_panel { + /// panel width in millimeter + int width_mm; + /// panel height in millimeter + int height_mm; + /// panel has a mini-LED backllight + bool has_mini_led; +}; + /* TODO: move IOMFB members to its own struct */ struct apple_dcp { struct device *dev; @@ -146,9 +156,6 @@ struct apple_dcp { /* clear all surfaces on init */ bool surfaces_cleared; - /* panel has a mini-LED backllight */ - bool has_mini_led; - /* Modes valid for the connected display */ struct dcp_display_mode *modes; unsigned int nr_modes; @@ -173,6 +180,9 @@ struct apple_dcp { /* Workqueue for updating the initial initial brightness */ struct work_struct bl_register_wq; struct mutex bl_register_mutex; + + /* integrated panel if present */ + struct dcp_panel panel; }; int dcp_backlight_register(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4afc4ce53593bb..730e351209f1f6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -57,14 +57,21 @@ void dcp_drm_crtc_vblank(struct apple_crtc *crtc) void dcp_set_dimensions(struct apple_dcp *dcp) { int i; + int width_mm = dcp->width_mm; + int height_mm = dcp->height_mm; + + if (width_mm == 0 || height_mm == 0) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } /* Set the connector info */ if (dcp->connector) { struct drm_connector *connector = &dcp->connector->base; mutex_lock(&connector->dev->mode_config.mutex); - connector->display_info.width_mm = dcp->width_mm; - connector->display_info.height_mm = dcp->height_mm; + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; mutex_unlock(&connector->dev->mode_config.mutex); } @@ -74,8 +81,8 @@ void dcp_set_dimensions(struct apple_dcp *dcp) * DisplayAttributes, and TimingElements may be sent first */ for (i = 0; i < dcp->nr_modes; ++i) { - dcp->modes[i].mode.width_mm = dcp->width_mm; - dcp->modes[i].mode.height_mm = dcp->height_mm; + dcp->modes[i].mode.width_mm = width_mm; + dcp->modes[i].mode.height_mm = height_mm; } } @@ -434,7 +441,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp->brightness.scale = 65536; panel_np = of_get_compatible_child(dev->of_node, "apple,panel-mini-led"); if (panel_np) - dcp->has_mini_led = true; + dcp->panel.has_mini_led = true; else panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); @@ -448,10 +455,10 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dev_err(dev, "Missing property 'apple,max-brightness'\n"); } - of_property_read_u32(panel_np, "width-mm", &dcp->width_mm); + of_property_read_u32(panel_np, "width-mm", &dcp->panel.width_mm); /* use adjusted height as long as the notch is hidden */ of_property_read_u32(panel_np, height_prop[!dcp->notch_height], - &dcp->height_mm); + &dcp->panel.height_mm); of_node_put(panel_np); dcp->connector_type = DRM_MODE_CONNECTOR_eDP; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ed126b10b209c2..89651cea1454d6 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -460,7 +460,7 @@ dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) .value = 0 }; - if (dcp->has_mini_led && + if (dcp->panel.has_mini_led && memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { /* From 9580e457dc71935304ed3bd7d484b7d896d612ad Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 22 Mar 2023 16:09:09 +0900 Subject: [PATCH 268/635] drm/apple: Align buffers to 16K page size Signed-off-by: Asahi Lina --- drivers/gpu/drm/apple/apple_drv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index a8daf46b94754e..a4ae7afbad73bb 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -49,12 +49,14 @@ struct apple_drm_private { DEFINE_DRM_GEM_DMA_FOPS(apple_fops); +#define DART_PAGE_SIZE 16384 + static int apple_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, struct drm_mode_create_dumb *args) { args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); - args->size = args->pitch * args->height; + args->size = round_up(args->pitch * args->height, DART_PAGE_SIZE); return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } From 59f823f79ea6c6acf4bd1d29d6336ea5d96a809a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 28 Feb 2023 20:34:03 +0100 Subject: [PATCH 269/635] drm/apple: purge unused dcp_update_notify_clients_dcp Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 5 ----- drivers/gpu/drm/apple/iomfb.h | 18 ------------------ 2 files changed, 23 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 89651cea1454d6..2591c7b4a3cdb0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -170,7 +170,6 @@ static u8 dcp_pop_depth(u8 *depth) const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { DCP_METHOD("A000", dcpep_late_init_signal), DCP_METHOD("A029", dcpep_setup_video_limits), - DCP_METHOD("A034", dcpep_update_notify_clients_dcp), DCP_METHOD("A131", iomfbep_a131_pmu_service_matched), DCP_METHOD("A132", iomfbep_a132_backlight_service_matched), DCP_METHOD("A357", dcpep_set_create_dfb), @@ -305,10 +304,6 @@ DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); -__attribute__((unused)) -DCP_THUNK_IN(dcp_update_notify_clients_dcp, dcpep_update_notify_clients_dcp, - struct dcp_update_notify_clients_dcp); - DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, struct dcp_set_parameter_dcp, u32); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index a7e9b62425b2c4..54b34fbba94894 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -211,7 +211,6 @@ enum dcpep_method { dcpep_flush_supports_power, dcpep_set_power_state, dcpep_first_client_open, - dcpep_update_notify_clients_dcp, dcpep_set_parameter_dcp, dcpep_enable_disable_video_power_savings, dcpep_is_main_display, @@ -395,23 +394,6 @@ struct dcp_set_dcpav_prop_end_req { char key[0x40]; } __packed; -struct dcp_update_notify_clients_dcp { - u32 client_0; - u32 client_1; - u32 client_2; - u32 client_3; - u32 client_4; - u32 client_5; - u32 client_6; - u32 client_7; - u32 client_8; - u32 client_9; - u32 client_a; - u32 client_b; - u32 client_c; - u32 client_d; -} __packed; - struct dcp_set_parameter_dcp { u32 param; u32 value[8]; From e675b34944b60f6736d5f8ec77591ddb794fc653 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 8 Jan 2023 21:30:22 +0100 Subject: [PATCH 270/635] drm/apple: Add callbacks triggered by last_client_close_dcp() Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 2591c7b4a3cdb0..962b3c619254d4 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1330,9 +1330,14 @@ bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, [576] = trampoline_hotplug, [577] = trampoline_nop, /* powerstate_notify */ [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ [589] = trampoline_swap_complete, [591] = trampoline_swap_complete_intent_gated, [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ [598] = trampoline_nop, /* find_swap_function_gated */ }; From 1a5b5d6ae3bc721b6031eac4e638806b46589b46 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 17 Feb 2023 23:17:10 +0100 Subject: [PATCH 271/635] drm/apple: Add support for the macOS 13.2 DCP firmware This adds support for multiple incompatible DCP firmware versions. The approach taken here duplicates more than necessary. Unmodified calls do not need to be templated. For simplicity and in the expectation that more calls and callbacks are modified in the future everything is templated. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/Makefile | 2 + drivers/gpu/drm/apple/dcp-internal.h | 11 +- drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/iomfb.c | 1475 ++---------------------- drivers/gpu/drm/apple/iomfb.h | 123 +- drivers/gpu/drm/apple/iomfb_internal.h | 123 ++ drivers/gpu/drm/apple/iomfb_template.c | 1344 +++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.h | 181 +++ drivers/gpu/drm/apple/iomfb_v12_3.c | 105 ++ drivers/gpu/drm/apple/iomfb_v12_3.h | 17 + drivers/gpu/drm/apple/iomfb_v13_2.c | 105 ++ drivers/gpu/drm/apple/iomfb_v13_2.h | 17 + drivers/gpu/drm/apple/version_utils.h | 14 + 13 files changed, 2023 insertions(+), 1496 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_internal.h create mode 100644 drivers/gpu/drm/apple/iomfb_template.c create mode 100644 drivers/gpu/drm/apple/iomfb_template.h create mode 100644 drivers/gpu/drm/apple/iomfb_v12_3.c create mode 100644 drivers/gpu/drm/apple/iomfb_v12_3.h create mode 100644 drivers/gpu/drm/apple/iomfb_v13_2.c create mode 100644 drivers/gpu/drm/apple/iomfb_v13_2.h create mode 100644 drivers/gpu/drm/apple/version_utils.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 2c02e4dcfd076d..45ef064b3e6fa5 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -5,6 +5,8 @@ CFLAGS_trace.o = -I$(src) appledrm-y := apple_drv.o apple_dcp-y := dcp.o dcp_backlight.o iomfb.o parser.o +apple_dcp-y += iomfb_v12_3.o +apple_dcp-y += iomfb_v13_2.o apple_dcp-$(CONFIG_TRACING) += trace.o apple_piodma-y := dummy-piodma.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index b34294816ec1ff..59777809a7f370 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -11,6 +11,8 @@ #include #include "iomfb.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_2.h" #define DCP_MAX_PLANES 2 @@ -19,6 +21,7 @@ struct apple_dcp; enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, + DCP_FIRMWARE_V_13_2, }; enum { @@ -134,11 +137,17 @@ struct apple_dcp { struct dcp_channel ch_cmd, ch_oobcmd; struct dcp_channel ch_cb, ch_oobcb, ch_async; + /* iomfb EP callback handlers */ + const iomfb_cb_handler *cb_handlers; + /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ - struct dcp_swap_submit_req swap; + union { + struct dcp_swap_submit_req_v12_3 v12_3; + struct dcp_swap_submit_req_v13_2 v13_2; + } swap; /* Current display mode */ bool valid_mode; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 730e351209f1f6..d9d847ac819d14 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -404,6 +404,8 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) return DCP_FIRMWARE_V_12_3; + if (strncmp(compat_str, "13.2.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_13_2; dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", compat_str, fw_str); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 962b3c619254d4..868ed2598a4d7f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1,19 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright 2021 Alyssa Rosenzweig */ +#include #include #include -#include -#include -#include +#include #include #include #include #include -#include -#include +#include +#include +#include +#include #include -#include #include #include @@ -28,25 +28,6 @@ #include "parser.h" #include "trace.h" -/* Register defines used in bandwidth setup structure */ -#define REG_SCRATCH (0x14) -#define REG_SCRATCH_T600X (0x988) -#define REG_DOORBELL (0x0) -#define REG_DOORBELL_BIT (2) - -struct dcp_wait_cookie { - struct kref refcount; - struct completion done; -}; - -static void release_wait_cookie(struct kref *ref) -{ - struct dcp_wait_cookie *cookie; - cookie = container_of(ref, struct dcp_wait_cookie, refcount); - - kfree(cookie); -} - static int dcp_tx_offset(enum dcp_context_id id) { switch (id) { @@ -165,33 +146,8 @@ static u8 dcp_pop_depth(u8 *depth) return --(*depth); } -#define DCP_METHOD(tag, name) [name] = { #name, tag } - -const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { - DCP_METHOD("A000", dcpep_late_init_signal), - DCP_METHOD("A029", dcpep_setup_video_limits), - DCP_METHOD("A131", iomfbep_a131_pmu_service_matched), - DCP_METHOD("A132", iomfbep_a132_backlight_service_matched), - DCP_METHOD("A357", dcpep_set_create_dfb), - DCP_METHOD("A358", iomfbep_a358_vi_set_temperature_hint), - DCP_METHOD("A401", dcpep_start_signal), - DCP_METHOD("A407", dcpep_swap_start), - DCP_METHOD("A408", dcpep_swap_submit), - DCP_METHOD("A410", dcpep_set_display_device), - DCP_METHOD("A411", dcpep_is_main_display), - DCP_METHOD("A412", dcpep_set_digital_out_mode), - DCP_METHOD("A426", iomfbep_get_color_remap_mode), - DCP_METHOD("A439", dcpep_set_parameter_dcp), - DCP_METHOD("A443", dcpep_create_default_fb), - DCP_METHOD("A447", dcpep_enable_disable_video_power_savings), - DCP_METHOD("A454", dcpep_first_client_open), - DCP_METHOD("A460", dcpep_set_display_refresh_properties), - DCP_METHOD("A463", dcpep_flush_supports_power), - DCP_METHOD("A468", dcpep_set_power_state), -}; - /* Call a DCP function given by a tag */ -static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { @@ -203,10 +159,10 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, .out_len = out_len, /* Tag is reversed due to endianness of the fourcc */ - .tag[0] = dcp_methods[method].tag[3], - .tag[1] = dcp_methods[method].tag[2], - .tag[2] = dcp_methods[method].tag[1], - .tag[3] = dcp_methods[method].tag[0], + .tag[0] = call->tag[3], + .tag[1] = call->tag[2], + .tag[2] = call->tag[1], + .tag[3] = call->tag[0], }; u8 depth = dcp_push_depth(&ch->depth); @@ -221,7 +177,7 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, if (in_len > 0) memcpy(out_data, data, in_len); - trace_iomfb_push(dcp, &dcp_methods[method], context, offset, depth); + trace_iomfb_push(dcp, call, context, offset, depth); ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; @@ -232,88 +188,8 @@ static void dcp_push(struct apple_dcp *dcp, bool oob, enum dcpep_method method, dcpep_msg(context, data_len, offset)); } -#define DCP_THUNK_VOID(func, handle) \ - static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ - void *cookie) \ - { \ - dcp_push(dcp, oob, handle, 0, 0, NULL, cb, cookie); \ - } - -#define DCP_THUNK_OUT(func, handle, T) \ - static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ - void *cookie) \ - { \ - dcp_push(dcp, oob, handle, 0, sizeof(T), NULL, cb, cookie); \ - } - -#define DCP_THUNK_IN(func, handle, T) \ - static void func(struct apple_dcp *dcp, bool oob, T *data, \ - dcp_callback_t cb, void *cookie) \ - { \ - dcp_push(dcp, oob, handle, sizeof(T), 0, data, cb, cookie); \ - } - -#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ - static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ - dcp_callback_t cb, void *cookie) \ - { \ - dcp_push(dcp, oob, handle, sizeof(T_in), sizeof(T_out), data, \ - cb, cookie); \ - } - -#define IOMFB_THUNK_INOUT(name) \ - static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, \ - struct iomfb_ ## name ## _req *data, \ - dcp_callback_t cb, void *cookie) \ - { \ - dcp_push(dcp, oob, iomfbep_ ## name, \ - sizeof(struct iomfb_ ## name ## _req), \ - sizeof(struct iomfb_ ## name ## _resp), \ - data, cb, cookie); \ - } - -DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); -DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); -DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); - -IOMFB_THUNK_INOUT(get_color_remap_mode); - -DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct dcp_swap_submit_req, - struct dcp_swap_submit_resp); - -DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, - struct dcp_swap_start_resp); - -DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, - struct dcp_set_power_state_req, - struct dcp_set_power_state_resp); - -DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, - struct dcp_set_digital_out_mode_req, u32); - -DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); - -DCP_THUNK_OUT(dcp_set_display_refresh_properties, - dcpep_set_display_refresh_properties, u32); - -DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); -DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); -DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); -DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); -DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); -DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); -DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); - -DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, - struct dcp_set_parameter_dcp, u32); - -DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, - dcpep_enable_disable_video_power_savings, u32, int); - -DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); - /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ -static int dcp_parse_tag(char tag[4]) +int dcp_parse_tag(char tag[4]) { u32 d[3]; int i; @@ -332,7 +208,7 @@ static int dcp_parse_tag(char tag[4]) } /* Ack a callback from the DCP */ -static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) +void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) { struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -341,776 +217,54 @@ static void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) dcpep_ack(context)); } -/* DCP callback handlers */ -static void dcpep_cb_nop(struct apple_dcp *dcp) -{ - /* No operation */ -} - -static u8 dcpep_cb_true(struct apple_dcp *dcp) -{ - return true; -} - -static u8 dcpep_cb_false(struct apple_dcp *dcp) -{ - return false; -} - -static u32 dcpep_cb_zero(struct apple_dcp *dcp) -{ - return 0; -} - -static void dcpep_cb_swap_complete(struct apple_dcp *dcp, - struct dc_swap_complete_resp *resp) -{ - trace_iomfb_swap_complete(dcp, resp->swap_id); - - dcp_drm_crtc_vblank(dcp->crtc); -} - -/* special */ -static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) -{ - // ack D100 cb_match_pmu_service - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *out, void *in) -{ - trace_iomfb_callback(dcp, tag, __func__); - iomfb_a358_vi_set_temperature_hint(dcp, false, - complete_vi_set_temperature_hint, - NULL); - - // return false for deferred ACK - return false; -} - -static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +void dcp_sleep(struct apple_dcp *dcp) { - struct dcp_channel *ch = &dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - - // ack D206 cb_match_pmu_service_2 - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -static bool iomfbep_cb_match_pmu_service_2(struct apple_dcp *dcp, int tag, void *out, void *in) -{ - trace_iomfb_callback(dcp, tag, __func__); - - iomfb_a131_pmu_service_matched(dcp, false, complete_pmu_service_matched, - out); - - // return false for deferred ACK - return false; -} - -static void complete_backlight_service_matched(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_channel *ch = &dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - - // ack D206 cb_match_backlight_service - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, void *out, void *in) -{ - trace_iomfb_callback(dcp, tag, __func__); - - iomfb_a132_backlight_service_matched(dcp, false, complete_backlight_service_matched, out); - - // return false for deferred ACK - return false; -} - -static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) -{ - switch (prop->id) { - case IOMFB_PROPERTY_NITS: - { - dcp->brightness.nits = prop->value / dcp->brightness.scale; - /* notify backlight device of the initial brightness */ - if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) - schedule_work(&dcp->bl_register_wq); - trace_iomfb_brightness(dcp, prop->value); + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_sleep_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_2: + iomfb_sleep_v13_2(dcp); break; - } default: - dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); - } -} - -static struct dcp_get_uint_prop_resp -dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) -{ - struct dcp_get_uint_prop_resp resp = (struct dcp_get_uint_prop_resp){ - .value = 0 - }; - - if (dcp->panel.has_mini_led && - memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ - if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { - /* - * TODO: value from j314c, find out if it is temperature in - * centigrade C and which temperature sensor reports it - */ - resp.value = 3029; - resp.ret = true; - } - } - - return resp; -} - -static u8 iomfbep_cb_sr_set_property_int(struct apple_dcp *dcp, - struct iomfb_sr_set_property_int_req *req) -{ - if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ - if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { - if (!req->value_null) - dcp->brightness.scale = req->value; - } - } - - return 1; -} - -static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) -{ - // TODO: trace this, see if there properties which needs to used later -} - -/* - * Callback to map a buffer allocated with allocate_buf for PIODMA usage. - * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated - * stream of the display DART, rather than the expected DCP DART. - * - * XXX: This relies on dma_get_sgtable in concert with dma_map_sgtable, which - * is a "fundamentally unsafe" operation according to the docs. And yet - * everyone does it... - */ -static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, - struct dcp_map_buf_req *req) -{ - struct sg_table *map; - int ret; - - if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) - goto reject; - - map = &dcp->memdesc[req->buffer].map; - - if (!map->sgl) - goto reject; - - /* Use PIODMA device instead of DCP to map against the right IOMMU. */ - ret = dma_map_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); - - if (ret) - goto reject; - - return (struct dcp_map_buf_resp){ .dva = sg_dma_address(map->sgl) }; - -reject: - dev_err(dcp->dev, "denying map of invalid buffer %llx for pidoma\n", - req->buffer); - return (struct dcp_map_buf_resp){ .ret = EINVAL }; -} - -static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, - struct dcp_unmap_buf_resp *resp) -{ - struct sg_table *map; - dma_addr_t dma_addr; - - if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { - dev_warn(dcp->dev, "unmap request for out of range buffer %llu", - resp->buffer); - return; - } - - map = &dcp->memdesc[resp->buffer].map; - - if (!map->sgl) { - dev_warn(dcp->dev, - "unmap for non-mapped buffer %llu iova:0x%08llx", - resp->buffer, resp->dva); - return; - } - - dma_addr = sg_dma_address(map->sgl); - if (dma_addr != resp->dva) { - dev_warn(dcp->dev, "unmap buffer %llu address mismatch dma_addr:%llx dva:%llx", - resp->buffer, dma_addr, resp->dva); - return; - } - - /* Use PIODMA device instead of DCP to unmap from the right IOMMU. */ - dma_unmap_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); -} - -/* - * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be - * physically contigiuous, however we should save the sgtable in case the - * buffer needs to be later mapped for PIODMA. - */ -static struct dcp_allocate_buffer_resp -dcpep_cb_allocate_buffer(struct apple_dcp *dcp, - struct dcp_allocate_buffer_req *req) -{ - struct dcp_allocate_buffer_resp resp = { 0 }; - struct dcp_mem_descriptor *memdesc; - u32 id; - - resp.dva_size = ALIGN(req->size, 4096); - resp.mem_desc_id = - find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - - if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { - dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring"); - resp.dva_size = 0; - resp.mem_desc_id = 0; - return resp; - } - id = resp.mem_desc_id; - set_bit(id, dcp->memdesc_map); - - memdesc = &dcp->memdesc[id]; - - memdesc->size = resp.dva_size; - memdesc->buf = dma_alloc_coherent(dcp->dev, memdesc->size, - &memdesc->dva, GFP_KERNEL); - - dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, - memdesc->size); - resp.dva = memdesc->dva; - - return resp; -} - -static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) -{ - struct dcp_mem_descriptor *memdesc; - u32 id = *mem_desc_id; - - if (id >= DCP_MAX_MAPPINGS) { - dev_warn(dcp->dev, - "unmap request for out of range mem_desc_id %u", id); - return 0; - } - - if (!test_and_clear_bit(id, dcp->memdesc_map)) { - dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u", - id); - return 0; - } - - memdesc = &dcp->memdesc[id]; - if (memdesc->buf) { - dma_free_coherent(dcp->dev, memdesc->size, memdesc->buf, - memdesc->dva); - - memdesc->buf = NULL; - memset(&memdesc->map, 0, sizeof(memdesc->map)); - } else { - memdesc->reg = 0; - } - - memdesc->size = 0; - - return 1; -} - -/* Validate that the specified region is a display register */ -static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) -{ - int i; - - for (i = 0; i < dcp->nr_disp_registers; ++i) { - struct resource *r = dcp->disp_registers[i]; - - if ((start >= r->start) && (end <= r->end)) - return true; - } - - return false; -} - -/* - * Map contiguous physical memory into the DCP's address space. The firmware - * uses this to map the display registers we advertise in - * sr_map_device_memory_with_index, so we bounds check against that to guard - * safe against malicious coprocessors. - */ -static struct dcp_map_physical_resp -dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) -{ - int size = ALIGN(req->size, 4096); - u32 id; - - if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { - dev_err(dcp->dev, "refusing to map phys address %llx size %llx", - req->paddr, req->size); - return (struct dcp_map_physical_resp){}; - } - - id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - set_bit(id, dcp->memdesc_map); - dcp->memdesc[id].size = size; - dcp->memdesc[id].reg = req->paddr; - - return (struct dcp_map_physical_resp){ - .dva_size = size, - .mem_desc_id = id, - .dva = dma_map_resource(dcp->dev, req->paddr, size, - DMA_BIDIRECTIONAL, 0), - }; -} - -static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) -{ - return clk_get_rate(dcp->clk); -} - -static struct dcp_map_reg_resp dcpep_cb_map_reg(struct apple_dcp *dcp, - struct dcp_map_reg_req *req) -{ - if (req->index >= dcp->nr_disp_registers) { - dev_warn(dcp->dev, "attempted to read invalid reg index %u", - req->index); - - return (struct dcp_map_reg_resp){ .ret = 1 }; - } else { - struct resource *rsrc = dcp->disp_registers[req->index]; - - return (struct dcp_map_reg_resp){ - .addr = rsrc->start, .length = resource_size(rsrc) - }; - } -} - -static struct dcp_read_edt_data_resp -dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) -{ - return (struct dcp_read_edt_data_resp){ - .value[0] = req->value[0], - .ret = 0, - }; -} - -static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, - u8 *enabled) -{ - /* - * update backlight brightness on next swap, on non mini-LED displays - * DCP seems to set an invalid iDAC value after coming out of DPMS. - * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" - */ - dcp->brightness.update = true; -} - -/* Chunked data transfer for property dictionaries */ -static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) -{ - if (dcp->chunks.data != NULL) { - dev_warn(dcp->dev, "ignoring spurious transfer start\n"); - return false; - } - - dcp->chunks.length = *length; - dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); - - if (!dcp->chunks.data) { - dev_warn(dcp->dev, "failed to allocate chunks\n"); - return false; - } - - return true; -} - -static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, - struct dcp_set_dcpav_prop_chunk_req *req) -{ - if (!dcp->chunks.data) { - dev_warn(dcp->dev, "ignoring spurious chunk\n"); - return false; - } - - if (req->offset + req->length > dcp->chunks.length) { - dev_warn(dcp->dev, "ignoring overflowing chunk\n"); - return false; - } - - memcpy(dcp->chunks.data + req->offset, req->data, req->length); - return true; -} - -static bool dcpep_process_chunks(struct apple_dcp *dcp, - struct dcp_set_dcpav_prop_end_req *req) -{ - struct dcp_parse_ctx ctx; - int ret; - - if (!dcp->chunks.data) { - dev_warn(dcp->dev, "ignoring spurious end\n"); - return false; - } - - /* used just as opaque pointer for tracing */ - ctx.dcp = dcp; - - ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); - - if (ret) { - dev_warn(dcp->dev, "bad header on dcpav props\n"); - return false; - } - - if (!strcmp(req->key, "TimingElements")) { - dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, - dcp->width_mm, dcp->height_mm, - dcp->notch_height); - - if (IS_ERR(dcp->modes)) { - dev_warn(dcp->dev, "failed to parse modes\n"); - dcp->modes = NULL; - dcp->nr_modes = 0; - return false; - } - } else if (!strcmp(req->key, "DisplayAttributes")) { - /* DisplayAttributes are empty for integrated displays, use - * display dimensions read from the devicetree - */ - if (dcp->main_display) { - ret = parse_display_attributes(&ctx, &dcp->width_mm, - &dcp->height_mm); - - if (ret) { - dev_warn(dcp->dev, "failed to parse display attribs\n"); - return false; - } - } - - dcp_set_dimensions(dcp); - } - - return true; -} - -static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, - struct dcp_set_dcpav_prop_end_req *req) -{ - u8 resp = dcpep_process_chunks(dcp, req); - - /* Reset for the next transfer */ - devm_kfree(dcp->dev, dcp->chunks.data); - dcp->chunks.data = NULL; - - return resp; -} - -/* Boot sequence */ -static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_channel *ch = &dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - dev_dbg(dcp->dev, "boot done"); - - *succ = true; - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); -} - -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_late_init_signal(dcp, false, boot_5, NULL); -} - -static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) -{ - u32 v_true = true; - - dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); -} - -static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_setup_video_limits(dcp, false, boot_3, NULL); -} - -static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_create_default_fb(dcp, false, boot_2, NULL); -} - -/* Use special function signature to defer the ACK */ -static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) -{ - trace_iomfb_callback(dcp, tag, __func__); - dcp_set_create_dfb(dcp, false, boot_1_5, NULL); - return false; -} - -static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) -{ - if (dcp->disp_registers[5] && dcp->disp_registers[6]) - return (struct dcp_rt_bandwidth){ - .reg_scratch = - dcp->disp_registers[5]->start + REG_SCRATCH, - .reg_doorbell = - dcp->disp_registers[6]->start + REG_DOORBELL, - .doorbell_bit = REG_DOORBELL_BIT, - - .padding[3] = 0x4, // XXX: required by 11.x firmware - }; - else if (dcp->disp_registers[4]) - return (struct dcp_rt_bandwidth){ - .reg_scratch = dcp->disp_registers[4]->start + - REG_SCRATCH_T600X, - .reg_doorbell = 0, - .doorbell_bit = 0, - }; - else - return (struct dcp_rt_bandwidth){ - .reg_scratch = 0, - .reg_doorbell = 0, - .doorbell_bit = 0, - }; -} - -/* Callback to get the current time as milliseconds since the UNIX epoch */ -static u64 dcpep_cb_get_time(struct apple_dcp *dcp) -{ - return ktime_to_ms(ktime_get_real()); -} - -struct dcp_swap_cookie { - struct kref refcount; - struct completion done; - u32 swap_id; -}; - -static void release_swap_cookie(struct kref *ref) -{ - struct dcp_swap_cookie *cookie; - cookie = container_of(ref, struct dcp_swap_cookie, refcount); - - kfree(cookie); -} - -static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_submit_resp *resp = data; - dev_dbg(dcp->dev, "%s", __func__); - - if (cookie) { - struct dcp_swap_cookie *info = cookie; - complete(&info->done); - kref_put(&info->refcount, release_swap_cookie); - } - - if (resp->ret) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} - -static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, - void *cookie) -{ - struct dcp_swap_start_resp *resp = data; - dev_dbg(dcp->dev, "%s swap_id: %u", __func__, resp->swap_id); - dcp->swap.swap.swap_id = resp->swap_id; - - if (cookie) { - struct dcp_swap_cookie *info = cookie; - info->swap_id = resp->swap_id; - } - - dcp_swap_submit(dcp, false, &dcp->swap, dcp_swap_cleared, cookie); -} - -static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_wait_cookie *wait = cookie; - dev_dbg(dcp->dev, "%s", __func__); - - if (wait) { - complete(&wait->done); - kref_put(&wait->refcount, release_wait_cookie); + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; } } -static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_set_power_state_req req = { - .unklong = 1, - }; - dev_dbg(dcp->dev, "%s", __func__); - - dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); -} - -static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_set_parameter_dcp param = { - .param = 14, - .value = { 0 }, - .count = 1, - }; - dev_dbg(dcp->dev, "%s", __func__); - - dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_power_state, cookie); -} - void dcp_poweron(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - struct dcp_wait_cookie *cookie; - int ret; - u32 handle; - dev_dbg(dcp->dev, "%s", __func__); - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) - return; - - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - - if (dcp->main_display) { - handle = 0; - dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, - cookie); - } else { - handle = 2; - dcp_set_display_device(dcp, false, &handle, - dcp_on_set_parameter, cookie); + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweron_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_2: + iomfb_poweron_v13_2(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; } - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); - - if (ret == 0) - dev_warn(dcp->dev, "wait for power timed out"); - - kref_put(&cookie->refcount, release_wait_cookie);; - - /* Force a brightness update after poweron, to restore the brightness */ - dcp->brightness.update = true; } EXPORT_SYMBOL(dcp_poweron); -static void complete_set_powerstate(struct apple_dcp *dcp, void *out, - void *cookie) -{ - struct dcp_wait_cookie *wait = cookie; - - if (wait) { - complete(&wait->done); - kref_put(&wait->refcount, release_wait_cookie); - } -} - void dcp_poweroff(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - int ret, swap_id; - struct dcp_set_power_state_req power_req = { - .unklong = 0, - }; - struct dcp_swap_cookie *cookie; - struct dcp_wait_cookie *poff_cookie; - struct dcp_swap_start_req swap_req = { 0 }; - - dev_dbg(dcp->dev, "%s", __func__); - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) - return; - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - // clear surfaces - memset(&dcp->swap, 0, sizeof(dcp->swap)); - - dcp->swap.swap.swap_enabled = - dcp->swap.swap.swap_completed = IOMFB_SET_BACKGROUND | 0xF; - dcp->swap.swap.bg_color = 0xFF000000; - - /* - * Turn off the backlight. This matters because the DCP's idea of - * backlight brightness gets desynced after a power change, and it - * needs to be told it's going to turn off so it will consider the - * subsequent update on poweron an actual change and restore the - * brightness. - */ - dcp->swap.swap.bl_unk = 1; - dcp->swap.swap.bl_value = 0; - dcp->swap.swap.bl_power = 0; - - for (int l = 0; l < SWAP_SURFACES; l++) - dcp->swap.surf_null[l] = true; - - dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); - - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); - swap_id = cookie->swap_id; - kref_put(&cookie->refcount, release_swap_cookie); - if (ret <= 0) { - dcp->crashed = true; - return; + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweroff_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_2: + iomfb_poweroff_v13_2(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; } - - dev_dbg(dcp->dev, "%s: clear swap submitted: %u", __func__, swap_id); - - poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); - if (!poff_cookie) - return; - init_completion(&poff_cookie->done); - kref_init(&poff_cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&poff_cookie->refcount); - - dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, - poff_cookie); - ret = wait_for_completion_timeout(&poff_cookie->done, - msecs_to_jiffies(1000)); - - if (ret == 0) - dev_warn(dcp->dev, "setPowerState(0) timeout %u ms", 1000); - else if (ret > 0) - dev_dbg(dcp->dev, - "setPowerState(0) finished with %d ms to spare", - jiffies_to_msecs(ret)); - - kref_put(&poff_cookie->refcount, release_wait_cookie); - dev_dbg(dcp->dev, "%s: setPowerState(0) done", __func__); } EXPORT_SYMBOL(dcp_poweroff); @@ -1148,199 +302,6 @@ void dcp_hotplug(struct work_struct *work) } EXPORT_SYMBOL_GPL(dcp_hotplug); -static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) -{ - struct apple_connector *connector = dcp->connector; - - /* DCP issues hotplug_gated callbacks after SetPowerState() calls on - * devices with display (macbooks, imacs). This must not result in - * connector state changes on DRM side. Some applications won't enable - * a CRTC with a connector in disconnected state. Weston after DPMS off - * is one example. dcp_is_main_display() returns true on devices with - * integrated display. Ignore the hotplug_gated() callbacks there. - */ - if (dcp->main_display) - return; - - /* Hotplug invalidates mode. DRM doesn't always handle this. */ - if (!(*connected)) { - dcp->valid_mode = false; - /* after unplug swap will not complete until the next - * set_digital_out_mode */ - schedule_work(&dcp->vblank_wq); - } - - if (connector && connector->connected != !!(*connected)) { - connector->connected = !!(*connected); - dcp->valid_mode = false; - schedule_work(&connector->hotplug_wq); - } -} - -static void -dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, - struct dcp_swap_complete_intent_gated *info) -{ - trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, - info->width, info->height); -} - -#define DCPEP_MAX_CB (1000) - -/* - * Define type-safe trampolines. Define typedefs to enforce type-safety on the - * input data (so if the types don't match, gcc errors out). - */ - -#define TRAMPOLINE_VOID(func, handler) \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - trace_iomfb_callback(dcp, tag, #handler); \ - handler(dcp); \ - return true; \ - } - -#define TRAMPOLINE_IN(func, handler, T_in) \ - typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ - \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - callback_##handler cb = handler; \ - \ - trace_iomfb_callback(dcp, tag, #handler); \ - cb(dcp, in); \ - return true; \ - } - -#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ - typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ - \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - T_out *typed_out = out; \ - callback_##handler cb = handler; \ - \ - trace_iomfb_callback(dcp, tag, #handler); \ - *typed_out = cb(dcp, in); \ - return true; \ - } - -#define TRAMPOLINE_OUT(func, handler, T_out) \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ - { \ - T_out *typed_out = out; \ - \ - trace_iomfb_callback(dcp, tag, #handler); \ - *typed_out = handler(dcp); \ - return true; \ - } - -TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); -TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); -TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); -TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); -TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, - struct dc_swap_complete_resp); -TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, - struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); -TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, - struct iomfb_set_fx_prop_req) -TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct dcp_map_buf_req, struct dcp_map_buf_resp); -TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, - struct dcp_unmap_buf_resp); -TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, - struct iomfb_sr_set_property_int_req, u8); -TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, - struct dcp_allocate_buffer_req, - struct dcp_allocate_buffer_resp); -TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, - struct dcp_map_physical_req, struct dcp_map_physical_resp); -TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, - u8); -TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, struct dcp_map_reg_req, - struct dcp_map_reg_resp); -TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, - struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); -TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); -TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, - struct dcp_set_dcpav_prop_chunk_req, u8); -TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, - struct dcp_set_dcpav_prop_end_req, u8); -TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, - struct dcp_rt_bandwidth); -TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); -TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); -TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); -TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, - dcpep_cb_swap_complete_intent_gated, - struct dcp_swap_complete_intent_gated); -TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, - iomfbep_cb_enable_backlight_message_ap_gated, u8); -TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, - struct iomfb_property); - -bool (*const dcpep_cb_handlers[DCPEP_MAX_CB])(struct apple_dcp *, int, void *, - void *) = { - [0] = trampoline_true, /* did_boot_signal */ - [1] = trampoline_true, /* did_power_on_signal */ - [2] = trampoline_nop, /* will_power_off_signal */ - [3] = trampoline_rt_bandwidth, - [100] = iomfbep_cb_match_pmu_service, - [101] = trampoline_zero, /* get_display_default_stride */ - [102] = trampoline_nop, /* set_number_property */ - [103] = trampoline_nop, /* set_boolean_property */ - [106] = trampoline_nop, /* remove_property */ - [107] = trampoline_true, /* create_provider_service */ - [108] = trampoline_true, /* create_product_service */ - [109] = trampoline_true, /* create_pmu_service */ - [110] = trampoline_true, /* create_iomfb_service */ - [111] = trampoline_true, /* create_backlight_service */ - [116] = dcpep_cb_boot_1, - [117] = trampoline_false, /* is_dark_boot */ - [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [120] = trampoline_read_edt_data, - [122] = trampoline_prop_start, - [123] = trampoline_prop_chunk, - [124] = trampoline_prop_end, - [201] = trampoline_map_piodma, - [202] = trampoline_unmap_piodma, - [206] = iomfbep_cb_match_pmu_service_2, - [207] = iomfbep_cb_match_backlight_service, - [208] = trampoline_get_time, - [211] = trampoline_nop, /* update_backlight_factor_prop */ - [300] = trampoline_pr_publish, - [401] = trampoline_get_uint_prop, - [404] = trampoline_nop, /* sr_set_uint_prop */ - [406] = trampoline_set_fx_prop, - [408] = trampoline_get_frequency, - [411] = trampoline_map_reg, - [413] = trampoline_true, /* sr_set_property_dict */ - [414] = trampoline_sr_set_property_int, - [415] = trampoline_true, /* sr_set_property_bool */ - [451] = trampoline_allocate_buffer, - [452] = trampoline_map_physical, - [456] = trampoline_release_mem_desc, - [552] = trampoline_true, /* set_property_dict_0 */ - [561] = trampoline_true, /* set_property_dict */ - [563] = trampoline_true, /* set_property_int */ - [565] = trampoline_true, /* set_property_bool */ - [567] = trampoline_true, /* set_property_str */ - [574] = trampoline_zero, /* power_up_dart */ - [576] = trampoline_hotplug, - [577] = trampoline_nop, /* powerstate_notify */ - [582] = trampoline_true, /* create_default_fb_surface */ - [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ - [588] = trampoline_nop, /* resize_default_fb_surface_gated */ - [589] = trampoline_swap_complete, - [591] = trampoline_swap_complete_intent_gated, - [593] = trampoline_enable_backlight_message_ap_gated, - [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ - [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ - [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ - [598] = trampoline_nop, /* find_swap_function_gated */ -}; - static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, void *data, u32 length, u16 offset) { @@ -1351,7 +312,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - if (tag < 0 || tag >= DCPEP_MAX_CB || !dcpep_cb_handlers[tag]) { + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); return; @@ -1369,7 +330,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, ch->output[depth] = out; ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); - if (dcpep_cb_handlers[tag](dcp, tag, out, in)) + if (dcp->cb_handlers[tag](dcp, tag, out, in)) dcp_ack(dcp, context); } @@ -1425,48 +386,12 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) dcpep_handle_cb(dcp, ctx_id, data, length, offset); } -/* - * Callback for swap requests. If a swap failed, we'll never get a swap - * complete event so we need to fake a vblank event early to avoid a hang. - */ - -static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_submit_resp *resp = data; - - if (resp->ret) { - dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} - -static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_start_resp *resp = data; - - dcp->swap.swap.swap_id = resp->swap_id; - - trace_iomfb_swap_submit(dcp, resp->swap_id); - dcp_swap_submit(dcp, false, &dcp->swap, dcp_swapped, NULL); -} - /* * DRM specifies rectangles as start and end coordinates. DCP specifies * rectangles as a start coordinate and a width/height. Convert a DRM rectangle * to a DCP rectangle. */ -static struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect) +struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect) { return (struct dcp_rect){ .x = rect->x1, .y = rect->y1, @@ -1474,7 +399,7 @@ static struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect) .h = drm_rect_height(rect) }; } -static u32 drm_format_to_dcp(u32 drm) +u32 drm_format_to_dcp(u32 drm) { switch (drm) { case DRM_FORMAT_XRGB8888: @@ -1520,7 +445,7 @@ int dcp_get_modes(struct drm_connector *connector) EXPORT_SYMBOL_GPL(dcp_get_modes); /* The user may own drm_display_mode, so we need to search for our copy */ -static struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, +struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode) { int i; @@ -1559,46 +484,11 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } EXPORT_SYMBOL(dcp_crtc_mode_fixup); -/* Helpers to modeset and swap, used to flush */ -static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct dcp_swap_start_req start_req = { 0 }; - dev_dbg(dcp->dev, "%s", __func__); - - if (dcp->connector && dcp->connector->connected) - dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); - else - dcp_drm_crtc_vblank(dcp->crtc); -} - -static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, - void *cookie) -{ - struct dcp_wait_cookie *wait = cookie; - dev_dbg(dcp->dev, "%s", __func__); - - if (wait) { - complete(&wait->done); - kref_put(&wait->refcount, release_wait_cookie); - } -} void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); - struct drm_plane *plane; - struct drm_plane_state *new_state, *old_state; - struct drm_crtc_state *crtc_state; - struct dcp_swap_submit_req *req = &dcp->swap; - int plane_idx, l; - int has_surface = 0; - bool modeset; - dev_dbg(dcp->dev, "%s", __func__); - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; if (dcp_channel_busy(&dcp->ch_cmd)) { @@ -1610,191 +500,34 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) return; } - /* Reset to defaults */ - memset(req, 0, sizeof(*req)); - for (l = 0; l < SWAP_SURFACES; l++) - req->surf_null[l] = true; - - /* - * Clear all surfaces on startup. The boot framebuffer in surface 0 - * sticks around. - */ - if (!dcp->surfaces_cleared) { - req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0xF; - req->swap.bg_color = 0xFF000000; - dcp->surfaces_cleared = true; - } - - // Surface 0 has limitations at least on t600x. - l = 1; - for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { - struct drm_framebuffer *fb = new_state->fb; - struct drm_gem_dma_object *obj; - struct drm_rect src_rect; - bool is_premultiplied = false; - - /* skip planes not for this crtc */ - if (old_state->crtc != crtc && new_state->crtc != crtc) - continue; - - WARN_ON(l >= SWAP_SURFACES); - - req->swap.swap_enabled |= BIT(l); - - if (old_state->fb && fb != old_state->fb) { - /* - * Race condition between a framebuffer unbind getting - * swapped out and GEM unreferencing a framebuffer. If - * we lose the race, the display gets IOVA faults and - * the DCP crashes. We need to extend the lifetime of - * the drm_framebuffer (and hence the GEM object) until - * after we get a swap complete for the swap unbinding - * it. - */ - struct dcp_fb_reference *entry = - kzalloc(sizeof(*entry), GFP_KERNEL); - if (entry) { - entry->fb = old_state->fb; - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); - } - drm_framebuffer_get(old_state->fb); - } - - if (!new_state->fb) { - l += 1; - continue; - } - req->surf_null[l] = false; - has_surface = 1; - - /* - * DCP doesn't support XBGR8 / XRGB8 natively. Blending as - * pre-multiplied alpha with a black background can be used as - * workaround for the bottommost plane. - */ - if (fb->format->format == DRM_FORMAT_XRGB8888 || - fb->format->format == DRM_FORMAT_XBGR8888) - is_premultiplied = true; - - drm_rect_fp_to_int(&src_rect, &new_state->src); - - req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); - req->swap.dst_rect[l] = drm_to_dcp_rect(&new_state->dst); - - if (dcp->notch_height > 0) - req->swap.dst_rect[l].y += dcp->notch_height; - - /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts - * the address for source x/y offsets. Since IOMFB has a direct - * support source position prefer that. - */ - obj = drm_fb_dma_get_gem_obj(fb, 0); - if (obj) - req->surf_iova[l] = obj->dma_addr + fb->offsets[0]; - - req->surf[l] = (struct dcp_surface){ - .is_premultiplied = is_premultiplied, - .format = drm_format_to_dcp(fb->format->format), - .xfer_func = DCP_XFER_FUNC_SDR, - .colorspace = DCP_COLORSPACE_NATIVE, - .stride = fb->pitches[0], - .width = fb->width, - .height = fb->height, - .buf_size = fb->height * fb->pitches[0], - .surface_id = req->swap.surf_ids[l], - - /* Only used for compressed or multiplanar surfaces */ - .pix_size = 1, - .pel_w = 1, - .pel_h = 1, - .has_comp = 1, - .has_planes = 1, - }; - - l += 1; - } - - if (modeset) { - struct dcp_display_mode *mode; - struct dcp_wait_cookie *cookie; - int ret; - - mode = lookup_mode(dcp, &crtc_state->mode); - if (!mode) { - dev_warn(dcp->dev, "no match for " DRM_MODE_FMT, - DRM_MODE_ARG(&crtc_state->mode)); - schedule_work(&dcp->vblank_wq); - return; - } - - dev_info(dcp->dev, "set_digital_out_mode(color:%d timing:%d)", - mode->color_mode_id, mode->timing_mode_id); - dcp->mode = (struct dcp_set_digital_out_mode_req){ - .color_mode_id = mode->color_mode_id, - .timing_mode_id = mode->timing_mode_id - }; - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) { - schedule_work(&dcp->vblank_wq); - return; - } - - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - - dcp_set_digital_out_mode(dcp, false, &dcp->mode, - complete_set_digital_out_mode, cookie); - - dev_dbg(dcp->dev, "%s - wait for modeset", __func__); - ret = wait_for_completion_timeout(&cookie->done, - msecs_to_jiffies(500)); - - kref_put(&cookie->refcount, release_wait_cookie); - - if (ret == 0) { - dev_dbg(dcp->dev, "set_digital_out_mode 200 ms"); - schedule_work(&dcp->vblank_wq); - return; - } else if (ret > 0) { - dev_dbg(dcp->dev, - "set_digital_out_mode finished with %d to spare", - jiffies_to_msecs(ret)); - } - - dcp->valid_mode = true; - } - - if (!has_surface && !crtc_state->color_mgmt_changed) { - if (crtc_state->enable && crtc_state->active && - !crtc_state->planes_changed) { - schedule_work(&dcp->vblank_wq); - return; - } - - /* Set black background */ - req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; - req->swap.bg_color = 0xFF000000; - req->clear = 1; + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_flush_v12_3(dcp, crtc, state); + break; + case DCP_FIRMWARE_V_13_2: + iomfb_flush_v13_2(dcp, crtc, state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; } +} +EXPORT_SYMBOL_GPL(dcp_flush); - /* These fields should be set together */ - req->swap.swap_completed = req->swap.swap_enabled; - - /* update brightness if changed */ - if (dcp->brightness.update) { - req->swap.bl_unk = 1; - req->swap.bl_value = dcp->brightness.dac; - req->swap.bl_power = 0x40; - dcp->brightness.update = false; +void iomfb_start(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_start_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_2: + iomfb_start_v13_2(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; } - - do_swap(dcp, NULL, NULL); } -EXPORT_SYMBOL_GPL(dcp_flush); bool dcp_is_initialized(struct platform_device *pdev) { @@ -1804,58 +537,12 @@ bool dcp_is_initialized(struct platform_device *pdev) } EXPORT_SYMBOL_GPL(dcp_is_initialized); -static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct apple_connector *connector; - int result = *(int *)out; - dev_info(dcp->dev, "DCP is_main_display: %d\n", result); - - dcp->main_display = result != 0; - - connector = dcp->connector; - if (connector) { - connector->connected = dcp->nr_modes > 0; - schedule_work(&connector->hotplug_wq); - } - - dcp->active = true; - complete(&dcp->start_done); -} - -static void init_3(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_is_main_display(dcp, false, res_is_main_display, NULL); -} - -static void init_2(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_first_client_open(dcp, false, init_3, NULL); -} - -static void init_1(struct apple_dcp *dcp, void *out, void *cookie) -{ - u32 val = 0; - dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); -} - -static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) -{ - struct iomfb_get_color_remap_mode_req color_remap = - (struct iomfb_get_color_remap_mode_req){ - .mode = 6, - }; - - dev_info(dcp->dev, "DCP booted\n"); - - iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); -} - void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) - dcp_start_signal(dcp, false, dcp_started, NULL); + iomfb_start(dcp); else if (type == IOMFB_MESSAGE_TYPE_MSG) dcpep_got_msg(dcp, message); else @@ -1878,13 +565,19 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) void iomfb_shutdown(struct apple_dcp *dcp) { - struct dcp_set_power_state_req req = { - /* defaults are ok */ - }; - /* We're going down */ dcp->active = false; dcp->valid_mode = false; - dcp_set_power_state(dcp, false, &req, NULL, NULL); + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_shutdown_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_2: + iomfb_shutdown_v13_2(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } } diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 54b34fbba94894..0c8061bb96e3e0 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -6,6 +6,8 @@ #include +#include "version_utils.h" + /* Fixed size of shared memory between DCP and AP */ #define DCP_SHMEM_SIZE 0x100000 @@ -106,35 +108,6 @@ struct dcp_rect { */ #define IOMFB_SET_BACKGROUND BIT(31) -struct dcp_swap { - u64 ts1; - u64 ts2; - u64 unk_10[6]; - u64 flags1; - u64 flags2; - - u32 swap_id; - - u32 surf_ids[SWAP_SURFACES]; - struct dcp_rect src_rect[SWAP_SURFACES]; - u32 surf_flags[SWAP_SURFACES]; - u32 surf_unk[SWAP_SURFACES]; - struct dcp_rect dst_rect[SWAP_SURFACES]; - u32 swap_enabled; - u32 swap_completed; - - u32 bg_color; - u8 unk_110[0x1b8]; - u32 unk_2c8; - u8 unk_2cc[0x14]; - u32 unk_2e0; - u16 unk_2e2; - u64 bl_unk; - u32 bl_value; // min value is 0x10000000 - u8 bl_power; // constant 0x40 for on - u8 unk_2f3[0x2d]; -} __packed; - /* Information describing a plane of a planar compressed surface */ struct dcp_plane_info { u32 width; @@ -154,38 +127,6 @@ struct dcp_component_types { u8 types[7]; } __packed; -/* Information describing a surface */ -struct dcp_surface { - u8 is_tiled; - u8 is_tearing_allowed; - u8 is_premultiplied; - u32 plane_cnt; - u32 plane_cnt2; - u32 format; /* DCP fourcc */ - u32 ycbcr_matrix; - u8 xfer_func; - u8 colorspace; - u32 stride; - u16 pix_size; - u8 pel_w; - u8 pel_h; - u32 offset; - u32 width; - u32 height; - u32 buf_size; - u64 protection_opts; - u32 surface_id; - struct dcp_component_types comp_types[MAX_PLANES]; - u64 has_comp; - struct dcp_plane_info planes[MAX_PLANES]; - u64 has_planes; - u32 compression_info[MAX_PLANES][13]; - u64 has_compr_info; - u32 unk_num; - u32 unk_denom; - u8 padding[7]; -} __packed; - struct dcp_rt_bandwidth { u64 unk1; u64 reg_scratch; @@ -218,14 +159,22 @@ enum dcpep_method { iomfbep_a132_backlight_service_matched, iomfbep_a358_vi_set_temperature_hint, iomfbep_get_color_remap_mode, + iomfbep_last_client_close, dcpep_num_methods }; +#define IOMFB_METHOD(tag, name) [name] = { #name, tag } + struct dcp_method_entry { const char *name; char tag[4]; }; +#define IOMFB_MAX_CB (1000) +struct apple_dcp; + +typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); + /* Prototypes */ struct dcp_set_digital_out_mode_req { @@ -287,21 +236,6 @@ struct dcp_map_physical_resp { u32 mem_desc_id; } __packed; -struct dcp_map_reg_req { - char obj[4]; - u32 index; - u32 flags; - u8 addr_null; - u8 length_null; - u8 padding[2]; -} __packed; - -struct dcp_map_reg_resp { - u64 addr; - u64 length; - u32 ret; -} __packed; - struct dcp_swap_start_req { u32 swap_id; struct dcp_iouserclient client; @@ -316,34 +250,6 @@ struct dcp_swap_start_resp { u32 ret; } __packed; -struct dcp_swap_submit_req { - struct dcp_swap swap; - struct dcp_surface surf[SWAP_SURFACES]; - u64 surf_iova[SWAP_SURFACES]; - u8 unkbool; - u64 unkdouble; - u32 clear; // or maybe switch to default fb? - u8 swap_null; - u8 surf_null[SWAP_SURFACES]; - u8 unkoutbool_null; - u8 padding[1]; -} __packed; - -struct dcp_swap_submit_resp { - u8 unkoutbool; - u32 ret; - u8 padding[3]; -} __packed; - -struct dc_swap_complete_resp { - u32 swap_id; - u8 unkbool; - u64 swap_data; - u8 swap_info[0x6c4]; - u32 unkint; - u8 swap_info_null; -} __packed; - struct dcp_get_uint_prop_req { char obj[4]; char key[0x40]; @@ -435,4 +341,13 @@ struct iomfb_get_color_remap_mode_resp { u32 ret; } __packed; +struct iomfb_last_client_close_req { + u8 unkint_null; + u8 padding[3]; +} __packed; + +struct iomfb_last_client_close_resp { + u32 unkint; +} __packed; + #endif diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h new file mode 100644 index 00000000000000..401b6ec32848d3 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include +#include + +#include "dcp-internal.h" + +struct apple_dcp; + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + + +#define DCP_THUNK_VOID(func, handle) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], 0, 0, NULL, cb, cookie); \ + } + +#define DCP_THUNK_OUT(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], 0, sizeof(T), NULL, cb, cookie); \ + } + +#define DCP_THUNK_IN(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, T *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], sizeof(T), 0, data, cb, cookie); \ + } + +#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ + static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], sizeof(T_in), sizeof(T_out), data, \ + cb, cookie); \ + } + +#define IOMFB_THUNK_INOUT(name) \ + static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, \ + struct iomfb_ ## name ## _req *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[iomfbep_ ## name], \ + sizeof(struct iomfb_ ## name ## _req), \ + sizeof(struct iomfb_ ## name ## _resp), \ + data, cb, cookie); \ + } + +/* + * Define type-safe trampolines. Define typedefs to enforce type-safety on the + * input data (so if the types don't match, gcc errors out). + */ + +#define TRAMPOLINE_VOID(func, handler) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + trace_iomfb_callback(dcp, tag, #handler); \ + handler(dcp); \ + return true; \ + } + +#define TRAMPOLINE_IN(func, handler, T_in) \ + typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + callback_##handler cb = handler; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ + typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + callback_##handler cb = handler; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + *typed_out = cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_OUT(func, handler, T_out) \ + static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + *typed_out = handler(dcp); \ + return true; \ + } + +/* Call a DCP function given by a tag */ +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie); + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +int dcp_parse_tag(char tag[4]); + +void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context); + +/* + * DRM specifies rectangles as start and end coordinates. DCP specifies + * rectangles as a start coordinate and a width/height. Convert a DRM rectangle + * to a DCP rectangle. + */ +struct dcp_rect drm_to_dcp_rect(struct drm_rect *rect); + +u32 drm_format_to_dcp(u32 drm); + +/* The user may own drm_display_mode, so we need to search for our copy */ +struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + const struct drm_display_mode *mode); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c new file mode 100644 index 00000000000000..c6c62e4bc4c125 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -0,0 +1,1344 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright 2021 Alyssa Rosenzweig + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "iomfb_internal.h" +#include "parser.h" +#include "trace.h" +#include "version_utils.h" + +/* Register defines used in bandwidth setup structure */ +#define REG_SCRATCH (0x14) +#define REG_SCRATCH_T600X (0x988) +#define REG_DOORBELL (0x0) +#define REG_DOORBELL_BIT (2) + +struct dcp_wait_cookie { + struct kref refcount; + struct completion done; +}; + +static void release_wait_cookie(struct kref *ref) +{ + struct dcp_wait_cookie *cookie; + cookie = container_of(ref, struct dcp_wait_cookie, refcount); + + kfree(cookie); +} + +DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); +DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); +DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); + +IOMFB_THUNK_INOUT(get_color_remap_mode); +IOMFB_THUNK_INOUT(last_client_close); + +DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, + struct DCP_FW_NAME(dcp_swap_submit_req), + struct DCP_FW_NAME(dcp_swap_submit_resp)); + +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, + struct dcp_swap_start_resp); + +DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, + struct dcp_set_power_state_req, + struct dcp_set_power_state_resp); + +DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, + struct dcp_set_digital_out_mode_req, u32); + +DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); + +DCP_THUNK_OUT(dcp_set_display_refresh_properties, + dcpep_set_display_refresh_properties, u32); + +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +DCP_THUNK_INOUT(dcp_late_init_signal, dcpep_late_init_signal, u32, u32); +#else +DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); +#endif +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); +DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); +DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); +DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); +DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); + +DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, + struct dcp_set_parameter_dcp, u32); + +DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, + dcpep_enable_disable_video_power_savings, u32, int); + +DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); + +/* DCP callback handlers */ +static void dcpep_cb_nop(struct apple_dcp *dcp) +{ + /* No operation */ +} + +static u8 dcpep_cb_true(struct apple_dcp *dcp) +{ + return true; +} + +static u8 dcpep_cb_false(struct apple_dcp *dcp) +{ + return false; +} + +static u32 dcpep_cb_zero(struct apple_dcp *dcp) +{ + return 0; +} + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + trace_iomfb_swap_complete(dcp, resp->swap_id); + + dcp_drm_crtc_vblank(dcp->crtc); +} + +/* special */ +static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) +{ + // ack D100 cb_match_pmu_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + iomfb_a358_vi_set_temperature_hint(dcp, false, + complete_vi_set_temperature_hint, + NULL); + + // return false for deferred ACK + return false; +} + +static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_pmu_service_2 + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service_2(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a131_pmu_service_matched(dcp, false, complete_pmu_service_matched, + out); + + // return false for deferred ACK + return false; +} + +static void complete_backlight_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_backlight_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a132_backlight_service_matched(dcp, false, complete_backlight_service_matched, out); + + // return false for deferred ACK + return false; +} + +static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) +{ + switch (prop->id) { + case IOMFB_PROPERTY_NITS: + { + dcp->brightness.nits = prop->value / dcp->brightness.scale; + /* notify backlight device of the initial brightness */ + if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); + trace_iomfb_brightness(dcp, prop->value); + break; + } + default: + dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); + } +} + +static struct dcp_get_uint_prop_resp +dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) +{ + struct dcp_get_uint_prop_resp resp = (struct dcp_get_uint_prop_resp){ + .value = 0 + }; + + if (dcp->panel.has_mini_led && + memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ + if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { + /* + * TODO: value from j314c, find out if it is temperature in + * centigrade C and which temperature sensor reports it + */ + resp.value = 3029; + resp.ret = true; + } + } + + return resp; +} + +static u8 iomfbep_cb_sr_set_property_int(struct apple_dcp *dcp, + struct iomfb_sr_set_property_int_req *req) +{ + if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ + if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { + if (!req->value_null) + dcp->brightness.scale = req->value; + } + } + + return 1; +} + +static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) +{ + // TODO: trace this, see if there properties which needs to used later +} + +/* + * Callback to map a buffer allocated with allocate_buf for PIODMA usage. + * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated + * stream of the display DART, rather than the expected DCP DART. + * + * XXX: This relies on dma_get_sgtable in concert with dma_map_sgtable, which + * is a "fundamentally unsafe" operation according to the docs. And yet + * everyone does it... + */ +static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct sg_table *map; + int ret; + + if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) + goto reject; + + map = &dcp->memdesc[req->buffer].map; + + if (!map->sgl) + goto reject; + + /* Use PIODMA device instead of DCP to map against the right IOMMU. */ + ret = dma_map_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); + + if (ret) + goto reject; + + return (struct dcp_map_buf_resp){ .dva = sg_dma_address(map->sgl) }; + +reject: + dev_err(dcp->dev, "denying map of invalid buffer %llx for pidoma\n", + req->buffer); + return (struct dcp_map_buf_resp){ .ret = EINVAL }; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *resp) +{ + struct sg_table *map; + dma_addr_t dma_addr; + + if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu", + resp->buffer); + return; + } + + map = &dcp->memdesc[resp->buffer].map; + + if (!map->sgl) { + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx", + resp->buffer, resp->dva); + return; + } + + dma_addr = sg_dma_address(map->sgl); + if (dma_addr != resp->dva) { + dev_warn(dcp->dev, "unmap buffer %llu address mismatch dma_addr:%llx dva:%llx", + resp->buffer, dma_addr, resp->dva); + return; + } + + /* Use PIODMA device instead of DCP to unmap from the right IOMMU. */ + dma_unmap_sgtable(&dcp->piodma->dev, map, DMA_BIDIRECTIONAL, 0); +} + +/* + * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be + * physically contigiuous, however we should save the sgtable in case the + * buffer needs to be later mapped for PIODMA. + */ +static struct dcp_allocate_buffer_resp +dcpep_cb_allocate_buffer(struct apple_dcp *dcp, + struct dcp_allocate_buffer_req *req) +{ + struct dcp_allocate_buffer_resp resp = { 0 }; + struct dcp_mem_descriptor *memdesc; + u32 id; + + resp.dva_size = ALIGN(req->size, 4096); + resp.mem_desc_id = + find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + + if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring"); + resp.dva_size = 0; + resp.mem_desc_id = 0; + return resp; + } + id = resp.mem_desc_id; + set_bit(id, dcp->memdesc_map); + + memdesc = &dcp->memdesc[id]; + + memdesc->size = resp.dva_size; + memdesc->buf = dma_alloc_coherent(dcp->dev, memdesc->size, + &memdesc->dva, GFP_KERNEL); + + dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, + memdesc->size); + resp.dva = memdesc->dva; + + return resp; +} + +static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) +{ + struct dcp_mem_descriptor *memdesc; + u32 id = *mem_desc_id; + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, + "unmap request for out of range mem_desc_id %u", id); + return 0; + } + + if (!test_and_clear_bit(id, dcp->memdesc_map)) { + dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u", + id); + return 0; + } + + memdesc = &dcp->memdesc[id]; + if (memdesc->buf) { + dma_free_coherent(dcp->dev, memdesc->size, memdesc->buf, + memdesc->dva); + + memdesc->buf = NULL; + memset(&memdesc->map, 0, sizeof(memdesc->map)); + } else { + memdesc->reg = 0; + } + + memdesc->size = 0; + + return 1; +} + +/* Validate that the specified region is a display register */ +static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +{ + int i; + + for (i = 0; i < dcp->nr_disp_registers; ++i) { + struct resource *r = dcp->disp_registers[i]; + + if ((start >= r->start) && (end <= r->end)) + return true; + } + + return false; +} + +/* + * Map contiguous physical memory into the DCP's address space. The firmware + * uses this to map the display registers we advertise in + * sr_map_device_memory_with_index, so we bounds check against that to guard + * safe against malicious coprocessors. + */ +static struct dcp_map_physical_resp +dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) +{ + int size = ALIGN(req->size, 4096); + u32 id; + + if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + dev_err(dcp->dev, "refusing to map phys address %llx size %llx", + req->paddr, req->size); + return (struct dcp_map_physical_resp){}; + } + + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + set_bit(id, dcp->memdesc_map); + dcp->memdesc[id].size = size; + dcp->memdesc[id].reg = req->paddr; + + return (struct dcp_map_physical_resp){ + .dva_size = size, + .mem_desc_id = id, + .dva = dma_map_resource(dcp->dev, req->paddr, size, + DMA_BIDIRECTIONAL, 0), + }; +} + +static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) +{ + return clk_get_rate(dcp->clk); +} + +static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *dcp, + struct DCP_FW_NAME(dcp_map_reg_req) *req) +{ + if (req->index >= dcp->nr_disp_registers) { + dev_warn(dcp->dev, "attempted to read invalid reg index %u", + req->index); + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; + } else { + struct resource *rsrc = dcp->disp_registers[req->index]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), + DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); +#endif + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ + .addr = rsrc->start, + .length = resource_size(rsrc), +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .dva = dva, +#endif + }; + } +} + +static struct dcp_read_edt_data_resp +dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) +{ + return (struct dcp_read_edt_data_resp){ + .value[0] = req->value[0], + .ret = 0, + }; +} + +static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, + u8 *enabled) +{ + /* + * update backlight brightness on next swap, on non mini-LED displays + * DCP seems to set an invalid iDAC value after coming out of DPMS. + * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" + */ + dcp->brightness.update = true; +} + +/* Chunked data transfer for property dictionaries */ +static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) +{ + if (dcp->chunks.data != NULL) { + dev_warn(dcp->dev, "ignoring spurious transfer start\n"); + return false; + } + + dcp->chunks.length = *length; + dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "failed to allocate chunks\n"); + return false; + } + + return true; +} + +static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_chunk_req *req) +{ + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious chunk\n"); + return false; + } + + if (req->offset + req->length > dcp->chunks.length) { + dev_warn(dcp->dev, "ignoring overflowing chunk\n"); + return false; + } + + memcpy(dcp->chunks.data + req->offset, req->data, req->length); + return true; +} + +static bool dcpep_process_chunks(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + struct dcp_parse_ctx ctx; + int ret; + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious end\n"); + return false; + } + + /* used just as opaque pointer for tracing */ + ctx.dcp = dcp; + + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); + + if (ret) { + dev_warn(dcp->dev, "bad header on dcpav props\n"); + return false; + } + + if (!strcmp(req->key, "TimingElements")) { + dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, + dcp->width_mm, dcp->height_mm, + dcp->notch_height); + + if (IS_ERR(dcp->modes)) { + dev_warn(dcp->dev, "failed to parse modes\n"); + dcp->modes = NULL; + dcp->nr_modes = 0; + return false; + } + } else if (!strcmp(req->key, "DisplayAttributes")) { + /* DisplayAttributes are empty for integrated displays, use + * display dimensions read from the devicetree + */ + if (dcp->main_display) { + ret = parse_display_attributes(&ctx, &dcp->width_mm, + &dcp->height_mm); + + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } + } + + dcp_set_dimensions(dcp); + } + + return true; +} + +static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + u8 resp = dcpep_process_chunks(dcp, req); + + /* Reset for the next transfer */ + devm_kfree(dcp->dev, dcp->chunks.data); + dcp->chunks.data = NULL; + + return resp; +} + +/* Boot sequence */ +static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + dev_dbg(dcp->dev, "boot done"); + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); +} + +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 v_true = 1; + dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); +#else + dcp_late_init_signal(dcp, false, boot_5, NULL); +#endif +} + +static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 v_true = true; + + dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); +} + +static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_setup_video_limits(dcp, false, boot_3, NULL); +} + +static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_create_default_fb(dcp, false, boot_2, NULL); +} + +/* Use special function signature to defer the ACK */ +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); + return false; +} + +static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) +{ + if (dcp->disp_registers[5] && dcp->disp_registers[6]) + return (struct dcp_rt_bandwidth){ + .reg_scratch = + dcp->disp_registers[5]->start + REG_SCRATCH, + .reg_doorbell = + dcp->disp_registers[6]->start + REG_DOORBELL, + .doorbell_bit = REG_DOORBELL_BIT, + + .padding[3] = 0x4, // XXX: required by 11.x firmware + }; + else if (dcp->disp_registers[4]) + return (struct dcp_rt_bandwidth){ + .reg_scratch = dcp->disp_registers[4]->start + + REG_SCRATCH_T600X, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; + else + return (struct dcp_rt_bandwidth){ + .reg_scratch = 0, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; +} + +/* Callback to get the current time as milliseconds since the UNIX epoch */ +static u64 dcpep_cb_get_time(struct apple_dcp *dcp) +{ + return ktime_to_ms(ktime_get_real()); +} + +struct dcp_swap_cookie { + struct kref refcount; + struct completion done; + u32 swap_id; +}; + +static void release_swap_cookie(struct kref *ref) +{ + struct dcp_swap_cookie *cookie; + cookie = container_of(ref, struct dcp_swap_cookie, refcount); + + kfree(cookie); +} + +static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + dev_dbg(dcp->dev, "%s", __func__); + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + + if (resp->ret) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + dev_dbg(dcp->dev, "%s swap_id: %u", __func__, resp->swap_id); + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + info->swap_id = resp->swap_id; + } + + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); +} + +static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + dev_dbg(dcp->dev, "%s", __func__); + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + dev_dbg(dcp->dev, "%s", __func__); + + dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); +} + +static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = 14, + .value = { 0 }, +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .count = 3, +#else + .count = 1, +#endif + }; + dev_dbg(dcp->dev, "%s", __func__); + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_power_state, cookie); +} + +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + int ret; + u32 handle; + dev_err(dcp->dev, "dcp_poweron() starting\n"); + + dev_dbg(dcp->dev, "%s", __func__); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + if (dcp->main_display) { + handle = 0; + dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, + cookie); + } else { + handle = 2; + dcp_set_display_device(dcp, false, &handle, + dcp_on_set_parameter, cookie); + } + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); + + if (ret == 0) + dev_warn(dcp->dev, "wait for power timed out"); + + kref_put(&cookie->refcount, release_wait_cookie);; + + /* Force a brightness update after poweron, to restore the brightness */ + dcp->brightness.update = true; +} + +static void complete_set_powerstate(struct apple_dcp *dcp, void *out, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + cookie); +} + +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) +{ + int ret, swap_id; + struct iomfb_last_client_close_req last_client_req = {}; + struct dcp_swap_cookie *cookie; + struct dcp_wait_cookie *poff_cookie; + struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); + + dev_dbg(dcp->dev, "%s", __func__); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + // clear surfaces + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0xF; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; + swap->unkU32Ptr_null = true; + swap->unkU32out_null = true; +#endif + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + swap_id = cookie->swap_id; + kref_put(&cookie->refcount, release_swap_cookie); + if (ret <= 0) { + dcp->crashed = true; + return; + } + + dev_dbg(dcp->dev, "%s: clear swap submitted: %u", __func__, swap_id); + + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); + if (!poff_cookie) + return; + init_completion(&poff_cookie->done); + kref_init(&poff_cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&poff_cookie->refcount); + + iomfb_last_client_close(dcp, false, &last_client_req, + last_client_closed_poff, poff_cookie); + ret = wait_for_completion_timeout(&poff_cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setPowerState(0) timeout %u ms", 1000); + else if (ret > 0) + dev_dbg(dcp->dev, + "setPowerState(0) finished with %d ms to spare", + jiffies_to_msecs(ret)); + + kref_put(&poff_cookie->refcount, release_wait_cookie); + dev_dbg(dcp->dev, "%s: setPowerState(0) done", __func__); + + dev_err(dcp->dev, "dcp_poweroff() done\n"); +} + +static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); +} + +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) +{ + int ret; + struct iomfb_last_client_close_req req = {}; + + struct dcp_wait_cookie *cookie; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + iomfb_last_client_close(dcp, false, &req, last_client_closed_sleep, + cookie); + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setDCPPower(0) timeout %u ms", 1000); + + kref_put(&cookie->refcount, release_wait_cookie); + dev_dbg(dcp->dev, "%s: setDCPPower(0) done", __func__); + + dev_err(dcp->dev, "dcp_sleep() done\n"); +} + +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + struct apple_connector *connector = dcp->connector; + + /* DCP issues hotplug_gated callbacks after SetPowerState() calls on + * devices with display (macbooks, imacs). This must not result in + * connector state changes on DRM side. Some applications won't enable + * a CRTC with a connector in disconnected state. Weston after DPMS off + * is one example. dcp_is_main_display() returns true on devices with + * integrated display. Ignore the hotplug_gated() callbacks there. + */ + if (dcp->main_display) + return; + + /* Hotplug invalidates mode. DRM doesn't always handle this. */ + if (!(*connected)) { + dcp->valid_mode = false; + /* after unplug swap will not complete until the next + * set_digital_out_mode */ + schedule_work(&dcp->vblank_wq); + } + + if (connector && connector->connected != !!(*connected)) { + connector->connected = !!(*connected); + dcp->valid_mode = false; + schedule_work(&connector->hotplug_wq); + } +} + +static void +dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, + struct dcp_swap_complete_intent_gated *info) +{ + trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, + info->width, info->height); +} + +TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); +TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); +TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); +TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); +TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, + struct DCP_FW_NAME(dc_swap_complete_resp)); +TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, + struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, + struct iomfb_set_fx_prop_req) +TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, + struct dcp_map_buf_req, struct dcp_map_buf_resp); +TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, + struct dcp_unmap_buf_resp); +TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, + struct iomfb_sr_set_property_int_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, + struct dcp_allocate_buffer_req, + struct dcp_allocate_buffer_resp); +TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, + struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, + u8); +TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, + struct DCP_FW_NAME(dcp_map_reg_req), + struct DCP_FW_NAME(dcp_map_reg_resp)); +TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, + struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); +TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); +TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, + struct dcp_set_dcpav_prop_chunk_req, u8); +TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, + struct dcp_set_dcpav_prop_end_req, u8); +TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, + struct dcp_rt_bandwidth); +TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); +TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, + dcpep_cb_swap_complete_intent_gated, + struct dcp_swap_complete_intent_gated); +TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, + iomfbep_cb_enable_backlight_message_ap_gated, u8); +TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, + struct iomfb_property); + +/* + * Callback for swap requests. If a swap failed, we'll never get a swap + * complete event so we need to fake a vblank event early to avoid a hang. + */ + +static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + + if (resp->ret) { + dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + + trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); +} + +/* Helpers to modeset and swap, used to flush */ +static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_req start_req = { 0 }; + dev_dbg(dcp->dev, "%s", __func__); + + if (dcp->connector && dcp->connector->connected) + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + else + dcp_drm_crtc_vblank(dcp->crtc); +} + +static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + dev_dbg(dcp->dev, "%s", __func__); + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct drm_plane *plane; + struct drm_plane_state *new_state, *old_state; + struct drm_crtc_state *crtc_state; + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + int plane_idx, l; + int has_surface = 0; + bool modeset; + dev_dbg(dcp->dev, "%s", __func__); + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + + /* Reset to defaults */ + memset(req, 0, sizeof(*req)); + for (l = 0; l < SWAP_SURFACES; l++) + req->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (l = 0; l < 5; l++) + req->surf2_null[l] = true; + req->unkU32Ptr_null = true; + req->unkU32out_null = true; +#endif + + /* + * Clear all surfaces on startup. The boot framebuffer in surface 0 + * sticks around. + */ + if (!dcp->surfaces_cleared) { + req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0xF; + req->swap.bg_color = 0xFF000000; + dcp->surfaces_cleared = true; + } + + // Surface 0 has limitations at least on t600x. + l = 1; + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { + struct drm_framebuffer *fb = new_state->fb; + struct drm_gem_dma_object *obj; + struct drm_rect src_rect; + bool is_premultiplied = false; + + /* skip planes not for this crtc */ + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + + WARN_ON(l >= SWAP_SURFACES); + + req->swap.swap_enabled |= BIT(l); + + if (old_state->fb && fb != old_state->fb) { + /* + * Race condition between a framebuffer unbind getting + * swapped out and GEM unreferencing a framebuffer. If + * we lose the race, the display gets IOVA faults and + * the DCP crashes. We need to extend the lifetime of + * the drm_framebuffer (and hence the GEM object) until + * after we get a swap complete for the swap unbinding + * it. + */ + struct dcp_fb_reference *entry = + kzalloc(sizeof(*entry), GFP_KERNEL); + if (entry) { + entry->fb = old_state->fb; + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + } + drm_framebuffer_get(old_state->fb); + } + + if (!new_state->fb) { + l += 1; + continue; + } + req->surf_null[l] = false; + has_surface = 1; + + /* + * DCP doesn't support XBGR8 / XRGB8 natively. Blending as + * pre-multiplied alpha with a black background can be used as + * workaround for the bottommost plane. + */ + if (fb->format->format == DRM_FORMAT_XRGB8888 || + fb->format->format == DRM_FORMAT_XBGR8888) + is_premultiplied = true; + + drm_rect_fp_to_int(&src_rect, &new_state->src); + + req->swap.src_rect[l] = drm_to_dcp_rect(&src_rect); + req->swap.dst_rect[l] = drm_to_dcp_rect(&new_state->dst); + + if (dcp->notch_height > 0) + req->swap.dst_rect[l].y += dcp->notch_height; + + /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts + * the address for source x/y offsets. Since IOMFB has a direct + * support source position prefer that. + */ + obj = drm_fb_dma_get_gem_obj(fb, 0); + if (obj) + req->surf_iova[l] = obj->dma_addr + fb->offsets[0]; + + req->surf[l] = (struct DCP_FW_NAME(dcp_surface)){ + .is_premultiplied = is_premultiplied, + .format = drm_format_to_dcp(fb->format->format), + .xfer_func = DCP_XFER_FUNC_SDR, + .colorspace = DCP_COLORSPACE_NATIVE, + .stride = fb->pitches[0], + .width = fb->width, + .height = fb->height, + .buf_size = fb->height * fb->pitches[0], + .surface_id = req->swap.surf_ids[l], + + /* Only used for compressed or multiplanar surfaces */ + .pix_size = 1, + .pel_w = 1, + .pel_h = 1, + .has_comp = 1, + .has_planes = 1, + }; + + l += 1; + } + + if (modeset) { + struct dcp_display_mode *mode; + struct dcp_wait_cookie *cookie; + int ret; + + mode = lookup_mode(dcp, &crtc_state->mode); + if (!mode) { + dev_warn(dcp->dev, "no match for " DRM_MODE_FMT, + DRM_MODE_ARG(&crtc_state->mode)); + schedule_work(&dcp->vblank_wq); + return; + } + + dev_info(dcp->dev, "set_digital_out_mode(color:%d timing:%d)", + mode->color_mode_id, mode->timing_mode_id); + dcp->mode = (struct dcp_set_digital_out_mode_req){ + .color_mode_id = mode->color_mode_id, + .timing_mode_id = mode->timing_mode_id + }; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) { + schedule_work(&dcp->vblank_wq); + return; + } + + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); + + dev_dbg(dcp->dev, "%s - wait for modeset", __func__); + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(500)); + + kref_put(&cookie->refcount, release_wait_cookie); + + if (ret == 0) { + dev_dbg(dcp->dev, "set_digital_out_mode 200 ms"); + schedule_work(&dcp->vblank_wq); + return; + } else if (ret > 0) { + dev_dbg(dcp->dev, + "set_digital_out_mode finished with %d to spare", + jiffies_to_msecs(ret)); + } + + dcp->valid_mode = true; + } + + if (!has_surface && !crtc_state->color_mgmt_changed) { + if (crtc_state->enable && crtc_state->active && + !crtc_state->planes_changed) { + schedule_work(&dcp->vblank_wq); + return; + } + + /* Set black background */ + req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; + req->swap.bg_color = 0xFF000000; + req->clear = 1; + } + + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + + /* update brightness if changed */ + if (dcp->brightness.update) { + req->swap.bl_unk = 1; + req->swap.bl_value = dcp->brightness.dac; + req->swap.bl_power = 0x40; + dcp->brightness.update = false; + } + + do_swap(dcp, NULL, NULL); +} + +static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct apple_connector *connector; + int result = *(int *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); + + dcp->main_display = result != 0; + + connector = dcp->connector; + if (connector) { + connector->connected = dcp->nr_modes > 0; + schedule_work(&connector->hotplug_wq); + } + + dcp->active = true; + complete(&dcp->start_done); +} + +static void init_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_is_main_display(dcp, false, res_is_main_display, NULL); +} + +static void init_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_first_client_open(dcp, false, init_3, NULL); +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 val = 0; + dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); +} + +static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct iomfb_get_color_remap_mode_req color_remap = + (struct iomfb_get_color_remap_mode_req){ + .mode = 6, + }; + + dev_info(dcp->dev, "DCP booted\n"); + + iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); +} + +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) +{ + struct dcp_set_power_state_req req = { + /* defaults are ok */ + }; + + dcp_set_power_state(dcp, false, &req, NULL, NULL); +} diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h new file mode 100644 index 00000000000000..539ec65e5825f4 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +/* + * This file is intended to be included multiple times with IOMFB_VER + * defined to declare DCP firmware version dependent structs. + */ + +#ifdef DCP_FW_VER + +#include + +#include + +#include "iomfb.h" +#include "version_utils.h" + +struct DCP_FW_NAME(dcp_swap) { + u64 ts1; + u64 ts2; + u64 unk_10[6]; + u64 flags1; + u64 flags2; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 bg_color; + u8 unk_110[0x1b8]; + u32 unk_2c8; + u8 unk_2cc[0x14]; + u32 unk_2e0; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u16 unk_2e2; +#else + u8 unk_2e2[3]; +#endif + u64 bl_unk; + u32 bl_value; // min value is 0x10000000 + u8 bl_power; // constant 0x40 for on + u8 unk_2f3[0x2d]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unk_320[0x13f]; +#endif +} __packed; + +/* Information describing a surface */ +struct DCP_FW_NAME(dcp_surface) { + u8 is_tiled; + u8 is_tearing_allowed; + u8 is_premultiplied; + u32 plane_cnt; + u32 plane_cnt2; + u32 format; /* DCP fourcc */ + u32 ycbcr_matrix; + u8 xfer_func; + u8 colorspace; + u32 stride; + u16 pix_size; + u8 pel_w; + u8 pel_h; + u32 offset; + u32 width; + u32 height; + u32 buf_size; + u64 protection_opts; + u32 surface_id; + struct dcp_component_types comp_types[MAX_PLANES]; + u64 has_comp; + struct dcp_plane_info planes[MAX_PLANES]; + u64 has_planes; + u32 compression_info[MAX_PLANES][13]; + u64 has_compr_info; + u32 unk_num; + u32 unk_denom; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u8 padding[7]; +#else + u8 padding[47]; +#endif +} __packed; + +/* Prototypes */ + +struct DCP_FW_NAME(dcp_swap_submit_req) { + struct DCP_FW_NAME(dcp_swap) swap; + struct DCP_FW_NAME(dcp_surface) surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 unk_u64_a[SWAP_SURFACES]; + struct DCP_FW_NAME(dcp_surface) surf2[5]; + u64 surf2_iova[5]; +#endif + u8 unkbool; + u64 unkdouble; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 unkU64; + u8 unkbool2; +#endif + u32 clear; // or maybe switch to default fb? +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 unkU32Ptr; +#endif + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 surf2_null[5]; +#endif + u8 unkoutbool_null; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unkU32Ptr_null; + u8 unkU32out_null; +#endif + u8 padding[1]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_submit_resp) { + u8 unkoutbool; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 unkU32out; +#endif + u32 ret; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; + u8 unkbool; + u64 swap_data; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u8 swap_info[0x6c4]; +#else + u8 swap_info[0x6c5]; +#endif + u32 unkint; + u8 swap_info_null; +} __packed; + +struct DCP_FW_NAME(dcp_map_reg_req) { + char obj[4]; + u32 index; + u32 flags; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unk_u64_null; +#endif + u8 addr_null; + u8 length_null; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 padding[1]; +#else + u8 padding[2]; +#endif +} __packed; + +struct DCP_FW_NAME(dcp_map_reg_resp) { +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 dva; +#endif + u64 addr; + u64 length; + u32 ret; +} __packed; + + +struct apple_dcp; + +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state); +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c new file mode 100644 index 00000000000000..354abbfdb24c36 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "iomfb_v12_3.h" +#include "iomfb_v13_2.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A357", dcpep_set_create_dfb), + IOMFB_METHOD("A358", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A407", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A410", dcpep_set_display_device), + IOMFB_METHOD("A411", dcpep_is_main_display), + IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A439", dcpep_set_parameter_dcp), + IOMFB_METHOD("A443", dcpep_create_default_fb), + IOMFB_METHOD("A447", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A454", dcpep_first_client_open), + IOMFB_METHOD("A455", iomfbep_last_client_close), + IOMFB_METHOD("A460", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A463", dcpep_flush_supports_power), + IOMFB_METHOD("A468", dcpep_set_power_state), +}; + +#define DCP_FW v12_3 +#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_true, /* create_backlight_service */ + [116] = dcpep_cb_boot_1, + [117] = trampoline_false, /* is_dark_boot */ + [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [120] = trampoline_read_edt_data, + [122] = trampoline_prop_start, + [123] = trampoline_prop_chunk, + [124] = trampoline_prop_end, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_get_time, + [211] = trampoline_nop, /* update_backlight_factor_prop */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [456] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; + +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.h b/drivers/gpu/drm/apple/iomfb_v12_3.h new file mode 100644 index 00000000000000..7359685d981fe5 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v12_3.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V12_3_H__ +#define __APPLE_IOMFB_V12_3_H__ + +#include "version_utils.h" + +#define DCP_FW v12_3 +#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V12_3_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v13_2.c b/drivers/gpu/drm/apple/iomfb_v13_2.c new file mode 100644 index 00000000000000..27f1d84e928a69 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v13_2.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "iomfb_v12_3.h" +#include "iomfb_v13_2.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A373", dcpep_set_create_dfb), + IOMFB_METHOD("A374", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A407", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A410", dcpep_set_display_device), + IOMFB_METHOD("A411", dcpep_is_main_display), + IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A441", dcpep_set_parameter_dcp), + IOMFB_METHOD("A445", dcpep_create_default_fb), + IOMFB_METHOD("A449", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A456", dcpep_first_client_open), + IOMFB_METHOD("A457", iomfbep_last_client_close), + IOMFB_METHOD("A462", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A465", dcpep_flush_supports_power), + IOMFB_METHOD("A471", dcpep_set_power_state), +}; + +#define DCP_FW v13_2 +#define DCP_FW_VER DCP_FW_VERSION(13, 2, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_true, /* create_backlight_service */ + [112] = trampoline_true, /* create_nvram_servce? */ + [119] = dcpep_cb_boot_1, + [120] = trampoline_false, /* is_dark_boot */ + [121] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [123] = trampoline_read_edt_data, + [125] = trampoline_prop_start, + [126] = trampoline_prop_chunk, + [127] = trampoline_prop_end, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_get_time, + [211] = trampoline_nop, /* update_backlight_factor_prop */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [456] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v13_2.h b/drivers/gpu/drm/apple/iomfb_v13_2.h new file mode 100644 index 00000000000000..f3810b727235bc --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v13_2.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V13_2_H__ +#define __APPLE_IOMFB_V13_2_H__ + +#include "version_utils.h" + +#define DCP_FW v13_2 +#define DCP_FW_VER DCP_FW_VERSION(13, 2, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V13_2_H__ */ diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h new file mode 100644 index 00000000000000..c85baea729414d --- /dev/null +++ b/drivers/gpu/drm/apple/version_utils.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_VERSION_UTILS_H__ +#define __APPLE_VERSION_UTILS_H__ + +#include + +#define DCP_FW_UNION(u) (u).DCP_FW +#define DCP_FW_SUFFIX CONCATENATE(_, DCP_FW) +#define DCP_FW_NAME(name) CONCATENATE(name, DCP_FW_SUFFIX) +#define DCP_FW_VERSION(x, y, z) ( ((x) << 16) | ((y) << 8) | (z) ) + +#endif /*__APPLE_VERSION_UTILS_H__*/ From 5d30346e2885665b25294cb9b805ba613944fc63 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 9 Mar 2023 12:44:51 +0100 Subject: [PATCH 272/635] drm/apple: ignore surf[3] in clear swap calls MacOS 13.2 does the same and it is unclear if surf[3] can be used at all. PRobably not necessary but found during debugging to firmware 13.2. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/iomfb_template.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index c6c62e4bc4c125..fb2be2669a7e5d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -841,7 +841,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) memset(swap, 0, sizeof(*swap)); swap->swap.swap_enabled = - swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0xF; + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; swap->swap.bg_color = 0xFF000000; /* @@ -1113,7 +1113,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru * sticks around. */ if (!dcp->surfaces_cleared) { - req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0xF; + req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0x7; req->swap.bg_color = 0xFF000000; dcp->surfaces_cleared = true; } From 6e8dbe0ff7b2a044f4da1a52d7e6adf615c3a9ea Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 12 Mar 2023 21:38:50 +0100 Subject: [PATCH 273/635] drm/apple: Support color transformation matrices kwin 5.27.3 adds support for "Night Color" via drm "CTM" properties. Wire CTM support up via the "set_matrix" iomfb call. Link: https://bugs.kde.org/show_bug.cgi?id=455720 Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 1 + drivers/gpu/drm/apple/iomfb.h | 14 ++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 20 +++++++++++++++++++- drivers/gpu/drm/apple/iomfb_v12_3.c | 1 + drivers/gpu/drm/apple/iomfb_v13_2.c | 1 + 5 files changed, 36 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index a4ae7afbad73bb..f52ecf62880f90 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -334,6 +334,7 @@ static int apple_probe_per_dcp(struct device *dev, return ret; drm_crtc_helper_add(&crtc->base, &apple_crtc_helper_funcs); + drm_crtc_enable_color_mgmt(&crtc->base, 0, true, 0); enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, DRM_MODE_ENCODER_TMDS); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 0c8061bb96e3e0..7cda9124bcf96b 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -160,6 +160,7 @@ enum dcpep_method { iomfbep_a358_vi_set_temperature_hint, iomfbep_get_color_remap_mode, iomfbep_last_client_close, + iomfbep_set_matrix, dcpep_num_methods }; @@ -350,4 +351,17 @@ struct iomfb_last_client_close_resp { u32 unkint; } __packed; +struct iomfb_set_matrix_req { + u32 unk_u32; // maybe length? + u64 r[3]; + u64 g[3]; + u64 b[3]; + u8 matrix_null; + u8 padding[3]; +} __packed; + +struct iomfb_set_matrix_resp { + u32 ret; +} __packed; + #endif diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index fb2be2669a7e5d..ea95fa37eee079 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -55,6 +55,7 @@ DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); +IOMFB_THUNK_INOUT(set_matrix); IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); @@ -1285,7 +1286,24 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru dcp->brightness.update = false; } - do_swap(dcp, NULL, NULL); + if (crtc_state->color_mgmt_changed && crtc_state->ctm) { + struct iomfb_set_matrix_req mat; + struct drm_color_ctm *ctm = (struct drm_color_ctm *)crtc_state->ctm->data; + + mat.unk_u32 = 9; + mat.r[0] = ctm->matrix[0]; + mat.r[1] = ctm->matrix[1]; + mat.r[2] = ctm->matrix[2]; + mat.g[0] = ctm->matrix[3]; + mat.g[1] = ctm->matrix[4]; + mat.g[2] = ctm->matrix[5]; + mat.b[0] = ctm->matrix[6]; + mat.b[1] = ctm->matrix[7]; + mat.b[2] = ctm->matrix[8]; + + iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); + } else + do_swap(dcp, NULL, NULL); } static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 354abbfdb24c36..c226a1139a84c8 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -18,6 +18,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A410", dcpep_set_display_device), IOMFB_METHOD("A411", dcpep_is_main_display), IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A422", iomfbep_set_matrix), IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), IOMFB_METHOD("A439", dcpep_set_parameter_dcp), IOMFB_METHOD("A443", dcpep_create_default_fb), diff --git a/drivers/gpu/drm/apple/iomfb_v13_2.c b/drivers/gpu/drm/apple/iomfb_v13_2.c index 27f1d84e928a69..63ae1e79adda10 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_2.c +++ b/drivers/gpu/drm/apple/iomfb_v13_2.c @@ -18,6 +18,7 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A410", dcpep_set_display_device), IOMFB_METHOD("A411", dcpep_is_main_display), IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A422", iomfbep_set_matrix), IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), IOMFB_METHOD("A441", dcpep_set_parameter_dcp), IOMFB_METHOD("A445", dcpep_create_default_fb), From a8a5aa702ab3f1adadabb6761e612325799bc86a Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 23 Mar 2023 08:40:42 +0100 Subject: [PATCH 274/635] drm/apple: Drop unsupported DRM_FORMAT_ARGB2101010 Depends on https://gitlab.freedesktop.org/asahi/mesa/-/merge_requests/5 Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 1 - drivers/gpu/drm/apple/iomfb.c | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index f52ecf62880f90..85fcaa47fb14ca 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -145,7 +145,6 @@ static const struct drm_plane_funcs apple_plane_funcs = { */ static const u32 dcp_formats[] = { DRM_FORMAT_XRGB2101010, - DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, DRM_FORMAT_ARGB8888, DRM_FORMAT_XBGR8888, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 868ed2598a4d7f..d9bd4b52dda701 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -410,7 +410,6 @@ u32 drm_format_to_dcp(u32 drm) case DRM_FORMAT_ABGR8888: return fourcc_code('A', 'B', 'G', 'R'); - case DRM_FORMAT_ARGB2101010: case DRM_FORMAT_XRGB2101010: return fourcc_code('r', '0', '3', 'w'); } From bec9b9c814220f58469588bbee589804926b474a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 21:57:22 +0900 Subject: [PATCH 275/635] dcp: Allow unused trampolines Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/iomfb_internal.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index 401b6ec32848d3..09f8857d30c341 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -57,7 +57,7 @@ typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); */ #define TRAMPOLINE_VOID(func, handler) \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ trace_iomfb_callback(dcp, tag, #handler); \ handler(dcp); \ @@ -67,7 +67,7 @@ typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); #define TRAMPOLINE_IN(func, handler, T_in) \ typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ callback_##handler cb = handler; \ \ @@ -79,7 +79,7 @@ typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); #define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ T_out *typed_out = out; \ callback_##handler cb = handler; \ @@ -90,7 +90,7 @@ typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); } #define TRAMPOLINE_OUT(func, handler, T_out) \ - static bool func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ { \ T_out *typed_out = out; \ \ From 4cd47de1f95b3314a44fbda70c363fc42d1d42d3 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 21:57:38 +0900 Subject: [PATCH 276/635] dcp: Add get_tiling_state Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/iomfb.h | 13 +++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++++++++ drivers/gpu/drm/apple/iomfb_v13_2.c | 2 ++ 3 files changed, 27 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 7cda9124bcf96b..6602bf19107d43 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -364,4 +364,17 @@ struct iomfb_set_matrix_resp { u32 ret; } __packed; +struct dcpep_get_tiling_state_req { + u32 event; + u32 param; + u32 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcpep_get_tiling_state_resp { + u32 value; + u32 ret; +} __packed; + #endif diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ea95fa37eee079..8403a65d056868 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -977,6 +977,16 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, info->width, info->height); } +static struct dcpep_get_tiling_state_resp +dcpep_cb_get_tiling_state(struct apple_dcp *dcp, + struct dcpep_get_tiling_state_req *req) +{ + return (struct dcpep_get_tiling_state_resp){ + .value = 0, + .ret = 1, + }; +} + TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); @@ -1022,6 +1032,8 @@ TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, iomfbep_cb_enable_backlight_message_ap_gated, u8); TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, struct iomfb_property); +TRAMPOLINE_INOUT(trampoline_get_tiling_state, dcpep_cb_get_tiling_state, + struct dcpep_get_tiling_state_req, struct dcpep_get_tiling_state_resp); /* * Callback for swap requests. If a swap failed, we'll never get a swap diff --git a/drivers/gpu/drm/apple/iomfb_v13_2.c b/drivers/gpu/drm/apple/iomfb_v13_2.c index 63ae1e79adda10..356a2aa2433be0 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_2.c +++ b/drivers/gpu/drm/apple/iomfb_v13_2.c @@ -51,6 +51,8 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [110] = trampoline_true, /* create_iomfb_service */ [111] = trampoline_true, /* create_backlight_service */ [112] = trampoline_true, /* create_nvram_servce? */ + [113] = trampoline_get_tiling_state, + [114] = trampoline_false, /* set_tiling_state */ [119] = dcpep_cb_boot_1, [120] = trampoline_false, /* is_dark_boot */ [121] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ From 210457a76c5e8dba8b9ce4c2c6ab87668a239e45 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 22:54:28 +0900 Subject: [PATCH 277/635] dcp: 42-bit DMA masks Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/apple_drv.c | 2 +- drivers/gpu/drm/apple/dcp.c | 2 +- drivers/gpu/drm/apple/dummy-piodma.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 85fcaa47fb14ca..69900f3da61e81 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -459,7 +459,7 @@ static int apple_drm_init(struct device *dev) resource_size_t fb_size; int ret; - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d9d847ac819d14..2e1e077a8f274f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -420,7 +420,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) u32 cpu_ctrl; int ret; - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dummy-piodma.c b/drivers/gpu/drm/apple/dummy-piodma.c index ec5d4fecf356c0..eaa0476854a603 100644 --- a/drivers/gpu/drm/apple/dummy-piodma.c +++ b/drivers/gpu/drm/apple/dummy-piodma.c @@ -26,7 +26,7 @@ static const struct component_ops dcp_piodma_comp_ops = { }; static int dcp_piodma_probe(struct platform_device *pdev) { - int ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); + int ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); if (ret) return ret; From 613ee7ed38cf46141b9b60030eaad63cacaa66e0 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 22:55:25 +0900 Subject: [PATCH 278/635] dcp: T602X bwreq support Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/iomfb_template.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 8403a65d056868..f0fdc01fae793c 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -35,6 +35,7 @@ /* Register defines used in bandwidth setup structure */ #define REG_SCRATCH (0x14) #define REG_SCRATCH_T600X (0x988) +#define REG_SCRATCH_T602X (0x1208) #define REG_DOORBELL (0x0) #define REG_DOORBELL_BIT (2) @@ -636,7 +637,7 @@ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) { - if (dcp->disp_registers[5] && dcp->disp_registers[6]) + if (dcp->disp_registers[5] && dcp->disp_registers[6]) { return (struct dcp_rt_bandwidth){ .reg_scratch = dcp->disp_registers[5]->start + REG_SCRATCH, @@ -646,19 +647,24 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) .padding[3] = 0x4, // XXX: required by 11.x firmware }; - else if (dcp->disp_registers[4]) + } else if (dcp->disp_registers[4]) { + u32 offset = REG_SCRATCH_T600X; + if (of_device_is_compatible(dcp->dev->of_node, "apple,t6020-dcp")) + offset = REG_SCRATCH_T602X; + return (struct dcp_rt_bandwidth){ .reg_scratch = dcp->disp_registers[4]->start + - REG_SCRATCH_T600X, + offset, .reg_doorbell = 0, .doorbell_bit = 0, }; - else + } else { return (struct dcp_rt_bandwidth){ .reg_scratch = 0, .reg_doorbell = 0, .doorbell_bit = 0, }; + } } /* Callback to get the current time as milliseconds since the UNIX epoch */ From 9a925bc858efc54c8e46241543102353ed79382a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 22:55:46 +0900 Subject: [PATCH 279/635] dcp: Warn if DMA mapping fails Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/iomfb_template.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f0fdc01fae793c..0e718742a92e02 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -412,6 +412,7 @@ static struct dcp_map_physical_resp dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) { int size = ALIGN(req->size, 4096); + dma_addr_t dva; u32 id; if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { @@ -425,11 +426,13 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); + return (struct dcp_map_physical_resp){ .dva_size = size, .mem_desc_id = id, - .dva = dma_map_resource(dcp->dev, req->paddr, size, - DMA_BIDIRECTIONAL, 0), + .dva = dva, }; } From 7582f4b9e2fb4dd4384fda101d7b47bfb5ff21fc Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 20 Apr 2023 23:24:59 +0200 Subject: [PATCH 280/635] PCI: apple: Skip controller port setup for online links U-boot gained recently support for PCIe controller on Apple silicon devices. It is currently unkown how to reset / retrain already brought up ports. Redoing the controller level setup breaks the links. Check the link status before performing controller level port/link setup. Link: https://lore.kernel.org/u-boot/20230121192800.82428-1-kettenis@openbsd.org/ Signed-off-by: Janne Grunau --- drivers/pci/controller/pcie-apple.c | 88 ++++++++++++++++++----------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index d2fa591f3a7f9a..0f1f5d0fe80517 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -605,15 +605,13 @@ static int apple_pcie_probe_port(struct device_node *np) return 0; } -static int apple_pcie_setup_port(struct apple_pcie *pcie, +static int apple_pcie_setup_link(struct apple_pcie *pcie, + struct apple_pcie_port *port, struct device_node *np) { - struct platform_device *platform = to_platform_device(pcie->dev); - struct apple_pcie_port *port; - struct gpio_desc *reset, *pwren = NULL; - u32 stat, idx; - int ret, i; - char name[16]; + struct gpio_desc *reset, *pwren; + u32 stat; + int ret; reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", GPIOD_OUT_LOW, "PERST#"); @@ -629,32 +627,6 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, return PTR_ERR(pwren); } - port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); - if (!port) - return -ENOMEM; - - ret = of_property_read_u32_index(np, "reg", 0, &idx); - if (ret) - return ret; - - /* Use the first reg entry to work out the port index */ - port->idx = idx >> 11; - port->pcie = pcie; - port->np = np; - - snprintf(name, sizeof(name), "port%d", port->idx); - port->base = devm_platform_ioremap_resource_byname(platform, name); - if (IS_ERR(port->base)) - port->base = devm_platform_ioremap_resource(platform, port->idx + 2); - if (IS_ERR(port->base)) { - return PTR_ERR(port->base); - } - - snprintf(name, sizeof(name), "phy%d", port->idx); - port->phy = devm_platform_ioremap_resource_byname(platform, name); - if (IS_ERR(port->phy)) - port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); - rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); /* Assert PERST# before setting up the clock */ @@ -690,6 +662,52 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, return ret; } + return 0; +} + +static int apple_pcie_setup_port(struct apple_pcie *pcie, + struct device_node *np) +{ + struct platform_device *platform = to_platform_device(pcie->dev); + struct apple_pcie_port *port; + u32 link_stat, idx; + int ret, i; + char name[16]; + + port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + ret = of_property_read_u32_index(np, "reg", 0, &idx); + if (ret) + return ret; + + /* Use the first reg entry to work out the port index */ + port->idx = idx >> 11; + port->pcie = pcie; + port->np = np; + + snprintf(name, sizeof(name), "port%d", port->idx); + port->base = devm_platform_ioremap_resource_byname(platform, name); + if (IS_ERR(port->base)) + port->base = devm_platform_ioremap_resource(platform, port->idx + 2); + if (IS_ERR(port->base)) { + return PTR_ERR(port->base); + } + + snprintf(name, sizeof(name), "phy%d", port->idx); + port->phy = devm_platform_ioremap_resource_byname(platform, name); + if (IS_ERR(port->phy)) + port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); + + /* link might be already brought up by u-boot, skip setup then */ + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + ret = apple_pcie_setup_link(pcie, port, np); + if (ret) + return ret; + } + ret = apple_pcie_port_setup_irq(port); if (ret) return ret; @@ -714,6 +732,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); + if (link_stat & PORT_LINKSTS_UP) + return 0; + + /* start link training */ writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) From 06ef11a8693b8c03ca249ef7db272334bd61cee8 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 2 Feb 2023 20:26:49 +0100 Subject: [PATCH 281/635] dt-bindings: dma: apple,admac: Add t8112-admac compatible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The block found on Apple's M2 SoC is compatible with the existing driver so add its per-SoC compatible. Acked-by: Vinod Koul Acked-by: Martin Povišer Acked-by: Krzysztof Kozlowski Signed-off-by: Janne Grunau --- Documentation/devicetree/bindings/dma/apple,admac.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/apple,admac.yaml b/Documentation/devicetree/bindings/dma/apple,admac.yaml index 05163d124ec323..ab193bc8bdbb3e 100644 --- a/Documentation/devicetree/bindings/dma/apple,admac.yaml +++ b/Documentation/devicetree/bindings/dma/apple,admac.yaml @@ -26,6 +26,7 @@ properties: - enum: - apple,t6000-admac - apple,t8103-admac + - apple,t8112-admac - const: apple,admac reg: From 7896bee60d758d8e54dffb74e3ecb75eca94b17d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 24 Apr 2023 23:31:20 +0900 Subject: [PATCH 282/635] arm64: dts: apple: t8112: Remove always-on from the PMP node This should now work properly with power domain dependencies. --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 7c050c6f2707a1..d36f6094698ddf 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -773,7 +773,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "pmp"; - apple,always-on; }; ps_pms_sram: power-controller@418 { From 84da58345778ab4c7c51e95292a553c11b58aa98 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 14 Feb 2023 10:07:49 +0100 Subject: [PATCH 283/635] arm64: dts: apple: t8112: Add wlan/bt PCIe device nodes Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-j473.dts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 06fe257f08be49..4362d9081bd7b9 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -17,7 +17,9 @@ model = "Apple Mac mini (M2, 2023)"; aliases { + bluetooth0 = &bluetooth0; ethernet0 = ðernet0; + wifi0 = &wifi0; }; }; @@ -28,6 +30,22 @@ */ &port00 { bus-range = <1 1>; + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + apple,antenna-sku = "XX"; + brcm,board-type = "apple,miyake"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + brcm,board-type = "apple,miyake"; + }; }; &port01 { From f72b15914e6b1fc797993e8ac40a05fb1fdddf4f Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 18:54:35 +0900 Subject: [PATCH 284/635] arm64: dts: apple: t8112: Add PMU NVMEM and SMC RTC/reboot nodes Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8112.dtsi | 78 ++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 1666e6ab250bc0..e68f5042e1b339 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -641,6 +641,84 @@ }; }; + nub_spmi: spmi@23d0d9300 { + compatible = "apple,t8112-spmi", "apple,spmi"; + reg = <0x2 0x3d714000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + interrupt-parent = <&aic>; + interrupts = , + ; + + pmu1: pmu@e { + compatible = "apple,stowe-pmu", "apple,spmi-pmu"; + reg = <0xe SPMI_USID>; + #address-cells = <1>; + #size-cells = <1>; + + rtc_nvmem@f800 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0xf800 0x300>; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@1 { + reg = <0x1 0x1>; + }; + + rtc_offset: rtc-offset@100 { + reg = <0x100 0x6>; + }; + }; + + legacy_nvmem@f700 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0xf700 0x20>; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@1 { + reg = <0x1 0x1>; + }; + + boot_error_count: boot-error-count@2 { + reg = <0x2 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@2 { + reg = <0x2 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@3 { + reg = <0x3 0x1>; + }; + + shutdown_flag: shutdown-flag@f { + reg = <0xf 0x1>; + bits = <3 1>; + }; + }; + + scrpad_nvmem@8000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x8000 0x2800>; + #address-cells = <1>; + #size-cells = <1>; + + fault_shadow: fault-shadow@67b { + reg = <0x67b 0x10>; + }; + + socd: socd@b00 { + reg = <0xb00 0x400>; + }; + }; + + }; + }; + pinctrl_nub: pinctrl@23d1f0000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3d1f0000 0x0 0x4000>; From 107c2a619a7fa2763ebb8c6ce921a17757c52abc Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 2 Feb 2023 14:33:26 +0100 Subject: [PATCH 285/635] arm64: dts: apple: t8112-j493: Add spi3 node Used for the touchbar, clock frequency is probably wrong. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112.dtsi | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index e68f5042e1b339..c1bf739c70c265 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -359,6 +359,13 @@ clock-output-names = "nco_ref"; }; + clk_120m: clock-120m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <120000000>; + clock-output-names = "clk_120m"; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -467,6 +474,20 @@ status = "disabled"; }; + spi3: spi@23510c000 { + compatible = "apple,t8112-spi", "apple,spi"; + reg = <0x2 0x3510c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_120m>; + pinctrl-0 = <&spi3_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi3>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + serial0: serial@235200000 { compatible = "apple,s5l-uart"; reg = <0x2 0x35200000 0x0 0x1000>; From 0f6515daa79b17e61157d3dd259688098eaaeebf Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Feb 2022 12:59:39 +0900 Subject: [PATCH 286/635] arm64: dts: apple: t8112: Add SMC node to devicetree Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8112-j413.dts | 1 + arch/arm64/boot/dts/apple/t8112-j473.dts | 2 ++ arch/arm64/boot/dts/apple/t8112-j493.dts | 1 + arch/arm64/boot/dts/apple/t8112.dtsi | 38 ++++++++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 6f69658623bf89..66049c75b8d0e1 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -42,6 +42,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 4362d9081bd7b9..33904f3c9ff8f4 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -30,6 +30,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4434"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -50,6 +51,7 @@ &port01 { bus-range = <2 2>; + pwren-gpios = <&smc_gpio 24 GPIO_ACTIVE_HIGH>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index 0ad908349f5540..1ca48f9d71c75f 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -42,6 +42,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index c1bf739c70c265..0fc995fa486544 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -778,6 +778,44 @@ interrupts = ; }; + smc_mbox: mbox@23e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x3e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@23e400000 { + compatible = "apple,t8112-smc", "apple,smc"; + reg = <0x2 0x3e400000 0x0 0x4000>, + <0x2 0x3fe00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + gpio-controller; + #gpio-cells = <2>; + }; + + smc_rtc: rtc { + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + + smc_reboot: reboot { + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>, <&pm_setting>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count", "pm_setting"; + }; + }; + pinctrl_smc: pinctrl@23e820000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3e820000 0x0 0x4000>; From e2662bd260cd617f4e9cbf762fbb651ea3370f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sat, 19 Feb 2022 09:49:59 +0100 Subject: [PATCH 287/635] arm64: dts: apple: t8112*: Put in audio nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-j413.dts | 84 +++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8112-j473.dts | 52 +++++++++++++++ arch/arm64/boot/dts/apple/t8112-j493.dts | 85 ++++++++++++++++++++++++ 3 files changed, 221 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 66049c75b8d0e1..7554d78d678c47 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -72,6 +72,51 @@ }; }; +&i2c1 { + speaker_left_woof: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + }; +}; + +&i2c3 { + speaker_right_woof: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -79,3 +124,42 @@ &fpwm1 { status = "okay"; }; + +/ { + sound { + compatible = "apple,j413-macaudio", "apple,macaudio"; + model = "MacBook Air J413 integrated audio"; + + dai-link@0 { + link-name = "Speakers"; + + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_tweet>, <&speaker_left_woof>, + <&speaker_right_tweet>, <&speaker_right_woof>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 33904f3c9ff8f4..83b254b0a6e4dd 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -72,3 +72,55 @@ &pcie2_dart { status = "okay"; }; + +&i2c1 { + speaker_amp: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <149 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J473"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index 1ca48f9d71c75f..fc0b274991316b 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -61,6 +61,51 @@ }; }; +&i2c1 { + speaker_left_rear: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + }; + + speaker_left_front: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + }; +}; + +&i2c3 { + speaker_right_rear: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + }; + + speaker_right_front: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -68,3 +113,43 @@ &fpwm1 { status = "okay"; }; + +/ { + sound { + compatible = "apple,j493-macaudio", "apple,macaudio"; + model = "MacBook Pro J493 integrated audio"; + + dai-link@0 { + link-name = "Speakers"; + + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + From 2fa0ea9b92c95985f9bf3c7bfee62db11bd4ecc3 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 2 Feb 2023 12:50:56 +0100 Subject: [PATCH 288/635] arm64: dts: apple: t8112: Add dwc3 nodes Signed-off-by: Hector Martin Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-j413.dts | 12 +++++ arch/arm64/boot/dts/apple/t8112-j473.dts | 12 +++++ arch/arm64/boot/dts/apple/t8112-j493.dts | 12 +++++ arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 51 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 60 +++++++++++++++++++++++ 5 files changed, 147 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 7554d78d678c47..b20b795950be3d 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -61,6 +61,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c0 { /* MagSafe port */ hpm5: usb-pd@3a { diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 83b254b0a6e4dd..4705ec980211c4 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -23,6 +23,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-left"; +}; + +&typec1 { + label = "USB-C Back-right"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index fc0b274991316b..f158a004a28ca3 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -61,6 +61,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c1 { speaker_left_rear: codec@38 { compatible = "ti,sn012776", "ti,tas2764"; diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index f5edf61113e7aa..2c14f4479c401f 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -53,6 +53,23 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_con_hs: endpoint { + remote-endpoint = <&typec0_usb_hs>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -61,6 +78,40 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_con_hs: endpoint { + remote-endpoint = <&typec1_usb_hs>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + port { + typec0_usb_hs: endpoint { + remote-endpoint = <&typec0_con_hs>; + }; + }; +}; + +&dwc3_1 { + port { + typec1_usb_hs: endpoint { + remote-endpoint = <&typec1_con_hs>; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 0fc995fa486544..f1ed8f408644f1 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -892,6 +892,66 @@ resets = <&ps_ans>; }; + dwc3_0: usb@382280000 { + compatible = "apple,t8112-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x3 0x82280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_0: iommu@382f00000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x3 0x82f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_1: iommu@382f80000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x3 0x82f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@502280000 { + compatible = "apple,t8112-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x5 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_0: iommu@502f00000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x5 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_1: iommu@502f80000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x5 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart: iommu@681008000 { compatible = "apple,t8110-dart"; reg = <0x6 0x81008000 0x0 0x4000>; From 042842b927cb4269e9c335466e9d62651a2016af Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 21 Feb 2022 23:01:36 +0100 Subject: [PATCH 289/635] arm64: dts: apple: Add backlight node to j413/j493 It can be turned off with: echo 1 > /sys/class/backlight/backlight/bl_power It can be turned on with: echo 0 > /sys/class/backlight/backlight/bl_power Needs CONFIG_BACKLIGHT_GPIO=m. Signed-off-by: Thomas Glanzmann Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-j413.dts | 10 ++++++++++ arch/arm64/boot/dts/apple/t8112-j493.dts | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index b20b795950be3d..ba070eef1eb4f6 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -138,6 +138,12 @@ }; / { + backlight: gpio-bl { + compatible = "gpio-backlight"; + gpios = <&smc_gpio 18 GPIO_ACTIVE_HIGH>; + default-on; + }; + sound { compatible = "apple,j413-macaudio", "apple,macaudio"; model = "MacBook Air J413 integrated audio"; @@ -175,3 +181,7 @@ }; }; }; + +&framebuffer0 { + backlight = <&backlight>; +}; diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index f158a004a28ca3..e5e596e02be014 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -127,6 +127,12 @@ }; / { + backlight: gpio-bl { + compatible = "gpio-backlight"; + gpios = <&smc_gpio 18 GPIO_ACTIVE_HIGH>; + default-on; + }; + sound { compatible = "apple,j493-macaudio", "apple,macaudio"; model = "MacBook Pro J493 integrated audio"; @@ -165,3 +171,6 @@ }; }; +&framebuffer0 { + backlight = <&backlight>; +}; From cdf5f1bce352224fe825fd56e0d161b74766a4c5 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 2 Feb 2023 11:15:35 +0100 Subject: [PATCH 290/635] arm64: dts: apple: t8112: Add mtp device nodes for j413/j493 Those provide trackpad and keyboard for j413/j493. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-j413.dts | 32 +++++++++++ arch/arm64/boot/dts/apple/t8112-j493.dts | 32 +++++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 73 ++++++++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index ba070eef1eb4f6..27ed249f5aaaeb 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -185,3 +185,35 @@ &framebuffer0 { backlight = <&backlight>; }; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + multi-touch { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + firmware-name = "apple/tpmtfw-j413.bin"; + }; + + keyboard { + }; + + stm { + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index e5e596e02be014..7e053a80602e41 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -174,3 +174,35 @@ &framebuffer0 { backlight = <&backlight>; }; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + multi-touch { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + firmware-name = "apple/tpmtfw-j493.bin"; + }; + + keyboard { + }; + + stm { + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index f1ed8f408644f1..749231a16f0a36 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -858,6 +858,79 @@ ; }; + mtp: mtp@24e400000 { + compatible = "apple,t8112-mtp", "apple,t8112-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0x4e400000 0x0 0x4000>, + <0x2 0x4ec00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@24e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@24e808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4e808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@24eb14000 { + compatible = "apple,t8112-dockchannel", "apple,dockchannel"; + reg = <0x2 0x4eb14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0x4eb28000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; From 690b532bcb1ce3a8c68305b65641adb2df938feb Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 26 Nov 2021 15:37:23 +0900 Subject: [PATCH 291/635] arm64: dts: apple: t8103: Add dwc3 nodes Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103-j274.dts | 12 +++++ arch/arm64/boot/dts/apple/t8103-j293.dts | 12 +++++ arch/arm64/boot/dts/apple/t8103-j313.dts | 12 +++++ arch/arm64/boot/dts/apple/t8103-j456.dts | 12 +++++ arch/arm64/boot/dts/apple/t8103-j457.dts | 12 +++++ arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 51 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 60 +++++++++++++++++++++++ 7 files changed, 171 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 1c3e37f86d46d7..968fe22163d443 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -29,6 +29,18 @@ brcm,board-type = "apple,atlantisb"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-left"; +}; + +&typec1 { + label = "USB-C Back-right"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 56b0c67bfcda32..58ab9b4f765ef7 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -38,6 +38,18 @@ brcm,board-type = "apple,honshu"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; + &i2c2 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 97a4344d8dca68..bce9b911009e2b 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -41,3 +41,15 @@ &fpwm1 { status = "okay"; }; + +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Left-back"; +}; + +&typec1 { + label = "USB-C Left-front"; +}; diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 58c8e43789b486..9983e11cacdf19 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -47,6 +47,18 @@ }; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-right-middle"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 152f95fd49a211..a622ff607d4075 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -29,6 +29,18 @@ brcm,board-type = "apple,santorini"; }; +/* + * Provide labels for the USB type C ports. + */ + +&typec0 { + label = "USB-C Back-right"; +}; + +&typec1 { + label = "USB-C Back-left"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 5988a4eb6efaa0..36dba76c06fd47 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -53,6 +53,23 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <106 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_con_hs: endpoint { + remote-endpoint = <&typec0_usb_hs>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -61,6 +78,40 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <106 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_con_hs: endpoint { + remote-endpoint = <&typec1_usb_hs>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + port { + typec0_usb_hs: endpoint { + remote-endpoint = <&typec0_con_hs>; + }; + }; +}; + +&dwc3_1 { + port { + typec1_usb_hs: endpoint { + remote-endpoint = <&typec1_con_hs>; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 9b0dad6b618444..4580f207fa1f07 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -717,6 +717,66 @@ resets = <&ps_ans2>; }; + dwc3_0: usb@382280000 { + compatible = "apple,t8103-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x3 0x82280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_0: iommu@382f00000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0x82f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_0_dart_1: iommu@382f80000 { + compatible = "apple,t8103-dart"; + reg = <0x3 0x82f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + + dwc3_1: usb@502280000 { + compatible = "apple,t8103-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x5 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_0: iommu@502f00000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + + dwc3_1_dart_1: iommu@502f80000 { + compatible = "apple,t8103-dart"; + reg = <0x5 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart_0: iommu@681008000 { compatible = "apple,t8103-dart"; reg = <0x6 0x81008000 0x0 0x4000>; From 11f350a2b263717c4a1778b8a613402726cde687 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 26 Nov 2021 00:24:15 +0100 Subject: [PATCH 292/635] arm64: dts: apple: t8103: Add spi3/keyboard nodes Enables keyboard and touchpad input on MacBook Air (M1, 2020) and MacBook Pro (13-inch, M1, 2020). Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-j293.dts | 21 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8103-j313.dts | 21 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8103.dtsi | 28 ++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 58ab9b4f765ef7..0be0437c3ac230 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -50,6 +50,27 @@ label = "USB-C Left-front"; }; +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + &i2c2 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index bce9b911009e2b..7b13e16957ead7 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -53,3 +53,24 @@ &typec1 { label = "USB-C Left-front"; }; + +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 4580f207fa1f07..4318c89fc6cbe4 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -326,6 +326,13 @@ clock-output-names = "clkref"; }; + clk_120m: clock-120m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <120000000>; + clock-output-names = "clk_120m"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -441,6 +448,20 @@ status = "disabled"; }; + spi3: spi@23510c000 { + compatible = "apple,t8103-spi", "apple,spi"; + reg = <0x2 0x3510c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_120m>; + pinctrl-0 = <&spi3_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi3>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; /* only used in J293/J313 */ + }; + serial0: serial@235200000 { compatible = "apple,s5l-uart"; reg = <0x2 0x35200000 0x0 0x1000>; @@ -597,6 +618,13 @@ ; }; + spi3_pins: spi3-pins { + pinmux = , + , + , + ; + }; + pcie_pins: pcie-pins { pinmux = , , From b9f8e9260bc3ad9e3b20dd25a2c0408bb9925367 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 6 Feb 2022 21:22:29 +0900 Subject: [PATCH 293/635] arm64: dts: apple: Add PCI power enable GPIOs t8103: - WLAN (SMC PMU GPIO #13) t600x: - WLAN (SMC PMU GPIO #13) - SD (SMC PMU GPIO #26) Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 36dba76c06fd47..b70cd5f64b4cc6 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -122,6 +122,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: network@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; From 311083995bcf9dd0cf94ec2c1b847dacaa027629 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Feb 2022 12:59:39 +0900 Subject: [PATCH 294/635] arm64: dts: apple: Add SMC node to t8103/t6001 devicetrees Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103.dtsi | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 4318c89fc6cbe4..e28d07e6954d19 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -669,6 +669,32 @@ interrupts = ; }; + smc_mbox: mbox@23e408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x3e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@23e400000 { + compatible = "apple,t8103-smc", "apple,smc"; + reg = <0x2 0x3e400000 0x0 0x4000>, + <0x2 0x3fe00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + gpio-controller; + #gpio-cells = <2>; + }; + }; + pinctrl_smc: pinctrl@23e820000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x3e820000 0x0 0x4000>; From d65eda85305a407443cd4c72e83d7e32943f6303 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 18:54:35 +0900 Subject: [PATCH 295/635] arm64: dts: apple: Add PMU NVMEM and SMC RTC/reboot nodes Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103.dtsi | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index e28d07e6954d19..db00ef3de56385 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include / { compatible = "apple,t8103", "apple,arm-platform"; @@ -632,6 +633,81 @@ }; }; + nub_spmi: spmi@23d0d9300 { + compatible = "apple,t8103-spmi", "apple,spmi"; + reg = <0x2 0x3d0d9300 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + + pmu1: pmu@f { + compatible = "apple,sera-pmu", "apple,spmi-pmu"; + reg = <0xf SPMI_USID>; + #address-cells = <1>; + #size-cells = <1>; + + rtc_nvmem@d000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0xd000 0x300>; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@1 { + reg = <0x1 0x1>; + }; + + rtc_offset: rtc-offset@100 { + reg = <0x100 0x6>; + }; + }; + + legacy_nvmem@9f00 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x9f00 0x20>; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@1 { + reg = <0x1 0x1>; + }; + + boot_error_count: boot-error-count@2 { + reg = <0x2 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@2 { + reg = <0x2 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@3 { + reg = <0x3 0x1>; + }; + + shutdown_flag: shutdown-flag@f { + reg = <0xf 0x1>; + bits = <3 1>; + }; + }; + + scrpad_nvmem@a000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0xa000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + fault_shadow: fault-shadow@67b { + reg = <0x67b 0x10>; + }; + + socd: socd@b00 { + reg = <0xb00 0x400>; + }; + }; + + }; + }; + pinctrl_nub: pinctrl@23d1f0000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x3d1f0000 0x0 0x4000>; @@ -693,6 +769,18 @@ gpio-controller; #gpio-cells = <2>; }; + + smc_rtc: rtc { + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + + smc_reboot: reboot { + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>, <&pm_setting>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count", "pm_setting"; + }; }; pinctrl_smc: pinctrl@23e820000 { From 9d49b14b769dbe3a7b04a8f3a4886281a5396b58 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 3 Mar 2022 02:20:39 +0900 Subject: [PATCH 296/635] arm64: dts: apple: Mark ATC USB AON domains as always-on Shutting these down breaks dwc3 init done by the firmware. We probably never want to do this anyway. It might be possible remove this once a PHY driver is in place to do the init properly, but it may not be worth it. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 9645861a858c1a..1646e82bdc3692 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -1103,6 +1103,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc0_usb_aon"; + apple,always-on; /* Needs to stay on for dwc3 to work */ }; ps_atc1_usb_aon: power-controller@90 { @@ -1111,6 +1112,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "atc1_usb_aon"; + apple,always-on; /* Needs to stay on for dwc3 to work */ }; ps_atc0_usb: power-controller@98 { From d76d87a83c5a5b58ef9d487c306dc20085b27d28 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 21 Feb 2022 23:01:36 +0100 Subject: [PATCH 297/635] arm64: dts: apple: Add backlight node to j293/j313 It can be turned off with: echo 1 > /sys/class/backlight/backlight/bl_power It can be turned on with: echo 0 > /sys/class/backlight/backlight/bl_power Needs CONFIG_BACKLIGHT_GPIO=m. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8103-j293.dts | 12 ++++++++++++ arch/arm64/boot/dts/apple/t8103-j313.dts | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 0be0437c3ac230..bf39433ca10dda 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -82,3 +82,15 @@ &fpwm1 { status = "okay"; }; + +/ { + backlight: gpio-bl { + compatible = "gpio-backlight"; + gpios = <&smc_gpio 18 GPIO_ACTIVE_HIGH>; + default-on; + }; +}; + +&framebuffer0 { + backlight = <&backlight>; +}; diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 7b13e16957ead7..ff3dc749418878 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -74,3 +74,15 @@ interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; }; }; + +/ { + backlight: gpio-bl { + compatible = "gpio-backlight"; + gpios = <&smc_gpio 18 GPIO_ACTIVE_HIGH>; + default-on; + }; +}; + +&framebuffer0 { + backlight = <&backlight>; +}; From c5de5c9a05d17f1f64424f0de2e1c6b19dd3f9ad Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 17 Mar 2022 23:49:07 +0900 Subject: [PATCH 298/635] arm64: dts: apple: Keep PCIe power domain on This causes flakiness if shut down; don't do it until we find out what's going on. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 1646e82bdc3692..18c1814585eb24 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -717,6 +717,7 @@ #reset-cells = <0>; label = "apcie_gp"; power-domains = <&ps_apcie>; + apple,always-on; /* Breaks things if shut down */ }; ps_ans2: power-controller@3f0 { From 9ed48f311f5615227338f4ec410e361ea0d6e881 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 4 Feb 2022 12:59:39 +0900 Subject: [PATCH 299/635] arm64: dts: apple: Add SMC node to t600x devicetrees Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index b1c875e692c8fb..f787d9267c2fc8 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -24,6 +24,32 @@ power-domains = <&ps_aic>; }; + smc_mbox: mbox@290408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x90408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@290400000 { + compatible = "apple,t6000-smc", "apple,smc"; + reg = <0x2 0x90400000 0x0 0x4000>, + <0x2 0x91e00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + gpio-controller; + #gpio-cells = <2>; + }; + }; + pinctrl_smc: pinctrl@290820000 { compatible = "apple,t6000-pinctrl", "apple,pinctrl"; reg = <0x2 0x90820000 0x0 0x4000>; From 8afd779d17641526f6e70e94da98a5b4237cb398 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 15 Feb 2022 18:54:35 +0900 Subject: [PATCH 300/635] arm64: dts: apple: Add PMU NVMEM and SMC RTC/reboot nodes Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t6001.dtsi | 1 + arch/arm64/boot/dts/apple/t6002.dtsi | 1 + arch/arm64/boot/dts/apple/t600x-die0.dtsi | 87 +++++++++++++++++++++++ 3 files changed, 89 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 620b17e4031f06..d2cf81926f284c 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include "multi-die-cpp.h" diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index a963a5011799a0..e36f422d257d8f 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include "multi-die-cpp.h" diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index f787d9267c2fc8..6fa873ffcbb5c0 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -48,6 +48,18 @@ gpio-controller; #gpio-cells = <2>; }; + + smc_rtc: rtc { + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + + smc_reboot: reboot { + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>, <&pm_setting>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count", "pm_setting"; + }; }; pinctrl_smc: pinctrl@290820000 { @@ -79,6 +91,81 @@ interrupts = ; }; + nub_spmi0: spmi@2920a1300 { + compatible = "apple,t6000-spmi", "apple,spmi"; + reg = <0x2 0x920a1300 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + + pmu1: pmu@f { + compatible = "apple,maverick-pmu", "apple,spmi-pmu"; + reg = <0xf SPMI_USID>; + #address-cells = <1>; + #size-cells = <1>; + + rtc_nvmem@1400 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x1400 0x20>; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@5 { + reg = <0x5 0x1>; + }; + + rtc_offset: rtc-offset@11 { + reg = <0x11 0x6>; + }; + }; + + legacy_nvmem@6000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x6000 0x20>; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@1 { + reg = <0x1 0x1>; + }; + + boot_error_count: boot-error-count@2 { + reg = <0x2 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@2 { + reg = <0x2 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@3 { + reg = <0x3 0x1>; + }; + + shutdown_flag: shutdown-flag@f { + reg = <0xf 0x1>; + bits = <3 1>; + }; + }; + + scrpad_nvmem@8000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x8000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + fault_shadow: fault-shadow@67b { + reg = <0x67b 0x10>; + }; + + socd: socd@b00 { + reg = <0xb00 0x400>; + }; + }; + + }; + }; + sio_dart_0: iommu@39b004000 { compatible = "apple,t6000-dart"; reg = <0x3 0x9b004000 0x0 0x4000>; From 6d37df837040f2fa9cd31a6352a564893c5658cb Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 9 Dec 2021 21:58:10 +0900 Subject: [PATCH 301/635] arm64: dts: apple: t6000: Add spi1 node Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-common.dtsi | 7 +++++++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 14 ++++++++++++++ arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi | 7 +++++++ 3 files changed, 28 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index fa8ead69936366..87dfc13d74171f 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -362,6 +362,13 @@ clock-output-names = "clkref"; }; + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 6fa873ffcbb5c0..ac30b5933e8a0e 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -276,6 +276,20 @@ status = "disabled"; }; + spi1: spi@39b104000 { + compatible = "apple,t6000-spi", "apple,spi"; + reg = <0x3 0x9b104000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk_200m>; + pinctrl-0 = <&spi1_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi1>; + status = "disabled"; + }; + serial0: serial@39b200000 { compatible = "apple,s5l-uart"; reg = <0x3 0x9b200000 0x0 0x1000>; diff --git a/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi index b31f1a7a2b3fc3..855dcf30a50292 100644 --- a/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi @@ -36,6 +36,13 @@ ; }; + spi1_pins: spi1-pins { + pinmux = , + , + , + ; + }; + pcie_pins: pcie-pins { pinmux = , , From 19ffa1c9dd45c0ec6bd8b45c27971ea2449e5bd8 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 9 Dec 2021 21:58:29 +0900 Subject: [PATCH 302/635] arm64: dts: apple: t600x-j314-j316: Add NOR flash node Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 2e471dfe43cf88..c1f1a4b385fca7 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -94,6 +94,18 @@ clock-frequency = <1068000000>; }; +&spi1 { + status = "disabled"; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0x0>; + spi-max-frequency = <25000000>; + #address-cells = <1>; + #size-cells = <1>; + }; +}; + /* PCIe devices */ &port00 { /* WLAN */ From 0be9954f124ba79e47320e682856bcf0a4f24abb Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 11 Nov 2021 21:31:21 +0100 Subject: [PATCH 303/635] arm64: dts: apple: t600x: Add spi3 node Used for keyboard and touchpad input on MacBook Pro (14/16-inch, M1 Pro/Max, 2021). Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 14 ++++++++++++++ arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index ac30b5933e8a0e..152c878ab3e005 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -290,6 +290,20 @@ status = "disabled"; }; + spi3: spi@39b10c000 { + compatible = "apple,t6000-spi", "apple,spi"; + reg = <0x3 0x9b10c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clkref>; + pinctrl-0 = <&spi3_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi3>; + status = "disabled"; + }; + serial0: serial@39b200000 { compatible = "apple,s5l-uart"; reg = <0x3 0x9b200000 0x0 0x1000>; diff --git a/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi index 855dcf30a50292..1a994c3c1b79f0 100644 --- a/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-gpio-pins.dtsi @@ -43,6 +43,13 @@ ; }; + spi3_pins: spi3-pins { + pinmux = , + , + , + ; + }; + pcie_pins: pcie-pins { pinmux = , , From e608cb643913f6058c2f455e776cd97da2e1e311 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Thu, 11 Nov 2021 21:31:21 +0100 Subject: [PATCH 304/635] arm64: dts: apple: j31[46]: Add keyboard nodes Enables keyboard and touchpad input on MacBook Pro (14/16-inch, M1 Pro/Max, 2021). Signed-off-by: Janne Grunau --- .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index c1f1a4b385fca7..02571363c7ae71 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -106,6 +106,27 @@ }; }; +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 194 0>; + interrupts-extended = <&pinctrl_nub 6 IRQ_TYPE_LEVEL_LOW>; + }; +}; + /* PCIe devices */ &port00 { /* WLAN */ From 9dc5882846344afd37e314eea40d5ba3c1e7b7b6 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 17 May 2022 23:54:26 +0200 Subject: [PATCH 305/635] arm64: dts: apple: t600x: Add dwc3 nodes Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t6002-j375d.dts | 64 +++++++++ arch/arm64/boot/dts/apple/t600x-dieX.dtsi | 124 ++++++++++++++++++ .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 92 +++++++++++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 105 +++++++++++++++ 4 files changed, 385 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6002-j375d.dts b/arch/arm64/boot/dts/apple/t6002-j375d.dts index 3365429bdc8be9..d3d4f4461deecb 100644 --- a/arch/arm64/boot/dts/apple/t6002-j375d.dts +++ b/arch/arm64/boot/dts/apple/t6002-j375d.dts @@ -26,6 +26,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec4: connector { + compatible = "usb-c-connector"; + label = "USB-C Front Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec4_con_hs: endpoint { + remote-endpoint = <&typec4_usb_hs>; + }; + }; + }; + }; }; /* front-left */ @@ -35,9 +53,55 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec5: connector { + compatible = "usb-c-connector"; + label = "USB-C Front Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec5_con_hs: endpoint { + remote-endpoint = <&typec5_usb_hs>; + }; + }; + }; + }; + }; +}; + +/* USB controllers on die 1 */ +&dwc3_0_die1 { + port { + typec4_usb_hs: endpoint { + remote-endpoint = <&typec4_con_hs>; + }; }; }; +&dwc3_1_die1 { + port { + typec5_usb_hs: endpoint { + remote-endpoint = <&typec5_con_hs>; + }; + }; +}; + +/* delete unused USB nodes on die 1 */ + +/delete-node/ &dwc3_2_dart_0_die1; +/delete-node/ &dwc3_2_dart_1_die1; +/delete-node/ &dwc3_2_die1; + +/delete-node/ &dwc3_3_dart_0_die1; +/delete-node/ &dwc3_3_dart_1_die1; +/delete-node/ &dwc3_3_die1; + + /* delete unused always-on power-domains on die 1 */ /delete-node/ &ps_atc2_usb_aon_die1; diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index a32ff0c9d7b0c2..a5f2ef5aab7929 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -119,3 +119,127 @@ interrupt-controller; #interrupt-cells = <2>; }; + + DIE_NODE(dwc3_0_dart_0): iommu@702f00000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0_dart_1): iommu@702f80000 { + compatible = "apple,t6000-dart"; + reg = <0x7 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0): usb@702280000 { + compatible = "apple,t6000-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x7 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + /* dr_mode = "otg"; */ + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, + <&DIE_NODE(dwc3_0_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + }; + + DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1_dart_1): iommu@b02f80000 { + compatible = "apple,t6000-dart"; + reg = <0xb 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1): usb@b02280000 { + compatible = "apple,t6000-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0xb 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + /* dr_mode = "otg"; */ + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, + <&DIE_NODE(dwc3_1_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + }; + + DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2_dart_1): iommu@f02f80000 { + compatible = "apple,t6000-dart"; + reg = <0xf 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2): usb@f02280000 { + compatible = "apple,t6000-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0xf 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + /* dr_mode = "otg"; */ + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, + <&DIE_NODE(dwc3_2_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + }; + + DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3_dart_1): iommu@1302f80000 { + compatible = "apple,t6000-dart"; + reg = <0x13 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3): usb@1302280000 { + compatible = "apple,t6000-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x13 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + /* dr_mode = "otg"; */ + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, + <&DIE_NODE(dwc3_3_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 02571363c7ae71..ebd7db2e08efa4 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -62,6 +62,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left Rear"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_con_hs: endpoint { + remote-endpoint = <&typec0_usb_hs>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -70,6 +88,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left Front"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_con_hs: endpoint { + remote-endpoint = <&typec1_usb_hs>; + }; + }; + }; + }; }; hpm2: usb-pd@3b { @@ -78,6 +114,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_con_hs: endpoint { + remote-endpoint = <&typec2_usb_hs>; + }; + }; + }; + }; }; /* MagSafe port */ @@ -152,3 +206,41 @@ &fpwm0 { status = "okay"; }; + +/* USB controllers */ +&dwc3_0 { + port { + typec0_usb_hs: endpoint { + remote-endpoint = <&typec0_con_hs>; + }; + }; +}; + +&dwc3_1 { + port { + typec1_usb_hs: endpoint { + remote-endpoint = <&typec1_con_hs>; + }; + }; +}; + +&dwc3_2 { + port { + typec2_usb_hs: endpoint { + remote-endpoint = <&typec2_con_hs>; + }; + }; +}; + +/* ATC3 is used for DisplayPort -> HDMI only */ +&dwc3_3_dart_0 { + status = "disabled"; +}; + +&dwc3_3_dart_1 { + status = "disabled"; +}; + +&dwc3_3 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 1e5a19e49b089d..68cbe5bd801d25 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -48,6 +48,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Left"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_con_hs: endpoint { + remote-endpoint = <&typec0_usb_hs>; + }; + }; + }; + }; }; hpm1: usb-pd@3f { @@ -56,6 +74,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Left Middle"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec1_con_hs: endpoint { + remote-endpoint = <&typec1_usb_hs>; + }; + }; + }; + }; }; hpm2: usb-pd@3b { @@ -64,6 +100,24 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec2: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Right Middle"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec2_con_hs: endpoint { + remote-endpoint = <&typec2_usb_hs>; + }; + }; + }; + }; }; hpm3: usb-pd@3c { @@ -72,6 +126,57 @@ interrupt-parent = <&pinctrl_ap>; interrupts = <174 IRQ_TYPE_LEVEL_LOW>; interrupt-names = "irq"; + + typec3: connector { + compatible = "usb-c-connector"; + label = "USB-C Back Right"; + power-role = "dual"; + data-role = "dual"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec3_con_hs: endpoint { + remote-endpoint = <&typec3_usb_hs>; + }; + }; + }; + }; + }; +}; + +/* USB controllers */ +&dwc3_0 { + port { + typec0_usb_hs: endpoint { + remote-endpoint = <&typec0_con_hs>; + }; + }; +}; + +&dwc3_1 { + port { + typec1_usb_hs: endpoint { + remote-endpoint = <&typec1_con_hs>; + }; + }; +}; + +&dwc3_2 { + port { + typec2_usb_hs: endpoint { + remote-endpoint = <&typec2_con_hs>; + }; + }; +}; + +&dwc3_3 { + port { + typec3_usb_hs: endpoint { + remote-endpoint = <&typec3_con_hs>; + }; }; }; From b764817592d858ae1d183ba3cd59646cba1cf6b1 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 21 Dec 2021 17:07:17 +0900 Subject: [PATCH 306/635] arm64: dts: apple: Add WiFi module and antenna properties Add the new module-instance/antenna-sku properties required to select WiFi firmwares properly to all board device trees. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 4 ++++ arch/arm64/boot/dts/apple/t6000-j316s.dts | 4 ++++ arch/arm64/boot/dts/apple/t6001-j314c.dts | 4 ++++ arch/arm64/boot/dts/apple/t6001-j316c.dts | 4 ++++ arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 2 ++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 3 +++ 6 files changed, 21 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index c9e192848fe3f9..ac35870ca129ce 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -16,3 +16,7 @@ compatible = "apple,j314s", "apple,t6000", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M1 Pro, 2021)"; }; + +&wifi0 { + brcm,board-type = "apple,maldives"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index ff1803ce23001c..77d6d8c14d741e 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -16,3 +16,7 @@ compatible = "apple,j316s", "apple,t6000", "apple,arm-platform"; model = "Apple MacBook Pro (16-inch, M1 Pro, 2021)"; }; + +&wifi0 { + brcm,board-type = "apple,madagascar"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index 1761d15b98c12f..0a5655792a8f1c 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -16,3 +16,7 @@ compatible = "apple,j314c", "apple,t6001", "apple,arm-platform"; model = "Apple MacBook Pro (14-inch, M1 Max, 2021)"; }; + +&wifi0 { + brcm,board-type = "apple,maldives"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 750e9beeffc0aa..9c215531ea543e 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -16,3 +16,7 @@ compatible = "apple,j316c", "apple,t6001", "apple,arm-platform"; model = "Apple MacBook Pro (16-inch, M1 Max, 2021)"; }; + +&wifi0 { + brcm,board-type = "apple,madagascar"; +}; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index ebd7db2e08efa4..e3a7ed2706d7c6 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -186,9 +186,11 @@ /* WLAN */ bus-range = <1 1>; wifi0: wifi@0,0 { + compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; + apple,antenna-sku = "XX"; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 68cbe5bd801d25..bb2f1efce4a8a4 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -190,6 +190,9 @@ bus-range = <1 1>; wifi0: wifi@0,0 { reg = <0x10000 0x0 0x0 0x0 0x0>; + compatible = "pci14e4,4433"; + brcm,board-type = "apple,okinawa"; + apple,antenna-sku = "XX"; /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; }; From 77134f64726e8d4f549c881cc096798b1aee34b4 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 6 Feb 2022 21:22:29 +0900 Subject: [PATCH 307/635] arm64: dts: apple: Add PCI power enable GPIOs t8103: - WLAN (SMC PMU GPIO #13) t600x: - WLAN (SMC PMU GPIO #13) - SD (SMC PMU GPIO #26) Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 2 ++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 3 +++ 2 files changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index e3a7ed2706d7c6..2343ea2d057e42 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -185,6 +185,7 @@ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -197,6 +198,7 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index bb2f1efce4a8a4..dea1d0be740468 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -188,6 +188,7 @@ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { reg = <0x10000 0x0 0x0 0x0 0x0>; compatible = "pci14e4,4433"; @@ -201,6 +202,7 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -223,6 +225,7 @@ &port03 { /* USB xHCI */ bus-range = <4 4>; + pwren-gpios = <&smc_gpio 20 GPIO_ACTIVE_HIGH>; status = "okay"; }; From e4858262b00699ad428a1e8529107e68ee1d02e5 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 3 Mar 2022 03:24:36 +0900 Subject: [PATCH 308/635] arm64: dts: apple: Add backlight node to j314/j316 Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 2343ea2d057e42..e34469179738bd 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -28,6 +28,7 @@ compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ + backlight = <&backlight>; status = "disabled"; }; }; @@ -248,3 +249,11 @@ &dwc3_3 { status = "disabled"; }; + +/ { + backlight: gpio-bl { + compatible = "gpio-backlight"; + gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; + default-on; + }; +}; From ac3ef0182e6ae6dc4f6c508eb2cc86c6fd9ade3b Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 27 Jun 2022 22:21:34 +0900 Subject: [PATCH 309/635] arm64: dts: apple: t8103: Fix spi4 power domain sort order Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 18c1814585eb24..299b1f51b54179 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -387,6 +387,15 @@ power-domains = <&ps_sio>, <&ps_spi_p>; }; + ps_spi4: power-controller@260 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi4"; + power-domains = <&ps_sio>, <&ps_spi_p>; + }; + ps_uart_n: power-controller@268 { compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; @@ -558,15 +567,6 @@ apple,always-on; /* Memory controller */ }; - ps_spi4: power-controller@260 { - compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; - reg = <0x260 4>; - #power-domain-cells = <0>; - #reset-cells = <0>; - label = "spi4"; - power-domains = <&ps_sio>, <&ps_spi_p>; - }; - ps_dcs0: power-controller@300 { compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; From 24dda5ca87fc77b8a1cceb279c0bcafb4e0a6d8f Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 11 Jul 2022 20:05:02 +0900 Subject: [PATCH 310/635] arm64: dts: apple: t600x: Add bluetooth device trees Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 4 ++++ arch/arm64/boot/dts/apple/t6000-j316s.dts | 4 ++++ arch/arm64/boot/dts/apple/t6001-j314c.dts | 4 ++++ arch/arm64/boot/dts/apple/t6001-j316c.dts | 4 ++++ arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 8 ++++++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 9 +++++++++ 6 files changed, 33 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index ac35870ca129ce..1430b91ff1b152 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -20,3 +20,7 @@ &wifi0 { brcm,board-type = "apple,maldives"; }; + +&bluetooth0 { + brcm,board-type = "apple,maldives"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index 77d6d8c14d741e..da0cbe7d96736b 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -20,3 +20,7 @@ &wifi0 { brcm,board-type = "apple,madagascar"; }; + +&bluetooth0 { + brcm,board-type = "apple,madagascar"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index 0a5655792a8f1c..c37097dcfdb304 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -20,3 +20,7 @@ &wifi0 { brcm,board-type = "apple,maldives"; }; + +&bluetooth0 { + brcm,board-type = "apple,maldives"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 9c215531ea543e..3bc6e0c3294cf9 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -20,3 +20,7 @@ &wifi0 { brcm,board-type = "apple,madagascar"; }; + +&bluetooth0 { + brcm,board-type = "apple,madagascar"; +}; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index e34469179738bd..2abdcff1fe6ec8 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -13,6 +13,7 @@ / { aliases { + bluetooth0 = &bluetooth0; serial0 = &serial0; wifi0 = &wifi0; }; @@ -194,6 +195,13 @@ local-mac-address = [00 10 18 00 00 10]; apple,antenna-sku = "XX"; }; + + bluetooth0: network@0,1 { + compatible = "pci14e4,5f71"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; }; &port01 { diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index dea1d0be740468..e9260a84695abe 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -11,6 +11,7 @@ / { aliases { + bluetooth0 = &bluetooth0; serial0 = &serial0; wifi0 = &wifi0; }; @@ -197,6 +198,14 @@ /* To be filled by the loader */ local-mac-address = [00 10 18 00 00 10]; }; + + bluetooth0: network@0,1 { + compatible = "pci14e4,5f71"; + brcm,board-type = "apple,okinawa"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; }; &port01 { From 035f985df9b53a5f44b50e8707e42876b3265874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Sat, 19 Feb 2022 09:49:59 +0100 Subject: [PATCH 311/635] arm64: dts: apple: t8103*: Put in audio nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- arch/arm64/boot/dts/apple/t8103-j274.dts | 50 ++++++++++++++ arch/arm64/boot/dts/apple/t8103-j293.dts | 84 ++++++++++++++++++++++++ arch/arm64/boot/dts/apple/t8103-j313.dts | 66 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8103-j456.dts | 31 +++++++++ arch/arm64/boot/dts/apple/t8103-j457.dts | 31 +++++++++ 5 files changed, 262 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 968fe22163d443..6782d4bed42995 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -70,6 +70,56 @@ status = "okay"; }; +&i2c1 { + speaker_amp: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j274-macaudio", "apple,macaudio"; + model = "Mac mini J274 integrated audio"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index bf39433ca10dda..9286a8697529b8 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -71,8 +71,55 @@ }; }; +&i2c1 { + speaker_left_rear: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + }; + + speaker_left_front: codec@32 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x32>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + speaker_right_rear: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + }; + + speaker_right_front: codec@35 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x35>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + }; }; &i2c4 { @@ -89,6 +136,43 @@ gpios = <&smc_gpio 18 GPIO_ACTIVE_HIGH>; default-on; }; + + sound { + compatible = "apple,j293-macaudio", "apple,macaudio"; + model = "MacBook Pro J293 integrated audio"; + + dai-link@0 { + link-name = "Speakers"; + + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; }; &framebuffer0 { diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index ff3dc749418878..4262d9874bae4a 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -75,12 +75,78 @@ }; }; +&i2c1 { + speaker_left: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left"; + }; +}; + +&i2c3 { + speaker_right: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right"; + }; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + / { backlight: gpio-bl { compatible = "gpio-backlight"; gpios = <&smc_gpio 18 GPIO_ACTIVE_HIGH>; default-on; }; + + sound { + compatible = "apple,j313-macaudio", "apple,macaudio"; + model = "MacBook Air J313 integrated audio"; + + dai-link@0 { + link-name = "Speakers"; + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left>, <&speaker_right>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; }; &framebuffer0 { diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 9983e11cacdf19..fbecc8bcfad3d7 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -87,3 +87,34 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j456-macaudio", "apple,macaudio"; + model = "iMac J456 integrated audio"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index a622ff607d4075..bba32fffd378a6 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -60,3 +60,34 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j457-macaudio", "apple,macaudio"; + model = "iMac J457 integrated audio"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; From ae17e026f58aa2f8fa44fc0312509dc1a7da956d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 11 Mar 2022 22:16:25 +0100 Subject: [PATCH 312/635] arm64: dts: apple: t600x-jxxx: Put in audio nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Povišer --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 4 + arch/arm64/boot/dts/apple/t6000-j316s.dts | 4 + arch/arm64/boot/dts/apple/t6001-j314c.dts | 4 + arch/arm64/boot/dts/apple/t6001-j316c.dts | 4 + .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 115 ++++++++++++++++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 61 ++++++++++ 6 files changed, 192 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index 1430b91ff1b152..6c016b3b0ff11f 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -24,3 +24,7 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&sound { + model = "MacBook Pro J314 integrated audio"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index da0cbe7d96736b..d20630560e71c2 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -24,3 +24,7 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&sound { + model = "MacBook Pro J316 integrated audio"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index c37097dcfdb304..ef5846baf7fdd6 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -24,3 +24,7 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&sound { + model = "MacBook Pro J314 integrated audio"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 3bc6e0c3294cf9..df0cee05a708a4 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -24,3 +24,7 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&sound { + model = "MacBook Pro J316 integrated audio"; +}; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 2abdcff1fe6ec8..6ef465e6fda7fe 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -146,6 +146,81 @@ }; }; +&i2c1 { + status = "okay"; + + speaker_left_tweet: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + }; + + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + }; + + speaker_left_woof2: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + status = "okay"; + + speaker_right_tweet: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + }; + + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + }; + + speaker_right_woof2: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + }; +}; + &nco_clkref { clock-frequency = <1068000000>; }; @@ -264,4 +339,44 @@ gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; default-on; }; + + sound: sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + + dai-link@0 { + link-name = "Speakers"; + + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index e9260a84695abe..4ff38be6b68d5c 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -181,10 +181,71 @@ }; }; +/* Audio */ +&i2c1 { + status = "okay"; + + speaker: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &nco_clkref { clock-frequency = <1068000000>; }; +/ { + sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375 integrated audio"; + + dai-link@0 { + link-name = "Speaker"; + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + /* PCIe devices */ &port00 { /* WLAN */ From d0eede2887caef9724c9100b689c0c0c9f484020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Tue, 30 Aug 2022 09:59:59 +0200 Subject: [PATCH 313/635] arm64: dts: apple: Drop 'integrated audio' from sound models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Even though my preference would be to keep it in, the long name crops in a bunch of places and the verbiage at the end needs to go. Signed-off-by: Martin Povišer --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 2 +- arch/arm64/boot/dts/apple/t6000-j316s.dts | 2 +- arch/arm64/boot/dts/apple/t6001-j314c.dts | 2 +- arch/arm64/boot/dts/apple/t6001-j316c.dts | 2 +- arch/arm64/boot/dts/apple/t600x-j375.dtsi | 2 +- arch/arm64/boot/dts/apple/t8103-j274.dts | 2 +- arch/arm64/boot/dts/apple/t8103-j293.dts | 2 +- arch/arm64/boot/dts/apple/t8103-j313.dts | 2 +- arch/arm64/boot/dts/apple/t8103-j456.dts | 2 +- arch/arm64/boot/dts/apple/t8103-j457.dts | 2 +- arch/arm64/boot/dts/apple/t8112-j413.dts | 2 +- arch/arm64/boot/dts/apple/t8112-j493.dts | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index 6c016b3b0ff11f..b514b1114c4d24 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -26,5 +26,5 @@ }; &sound { - model = "MacBook Pro J314 integrated audio"; + model = "MacBook Pro J314"; }; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index d20630560e71c2..7f5c91bb5f3210 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -26,5 +26,5 @@ }; &sound { - model = "MacBook Pro J316 integrated audio"; + model = "MacBook Pro J316"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index ef5846baf7fdd6..35ff978b998702 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -26,5 +26,5 @@ }; &sound { - model = "MacBook Pro J314 integrated audio"; + model = "MacBook Pro J314"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index df0cee05a708a4..6a35c87aa7ab02 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -26,5 +26,5 @@ }; &sound { - model = "MacBook Pro J316 integrated audio"; + model = "MacBook Pro J316"; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 4ff38be6b68d5c..8a07ed81cea5f5 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -213,7 +213,7 @@ / { sound { compatible = "apple,j375-macaudio", "apple,macaudio"; - model = "Mac Studio J375 integrated audio"; + model = "Mac Studio J375"; dai-link@0 { link-name = "Speaker"; diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 6782d4bed42995..b214075000fb8a 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -97,7 +97,7 @@ / { sound { compatible = "apple,j274-macaudio", "apple,macaudio"; - model = "Mac mini J274 integrated audio"; + model = "Mac mini J274"; dai-link@0 { link-name = "Speaker"; diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 9286a8697529b8..8467ee5b57cf7d 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -139,7 +139,7 @@ sound { compatible = "apple,j293-macaudio", "apple,macaudio"; - model = "MacBook Pro J293 integrated audio"; + model = "MacBook Pro J293"; dai-link@0 { link-name = "Speakers"; diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 4262d9874bae4a..3b92bc43776fd4 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -115,7 +115,7 @@ sound { compatible = "apple,j313-macaudio", "apple,macaudio"; - model = "MacBook Air J313 integrated audio"; + model = "MacBook Air J313"; dai-link@0 { link-name = "Speakers"; diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index fbecc8bcfad3d7..7cf103418455a8 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -104,7 +104,7 @@ / { sound { compatible = "apple,j456-macaudio", "apple,macaudio"; - model = "iMac J456 integrated audio"; + model = "iMac J456"; dai-link@1 { link-name = "Headphone Jack"; diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index bba32fffd378a6..3969d7449f5c13 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -77,7 +77,7 @@ / { sound { compatible = "apple,j457-macaudio", "apple,macaudio"; - model = "iMac J457 integrated audio"; + model = "iMac J457"; dai-link@1 { link-name = "Headphone Jack"; diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 27ed249f5aaaeb..92f5e7f2bf24ed 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -146,7 +146,7 @@ sound { compatible = "apple,j413-macaudio", "apple,macaudio"; - model = "MacBook Air J413 integrated audio"; + model = "MacBook Air J413"; dai-link@0 { link-name = "Speakers"; diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index 7e053a80602e41..d80a403279aeda 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -135,7 +135,7 @@ sound { compatible = "apple,j493-macaudio", "apple,macaudio"; - model = "MacBook Pro J493 integrated audio"; + model = "MacBook Pro J493"; dai-link@0 { link-name = "Speakers"; From 2bb3fddd62ebd47d727496f3018cb9025b4c5d3a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 17 Oct 2022 18:29:28 +0900 Subject: [PATCH 314/635] arm64: dts: apple: t6001-j375c: Add USB3 hub GPIO initialization The Mac Studio M1 Max (t6001) model has a built-in USB3 hub. This hub has a firmware flash which is also connected to an AP SPI controller. The hub starts out in reset and the host is expected to bring it out of reset, potentially after upgrading/validating the firmware. We won't be doing anything with the firmware, so just use gpio-hog to flip the two GPIOs needed to bring up the hub chip. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t6001-j375c.dts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6001-j375c.dts b/arch/arm64/boot/dts/apple/t6001-j375c.dts index 62ea437b58b25c..e32a2715356feb 100644 --- a/arch/arm64/boot/dts/apple/t6001-j375c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j375c.dts @@ -16,3 +16,19 @@ compatible = "apple,j375c", "apple,t6001", "apple,arm-platform"; model = "Apple Mac Studio (M1 Max, 2022)"; }; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <230 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <231 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; From 411676196adc88fd58b3a20062710e4c4f9719f2 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 25 Jan 2022 21:50:59 +0100 Subject: [PATCH 315/635] arch: arm64: apple: Add missing power state deps for display The dcp co-processor crashes on HDMI unplug while it apparently tries to notify pmp. Handle "notify_pmp" as a parent dependency for "ps_disp0_fe" and "ps_dispext_fe". Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 299b1f51b54179..0966322c5c8e3f 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -645,7 +645,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_rmx>; + power-domains = <&ps_rmx>, <&ps_pmp>; apple,always-on; /* TODO: figure out if we can enable PM here */ }; @@ -655,7 +655,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_rmx>; + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@378 { From 25c48a0dd0fa1e57dfbb477b8698274520133aa7 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 24 Apr 2022 11:20:31 +0200 Subject: [PATCH 316/635] arch: arm64: apple: t600x: Mark USB and PCIe as "dma-coherent" Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 2 ++ arch/arm64/boot/dts/apple/t600x-dieX.dtsi | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 152c878ab3e005..40931fe6c93651 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -435,6 +435,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index a5f2ef5aab7929..5c25d843284ca0 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -149,6 +149,7 @@ iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, <&DIE_NODE(dwc3_0_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc0_usb)>; + dma-coherent; }; DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { @@ -180,6 +181,7 @@ iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, <&DIE_NODE(dwc3_1_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc1_usb)>; + dma-coherent; }; DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { @@ -211,6 +213,7 @@ iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, <&DIE_NODE(dwc3_2_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc2_usb)>; + dma-coherent; }; DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { @@ -242,4 +245,5 @@ iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, <&DIE_NODE(dwc3_3_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc3_usb)>; + dma-coherent; }; From 37578f9f8cf5453216ede1614fd202a4696f412e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 20 Sep 2021 02:27:09 +0900 Subject: [PATCH 317/635] arch: arm64: apple: Add display controller related device tree nodes The display system is initialized by the bootloader to provide a simple framebuffer at startup. Memory for the framebuffer and heap for the display co-processor are alreay mapped through the IOMMU. IOMMU intialization must preserve this mappings to avoid crashing the display co-processor. The exisitng mappings are caried in the devicetree. They are applied during device attach to ensure the IOMMU framework is aware of these mapping. Mappings are filled by m1n1 during boot. Based on https://lore.kernel.org/asahi/20220923123557.866972-1-thierry.reding@gmail.com Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 10 +++ arch/arm64/boot/dts/apple/t8103.dtsi | 75 +++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index b70cd5f64b4cc6..727cbd8f206d12 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -12,6 +12,9 @@ / { aliases { bluetooth0 = &bluetooth0; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; wifi0 = &wifi0; @@ -32,6 +35,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index db00ef3de56385..10a2ee3e4ca78d 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -334,6 +334,14 @@ clock-output-names = "clk_120m"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -364,6 +372,73 @@ #performance-domain-cells = <0>; }; + disp0_piodma: mapper@230000000 { + compatible = "apple,t8103-dcp-piodma", "apple,dcp-piodma"; + iommus = <&disp0_dart 4>; + }; + + disp0_dart: iommu@231304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "disp-5", "doorbell"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>, + <0x2 0x3b738000 0x0 0x1000>, + <0x2 0x3bc3c000 0x0 0x1000>; + power-domains = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + apple,asc-dram-mask = <0xf 0x00000000>; + apple,piodma-mapper = <&disp0_piodma>; + phandle = <&dcp>; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8103-dart"; reg = <0x2 0x35004000 0x0 0x4000>; From 26bf72f4b0df551df43ffbb279b5e3c9f64f5479 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 11 Mar 2022 22:14:52 +0100 Subject: [PATCH 318/635] arch: arm64: apple: t600x: Add display controller related device tree nodes The display system is initialized by the bootloader to provide a simple framebuffer at startup. Memory for the framebuffer and heap for the display co-processor are alreay mapped through the IOMMU. IOMMU intialization must preserve this mappings to avoid crashing the display co-processor. The exisitng mappings are caried in the devicetree. They are applied during device attach to ensure the IOMMU framework is aware of these mapping. Mappings are filled by m1n1 during boot. Based on https://lore.kernel.org/asahi/20220923123557.866972-1-thierry.reding@gmail.com Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t600x-common.dtsi | 6 ++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 64 +++++++++++++++++++ .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 10 +++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 10 +++ 4 files changed, 90 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index 87dfc13d74171f..01385ef831ca91 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -369,6 +369,12 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <237333328>; + clock-output-names = "clk_disp0"; + }; /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 40931fe6c93651..259cf535a7fadc 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -166,6 +166,70 @@ }; }; + disp0_piodma: mapper@38a000000 { + compatible = "apple,t6000-dcp-piodma", "apple,dcp-piodma"; + iommus = <&disp0_dart 4>; + }; + + disp0_dart: iommu@38b304000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x8b304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + dcp_dart: iommu@38b30c000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x8b30c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + dcp_mbox: mbox@38bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x8bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@38bc00000 { + compatible = "apple,t6000-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x3 0x8bc00000 0x0 0x4000>, + <0x3 0x8a000000 0x0 0x3000000>, + <0x3 0x8b320000 0x0 0x4000>, + <0x3 0x8b344000 0x0 0x4000>, + <0x3 0x8b800000 0x0 0x800000>, + <0x2 0x8e3d0000 0x0 0x4000>; + power-domains = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + apple,asc-dram-mask = <0x1f0 0x00000000>; + apple,piodma-mapper = <&disp0_piodma>; + phandle = <&dcp>; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + sio_dart_0: iommu@39b004000 { compatible = "apple,t6000-dart"; reg = <0x3 0x9b004000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 6ef465e6fda7fe..065670b333ae1a 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -14,6 +14,9 @@ / { aliases { bluetooth0 = &bluetooth0; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; wifi0 = &wifi0; }; @@ -34,6 +37,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@10000000000 { device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 8a07ed81cea5f5..a84cb24f87e2b8 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -12,6 +12,9 @@ / { aliases { bluetooth0 = &bluetooth0; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; wifi0 = &wifi0; }; @@ -31,6 +34,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@10000000000 { device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ From d56faf26884e8a36d6464f5b0b463f38c6f8583f Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 3 Oct 2022 17:44:37 +0200 Subject: [PATCH 319/635] arch: arm64: apple: t8103: Add connector type property for DCP* Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-j274.dts | 4 ++++ arch/arm64/boot/dts/apple/t8103-j293.dts | 4 ++++ arch/arm64/boot/dts/apple/t8103-j313.dts | 4 ++++ arch/arm64/boot/dts/apple/t8103-j456.dts | 4 ++++ arch/arm64/boot/dts/apple/t8103-j457.dts | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index b214075000fb8a..7169563c1b351a 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -21,6 +21,10 @@ }; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + &bluetooth0 { brcm,board-type = "apple,atlantisb"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 8467ee5b57cf7d..013dd46766f8b3 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -30,6 +30,10 @@ }; }; +&dcp { + apple,connector-type = "eDP"; +}; + &bluetooth0 { brcm,board-type = "apple,honshu"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 3b92bc43776fd4..e8ca92270dc9d2 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -30,6 +30,10 @@ }; }; +&dcp { + apple,connector-type = "eDP"; +}; + &bluetooth0 { brcm,board-type = "apple,shikoku"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 7cf103418455a8..ca74b7f87c0c3b 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -21,6 +21,10 @@ }; }; +&dcp { + apple,connector-type = "eDP"; +}; + &bluetooth0 { brcm,board-type = "apple,capri"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 3969d7449f5c13..2f248178ce5885 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -21,6 +21,10 @@ }; }; +&dcp { + apple,connector-type = "eDP"; +}; + &bluetooth0 { brcm,board-type = "apple,santorini"; }; From 7351eb1630b1d6e45e7b3ffe3908565adc2d6207 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 22 Oct 2022 09:33:04 +0200 Subject: [PATCH 320/635] arch: arm64: apple: t600x: Add connector type property for DCP* Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 4 ++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 065670b333ae1a..6a9c78893d6e45 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -66,6 +66,10 @@ status = "okay"; }; +&dcp { + apple,connector-type = "eDP"; +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index a84cb24f87e2b8..b5fbad0379e16b 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -51,6 +51,10 @@ status = "okay"; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { From c73b08108b469d251d9a1525ca333f5b5820ec15 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Tue, 15 Nov 2022 12:09:48 +0100 Subject: [PATCH 321/635] arm64: dts: apple: t8103: Add eFuses node --- arch/arm64/boot/dts/apple/t8103.dtsi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 10a2ee3e4ca78d..d814b9b8393387 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -934,6 +934,13 @@ resets = <&ps_ans2>; }; + efuse@23d2bc000 { + compatible = "apple,t8103-efuses", "apple,efuses"; + reg = <0x2 0x3d2bc000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + }; + dwc3_0: usb@382280000 { compatible = "apple,t8103-dwc3", "apple,dwc3", "snps,dwc3"; reg = <0x3 0x82280000 0x0 0x100000>; From 2e1b054d96f3a51795b151206ddf72cd06924078 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 30 Nov 2022 22:11:09 +0100 Subject: [PATCH 322/635] arm64: dts: apple: t8103: Add ATCPHY node Signed-off-by: Sven Peter --- arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 31 ++++ arch/arm64/boot/dts/apple/t8103.dtsi | 179 ++++++++++++++++++++++ 2 files changed, 210 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 727cbd8f206d12..1ad2a57698afb9 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -18,6 +18,8 @@ serial0 = &serial0; serial2 = &serial2; wifi0 = &wifi0; + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; }; chosen { @@ -78,6 +80,12 @@ remote-endpoint = <&typec0_usb_hs>; }; }; + port@1 { + reg = <1>; + typec0_con_ss: endpoint { + remote-endpoint = <&typec0_usb_ss>; + }; + }; }; }; }; @@ -103,6 +111,12 @@ remote-endpoint = <&typec1_usb_hs>; }; }; + port@1 { + reg = <1>; + typec1_con_ss: endpoint { + remote-endpoint = <&typec1_usb_ss>; + }; + }; }; }; }; @@ -125,6 +139,23 @@ }; }; +/* Type-C PHYs */ +&atcphy0 { + port { + typec0_usb_ss: endpoint { + remote-endpoint = <&typec0_con_ss>; + }; + }; +}; + +&atcphy1 { + port { + typec1_usb_ss: endpoint { + remote-endpoint = <&typec1_con_ss>; + }; + }; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index d814b9b8393387..876e03dbe6d3e3 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -12,6 +12,7 @@ #include #include #include +#include / { compatible = "apple,t8103", "apple,arm-platform"; @@ -939,6 +940,100 @@ reg = <0x2 0x3d2bc000 0x0 0x1000>; #address-cells = <1>; #size-cells = <1>; + atcphy0_auspll_rodco_bias_adjust: efuse@430,26 { + reg = <0x430 4>; + bits = <26 3>; + }; + + atcphy0_auspll_rodco_encap: efuse@430,29 { + reg = <0x430 4>; + bits = <29 2>; + }; + + atcphy0_auspll_dtc_vreg_adjust: efuse@430,31 { + reg = <0x430 8>; + bits = <31 3>; + }; + + atcphy0_auspll_fracn_dll_start_capcode: efuse@434,2 { + reg = <0x434 4>; + bits = <2 2>; + }; + + atcphy0_aus_cmn_shm_vreg_trim: efuse@434,4 { + reg = <0x434 4>; + bits = <4 5>; + }; + + atcphy0_cio3pll_dco_coarsebin0: efuse@434,9 { + reg = <0x434 4>; + bits = <9 6>; + }; + + atcphy0_cio3pll_dco_coarsebin1: efuse@434,15 { + reg = <0x434 4>; + bits = <15 6>; + }; + + atcphy0_cio3pll_dll_start_capcode: efuse@434,21 { + reg = <0x434 4>; + bits = <21 2>; + }; + + atcphy0_cio3pll_dtc_vreg_adjust: efuse@434,23 { + reg = <0x434 0x4>; + bits = <23 3>; + }; + + atcphy1_auspll_rodco_bias_adjust: efuse@438,4 { + reg = <0x438 4>; + bits = <4 3>; + }; + + atcphy1_auspll_rodco_encap: efuse@438,7 { + reg = <0x438 4>; + bits = <7 2>; + }; + + atcphy1_auspll_dtc_vreg_adjust: efuse@438,9 { + reg = <0x438 4>; + bits = <9 3>; + }; + + atcphy1_auspll_fracn_dll_start_capcode: efuse@438,12 { + reg = <0x438 4>; + bits = <12 2>; + }; + + atcphy1_aus_cmn_shm_vreg_trim: efuse@438,14 { + reg = <0x438 4>; + bits = <14 5>; + }; + + atcphy1_cio3pll_dco_coarsebin0: efuse@438,19 { + reg = <0x438 4>; + bits = <19 6>; + }; + + atcphy1_cio3pll_dco_coarsebin1: efuse@438,25 { + reg = <0x438 4>; + bits = <25 6>; + }; + + atcphy1_cio3pll_dll_start_capcode: efuse@438,31 { + reg = <0x438 4>; + bits = <31 1>; + }; + + atcphy1_cio3pll_dll_start_capcode_workaround: efuse@43c,0 { + reg = <0x43c 0x4>; + bits = <0 1>; + }; + + atcphy1_cio3pll_dtc_vreg_adjust: efuse@43c,1 { + reg = <0x43c 0x4>; + bits = <1 3>; + }; }; dwc3_0: usb@382280000 { @@ -951,6 +1046,9 @@ role-switch-default-mode = "host"; iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; }; dwc3_0_dart_0: iommu@382f00000 { @@ -971,6 +1069,44 @@ power-domains = <&ps_atc0_usb>; }; + atcphy0: phy@383000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x3 0x83000000 0x0 0x4c000>, + <0x3 0x83050000 0x0 0x8000>, + <0x3 0x80000000 0x0 0x4000>, + <0x3 0x82a90000 0x0 0x4000>, + <0x3 0x82a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + nvmem-cells = <&atcphy0_aus_cmn_shm_vreg_trim>, + <&atcphy0_auspll_rodco_encap>, + <&atcphy0_auspll_rodco_bias_adjust>, + <&atcphy0_auspll_fracn_dll_start_capcode>, + <&atcphy0_auspll_dtc_vreg_adjust>, + <&atcphy0_cio3pll_dco_coarsebin0>, + <&atcphy0_cio3pll_dco_coarsebin1>, + <&atcphy0_cio3pll_dll_start_capcode>, + <&atcphy0_cio3pll_dtc_vreg_adjust>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&ps_atc0_usb>; + }; + dwc3_1: usb@502280000 { compatible = "apple,t8103-dwc3", "apple,dwc3", "snps,dwc3"; reg = <0x5 0x02280000 0x0 0x100000>; @@ -981,6 +1117,9 @@ role-switch-default-mode = "host"; iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; }; dwc3_1_dart_0: iommu@502f00000 { @@ -1001,6 +1140,46 @@ power-domains = <&ps_atc1_usb>; }; + atcphy1: phy@503000000 { + compatible = "apple,t8103-atcphy"; + reg = <0x5 0x03000000 0x0 0x4c000>, + <0x5 0x03050000 0x0 0x8000>, + <0x5 0x0 0x0 0x4000>, + <0x5 0x02a90000 0x0 0x4000>, + <0x5 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + nvmem-cells = <&atcphy1_aus_cmn_shm_vreg_trim>, + <&atcphy1_auspll_rodco_encap>, + <&atcphy1_auspll_rodco_bias_adjust>, + <&atcphy1_auspll_fracn_dll_start_capcode>, + <&atcphy1_auspll_dtc_vreg_adjust>, + <&atcphy1_cio3pll_dco_coarsebin0>, + <&atcphy1_cio3pll_dco_coarsebin1>, + <&atcphy1_cio3pll_dll_start_capcode>, + <&atcphy1_cio3pll_dtc_vreg_adjust>, + <&atcphy1_cio3pll_dll_start_capcode_workaround>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust", + "cio3pll_dll_start_capcode_workaround"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart_0: iommu@681008000 { compatible = "apple,t8103-dart"; reg = <0x6 0x81008000 0x0 0x4000>; From 3e5bf650ef7352a8394f7075b9f35101f924af9d Mon Sep 17 00:00:00 2001 From: R Date: Tue, 15 Nov 2022 12:09:52 +0100 Subject: [PATCH 323/635] arch: arm64: dts: apple: t6000: Add eFuses node Signed-off-by: R --- arch/arm64/boot/dts/apple/t600x-dieX.dtsi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index 5c25d843284ca0..d135a437d513ce 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -74,6 +74,13 @@ reg = <0x2 0x92280000 0 0x4000>; }; + DIE_NODE(efuse): efuse@2922bc000 { + compatible = "apple,t6000-efuses", "apple,efuses"; + reg = <0x2 0x922bc000 0x0 0x2000>; + #address-cells = <1>; + #size-cells = <1>; + }; + DIE_NODE(pinctrl_aop): pinctrl@293820000 { compatible = "apple,t6000-pinctrl", "apple,pinctrl"; reg = <0x2 0x93820000 0x0 0x4000>; From 7a9a7b2aa6f6f4c7d8e527373ea52ea93de61b65 Mon Sep 17 00:00:00 2001 From: R Date: Tue, 15 Nov 2022 12:09:54 +0100 Subject: [PATCH 324/635] arch: arm64: dts: apple: t600x: Add ATCPHY nodes Signed-off-by: R --- arch/arm64/boot/dts/apple/t6001.dtsi | 1 + arch/arm64/boot/dts/apple/t6002-j375d.dts | 35 ++ arch/arm64/boot/dts/apple/t6002.dtsi | 1 + arch/arm64/boot/dts/apple/t600x-dieX.dtsi | 352 +++++++++++++++++- .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 47 +++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 61 +++ 6 files changed, 493 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index d2cf81926f284c..0bdd1966f5302e 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "multi-die-cpp.h" diff --git a/arch/arm64/boot/dts/apple/t6002-j375d.dts b/arch/arm64/boot/dts/apple/t6002-j375d.dts index d3d4f4461deecb..abf6c53bb78faa 100644 --- a/arch/arm64/boot/dts/apple/t6002-j375d.dts +++ b/arch/arm64/boot/dts/apple/t6002-j375d.dts @@ -15,6 +15,10 @@ / { compatible = "apple,j375d", "apple,t6002", "apple,arm-platform"; model = "Apple Mac Studio (M1 Ultra, 2022)"; + aliases { + atcphy4 = &atcphy0_die1; + atcphy5 = &atcphy1_die1; + }; }; /* USB Type C */ @@ -42,6 +46,12 @@ remote-endpoint = <&typec4_usb_hs>; }; }; + port@1 { + reg = <1>; + typec4_con_ss: endpoint { + remote-endpoint = <&typec4_usb_ss>; + }; + }; }; }; }; @@ -69,6 +79,12 @@ remote-endpoint = <&typec5_usb_hs>; }; }; + port@1 { + reg = <1>; + typec5_con_ss: endpoint { + remote-endpoint = <&typec5_usb_ss>; + }; + }; }; }; }; @@ -91,15 +107,34 @@ }; }; +/* Type-C PHYs */ +&atcphy0_die1 { + port { + typec4_usb_ss: endpoint { + remote-endpoint = <&typec4_con_ss>; + }; + }; +}; + +&atcphy1_die1 { + port { + typec5_usb_ss: endpoint { + remote-endpoint = <&typec5_con_ss>; + }; + }; +}; + /* delete unused USB nodes on die 1 */ /delete-node/ &dwc3_2_dart_0_die1; /delete-node/ &dwc3_2_dart_1_die1; /delete-node/ &dwc3_2_die1; +/delete-node/ &atcphy2_die1; /delete-node/ &dwc3_3_dart_0_die1; /delete-node/ &dwc3_3_dart_1_die1; /delete-node/ &dwc3_3_die1; +/delete-node/ &atcphy3_die1; /* delete unused always-on power-domains on die 1 */ diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index e36f422d257d8f..8fa2d8dd72ff7f 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "multi-die-cpp.h" diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index d135a437d513ce..3fca8efb2dcf17 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -79,6 +79,186 @@ reg = <0x2 0x922bc000 0x0 0x2000>; #address-cells = <1>; #size-cells = <1>; + + DIE_NODE(atcphy0_auspll_rodco_bias_adjust): efuse@a10,22 { + reg = <0xa10 4>; + bits = <22 3>; + }; + + DIE_NODE(atcphy0_auspll_rodco_encap): efuse@a10,25 { + reg = <0xa10 4>; + bits = <25 2>; + }; + + DIE_NODE(atcphy0_auspll_dtc_vreg_adjust): efuse@a10,27 { + reg = <0xa10 4>; + bits = <27 3>; + }; + + DIE_NODE(atcphy0_auspll_fracn_dll_start_capcode): efuse@a10,30 { + reg = <0xa10 4>; + bits = <30 2>; + }; + + DIE_NODE(atcphy0_aus_cmn_shm_vreg_trim): efuse@a14,0 { + reg = <0xa14 4>; + bits = <0 5>; + }; + + DIE_NODE(atcphy0_cio3pll_dco_coarsebin0): efuse@a14,5 { + reg = <0xa14 4>; + bits = <5 6>; + }; + + DIE_NODE(atcphy0_cio3pll_dco_coarsebin1): efuse@a14,11 { + reg = <0xa14 4>; + bits = <11 6>; + }; + + DIE_NODE(atcphy0_cio3pll_dll_start_capcode): efuse@a14,17 { + reg = <0xa14 4>; + bits = <17 2>; + }; + + DIE_NODE(atcphy0_cio3pll_dtc_vreg_adjust): efuse@a14,19 { + reg = <0xa14 4>; + bits = <19 3>; + }; + + DIE_NODE(atcphy1_auspll_rodco_bias_adjust): efuse@a18,0 { + reg = <0xa18 4>; + bits = <0 3>; + }; + + DIE_NODE(atcphy1_auspll_rodco_encap): efuse@a18,3 { + reg = <0xa18 4>; + bits = <3 2>; + }; + + DIE_NODE(atcphy1_auspll_dtc_vreg_adjust): efuse@a18,5 { + reg = <0xa18 4>; + bits = <5 3>; + }; + + DIE_NODE(atcphy1_auspll_fracn_dll_start_capcode): efuse@a18,8 { + reg = <0xa18 4>; + bits = <8 2>; + }; + + DIE_NODE(atcphy1_aus_cmn_shm_vreg_trim): efuse@a18,10 { + reg = <0xa18 4>; + bits = <10 5>; + }; + + DIE_NODE(atcphy1_cio3pll_dco_coarsebin0): efuse@a18,15 { + reg = <0xa18 4>; + bits = <15 6>; + }; + + DIE_NODE(atcphy1_cio3pll_dco_coarsebin1): efuse@a18,21 { + reg = <0xa18 4>; + bits = <21 6>; + }; + + DIE_NODE(atcphy1_cio3pll_dll_start_capcode): efuse@a18,27 { + reg = <0xa18 4>; + bits = <27 2>; + }; + + DIE_NODE(atcphy1_cio3pll_dtc_vreg_adjust): efuse@a18,29 { + reg = <0xa18 4>; + bits = <29 3>; + }; + + DIE_NODE(atcphy2_auspll_rodco_bias_adjust): efuse@a1c,10 { + reg = <0xa1c 4>; + bits = <10 3>; + }; + + DIE_NODE(atcphy2_auspll_rodco_encap): efuse@a1c,13 { + reg = <0xa1c 4>; + bits = <13 2>; + }; + + DIE_NODE(atcphy2_auspll_dtc_vreg_adjust): efuse@a1c,15 { + reg = <0xa1c 4>; + bits = <15 3>; + }; + + DIE_NODE(atcphy2_auspll_fracn_dll_start_capcode): efuse@a1c,18 { + reg = <0xa1c 4>; + bits = <18 2>; + }; + + DIE_NODE(atcphy2_aus_cmn_shm_vreg_trim): efuse@a1c,20 { + reg = <0xa1c 4>; + bits = <20 5>; + }; + + DIE_NODE(atcphy2_cio3pll_dco_coarsebin0): efuse@a1c,25 { + reg = <0xa1c 4>; + bits = <25 6>; + }; + + DIE_NODE(atcphy2_cio3pll_dco_coarsebin1): efuse@a1c,31 { + reg = <0xa1c 8>; + bits = <31 6>; + }; + + DIE_NODE(atcphy2_cio3pll_dll_start_capcode): efuse@a20,5 { + reg = <0xa20 4>; + bits = <5 2>; + }; + + DIE_NODE(atcphy2_cio3pll_dtc_vreg_adjust): efuse@a20,7 { + reg = <0xa20 4>; + bits = <7 3>; + }; + + DIE_NODE(atcphy3_auspll_rodco_bias_adjust): efuse@a20,20 { + reg = <0xa20 4>; + bits = <20 3>; + }; + + DIE_NODE(atcphy3_auspll_rodco_encap): efuse@a20,23 { + reg = <0xa20 4>; + bits = <23 2>; + }; + + DIE_NODE(atcphy3_auspll_dtc_vreg_adjust): efuse@a20,25 { + reg = <0xa20 4>; + bits = <25 3>; + }; + + DIE_NODE(atcphy3_auspll_fracn_dll_start_capcode): efuse@a20,28 { + reg = <0xa20 4>; + bits = <28 2>; + }; + + DIE_NODE(atcphy3_aus_cmn_shm_vreg_trim): efuse@a20,30 { + reg = <0xa20 8>; + bits = <30 5>; + }; + + DIE_NODE(atcphy3_cio3pll_dco_coarsebin0): efuse@a24,3 { + reg = <0xa24 4>; + bits = <3 6>; + }; + + DIE_NODE(atcphy3_cio3pll_dco_coarsebin1): efuse@a24,9 { + reg = <0xa24 4>; + bits = <9 6>; + }; + + DIE_NODE(atcphy3_cio3pll_dll_start_capcode): efuse@a24,15 { + reg = <0xa24 4>; + bits = <15 2>; + }; + + DIE_NODE(atcphy3_cio3pll_dtc_vreg_adjust): efuse@a24,17 { + reg = <0xa24 4>; + bits = <17 3>; + }; }; DIE_NODE(pinctrl_aop): pinctrl@293820000 { @@ -150,13 +330,54 @@ reg = <0x7 0x02280000 0x0 0x100000>; interrupt-parent = <&aic>; interrupts = ; - /* dr_mode = "otg"; */ + dr_mode = "otg"; usb-role-switch; role-switch-default-mode = "host"; iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, <&DIE_NODE(dwc3_0_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc0_usb)>; dma-coherent; + resets = <&DIE_NODE(atcphy0)>; + phys = <&DIE_NODE(atcphy0) PHY_TYPE_USB2>, <&DIE_NODE(atcphy0) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy0): phy@703000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0x7 0x03000000 0x0 0x4c000>, + <0x7 0x03050000 0x0 0x8000>, + <0x7 0x00000000 0x0 0x4000>, + <0x7 0x02a90000 0x0 0x4000>, + <0x7 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + nvmem-cells = <&DIE_NODE(atcphy0_aus_cmn_shm_vreg_trim)>, + <&DIE_NODE(atcphy0_auspll_rodco_encap)>, + <&DIE_NODE(atcphy0_auspll_rodco_bias_adjust)>, + <&DIE_NODE(atcphy0_auspll_fracn_dll_start_capcode)>, + <&DIE_NODE(atcphy0_auspll_dtc_vreg_adjust)>, + <&DIE_NODE(atcphy0_cio3pll_dco_coarsebin0)>, + <&DIE_NODE(atcphy0_cio3pll_dco_coarsebin1)>, + <&DIE_NODE(atcphy0_cio3pll_dll_start_capcode)>, + <&DIE_NODE(atcphy0_cio3pll_dtc_vreg_adjust)>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; }; DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { @@ -182,13 +403,54 @@ reg = <0xb 0x02280000 0x0 0x100000>; interrupt-parent = <&aic>; interrupts = ; - /* dr_mode = "otg"; */ + dr_mode = "otg"; usb-role-switch; role-switch-default-mode = "host"; iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, <&DIE_NODE(dwc3_1_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc1_usb)>; dma-coherent; + resets = <&DIE_NODE(atcphy1)>; + phys = <&DIE_NODE(atcphy1) PHY_TYPE_USB2>, <&DIE_NODE(atcphy1) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy1): phy@b03000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0xb 0x03000000 0x0 0x4c000>, + <0xb 0x03050000 0x0 0x8000>, + <0xb 0x00000000 0x0 0x4000>, + <0xb 0x02a90000 0x0 0x4000>, + <0xb 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + nvmem-cells = <&DIE_NODE(atcphy1_aus_cmn_shm_vreg_trim)>, + <&DIE_NODE(atcphy1_auspll_rodco_encap)>, + <&DIE_NODE(atcphy1_auspll_rodco_bias_adjust)>, + <&DIE_NODE(atcphy1_auspll_fracn_dll_start_capcode)>, + <&DIE_NODE(atcphy1_auspll_dtc_vreg_adjust)>, + <&DIE_NODE(atcphy1_cio3pll_dco_coarsebin0)>, + <&DIE_NODE(atcphy1_cio3pll_dco_coarsebin1)>, + <&DIE_NODE(atcphy1_cio3pll_dll_start_capcode)>, + <&DIE_NODE(atcphy1_cio3pll_dtc_vreg_adjust)>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; }; DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { @@ -214,13 +476,54 @@ reg = <0xf 0x02280000 0x0 0x100000>; interrupt-parent = <&aic>; interrupts = ; - /* dr_mode = "otg"; */ + dr_mode = "otg"; usb-role-switch; role-switch-default-mode = "host"; iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, <&DIE_NODE(dwc3_2_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc2_usb)>; dma-coherent; + resets = <&DIE_NODE(atcphy2)>; + phys = <&DIE_NODE(atcphy2) PHY_TYPE_USB2>, <&DIE_NODE(atcphy2) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy2): phy@f03000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0xf 0x03000000 0x0 0x4c000>, + <0xf 0x03050000 0x0 0x8000>, + <0xf 0x00000000 0x0 0x4000>, + <0xf 0x02a90000 0x0 0x4000>, + <0xf 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + nvmem-cells = <&DIE_NODE(atcphy2_aus_cmn_shm_vreg_trim)>, + <&DIE_NODE(atcphy2_auspll_rodco_encap)>, + <&DIE_NODE(atcphy2_auspll_rodco_bias_adjust)>, + <&DIE_NODE(atcphy2_auspll_fracn_dll_start_capcode)>, + <&DIE_NODE(atcphy2_auspll_dtc_vreg_adjust)>, + <&DIE_NODE(atcphy2_cio3pll_dco_coarsebin0)>, + <&DIE_NODE(atcphy2_cio3pll_dco_coarsebin1)>, + <&DIE_NODE(atcphy2_cio3pll_dll_start_capcode)>, + <&DIE_NODE(atcphy2_cio3pll_dtc_vreg_adjust)>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; }; DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { @@ -246,11 +549,52 @@ reg = <0x13 0x02280000 0x0 0x100000>; interrupt-parent = <&aic>; interrupts = ; - /* dr_mode = "otg"; */ + dr_mode = "otg"; usb-role-switch; role-switch-default-mode = "host"; iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, <&DIE_NODE(dwc3_3_dart_1) 1>; power-domains = <&DIE_NODE(ps_atc3_usb)>; dma-coherent; + resets = <&DIE_NODE(atcphy3)>; + phys = <&DIE_NODE(atcphy3) PHY_TYPE_USB2>, <&DIE_NODE(atcphy3) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy3): phy@1303000000 { + compatible = "apple,t6000-atcphy", "apple,t8103-atcphy"; + reg = <0x13 0x03000000 0x0 0x4c000>, + <0x13 0x03050000 0x0 0x8000>, + <0x13 0x00000000 0x0 0x4000>, + <0x13 0x02a90000 0x0 0x4000>, + <0x13 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + nvmem-cells = <&DIE_NODE(atcphy3_aus_cmn_shm_vreg_trim)>, + <&DIE_NODE(atcphy3_auspll_rodco_encap)>, + <&DIE_NODE(atcphy3_auspll_rodco_bias_adjust)>, + <&DIE_NODE(atcphy3_auspll_fracn_dll_start_capcode)>, + <&DIE_NODE(atcphy3_auspll_dtc_vreg_adjust)>, + <&DIE_NODE(atcphy3_cio3pll_dco_coarsebin0)>, + <&DIE_NODE(atcphy3_cio3pll_dco_coarsebin1)>, + <&DIE_NODE(atcphy3_cio3pll_dll_start_capcode)>, + <&DIE_NODE(atcphy3_cio3pll_dtc_vreg_adjust)>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 6a9c78893d6e45..334ae2fea619ce 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -13,6 +13,10 @@ / { aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; dcp = &dcp; disp0 = &display; @@ -94,6 +98,12 @@ remote-endpoint = <&typec0_usb_hs>; }; }; + port@1 { + reg = <1>; + typec0_con_ss: endpoint { + remote-endpoint = <&typec0_usb_ss>; + }; + }; }; }; }; @@ -120,6 +130,12 @@ remote-endpoint = <&typec1_usb_hs>; }; }; + port@1 { + reg = <1>; + typec1_con_ss: endpoint { + remote-endpoint = <&typec1_usb_ss>; + }; + }; }; }; }; @@ -146,6 +162,12 @@ remote-endpoint = <&typec2_usb_hs>; }; }; + port@1 { + reg = <1>; + typec2_con_ss: endpoint { + remote-endpoint = <&typec2_usb_ss>; + }; + }; }; }; }; @@ -334,6 +356,31 @@ }; }; +/* Type-C PHYs */ +&atcphy0 { + port { + typec0_usb_ss: endpoint { + remote-endpoint = <&typec0_con_ss>; + }; + }; +}; + +&atcphy1 { + port { + typec1_usb_ss: endpoint { + remote-endpoint = <&typec1_con_ss>; + }; + }; +}; + +&atcphy2 { + port { + typec2_usb_ss: endpoint { + remote-endpoint = <&typec2_con_ss>; + }; + }; +}; + /* ATC3 is used for DisplayPort -> HDMI only */ &dwc3_3_dart_0 { status = "disabled"; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index b5fbad0379e16b..fe003fa6227d7f 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -11,6 +11,10 @@ / { aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; + atcphy2 = &atcphy2; + atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; dcp = &dcp; disp0 = &display; @@ -79,6 +83,12 @@ remote-endpoint = <&typec0_usb_hs>; }; }; + port@1 { + reg = <1>; + typec0_con_ss: endpoint { + remote-endpoint = <&typec0_usb_ss>; + }; + }; }; }; }; @@ -105,6 +115,12 @@ remote-endpoint = <&typec1_usb_hs>; }; }; + port@1 { + reg = <1>; + typec1_con_ss: endpoint { + remote-endpoint = <&typec1_usb_ss>; + }; + }; }; }; }; @@ -131,6 +147,12 @@ remote-endpoint = <&typec2_usb_hs>; }; }; + port@1 { + reg = <1>; + typec2_con_ss: endpoint { + remote-endpoint = <&typec2_usb_ss>; + }; + }; }; }; }; @@ -157,6 +179,12 @@ remote-endpoint = <&typec3_usb_hs>; }; }; + port@1 { + reg = <1>; + typec3_con_ss: endpoint { + remote-endpoint = <&typec3_usb_ss>; + }; + }; }; }; }; @@ -195,6 +223,39 @@ }; }; +/* Type-C PHYs */ +&atcphy0 { + port { + typec0_usb_ss: endpoint { + remote-endpoint = <&typec0_con_ss>; + }; + }; +}; + +&atcphy1 { + port { + typec1_usb_ss: endpoint { + remote-endpoint = <&typec1_con_ss>; + }; + }; +}; + +&atcphy2 { + port { + typec2_usb_ss: endpoint { + remote-endpoint = <&typec2_con_ss>; + }; + }; +}; + +&atcphy3 { + port { + typec3_usb_ss: endpoint { + remote-endpoint = <&typec3_con_ss>; + }; + }; +}; + /* Audio */ &i2c1 { status = "okay"; From 0f2cff4fd6336ddc31183027ec0ba88a5bfb7493 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 31 Oct 2022 01:19:36 +0100 Subject: [PATCH 325/635] arch: arm64: apple: Add dcp panel node for t8103 based laptops and imacs The panel node will contain among other properties backlight control related properties from the "backlight" node in the ADT. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-j293.dts | 7 ++++++- arch/arm64/boot/dts/apple/t8103-j313.dts | 7 ++++++- arch/arm64/boot/dts/apple/t8103-j456.dts | 7 ++++++- arch/arm64/boot/dts/apple/t8103-j457.dts | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 013dd46766f8b3..c4e861f9f65070 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -31,7 +31,12 @@ }; &dcp { - apple,connector-type = "eDP"; + panel: panel { + compatible = "apple,panel-j293", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; }; &bluetooth0 { diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index e8ca92270dc9d2..7836d1607294fc 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -31,7 +31,12 @@ }; &dcp { - apple,connector-type = "eDP"; + panel: panel { + compatible = "apple,panel-j313", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <420>; + }; }; &bluetooth0 { diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index ca74b7f87c0c3b..19f14208a80e16 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -22,7 +22,12 @@ }; &dcp { - apple,connector-type = "eDP"; + panel: panel { + compatible = "apple,panel-j456", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; }; &bluetooth0 { diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 2f248178ce5885..72a0a1b9e2db25 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -22,7 +22,12 @@ }; &dcp { - apple,connector-type = "eDP"; + panel: panel { + compatible = "apple,panel-j457", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; }; &bluetooth0 { From 9ea709a30db187cf96866ec8cf5edcfceb926a43 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 31 Oct 2022 01:21:34 +0100 Subject: [PATCH 326/635] arch: arm64: apple: Add dcp panel node for t600x based laptops The panel node will contain among other properties backlight control related properties from the "backlight" node in the ADT. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t6000-j314s.dts | 7 +++++++ arch/arm64/boot/dts/apple/t6000-j316s.dts | 7 +++++++ arch/arm64/boot/dts/apple/t6001-j314c.dts | 7 +++++++ arch/arm64/boot/dts/apple/t6001-j316c.dts | 7 +++++++ arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 4 +++- 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index b514b1114c4d24..a2ccc654216164 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -25,6 +25,13 @@ brcm,board-type = "apple,maldives"; }; +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + &sound { model = "MacBook Pro J314"; }; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index 7f5c91bb5f3210..99a76392b3b791 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -25,6 +25,13 @@ brcm,board-type = "apple,madagascar"; }; +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + &sound { model = "MacBook Pro J316"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index 35ff978b998702..82d851d4cd3857 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -25,6 +25,13 @@ brcm,board-type = "apple,maldives"; }; +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + &sound { model = "MacBook Pro J314"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 6a35c87aa7ab02..a6987c8324dbd7 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -25,6 +25,13 @@ brcm,board-type = "apple,madagascar"; }; +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + &sound { model = "MacBook Pro J316"; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 334ae2fea619ce..30cfc36dd2b54a 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -71,7 +71,9 @@ }; &dcp { - apple,connector-type = "eDP"; + panel: panel { + apple,max-brightness = <500>; + }; }; /* USB Type C */ From 2b41789384499a817e261c829c1485ffd57f6115 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 15 Nov 2022 20:12:33 +0100 Subject: [PATCH 327/635] arm64: dts: apple: t8112: Add eFuses node Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112.dtsi | 97 ++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 749231a16f0a36..b2c124080c145e 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -778,6 +778,103 @@ interrupts = ; }; + efuse@23d2c8000 { + compatible = "apple,t8112-efuses", "apple,efuses"; + reg = <0x2 0x3d2c8000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + atcphy0_auspll_rodco_bias_adjust: efuse@480,20 { + reg = <0x480 4>; + bits = <20 3>; + }; + + atcphy0_auspll_rodco_encap: efuse@480,23 { + reg = <0x480 4>; + bits = <23 2>; + }; + + atcphy0_auspll_dtc_vreg_adjust: efuse@480,25 { + reg = <0x480 4>; + bits = <25 3>; + }; + + atcphy0_auspll_fracn_dll_start_capcode: efuse@480,28 { + reg = <0x480 4>; + bits = <28 2>; + }; + + atcphy0_aus_cmn_shm_vreg_trim: efuse@480,30 { + reg = <0x480 8>; + bits = <30 5>; + }; + + atcphy0_cio3pll_dco_coarsebin0: efuse@484,3 { + reg = <0x484 4>; + bits = <3 6>; + }; + + atcphy0_cio3pll_dco_coarsebin1: efuse@484,9 { + reg = <0x484 4>; + bits = <9 6>; + }; + + atcphy0_cio3pll_dll_start_capcode: efuse@484,15 { + reg = <0x484 4>; + bits = <15 2>; + }; + + atcphy0_cio3pll_dtc_vreg_adjust: efuse@484,17 { + reg = <0x484 0x4>; + bits = <17 3>; + }; + + atcphy1_auspll_rodco_bias_adjust: efuse@484,30 { + reg = <0x484 8>; + bits = <30 3>; + }; + + atcphy1_auspll_rodco_encap: efuse@488,1 { + reg = <0x488 8>; + bits = <1 2>; + }; + + atcphy1_auspll_dtc_vreg_adjust: efuse@488,3 { + reg = <0x488 4>; + bits = <3 3>; + }; + + atcphy1_auspll_fracn_dll_start_capcode: efuse@488,6 { + reg = <0x488 4>; + bits = <6 2>; + }; + + atcphy1_aus_cmn_shm_vreg_trim: efuse@488,8 { + reg = <0x488 4>; + bits = <8 5>; + }; + + atcphy1_cio3pll_dco_coarsebin0: efuse@488,13 { + reg = <0x488 4>; + bits = <13 6>; + }; + + atcphy1_cio3pll_dco_coarsebin1: efuse@488,19 { + reg = <0x488 4>; + bits = <19 6>; + }; + + atcphy1_cio3pll_dll_start_capcode: efuse@488,25 { + reg = <0x488 4>; + bits = <25 2>; + }; + + atcphy1_cio3pll_dtc_vreg_adjust: efuse@488,27 { + reg = <0x488 0x4>; + bits = <27 3>; + }; + }; + smc_mbox: mbox@23e408000 { compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x3e408000 0x0 0x4000>; From 7756f56c9e528924a110393a97f37c1b13f99a16 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 15 Nov 2022 20:14:27 +0100 Subject: [PATCH 328/635] arm64: dts: apple: t8112: Add ATCPHY nodes Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 31 +++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 83 +++++++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 2c14f4479c401f..1d0d1ab4a97cc9 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -11,6 +11,8 @@ / { aliases { + atcphy0 = &atcphy0; + atcphy1 = &atcphy1; serial0 = &serial0; serial2 = &serial2; }; @@ -68,6 +70,12 @@ remote-endpoint = <&typec0_usb_hs>; }; }; + port@1 { + reg = <1>; + typec0_con_ss: endpoint { + remote-endpoint = <&typec0_usb_ss>; + }; + }; }; }; }; @@ -93,6 +101,12 @@ remote-endpoint = <&typec1_usb_hs>; }; }; + port@1 { + reg = <1>; + typec1_con_ss: endpoint { + remote-endpoint = <&typec1_usb_ss>; + }; + }; }; }; }; @@ -115,6 +129,23 @@ }; }; +/* Type-C PHYs */ +&atcphy0 { + port { + typec0_usb_ss: endpoint { + remote-endpoint = <&typec0_con_ss>; + }; + }; +}; + +&atcphy1 { + port { + typec1_usb_ss: endpoint { + remote-endpoint = <&typec1_con_ss>; + }; + }; +}; + &i2c1 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index b2c124080c145e..5d7220318171d1 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include #include / { @@ -1072,6 +1073,9 @@ role-switch-default-mode = "host"; iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; }; dwc3_0_dart_0: iommu@382f00000 { @@ -1092,6 +1096,44 @@ power-domains = <&ps_atc0_usb>; }; + atcphy0: phy@383000000 { + compatible = "apple,t8112-atcphy", "apple,t8103-atcphy"; + reg = <0x3 0x83000000 0x0 0x4c000>, + <0x3 0x83050000 0x0 0x8000>, + <0x3 0x80000000 0x0 0x4000>, + <0x3 0x82a90000 0x0 0x4000>, + <0x3 0x82a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + nvmem-cells = <&atcphy0_aus_cmn_shm_vreg_trim>, + <&atcphy0_auspll_rodco_encap>, + <&atcphy0_auspll_rodco_bias_adjust>, + <&atcphy0_auspll_fracn_dll_start_capcode>, + <&atcphy0_auspll_dtc_vreg_adjust>, + <&atcphy0_cio3pll_dco_coarsebin0>, + <&atcphy0_cio3pll_dco_coarsebin1>, + <&atcphy0_cio3pll_dll_start_capcode>, + <&atcphy0_cio3pll_dtc_vreg_adjust>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&ps_atc0_usb>; + }; + dwc3_1: usb@502280000 { compatible = "apple,t8112-dwc3", "apple,dwc3", "snps,dwc3"; reg = <0x5 0x02280000 0x0 0x100000>; @@ -1102,6 +1144,9 @@ role-switch-default-mode = "host"; iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; }; dwc3_1_dart_0: iommu@502f00000 { @@ -1122,6 +1167,44 @@ power-domains = <&ps_atc1_usb>; }; + atcphy1: phy@503000000 { + compatible = "apple,t8112-atcphy", "apple,t8103-atcphy"; + reg = <0x5 0x03000000 0x0 0x4c000>, + <0x5 0x03050000 0x0 0x8000>, + <0x5 0x0 0x0 0x4000>, + <0x5 0x02a90000 0x0 0x4000>, + <0x5 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + nvmem-cells = <&atcphy1_aus_cmn_shm_vreg_trim>, + <&atcphy1_auspll_rodco_encap>, + <&atcphy1_auspll_rodco_bias_adjust>, + <&atcphy1_auspll_fracn_dll_start_capcode>, + <&atcphy1_auspll_dtc_vreg_adjust>, + <&atcphy1_cio3pll_dco_coarsebin0>, + <&atcphy1_cio3pll_dco_coarsebin1>, + <&atcphy1_cio3pll_dll_start_capcode>, + <&atcphy1_cio3pll_dtc_vreg_adjust>; + nvmem-cell-names = "aus_cmn_shm_vreg_trim", + "auspll_rodco_encap", + "auspll_rodco_bias_adjust", + "auspll_fracn_dll_start_capcode", + "auspll_dtc_vreg_adjust", + "cio3pll_dco_coarsebin0", + "cio3pll_dco_coarsebin1", + "cio3pll_dll_start_capcode", + "cio3pll_dtc_vreg_adjust"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart: iommu@681008000 { compatible = "apple,t8110-dart"; reg = <0x6 0x81008000 0x0 0x4000>; From e8fac9f1bd4524e01b17abf6c0256153550690c1 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 20 Nov 2022 20:22:57 +0100 Subject: [PATCH 329/635] arm64: dts: apple: t8112: Add dcp/disp0 nodes Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-j413.dts | 10 +++ arch/arm64/boot/dts/apple/t8112-j493.dts | 9 +++ arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 10 +++ arch/arm64/boot/dts/apple/t8112.dtsi | 76 +++++++++++++++++++++++ 4 files changed, 105 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 92f5e7f2bf24ed..93b4e2aa5fcf24 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -35,6 +35,16 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j413", "apple,panel"; + width-mm = <290>; + height-mm = <189>; + adj-height-mm = <181>; + apple,max-brightness = <525>; + }; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index d80a403279aeda..8b1369781b3b83 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -35,6 +35,15 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j493", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 1d0d1ab4a97cc9..7edabecc862e46 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -13,6 +13,9 @@ aliases { atcphy0 = &atcphy0; atcphy1 = &atcphy1; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; }; @@ -32,6 +35,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 5d7220318171d1..a468641bd40121 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -367,6 +367,14 @@ clock-output-names = "clk_120m"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -387,6 +395,74 @@ #performance-domain-cells = <0>; }; + disp0_piodma: mapper@230000000 { + compatible = "apple,t8112-dcp-piodma", "apple,dcp-piodma"; + iommus = <&disp0_dart 4>; + }; + + disp0_dart: iommu@231304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8112-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "disp-5", "doorbell"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x61c000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>, + <0x2 0x3b730000 0x0 0x1000>, + <0x2 0x3bc3c000 0x0 0x1000>; + power-domains = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + apple,asc-dram-mask = <0x0 0x0>; + apple,piodma-mapper = <&disp0_piodma>; + phandle = <&dcp>; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8110-dart"; reg = <0x2 0x35004000 0x0 0x4000>; From f91172ba2b71d40dc3a9a80f3ca10dd91c387436 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 2 Nov 2022 15:58:07 +0900 Subject: [PATCH 330/635] scripts/dtc: Add support for floating-point literals Signed-off-by: Asahi Lina --- scripts/dtc/data.c | 27 +++++++++++++++++++++++++++ scripts/dtc/dtc-lexer.l | 22 ++++++++++++++++++++++ scripts/dtc/dtc-parser.y | 16 ++++++++++++++++ scripts/dtc/dtc.h | 1 + 4 files changed, 66 insertions(+) diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c index 14734233ad8b7e..d12c1f0146bedf 100644 --- a/scripts/dtc/data.c +++ b/scripts/dtc/data.c @@ -184,6 +184,33 @@ struct data data_append_integer(struct data d, uint64_t value, int bits) } } +struct data data_append_float(struct data d, double value, int bits) +{ + float f32; + uint32_t u32; + double f64; + uint64_t u64; + fdt32_t value_32; + fdt64_t value_64; + + switch (bits) { + case 32: + f32 = value; + memcpy(&u32, &f32, sizeof(u32)); + value_32 = cpu_to_fdt32(u32); + return data_append_data(d, &value_32, 4); + + case 64: + f64 = value; + memcpy(&u64, &f64, sizeof(u64)); + value_64 = cpu_to_fdt64(u64); + return data_append_data(d, &value_64, 8); + + default: + die("Invalid literal size (%d)\n", bits); + } +} + struct data data_append_re(struct data d, uint64_t address, uint64_t size) { struct fdt_reserve_entry re; diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index de60a70b6bdbcb..ac0fadff20802d 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -151,6 +151,28 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); return DT_LABEL; } +[-+]?(([0-9]+\.[0-9]*)|([0-9]*\.[0-9]+))(e[-+]?[0-9]+)?f? { + char *e; + DPRINT("Floating-point Literal: '%s'\n", yytext); + + errno = 0; + yylval.floating = strtod(yytext, &e); + + if (*e && (*e != 'f' || e[1])) { + lexical_error("Bad floating-point literal '%s'", + yytext); + } + + if (errno == ERANGE) + lexical_error("Floating-point literal '%s' out of range", + yytext); + else + /* ERANGE is the only strtod error triggerable + * by strings matching the pattern */ + assert(errno == 0); + return DT_FP_LITERAL; + } + ([0-9]+|0[xX][0-9a-fA-F]+)(U|L|UL|LL|ULL)? { char *e; DPRINT("Integer Literal: '%s'\n", yytext); diff --git a/scripts/dtc/dtc-parser.y b/scripts/dtc/dtc-parser.y index bff1337ec2665a..398451f2fab54b 100644 --- a/scripts/dtc/dtc-parser.y +++ b/scripts/dtc/dtc-parser.y @@ -48,6 +48,7 @@ static bool is_ref_relative(const char *ref) struct node *nodelist; struct reserve_info *re; uint64_t integer; + double floating; unsigned int flags; } @@ -61,6 +62,7 @@ static bool is_ref_relative(const char *ref) %token DT_OMIT_NO_REF %token DT_PROPNODENAME %token DT_LITERAL +%token DT_FP_LITERAL %token DT_CHAR_LITERAL %token DT_BYTE %token DT_STRING @@ -86,6 +88,7 @@ static bool is_ref_relative(const char *ref) %type subnode %type subnodes +%type floating_prim %type integer_prim %type integer_unary %type integer_mul @@ -392,6 +395,15 @@ arrayprefix: $$.data = data_add_marker(empty_data, TYPE_UINT32, NULL); $$.bits = 32; } + | arrayprefix floating_prim + { + if ($1.bits < 32) { + ERROR(&@2, "Floating-point values must be" + " 32-bit or 64-bit"); + } + + $$.data = data_append_float($1.data, $2, $1.bits); + } | arrayprefix integer_prim { if ($1.bits < 64) { @@ -436,6 +448,10 @@ arrayprefix: } ; +floating_prim: + DT_FP_LITERAL + ; + integer_prim: DT_LITERAL | DT_CHAR_LITERAL diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h index 0a1f54991026bd..1054d922207cc5 100644 --- a/scripts/dtc/dtc.h +++ b/scripts/dtc/dtc.h @@ -177,6 +177,7 @@ struct data data_insert_at_marker(struct data d, struct marker *m, struct data data_merge(struct data d1, struct data d2); struct data data_append_cell(struct data d, cell_t word); struct data data_append_integer(struct data d, uint64_t word, int bits); +struct data data_append_float(struct data d, double value, int bits); struct data data_append_re(struct data d, uint64_t address, uint64_t size); struct data data_append_addr(struct data d, uint64_t addr); struct data data_append_byte(struct data d, uint8_t byte); From 738b795fc99424540d8b7dafd9f426a7e6f79690 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 18 Aug 2022 02:15:43 +0900 Subject: [PATCH 331/635] arm64: dts: apple: t8103*: Add GPU nodes Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t8103.dtsi | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 876e03dbe6d3e3..476efeb1ccd5e6 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -353,6 +353,24 @@ clock-output-names = "nco_ref"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0x9 0xfff74000 0 0x4000>; + }; + + uat_pagetables: uat-pagetables { + reg = <0x9 0xfff78000 0 0x40000>; + }; + + uat_ttbs: uat-ttbs { + reg = <0x9 0xfffb8000 0 0x4000>; + }; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -361,6 +379,36 @@ ranges; nonposted-mmio; + agx: gpu@206400000 { + compatible = "apple,agx-t8103", "apple,agx-g13g"; + reg = <0x2 0x6400000 0 0x40000>, + <0x2 0x4000000 0 0x1000000>; + reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + mboxes = <&agx_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + }; + + agx_mbox: mbox@206408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + cpufreq_e: performance-controller@210e20000 { compatible = "apple,t8103-cluster-cpufreq", "apple,cluster-cpufreq"; reg = <0x2 0x10e20000 0 0x1000>; From ed7a3f9f442f3401c1cfa53b6f18c8c759ab198b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 22 Oct 2022 00:15:22 +0900 Subject: [PATCH 332/635] arm64: dts: Add GPU performance data to t8103.dts Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t8103.dtsi | 52 +++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 476efeb1ccd5e6..e535f416954709 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -20,6 +20,10 @@ #address-cells = <2>; #size-cells = <2>; + aliases { + gpu = &gpu; + }; + cpus { #address-cells = <2>; #size-cells = <0>; @@ -299,6 +303,50 @@ #endif }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + apple,opp-rel-power = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <396000000>; + opp-microvolt = <603000>; + apple,opp-rel-power = <19>; + }; + opp02 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <640000>; + apple,opp-rel-power = <26>; + }; + opp03 { + opp-hz = /bits/ 64 <720000000>; + opp-microvolt = <690000>; + apple,opp-rel-power = <38>; + }; + opp04 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <784000>; + apple,opp-rel-power = <60>; + }; + opp05 { + opp-hz = /bits/ 64 <1128000000>; + opp-microvolt = <862000>; + apple,opp-rel-power = <87>; + }; + opp06 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <931000>; + apple,opp-rel-power = <100>; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -379,7 +427,7 @@ ranges; nonposted-mmio; - agx: gpu@206400000 { + gpu: gpu@206400000 { compatible = "apple,agx-t8103", "apple,agx-g13g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; @@ -394,6 +442,8 @@ power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; + + operating-points-v2 = <&gpu_opp>; }; agx_mbox: mbox@206408000 { From ef8d30790e2a80b7af655265c188c6213c9f0e18 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 2 Nov 2022 23:32:46 +0900 Subject: [PATCH 333/635] arm64: dts: Add power data for t8103 Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t8103-j274.dts | 4 +++ arch/arm64/boot/dts/apple/t8103-j456.dts | 4 +++ arch/arm64/boot/dts/apple/t8103-j457.dts | 4 +++ arch/arm64/boot/dts/apple/t8103.dtsi | 40 +++++++++++++++++++----- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 7169563c1b351a..0a69e2962c7c7f 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -127,3 +127,7 @@ }; }; + +&gpu { + apple,perf-base-pstate = <3>; +}; diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 19f14208a80e16..c16a0594b1d2ca 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -127,3 +127,7 @@ }; }; }; + +&gpu { + apple,perf-base-pstate = <3>; +}; diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index 72a0a1b9e2db25..c1d1201ecbe2cc 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -100,3 +100,7 @@ }; }; }; + +&gpu { + apple,perf-base-pstate = <3>; +}; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index e535f416954709..e0d724bccf8293 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -313,37 +313,37 @@ opp00 { opp-hz = /bits/ 64 <0>; opp-microvolt = <400000>; - apple,opp-rel-power = <0>; + opp-microwatt = <0>; }; opp01 { opp-hz = /bits/ 64 <396000000>; opp-microvolt = <603000>; - apple,opp-rel-power = <19>; + opp-microwatt = <3714690>; }; opp02 { opp-hz = /bits/ 64 <528000000>; opp-microvolt = <640000>; - apple,opp-rel-power = <26>; + opp-microwatt = <5083260>; }; opp03 { opp-hz = /bits/ 64 <720000000>; opp-microvolt = <690000>; - apple,opp-rel-power = <38>; + opp-microwatt = <7429380>; }; opp04 { opp-hz = /bits/ 64 <924000000>; opp-microvolt = <784000>; - apple,opp-rel-power = <60>; + opp-microwatt = <11730600>; }; opp05 { opp-hz = /bits/ 64 <1128000000>; opp-microvolt = <862000>; - apple,opp-rel-power = <87>; + opp-microwatt = <17009370>; }; opp06 { opp-hz = /bits/ 64 <1278000000>; opp-microvolt = <931000>; - apple,opp-rel-power = <100>; + opp-microwatt = <19551000>; }; }; @@ -444,6 +444,32 @@ memory-region-names = "ttbs", "pagetables", "handoff"; operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <850000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <7.5>; + apple,avg-power-kp = <4.0>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,power-zones = <30000 100 6875>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <91.5>; + apple,ppm-kp = <6.9>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; }; agx_mbox: mbox@206408000 { From 58e8c12cef9403139b517120bc9f44fa13abb573 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 3 Nov 2022 01:03:44 +0900 Subject: [PATCH 334/635] arm64: dts: Add t600x GPU nodes Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t6000.dtsi | 6 ++ arch/arm64/boot/dts/apple/t6001.dtsi | 26 ++++++++ arch/arm64/boot/dts/apple/t6002.dtsi | 8 +++ arch/arm64/boot/dts/apple/t600x-common.dtsi | 66 +++++++++++++++++++++ arch/arm64/boot/dts/apple/t600x-die0.dtsi | 64 ++++++++++++++++++++ arch/arm64/boot/dts/apple/t600x-j375.dtsi | 9 +++ 6 files changed, 179 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 89c3b211b116e9..c9e4e52d9aac92 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6001, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6001.dtsi" / { @@ -16,3 +18,7 @@ }; /delete-node/ &pmgr_south; + +&gpu { + compatible = "apple,agx-t6000", "apple,agx-g13x"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 0bdd1966f5302e..6e7e7cdeacf943 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -16,11 +16,33 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { compatible = "apple,t6001", "apple,arm-platform"; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -63,3 +85,7 @@ }; }; }; + +&gpu { + compatible = "apple,agx-t6001", "apple,agx-g13x"; +}; diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index 8fa2d8dd72ff7f..f1164315be755a 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -16,6 +16,10 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -301,3 +305,7 @@ // On t6002, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; + +&gpu { + compatible = "apple,agx-t6002", "apple,agx-g13x"; +}; diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index 01385ef831ca91..c76ab2fc8f96dd 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -11,6 +11,10 @@ #address-cells = <2>; #size-cells = <2>; + aliases { + gpu = &gpu; + }; + cpus { #address-cells = <2>; #size-cells = <0>; @@ -333,6 +337,50 @@ */ }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <388800000>; + opp-microvolt = GPU_REPEAT(634000); + opp-microwatt = <25011450>; + }; + opp02 { + opp-hz = /bits/ 64 <486000000>; + opp-microvolt = GPU_REPEAT(650000); + opp-microwatt = <31681170>; + }; + opp03 { + opp-hz = /bits/ 64 <648000000>; + opp-microvolt = GPU_REPEAT(668000); + opp-microwatt = <41685750>; + }; + opp04 { + opp-hz = /bits/ 64 <777600000>; + opp-microvolt = GPU_REPEAT(715000); + opp-microwatt = <56692620>; + }; + opp05 { + opp-hz = /bits/ 64 <972000000>; + opp-microvolt = GPU_REPEAT(778000); + opp-microwatt = <83371500>; + }; + opp06 { + opp-hz = /bits/ 64 <1296000000>; + opp-microvolt = GPU_REPEAT(903000); + opp-microwatt = <166743000>; + }; + }; + pmu-e { compatible = "apple,icestorm-pmu"; interrupt-parent = <&aic>; @@ -384,4 +432,22 @@ #clock-cells = <0>; clock-output-names = "nco_ref"; }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 259cf535a7fadc..6c3ce219e46dda 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -422,6 +422,70 @@ #sound-dai-cells = <1>; }; + gpu: gpu@406400000 { + compatible = "apple,agx-g13x"; + reg = <0x4 0x6400000 0 0x40000>, + <0x4 0x4000000 0 0x1000000>; + reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + mboxes = <&agx_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <790000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <2.4>; + apple,avg-power-kp = <1.5>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <500.0>; + apple,fast-die0-proportional-gain = <72.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain = <6.3>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <15.75>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <30>; + apple,ppm-kp = <1.5>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); + }; + + agx_mbox: mbox@406408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + pcie0_dart_0: iommu@581008000 { compatible = "apple,t6000-dart"; reg = <0x5 0x81008000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index fe003fa6227d7f..23706792267e22 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -382,3 +382,12 @@ &pcie0_dart_3 { status = "okay"; }; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; From 70192c556cacd7d2d6f3ab6748aeea68fb4f7344 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 4 Nov 2022 21:56:26 +0900 Subject: [PATCH 335/635] arm64: dts: t8103: Add GPU leak coefficients Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t8103.dtsi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index e0d724bccf8293..b8db39a245377a 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -470,6 +470,9 @@ apple,pwr-integral-min-clamp = <0>; apple,pwr-min-duty-cycle = <40>; apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = <1000.0>; + apple,sram-leak-coef = <45.0>; }; agx_mbox: mbox@206408000 { From e9507db4aed611962c2202b6c9e93e4252376d57 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 9 Nov 2022 10:44:43 +0900 Subject: [PATCH 336/635] arm64: dts: apple: Add no-map to GPU reserved-memory nodes Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t600x-common.dtsi | 3 +++ arch/arm64/boot/dts/apple/t8103.dtsi | 3 +++ 2 files changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index c76ab2fc8f96dd..279ba91d8abacd 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -440,14 +440,17 @@ uat_handoff: uat-handoff { reg = <0 0 0 0>; + no-map; }; uat_pagetables: uat-pagetables { reg = <0 0 0 0>; + no-map; }; uat_ttbs: uat-ttbs { reg = <0 0 0 0>; + no-map; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index b8db39a245377a..58a6b7ad2c5f60 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -408,14 +408,17 @@ uat_handoff: uat-handoff { reg = <0x9 0xfff74000 0 0x4000>; + no-map; }; uat_pagetables: uat-pagetables { reg = <0x9 0xfff78000 0 0x40000>; + no-map; }; uat_ttbs: uat-ttbs { reg = <0x9 0xfffb8000 0 0x4000>; + no-map; }; }; From 103fd4b5be56a92ed7733fc38cbdd687144f9454 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 25 Nov 2022 23:06:59 +0900 Subject: [PATCH 337/635] arm64: dts: apple: Add GPU nodes to T8112 Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t8112.dtsi | 141 +++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index a468641bd40121..3680cd5cb10f29 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -20,6 +20,10 @@ #address-cells = <2>; #size-cells = <2>; + aliases { + gpu = &gpu; + }; + cpus { #address-cells = <2>; #size-cells = <0>; @@ -321,6 +325,60 @@ #endif }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = <603000>; + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = <675000>; + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = <710000>; + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = <775000>; + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = <820000>; + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = <875000>; + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <915000>; + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = <950000>; + opp-microwatt = <22800000>; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -375,6 +433,27 @@ clock-output-names = "clk_disp0"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0x9 0xfff74000 0 0x4000>; + no-map; + }; + + uat_pagetables: uat-pagetables { + reg = <0x9 0xfff78000 0 0x40000>; + no-map; + }; + + uat_ttbs: uat-ttbs { + reg = <0x9 0xfffb8000 0 0x4000>; + no-map; + }; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -383,6 +462,68 @@ ranges; nonposted-mmio; + gpu: gpu@206400000 { + compatible = "apple,agx-t8112", "apple,agx-g14g"; + reg = <0x2 0x6400000 0 0x40000>, + <0x2 0x4000000 0 0x1000000>; + reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + ; + mboxes = <&agx_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + + apple,firmware-version = <12 4 0>; + apple,firmware-compat = <12 4 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <780000>; + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <9.375>; + apple,avg-power-kp = <3.22>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <5.94>; + apple,perf-integral-gain2 = <5.94>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <14.85>; + apple,perf-proportional-gain2 = <14.85>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <205.0>; + apple,ppm-kp = <0.75>; + apple,pwr-min-duty-cycle = <40>; + apple,core-leak-coef = <1920.0>; + apple,sram-leak-coef = <74.0>; + }; + + agx_mbox: mbox@206408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + cpufreq_e: cpufreq@210e20000 { compatible = "apple,t8112-cluster-cpufreq", "apple,cluster-cpufreq"; reg = <0x2 0x10e20000 0 0x1000>; From 9b36c2f4fcedc4270b8d07b661e9c10a04f9d573 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 25 Nov 2022 23:07:22 +0900 Subject: [PATCH 338/635] arm64: dts: apple: Add GPU firmware versions to t8113/t600x Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 3 +++ arch/arm64/boot/dts/apple/t8103.dtsi | 3 +++ 2 files changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 6c3ce219e46dda..f257fecfb18e0f 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -438,6 +438,9 @@ memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + operating-points-v2 = <&gpu_opp>; apple,perf-base-pstate = <1>; apple,min-sram-microvolt = <790000>; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 58a6b7ad2c5f60..57d79308ae7aa4 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -446,6 +446,9 @@ memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + operating-points-v2 = <&gpu_opp>; apple,perf-base-pstate = <1>; apple,min-sram-microvolt = <850000>; From 7a00949ad86279f7af732f8f2a916bd453b972bd Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 29 Oct 2022 23:18:27 +0300 Subject: [PATCH 339/635] arm64: dts: apple: t600x: Add the NVRAM bindings Add the SPI controller and the nvram partition bindings for M1 Pro/Max/Ultra Macs Signed-off-by: Sasha Finkelstein --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 1 - arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 13 +++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index f257fecfb18e0f..5c7d78b2dc5c7e 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -351,7 +351,6 @@ pinctrl-0 = <&spi1_pins>; pinctrl-names = "default"; power-domains = <&ps_spi1>; - status = "disabled"; }; spi3: spi@39b10c000 { diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 30cfc36dd2b54a..c8f09403e46efc 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -264,14 +264,23 @@ }; &spi1 { - status = "disabled"; - flash@0 { compatible = "jedec,spi-nor"; reg = <0x0>; spi-max-frequency = <25000000>; #address-cells = <1>; #size-cells = <1>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@700000 { + label = "nvram"; + reg = <0x700000 0x20000>; + }; + }; }; }; From 45b52354bf4e98dbe98716e4bf0f2ed29c402c4e Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 29 Oct 2022 23:17:40 +0300 Subject: [PATCH 340/635] arm64: dts: apple: t8112: Add the NVRAM bindings Add the SPI controller and the nvram partition bindings for M2 Macs Signed-off-by: Sasha Finkelstein --- arch/arm64/boot/dts/apple/t8112.dtsi | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 3680cd5cb10f29..21855e8a5bd863 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -454,6 +454,13 @@ }; }; + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -692,6 +699,38 @@ status = "disabled"; }; + spi1: spi@235104000 { + compatible = "apple,t8112-spi", "apple,spi"; + reg = <0x2 0x35104000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + pinctrl-0 = <&spi1_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi1>; + #address-cells = <1>; + #size-cells = <0>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0x0>; + spi-max-frequency = <25000000>; + #address-cells = <1>; + #size-cells = <1>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@700000 { + label = "nvram"; + reg = <0x700000 0x20000>; + }; + }; + }; + }; + spi3: spi@23510c000 { compatible = "apple,t8112-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -865,6 +904,13 @@ ; }; + spi1_pins: spi1-pins { + pinmux = , + , + , + ; + }; + spi3_pins: spi3-pins { pinmux = , , From 58314c4ea42f0b947acfa26736b5ef8280f7e54f Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Sat, 29 Oct 2022 23:14:39 +0300 Subject: [PATCH 341/635] arm64: dts: apple: t8103: Add the NVRAM bindings Add the SPI controller and the nvram partition bindings for M1 Macs Signed-off-by: Sasha Finkelstein --- arch/arm64/boot/dts/apple/t8103.dtsi | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 57d79308ae7aa4..7300a2f4d6e8e5 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -383,6 +383,13 @@ clock-output-names = "clk_120m"; }; + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ clk_disp0: clock-disp0 { compatible = "fixed-clock"; @@ -658,6 +665,38 @@ status = "disabled"; }; + spi1: spi@235104000 { + compatible = "apple,t8103-spi", "apple,spi"; + reg = <0x2 0x35104000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + pinctrl-0 = <&spi1_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi1>; + #address-cells = <1>; + #size-cells = <0>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0x0>; + spi-max-frequency = <25000000>; + #address-cells = <1>; + #size-cells = <1>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@700000 { + label = "nvram"; + reg = <0x700000 0x20000>; + }; + }; + }; + }; + spi3: spi@23510c000 { compatible = "apple,t8103-spi", "apple,spi"; reg = <0x2 0x3510c000 0x0 0x4000>; @@ -828,6 +867,13 @@ ; }; + spi1_pins: spi1-pins { + pinmux = , + , + , + ; + }; + spi3_pins: spi3-pins { pinmux = , , From 6fe110ada8290f125cdae7c52fe43178352cf0d6 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 13 Dec 2022 00:17:35 +0900 Subject: [PATCH 342/635] arm64: dts: apple: t600x: Add DCP power domain to missing devices Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 2 ++ arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 1 + arch/arm64/boot/dts/apple/t600x-j375.dtsi | 1 + arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 2 -- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 5c7d78b2dc5c7e..b1052485cae1bf 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -178,6 +178,7 @@ interrupt-parent = <&aic>; interrupts = ; status = "disabled"; + power-domains = <&ps_disp0_cpu0>; }; dcp_dart: iommu@38b30c000 { @@ -186,6 +187,7 @@ #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_disp0_cpu0>; }; dcp_mbox: mbox@38bc08000 { diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index c8f09403e46efc..a4963e1e9aba97 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -38,6 +38,7 @@ /* Format properties will be added by loader */ backlight = <&backlight>; status = "disabled"; + power-domains = <&ps_disp0_cpu0>; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 23706792267e22..a23462f8d3831d 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -35,6 +35,7 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + power-domains = <&ps_disp0_cpu0>; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 0bd44753b76a0c..00b317c2355b8c 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -1293,7 +1293,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_fe); power-domains = <&DIE_NODE(ps_afnc2_lw0)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; DIE_NODE(ps_disp0_cpu0): power-controller@350 { @@ -1303,7 +1302,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_cpu0); power-domains = <&DIE_NODE(ps_disp0_fe)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; From 2bd4b15751fa8f04a71d43f3917e846f9eb08970 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 13 Dec 2022 00:17:35 +0900 Subject: [PATCH 343/635] arm64: dts: apple: t8103: Add DCP power domain to missing devices Removes the "apple,always-on" property from ps_disp0_fe/cpu0. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-jxxx.dtsi | 1 + arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 2 -- arch/arm64/boot/dts/apple/t8103.dtsi | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 1ad2a57698afb9..6f78ab9b211d60 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -32,6 +32,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 0966322c5c8e3f..ea0ee0224b1cb1 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -646,7 +646,6 @@ #reset-cells = <0>; label = "disp0_fe"; power-domains = <&ps_rmx>, <&ps_pmp>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; ps_dispext_fe: power-controller@368 { @@ -1001,7 +1000,6 @@ #reset-cells = <0>; label = "disp0_cpu0"; power-domains = <&ps_disp0_fe>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; }; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 7300a2f4d6e8e5..59c1e79dd2191f 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -524,6 +524,7 @@ #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_disp0_cpu0>; status = "disabled"; }; @@ -533,6 +534,7 @@ #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_disp0_cpu0>; }; dcp_mbox: mbox@231c08000 { From 306b3599cc4e9b7c704701551cfe1ebc71664d6c Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 13 Dec 2022 00:17:35 +0900 Subject: [PATCH 344/635] arm64: dts: apple: t8112: Add DCP power domain to missing devices Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-jxxx.dtsi | 1 + arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 2 -- arch/arm64/boot/dts/apple/t8112.dtsi | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 7edabecc862e46..3c0580db1b05ea 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -30,6 +30,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index d36f6094698ddf..32232b4f8ab652 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -663,7 +663,6 @@ #reset-cells = <0>; label = "disp0_sys"; power-domains = <&ps_rmx1>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; ps_disp0_fe: power-controller@378 { @@ -673,7 +672,6 @@ #reset-cells = <0>; label = "disp0_fe"; power-domains = <&ps_disp0_sys>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; ps_dispext_sys: power-controller@380 { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 21855e8a5bd863..64f0bc10d35777 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -554,6 +554,7 @@ #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_disp0_cpu0>; status = "disabled"; }; @@ -563,6 +564,7 @@ #iommu-cells = <1>; interrupt-parent = <&aic>; interrupts = ; + power-domains = <&ps_disp0_cpu0>; }; dcp_mbox: mbox@231c08000 { From d84c35270f702368137f8a1244a56eea8e2022bc Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 30 Dec 2022 01:15:57 +0100 Subject: [PATCH 345/635] arm64: dts: apple: t8103: Add missing ps_pmp dependency to ps_gfx AGX' ASC crashes shortly after ps_pmp is powered down due to dcp runtime PM suspend. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index ea0ee0224b1cb1..724e7fd559e7a1 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -733,6 +733,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "gfx"; + power-domains = <&ps_pmp>; }; ps_dcs4: power-controller@320 { From 5bd6fabd807331807acf175dbccf9978a8ada021 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 9 Jan 2023 00:03:49 +0100 Subject: [PATCH 346/635] arm64: dts: apple: t600x: Add "ps_disp0_cpu0" as resets for dcp Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index b1052485cae1bf..79ac3ca44b21fb 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -219,6 +219,7 @@ <0x3 0x8b800000 0x0 0x800000>, <0x2 0x8e3d0000 0x0 0x4000>; power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; clocks = <&clk_disp0>; apple,asc-dram-mask = <0x1f0 0x00000000>; apple,piodma-mapper = <&disp0_piodma>; From ff60dbf01cd6ed74457b662da435e73c0279cedc Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 9 Jan 2023 00:05:06 +0100 Subject: [PATCH 347/635] arm64: dts: apple: t8103: Add "ps_disp0_cpu0" as resets for dcp Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8103.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 59c1e79dd2191f..95bb8973c07037 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -569,6 +569,7 @@ <0x2 0x3b738000 0x0 0x1000>, <0x2 0x3bc3c000 0x0 0x1000>; power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; clocks = <&clk_disp0>; apple,asc-dram-mask = <0xf 0x00000000>; apple,piodma-mapper = <&disp0_piodma>; From 6ef6b4b11faeb751ff790a8556e822becb5dec0e Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 9 Jan 2023 00:05:30 +0100 Subject: [PATCH 348/635] arm64: dts: apple: t8112: Add "ps_disp0_cpu0" as resets for dcp Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 64f0bc10d35777..fd8a9eac5cf857 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -599,6 +599,7 @@ <0x2 0x3b730000 0x0 0x1000>, <0x2 0x3bc3c000 0x0 0x1000>; power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; clocks = <&clk_disp0>; apple,asc-dram-mask = <0x0 0x0>; apple,piodma-mapper = <&disp0_piodma>; From 3e4c13136d8b6f05341ba63ac8330979a6596968 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 11 Mar 2023 21:45:11 +0900 Subject: [PATCH 349/635] arm64: dts: apple: j314/j316: Disable ATC3_USB_AON power domain These power domains are normally always on for real Thunderbolt ports (or else dwc3 breaks), but not for the port that's hardwired to the HDMI bridge. Fixes some dmesg spam: apple-pmgr-pwrstate 292280000.power-management:power-controller@a0: always-on domain atc3_usb_aon is not on at boot Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index a4963e1e9aba97..91e40b9aa7f2ac 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -406,6 +406,10 @@ status = "disabled"; }; +&ps_atc3_usb_aon { + /delete-property/ apple,always-on; +}; + / { backlight: gpio-bl { compatible = "gpio-backlight"; From baf67c5b13e09fffcc055d5ebbb27af116650c4d Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 19 Mar 2023 19:12:52 +0900 Subject: [PATCH 350/635] arm64: dts: apple: Add keyboard alias & layout props for t8112 laptops Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8112-j413.dts | 5 ++++- arch/arm64/boot/dts/apple/t8112-j493.dts | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 93b4e2aa5fcf24..1b95c320247573 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -20,6 +20,7 @@ aliases { bluetooth0 = &bluetooth0; wifi0 = &wifi0; + keyboard = &keyboard; }; led-controller { @@ -214,7 +215,9 @@ firmware-name = "apple/tpmtfw-j413.bin"; }; - keyboard { + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; }; stm { diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index 8b1369781b3b83..50d0b4be3fd32b 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -20,6 +20,7 @@ aliases { bluetooth0 = &bluetooth0; wifi0 = &wifi0; + keyboard = &keyboard; }; led-controller { @@ -202,7 +203,9 @@ firmware-name = "apple/tpmtfw-j493.bin"; }; - keyboard { + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; }; stm { From 0525e216590f4838473be05f2ca667e9e35ee26e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 9 Apr 2023 23:49:01 +0900 Subject: [PATCH 351/635] arm64: dts: apple: Fix t600x mca IRQs Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 79ac3ca44b21fb..469cafd4a7e2e1 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -414,10 +414,10 @@ "tx2a", "rx2a", "tx2b", "rx2b", "tx3a", "rx3a", "tx3b", "rx3b"; interrupt-parent = <&aic>; - interrupts = , + interrupts = , + , , - , - ; + ; power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, <&ps_mca2>, <&ps_mca3>; resets = <&ps_audio_p>; From 9f021221600a9703dd8a1ce5fd1bddc2e022ed24 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Fri, 14 Apr 2023 08:07:52 +0200 Subject: [PATCH 352/635] WIP: drm/apple: Port to incompatible V13.3 firmware interface Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/dcp-internal.h | 6 +-- drivers/gpu/drm/apple/dcp.c | 4 +- drivers/gpu/drm/apple/iomfb.c | 24 ++++----- drivers/gpu/drm/apple/iomfb.h | 14 +++++ drivers/gpu/drm/apple/iomfb_template.c | 10 ++++ drivers/gpu/drm/apple/iomfb_template.h | 1 + drivers/gpu/drm/apple/iomfb_v12_3.c | 2 +- .../apple/{iomfb_v13_2.c => iomfb_v13_3.c} | 52 ++++++++++--------- .../apple/{iomfb_v13_2.h => iomfb_v13_3.h} | 10 ++-- 10 files changed, 76 insertions(+), 49 deletions(-) rename drivers/gpu/drm/apple/{iomfb_v13_2.c => iomfb_v13_3.c} (73%) rename drivers/gpu/drm/apple/{iomfb_v13_2.h => iomfb_v13_3.h} (52%) diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 45ef064b3e6fa5..71e1aa3766f37b 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -6,7 +6,7 @@ appledrm-y := apple_drv.o apple_dcp-y := dcp.o dcp_backlight.o iomfb.o parser.o apple_dcp-y += iomfb_v12_3.o -apple_dcp-y += iomfb_v13_2.o +apple_dcp-y += iomfb_v13_3.o apple_dcp-$(CONFIG_TRACING) += trace.o apple_piodma-y := dummy-piodma.o diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 59777809a7f370..fee15867b5c0cf 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -12,7 +12,7 @@ #include "iomfb.h" #include "iomfb_v12_3.h" -#include "iomfb_v13_2.h" +#include "iomfb_v13_3.h" #define DCP_MAX_PLANES 2 @@ -21,7 +21,7 @@ struct apple_dcp; enum dcp_firmware_version { DCP_FIRMWARE_UNKNOWN, DCP_FIRMWARE_V_12_3, - DCP_FIRMWARE_V_13_2, + DCP_FIRMWARE_V_13_3, }; enum { @@ -146,7 +146,7 @@ struct apple_dcp { /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ union { struct dcp_swap_submit_req_v12_3 v12_3; - struct dcp_swap_submit_req_v13_2 v13_2; + struct dcp_swap_submit_req_v13_3 v13_3; } swap; /* Current display mode */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2e1e077a8f274f..9724bc0a48e039 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -404,8 +404,8 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) return DCP_FIRMWARE_V_12_3; - if (strncmp(compat_str, "13.2.0", sizeof(compat_str)) == 0) - return DCP_FIRMWARE_V_13_2; + if (strncmp(compat_str, "13.3.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_13_3; dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", compat_str, fw_str); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d9bd4b52dda701..d31ebf7e301717 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -223,8 +223,8 @@ void dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_V_12_3: iomfb_sleep_v12_3(dcp); break; - case DCP_FIRMWARE_V_13_2: - iomfb_sleep_v13_2(dcp); + case DCP_FIRMWARE_V_13_3: + iomfb_sleep_v13_3(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -240,8 +240,8 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_12_3: iomfb_poweron_v12_3(dcp); break; - case DCP_FIRMWARE_V_13_2: - iomfb_poweron_v13_2(dcp); + case DCP_FIRMWARE_V_13_3: + iomfb_poweron_v13_3(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -258,8 +258,8 @@ void dcp_poweroff(struct platform_device *pdev) case DCP_FIRMWARE_V_12_3: iomfb_poweroff_v12_3(dcp); break; - case DCP_FIRMWARE_V_13_2: - iomfb_poweroff_v13_2(dcp); + case DCP_FIRMWARE_V_13_3: + iomfb_poweroff_v13_3(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -503,8 +503,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) case DCP_FIRMWARE_V_12_3: iomfb_flush_v12_3(dcp, crtc, state); break; - case DCP_FIRMWARE_V_13_2: - iomfb_flush_v13_2(dcp, crtc, state); + case DCP_FIRMWARE_V_13_3: + iomfb_flush_v13_3(dcp, crtc, state); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -519,8 +519,8 @@ void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_12_3: iomfb_start_v12_3(dcp); break; - case DCP_FIRMWARE_V_13_2: - iomfb_start_v13_2(dcp); + case DCP_FIRMWARE_V_13_3: + iomfb_start_v13_3(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -572,8 +572,8 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_12_3: iomfb_shutdown_v12_3(dcp); break; - case DCP_FIRMWARE_V_13_2: - iomfb_shutdown_v13_2(dcp); + case DCP_FIRMWARE_V_13_3: + iomfb_shutdown_v13_3(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 6602bf19107d43..e8168338d374ef 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -136,6 +136,20 @@ struct dcp_rt_bandwidth { u32 padding[7]; } __packed; +struct frame_sync_props { + u8 unk[28]; +}; + +struct dcp_set_frame_sync_props_req { + struct frame_sync_props props; + u8 frame_sync_props_null; + u8 padding[3]; +} __packed; + +struct dcp_set_frame_sync_props_resp { + struct frame_sync_props props; +} __packed; + /* Method calls */ enum dcpep_method { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 0e718742a92e02..b2bd5d384b5c39 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -670,6 +670,13 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) } } +static struct dcp_set_frame_sync_props_resp +dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req *req) +{ + return (struct dcp_set_frame_sync_props_resp){}; +} + /* Callback to get the current time as milliseconds since the UNIX epoch */ static u64 dcpep_cb_get_time(struct apple_dcp *dcp) { @@ -1031,6 +1038,9 @@ TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, struct dcp_set_dcpav_prop_end_req, u8); TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, struct dcp_rt_bandwidth); +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, + struct dcp_set_frame_sync_props_req, + struct dcp_set_frame_sync_props_resp); TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 539ec65e5825f4..e9c249609f46cb 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -48,6 +48,7 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_2f3[0x2d]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unk_320[0x13f]; + u64 unk_1; #endif } __packed; diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index c226a1139a84c8..8188321004a63f 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -2,7 +2,7 @@ /* Copyright The Asahi Linux Contributors */ #include "iomfb_v12_3.h" -#include "iomfb_v13_2.h" +#include "iomfb_v13_3.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v13_2.c b/drivers/gpu/drm/apple/iomfb_v13_3.c similarity index 73% rename from drivers/gpu/drm/apple/iomfb_v13_2.c rename to drivers/gpu/drm/apple/iomfb_v13_3.c index 356a2aa2433be0..18020c6cd39493 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_2.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -2,7 +2,7 @@ /* Copyright The Asahi Linux Contributors */ #include "iomfb_v12_3.h" -#include "iomfb_v13_2.h" +#include "iomfb_v13_3.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { @@ -25,13 +25,13 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A449", dcpep_enable_disable_video_power_savings), IOMFB_METHOD("A456", dcpep_first_client_open), IOMFB_METHOD("A457", iomfbep_last_client_close), - IOMFB_METHOD("A462", dcpep_set_display_refresh_properties), - IOMFB_METHOD("A465", dcpep_flush_supports_power), - IOMFB_METHOD("A471", dcpep_set_power_state), + IOMFB_METHOD("A463", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A466", dcpep_flush_supports_power), + IOMFB_METHOD("A472", dcpep_set_power_state), }; -#define DCP_FW v13_2 -#define DCP_FW_VER DCP_FW_VERSION(13, 2, 0) +#define DCP_FW v13_3 +#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) #include "iomfb_template.c" @@ -40,32 +40,34 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [1] = trampoline_true, /* did_power_on_signal */ [2] = trampoline_nop, /* will_power_off_signal */ [3] = trampoline_rt_bandwidth, + [6] = trampoline_set_frame_sync_props, [100] = iomfbep_cb_match_pmu_service, [101] = trampoline_zero, /* get_display_default_stride */ [102] = trampoline_nop, /* set_number_property */ - [103] = trampoline_nop, /* set_boolean_property */ - [106] = trampoline_nop, /* remove_property */ - [107] = trampoline_true, /* create_provider_service */ - [108] = trampoline_true, /* create_product_service */ - [109] = trampoline_true, /* create_pmu_service */ - [110] = trampoline_true, /* create_iomfb_service */ - [111] = trampoline_true, /* create_backlight_service */ - [112] = trampoline_true, /* create_nvram_servce? */ - [113] = trampoline_get_tiling_state, - [114] = trampoline_false, /* set_tiling_state */ - [119] = dcpep_cb_boot_1, - [120] = trampoline_false, /* is_dark_boot */ - [121] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [123] = trampoline_read_edt_data, - [125] = trampoline_prop_start, - [126] = trampoline_prop_chunk, - [127] = trampoline_prop_end, + [103] = trampoline_nop, /* trigger_user_cal_loader */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, /* remove_property */ + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_pmu_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_true, /* create_backlight_service */ + [113] = trampoline_true, /* create_nvram_servce? */ + [114] = trampoline_get_tiling_state, + [115] = trampoline_false, /* set_tiling_state */ + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, /* is_dark_boot */ + [122] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, [201] = trampoline_map_piodma, [202] = trampoline_unmap_piodma, [206] = iomfbep_cb_match_pmu_service_2, [207] = iomfbep_cb_match_backlight_service, - [208] = trampoline_get_time, - [211] = trampoline_nop, /* update_backlight_factor_prop */ + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, [300] = trampoline_pr_publish, [401] = trampoline_get_uint_prop, [404] = trampoline_nop, /* sr_set_uint_prop */ diff --git a/drivers/gpu/drm/apple/iomfb_v13_2.h b/drivers/gpu/drm/apple/iomfb_v13_3.h similarity index 52% rename from drivers/gpu/drm/apple/iomfb_v13_2.h rename to drivers/gpu/drm/apple/iomfb_v13_3.h index f3810b727235bc..bbb3156b40f893 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_2.h +++ b/drivers/gpu/drm/apple/iomfb_v13_3.h @@ -1,17 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright The Asahi Linux Contributors */ -#ifndef __APPLE_IOMFB_V13_2_H__ -#define __APPLE_IOMFB_V13_2_H__ +#ifndef __APPLE_IOMFB_V13_3_H__ +#define __APPLE_IOMFB_V13_3_H__ #include "version_utils.h" -#define DCP_FW v13_2 -#define DCP_FW_VER DCP_FW_VERSION(13, 2, 0) +#define DCP_FW v13_3 +#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) #include "iomfb_template.h" #undef DCP_FW_VER #undef DCP_FW -#endif /* __APPLE_IOMFB_V13_2_H__ */ +#endif /* __APPLE_IOMFB_V13_3_H__ */ From efa5a0687157ef6b5cbbde293ec88c26c77c1fdd Mon Sep 17 00:00:00 2001 From: Zongmin Zhou Date: Tue, 28 Mar 2023 10:31:29 +0800 Subject: [PATCH 353/635] drm/probe_helper: fix the warning reported when calling drm_kms_helper_poll_disable during suspend When drivers call drm_kms_helper_poll_disable from their device suspend implementation without enabled output polling before, following warning will be reported,due to work->func not be initialized: [ 55.141361] WARNING: CPU: 3 PID: 372 at kernel/workqueue.c:3066 __flush_work+0x22f/0x240 [ 55.141382] Modules linked in: nls_iso8859_1 snd_hda_codec_generic ledtrig_audio snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi snd_hda_codec snd_hda_core snd_hwdep snd_pcm snd_seq_midi snd_seq_midi_event snd_rawmidi snd_seq intel_rapl_msr intel_rapl_common bochs drm_vram_helper drm_ttm_helper snd_seq_device nfit ttm crct10dif_pclmul snd_timer ghash_clmulni_intel binfmt_misc sha512_ssse3 aesni_intel drm_kms_helper joydev input_leds syscopyarea crypto_simd snd cryptd sysfillrect sysimgblt mac_hid serio_raw soundcore qemu_fw_cfg sch_fq_codel msr parport_pc ppdev lp parport drm ramoops reed_solomon pstore_blk pstore_zone efi_pstore virtio_rng ip_tables x_tables autofs4 hid_generic usbhid hid ahci virtio_net i2c_i801 crc32_pclmul psmouse virtio_scsi libahci i2c_smbus lpc_ich xhci_pci net_failover virtio_blk xhci_pci_renesas failover [ 55.141430] CPU: 3 PID: 372 Comm: kworker/u16:9 Not tainted 6.2.0-rc6+ #16 [ 55.141433] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.12.1-0-ga5cab58e9a3f-prebuilt.qemu.org 04/01/2014 [ 55.141435] Workqueue: events_unbound async_run_entry_fn [ 55.141441] RIP: 0010:__flush_work+0x22f/0x240 [ 55.141444] Code: 8b 43 28 48 8b 53 30 89 c1 e9 f9 fe ff ff 4c 89 f7 e8 b5 95 d9 00 e8 00 53 08 00 45 31 ff e9 11 ff ff ff 0f 0b e9 0a ff ff ff <0f> 0b 45 31 ff e9 00 ff ff ff e8 e2 54 d8 00 66 90 90 90 90 90 90 [ 55.141446] RSP: 0018:ff59221940833c18 EFLAGS: 00010246 [ 55.141449] RAX: 0000000000000000 RBX: 0000000000000000 RCX: ffffffff9b72bcbe [ 55.141450] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ff3ea01e4265e330 [ 55.141451] RBP: ff59221940833c90 R08: 0000000000000000 R09: 8080808080808080 [ 55.141453] R10: ff3ea01e42b3caf4 R11: 000000000000000f R12: ff3ea01e4265e330 [ 55.141454] R13: 0000000000000001 R14: ff3ea01e505e5e80 R15: 0000000000000001 [ 55.141455] FS: 0000000000000000(0000) GS:ff3ea01fb7cc0000(0000) knlGS:0000000000000000 [ 55.141456] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 55.141458] CR2: 0000563543ad1546 CR3: 000000010ee82005 CR4: 0000000000771ee0 [ 55.141464] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 55.141465] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 55.141466] PKRU: 55555554 [ 55.141467] Call Trace: [ 55.141469] [ 55.141472] ? pcie_wait_cmd+0xdf/0x220 [ 55.141478] ? mptcp_seq_show+0xe0/0x180 [ 55.141484] __cancel_work_timer+0x124/0x1b0 [ 55.141487] cancel_delayed_work_sync+0x17/0x20 [ 55.141490] drm_kms_helper_poll_disable+0x26/0x40 [drm_kms_helper] [ 55.141516] drm_mode_config_helper_suspend+0x25/0x90 [drm_kms_helper] [ 55.141531] ? __pm_runtime_resume+0x64/0x90 [ 55.141536] bochs_pm_suspend+0x16/0x20 [bochs] [ 55.141540] pci_pm_suspend+0x8b/0x1b0 [ 55.141545] ? __pfx_pci_pm_suspend+0x10/0x10 [ 55.141547] dpm_run_callback+0x4c/0x160 [ 55.141550] __device_suspend+0x14c/0x4c0 [ 55.141553] async_suspend+0x24/0xa0 [ 55.141555] async_run_entry_fn+0x34/0x120 [ 55.141557] process_one_work+0x21a/0x3f0 [ 55.141560] worker_thread+0x4e/0x3c0 [ 55.141563] ? __pfx_worker_thread+0x10/0x10 [ 55.141565] kthread+0xf2/0x120 [ 55.141568] ? __pfx_kthread+0x10/0x10 [ 55.141570] ret_from_fork+0x29/0x50 [ 55.141575] [ 55.141575] ---[ end trace 0000000000000000 ]--- Fixes: a4e771729a51 ("drm/probe_helper: sort out poll_running vs poll_enabled") Signed-off-by: Zongmin Zhou --- drivers/gpu/drm/drm_probe_helper.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c index 8127be134c39ee..ac72b18e225710 100644 --- a/drivers/gpu/drm/drm_probe_helper.c +++ b/drivers/gpu/drm/drm_probe_helper.c @@ -855,7 +855,8 @@ void drm_kms_helper_poll_disable(struct drm_device *dev) if (dev->mode_config.poll_running) drm_kms_helper_disable_hpd(dev); - cancel_delayed_work_sync(&dev->mode_config.output_poll_work); + if (dev->mode_config.poll_enabled) + cancel_delayed_work_sync(&dev->mode_config.output_poll_work); dev->mode_config.poll_running = false; } From e180f55bc7cd7f4bbb2abf81f1fbd41b9641681f Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sat, 15 Apr 2023 16:43:39 +0200 Subject: [PATCH 354/635] drm/apple: Remove simpledrm framebuffer before DRM device alloc Should result in drm apple to be registered as first DRM device replacing simpledrm. Should resolve problems with userspace assuming that card0 is the main displays device. Signed-off-by: Janne Grunau --- drivers/gpu/drm/apple/apple_drv.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 69900f3da61e81..861c5a9b1fb142 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -467,6 +467,14 @@ static int apple_drm_init(struct device *dev) if (ret) return ret; + fb_size = fb_r.end - fb_r.start + 1; + ret = drm_aperture_remove_conflicting_framebuffers(fb_r.start, fb_size, + false, &apple_drm_driver); + if (ret) { + dev_err(dev, "Failed remove fb: %d\n", ret); + goto err_unbind; + } + apple = devm_drm_dev_alloc(dev, &apple_drm_driver, struct apple_drm_private, drm); if (IS_ERR(apple)) @@ -478,14 +486,6 @@ static int apple_drm_init(struct device *dev) if (ret) return ret; - fb_size = fb_r.end - fb_r.start + 1; - ret = drm_aperture_remove_conflicting_framebuffers(fb_r.start, fb_size, - false, &apple_drm_driver); - if (ret) { - dev_err(dev, "Failed remove fb: %d\n", ret); - goto err_unbind; - } - ret = drmm_mode_config_init(&apple->drm); if (ret) goto err_unload; From 594271bc4b64c56a0fcaa98bdd3edcde1dfe3672 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 25 Apr 2023 01:49:14 +0900 Subject: [PATCH 355/635] drm/apple: Mark DCP as being in the wakeup path This prevents the PD from being shut down on suspend, which we need until we support runtime PM properly again. Signed-off-by: Hector Martin --- drivers/gpu/drm/apple/dcp.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9724bc0a48e039..623e2d21d98caa 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -585,6 +585,26 @@ static void dcp_platform_shutdown(struct platform_device *pdev) component_del(&pdev->dev, &dcp_comp_ops); } +static __maybe_unused int dcp_platform_suspend(struct device *dev) +{ + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); + + return 0; +} + +static __maybe_unused int dcp_platform_resume(struct device *dev) +{ + return 0; +} + +static SIMPLE_DEV_PM_OPS(dcp_platform_pm_ops, + dcp_platform_suspend, dcp_platform_resume); + static const struct of_device_id of_match[] = { { .compatible = "apple,dcp" }, {} @@ -598,6 +618,7 @@ static struct platform_driver apple_platform_driver = { .driver = { .name = "apple-dcp", .of_match_table = of_match, + .pm = pm_sleep_ptr(&dcp_platform_pm_ops), }, }; From 95f67ad42649acc8ab42f7ba2a05dc169c4a7e28 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Wed, 26 Apr 2023 00:19:15 +0900 Subject: [PATCH 356/635] fixup! cpuidle: apple: Add Apple SoC cpuidle driver --- drivers/cpuidle/cpuidle-apple.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index b430174c724bbf..cd1c7089c7d4c2 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -33,9 +33,13 @@ asm( "orr x0, x0, #(3L << 24)\n" "msr s3_5_c15_c5_0, x0\n" + "1:\n" "dsb sy\n" "wfi\n" + "mrs x0, ISR_EL1\n" + "cbz x0, 1b\n" + "mrs x0, s3_5_c15_c5_0\n" "bic x0, x0, #(1L << 24)\n" "msr s3_5_c15_c5_0, x0\n" @@ -68,16 +72,7 @@ static __cpuidle int apple_enter_idle(struct cpuidle_device *dev, struct cpuidle cpu_do_idle(); break; case STATE_PWRDOWN: - /* - * If we probed this driver in EL1, that's the m1n1 hypervisor. - * Go through the motions but do a normal WFI idle, since it will - * trap out the WFI idle stuff anyway. This avoids noise and slowness. - */ - if (is_kernel_in_hyp_mode()) - apple_cpu_deep_wfi(); - else - cpu_do_idle(); - + apple_cpu_deep_wfi(); break; default: WARN_ON(1); From cc7e40308b6402ce7de67b2e5957c37ef58c5729 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 9 Apr 2023 23:48:38 +0900 Subject: [PATCH 357/635] arm64: dts: apple: Add initial t602x device trees Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/Makefile | 5 + arch/arm64/boot/dts/apple/t600x-die0.dtsi | 2 +- .../arm64/boot/dts/apple/t600x-j314-j316.dtsi | 11 +- arch/arm64/boot/dts/apple/t600x-j375.dtsi | 6 +- arch/arm64/boot/dts/apple/t6020-j414s.dts | 37 + arch/arm64/boot/dts/apple/t6020-j416s.dts | 37 + arch/arm64/boot/dts/apple/t6020-j474s.dts | 92 + arch/arm64/boot/dts/apple/t6020.dtsi | 24 + arch/arm64/boot/dts/apple/t6021-j414c.dts | 37 + arch/arm64/boot/dts/apple/t6021-j416c.dts | 53 + arch/arm64/boot/dts/apple/t6021.dtsi | 91 + arch/arm64/boot/dts/apple/t602x-common.dtsi | 489 ++++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 699 +++++ arch/arm64/boot/dts/apple/t602x-dieX.dtsi | 344 +++ .../arm64/boot/dts/apple/t602x-gpio-pins.dtsi | 81 + .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 120 + arch/arm64/boot/dts/apple/t602x-nvme.dtsi | 42 + arch/arm64/boot/dts/apple/t602x-pmgr.dtsi | 2261 +++++++++++++++++ 18 files changed, 4428 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t6020-j414s.dts create mode 100644 arch/arm64/boot/dts/apple/t6020-j416s.dts create mode 100644 arch/arm64/boot/dts/apple/t6020-j474s.dts create mode 100644 arch/arm64/boot/dts/apple/t6020.dtsi create mode 100644 arch/arm64/boot/dts/apple/t6021-j414c.dts create mode 100644 arch/arm64/boot/dts/apple/t6021-j416c.dts create mode 100644 arch/arm64/boot/dts/apple/t6021.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-common.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-die0.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-dieX.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-nvme.dtsi create mode 100644 arch/arm64/boot/dts/apple/t602x-pmgr.dtsi diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index aec5e29cdfb737..e3e62c672d53b7 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -13,3 +13,8 @@ dtb-$(CONFIG_ARCH_APPLE) += t6002-j375d.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j413.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j473.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j493.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6020-j414s.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6021-j414c.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6020-j416s.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6021-j416c.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6020-j474s.dtb diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 469cafd4a7e2e1..9e11a5c92e5284 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -221,7 +221,7 @@ power-domains = <&ps_disp0_cpu0>; resets = <&ps_disp0_cpu0>; clocks = <&clk_disp0>; - apple,asc-dram-mask = <0x1f0 0x00000000>; + apple,asc-dram-mask = <0>; apple,piodma-mapper = <&disp0_piodma>; phandle = <&dcp>; }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index 91e40b9aa7f2ac..03e3f6744f0c9a 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -265,6 +265,8 @@ }; &spi1 { + status = "okay"; + flash@0 { compatible = "jedec,spi-nor"; reg = <0x0>; @@ -277,7 +279,7 @@ #address-cells = <1>; #size-cells = <1>; - partition@700000 { + nvram: partition@700000 { label = "nvram"; reg = <0x700000 0x20000>; }; @@ -285,6 +287,7 @@ }; }; +#ifndef NO_SPI_TRACKPAD &spi3 { status = "okay"; @@ -305,6 +308,7 @@ interrupts-extended = <&pinctrl_nub 6 IRQ_TYPE_LEVEL_LOW>; }; }; +#endif /* PCIe devices */ &port00 { @@ -331,6 +335,7 @@ /* SD card reader */ bus-range = <2 2>; pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; + status = "okay"; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -343,6 +348,10 @@ status = "okay"; }; +&pcie0_dart_1 { + status = "okay"; +}; + /* USB controllers */ &dwc3_0 { port { diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index a23462f8d3831d..4abd3edc013148 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -287,7 +287,7 @@ }; / { - sound { + sound: sound { compatible = "apple,j375-macaudio", "apple,macaudio"; model = "Mac Studio J375"; @@ -345,6 +345,7 @@ }; }; +#ifndef NO_PCIE_SDHC &port01 { /* SD card reader */ bus-range = <2 2>; @@ -356,6 +357,7 @@ wp-inverted; }; }; +#endif &port02 { /* 10 Gbit Ethernet */ @@ -384,6 +386,7 @@ status = "okay"; }; +#ifndef NO_GPU &gpu { apple,avg-power-ki-only = <0.6375>; apple,avg-power-kp = <0.58>; @@ -392,3 +395,4 @@ apple,ppm-ki = <5.8>; apple,ppm-kp = <0.355>; }; +#endif diff --git a/arch/arm64/boot/dts/apple/t6020-j414s.dts b/arch/arm64/boot/dts/apple/t6020-j414s.dts new file mode 100644 index 00000000000000..1ce122d7ac5416 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6020-j414s.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14-inch, M2 Pro, 2023) + * + * target-type: J414s + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6020.dtsi" +#include "t602x-j414-j416.dtsi" + +/ { + compatible = "apple,j414s", "apple,t6020", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M2 Pro, 2023)"; +}; + +&wifi0 { + brcm,board-type = "apple,tokara"; +}; + +&bluetooth0 { + brcm,board-type = "apple,tokara"; +}; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&sound { + model = "MacBook Pro J414"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j416s.dts b/arch/arm64/boot/dts/apple/t6020-j416s.dts new file mode 100644 index 00000000000000..8a3b8bd946561d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6020-j416s.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (16-inch, M2 Pro, 2023) + * + * target-type: J416s + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6020.dtsi" +#include "t602x-j414-j416.dtsi" + +/ { + compatible = "apple,j416s", "apple,t6020", "apple,arm-platform"; + model = "Apple MacBook Pro (16-inch, M2 Pro, 2023)"; +}; + +&wifi0 { + brcm,board-type = "apple,amami"; +}; + +&bluetooth0 { + brcm,board-type = "apple,amami"; +}; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&sound { + model = "MacBook Pro J416"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j474s.dts b/arch/arm64/boot/dts/apple/t6020-j474s.dts new file mode 100644 index 00000000000000..37d5a2a50eeab6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6020-j474s.dts @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Mac Mini (M2 Pro, 2023) + * + * target-type: J474s + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6020.dtsi" + +/* + * These model is very similar to the previous generation Mac Studio, other than + * the GPIO indices. + */ + +#define NO_PCIE_SDHC +#define NO_GPU +#include "t600x-j375.dtsi" + +/ { + compatible = "apple,j474s", "apple,t6020", "apple,arm-platform"; + model = "Apple Mac Mini (M2 Pro, 2023)"; + + aliases { + ethernet0 = ðernet0; + }; +}; + +&hpm0 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&hpm1 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&hpm2 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&hpm3 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&wifi0 { + compatible = "pci14e4,4434"; + brcm,board-type = "apple,tasmania"; +}; + +&bluetooth0 { + compatible = "pci14e4,5f72"; + brcm,board-type = "apple,tasmania"; +}; + +/* PCIe devices */ +&port00 { + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; +}; + +&port02 { + /* 10 Gbit Ethernet */ + bus-range = <3 3>; + status = "okay"; + ethernet0: ethernet@0,0 { + reg = <0x30000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&port03 { + /* USB xHCI */ + pwren-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; +}; + + +&speaker { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; + +&sound { + model = "Mac mini J474"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi new file mode 100644 index 00000000000000..3a864ebd91bb2f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6020 "M2 Pro" SoC + * + * Other names: H14J, "Rhodes Chop" + * + * Copyright The Asahi Linux Contributors + */ + +/* This chip is just a cut down version of t6021, so include it and disable the missing parts */ + +#define GPU_REPEAT(x) + +#include "t6021.dtsi" + +/ { + compatible = "apple,t6020", "apple,arm-platform"; +}; + +/delete-node/ &pmgr_south; + +&gpu { + compatible = "apple,agx-t6020", "apple,agx-g14x"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts new file mode 100644 index 00000000000000..d7480fd8459b55 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14-inch, M2 Max, 2023) + * + * target-type: J414c + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6021.dtsi" +#include "t602x-j414-j416.dtsi" + +/ { + compatible = "apple,j414c", "apple,t6021", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M2 Max, 2023)"; +}; + +&wifi0 { + brcm,board-type = "apple,tokara"; +}; + +&bluetooth0 { + brcm,board-type = "apple,tokara"; +}; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&sound { + model = "MacBook Pro J414"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j416c.dts b/arch/arm64/boot/dts/apple/t6021-j416c.dts new file mode 100644 index 00000000000000..cc6e4c27502619 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6021-j416c.dts @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (16-inch, M2 Max, 2022) + * + * target-type: J416c + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6021.dtsi" +#include "t602x-j414-j416.dtsi" + +/ { + compatible = "apple,j416c", "apple,t6021", "apple,arm-platform"; + model = "Apple MacBook Pro (16-inch, M2 Max, 2023)"; +}; + +/* This machine model (only) has two extra boost CPU P-states */ +&avalanche_opp { + opp18 { + opp-hz = /bits/ 64 <3528000000>; + opp-level = <18>; + clock-latency-ns = <67000>; + turbo-mode; + }; + opp19 { + opp-hz = /bits/ 64 <3696000000>; + opp-level = <19>; + clock-latency-ns = <67000>; + turbo-mode; + }; +}; + +&wifi0 { + brcm,board-type = "apple,amami"; +}; + +&bluetooth0 { + brcm,board-type = "apple,amami"; +}; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&sound { + model = "MacBook Pro J416"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi new file mode 100644 index 00000000000000..3c7d691fbad50e --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6021 "M2 Max" SoC + * + * Other names: H14J, "Jade" + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "multi-die-cpp.h" + +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + +#include "t602x-common.dtsi" + +/ { + compatible = "apple,t6001", "apple,arm-platform"; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges; + nonposted-mmio; + + // filled via templated includes at the end of the file + }; +}; + +#define DIE +#define DIE_NO 0 + +&{/soc} { + #include "t602x-die0.dtsi" + #include "t602x-dieX.dtsi" + #include "t602x-nvme.dtsi" +}; + +#include "t602x-gpio-pins.dtsi" +#include "t602x-pmgr.dtsi" + +#undef DIE +#undef DIE_NO + + +&aic { + affinities { + e-core-pmu-affinity { + apple,fiq-index = ; + cpus = <&cpu_e00 &cpu_e01 &cpu_e02 &cpu_e03>; + }; + + p-core-pmu-affinity { + apple,fiq-index = ; + cpus = <&cpu_p00 &cpu_p01 &cpu_p02 &cpu_p03 + &cpu_p10 &cpu_p11 &cpu_p12 &cpu_p13>; + }; + }; +}; + +&gpu { + compatible = "apple,agx-t6021", "apple,agx-g14x"; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi new file mode 100644 index 00000000000000..1224e4f12a5347 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Nodes common to all T602x family SoCs (M2 Pro/Max/Ultra) + * + * Copyright The Asahi Linux Contributors + */ + + / { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e00>; + }; + core1 { + cpu = <&cpu_e01>; + }; + core2 { + cpu = <&cpu_e02>; + }; + core3 { + cpu = <&cpu_e03>; + }; + }; + cluster1 { + core0 { + cpu = <&cpu_p00>; + }; + core1 { + cpu = <&cpu_p01>; + }; + core2 { + cpu = <&cpu_p02>; + }; + core3 { + cpu = <&cpu_p03>; + }; + }; + + cluster2 { + core0 { + cpu = <&cpu_p10>; + }; + core1 { + cpu = <&cpu_p11>; + }; + core2 { + cpu = <&cpu_p12>; + }; + core3 { + cpu = <&cpu_p13>; + }; + }; + }; + + cpu_e00: cpu@0 { + compatible = "apple,blizzard"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&blizzard_opp>; + capacity-dmips-mhz = <756>; + performance-domains = <&cpufreq_e>; + }; + + cpu_e01: cpu@1 { + compatible = "apple,blizzard"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&blizzard_opp>; + capacity-dmips-mhz = <756>; + performance-domains = <&cpufreq_e>; + }; + + cpu_e02: cpu@2 { + compatible = "apple,blizzard"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&blizzard_opp>; + capacity-dmips-mhz = <756>; + performance-domains = <&cpufreq_e>; + }; + + cpu_e03: cpu@3 { + compatible = "apple,blizzard"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + operating-points-v2 = <&blizzard_opp>; + capacity-dmips-mhz = <756>; + performance-domains = <&cpufreq_e>; + }; + + cpu_p00: cpu@10100 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p01: cpu@10101 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p02: cpu@10102 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p03: cpu@10103 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p10: cpu@10200 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10200>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p1>; + }; + + cpu_p11: cpu@10201 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10201>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p1>; + }; + + cpu_p12: cpu@10202 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10202>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p1>; + }; + + cpu_p13: cpu@10203 { + compatible = "apple,avalanche"; + device_type = "cpu"; + reg = <0x0 0x10203>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + operating-points-v2 = <&avalanche_opp>; + capacity-dmips-mhz = <1024>; + performance-domains = <&cpufreq_p1>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + + l2_cache_2: l2-cache-2 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + blizzard_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + /* pstate #1 is a dummy clone of #2 */ + opp02 { + opp-hz = /bits/ 64 <912000000>; + opp-level = <2>; + clock-latency-ns = <7700>; + }; + opp03 { + opp-hz = /bits/ 64 <1284000000>; + opp-level = <3>; + clock-latency-ns = <25000>; + }; + opp04 { + opp-hz = /bits/ 64 <1752000000>; + opp-level = <4>; + clock-latency-ns = <33000>; + }; + opp05 { + opp-hz = /bits/ 64 <2004000000>; + opp-level = <5>; + clock-latency-ns = <38000>; + }; + opp06 { + opp-hz = /bits/ 64 <2256000000>; + opp-level = <6>; + clock-latency-ns = <44000>; + }; + opp07 { + opp-hz = /bits/ 64 <2424000000>; + opp-level = <7>; + clock-latency-ns = <48000>; + }; + }; + + avalanche_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp01 { + opp-hz = /bits/ 64 <702000000>; + opp-level = <1>; + clock-latency-ns = <7400>; + }; + opp02 { + opp-hz = /bits/ 64 <948000000>; + opp-level = <2>; + clock-latency-ns = <18000>; + }; + opp03 { + opp-hz = /bits/ 64 <1188000000>; + opp-level = <3>; + clock-latency-ns = <21000>; + }; + opp04 { + opp-hz = /bits/ 64 <1452000000>; + opp-level = <4>; + clock-latency-ns = <24000>; + }; + opp05 { + opp-hz = /bits/ 64 <1704000000>; + opp-level = <5>; + clock-latency-ns = <28000>; + }; + opp06 { + opp-hz = /bits/ 64 <1968000000>; + opp-level = <6>; + clock-latency-ns = <31000>; + }; + opp07 { + opp-hz = /bits/ 64 <2208000000>; + opp-level = <7>; + clock-latency-ns = <33000>; + }; + opp08 { + opp-hz = /bits/ 64 <2400000000>; + opp-level = <8>; + clock-latency-ns = <45000>; + }; + opp09 { + opp-hz = /bits/ 64 <2568000000>; + opp-level = <9>; + clock-latency-ns = <47000>; + }; + opp10 { + opp-hz = /bits/ 64 <2724000000>; + opp-level = <10>; + clock-latency-ns = <50000>; + }; + opp11 { + opp-hz = /bits/ 64 <2868000000>; + opp-level = <11>; + clock-latency-ns = <52000>; + }; + opp12 { + opp-hz = /bits/ 64 <3000000000>; + opp-level = <12>; + clock-latency-ns = <57000>; + }; + opp13 { + opp-hz = /bits/ 64 <3132000000>; + opp-level = <13>; + clock-latency-ns = <60000>; + }; + opp14 { + opp-hz = /bits/ 64 <3264000000>; + opp-level = <14>; + clock-latency-ns = <64000>; + }; + opp15 { + opp-hz = /bits/ 64 <3360000000>; + opp-level = <15>; + clock-latency-ns = <64000>; + turbo-mode; + }; + opp16 { + opp-hz = /bits/ 64 <3408000000>; + opp-level = <16>; + clock-latency-ns = <64000>; + turbo-mode; + }; + opp17 { + opp-hz = /bits/ 64 <3504000000>; + opp-level = <17>; + clock-latency-ns = <64000>; + turbo-mode; + }; + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_REPEAT(637000); + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_REPEAT(656000); + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_REPEAT(687000); + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = GPU_REPEAT(725000); + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = GPU_REPEAT(790000); + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_REPEAT(843000); + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = GPU_REPEAT(887000); + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = GPU_REPEAT(918000); + opp-microwatt = <22800000>; + }; + }; + + pmu-e { + compatible = "apple,blizzard-pmu"; + interrupt-parent = <&aic>; + interrupts = ; + }; + + pmu-p { + compatible = "apple,avalanche-pmu"; + interrupt-parent = <&aic>; + interrupts = ; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <257142848>; /* TODO: check */ + clock-output-names = "clk_disp0"; + }; + + /* + * This is a fabulated representation of the input clock + * to NCO since we don't know the true clock tree. + */ + nco_clkref: clock-ref-nco { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-output-names = "nco_ref"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi new file mode 100644 index 00000000000000..722b51c2fb0f8a --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -0,0 +1,699 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * In anticipation of an M2 Ultra. Inspired by T600x. + * + * Obviously needs filling out, just the bare bones required + * to boot to a console in the HV. + * + * Copyright The Asahi Linux Contributors + */ + + nco: clock-controller@28e03c000 { + compatible = "apple,t6020-nco", "apple,nco"; + reg = <0x2 0x8e03c000 0x0 0x14000>; + clocks = <&nco_clkref>; + #clock-cells = <1>; + }; + + aic: interrupt-controller@28e100000 { + compatible = "apple,t6020-aic", "apple,aic2"; + #interrupt-cells = <4>; + interrupt-controller; + reg = <0x2 0x8e100000 0x0 0xc000>, + <0x2 0x8e10c000 0x0 0x1000>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + }; + + wdt: watchdog@29e2c4000 { + compatible = "apple,t6020-wdt", "apple,wdt"; + reg = <0x2 0x9e2c4000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + nub_spmi0: spmi@29e114000 { + compatible = "apple,t6020-spmi", "apple,spmi"; + reg = <0x2 0x9e114000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + interrupt-parent = <&aic>; + interrupts = , + ; + + pmu1: pmu@f { + compatible = "apple,maverick-pmu", "apple,spmi-pmu"; + reg = <0xb SPMI_USID>; + #address-cells = <1>; + #size-cells = <1>; + + rtc_nvmem@1400 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x1400 0x20>; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@5 { + reg = <0x5 0x1>; + }; + + rtc_offset: rtc-offset@11 { + reg = <0x11 0x6>; + }; + }; + + legacy_nvmem@6000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x6000 0x20>; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@1 { + reg = <0x1 0x1>; + }; + + boot_error_count: boot-error-count@2 { + reg = <0x2 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@2 { + reg = <0x2 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@3 { + reg = <0x3 0x1>; + }; + + shutdown_flag: shutdown-flag@f { + reg = <0xf 0x1>; + bits = <3 1>; + }; + }; + + scrpad_nvmem@8000 { + compatible = "apple,spmi-pmu-nvmem"; + reg = <0x8000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + + fault_shadow: fault-shadow@67b { + reg = <0x67b 0x10>; + }; + + socd: socd@b00 { + reg = <0xb00 0x400>; + }; + }; + + }; + }; + + smc_mbox: mbox@2a2408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa2408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@2a2400000 { + compatible = "apple,t6020-smc", "apple,smc"; + reg = <0x2 0xa2400000 0x0 0x4000>, + <0x2 0xa3e00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + gpio-controller; + #gpio-cells = <2>; + }; + + smc_rtc: rtc { + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + + smc_reboot: reboot { + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>, <&pm_setting>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count", "pm_setting"; + }; + }; + + pinctrl_smc: pinctrl@2a2820000 { + compatible = "apple,t6020-pinctrl", "apple,pinctrl"; + reg = <0x2 0xa2820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 30>; + apple,npins = <30>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + disp0_piodma: mapper@388000000 { + compatible = "apple,t6020-dcp-piodma", "apple,dcp-piodma"; + iommus = <&disp0_dart 4>; + }; + + disp0_dart: iommu@389304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_dart: iommu@38930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_mbox: mbox@389c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@389c00000 { + compatible = "apple,t6020-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x3 0x89c00000 0x0 0x4000>, // check? + <0x3 0x88000000 0x0 0x61c000>, + <0x3 0x89320000 0x0 0x4000>, + <0x3 0x89344000 0x0 0x4000>, + <0x3 0x89800000 0x0 0x800000>, + <0x2 0x8e3d0000 0x0 0x4000>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + apple,asc-dram-mask = <0 0>; + apple,piodma-mapper = <&disp0_piodma>; + phandle = <&dcp>; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + sio_dart: iommu@39b008000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x9b008000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&ps_sio_cpu>; + }; + + fpwm0: pwm@39b030000 { + compatible = "apple,t6020-fpwm", "apple,s5l-fpwm"; + reg = <0x3 0x9b030000 0x0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + status = "disabled"; + }; + + i2c0: i2c@39b040000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b040000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c0>; + #address-cells = <0x1>; + #size-cells = <0x0>; + }; + + i2c1: i2c@39b044000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b044000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c1>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c2: i2c@39b048000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b048000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c2>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c3: i2c@39b04c000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b04c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c3>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c4: i2c@39b050000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b050000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c4_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c4>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c5: i2c@39b054000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b054000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c5_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c5>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c6: i2c@39b054000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b054000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c6_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c6>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c7: i2c@39b054000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b054000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c7_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c7>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c8: i2c@39b054000 { + compatible = "apple,t6020-i2c", "apple,i2c"; + reg = <0x3 0x9b054000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c8_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c8>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + spi1: spi@39b104000 { + compatible = "apple,t6020-spi", "apple,spi"; + reg = <0x3 0x9b104000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clk_200m>; + pinctrl-0 = <&spi1_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi1>; + status = "disabled"; + }; + + spi2: spi@39b108000 { + compatible = "apple,t6020-spi", "apple,spi"; + reg = <0x3 0x9b108000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clkref>; + pinctrl-0 = <&spi2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi2>; + status = "disabled"; + }; + + spi4: spi@39b110000 { + compatible = "apple,t6020-spi", "apple,spi"; + reg = <0x3 0x9b110000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&clkref>; + pinctrl-0 = <&spi4_pins>; + pinctrl-names = "default"; + power-domains = <&ps_spi4>; + status = "disabled"; + }; + + serial0: serial@39b200000 { + compatible = "apple,s5l-uart"; + reg = <0x3 0x9b200000 0x0 0x4000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + /* + * TODO: figure out the clocking properly, there may + * be a third selectable clock. + */ + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; + + admac: dma-controller@39b400000 { + compatible = "apple,t6020-admac", "apple,admac"; + reg = <0x3 0x9b400000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <&aic AIC_IRQ 0 1218 IRQ_TYPE_LEVEL_HIGH>, + <0>, + <0>; + iommus = <&sio_dart 2>; + power-domains = <&ps_sio_adma>; + resets = <&ps_audio_p>; + }; + + mca: mca@39b600000 { + compatible = "apple,t6020-mca", "apple,mca"; + reg = <0x3 0x9b600000 0x0 0x10000>, + <0x3 0x9b500000 0x0 0x20000>; + clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; + dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>, + <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>, + <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>, + <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>; + dma-names = "tx0a", "rx0a", "tx0b", "rx0b", + "tx1a", "rx1a", "tx1b", "rx1b", + "tx2a", "rx2a", "tx2b", "rx2b", + "tx3a", "rx3a", "tx3b", "rx3b"; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, + <&ps_mca2>, <&ps_mca3>; + resets = <&ps_audio_p>; + #sound-dai-cells = <1>; + }; + + pmgr_gfx: power-management@404e80000 { + compatible = "apple,t6021-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x4 0x4e80000 0 0x4000>; + }; + + gpu: gpu@406400000 { + compatible = "apple,agx-g14x"; + reg = <0x4 0x6400000 0 0x40000>, + <0x4 0x4000000 0 0x1000000>; + reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + mboxes = <&agx_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + + apple,firmware-version = <0 0 0>; + apple,firmware-compat = <0 0 0>; + + operating-points-v2 = <&gpu_opp>; + /* TODO perf stuff */ + apple,perf-base-pstate = <1>; + }; + + agx_mbox: mbox@406408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + pcie0: pcie@580000000 { + compatible = "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x5 0x80000000 0x0 0x1000000>, /* config */ + <0x5 0x91000000 0x0 0x4000>, /* rc */ + <0x5 0x94008000 0x0 0x4000>, /* port0 */ + <0x5 0x95008000 0x0 0x4000>, /* port1 */ + <0x5 0x96008000 0x0 0x4000>, /* port2 */ + <0x5 0x97008000 0x0 0x4000>, /* port3 */ + <0x5 0x9e00c000 0x0 0x4000>, /* phy0 */ + <0x5 0x9e010000 0x0 0x4000>, /* phy1 */ + <0x5 0x9e014000 0x0 0x4000>, /* phy2 */ + <0x5 0x9e018000 0x0 0x4000>, /* phy3 */ + <0x5 0x9401c000 0x0 0x1000>, /* ltssm0 */ + <0x5 0x9501c000 0x0 0x1000>, /* ltssm1 */ + <0x5 0x9601c000 0x0 0x1000>, /* ltssm2 */ + <0x5 0x9701c000 0x0 0x1000>; /* ltssm3 */ + reg-names = "config", "rc", + "port0", "port1", "port2", "port3", + "phy0", "phy1", "phy2", "phy3", + "ltssm0", "ltssm1", "ltssm2", "ltssm3"; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + + msi-controller; + msi-parent = <&pcie0>; + msi-ranges = <&aic AIC_IRQ 0 1672 IRQ_TYPE_EDGE_RISING 32>; + + + iommu-map = <0x100 &pcie0_dart_0 1 1>, + <0x200 &pcie0_dart_1 1 1>, + <0x300 &pcie0_dart_2 1 1>, + <0x400 &pcie0_dart_3 1 1>; + iommu-map-mask = <0xff00>; + + bus-range = <0 4>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x5 0xa0000000 0x5 0xa0000000 0x0 0x20000000>, + <0x02000000 0x0 0xc0000000 0x5 0xc0000000 0x0 0x40000000>; + + power-domains = <&ps_apcie_gp_sys>; + pinctrl-0 = <&pcie_pins>; + pinctrl-names = "default"; + + dma-coherent; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + + port01: pci@1,0 { + device_type = "pci"; + reg = <0x800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 5 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; + status = "disabled"; + }; + + port02: pci@2,0 { + device_type = "pci"; + reg = <0x1000 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 6 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; + status = "disabled"; + }; + + port03: pci@3,0 { + device_type = "pci"; + reg = <0x1800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 7 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port03 0 0 0 0>, + <0 0 0 2 &port03 0 0 0 1>, + <0 0 0 3 &port03 0 0 0 2>, + <0 0 0 4 &port03 0 0 0 3>; + status = "disabled"; + }; + }; + + pcie0_dart_0: iommu@594000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x5 0x94000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp_sys>; + }; + + pcie0_dart_1: iommu@595000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x5 0x95000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp_sys>; + status = "disabled"; + }; + + pcie0_dart_2: iommu@596000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x5 0x96000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp_sys>; + status = "disabled"; + }; + + pcie0_dart_3: iommu@597000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x5 0x97000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp_sys>; + status = "disabled"; + }; + + diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi new file mode 100644 index 00000000000000..203316df0d06f0 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Nodes present on both dies of a hypothetical T6022 (M2 Ultra) + * and present on M2 Pro/Max. + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(cpufreq_e): cpufreq@210e20000 { + compatible = "apple,t6020-cluster-cpufreq", "apple,t8112-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + DIE_NODE(cpufreq_p0): cpufreq@211e20000 { + compatible = "apple,t6020-cluster-cpufreq", "apple,t8112-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + DIE_NODE(cpufreq_p1): cpufreq@212e20000 { + compatible = "apple,t6020-cluster-cpufreq", "apple,t8112-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x12e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + DIE_NODE(pmgr): power-management@28e080000 { + compatible = "apple,t6020-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e080000 0 0x8000>; + }; + + DIE_NODE(pmgr_south): power-management@28e680000 { + compatible = "apple,t6020-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e680000 0 0x8000>; + }; + + DIE_NODE(pmgr_east): power-management@290280000 { + compatible = "apple,t6020-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x90280000 0 0xc000>; + }; + + DIE_NODE(pinctrl_nub): pinctrl@29e1f0000 { + compatible = "apple,t6000-pinctrl", "apple,pinctrl"; + reg = <0x2 0x9e1f0000 0x0 0x4000>; + power-domains = <&DIE_NODE(ps_nub_gpio)>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&DIE_NODE(pinctrl_nub) 0 0 30>; + apple,npins = <30>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + DIE_NODE(pmgr_mini): power-management@29e280000 { + compatible = "apple,t6000-pmgr", "apple,pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x9e280000 0 0x4000>; + }; + + DIE_NODE(efuse): efuse@29e2cc000 { + compatible = "apple,t6020-efuses", "apple,efuses"; + reg = <0x2 0x9e2cc000 0x0 0x2000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + DIE_NODE(pinctrl_aop): pinctrl@2a6820000 { + compatible = "apple,t6020-pinctrl", "apple,pinctrl"; + reg = <0x2 0xa6820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&DIE_NODE(pinctrl_aop) 0 0 72>; + apple,npins = <72>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { + compatible = "apple,t6020-pinctrl", "apple,pinctrl"; + reg = <0x3 0x9b028000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + clocks = <&clkref>; + power-domains = <&DIE_NODE(ps_gpio)>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&DIE_NODE(pinctrl_ap) 0 0 255>; + apple,npins = <255>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + DIE_NODE(dwc3_0_dart_0): iommu@702f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x7 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0_dart_1): iommu@702f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x7 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_0): usb@702280000 { + compatible = "apple,t6020-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x7 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_0_dart_0) 0>, + <&DIE_NODE(dwc3_0_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + dma-coherent; + resets = <&DIE_NODE(atcphy0)>; + phys = <&DIE_NODE(atcphy0) PHY_TYPE_USB2>, <&DIE_NODE(atcphy0) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy0): phy@703000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0x7 0x03000000 0x0 0x4c000>, + <0x7 0x03050000 0x0 0x8000>, + <0x7 0x00000000 0x0 0x4000>, + <0x7 0x02a90000 0x0 0x4000>, + <0x7 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + }; + + DIE_NODE(dwc3_1_dart_0): iommu@b02f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xb 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1_dart_1): iommu@b02f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xb 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_1): usb@b02280000 { + compatible = "apple,t6020-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0xb 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_1_dart_0) 0>, + <&DIE_NODE(dwc3_1_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + dma-coherent; + resets = <&DIE_NODE(atcphy1)>; + phys = <&DIE_NODE(atcphy1) PHY_TYPE_USB2>, <&DIE_NODE(atcphy1) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy1): phy@b03000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0xb 0x03000000 0x0 0x4c000>, + <0xb 0x03050000 0x0 0x8000>, + <0xb 0x00000000 0x0 0x4000>, + <0xb 0x02a90000 0x0 0x4000>, + <0xb 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + }; + + DIE_NODE(dwc3_2_dart_0): iommu@f02f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xf 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2_dart_1): iommu@f02f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0xf 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_2): usb@f02280000 { + compatible = "apple,t6020-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0xf 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_2_dart_0) 0>, + <&DIE_NODE(dwc3_2_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + dma-coherent; + resets = <&DIE_NODE(atcphy2)>; + phys = <&DIE_NODE(atcphy2) PHY_TYPE_USB2>, <&DIE_NODE(atcphy2) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy2): phy@f03000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0xf 0x03000000 0x0 0x4c000>, + <0xf 0x03050000 0x0 0x8000>, + <0xf 0x00000000 0x0 0x4000>, + <0xf 0x02a90000 0x0 0x4000>, + <0xf 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + }; + + DIE_NODE(dwc3_3_dart_0): iommu@1302f00000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x13 0x02f00000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3_dart_1): iommu@1302f80000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x13 0x02f80000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + #iommu-cells = <1>; + }; + + DIE_NODE(dwc3_3): usb@1302280000 { + compatible = "apple,t6020-dwc3", "apple,dwc3", "snps,dwc3"; + reg = <0x13 0x02280000 0x0 0x100000>; + interrupt-parent = <&aic>; + interrupts = ; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&DIE_NODE(dwc3_3_dart_0) 0>, + <&DIE_NODE(dwc3_3_dart_1) 1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + dma-coherent; + resets = <&DIE_NODE(atcphy3)>; + phys = <&DIE_NODE(atcphy3) PHY_TYPE_USB2>, <&DIE_NODE(atcphy3) PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + }; + + DIE_NODE(atcphy3): phy@1303000000 { + compatible = "apple,t6020-atcphy", "apple,t8103-atcphy"; + reg = <0x13 0x03000000 0x0 0x4c000>, + <0x13 0x03050000 0x0 0x8000>, + <0x13 0x00000000 0x0 0x4000>, + <0x13 0x02a90000 0x0 0x4000>, + <0x13 0x02a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", + "pipehandler"; + + #phy-cells = <1>; + #reset-cells = <0>; + + orientation-switch; + mode-switch; + svid = <0xff01>, <0x8087>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi new file mode 100644 index 00000000000000..acb133d1723d03 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * GPIO pin mappings for Apple T600x SoCs. + * + * Copyright The Asahi Linux Contributors + */ + +&pinctrl_ap { + i2c0_pins: i2c0-pins { + pinmux = , + ; + }; + + i2c1_pins: i2c1-pins { + pinmux = , + ; + }; + + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + + i2c3_pins: i2c3-pins { + pinmux = , + ; + }; + + i2c4_pins: i2c4-pins { + pinmux = , + ; + }; + + i2c5_pins: i2c5-pins { + pinmux = , + ; + }; + + i2c6_pins: i2c6-pins { + pinmux = , + ; + }; + + i2c7_pins: i2c7-pins { + pinmux = , + ; + }; + + i2c8_pins: i2c8-pins { + pinmux = , + ; + }; + + spi1_pins: spi1-pins { + pinmux = , /* SDI */ + , /* SDO */ + , /* SCK */ + ; /* CS */ + }; + + spi2_pins: spi2-pins { + pinmux = , /* SDI */ + , /* SDO */ + , /* SCK */ + ; /* CS */ + }; + + spi4_pins: spi4-pins { + pinmux = , /* SDI */ + , /* SDO */ + , /* SCK */ + ; /* CS */ + }; + + pcie_pins: pcie-pins { + pinmux = , + , + , + ; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi new file mode 100644 index 00000000000000..eecef8ae7c1f6d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14/16-inch, 2022) + * + * This file contains the parts common to J414 and J416 devices with both t6020 and t6021. + * + * target-type: J414s / J414c / J416s / J416c + * + * Copyright The Asahi Linux Contributors + */ + +/* + * These models are essentially identical to the previous generation, other than + * the GPIO indices. + */ + +#define NO_SPI_TRACKPAD +#include "t600x-j314-j316.dtsi" + +&nvram { + /* nvram is larger than on previous machines */ + reg = <0x700000 0x100000>; +}; + +&hpm0 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&hpm1 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&hpm2 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&hpm5 { + interrupts = <44 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_tweet { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof1 { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof2 { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_tweet { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof1 { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof2 { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; + +&wifi0 { + compatible = "pci14e4,4434"; +}; + +&bluetooth0 { + compatible = "pci14e4,5f72"; +}; + +&port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; +}; + +/* enable ISP power-domains */ + +&ps_isp_sys { + status = "okay"; +}; + +&ps_isp_cpu { + status = "okay"; +}; + +&ps_isp_fe { + status = "okay"; +}; + +&ps_dprx { + status = "okay"; +}; + +&ps_isp_vis { + status = "okay"; +}; + +&ps_isp_be { + status = "okay"; +}; + +&ps_isp_raw { + status = "okay"; +}; + +&ps_isp_clr { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi new file mode 100644 index 00000000000000..756a971bde48ae --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * NVMe related devices for Apple T602x SoCs. + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(ans_mbox): mbox@347408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x47408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&DIE_NODE(ps_ans2)>; + #mbox-cells = <0>; + }; + + DIE_NODE(sart): sart@34bc50000 { + compatible = "apple,t6020-sart", "apple,t6000-sart"; + reg = <0x3 0x4bc50000 0x0 0x10000>; + power-domains = <&DIE_NODE(ps_ans2)>; + }; + + DIE_NODE(nvme): nvme@34bcc0000 { + compatible = "apple,t6020-nvme-ans2", "apple,nvme-ans2"; + reg = <0x3 0x4bcc0000 0x0 0x40000>, <0x3 0x47400000 0x0 0x4000>; + reg-names = "nvme", "ans"; + interrupt-parent = <&aic>; + /* The NVME interrupt is always routed to die 0 */ + interrupts = ; + mboxes = <&DIE_NODE(ans_mbox)>; + apple,sart = <&DIE_NODE(sart)>; + power-domains = <&DIE_NODE(ps_ans2)>, + <&DIE_NODE(ps_apcie_st_sys)>, + <&DIE_NODE(ps_apcie_st1_sys)>; + power-domain-names = "ans", "apcie0", "apcie1"; + resets = <&DIE_NODE(ps_ans2)>; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi new file mode 100644 index 00000000000000..84e5c19f90d7b3 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi @@ -0,0 +1,2261 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PMGR Power domains for the Apple T6001 "M1 Max" SoC + * + * Copyright The Asahi Linux Contributors + */ + +&pmgr { + DIE_NODE(ps_afi): power-controller@100 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afi); + apple,always-on; /* Apple Fabric, CPU interface is here */ + }; + + DIE_NODE(ps_aic): power-controller@108 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(aic); + apple,always-on; /* Core device */ + }; + + DIE_NODE(ps_dwi): power-controller@110 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dwi); + }; + + DIE_NODE(ps_pms): power-controller@118 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms); + apple,always-on; /* Core device */ + }; + + DIE_NODE(ps_gpio): power-controller@120 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(gpio); + power-domains = <&DIE_NODE(ps_sio)>, <&DIE_NODE(ps_pms)>; + }; + + DIE_NODE(ps_soc_dpe): power-controller@128 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(soc_dpe); + apple,always-on; /* Core device */ + }; + + DIE_NODE(ps_pms_c1ppt): power-controller@130 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_c1ppt); + apple,always-on; /* Core device */ + }; + + DIE_NODE(ps_pmgr_soc_ocla): power-controller@138 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pmgr_soc_ocla); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_amcc0): power-controller@168 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc0); + apple,always-on; /* Memory controller */ + }; + + DIE_NODE(ps_amcc2): power-controller@170 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc2); + apple,always-on; /* Memory controller */ + }; + + DIE_NODE(ps_dcs_00): power-controller@178 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_00); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_01): power-controller@180 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_01); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_02): power-controller@188 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_02); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_03): power-controller@190 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_03); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_08): power-controller@198 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_08); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_09): power-controller@1a0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_09); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_10): power-controller@1a8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_10); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_11): power-controller@1b0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_11); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_afnc1_ioa): power-controller@1b8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_ioa); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afc): power-controller@1d0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afc); + apple,always-on; /* Apple Fabric, CPU interface is here */ + }; + + DIE_NODE(ps_afnc0_ioa): power-controller@1e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc0_ioa); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc1_ls): power-controller@1f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_ls); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc1_ioa)>; + }; + + DIE_NODE(ps_afnc0_ls): power-controller@1f8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc0_ls); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc0_ioa)>; + }; + + DIE_NODE(ps_afnc1_lw0): power-controller@200 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_lw0); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc1_ls)>; + }; + + DIE_NODE(ps_afnc1_lw1): power-controller@208 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_lw1); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc1_ls)>; + }; + + DIE_NODE(ps_afnc1_lw2): power-controller@210 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_lw2); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc1_ls)>; + }; + + DIE_NODE(ps_afnc0_lw0): power-controller@218 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc0_lw0); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc0_ls)>; + }; + + DIE_NODE(ps_scodec): power-controller@220 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(scodec); + power-domains = <&DIE_NODE(ps_afnc1_lw0)>; + }; + + DIE_NODE(ps_atc0_common): power-controller@228 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_common); + power-domains = <&DIE_NODE(ps_afnc1_lw1)>; + }; + + DIE_NODE(ps_atc1_common): power-controller@230 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_common); + power-domains = <&DIE_NODE(ps_afnc1_lw1)>; + }; + + DIE_NODE(ps_atc2_common): power-controller@238 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_common); + power-domains = <&DIE_NODE(ps_afnc1_lw1)>; + }; + + DIE_NODE(ps_atc3_common): power-controller@240 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_common); + power-domains = <&DIE_NODE(ps_afnc1_lw1)>; + }; + + DIE_NODE(ps_dispext1_sys): power-controller@248 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext1_sys); + power-domains = <&DIE_NODE(ps_afnc1_lw2)>; + }; + + DIE_NODE(ps_pms_bridge): power-controller@250 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_bridge); + apple,always-on; /* Core device */ + power-domains = <&DIE_NODE(ps_afnc0_lw0)>; + }; + + DIE_NODE(ps_dispext0_sys): power-controller@258 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext0_sys); + power-domains = <&DIE_NODE(ps_afnc0_lw0)>, <&DIE_NODE(ps_afr)>; + }; + + DIE_NODE(ps_ane_sys): power-controller@260 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_sys); + power-domains = <&DIE_NODE(ps_afnc0_lw0)>; + }; + + DIE_NODE(ps_avd_sys): power-controller@268 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(avd_sys); + power-domains = <&DIE_NODE(ps_afnc0_lw0)>; + }; + + DIE_NODE(ps_atc0_cio): power-controller@270 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_cio); + power-domains = <&DIE_NODE(ps_atc0_common)>; + }; + + DIE_NODE(ps_atc0_pcie): power-controller@278 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_pcie); + power-domains = <&DIE_NODE(ps_atc0_common)>; + }; + + DIE_NODE(ps_atc1_cio): power-controller@280 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_cio); + power-domains = <&DIE_NODE(ps_atc1_common)>; + }; + + DIE_NODE(ps_atc1_pcie): power-controller@288 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_pcie); + power-domains = <&DIE_NODE(ps_atc1_common)>; + }; + + DIE_NODE(ps_atc2_cio): power-controller@290 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_cio); + power-domains = <&DIE_NODE(ps_atc2_common)>; + }; + + DIE_NODE(ps_atc2_pcie): power-controller@298 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_pcie); + power-domains = <&DIE_NODE(ps_atc2_common)>; + }; + + DIE_NODE(ps_atc3_cio): power-controller@2a0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_cio); + power-domains = <&DIE_NODE(ps_atc3_common)>; + }; + + DIE_NODE(ps_atc3_pcie): power-controller@2a8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_pcie); + power-domains = <&DIE_NODE(ps_atc3_common)>; + }; + + DIE_NODE(ps_dispext1_fe): power-controller@2b0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext1_fe); + power-domains = <&DIE_NODE(ps_dispext1_sys)>; + }; + + DIE_NODE(ps_dispext1_cpu0): power-controller@2b8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext1_cpu0); + power-domains = <&DIE_NODE(ps_dispext1_sys)>; + }; + + DIE_NODE(ps_dispext0_fe): power-controller@2c0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext0_fe); + power-domains = <&DIE_NODE(ps_dispext0_sys)>; + }; + +#if DIE_NO == 0 + /* PMP is only present on die 0 of the M1 Ultra */ + DIE_NODE(ps_pmp): power-controller@2c8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pmp); + }; +#endif + + DIE_NODE(ps_pms_sram): power-controller@2d0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_sram); + }; + + DIE_NODE(ps_dispext0_cpu0): power-controller@2d8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext0_cpu0); + power-domains = <&DIE_NODE(ps_dispext0_sys)>; + }; + + DIE_NODE(ps_ane_cpu): power-controller@2e0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_cpu); + power-domains = <&DIE_NODE(ps_ane_sys)>; + }; + + DIE_NODE(ps_atc0_cio_pcie): power-controller@2e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_cio_pcie); + power-domains = <&DIE_NODE(ps_atc0_cio)>; + }; + + DIE_NODE(ps_atc0_cio_usb): power-controller@2f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_cio_usb); + power-domains = <&DIE_NODE(ps_atc0_cio)>; + }; + + DIE_NODE(ps_atc1_cio_pcie): power-controller@2f8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_cio_pcie); + power-domains = <&DIE_NODE(ps_atc1_cio)>; + }; + + DIE_NODE(ps_atc1_cio_usb): power-controller@300 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_cio_usb); + power-domains = <&DIE_NODE(ps_atc1_cio)>; + }; + + DIE_NODE(ps_atc2_cio_pcie): power-controller@308 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_cio_pcie); + power-domains = <&DIE_NODE(ps_atc2_cio)>; + }; + + DIE_NODE(ps_atc2_cio_usb): power-controller@310 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_cio_usb); + power-domains = <&DIE_NODE(ps_atc2_cio)>; + }; + + DIE_NODE(ps_atc3_cio_pcie): power-controller@318 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_cio_pcie); + power-domains = <&DIE_NODE(ps_atc3_cio)>; + }; + + DIE_NODE(ps_atc3_cio_usb): power-controller@320 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_cio_usb); + power-domains = <&DIE_NODE(ps_atc3_cio)>; + }; + + DIE_NODE(ps_trace_fab): power-controller@390 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(trace_fab); + }; + + DIE_NODE(ps_ane_sys_mpm): power-controller@4000 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_sys_mpm); + power-domains = <&DIE_NODE(ps_ane_sys)>; + }; + + DIE_NODE(ps_ane_td): power-controller@4008 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_td); + power-domains = <&DIE_NODE(ps_ane_sys)>; + }; + + DIE_NODE(ps_ane_base): power-controller@4010 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_base); + power-domains = <&DIE_NODE(ps_ane_td)>; + }; + + DIE_NODE(ps_ane_set1): power-controller@4018 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set1); + power-domains = <&DIE_NODE(ps_ane_base)>; + }; + + DIE_NODE(ps_ane_set2): power-controller@4020 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set2); + power-domains = <&DIE_NODE(ps_ane_set1)>; + }; + + DIE_NODE(ps_ane_set3): power-controller@4028 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set3); + power-domains = <&DIE_NODE(ps_ane_set2)>; + }; + + DIE_NODE(ps_ane_set4): power-controller@4030 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_set4); + power-domains = <&DIE_NODE(ps_ane_set3)>; + }; +}; + +&pmgr_south { + DIE_NODE(ps_amcc4): power-controller@100 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc4); + apple,always-on; + }; + + DIE_NODE(ps_amcc5): power-controller@108 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc5); + apple,always-on; + }; + + DIE_NODE(ps_amcc6): power-controller@110 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc6); + apple,always-on; + }; + + DIE_NODE(ps_amcc7): power-controller@118 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc7); + apple,always-on; + }; + + DIE_NODE(ps_dcs_16): power-controller@120 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_16); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_17): power-controller@128 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_17); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_18): power-controller@130 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_18); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_19): power-controller@138 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_19); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_20): power-controller@140 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_20); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_21): power-controller@148 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_21); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_22): power-controller@150 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_22); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_23): power-controller@158 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_23); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_24): power-controller@160 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_24); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_25): power-controller@168 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_25); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_26): power-controller@170 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_26); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_27): power-controller@178 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_27); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_28): power-controller@180 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_28); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_29): power-controller@188 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_29); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_30): power-controller@190 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_30); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_31): power-controller@198 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_31); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_afnc4_ioa): power-controller@1a0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc4_ioa); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc4_ls): power-controller@1a8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc4_ls); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc4_ioa)>; + }; + + DIE_NODE(ps_afnc4_lw0): power-controller@1b0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc4_lw0); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc4_ls)>; + }; + + DIE_NODE(ps_afnc5_ioa): power-controller@1b8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc5_ioa); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc5_ls): power-controller@1c0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc5_ls); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc5_ioa)>; + }; + + DIE_NODE(ps_afnc5_lw0): power-controller@1c8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc5_lw0); + apple,always-on; /* Apple Fabric */ + power-domains = <&DIE_NODE(ps_afnc5_ls)>; + }; + + DIE_NODE(ps_dispext2_sys): power-controller@1d0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext2_sys); + }; + + DIE_NODE(ps_msr1): power-controller@1d8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr1); + }; + + DIE_NODE(ps_dispext2_fe): power-controller@1e0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext2_fe); + power-domains = <&DIE_NODE(ps_dispext2_sys)>; + }; + + DIE_NODE(ps_dispext2_cpu0): power-controller@1e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext2_cpu0); + power-domains = <&DIE_NODE(ps_dispext2_fe)>; + }; + + DIE_NODE(ps_msr1_ase_core): power-controller@1f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr1_ase_core); + power-domains = <&DIE_NODE(ps_msr1)>; + }; + + DIE_NODE(ps_dispext3_sys): power-controller@220 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext3_sys); + }; + + DIE_NODE(ps_venc1_sys): power-controller@228 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_sys); + }; + + DIE_NODE(ps_dispext3_fe): power-controller@230 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext3_fe); + power-domains = <&DIE_NODE(ps_dispext3_sys)>; + }; + + DIE_NODE(ps_dispext3_cpu0): power-controller@238 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext3_cpu0); + power-domains = <&DIE_NODE(ps_dispext3_fe)>; + }; + + DIE_NODE(ps_venc1_dma): power-controller@4000 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_dma); + power-domains = <&DIE_NODE(ps_venc1_sys)>; + }; + + DIE_NODE(ps_venc1_pipe4): power-controller@4008 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_pipe4); + power-domains = <&DIE_NODE(ps_venc1_dma)>; + }; + + DIE_NODE(ps_venc1_pipe5): power-controller@4010 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_pipe5); + power-domains = <&DIE_NODE(ps_venc1_dma)>; + }; + + DIE_NODE(ps_venc1_me0): power-controller@4018 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_me0); + power-domains = <&DIE_NODE(ps_venc1_pipe5)>, <&DIE_NODE(ps_venc1_pipe4)>; + }; + + DIE_NODE(ps_venc1_me1): power-controller@4020 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_me1); + power-domains = <&DIE_NODE(ps_venc1_me0)>; + }; +}; + +&pmgr_east { + DIE_NODE(ps_clvr_spmi0): power-controller@100 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(clvr_spmi0); + apple,always-on; /* PCPU voltage regulator interface (used by SMC) */ + }; + + DIE_NODE(ps_clvr_spmi1): power-controller@108 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(clvr_spmi1); + apple,always-on; /* GPU voltage regulator interface (used by SMC) */ + }; + + DIE_NODE(ps_clvr_spmi2): power-controller@110 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(clvr_spmi2); + apple,always-on; /* ANE, fabric, AFR voltage regulator interface (used by SMC) */ + }; + + DIE_NODE(ps_clvr_spmi3): power-controller@118 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(clvr_spmi3); + apple,always-on; /* Additional voltage regulator, probably used on T6021 (SMC) */ + }; + + DIE_NODE(ps_clvr_spmi4): power-controller@120 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(clvr_spmi4); + apple,always-on; /* Additional voltage regulator, probably used on T6021 (SMC) */ + }; + + DIE_NODE(ps_ispsens0): power-controller@128 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ispsens0); + }; + + DIE_NODE(ps_ispsens1): power-controller@130 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ispsens1); + }; + + DIE_NODE(ps_ispsens2): power-controller@138 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ispsens2); + }; + + DIE_NODE(ps_ispsens3): power-controller@140 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ispsens3); + }; + + DIE_NODE(ps_afnc6_ioa): power-controller@148 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc6_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc6_ls): power-controller@150 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc6_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc6_ioa)>; + }; + + DIE_NODE(ps_afnc6_lw0): power-controller@158 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc6_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc6_ls)>; + }; + + DIE_NODE(ps_afnc2_ioa): power-controller@160 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_dcs_10)>; + }; + + DIE_NODE(ps_afnc2_ls): power-controller@168 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc2_ioa)>; + }; + + DIE_NODE(ps_afnc2_lw0): power-controller@170 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc2_ls)>; + }; + + DIE_NODE(ps_afnc2_lw1): power-controller@178 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_lw1); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc2_ls)>; + }; + + DIE_NODE(ps_afnc3_ioa): power-controller@180 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc3_ls): power-controller@188 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc3_ioa)>; + }; + + DIE_NODE(ps_afnc3_lw0): power-controller@190 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc3_ls)>; + }; + + DIE_NODE(ps_apcie_gp): power-controller@198 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_gp); + power-domains = <&DIE_NODE(ps_afnc6_lw0)>; + }; + + DIE_NODE(ps_apcie_st): power-controller@1a0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_st); + power-domains = <&DIE_NODE(ps_afnc6_lw0)>; + }; + + DIE_NODE(ps_ans2): power-controller@1a8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ans2); + power-domains = <&DIE_NODE(ps_afnc6_lw0)>; + }; + + DIE_NODE(ps_disp0_sys): power-controller@1b0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(disp0_sys); + power-domains = <&DIE_NODE(ps_afnc2_lw0)>; + }; + + DIE_NODE(ps_jpg): power-controller@1b8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(jpg); + power-domains = <&DIE_NODE(ps_afnc2_lw0)>; + }; + + DIE_NODE(ps_sio): power-controller@1c0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio); + power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + }; + + DIE_NODE(ps_isp_sys): power-controller@1c8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_sys); + power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + status = "disabled"; /* appears to be fused-off on j474s */ + }; + + DIE_NODE(ps_disp0_fe): power-controller@1d0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(disp0_fe); + power-domains = <&DIE_NODE(ps_disp0_sys)>; + }; + + DIE_NODE(ps_disp0_cpu0): power-controller@1d8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(disp0_cpu0); + power-domains = <&DIE_NODE(ps_disp0_fe)>; + apple,min-state = <4>; + }; + + DIE_NODE(ps_sio_cpu): power-controller@1e0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_cpu); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_fpwm0): power-controller@1e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(fpwm0); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_fpwm1): power-controller@1f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(fpwm1); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_fpwm2): power-controller@1f8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x1f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(fpwm2); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c0): power-controller@200 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c0); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c1): power-controller@208 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c1); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c2): power-controller@210 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c2); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c3): power-controller@218 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c3); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c4): power-controller@220 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c4); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c5): power-controller@228 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c5); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c6): power-controller@230 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c6); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c7): power-controller@238 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c7); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c8): power-controller@240 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c8); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_spi_p): power-controller@248 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi_p); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_sio_spmi0): power-controller@250 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_spmi0); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_sio_spmi1): power-controller@258 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_spmi1); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_sio_spmi2): power-controller@260 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_spmi2); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_uart_p): power-controller@268 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart_p); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_audio_p): power-controller@270 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(audio_p); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_sio_adma): power-controller@278 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_adma); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_aes): power-controller@280 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(aes); + apple,always-on; + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_dptx_phy_ps): power-controller@288 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dptx_phy_ps); + apple,always-on; + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_spi0): power-controller@2d8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi0); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi1): power-controller@2e0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi1); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi2): power-controller@2e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi2); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi3): power-controller@2f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi3); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi4): power-controller@2f8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi4); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi5): power-controller@300 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi5); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_uart_n): power-controller@308 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart_n); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart0): power-controller@310 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart0); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_amcc1): power-controller@318 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc1); + apple,always-on; + }; + + DIE_NODE(ps_amcc3): power-controller@320 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc3); + apple,always-on; + }; + + DIE_NODE(ps_dcs_04): power-controller@328 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_04); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_05): power-controller@330 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_05); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_06): power-controller@338 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_06); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_07): power-controller@340 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_07); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_12): power-controller@348 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_12); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_13): power-controller@350 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_13); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_14): power-controller@358 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_14); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_dcs_15): power-controller@360 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_15); + apple,always-on; /* LPDDR5 interface */ + }; + + DIE_NODE(ps_uart1): power-controller@368 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart1); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart2): power-controller@370 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart2); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart3): power-controller@378 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart3); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart4): power-controller@380 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart4); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart5): power-controller@388 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart5); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart6): power-controller@390 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart6); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_mca0): power-controller@398 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca0); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_mca1): power-controller@3a0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca1); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_mca2): power-controller@3a8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca2); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_mca3): power-controller@3b0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca3); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_dpa0): power-controller@3b8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa0); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa1): power-controller@3c0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa1); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa2): power-controller@3c8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa2); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa3): power-controller@3d0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa3); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_msr0): power-controller@3d8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr0); + }; + + DIE_NODE(ps_venc_sys): power-controller@3e0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc_sys); + }; + + DIE_NODE(ps_dpa4): power-controller@3e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa4); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_msr0_ase_core): power-controller@3f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr0_ase_core); + power-domains = <&DIE_NODE(ps_msr0)>; + }; + + DIE_NODE(ps_apcie_gpshr_sys): power-controller@3f8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x3f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_gpshr_sys); + power-domains = <&DIE_NODE(ps_apcie_gp)>; + }; + + DIE_NODE(ps_apcie_st_sys): power-controller@408 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_st_sys); + power-domains = <&DIE_NODE(ps_apcie_st)>, <&DIE_NODE(ps_ans2)>; + }; + + DIE_NODE(ps_apcie_st1_sys): power-controller@410 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_st1_sys); + power-domains = <&DIE_NODE(ps_apcie_st_sys)>; + }; + + DIE_NODE(ps_apcie_gp_sys): power-controller@418 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x418 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_gp_sys); + power-domains = <&DIE_NODE(ps_apcie_gpshr_sys)>; + apple,always-on; /* Breaks things if shut down */ + }; + + DIE_NODE(ps_apcie_ge_sys): power-controller@420 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x420 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_ge_sys); + power-domains = <&DIE_NODE(ps_apcie_gpshr_sys)>; + }; + + DIE_NODE(ps_apcie_phy_sw): power-controller@428 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x428 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_phy_sw); + apple,always-on; /* macOS does not turn this off */ + }; + + DIE_NODE(ps_sep): power-controller@c00 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sep); + apple,always-on; /* Locked on */ + }; + + DIE_NODE(ps_isp_cpu): power-controller@4000 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_cpu); + power-domains = <&DIE_NODE(ps_isp_sys)>; + status = "disabled"; + }; + + DIE_NODE(ps_isp_fe): power-controller@4008 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_fe); + power-domains = <&DIE_NODE(ps_isp_sys)>; + status = "disabled"; + }; + + DIE_NODE(ps_dprx): power-controller@4010 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dprx); + power-domains = <&DIE_NODE(ps_isp_sys)>; + status = "disabled"; + }; + + DIE_NODE(ps_isp_vis): power-controller@4018 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_vis); + power-domains = <&DIE_NODE(ps_isp_fe)>; + status = "disabled"; + }; + + DIE_NODE(ps_isp_be): power-controller@4020 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_be); + power-domains = <&DIE_NODE(ps_isp_fe)>; + status = "disabled"; + }; + + DIE_NODE(ps_isp_raw): power-controller@4028 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_raw); + power-domains = <&DIE_NODE(ps_isp_fe)>; + status = "disabled"; + }; + + DIE_NODE(ps_isp_clr): power-controller@4030 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_clr); + power-domains = <&DIE_NODE(ps_isp_be)>; + status = "disabled"; + }; + + DIE_NODE(ps_venc_dma): power-controller@8000 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc_dma); + power-domains = <&DIE_NODE(ps_venc_sys)>; + }; + + DIE_NODE(ps_venc_pipe4): power-controller@8008 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc_pipe4); + power-domains = <&DIE_NODE(ps_venc_dma)>; + }; + + DIE_NODE(ps_venc_pipe5): power-controller@8010 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc_pipe5); + power-domains = <&DIE_NODE(ps_venc_dma)>; + }; + + DIE_NODE(ps_venc_me0): power-controller@8018 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc_me0); + power-domains = <&DIE_NODE(ps_venc_pipe5)>, <&DIE_NODE(ps_venc_pipe4)>; + }; + + DIE_NODE(ps_venc_me1): power-controller@8020 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc_me1); + power-domains = <&DIE_NODE(ps_venc_me0)>; + }; + + DIE_NODE(ps_prores): power-controller@c000 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(prores); + power-domains = <&DIE_NODE(ps_afnc3_lw0)>; + }; +}; + +&pmgr_mini { + DIE_NODE(ps_debug): power-controller@58 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(debug); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_nub_spmi0): power-controller@60 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi0); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_nub_spmi1): power-controller@68 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi1); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_nub_aon): power-controller@70 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_aon); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_msg): power-controller@78 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msg); + }; + + DIE_NODE(ps_nub_gpio): power-controller@80 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_gpio); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_nub_fabric): power-controller@88 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_fabric); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_atc0_usb_aon): power-controller@90 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x90 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_usb_aon); + apple,always-on; /* Needs to stay on for dwc3 to work */ + }; + + DIE_NODE(ps_atc1_usb_aon): power-controller@98 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x98 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_usb_aon); + apple,always-on; /* Needs to stay on for dwc3 to work */ + }; + + DIE_NODE(ps_atc2_usb_aon): power-controller@a0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_usb_aon); + apple,always-on; /* Needs to stay on for dwc3 to work */ + }; + + DIE_NODE(ps_atc3_usb_aon): power-controller@a8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_usb_aon); + apple,always-on; /* Needs to stay on for dwc3 to work */ + }; + + DIE_NODE(ps_mtp_fabric): power-controller@b0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_fabric); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_fabric)>; + }; + + DIE_NODE(ps_nub_sram): power-controller@b8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xb8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_sram); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_debug_switch): power-controller@c0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(debug_switch); + apple,always-on; /* Core AON device */ + }; + + DIE_NODE(ps_atc0_usb): power-controller@c8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_usb); + power-domains = <&DIE_NODE(ps_atc0_common)>; + }; + + DIE_NODE(ps_atc1_usb): power-controller@d0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_usb); + power-domains = <&DIE_NODE(ps_atc1_common)>; + }; + + DIE_NODE(ps_atc2_usb): power-controller@d8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_usb); + power-domains = <&DIE_NODE(ps_atc2_common)>; + }; + + DIE_NODE(ps_atc3_usb): power-controller@e0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_usb); + power-domains = <&DIE_NODE(ps_atc3_common)>; + }; + +#if 0 + /* MTP stuff is self-managed */ + DIE_NODE(ps_mtp_gpio): power-controller@e8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xe8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_gpio); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_base): power-controller@f0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xf0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_base); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_periph): power-controller@f8 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0xf8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_periph); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_spi0): power-controller@100 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_spi0); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_i2cm0): power-controller@108 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_i2cm0); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_uart0): power-controller@110 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_uart0); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_cpu): power-controller@118 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_cpu); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_scm_fabric): power-controller@120 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_scm_fabric); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_periph)>; + }; + + DIE_NODE(ps_mtp_sram): power-controller@128 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_sram); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_scm_fabric)>, <&DIE_NODE(ps_mtp_cpu)>; + }; + + DIE_NODE(ps_mtp_dma): power-controller@130 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_dma); + apple,always-on; /* MTP always stays on */ + power-domains = <&DIE_NODE(ps_mtp_sram)>; + }; +#endif +}; + +&pmgr_gfx { + DIE_NODE(ps_gpx): power-controller@0 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(gpx); + apple,always-on; + }; + + DIE_NODE(ps_afr): power-controller@100 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afr); + /* Apple Fabric, media stuff: this can power down */ + }; + + DIE_NODE(ps_gfx): power-controller@108 { + compatible = "apple,t6020-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(gfx); + power-domains = <&DIE_NODE(ps_afr)>, <&DIE_NODE(ps_gpx)>; + }; +}; + From 47f65f138bf9c92b9070e0631870456fe1a0382b Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 18:15:33 +0900 Subject: [PATCH 358/635] arm64: dts: apple: Add MTP nodes to t6020x Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t6020-j414s.dts | 4 + arch/arm64/boot/dts/apple/t6020-j416s.dts | 4 + arch/arm64/boot/dts/apple/t6021-j414c.dts | 4 + arch/arm64/boot/dts/apple/t6021-j416c.dts | 4 + arch/arm64/boot/dts/apple/t602x-die0.dtsi | 76 +++++++++++++++++++ .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 37 +++++++++ arch/arm64/boot/dts/apple/t8112.dtsi | 1 + 7 files changed, 130 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6020-j414s.dts b/arch/arm64/boot/dts/apple/t6020-j414s.dts index 1ce122d7ac5416..54e767b6df4c64 100644 --- a/arch/arm64/boot/dts/apple/t6020-j414s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j414s.dts @@ -35,3 +35,7 @@ &sound { model = "MacBook Pro J414"; }; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j416s.dts b/arch/arm64/boot/dts/apple/t6020-j416s.dts index 8a3b8bd946561d..598f646bda6db6 100644 --- a/arch/arm64/boot/dts/apple/t6020-j416s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j416s.dts @@ -35,3 +35,7 @@ &sound { model = "MacBook Pro J416"; }; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index d7480fd8459b55..c8098897752bc5 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -35,3 +35,7 @@ &sound { model = "MacBook Pro J414"; }; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j416c.dts b/arch/arm64/boot/dts/apple/t6021-j416c.dts index cc6e4c27502619..aa659da4e72015 100644 --- a/arch/arm64/boot/dts/apple/t6021-j416c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j416c.dts @@ -51,3 +51,7 @@ &sound { model = "MacBook Pro J416"; }; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 722b51c2fb0f8a..3b79439ab21a6a 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -170,6 +170,82 @@ ; }; + mtp: mtp@2a9400000 { + compatible = "apple,t6020-mtp", "apple,t6020-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xa9400000 0x0 0x4000>, + <0x2 0xa9c00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2a9408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa9408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@2a9808000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0xa9808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2a9b14000 { + compatible = "apple,t6020-dockchannel", "apple,dockchannel"; + reg = <0x2 0xa9b14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xa9b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + disp0_piodma: mapper@388000000 { compatible = "apple,t6020-dcp-piodma", "apple,dcp-piodma"; iommus = <&disp0_dart 4>; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index eecef8ae7c1f6d..415379b028fddb 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -118,3 +118,40 @@ &ps_isp_clr { status = "okay"; }; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + mtp_mt: multi-touch { + apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index fd8a9eac5cf857..aee07240a8f229 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -1267,6 +1267,7 @@ interrupts = ; ranges = <0 0x2 0x4eb28000 0x20000>; + nonposted-mmio; #address-cells = <1>; #size-cells = <1>; From c9569efb605c01e93a0eb0a6511bee843a737a41 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 11 Apr 2023 02:34:01 +0900 Subject: [PATCH 359/635] arm64: dts: apple: Add identity dma-ranges mapping Without this, the OF core ends up limiting all DMA masks to the default 32-bit, since that runs before drivers set up the proper DMA mask. Skipping the highest page because it is impossible to express a full 64-bit range in the DT. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t6001.dtsi | 2 ++ arch/arm64/boot/dts/apple/t6002.dtsi | 4 ++++ arch/arm64/boot/dts/apple/t6021.dtsi | 2 ++ arch/arm64/boot/dts/apple/t8103.dtsi | 2 ++ arch/arm64/boot/dts/apple/t8112.dtsi | 2 ++ 5 files changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 6e7e7cdeacf943..316deb8a95be63 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -50,6 +50,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index f1164315be755a..a7dfc6196fa724 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -240,6 +240,8 @@ <0x5 0x80000000 0x5 0x80000000 0x1 0x80000000>, <0x7 0x0 0x7 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -251,6 +253,8 @@ ranges = <0x2 0x0 0x22 0x0 0x4 0x0>, <0x7 0x0 0x27 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 3c7d691fbad50e..5a46ca207a1d5c 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -50,6 +50,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index 95bb8973c07037..f38f02766a1329 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -436,6 +436,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { compatible = "apple,agx-t8103", "apple,agx-g13g"; diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index aee07240a8f229..3845736fc46bd7 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -468,6 +468,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { compatible = "apple,agx-t8112", "apple,agx-g14g"; From 66a0dbfcb360869a797efbdb266f007d89894b79 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 11 Apr 2023 21:39:50 +0200 Subject: [PATCH 360/635] DO NOT SUBMIT: arm64: dts: apple: t6020-j474s: Add unused PCIe port01 This works around a Linux bug which results in mismatched iommus on gaps in PCI(e) ports / bus numbers. Remove as soon as the bug is identified. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t6020-j474s.dts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t6020-j474s.dts b/arch/arm64/boot/dts/apple/t6020-j474s.dts index 37d5a2a50eeab6..dc40bee0025f5c 100644 --- a/arch/arm64/boot/dts/apple/t6020-j474s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j474s.dts @@ -60,6 +60,16 @@ pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; }; +&port01 { + /* + * TODO: do not enable port without device. This works around a Linux + * bug which results in mismatched iommus on gaps in PCI(e) ports / bus + * numbers. + */ + bus-range = <2 2>; + status = "okay"; +}; + &port02 { /* 10 Gbit Ethernet */ bus-range = <3 3>; From 17fbb8489635fd7fa0a0a7628bae5dea99ac70e6 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Apr 2023 05:05:57 +0900 Subject: [PATCH 361/635] arm64: dts: apple: Add pmgr-misc nodes to t60xx --- arch/arm64/boot/dts/apple/t600x-die0.dtsi | 10 ++++++++++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index 9e11a5c92e5284..ee2bf373c47ba5 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -24,6 +24,16 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c800 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + apple,dcs-min-ps = <7>; + }; + smc_mbox: mbox@290408000 { compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x90408000 0x0 0x4000>; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 3b79439ab21a6a..6ca16f05b3ede8 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -25,6 +25,15 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6020-pmgr-misc", "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c400 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + }; + wdt: watchdog@29e2c4000 { compatible = "apple,t6020-wdt", "apple,wdt"; reg = <0x2 0x9e2c4000 0x0 0x4000>; From 508efe711b5c51a4b327c74de7f41360afb50c00 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 24 Apr 2023 23:27:52 +0900 Subject: [PATCH 362/635] arm64: dts: apple: t600x: Remove obsolete comment in ans2 power domain Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 00b317c2355b8c..0555ba2eb3918c 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -1383,12 +1383,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(ans2); - /* - * The ADT makes ps_apcie_st[1]_sys depend on ps_ans2 instead, - * but we'd rather have a single power domain for the downstream - * device to depend on, so use this node as the child. - * This makes more sense anyway (since ANS2 uses APCIE_ST). - */ power-domains = <&DIE_NODE(ps_afnc2_lw0)>; }; From 1e2f96e1f73f8ed897c659c0c1636c5a5b4b01ae Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Wed, 26 Apr 2023 02:17:26 +0900 Subject: [PATCH 363/635] arm64: dts: apple: Make ps_msg always-on Apple has it that way, and it might be important. Let's not risk it. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-pmgr.dtsi | 1 + arch/arm64/boot/dts/apple/t602x-pmgr.dtsi | 1 + arch/arm64/boot/dts/apple/t8103-pmgr.dtsi | 1 + arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 1 + 4 files changed, 4 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 0555ba2eb3918c..af3baf871b22ee 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -1873,6 +1873,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(msg); + apple,always-on; /* Core AON device? */ }; DIE_NODE(ps_nub_gpio): power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi index 84e5c19f90d7b3..b61d25e77c8678 100644 --- a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi @@ -2006,6 +2006,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(msg); + apple,always-on; /* Core AON device? */ }; DIE_NODE(ps_nub_gpio): power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index 724e7fd559e7a1..5fb8c8601a9dcb 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -1095,6 +1095,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_atc0_usb_aon: power-controller@88 { diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 32232b4f8ab652..5062e305e17f24 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -1061,6 +1061,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_nub_gpio: power-controller@80 { From e73886e2e584c7f3092097cca42f676a7a9ce906 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 27 Apr 2023 13:53:35 +0900 Subject: [PATCH 364/635] arm64: dts: apple: t600x: Enable turbo CPU p-states These should work now that we have cpuidle. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t600x-common.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index 279ba91d8abacd..667c02724b8646 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -315,7 +315,6 @@ opp-level = <12>; clock-latency-ns = <56000>; }; - /* Not available until CPU deep sleep is implemented opp13 { opp-hz = /bits/ 64 <3132000000>; opp-level = <13>; @@ -334,7 +333,6 @@ clock-latency-ns = <56000>; turbo-mode; }; - */ }; gpu_opp: opp-table-gpu { From 5e1d0112298315de51c0f08907351b6554864975 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 27 Apr 2023 13:54:17 +0900 Subject: [PATCH 365/635] arm64: dts: apple: t8103: Enable turbo CPU p-states These should work now that we have cpuidle. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8103.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index f38f02766a1329..b3b13d5938a023 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -280,7 +280,6 @@ opp-level = <12>; clock-latency-ns = <55000>; }; -#if 0 /* Not available until CPU deep sleep is implemented */ opp13 { opp-hz = /bits/ 64 <3096000000>; @@ -300,7 +299,6 @@ clock-latency-ns = <56000>; turbo-mode; }; -#endif }; gpu_opp: opp-table-gpu { From 26bd4bfc34840de563c776e0488c3c4839dd5922 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 27 Apr 2023 13:54:30 +0900 Subject: [PATCH 366/635] arm64: dts: apple: t8112: Enable turbo CPU p-states These should work now that we have cpuidle. Signed-off-by: Hector Martin --- arch/arm64/boot/dts/apple/t8112.dtsi | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 3845736fc46bd7..8f51e2798e7daf 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -302,8 +302,6 @@ opp-level = <14>; clock-latency-ns = <46000>; }; - /* Not available until CPU deep sleep is implemented */ -#if 0 opp15 { opp-hz = /bits/ 64 <3324000000>; opp-level = <15>; @@ -322,7 +320,6 @@ clock-latency-ns = <62000>; turbo-mode; }; -#endif }; gpu_opp: opp-table-gpu { From 1d9d900b53057adf79ba22257f2bd551bf5dfed7 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 8 May 2023 20:58:57 +0900 Subject: [PATCH 367/635] arm64: Introduce scaffolding to add ACTLR_EL1 to thread state Some CPUs expose IMPDEF features in ACTLR_EL1 that can be meaningfully controlled per-thread (like TSO control on Apple cores). Add the basic scaffolding to save/restore this register as part of context switching. This mechanism is disabled by default both by config symbol and via a runtime check, which ensures it is never triggered unless the system is known to need it for some feature (which also implies that the layout of ACTLR_EL1 is uniform between all CPU core types). Signed-off-by: Hector Martin --- arch/arm64/Kconfig | 3 +++ arch/arm64/include/asm/cpufeature.h | 5 +++++ arch/arm64/include/asm/processor.h | 3 +++ arch/arm64/kernel/process.c | 17 +++++++++++++++++ arch/arm64/kernel/setup.c | 8 ++++++++ 5 files changed, 36 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1023e896d46b89..7523c3348c1e12 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -380,6 +380,9 @@ config KASAN_SHADOW_OFFSET config UNWIND_TABLES bool +config ARM64_ACTLR_STATE + bool + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 6bf013fb110d79..6e196b6baa7c86 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -915,6 +915,11 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) return 8; } +static __always_inline bool system_has_actlr_state(void) +{ + return false; +} + struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); extern struct arm64_ftr_override id_aa64mmfr1_override; diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 3918f2a6797074..eff8bd8651eb2e 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -179,6 +179,9 @@ struct thread_struct { u64 sctlr_user; u64 svcr; u64 tpidr2_el0; +#ifdef CONFIG_ARM64_ACTLR_STATE + u64 actlr; +#endif }; static inline unsigned int thread_get_vl(struct thread_struct *thread, diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 71d59b5abede11..999bd559385f27 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -374,6 +374,9 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) if (system_supports_tpidr2()) p->thread.tpidr2_el0 = read_sysreg_s(SYS_TPIDR2_EL0); + if (system_has_actlr_state()) + p->thread.actlr = read_sysreg(actlr_el1); + if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; @@ -516,6 +519,19 @@ void update_sctlr_el1(u64 sctlr) isb(); } +/* + * IMPDEF control register ACTLR_EL1 handling. Some CPUs use this to + * expose features that can be controlled by userspace. + */ +static void actlr_thread_switch(struct task_struct *next) +{ + if (!system_has_actlr_state()) + return; + + current->thread.actlr = read_sysreg(actlr_el1); + write_sysreg(next->thread.actlr, actlr_el1); +} + /* * Thread switching. */ @@ -533,6 +549,7 @@ struct task_struct *__switch_to(struct task_struct *prev, ssbs_thread_switch(next); erratum_1418040_thread_switch(next); ptrauth_thread_switch_user(next); + actlr_thread_switch(next); /* * Complete any pending TLB or cache maintenance on this CPU in case diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index b8ec7b3ac9cbe8..960b6fe1b467a1 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -380,6 +380,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) */ init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + /* Store the boot CPU ACTLR_EL1 value as the default. This will only + * be actually restored during context switching iff the platform is + * known to use ACTLR_EL1 for exposable features and its layout is + * known to be the same on all CPUs. + */ + init_task.thread.actlr = read_sysreg(actlr_el1); +#endif if (boot_args[1] || boot_args[2] || boot_args[3]) { pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" From b5c6de40368b847e8c2968e260fb98ac1b7b10d2 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 8 May 2023 21:01:36 +0900 Subject: [PATCH 368/635] prctl: Introduce PR_{SET,GET}_MEM_MODEL On some architectures, it is possible to query and/or change the CPU memory model. This allows userspace to switch to a stricter memory model for performance reasons, such as when emulating code for another architecture where that model is the default. Introduce two prctls to allow userspace to query and set the memory model for a thread. Two models are initially defined: - PR_SET_MEM_MODEL_DEFAULT requests the default memory model for the architecture. - PR_SET_MEM_MODEL_TSO requests the x86 TSO memory model. PR_SET_MEM_MODEL is allowed to set a stricter memory model than requested if available, in which case it will return successfully. If the requested memory model cannot be fulfilled, it will return an error. The memory model that was actually set can be queried by a subsequent call to PR_GET_MEM_MODEL. Examples: - On a CPU with not support for a memory model at least as strong as TSO, PR_SET_MEM_MODEL(PR_SET_MEM_MODEL_TSO) fails. - On a CPU with runtime-configurable TSO support, PR_SET_MEM_MODEL can toggle the memory model between DEFAULT and TSO at will. - On a CPU where the only memory model is at least as strict as TSO, PR_GET_MEM_MODEL will return PR_SET_MEM_MODEL_DEFAULT, and PR_SET_MEM_MODEL(PR_SET_MEM_MODEL_TSO) will return success but leave the memory model at PR_SET_MEM_MODEL_DEFAULT. This implies that the default is in fact at least as strict as TSO. Signed-off-by: Hector Martin --- include/uapi/linux/prctl.h | 5 +++++ kernel/sys.c | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 1312a137f7fb85..1303d90c38ffd4 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -290,4 +290,9 @@ struct prctl_mm_map { #define PR_SET_VMA 0x53564d41 # define PR_SET_VMA_ANON_NAME 0 +#define PR_GET_MEM_MODEL 0x6d4d444c +#define PR_SET_MEM_MODEL 0x4d4d444c +# define PR_SET_MEM_MODEL_DEFAULT 0 +# define PR_SET_MEM_MODEL_TSO 1 + #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/sys.c b/kernel/sys.c index 351de791630205..f1317f930cf909 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2388,6 +2388,16 @@ static inline int prctl_get_mdwe(unsigned long arg2, unsigned long arg3, PR_MDWE_REFUSE_EXEC_GAIN : 0; } +int __weak arch_prctl_mem_model_get(struct task_struct *t) +{ + return -EINVAL; +} + +int __weak arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + return -EINVAL; +} + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2672,6 +2682,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, case PR_SET_VMA: error = prctl_set_vma(arg2, arg3, arg4, arg5); break; + case PR_GET_MEM_MODEL: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_get(me); + break; + case PR_SET_MEM_MODEL: + if (arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_set(me, arg2); + break; default: error = -EINVAL; break; From 4ab692b22440dd1d9203e939a10fb9d5919b54d8 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 8 May 2023 21:11:01 +0900 Subject: [PATCH 369/635] arm64: Implement PR_{GET,SET}_MEM_MODEL for always-TSO CPUs Some ARM64 implementations are known to always use the TSO memory model. Add trivial support for the PR_{GET,SET}_MEM_MODEL prctl, which allows userspace to learn this fact. Known TSO implementations: - Nvidia Denver - Nvidia Carmel - Fujitsu A64FX Signed-off-by: Hector Martin --- arch/arm64/Kconfig | 9 +++++++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/cpufeature.c | 7 +++-- arch/arm64/kernel/cpufeature_impdef.c | 39 +++++++++++++++++++++++++++ arch/arm64/kernel/process.c | 23 ++++++++++++++++ arch/arm64/tools/cpucaps | 1 + 6 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/kernel/cpufeature_impdef.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7523c3348c1e12..4706b2bf472ebf 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2130,6 +2130,15 @@ config ARM64_DEBUG_PRIORITY_MASKING If unsure, say N endif # ARM64_PSEUDO_NMI +config ARM64_MEMORY_MODEL_CONTROL + bool "Runtime memory model control" + help + Some ARM64 CPUs support runtime switching of the CPU memory + model, which can be useful to emulate other CPU architectures + which have different memory models. Say Y to enable support + for the PR_SET_MEM_MODEL/PR_GET_MEM_MODEL prctl() calls on + CPUs with this feature. + config RELOCATABLE bool "Build a relocatable kernel image" if EXPERT select ARCH_HAS_RELR diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index ceba6792f5b3c4..e180b5cb44f488 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,7 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idreg-override.o idle.o \ - patching.o + patching.o cpufeature_impdef.o obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 2e3e5513977733..c8e1fa753c7bba 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -134,6 +134,8 @@ DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); */ static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly; +void __init init_cpu_hwcaps_indirect_list_impdef(void); + void dump_cpu_features(void) { /* file-wide pr_fmt adds "CPU features: " prefix */ @@ -946,7 +948,7 @@ static void init_cpu_ftr_reg(u32 sys_reg, u64 new) extern const struct arm64_cpu_capabilities arm64_errata[]; static const struct arm64_cpu_capabilities arm64_features[]; -static void __init +void __init init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) { @@ -1046,6 +1048,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) * before we handle the boot CPU below. */ init_cpu_hwcaps_indirect_list(); + init_cpu_hwcaps_indirect_list_impdef(); /* * Detect and enable early CPU capabilities based on the boot CPU, @@ -1414,7 +1417,7 @@ has_always(const struct arm64_cpu_capabilities *entry, int scope) return true; } -static bool +bool feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { int val = cpuid_feature_extract_field_width(reg, entry->field_pos, diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c new file mode 100644 index 00000000000000..4defe725a8621d --- /dev/null +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Contains implementation-defined CPU feature definitions. + */ + +#include + +void __init init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); +bool feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); + +bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope) +{ + /* List of CPUs that always use the TSO memory model */ + static const struct midr_range fixed_tso_list[] = { + MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + MIDR_ALL_VERSIONS(MIDR_FUJITSU_A64FX), + { /* sentinel */ } + }; + + return is_midr_in_range_list(read_cpuid_id(), fixed_tso_list); +} + +static const struct arm64_cpu_capabilities arm64_impdef_features[] = { +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + { + .desc = "TSO memory model (Fixed)", + .capability = ARM64_HAS_TSO_FIXED, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_tso_fixed, + }, +#endif + {}, +}; + +void __init init_cpu_hwcaps_indirect_list_impdef(void) +{ + init_cpu_hwcaps_indirect_list_from_array(arm64_impdef_features); +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 999bd559385f27..95fec4990c5c54 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -532,6 +532,25 @@ static void actlr_thread_switch(struct task_struct *next) write_sysreg(next->thread.actlr, actlr_el1); } +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +int arch_prctl_mem_model_get(struct task_struct *t) +{ + return PR_SET_MEM_MODEL_DEFAULT; +} + +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + if (cpus_have_const_cap(ARM64_HAS_TSO_FIXED) && + val == PR_SET_MEM_MODEL_TSO) + return 0; + + if (val == PR_SET_MEM_MODEL_DEFAULT) + return 0; + + return -EINVAL; +} +#endif + /* * Thread switching. */ @@ -671,6 +690,10 @@ void arch_setup_new_exec(void) arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS, PR_SPEC_ENABLE); } + + if (IS_ENABLED(CONFIG_ARM64_MEMORY_MODEL_CONTROL)) { + arch_prctl_mem_model_set(current, PR_SET_MEM_MODEL_DEFAULT); + } } #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 37b1340e964664..30271914831ab8 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -43,6 +43,7 @@ HAS_SB HAS_STAGE2_FWB HAS_TIDCP1 HAS_TLB_RANGE +HAS_TSO_FIXED HAS_VIRT_HOST_EXTN HAS_WFXT HW_DBM From 38015ad2af19088e66e0c81f7efd5a8b51814026 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 8 May 2023 21:17:12 +0900 Subject: [PATCH 370/635] arm64: Implement Apple IMPDEF TSO memory model control Apple CPUs may implement the TSO memory model as an optional configurable mode. This allows x86 emulators to simplify their load/store handling, greatly increasing performance. Expose this via the prctl PR_SET_MEM_MODEL_TSO mechanism. We use the Apple IMPDEF AIDR_EL1 register to check for the availability of TSO mode, and enable this codepath on all CPUs with an Apple implementer. This relies on the ACTLR_EL1 thread state scaffolding introduced earlier. Signed-off-by: Hector Martin --- arch/arm64/Kconfig | 2 ++ arch/arm64/include/asm/apple_cpufeature.h | 15 ++++++++++++++ arch/arm64/include/asm/cpufeature.h | 3 ++- arch/arm64/kernel/cpufeature_impdef.c | 23 +++++++++++++++++++++ arch/arm64/kernel/process.c | 25 +++++++++++++++++++++-- arch/arm64/tools/cpucaps | 1 + 6 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/include/asm/apple_cpufeature.h diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 4706b2bf472ebf..dc433e10639d32 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2132,6 +2132,8 @@ endif # ARM64_PSEUDO_NMI config ARM64_MEMORY_MODEL_CONTROL bool "Runtime memory model control" + default ARCH_APPLE + select ARM64_ACTLR_STATE help Some ARM64 CPUs support runtime switching of the CPU memory model, which can be useful to emulate other CPU architectures diff --git a/arch/arm64/include/asm/apple_cpufeature.h b/arch/arm64/include/asm/apple_cpufeature.h new file mode 100644 index 00000000000000..4370d91ffa3ec9 --- /dev/null +++ b/arch/arm64/include/asm/apple_cpufeature.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef __ASM_APPLE_CPUFEATURES_H +#define __ASM_APPLE_CPUFEATURES_H + +#include +#include + +#define AIDR_APPLE_TSO_SHIFT 9 +#define AIDR_APPLE_TSO BIT(9) + +#define ACTLR_APPLE_TSO_SHIFT 1 +#define ACTLR_APPLE_TSO BIT(1) + +#endif diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 6e196b6baa7c86..835b9cdc342e55 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -917,7 +917,8 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) static __always_inline bool system_has_actlr_state(void) { - return false; + return IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && + cpus_have_const_cap(ARM64_HAS_TSO_APPLE); } struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c index 4defe725a8621d..84c131fe23d116 100644 --- a/arch/arm64/kernel/cpufeature_impdef.c +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -4,10 +4,23 @@ */ #include +#include void __init init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); bool feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); +bool has_apple_feature(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + WARN_ON(scope != SCOPE_SYSTEM); + + if (read_cpuid_implementor() != ARM_CPU_IMP_APPLE) + return false; + + val = read_sysreg(aidr_el1); + return feature_matches(val, entry); +} + bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope) { /* List of CPUs that always use the TSO memory model */ @@ -23,6 +36,16 @@ bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope) static const struct arm64_cpu_capabilities arm64_impdef_features[] = { #ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + { + .desc = "TSO memory model (Apple)", + .capability = ARM64_HAS_TSO_APPLE, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_apple_feature, + .field_pos = AIDR_APPLE_TSO_SHIFT, + .field_width = 1, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + }, { .desc = "TSO memory model (Fixed)", .capability = ARM64_HAS_TSO_FIXED, diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 95fec4990c5c54..ee7b76c4490ac0 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -43,6 +43,7 @@ #include #include +#include #include #include #include @@ -535,15 +536,35 @@ static void actlr_thread_switch(struct task_struct *next) #ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL int arch_prctl_mem_model_get(struct task_struct *t) { + if (cpus_have_const_cap(ARM64_HAS_TSO_APPLE) && + t->thread.actlr & ACTLR_APPLE_TSO) + return PR_SET_MEM_MODEL_TSO; + return PR_SET_MEM_MODEL_DEFAULT; } int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) { - if (cpus_have_const_cap(ARM64_HAS_TSO_FIXED) && - val == PR_SET_MEM_MODEL_TSO) + if (cpus_have_const_cap(ARM64_HAS_TSO_FIXED) && val == PR_SET_MEM_MODEL_TSO) return 0; + if (cpus_have_const_cap(ARM64_HAS_TSO_APPLE)) { + WARN_ON(!system_has_actlr_state()); + + switch (val) { + case PR_SET_MEM_MODEL_TSO: + t->thread.actlr |= ACTLR_APPLE_TSO; + break; + case PR_SET_MEM_MODEL_DEFAULT: + t->thread.actlr &= ~ACTLR_APPLE_TSO; + break; + default: + return -EINVAL; + } + write_sysreg(t->thread.actlr, actlr_el1); + return 0; + } + if (val == PR_SET_MEM_MODEL_DEFAULT) return 0; diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 30271914831ab8..58c66da58afd72 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -43,6 +43,7 @@ HAS_SB HAS_STAGE2_FWB HAS_TIDCP1 HAS_TLB_RANGE +HAS_TSO_APPLE HAS_TSO_FIXED HAS_VIRT_HOST_EXTN HAS_WFXT From 88e42b51c8f9f692c6fa7beabc6b58e92a2a6919 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 18 May 2023 16:12:39 +0900 Subject: [PATCH 371/635] PCI: apple: Make link up timeout configurable, default to 500ms We're seeing link up timeouts and it looks like devices are just too slow. Let's just increase this. Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 0f1f5d0fe80517..739bd6f36bc226 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -31,6 +31,10 @@ #include #include +static int link_up_timeout = 500; +module_param(link_up_timeout, int, 0644); +MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); + #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) #define CORE_RC_PHYIF_STAT 0x00028 @@ -738,7 +742,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, /* start link training */ writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) + if (!wait_for_completion_timeout(&pcie->event, link_up_timeout * HZ / 1000)) dev_warn(pcie->dev, "%pOF link didn't come up\n", np); if (pcie->hw->port_refclk) From 250ac979b7b2aa530e2645904d7d5ef9be1034a0 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Mon, 12 Dec 2022 02:53:26 -0500 Subject: [PATCH 372/635] init/Kconfig: Only block on RANDSTRUCT for RUST When enabling Rust in the kernel, we only need to block on the RANDSTRUCT feature and GCC plugin. The rest of the GCC plugins are reasonably safe to enable. Signed-off-by: Neal Gompa --- init/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/Kconfig b/init/Kconfig index c88bb30a8b0b02..bcb7a7860f2b76 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1944,9 +1944,9 @@ config RUST bool "Rust support" depends on HAVE_RUST depends on RUST_IS_AVAILABLE + depends on RANDSTRUCT_NONE depends on !MODVERSIONS - depends on !GCC_PLUGINS - depends on !RANDSTRUCT + depends on !GCC_PLUGIN_RANDSTRUCT depends on !DEBUG_INFO_BTF || PAHOLE_HAS_LANG_EXCLUDE select CONSTRUCTORS help From 49e57e6eb761f760944a91c621cac881d45523dc Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 11 Mar 2023 20:19:53 +0900 Subject: [PATCH 373/635] rust: Fixes for 1.67.0 Signed-off-by: Hector Martin --- rust/alloc/boxed.rs | 6 +++--- rust/alloc/lib.rs | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/alloc/boxed.rs b/rust/alloc/boxed.rs index dcfe87b14f3a64..5e7518bffbfb53 100644 --- a/rust/alloc/boxed.rs +++ b/rust/alloc/boxed.rs @@ -1869,7 +1869,7 @@ impl ExactSizeIterator for Box FusedIterator for Box {} #[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl + ?Sized, A: Allocator> FnOnce for Box { +impl + ?Sized, A: Allocator> FnOnce for Box { type Output = >::Output; extern "rust-call" fn call_once(self, args: Args) -> Self::Output { @@ -1878,14 +1878,14 @@ impl + ?Sized, A: Allocator> FnOnce for Box { } #[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl + ?Sized, A: Allocator> FnMut for Box { +impl + ?Sized, A: Allocator> FnMut for Box { extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output { >::call_mut(self, args) } } #[stable(feature = "boxed_closure_impls", since = "1.35.0")] -impl + ?Sized, A: Allocator> Fn for Box { +impl + ?Sized, A: Allocator> Fn for Box { extern "rust-call" fn call(&self, args: Args) -> Self::Output { >::call(self, args) } diff --git a/rust/alloc/lib.rs b/rust/alloc/lib.rs index 3aebf83c9967ea..1db93fdfae5802 100644 --- a/rust/alloc/lib.rs +++ b/rust/alloc/lib.rs @@ -141,6 +141,7 @@ #![feature(trusted_len)] #![feature(trusted_random_access)] #![feature(try_trait_v2)] +#![feature(tuple_trait)] #![feature(unchecked_math)] #![feature(unicode_internals)] #![feature(unsize)] From 98decb0c6dddbe1cffa95d23647ef3cb9d24bd4c Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 24 Apr 2023 23:05:40 +0900 Subject: [PATCH 374/635] driver core: fw_devlink: Add fw_devlink_count_absent_consumers() Some platforms have power domains that are active on boot and must remain powered up until all of their consumers probe. The genpd core needs a way to count how many consumers haven't probed yet to avoid powering off such domains. Add a fw_devlink_count_absent_consumers() function, which returns the total count of consumer devices which either have not been created at all yet (only fwlinks exist) or have been created but have no driver bound and fully probed yet. Signed-off-by: Hector Martin --- drivers/base/core.c | 27 +++++++++++++++++++++++++++ include/linux/fwnode.h | 1 + 2 files changed, 28 insertions(+) diff --git a/drivers/base/core.c b/drivers/base/core.c index 6878dfcbf0d60b..a010a6cdd8e999 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -202,6 +202,33 @@ void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(fw_devlink_purge_absent_suppliers); +/** + * fw_devlink_count_absent_consumers - Return how many consumers have + * either not been created yet, or do not yet have a driver attached. + * @fwnode: fwnode of the supplier + */ +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode) +{ + struct fwnode_link *link, *tmp; + struct device_link *dlink, *dtmp; + struct device *sup_dev = get_dev_from_fwnode(fwnode); + int count = 0; + + list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) + count++; + + if (!sup_dev) + return count; + + list_for_each_entry_safe(dlink, dtmp, &sup_dev->links.consumers, s_node) + if (dlink->consumer->links.status != DL_DEV_DRIVER_BOUND) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fw_devlink_count_absent_consumers); + + /** * __fwnode_links_move_consumers - Move consumer from @from to @to fwnode_handle * @from: move consumers away from this fwnode diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 5700451b300fb5..215fa5690c493f 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -211,5 +211,6 @@ extern bool fw_devlink_is_strict(void); int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup); void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode); #endif From 2930844b6765e0243fbf8742fabb0f566c42caef Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 24 Apr 2023 23:08:22 +0900 Subject: [PATCH 375/635] PM: domains: Add a flag to defer power-off until all consumers probe In some cases, power domains are active on boot and must remain turned on until all their dependent drivers probe. Examples are: - Boot-time framebuffers - Devices that run coprocessors which are handed off already running - Parent power domains with children that are also on at boot The genpd core currently powers off the genpd as soon as a single consumer device probes and goes into runtime suspend or when general probing is complete, whichever comes first. That breaks any devices which haven't probed yet. To fix this, add a GENPD_FLAG_DEFER_OFF which requests that the genpd core refuse to power down a domain if there are any consumer devices that either haven't probed yet, or whose device nodes do not exist yet (but fwlinks do). Genpd providers can set this if they expect to be critical for devices (e.g. if they are powered on at boot). It is possible for a device to be runtime suspended from its probe callback. If this is the last device to probe, this is allowable. To account for this, check whether the device whose callbacks are being invoked in the probing state, and in that case, allow 1 instead of 0 pending devices. Signed-off-by: Hector Martin --- drivers/base/power/domain.c | 54 ++++++++++++++++++++++++++++++++----- include/linux/pm_domain.h | 8 ++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 32084e38b73d01..d03d96c04551e0 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "PM: " fmt #include +#include #include #include #include @@ -130,6 +131,7 @@ static const struct genpd_lock_ops genpd_spin_ops = { #define genpd_is_active_wakeup(genpd) (genpd->flags & GENPD_FLAG_ACTIVE_WAKEUP) #define genpd_is_cpu_domain(genpd) (genpd->flags & GENPD_FLAG_CPU_DOMAIN) #define genpd_is_rpm_always_on(genpd) (genpd->flags & GENPD_FLAG_RPM_ALWAYS_ON) +#define genpd_is_defer_off(genpd) (genpd->flags & GENPD_FLAG_DEFER_OFF) static inline bool irq_safe_dev_in_sleep_domain(struct device *dev, const struct generic_pm_domain *genpd) @@ -654,6 +656,27 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) queue_work(pm_wq, &genpd->power_off_work); } +/** + * genpd_must_defer - Check whether the genpd cannot be safely powered off. + * @genpd: PM domain about to be powered down. + * @one_dev_probing: True if we are being called from RPM callbacks on a device that + * is probing, to allow poweroff if that device is the sole remaining consumer probing. + * + * Returns true if the @genpd has the GENPD_FLAG_DEFER_OFF flag and there + * are any consumer devices which either do not exist yet (only represented + * by fwlinks) or whose drivers have not probed yet. + */ +static bool genpd_must_defer(struct generic_pm_domain *genpd, bool one_dev_probing) +{ + if (genpd_is_defer_off(genpd) && genpd->has_provider) { + int absent = fw_devlink_count_absent_consumers(genpd->provider); + + if (absent > (one_dev_probing ? 1 : 0)) + return true; + } + return false; +} + /** * genpd_power_off - Remove power from a given PM domain. * @genpd: PM domain to power down. @@ -667,7 +690,7 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) * have been powered down, remove power from @genpd. */ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, - unsigned int depth) + bool one_dev_probing, unsigned int depth) { struct pm_domain_data *pdd; struct gpd_link *link; @@ -717,6 +740,14 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on)) return -EBUSY; + /* + * Do not allow PM domain to be powered off if it is marked + * as GENPD_FLAG_DEFER_OFF and there are consumer devices + * which have not probed yet. + */ + if (genpd_must_defer(genpd, one_dev_probing)) + return -EBUSY; + if (genpd->gov && genpd->gov->power_down_ok) { if (!genpd->gov->power_down_ok(&genpd->domain)) return -EAGAIN; @@ -743,7 +774,7 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, list_for_each_entry(link, &genpd->child_links, child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } @@ -801,7 +832,7 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth) child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } @@ -868,7 +899,7 @@ static void genpd_power_off_work_fn(struct work_struct *work) genpd = container_of(work, struct generic_pm_domain, power_off_work); genpd_lock(genpd); - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } @@ -933,6 +964,7 @@ static int genpd_runtime_suspend(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool runtime_pm = pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -987,7 +1019,7 @@ static int genpd_runtime_suspend(struct device *dev) return 0; genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); @@ -1008,6 +1040,7 @@ static int genpd_runtime_resume(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool timed = td && pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1065,7 +1098,7 @@ static int genpd_runtime_resume(struct device *dev) err_poweroff: if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) { genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); } @@ -1131,6 +1164,9 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, || atomic_read(&genpd->sd_count) > 0) return; + if (genpd_must_defer(genpd, false)) + return; + /* Check that the children are in their deepest (powered-off) state. */ list_for_each_entry(link, &genpd->parent_links, parent_node) { struct generic_pm_domain *child = link->child; @@ -2096,6 +2132,12 @@ int pm_genpd_init(struct generic_pm_domain *genpd, return -EINVAL; } + /* Deferred-off power domains should be powered on at initialization. */ + if (genpd_is_defer_off(genpd) && !genpd_status_on(genpd)) { + pr_warn("deferred-off PM domain %s is not on at init\n", genpd->name); + genpd->flags &= ~GENPD_FLAG_DEFER_OFF; + } + /* Multiple states but no governor doesn't make sense. */ if (!gov && genpd->state_count > 1) pr_warn("%s: no governor for states\n", genpd->name); diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index f776fb93eaa0d7..cec1189b33073e 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -61,6 +61,13 @@ * GENPD_FLAG_MIN_RESIDENCY: Enable the genpd governor to consider its * components' next wakeup when determining the * optimal idle state. + * + * GENPD_FLAG_DEFER_OFF: Defer powerdown if there are any consumer + * device fwlinks indicating that some consumer + * devices have not yet probed. This is useful + * for power domains which are active at boot and + * must not be shut down until all consumers + * complete their probe sequence. */ #define GENPD_FLAG_PM_CLK (1U << 0) #define GENPD_FLAG_IRQ_SAFE (1U << 1) @@ -69,6 +76,7 @@ #define GENPD_FLAG_CPU_DOMAIN (1U << 4) #define GENPD_FLAG_RPM_ALWAYS_ON (1U << 5) #define GENPD_FLAG_MIN_RESIDENCY (1U << 6) +#define GENPD_FLAG_DEFER_OFF (1U << 7) enum gpd_status { GENPD_STATE_ON = 0, /* PM domain is on */ From 7763abbe4c06848657bb2929584f32ffa99661ba Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 24 Apr 2023 23:13:14 +0900 Subject: [PATCH 376/635] soc: apple: apple-pmgr-pwrstate: Mark on-at-boot PDs as DEFER_OFF We consider any domains that are found to be powered on at boot as potentially critical for probing consumer devices. This prevents badness like the boot-time display controller being powered down as soon as its IOMMU probes. Fixes a pile of PD probe order dependencies and races that have required ALWAYS_ON workaround hacks until now, including: - ANS2 (NVMe) breaking if left on at handoff. - DISP0/DCP (boot display) completely breaking. - PM domains failing to probe when their parent was inadvertently shut down before the child probed. - PCIe losing state/fuse info/etc when it powers down before the driver is ready. - Touch Bar (DFR) display controller losing bootloader-configured state before its driver can probe and save it. The downside is that any spuriously on domains will remain on if their drivers are missing. We consider missing drivers that never get loaded a downstream bug. For older kernels running on newer DTs with extra devices, this shouldn't cause any major problems other than perhaps slightly increased power consumption (and we can always fix it in the bootloader by powering down those PDs if they don't need to be left on, since the bootloader is updated together with the DTs). Signed-off-by: Hector Martin --- drivers/soc/apple/apple-pmgr-pwrstate.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/soc/apple/apple-pmgr-pwrstate.c b/drivers/soc/apple/apple-pmgr-pwrstate.c index a3e2bc1d2686b6..b8f7dd9ebf15fe 100644 --- a/drivers/soc/apple/apple-pmgr-pwrstate.c +++ b/drivers/soc/apple/apple-pmgr-pwrstate.c @@ -242,6 +242,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) /* Turn it on so pm_genpd_init does not fail */ active = apple_pmgr_ps_power_on(&ps->genpd) == 0; } + } else if (active) { + ps->genpd.flags |= GENPD_FLAG_DEFER_OFF; } /* Turn on auto-PM if the domain is already on */ From aa216af0b623c6a2f90e30ec1eddaf2aa3988180 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 25 Apr 2023 01:46:23 +0900 Subject: [PATCH 377/635] tty: serial: samsung_tty: Mark as wakeup_path on no_console_suspend Devices not in the wakeup path always have their power domains shut down on suspend, which breaks no_console_suspend. Use the wakeup path feature to stop this from happening. This is somewhat an abuse of the concept as named, but the end result is exactly what we desire. Signed-off-by: Hector Martin --- drivers/tty/serial/samsung_tty.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index 7d82c97f642b9e..394522e15039dc 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -2193,6 +2193,9 @@ static int __maybe_unused s3c24xx_serial_suspend(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); + if (!console_suspend_enabled && uart_console(port)) + device_set_wakeup_path(dev); + if (port) uart_suspend_port(&s3c24xx_uart_drv, port); From 5b48d9eb3e5910c73370f23424460cdfd214650e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 25 Apr 2023 01:40:11 +0900 Subject: [PATCH 378/635] soc: apple: apple-pmgr-pwrstate: Mark on-at-boot PDs as wakeup The genpd core does not have a generic mechanism for skipping genpd shutdown on system sleep, but it does have the wakeup path mechanism that is essentially the same thing. Mark all PDs that are on at boot as potentially wakeup-relevant, which means they can *optionally* stay on. Drivers have to opt into this with device_set_wakeup_path() to actually force them to remain on. Signed-off-by: Hector Martin --- drivers/soc/apple/apple-pmgr-pwrstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/soc/apple/apple-pmgr-pwrstate.c b/drivers/soc/apple/apple-pmgr-pwrstate.c index b8f7dd9ebf15fe..260ae06c70a441 100644 --- a/drivers/soc/apple/apple-pmgr-pwrstate.c +++ b/drivers/soc/apple/apple-pmgr-pwrstate.c @@ -243,7 +243,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) active = apple_pmgr_ps_power_on(&ps->genpd) == 0; } } else if (active) { - ps->genpd.flags |= GENPD_FLAG_DEFER_OFF; + ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; } /* Turn on auto-PM if the domain is already on */ From cddca026d05561a46de2bd635e45dbda4665157a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 12 May 2023 18:55:48 +0900 Subject: [PATCH 379/635] HACK: Lol libwebrtc Remove this in a year or two, hopefully the Chromium/CEF ecosystem of fail will have caught up with their libwebrtc version by then. Signed-off-by: Hector Martin --- fs/fcntl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/fcntl.c b/fs/fcntl.c index b622be119706a1..01444865ed5c77 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -325,6 +325,8 @@ static long do_fcntl(int fd, unsigned int cmd, unsigned long arg, err = f_dupfd(arg, filp, 0); break; case F_DUPFD_CLOEXEC: + if (arg >= 1024) + arg = 0; /* Lol libwebrtc */ err = f_dupfd(arg, filp, O_CLOEXEC); break; case F_GETFD: From a93c49a93a575b02e51319300d642df07b77d94a Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 18 May 2023 16:18:29 +0900 Subject: [PATCH 380/635] PCI: apple: Reorder & improve link-up logic Always re-check LINKSTS right before deciding whether to start the link training and wait for it, just in case the link happened to come up while we were setting up IRQs. Also, always do the clock-gate disable even if the link is already up. Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 739bd6f36bc226..4cf8ce9056491f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -736,14 +736,14 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); - if (link_stat & PORT_LINKSTS_UP) - return 0; - - /* start link training */ - writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + /* start link training */ + writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - if (!wait_for_completion_timeout(&pcie->event, link_up_timeout * HZ / 1000)) - dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + if (!wait_for_completion_timeout(&pcie->event, link_up_timeout * HZ / 1000)) + dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + } if (pcie->hw->port_refclk) rmw_clear(PORT_REFCLK_CGDIS, port->base + PORT_REFCLK); From 23c2572382423192517bc17e1670a7149605c231 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 18 May 2023 17:03:32 +0900 Subject: [PATCH 381/635] PCI: apple: Log the time it takes for links to come up Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 4cf8ce9056491f..7d7d8973d9a7e2 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -738,11 +738,18 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, link_stat = readl_relaxed(port->base + PORT_LINKSTS); if (!(link_stat & PORT_LINKSTS_UP)) { + unsigned long timeout, left; /* start link training */ writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - if (!wait_for_completion_timeout(&pcie->event, link_up_timeout * HZ / 1000)) + timeout = link_up_timeout * HZ / 1000; + left = wait_for_completion_timeout(&pcie->event, timeout); + if (!left) dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + else + dev_info(pcie->dev, "%pOF link up after %ldms\n", np, + (timeout - left) * 1000 / HZ); + } if (pcie->hw->port_refclk) From 14426d8d29c830974069a70cf4e6ec3472ab6304 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 2 May 2023 22:00:37 +0200 Subject: [PATCH 382/635] squash! arm64: dts: apple: t8112: Remove always-on from the PMP node With "apple,always-on" removed from ps_pmp add it as dependency for the dcp* power-domains. Fixes dcp crashes on power state changes. TODO: investigate if it is enough to power ps_pmp on during SetPowerState calls. Signed-off-by: Janne Grunau --- arch/arm64/boot/dts/apple/t8112-pmgr.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 5062e305e17f24..3828a1333dacae 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -671,7 +671,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_disp0_sys>; + power-domains = <&ps_disp0_sys>, <&ps_pmp>; }; ps_dispext_sys: power-controller@380 { @@ -689,7 +689,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_dispext_sys>; + power-domains = <&ps_dispext_sys>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@3c8 { From aa9c968577536d05439b72c9aaa6447c61375522 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 26 Apr 2023 22:06:26 +0200 Subject: [PATCH 383/635] watchdog: apple: Support suspend and resume With working s2idle the watchdog has to be disarmed on suspend and re-armed on resume to prevent spuries resets after the timeout expires in suspend. Signed-off-by: Janne Grunau --- drivers/watchdog/apple_wdt.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index eddeb0fede896d..2167e0ba705eab 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -173,6 +173,8 @@ static int apple_wdt_probe(struct platform_device *pdev) if (!wdt->clk_rate) return -EINVAL; + platform_set_drvdata(pdev, wdt); + wdt->wdd.ops = &apple_wdt_ops; wdt->wdd.info = &apple_wdt_info; wdt->wdd.max_timeout = U32_MAX / wdt->clk_rate; @@ -190,6 +192,28 @@ static int apple_wdt_probe(struct platform_device *pdev) return devm_watchdog_register_device(dev, &wdt->wdd); } +static int __maybe_unused apple_wdt_resume(struct device *dev) +{ + struct apple_wdt *wdt = dev_get_drvdata(dev); + + if (watchdog_active(&wdt->wdd) || watchdog_hw_running(&wdt->wdd)) + apple_wdt_start(&wdt->wdd); + + return 0; +} + +static int __maybe_unused apple_wdt_suspend(struct device *dev) +{ + struct apple_wdt *wdt = dev_get_drvdata(dev); + + if (watchdog_active(&wdt->wdd) || watchdog_hw_running(&wdt->wdd)) + apple_wdt_stop(&wdt->wdd); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(apple_wdt_pm_ops, apple_wdt_suspend, apple_wdt_resume); + static const struct of_device_id apple_wdt_of_match[] = { { .compatible = "apple,wdt" }, {}, @@ -200,6 +224,7 @@ static struct platform_driver apple_wdt_driver = { .driver = { .name = "apple-watchdog", .of_match_table = apple_wdt_of_match, + .pm = pm_sleep_ptr(&apple_wdt_pm_ops), }, .probe = apple_wdt_probe, }; From 065150da3fe99a14a1f2acb00d2a326767179f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Fri, 28 Apr 2023 19:10:56 +0200 Subject: [PATCH 384/635] iommu: apple-dart: Link to consumers with blanket RPM_ACTIVE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the RPM_ACTIVE flag, runtime PM core only seems to consider the link insofar as it prevents the DART from suspending in case of consumers *considered active by runtime PM*. Other devices, like those on which runtime PM has yet to be enabled, or which lack any runtime PM support, are not considered in preventing the DART from suspending. DART going through suspend/resume cycle with active consumers can break the consumers' operation by the DART being reset in its resume path, among other things. Add RPM_ACTIVE flag to the link to have the consumer in the link prevent the DART from being suspended, unless the consumer itself is runtime PM suspended. This supersedes an earlier PCIe-only workaround. (TODO: Does this mean devices without bound drivers will keep their DARTs up indefinitely? This depends on the timing of the iommu probe_device/release_device calls. Investigate.) Signed-off-by: Martin Povišer --- drivers/iommu/apple-dart.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 922254383a95ba..224d813350f58b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -818,24 +818,15 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_stream_map *stream_map; - u32 dl_flags = DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER; int i; if (!cfg) return ERR_PTR(-ENODEV); - /* - * Runtime PM behaves strangely with PCIe devices, pm_runtime_enabled() - * returns true here although the devices do not support runtime PM. - * This seems to confuse the PM core and DART is runtime suspended while - * the device is in use. The dart_hw_reset() in the resume path ends up - * breaking PCIe devices eventually. - */ - if (dev_is_pci(dev)) - dl_flags |= DL_FLAG_RPM_ACTIVE; - for_each_stream_map(i, cfg, stream_map) - device_link_add(dev, stream_map->dart->dev, dl_flags); + device_link_add(dev, stream_map->dart->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER | + DL_FLAG_RPM_ACTIVE); return &cfg->stream_maps[0].dart->iommu; } From 6d71bd5647b2063fed0d7aff627b3654db303711 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Apr 2023 17:14:05 +0900 Subject: [PATCH 385/635] macsmc: Fix race between backend and core on notifications The core enables notifications (NTAP) before returning from the probe function, before the backend has a chance to set smc->core. This leads to a race if a notification arrives early. We already rely on the core setting the drvdata to itself, so move that ahead of NTAP=1 and drop smc->core entirely. That way it's safe for the backend notification callback to fire before the core probe function returns. Signed-off-by: Hector Martin --- drivers/platform/apple/smc.h | 3 +-- drivers/platform/apple/smc_core.c | 18 +++++++++--------- drivers/platform/apple/smc_rtkit.c | 10 ++++------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/drivers/platform/apple/smc.h b/drivers/platform/apple/smc.h index 8ae51887b2c553..34131f77fe09cb 100644 --- a/drivers/platform/apple/smc.h +++ b/drivers/platform/apple/smc.h @@ -19,8 +19,7 @@ struct apple_smc_backend_ops { int (*get_key_info)(void *cookie, smc_key key, struct apple_smc_key_info *info); }; -struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_backend_ops *ops, - void *cookie); +int apple_smc_probe(struct device *dev, const struct apple_smc_backend_ops *ops, void *cookie); void *apple_smc_get_cookie(struct apple_smc *smc); int apple_smc_remove(struct apple_smc *smc); void apple_smc_event_received(struct apple_smc *smc, uint32_t event); diff --git a/drivers/platform/apple/smc_core.c b/drivers/platform/apple/smc_core.c index 27605398be3b24..ee1df85e0aecff 100644 --- a/drivers/platform/apple/smc_core.c +++ b/drivers/platform/apple/smc_core.c @@ -236,7 +236,7 @@ void *apple_smc_get_cookie(struct apple_smc *smc) } EXPORT_SYMBOL(apple_smc_get_cookie); -struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_backend_ops *ops, void *cookie) +int apple_smc_probe(struct device *dev, const struct apple_smc_backend_ops *ops, void *cookie) { struct apple_smc *smc; u32 count; @@ -244,7 +244,7 @@ struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_bac smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); if (!smc) - return ERR_PTR(-ENOMEM); + return -ENOMEM; smc->dev = dev; smc->be_cookie = cookie; @@ -254,16 +254,18 @@ struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_bac ret = apple_smc_read_u32(smc, SMC_KEY(#KEY), &count); if (ret) - return ERR_PTR(dev_err_probe(dev, ret, "Failed to get key count")); + return dev_err_probe(dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); ret = apple_smc_get_key_by_index(smc, 0, &smc->first_key); if (ret) - return ERR_PTR(dev_err_probe(dev, ret, "Failed to get first key")); + return dev_err_probe(dev, ret, "Failed to get first key"); ret = apple_smc_get_key_by_index(smc, smc->key_count - 1, &smc->last_key); if (ret) - return ERR_PTR(dev_err_probe(dev, ret, "Failed to get last key")); + return dev_err_probe(dev, ret, "Failed to get last key"); + + dev_set_drvdata(dev, smc); /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), 1); @@ -271,13 +273,11 @@ struct apple_smc *apple_smc_probe(struct device *dev, const struct apple_smc_bac dev_info(dev, "Initialized (%d keys %p4ch..%p4ch)\n", smc->key_count, &smc->first_key, &smc->last_key); - dev_set_drvdata(dev, smc); - ret = mfd_add_devices(dev, -1, apple_smc_devs, ARRAY_SIZE(apple_smc_devs), NULL, 0, NULL); if (ret) - return ERR_PTR(dev_err_probe(dev, ret, "Subdevice initialization failed")); + return dev_err_probe(dev, ret, "Subdevice initialization failed"); - return smc; + return 0; } EXPORT_SYMBOL(apple_smc_probe); diff --git a/drivers/platform/apple/smc_rtkit.c b/drivers/platform/apple/smc_rtkit.c index 583d65681c3a25..77ad53fcc9e637 100644 --- a/drivers/platform/apple/smc_rtkit.c +++ b/drivers/platform/apple/smc_rtkit.c @@ -39,7 +39,6 @@ struct apple_smc_rtkit { struct device *dev; - struct apple_smc *core; struct apple_rtkit *rtk; struct completion init_done; @@ -320,6 +319,7 @@ static bool apple_smc_rtkit_recv_early(void *cookie, u8 endpoint, u64 message) static void apple_smc_rtkit_recv(void *cookie, u8 endpoint, u64 message) { struct apple_smc_rtkit *smc = cookie; + struct apple_smc *core = dev_get_drvdata(smc->dev); if (endpoint != SMC_ENDPOINT) { dev_err(smc->dev, "Received message for unknown endpoint 0x%x\n", endpoint); @@ -331,7 +331,7 @@ static void apple_smc_rtkit_recv(void *cookie, u8 endpoint, u64 message) return; } - apple_smc_event_received(smc->core, FIELD_GET(SMC_DATA, message)); + apple_smc_event_received(core, FIELD_GET(SMC_DATA, message)); } static const struct apple_rtkit_ops apple_smc_rtkit_ops = { @@ -402,11 +402,9 @@ static int apple_smc_rtkit_probe(struct platform_device *pdev) goto cleanup; } - smc->core = apple_smc_probe(dev, &apple_smc_rtkit_be_ops, smc); - if (IS_ERR(smc->core)) { - ret = PTR_ERR(smc->core); + ret = apple_smc_probe(dev, &apple_smc_rtkit_be_ops, smc); + if (ret) goto cleanup; - } return 0; From bc095b4ad98cee50cf1980b2af5ed2edb422c3f8 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Apr 2023 04:18:14 +0900 Subject: [PATCH 386/635] power: supply: macsmc_power: Log power data on button presses This helps catch s2idle power stats, since we get early data when the system resumes due to a power button press. Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 64 +++++++++++++++++------------ 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index 3b30384c47f7d7..d928c127dd7dc5 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -72,6 +72,36 @@ static struct macsmc_power *g_power; #define ACSt_CAN_BOOT_AP BIT(2) #define ACSt_CAN_BOOT_IBOOT BIT(1) +static void macsmc_do_dbg(struct macsmc_power *power) +{ + int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; + s32 p_bat = 0; + s16 t_full = 0, t_empty = 0; + u8 charge = 0; + + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PDTR), &p_in, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSTR), &p_sys, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PMVR), &p_3v8, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PHPC), &p_cpu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSVR), &p_clvr, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPMC), &p_mpmu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPSC), &p_spmu, 1000); + apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &p_bat); + apple_smc_read_s16(power->smc, SMC_KEY(B0TE), &t_empty); + apple_smc_read_s16(power->smc, SMC_KEY(B0TF), &t_full); + apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &charge); + +#define FD3(x) ((x) / 1000), abs((x) % 1000) + dev_info(power->dev, + "In %2d.%03dW Sys %2d.%03dW 3V8 %2d.%03dW MPMU %2d.%03dW SPMU %2d.%03dW " + "CLVR %2d.%03dW CPU %2d.%03dW Batt %2d.%03dW %d%% T%s %dm\n", + FD3(p_in), FD3(p_sys), FD3(p_3v8), FD3(p_mpmu), FD3(p_spmu), FD3(p_clvr), + FD3(p_cpu), FD3(p_bat), charge, + t_full >= 0 ? "full" : "empty", + t_full >= 0 ? t_full : t_empty); +#undef FD3 +} + static int macsmc_battery_get_status(struct macsmc_power *power) { u64 nocharge_flags; @@ -501,34 +531,8 @@ static void macsmc_dbg_work(struct work_struct *wrk) { struct macsmc_power *power = container_of(to_delayed_work(wrk), struct macsmc_power, dbg_log_work); - int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; - s32 p_bat = 0; - s16 t_full = 0, t_empty = 0; - u8 charge = 0; - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PDTR), &p_in, 1000); - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSTR), &p_sys, 1000); - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PMVR), &p_3v8, 1000); - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PHPC), &p_cpu, 1000); - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSVR), &p_clvr, 1000); - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPMC), &p_mpmu, 1000); - apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPSC), &p_spmu, 1000); - apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &p_bat); - apple_smc_read_s16(power->smc, SMC_KEY(B0TE), &t_empty); - apple_smc_read_s16(power->smc, SMC_KEY(B0TF), &t_full); - apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &charge); - -#define FD3(x) ((x) / 1000), abs((x) % 1000) - - dev_info(power->dev, - "In %2d.%03dW Sys %2d.%03dW 3V8 %2d.%03dW MPMU %2d.%03dW SPMU %2d.%03dW " - "CLVR %2d.%03dW CPU %2d.%03dW Batt %2d.%03dW %d%% T%s %dm\n", - FD3(p_in), FD3(p_sys), FD3(p_3v8), FD3(p_mpmu), FD3(p_spmu), FD3(p_clvr), - FD3(p_cpu), FD3(p_bat), charge, - t_full >= 0 ? "full" : "empty", - t_full >= 0 ? t_full : t_empty); - -#undef FD3 + macsmc_do_dbg(power); if (log_power) schedule_delayed_work(&power->dbg_log_work, POWER_LOG_INTERVAL); @@ -621,6 +625,12 @@ static int macsmc_power_event(struct notifier_block *nb, unsigned long event, vo } else if ((event & 0xff000000) == 0x71000000) { dev_info(power->dev, "Unknown charger event 0x%lx\n", event); + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x72010000) { + /* Button event handled by macsmc-hid, but let's do a debug print */ + if (log_power) + macsmc_do_dbg(power); + return NOTIFY_OK; } From a73fd218670ab084b24bac5ccaee1c092a31bfc3 Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 7 Feb 2023 10:52:15 -0800 Subject: [PATCH 387/635] rust: sync: impl {Debug,Display} for {Unique,}Arc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows printing the inner data of `Arc` and its friends if the inner data implements `Display` or `Debug`. It's useful for logging and debugging purpose. Signed-off-by: Boqun Feng Reviewed-by: Vincenzo Palazzo Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Reviewed-by: Björn Roy Baron Link: https://lore.kernel.org/r/20230207185216.1314638-2-boqun.feng@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/sync/arc.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index f2f1c83d72ba21..a94e303217c66e 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -22,6 +22,7 @@ use crate::{ }; use alloc::boxed::Box; use core::{ + fmt, marker::{PhantomData, Unsize}, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, @@ -522,3 +523,27 @@ impl DerefMut for UniqueArc { unsafe { &mut self.inner.ptr.as_mut().data } } } + +impl fmt::Display for UniqueArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +impl fmt::Display for Arc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +impl fmt::Debug for UniqueArc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} + +impl fmt::Debug for Arc { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} From 67ba94da0c113990cf5a65faf198f2824596851f Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 7 Feb 2023 10:52:16 -0800 Subject: [PATCH 388/635] samples: rust: print: Add sample code for Arc printing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This both demonstrates the usage of different print format in Rust and serves as a selftest for the `Display` and `Debug` implementation of `Arc` and its friends. Signed-off-by: Boqun Feng Reviewed-by: Björn Roy Baron Reviewed-by: Finn Behrens Reviewed-by: Vincenzo Palazzo Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230207185216.1314638-3-boqun.feng@gmail.com [ Applied suggestions and reworded for fixing title typos. ] Signed-off-by: Miguel Ojeda --- samples/rust/rust_print.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/samples/rust/rust_print.rs b/samples/rust/rust_print.rs index 8b39d9cef6d110..67ed8ebf8e8e9f 100644 --- a/samples/rust/rust_print.rs +++ b/samples/rust/rust_print.rs @@ -15,6 +15,30 @@ module! { struct RustPrint; +fn arc_print() -> Result { + use kernel::sync::*; + + let a = Arc::try_new(1)?; + let b = UniqueArc::try_new("hello, world")?; + + // Prints the value of data in `a`. + pr_info!("{}", a); + + // Uses ":?" to print debug fmt of `b`. + pr_info!("{:?}", b); + + let a: Arc<&str> = b.into(); + let c = a.clone(); + + // Uses `dbg` to print, will move `c` (for temporary debugging purposes). + dbg!(c); + + // Pretty-prints the debug formatting with lower-case hexadecimal integers. + pr_info!("{:#x?}", a); + + Ok(()) +} + impl kernel::Module for RustPrint { fn init(_module: &'static ThisModule) -> Result { pr_info!("Rust printing macros sample (init)\n"); @@ -43,6 +67,8 @@ impl kernel::Module for RustPrint { pr_cont!(" is {}", "continued"); pr_cont!(" with {}\n", "args"); + arc_print()?; + Ok(RustPrint) } } From df480a1687b74c107f32007b46d2ecc9ce298c90 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 24 Feb 2023 17:09:47 +0900 Subject: [PATCH 389/635] rust: Enable the new_uninit feature for kernel and driver crates The unstable new_uninit feature enables various library APIs to create uninitialized containers, such as `Box::assume_init()`. This is necessary to build abstractions that directly initialize memory at the target location, instead of doing copies through the stack. Will be used by the DRM scheduler abstraction in the kernel crate, and by field-wise initialization (e.g. using `place!()` or a future replacement macro which may itself live in `kernel`) in driver crates. Link: https://github.com/Rust-for-Linux/linux/issues/879 Link: https://github.com/Rust-for-Linux/linux/issues/2 Link: https://github.com/rust-lang/rust/issues/63291 Signed-off-by: Asahi Lina Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Reviewed-by: Vincenzo Palazzo Link: https://lore.kernel.org/r/20230224-rust-new_uninit-v1-1-c951443d9e26@asahilina.net [ Reworded to use `Link` tags. ] Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 + scripts/Makefile.build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 223564f9f0cc38..1118cd3e0b5f52 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -17,6 +17,7 @@ #![feature(core_ffi_c)] #![feature(dispatch_from_dyn)] #![feature(generic_associated_types)] +#![feature(new_uninit)] #![feature(receiver_trait)] #![feature(unsize)] diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 76323201232a54..1364e3d905fc62 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := core_ffi_c +rust_allowed_features := core_ffi_c,new_uninit rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ From be522de1a53eb2186ccedfabdd020150860ce8ec Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 24 Feb 2023 18:11:47 +0900 Subject: [PATCH 390/635] rust: Import upstream `alloc::vec::set_len_on_drop` module This is a subset of the Rust standard library `alloc` crate, version 1.62.0, licensed under "Apache-2.0 OR MIT", from: https://github.com/rust-lang/rust/tree/1.62.0/library/alloc/src The file is copied as-is, with no modifications whatsoever (not even adding the SPDX identifiers). For copyright details, please see: https://github.com/rust-lang/rust/blob/1.62.0/COPYRIGHT Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-vec-v1-1-733b5b5a57c5@asahilina.net [ Import version 1.62.0 instead, to match the one in mainline. ] Signed-off-by: Miguel Ojeda --- rust/alloc/vec/set_len_on_drop.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 rust/alloc/vec/set_len_on_drop.rs diff --git a/rust/alloc/vec/set_len_on_drop.rs b/rust/alloc/vec/set_len_on_drop.rs new file mode 100644 index 00000000000000..8b66bc81212969 --- /dev/null +++ b/rust/alloc/vec/set_len_on_drop.rs @@ -0,0 +1,28 @@ +// Set the length of the vec when the `SetLenOnDrop` value goes out of scope. +// +// The idea is: The length field in SetLenOnDrop is a local variable +// that the optimizer will see does not alias with any stores through the Vec's data +// pointer. This is a workaround for alias analysis issue #32155 +pub(super) struct SetLenOnDrop<'a> { + len: &'a mut usize, + local_len: usize, +} + +impl<'a> SetLenOnDrop<'a> { + #[inline] + pub(super) fn new(len: &'a mut usize) -> Self { + SetLenOnDrop { local_len: *len, len } + } + + #[inline] + pub(super) fn increment_len(&mut self, increment: usize) { + self.local_len += increment; + } +} + +impl Drop for SetLenOnDrop<'_> { + #[inline] + fn drop(&mut self) { + *self.len = self.local_len; + } +} From d7f9279e3444397cd20afe5f47c915ef48c91b08 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 24 Feb 2023 18:11:48 +0900 Subject: [PATCH 391/635] rust: Import upstream `alloc::vec::spec_extend` module This is a subset of the Rust standard library `alloc` crate, version 1.62.0, licensed under "Apache-2.0 OR MIT", from: https://github.com/rust-lang/rust/tree/1.62.0/library/alloc/src The file is copied as-is, with no modifications whatsoever (not even adding the SPDX identifiers). For copyright details, please see: https://github.com/rust-lang/rust/blob/1.62.0/COPYRIGHT Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-vec-v1-2-733b5b5a57c5@asahilina.net [ Import version 1.62.0 instead, to match the one in mainline. ] Signed-off-by: Miguel Ojeda --- rust/alloc/vec/spec_extend.rs | 87 +++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 rust/alloc/vec/spec_extend.rs diff --git a/rust/alloc/vec/spec_extend.rs b/rust/alloc/vec/spec_extend.rs new file mode 100644 index 00000000000000..506ee0ecfa279c --- /dev/null +++ b/rust/alloc/vec/spec_extend.rs @@ -0,0 +1,87 @@ +use crate::alloc::Allocator; +use core::iter::TrustedLen; +use core::ptr::{self}; +use core::slice::{self}; + +use super::{IntoIter, SetLenOnDrop, Vec}; + +// Specialization trait used for Vec::extend +pub(super) trait SpecExtend { + fn spec_extend(&mut self, iter: I); +} + +impl SpecExtend for Vec +where + I: Iterator, +{ + default fn spec_extend(&mut self, iter: I) { + self.extend_desugared(iter) + } +} + +impl SpecExtend for Vec +where + I: TrustedLen, +{ + default fn spec_extend(&mut self, iterator: I) { + // This is the case for a TrustedLen iterator. + let (low, high) = iterator.size_hint(); + if let Some(additional) = high { + debug_assert_eq!( + low, + additional, + "TrustedLen iterator's size hint is not exact: {:?}", + (low, high) + ); + self.reserve(additional); + unsafe { + let mut ptr = self.as_mut_ptr().add(self.len()); + let mut local_len = SetLenOnDrop::new(&mut self.len); + iterator.for_each(move |element| { + ptr::write(ptr, element); + ptr = ptr.offset(1); + // Since the loop executes user code which can panic we have to bump the pointer + // after each step. + // NB can't overflow since we would have had to alloc the address space + local_len.increment_len(1); + }); + } + } else { + // Per TrustedLen contract a `None` upper bound means that the iterator length + // truly exceeds usize::MAX, which would eventually lead to a capacity overflow anyway. + // Since the other branch already panics eagerly (via `reserve()`) we do the same here. + // This avoids additional codegen for a fallback code path which would eventually + // panic anyway. + panic!("capacity overflow"); + } + } +} + +impl SpecExtend> for Vec { + fn spec_extend(&mut self, mut iterator: IntoIter) { + unsafe { + self.append_elements(iterator.as_slice() as _); + } + iterator.forget_remaining_elements(); + } +} + +impl<'a, T: 'a, I, A: Allocator + 'a> SpecExtend<&'a T, I> for Vec +where + I: Iterator, + T: Clone, +{ + default fn spec_extend(&mut self, iterator: I) { + self.spec_extend(iterator.cloned()) + } +} + +impl<'a, T: 'a, A: Allocator + 'a> SpecExtend<&'a T, slice::Iter<'a, T>> for Vec +where + T: Copy, +{ + fn spec_extend(&mut self, iterator: slice::Iter<'a, T>) { + let slice = iterator.as_slice(); + unsafe { self.append_elements(slice) }; + } +} From a377b3e4d72761886849e317b4ae2aa293000e06 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 24 Feb 2023 18:11:49 +0900 Subject: [PATCH 392/635] rust: Add SPDX headers to alloc::vec::{spec_extend, set_len_on_drop} Add the missing SPDX headers to these modules, which were just imported from the Rust stdlib. Doing this in a separate commit makes it easier to audit that the files have not been modified in the original import. See the preceding two commits for attribution and licensing details. Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-vec-v1-3-733b5b5a57c5@asahilina.net [ Reworded for typo. ] Signed-off-by: Miguel Ojeda --- rust/alloc/vec/set_len_on_drop.rs | 2 ++ rust/alloc/vec/spec_extend.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rust/alloc/vec/set_len_on_drop.rs b/rust/alloc/vec/set_len_on_drop.rs index 8b66bc81212969..448bf5076a0bf3 100644 --- a/rust/alloc/vec/set_len_on_drop.rs +++ b/rust/alloc/vec/set_len_on_drop.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + // Set the length of the vec when the `SetLenOnDrop` value goes out of scope. // // The idea is: The length field in SetLenOnDrop is a local variable diff --git a/rust/alloc/vec/spec_extend.rs b/rust/alloc/vec/spec_extend.rs index 506ee0ecfa279c..7bb3abb7ee7164 100644 --- a/rust/alloc/vec/spec_extend.rs +++ b/rust/alloc/vec/spec_extend.rs @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + use crate::alloc::Allocator; use core::iter::TrustedLen; use core::ptr::{self}; From 6980163418c7969b3bb05de0280c8927fc9254a1 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Fri, 24 Feb 2023 18:11:50 +0900 Subject: [PATCH 393/635] rust: alloc: vec: Add some try_* methods we need Add some missing fallible methods that we need. They are all marked as: #[stable(feature = "kernel", since = "1.0.0")] for easy identification. Lina: Extracted from commit 487d7578bd03 ("rust: alloc: add some `try_*` methods we need") in rust-for-linux/rust. Signed-off-by: Miguel Ojeda Signed-off-by: Asahi Lina Link: https://github.com/Rust-for-Linux/linux/commit/487d7578bd03 Link: https://lore.kernel.org/r/20230224-rust-vec-v1-4-733b5b5a57c5@asahilina.net [ Match the non-fallible methods from version 1.62.0, since those in commit 487d7578bd03 were written for 1.54.0-beta.1. ] Signed-off-by: Miguel Ojeda --- rust/alloc/vec/mod.rs | 137 +++++++++++++++++++++++++++++++++- rust/alloc/vec/spec_extend.rs | 85 +++++++++++++++++++++ 2 files changed, 219 insertions(+), 3 deletions(-) diff --git a/rust/alloc/vec/mod.rs b/rust/alloc/vec/mod.rs index f77c7368d53478..fe4fff5064bcd0 100644 --- a/rust/alloc/vec/mod.rs +++ b/rust/alloc/vec/mod.rs @@ -122,10 +122,8 @@ use self::spec_from_elem::SpecFromElem; #[cfg(not(no_global_oom_handling))] mod spec_from_elem; -#[cfg(not(no_global_oom_handling))] use self::set_len_on_drop::SetLenOnDrop; -#[cfg(not(no_global_oom_handling))] mod set_len_on_drop; #[cfg(not(no_global_oom_handling))] @@ -149,7 +147,8 @@ mod spec_from_iter; #[cfg(not(no_global_oom_handling))] use self::spec_extend::SpecExtend; -#[cfg(not(no_global_oom_handling))] +use self::spec_extend::TrySpecExtend; + mod spec_extend; /// A contiguous growable array type, written as `Vec`, short for 'vector'. @@ -1919,6 +1918,17 @@ impl Vec { self.len += count; } + /// Tries to append elements to `self` from other buffer. + #[inline] + unsafe fn try_append_elements(&mut self, other: *const [T]) -> Result<(), TryReserveError> { + let count = unsafe { (*other).len() }; + self.try_reserve(count)?; + let len = self.len(); + unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) }; + self.len += count; + Ok(()) + } + /// Removes the specified range from the vector in bulk, returning all /// removed elements as an iterator. If the iterator is dropped before /// being fully consumed, it drops the remaining removed elements. @@ -2340,6 +2350,45 @@ impl Vec { } } + /// Tries to resize the `Vec` in-place so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the `Vec` is extended by the + /// difference, with each additional slot filled with `value`. + /// If `new_len` is less than `len`, the `Vec` is simply truncated. + /// + /// This method requires `T` to implement [`Clone`], + /// in order to be able to clone the passed value. + /// If you need more flexibility (or want to rely on [`Default`] instead of + /// [`Clone`]), use [`Vec::resize_with`]. + /// If you only need to resize to a smaller size, use [`Vec::truncate`]. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!["hello"]; + /// vec.try_resize(3, "world").unwrap(); + /// assert_eq!(vec, ["hello", "world", "world"]); + /// + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.try_resize(2, 0).unwrap(); + /// assert_eq!(vec, [1, 2]); + /// + /// let mut vec = vec![42]; + /// let result = vec.try_resize(usize::MAX, 0); + /// assert!(result.is_err()); + /// ``` + #[stable(feature = "kernel", since = "1.0.0")] + pub fn try_resize(&mut self, new_len: usize, value: T) -> Result<(), TryReserveError> { + let len = self.len(); + + if new_len > len { + self.try_extend_with(new_len - len, ExtendElement(value)) + } else { + self.truncate(new_len); + Ok(()) + } + } + /// Clones and appends all elements in a slice to the `Vec`. /// /// Iterates over the slice `other`, clones each element, and then appends @@ -2365,6 +2414,30 @@ impl Vec { self.spec_extend(other.iter()) } + /// Tries to clone and append all elements in a slice to the `Vec`. + /// + /// Iterates over the slice `other`, clones each element, and then appends + /// it to this `Vec`. The `other` slice is traversed in-order. + /// + /// Note that this function is same as [`extend`] except that it is + /// specialized to work with slices instead. If and when Rust gets + /// specialization this function will likely be deprecated (but still + /// available). + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.try_extend_from_slice(&[2, 3, 4]).unwrap(); + /// assert_eq!(vec, [1, 2, 3, 4]); + /// ``` + /// + /// [`extend`]: Vec::extend + #[stable(feature = "kernel", since = "1.0.0")] + pub fn try_extend_from_slice(&mut self, other: &[T]) -> Result<(), TryReserveError> { + self.try_spec_extend(other.iter()) + } + /// Copies elements from `src` range to the end of the vector. /// /// # Panics @@ -2504,6 +2577,36 @@ impl Vec { // len set by scope guard } } + + /// Try to extend the vector by `n` values, using the given generator. + fn try_extend_with>(&mut self, n: usize, mut value: E) -> Result<(), TryReserveError> { + self.try_reserve(n)?; + + unsafe { + let mut ptr = self.as_mut_ptr().add(self.len()); + // Use SetLenOnDrop to work around bug where compiler + // might not realize the store through `ptr` through self.set_len() + // don't alias. + let mut local_len = SetLenOnDrop::new(&mut self.len); + + // Write all elements except the last one + for _ in 1..n { + ptr::write(ptr, value.next()); + ptr = ptr.offset(1); + // Increment the length in every step in case next() panics + local_len.increment_len(1); + } + + if n > 0 { + // We can write the last element directly without cloning needlessly + ptr::write(ptr, value.last()); + local_len.increment_len(1); + } + + // len set by scope guard + Ok(()) + } + } } impl Vec { @@ -2838,6 +2941,34 @@ impl Vec { } } + // leaf method to which various SpecFrom/SpecExtend implementations delegate when + // they have no further optimizations to apply + fn try_extend_desugared>(&mut self, mut iterator: I) -> Result<(), TryReserveError> { + // This is the case for a general iterator. + // + // This function should be the moral equivalent of: + // + // for item in iterator { + // self.push(item); + // } + while let Some(element) = iterator.next() { + let len = self.len(); + if len == self.capacity() { + let (lower, _) = iterator.size_hint(); + self.try_reserve(lower.saturating_add(1))?; + } + unsafe { + ptr::write(self.as_mut_ptr().add(len), element); + // Since next() executes user code which can panic we have to bump the length + // after each step. + // NB can't overflow since we would have had to alloc the address space + self.set_len(len + 1); + } + } + + Ok(()) + } + /// Creates a splicing iterator that replaces the specified range in the vector /// with the given `replace_with` iterator and yields the removed items. /// `replace_with` does not need to be the same length as `range`. diff --git a/rust/alloc/vec/spec_extend.rs b/rust/alloc/vec/spec_extend.rs index 7bb3abb7ee7164..5ce2d00991bcdc 100644 --- a/rust/alloc/vec/spec_extend.rs +++ b/rust/alloc/vec/spec_extend.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::alloc::Allocator; +use crate::collections::{TryReserveError, TryReserveErrorKind}; use core::iter::TrustedLen; use core::ptr::{self}; use core::slice::{self}; @@ -8,10 +9,17 @@ use core::slice::{self}; use super::{IntoIter, SetLenOnDrop, Vec}; // Specialization trait used for Vec::extend +#[cfg(not(no_global_oom_handling))] pub(super) trait SpecExtend { fn spec_extend(&mut self, iter: I); } +// Specialization trait used for Vec::try_extend +pub(super) trait TrySpecExtend { + fn try_spec_extend(&mut self, iter: I) -> Result<(), TryReserveError>; +} + +#[cfg(not(no_global_oom_handling))] impl SpecExtend for Vec where I: Iterator, @@ -21,6 +29,16 @@ where } } +impl TrySpecExtend for Vec +where + I: Iterator, +{ + default fn try_spec_extend(&mut self, iter: I) -> Result<(), TryReserveError> { + self.try_extend_desugared(iter) + } +} + +#[cfg(not(no_global_oom_handling))] impl SpecExtend for Vec where I: TrustedLen, @@ -59,6 +77,41 @@ where } } +impl TrySpecExtend for Vec +where + I: TrustedLen, +{ + default fn try_spec_extend(&mut self, iterator: I) -> Result<(), TryReserveError> { + // This is the case for a TrustedLen iterator. + let (low, high) = iterator.size_hint(); + if let Some(additional) = high { + debug_assert_eq!( + low, + additional, + "TrustedLen iterator's size hint is not exact: {:?}", + (low, high) + ); + self.try_reserve(additional)?; + unsafe { + let mut ptr = self.as_mut_ptr().add(self.len()); + let mut local_len = SetLenOnDrop::new(&mut self.len); + iterator.for_each(move |element| { + ptr::write(ptr, element); + ptr = ptr.offset(1); + // Since the loop executes user code which can panic we have to bump the pointer + // after each step. + // NB can't overflow since we would have had to alloc the address space + local_len.increment_len(1); + }); + } + Ok(()) + } else { + Err(TryReserveErrorKind::CapacityOverflow.into()) + } + } +} + +#[cfg(not(no_global_oom_handling))] impl SpecExtend> for Vec { fn spec_extend(&mut self, mut iterator: IntoIter) { unsafe { @@ -68,6 +121,17 @@ impl SpecExtend> for Vec { } } +impl TrySpecExtend> for Vec { + fn try_spec_extend(&mut self, mut iterator: IntoIter) -> Result<(), TryReserveError> { + unsafe { + self.try_append_elements(iterator.as_slice() as _)?; + } + iterator.forget_remaining_elements(); + Ok(()) + } +} + +#[cfg(not(no_global_oom_handling))] impl<'a, T: 'a, I, A: Allocator + 'a> SpecExtend<&'a T, I> for Vec where I: Iterator, @@ -78,6 +142,17 @@ where } } +impl<'a, T: 'a, I, A: Allocator + 'a> TrySpecExtend<&'a T, I> for Vec +where + I: Iterator, + T: Clone, +{ + default fn try_spec_extend(&mut self, iterator: I) -> Result<(), TryReserveError> { + self.try_spec_extend(iterator.cloned()) + } +} + +#[cfg(not(no_global_oom_handling))] impl<'a, T: 'a, A: Allocator + 'a> SpecExtend<&'a T, slice::Iter<'a, T>> for Vec where T: Copy, @@ -87,3 +162,13 @@ where unsafe { self.append_elements(slice) }; } } + +impl<'a, T: 'a, A: Allocator + 'a> TrySpecExtend<&'a T, slice::Iter<'a, T>> for Vec +where + T: Copy, +{ + fn try_spec_extend(&mut self, iterator: slice::Iter<'a, T>) -> Result<(), TryReserveError> { + let slice = iterator.as_slice(); + unsafe { self.try_append_elements(slice) } + } +} From 5a0ce478a35ea3cbc87b710470b3307169fb44e2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 23 Mar 2023 21:35:10 +0900 Subject: [PATCH 394/635] rust: macros: Allow specifying multiple module aliases Modules can (and usually do) have multiple alias tags, in order to specify multiple possible device matches for autoloading. Allow this by changing the alias ModuleInfo field to an Option>. Note: For normal device IDs this is autogenerated by modpost (which is not properly integrated with Rust support yet), so it is useful to be able to manually add device match aliases for now, and should still be useful in the future for corner cases that modpost does not handle. This pulls in the expect_group() helper from the rfl/rust branch (with credit to authors). Co-developed-by: Miguel Ojeda Signed-off-by: Miguel Ojeda Co-developed-by: Finn Behrens Signed-off-by: Finn Behrens Co-developed-by: Sumera Priyadarsini Signed-off-by: Sumera Priyadarsini Reviewed-by: Vincenzo Palazzo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-macros-v2-1-7396e8b7018d@asahilina.net Signed-off-by: Miguel Ojeda --- rust/macros/helpers.rs | 10 +++++++++- rust/macros/module.rs | 30 +++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index cf7ad950dc1e24..b2bdd4d8c958af 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use proc_macro::{token_stream, TokenTree}; +use proc_macro::{token_stream, Group, TokenTree}; pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option { if let Some(TokenTree::Ident(ident)) = it.next() { @@ -56,6 +56,14 @@ pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> String { string } +pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group { + if let TokenTree::Group(group) = it.next().expect("Reached end of token stream for Group") { + group + } else { + panic!("Expected Group"); + } +} + pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { if it.next().is_some() { panic!("Expected end"); diff --git a/rust/macros/module.rs b/rust/macros/module.rs index a7e363c2b04442..5e7ab83908a0df 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -1,9 +1,27 @@ // SPDX-License-Identifier: GPL-2.0 use crate::helpers::*; -use proc_macro::{token_stream, Literal, TokenStream, TokenTree}; +use proc_macro::{token_stream, Delimiter, Literal, TokenStream, TokenTree}; use std::fmt::Write; +fn expect_string_array(it: &mut token_stream::IntoIter) -> Vec { + let group = expect_group(it); + assert_eq!(group.delimiter(), Delimiter::Bracket); + let mut values = Vec::new(); + let mut it = group.stream().into_iter(); + + while let Some(val) = try_string(&mut it) { + assert!(val.is_ascii(), "Expected ASCII string"); + values.push(val); + match it.next() { + Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','), + None => break, + _ => panic!("Expected ',' or end of array"), + } + } + values +} + struct ModInfoBuilder<'a> { module: &'a str, counter: usize, @@ -78,7 +96,7 @@ struct ModuleInfo { name: String, author: Option, description: Option, - alias: Option, + alias: Option>, } impl ModuleInfo { @@ -112,7 +130,7 @@ impl ModuleInfo { "author" => info.author = Some(expect_string(it)), "description" => info.description = Some(expect_string(it)), "license" => info.license = expect_string_ascii(it), - "alias" => info.alias = Some(expect_string_ascii(it)), + "alias" => info.alias = Some(expect_string_array(it)), _ => panic!( "Unknown key \"{}\". Valid keys are: {:?}.", key, EXPECTED_KEYS @@ -163,8 +181,10 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { modinfo.emit("description", &description); } modinfo.emit("license", &info.license); - if let Some(alias) = info.alias { - modinfo.emit("alias", &alias); + if let Some(aliases) = info.alias { + for alias in aliases { + modinfo.emit("alias", &alias); + } } // Built-in modules also export the `file` modinfo string. From aa3e21874335735eec0acd5579286d8d2daa05eb Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 3 Apr 2023 19:01:11 +0900 Subject: [PATCH 395/635] rust: sync: arc: Implement Arc::downcast() This mirrors the standard library's alloc::sync::Arc::downcast(). Based on the Rust standard library implementation, ver 1.62.0, licensed under "Apache-2.0 OR MIT", from: https://github.com/rust-lang/rust/tree/1.62.0/library/alloc/src For copyright details, please see: https://github.com/rust-lang/rust/blob/1.62.0/COPYRIGHT Reviewed-by: Martin Rodriguez Reboredo Reviewed-by: Andreas Hindborg Reviewed-by: Vincenzo Palazzo Reviewed-by: Gary Guo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-arc-v2-1-5c97a865b276@asahilina.net [ Moved `mod std_vendor;` up. ] Signed-off-by: Miguel Ojeda --- rust/kernel/sync/arc.rs | 2 ++ rust/kernel/sync/arc/std_vendor.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 rust/kernel/sync/arc/std_vendor.rs diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index a94e303217c66e..4ab9b280cb3d86 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -30,6 +30,8 @@ use core::{ ptr::NonNull, }; +mod std_vendor; + /// A reference-counted pointer to an instance of `T`. /// /// The reference count is incremented when new instances of [`Arc`] are created, and decremented diff --git a/rust/kernel/sync/arc/std_vendor.rs b/rust/kernel/sync/arc/std_vendor.rs new file mode 100644 index 00000000000000..a66a0c2831b3ed --- /dev/null +++ b/rust/kernel/sync/arc/std_vendor.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . + +use crate::sync::{arc::ArcInner, Arc}; +use core::any::Any; + +impl Arc { + /// Attempt to downcast the `Arc` to a concrete type. + pub fn downcast(self) -> core::result::Result, Self> + where + T: Any + Send + Sync, + { + if (*self).is::() { + // SAFETY: We have just checked that the type is correct, so we can cast the pointer. + unsafe { + let ptr = self.ptr.cast::>(); + core::mem::forget(self); + Ok(Arc::from_inner(ptr)) + } + } else { + Err(self) + } + } +} From d93a69ff57e61b9254325377f33be9f7715035d5 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 3 Apr 2023 19:01:12 +0900 Subject: [PATCH 396/635] rust: sync: arc: Add UniqueArc::assume_init() We can already create `UniqueArc>` instances with `UniqueArc::try_new_uninit()` and write to them with `write()`. Add the missing unsafe `assume_init()` function to promote it to `UniqueArc`, so users can do piece-wise initialization of the contents instead of doing it all at once as long as they keep the invariants (the same requirements as `MaybeUninit::assume_init()`). This mirrors the std `Arc::assume_init()` function. In the kernel, since we have `UniqueArc`, arguably this only belongs there since most use cases will initialize it immediately after creating it, before demoting it to `Arc` to share it. [ Miguel: The "Rust pin-init API for pinned initialization of structs" patch series [1] from Benno Lossin contains a very similar patch: rust: sync: add `assume_init` to `UniqueArc` Adds the `assume_init` function to `UniqueArc>` that unsafely assumes the value to be initialized and yields a value of type `UniqueArc`. This function is used when manually initializing the pointee of an `UniqueArc`. To make that patch a noop and thus drop it, I adjusted the `SAFETY` comment here to be the same as in the current latest version of that series (v7). I have also brought the `Reviewed-by`s there into here, and reworded the `Co-authored-by` into `Co-developed-by`. ] Link: https://lore.kernel.org/r/20230408122429.1103522-5-y86-dev@protonmail.com [1] Co-developed-by: Benno Lossin Signed-off-by: Benno Lossin Signed-off-by: Asahi Lina Reviewed-by: Gary Guo Reviewed-by: Wedson Almeida Filho Reviewed-by: Andreas Hindborg Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20230224-rust-arc-v2-2-5c97a865b276@asahilina.net Signed-off-by: Miguel Ojeda --- rust/kernel/sync/arc.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 4ab9b280cb3d86..8ce6d6bf478e8c 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -492,6 +492,17 @@ impl UniqueArc> { /// Converts a `UniqueArc>` into a `UniqueArc` by writing a value into it. pub fn write(mut self, value: T) -> UniqueArc { self.deref_mut().write(value); + // SAFETY: We just wrote the value to be initialized. + unsafe { self.assume_init() } + } + + /// Unsafely assume that `self` is initialized. + /// + /// # Safety + /// + /// The caller guarantees that the value behind this pointer has been initialized. It is + /// *immediate* UB to call this when the value is not initialized. + pub unsafe fn assume_init(self) -> UniqueArc { let inner = ManuallyDrop::new(self).inner.ptr; UniqueArc { // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be From 00b4c9558ccd6fdeb6e10b65a886029d7fd51e8e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 3 Apr 2023 18:48:10 +0900 Subject: [PATCH 397/635] rust: error: Rename to_kernel_errno() -> to_errno() This is kernel code, so specifying "kernel" is redundant. Let's simplify things and just call it to_errno(). Reviewed-by: Gary Guo Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-error-v3-1-03779bddc02b@asahilina.net Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 2 +- rust/macros/module.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 5b9751d7ff1d9d..35894fa35efe7d 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -73,7 +73,7 @@ pub struct Error(core::ffi::c_int); impl Error { /// Returns the kernel error code. - pub fn to_kernel_errno(self) -> core::ffi::c_int { + pub fn to_errno(self) -> core::ffi::c_int { self.0 } } diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 5e7ab83908a0df..fb1244f8c2e694 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -278,7 +278,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { return 0; }} Err(e) => {{ - return e.to_kernel_errno(); + return e.to_errno(); }} }} }} From 4f69a2c35ff78e9f1c68d6078a5701a16d88fe76 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 3 Apr 2023 18:48:11 +0900 Subject: [PATCH 398/635] rust: error: Add Error::to_ptr() This is the Rust equivalent to ERR_PTR(), for use in C callbacks. Marked as #[allow(dead_code)] for now, since it does not have any consumers yet. Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20230224-rust-error-v3-2-03779bddc02b@asahilina.net Signed-off-by: Miguel Ojeda --- rust/helpers.c | 7 +++++++ rust/kernel/error.rs | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/rust/helpers.c b/rust/helpers.c index 09a4d93f9d62cc..89f4cd1e0df356 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -20,6 +20,7 @@ #include #include +#include #include __noreturn void rust_helper_BUG(void) @@ -46,6 +47,12 @@ bool rust_helper_refcount_dec_and_test(refcount_t *r) } EXPORT_SYMBOL_GPL(rust_helper_refcount_dec_and_test); +__force void *rust_helper_ERR_PTR(long err) +{ + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 35894fa35efe7d..154d0ca6e2dcbf 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -76,6 +76,13 @@ impl Error { pub fn to_errno(self) -> core::ffi::c_int { self.0 } + + /// Returns the error encoded as a pointer. + #[allow(dead_code)] + pub(crate) fn to_ptr(self) -> *mut T { + // SAFETY: self.0 is a valid error due to its invariant. + unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } + } } impl From for Error { From 301b88ac6e94a975719ba2bd47b608b197dd0fd9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Mon, 3 Apr 2023 18:48:12 +0900 Subject: [PATCH 399/635] rust: error: Add Error::from_errno{_unchecked}() Add a function to create `Error` values out of a kernel error return, which safely upholds the invariant that the error code is well-formed (negative and greater than -MAX_ERRNO). If a malformed code is passed in, it will be converted to EINVAL. Lina: Imported from rust-for-linux/rust as authored by Miguel and Fox with refactoring from Wedson, renamed from_kernel_errno() to from_errno(). Co-developed-by: Fox Chen Signed-off-by: Fox Chen Co-developed-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Signed-off-by: Miguel Ojeda Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-error-v3-3-03779bddc02b@asahilina.net [ Mark the new associated functions as `#[allow(dead_code)]`. ] Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 154d0ca6e2dcbf..0223d48ab43dc6 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -72,6 +72,38 @@ pub mod code { pub struct Error(core::ffi::c_int); impl Error { + /// Creates an [`Error`] from a kernel error code. + /// + /// It is a bug to pass an out-of-range `errno`. `EINVAL` would + /// be returned in such a case. + #[allow(dead_code)] + pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error { + if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { + // TODO: Make it a `WARN_ONCE` once available. + crate::pr_warn!( + "attempted to create `Error` with out of range `errno`: {}", + errno + ); + return code::EINVAL; + } + + // INVARIANT: The check above ensures the type invariant + // will hold. + Error(errno) + } + + /// Creates an [`Error`] from a kernel error code. + /// + /// # Safety + /// + /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). + #[allow(dead_code)] + unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error { + // INVARIANT: The contract ensures the type invariant + // will hold. + Error(errno) + } + /// Returns the kernel error code. pub fn to_errno(self) -> core::ffi::c_int { self.0 From 280a0b009338acf96fa851c52c2284825f527331 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Mon, 3 Apr 2023 18:48:13 +0900 Subject: [PATCH 400/635] rust: error: Add to_result() helper Add a to_result() helper to convert kernel C return values to a Rust Result, mapping >=0 values to Ok(()) and negative values to Err(...), with Error::from_errno() ensuring that the errno is within range. Lina: Imported from rust-for-linux/rust, originally developed by Wedson as part of the AMBA device driver support. Signed-off-by: Wedson Almeida Filho Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-error-v3-4-03779bddc02b@asahilina.net [ Add a removal of `#[allow(dead_code)]`. ] Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 0223d48ab43dc6..7e25871063df65 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -76,7 +76,6 @@ impl Error { /// /// It is a bug to pass an out-of-range `errno`. `EINVAL` would /// be returned in such a case. - #[allow(dead_code)] pub(crate) fn from_errno(errno: core::ffi::c_int) -> Error { if errno < -(bindings::MAX_ERRNO as i32) || errno >= 0 { // TODO: Make it a `WARN_ONCE` once available. @@ -180,3 +179,13 @@ impl From for Error { /// it should still be modeled as returning a `Result` rather than /// just an [`Error`]. pub type Result = core::result::Result; + +/// Converts an integer as returned by a C kernel function to an error if it's negative, and +/// `Ok(())` otherwise. +pub fn to_result(err: core::ffi::c_int) -> Result { + if err < 0 { + Err(Error::from_errno(err)) + } else { + Ok(()) + } +} From 24bcea4bcbb5a0c0d765e72842b31e3e357fcdec Mon Sep 17 00:00:00 2001 From: Sven Van Asbroeck Date: Mon, 3 Apr 2023 18:48:14 +0900 Subject: [PATCH 401/635] rust: error: Add a helper to convert a C ERR_PTR to a `Result` Some kernel C API functions return a pointer which embeds an optional `errno`. Callers are supposed to check the returned pointer with `IS_ERR()` and if this returns `true`, retrieve the `errno` using `PTR_ERR()`. Create a Rust helper function to implement the Rust equivalent: transform a `*mut T` to `Result<*mut T>`. Lina: Imported from rust-for-linux/linux, with subsequent refactoring and contributions squashed in and attributed below. Renamed the function to from_err_ptr(). Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Co-developed-by: Miguel Ojeda Signed-off-by: Miguel Ojeda Co-developed-by: Fox Chen Signed-off-by: Fox Chen Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Sven Van Asbroeck Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-error-v3-5-03779bddc02b@asahilina.net [ Add a removal of `#[allow(dead_code)]`. ] Signed-off-by: Miguel Ojeda --- rust/helpers.c | 12 +++++++++++ rust/kernel/error.rs | 50 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/rust/helpers.c b/rust/helpers.c index 89f4cd1e0df356..04b9be46e88778 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -53,6 +53,18 @@ __force void *rust_helper_ERR_PTR(long err) } EXPORT_SYMBOL_GPL(rust_helper_ERR_PTR); +bool rust_helper_IS_ERR(__force const void *ptr) +{ + return IS_ERR(ptr); +} +EXPORT_SYMBOL_GPL(rust_helper_IS_ERR); + +long rust_helper_PTR_ERR(__force const void *ptr) +{ + return PTR_ERR(ptr); +} +EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 7e25871063df65..67637b87426769 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -96,7 +96,6 @@ impl Error { /// # Safety /// /// `errno` must be within error code range (i.e. `>= -MAX_ERRNO && < 0`). - #[allow(dead_code)] unsafe fn from_errno_unchecked(errno: core::ffi::c_int) -> Error { // INVARIANT: The contract ensures the type invariant // will hold. @@ -189,3 +188,52 @@ pub fn to_result(err: core::ffi::c_int) -> Result { Ok(()) } } + +/// Transform a kernel "error pointer" to a normal pointer. +/// +/// Some kernel C API functions return an "error pointer" which optionally +/// embeds an `errno`. Callers are supposed to check the returned pointer +/// for errors. This function performs the check and converts the "error pointer" +/// to a normal pointer in an idiomatic fashion. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::from_err_ptr; +/// # use kernel::bindings; +/// fn devm_platform_ioremap_resource( +/// pdev: &mut PlatformDevice, +/// index: u32, +/// ) -> Result<*mut core::ffi::c_void> { +/// // SAFETY: FFI call. +/// unsafe { +/// from_err_ptr(bindings::devm_platform_ioremap_resource( +/// pdev.to_ptr(), +/// index, +/// )) +/// } +/// } +/// ``` +// TODO: Remove `dead_code` marker once an in-kernel client is available. +#[allow(dead_code)] +pub(crate) fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { + // CAST: Casting a pointer to `*const core::ffi::c_void` is always valid. + let const_ptr: *const core::ffi::c_void = ptr.cast(); + // SAFETY: The FFI function does not deref the pointer. + if unsafe { bindings::IS_ERR(const_ptr) } { + // SAFETY: The FFI function does not deref the pointer. + let err = unsafe { bindings::PTR_ERR(const_ptr) }; + // CAST: If `IS_ERR()` returns `true`, + // then `PTR_ERR()` is guaranteed to return a + // negative value greater-or-equal to `-bindings::MAX_ERRNO`, + // which always fits in an `i16`, as per the invariant above. + // And an `i16` always fits in an `i32`. So casting `err` to + // an `i32` can never overflow, and is always valid. + // + // SAFETY: `IS_ERR()` ensures `err` is a + // negative value greater-or-equal to `-bindings::MAX_ERRNO`. + #[allow(clippy::unnecessary_cast)] + return Err(unsafe { Error::from_errno_unchecked(err as core::ffi::c_int) }); + } + Ok(ptr) +} From 19909fc394bf9cd0fde9deed8385239ec8c8fe37 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Mon, 3 Apr 2023 18:48:15 +0900 Subject: [PATCH 402/635] rust: error: Add from_result() helper Add a helper function to easily return C result codes from a Rust function that calls functions which return a Result. Lina: Imported from rust-for-linux/rust, originally developed by Wedson as part of file_operations.rs. Added the allow() flags since there is no user in the kernel crate yet and fixed a typo in a comment. Replaced the macro with a function taking a closure, per discussion on the ML. Co-developed-by: Fox Chen Signed-off-by: Fox Chen Co-developed-by: Miguel Ojeda Signed-off-by: Miguel Ojeda Signed-off-by: Wedson Almeida Filho Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230224-rust-error-v3-6-03779bddc02b@asahilina.net Signed-off-by: Miguel Ojeda --- rust/kernel/error.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 67637b87426769..5f4114b30b94c7 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -237,3 +237,42 @@ pub(crate) fn from_err_ptr(ptr: *mut T) -> Result<*mut T> { } Ok(ptr) } + +/// Calls a closure returning a [`crate::error::Result`] and converts the result to +/// a C integer result. +/// +/// This is useful when calling Rust functions that return [`crate::error::Result`] +/// from inside `extern "C"` functions that need to return an integer error result. +/// +/// `T` should be convertible from an `i16` via `From`. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::from_result; +/// # use kernel::bindings; +/// unsafe extern "C" fn probe_callback( +/// pdev: *mut bindings::platform_device, +/// ) -> core::ffi::c_int { +/// from_result(|| { +/// let ptr = devm_alloc(pdev)?; +/// bindings::platform_set_drvdata(pdev, ptr); +/// Ok(0) +/// }) +/// } +/// ``` +// TODO: Remove `dead_code` marker once an in-kernel client is available. +#[allow(dead_code)] +pub(crate) fn from_result(f: F) -> T +where + T: From, + F: FnOnce() -> Result, +{ + match f() { + Ok(v) => v, + // NO-OVERFLOW: negative `errno`s are no smaller than `-bindings::MAX_ERRNO`, + // `-bindings::MAX_ERRNO` fits in an `i16` as per invariant above, + // therefore a negative `errno` always fits in an `i16` and will not overflow. + Err(e) => T::from(e.to_errno() as i16), + } +} From 878d67bb895fe459fda7248e1d24c538038ea851 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:25:18 +0000 Subject: [PATCH 403/635] rust: enable the `pin_macro` feature This feature enables the use of the `pin!` macro for the `stack_pin_init!` macro. This feature is already stabilized in Rust version 1.68. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Acked-by: Boqun Feng Link: https://lore.kernel.org/r/20230408122429.1103522-2-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 + scripts/Makefile.build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 1118cd3e0b5f52..518559a0767ed2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -18,6 +18,7 @@ #![feature(dispatch_from_dyn)] #![feature(generic_associated_types)] #![feature(new_uninit)] +#![feature(pin_macro)] #![feature(receiver_trait)] #![feature(unsize)] diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 1364e3d905fc62..da70f68ba9e447 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := core_ffi_c,new_uninit +rust_allowed_features := core_ffi_c,new_uninit,pin_macro rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ From 864b97e30dcb09a9d5988f468dba1d6ee39817f6 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Sat, 8 Apr 2023 12:25:23 +0000 Subject: [PATCH 404/635] rust: macros: add `quote!` macro Add the `quote!` macro for creating `TokenStream`s directly via the given Rust tokens. It also supports repetitions using iterators. It will be used by the pin-init API proc-macros to generate code. Signed-off-by: Gary Guo Signed-off-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20230408122429.1103522-3-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/macros/lib.rs | 2 + rust/macros/quote.rs | 145 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 rust/macros/quote.rs diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index c1d385e345b96d..82b520f024dd7d 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -2,6 +2,8 @@ //! Crate for all kernel procedural macros. +#[macro_use] +mod quote; mod concat_idents; mod helpers; mod module; diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs new file mode 100644 index 00000000000000..94a6277182ee6b --- /dev/null +++ b/rust/macros/quote.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro::{TokenStream, TokenTree}; + +pub(crate) trait ToTokens { + fn to_tokens(&self, tokens: &mut TokenStream); +} + +impl ToTokens for Option { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let Some(v) = self { + v.to_tokens(tokens); + } + } +} + +impl ToTokens for proc_macro::Group { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend([TokenTree::from(self.clone())]); + } +} + +impl ToTokens for TokenTree { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend([self.clone()]); + } +} + +impl ToTokens for TokenStream { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(self.clone()); + } +} + +/// Converts tokens into [`proc_macro::TokenStream`] and performs variable interpolations with +/// the given span. +/// +/// This is a similar to the +/// [`quote_spanned!`](https://docs.rs/quote/latest/quote/macro.quote_spanned.html) macro from the +/// `quote` crate but provides only just enough functionality needed by the current `macros` crate. +#[allow(unused_macros)] +macro_rules! quote_spanned { + ($span:expr => $($tt:tt)*) => { + #[allow(clippy::vec_init_then_push)] + { + let mut tokens = ::std::vec::Vec::new(); + let span = $span; + quote_spanned!(@proc tokens span $($tt)*); + ::proc_macro::TokenStream::from_iter(tokens) + }}; + (@proc $v:ident $span:ident) => {}; + (@proc $v:ident $span:ident #$id:ident $($tt:tt)*) => { + let mut ts = ::proc_macro::TokenStream::new(); + $crate::quote::ToTokens::to_tokens(&$id, &mut ts); + $v.extend(ts); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident #(#$id:ident)* $($tt:tt)*) => { + for token in $id { + let mut ts = ::proc_macro::TokenStream::new(); + $crate::quote::ToTokens::to_tokens(&token, &mut ts); + $v.extend(ts); + } + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident ( $($inner:tt)* ) $($tt:tt)*) => { + let mut tokens = ::std::vec::Vec::new(); + quote_spanned!(@proc tokens $span $($inner)*); + $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( + ::proc_macro::Delimiter::Parenthesis, + ::proc_macro::TokenStream::from_iter(tokens) + ))); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident [ $($inner:tt)* ] $($tt:tt)*) => { + let mut tokens = ::std::vec::Vec::new(); + quote_spanned!(@proc tokens $span $($inner)*); + $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( + ::proc_macro::Delimiter::Bracket, + ::proc_macro::TokenStream::from_iter(tokens) + ))); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident { $($inner:tt)* } $($tt:tt)*) => { + let mut tokens = ::std::vec::Vec::new(); + quote_spanned!(@proc tokens $span $($inner)*); + $v.push(::proc_macro::TokenTree::Group(::proc_macro::Group::new( + ::proc_macro::Delimiter::Brace, + ::proc_macro::TokenStream::from_iter(tokens) + ))); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident :: $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Joint) + )); + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident : $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new(':', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident , $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new(',', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident @ $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new('@', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident ! $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new('!', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; + (@proc $v:ident $span:ident $id:ident $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new(stringify!($id), $span))); + quote_spanned!(@proc $v $span $($tt)*); + }; +} + +/// Converts tokens into [`proc_macro::TokenStream`] and performs variable interpolations with +/// mixed site span ([`Span::mixed_site()`]). +/// +/// This is a similar to the [`quote!`](https://docs.rs/quote/latest/quote/macro.quote.html) macro +/// from the `quote` crate but provides only just enough functionality needed by the current +/// `macros` crate. +/// +/// [`Span::mixed_site()`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.mixed_site +#[allow(unused_macros)] +macro_rules! quote { + ($($tt:tt)*) => { + quote_spanned!(::proc_macro::Span::mixed_site() => $($tt)*) + } +} From 3b33667cb9bdfd2043bfb8723d902a4a1d34b62b Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:25:29 +0000 Subject: [PATCH 405/635] rust: sync: change error type of constructor functions Change the error type of the constructors of `Arc` and `UniqueArc` to be `AllocError` instead of `Error`. This makes the API more clear as to what can go wrong when calling `try_new` or its variants. Signed-off-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20230408122429.1103522-4-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/sync/arc.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 8ce6d6bf478e8c..34d0e7cbe62e59 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -17,11 +17,11 @@ use crate::{ bindings, - error::Result, types::{ForeignOwnable, Opaque}, }; use alloc::boxed::Box; use core::{ + alloc::AllocError, fmt, marker::{PhantomData, Unsize}, mem::{ManuallyDrop, MaybeUninit}, @@ -152,7 +152,7 @@ unsafe impl Sync for Arc {} impl Arc { /// Constructs a new reference counted instance of `T`. - pub fn try_new(contents: T) -> Result { + pub fn try_new(contents: T) -> Result { // INVARIANT: The refcount is initialised to a non-zero value. let value = ArcInner { // SAFETY: There are no safety requirements for this FFI call. @@ -472,7 +472,7 @@ pub struct UniqueArc { impl UniqueArc { /// Tries to allocate a new [`UniqueArc`] instance. - pub fn try_new(value: T) -> Result { + pub fn try_new(value: T) -> Result { Ok(Self { // INVARIANT: The newly-created object has a ref-count of 1. inner: Arc::try_new(value)?, @@ -480,7 +480,7 @@ impl UniqueArc { } /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. - pub fn try_new_uninit() -> Result>> { + pub fn try_new_uninit() -> Result>, AllocError> { Ok(UniqueArc::> { // INVARIANT: The newly-created object has a ref-count of 1. inner: Arc::try_new(MaybeUninit::uninit())?, From 648c44a53701c81629b904af9b5291eff0b3194e Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:25:39 +0000 Subject: [PATCH 406/635] rust: types: add `Opaque::raw_get` This function mirrors `UnsafeCell::raw_get`. It avoids creating a reference and allows solely using raw pointers. The `pin-init` API will be using this, since uninitialized memory requires raw pointers. Signed-off-by: Benno Lossin Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20230408122429.1103522-6-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/types.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 9d0fdbc558433e..ff2b2fac951d69 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -238,6 +238,14 @@ impl Opaque { pub fn get(&self) -> *mut T { UnsafeCell::raw_get(self.0.as_ptr()) } + + /// Gets the value behind `this`. + /// + /// This function is useful to get access to the value without creating intermediate + /// references. + pub const fn raw_get(this: *const Self) -> *mut T { + UnsafeCell::raw_get(this.cast::>()) + } } /// A sum type that always holds either a value of type `L` or `R`. From f32888adb4b79e57f838c64da42568730ae5ef2a Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:25:45 +0000 Subject: [PATCH 407/635] rust: add pin-init API core This API is used to facilitate safe pinned initialization of structs. It replaces cumbersome `unsafe` manual initialization with elegant safe macro invocations. Due to the size of this change it has been split into six commits: 1. This commit introducing the basic public interface: traits and functions to represent and create initializers. 2. Adds the `#[pin_data]`, `pin_init!`, `try_pin_init!`, `init!` and `try_init!` macros along with their internal types. 3. Adds the `InPlaceInit` trait that allows using an initializer to create an object inside of a `Box` and other smart pointers. 4. Adds the `PinnedDrop` trait and adds macro support for it in the `#[pin_data]` macro. 5. Adds the `stack_pin_init!` macro allowing to pin-initialize a struct on the stack. 6. Adds the `Zeroable` trait and `init::zeroed` function to initialize types that have `0x00` in all bytes as a valid bit pattern. -- In this section the problem that the new pin-init API solves is outlined. This message describes the entirety of the API, not just the parts introduced in this commit. For a more granular explanation and additional information on pinning and this issue, view [1]. Pinning is Rust's way of enforcing the address stability of a value. When a value gets pinned it will be impossible for safe code to move it to another location. This is done by wrapping pointers to said object with `Pin

`. This wrapper prevents safe code from creating mutable references to the object, preventing mutable access, which is needed to move the value. `Pin

` provides `unsafe` functions to circumvent this and allow modifications regardless. It is then the programmer's responsibility to uphold the pinning guarantee. Many kernel data structures require a stable address, because there are foreign pointers to them which would get invalidated by moving the structure. Since these data structures are usually embedded in structs to use them, this pinning property propagates to the container struct. Resulting in most structs in both Rust and C code needing to be pinned. So if we want to have a `mutex` field in a Rust struct, this struct also needs to be pinned, because a `mutex` contains a `list_head`. Additionally initializing a `list_head` requires already having the final memory location available, because it is initialized by pointing it to itself. But this presents another challenge in Rust: values have to be initialized at all times. There is the `MaybeUninit` wrapper type, which allows handling uninitialized memory, but this requires using the `unsafe` raw pointers and a casting the type to the initialized variant. This problem gets exacerbated when considering encapsulation and the normal safety requirements of Rust code. The fields of the Rust `Mutex` should not be accessible to normal driver code. After all if anyone can modify the fields, there is no way to ensure the invariants of the `Mutex` are upheld. But if the fields are inaccessible, then initialization of a `Mutex` needs to be somehow achieved via a function or a macro. Because the `Mutex` must be pinned in memory, the function cannot return it by value. It also cannot allocate a `Box` to put the `Mutex` into, because that is an unnecessary allocation and indirection which would hurt performance. The solution in the rust tree (e.g. this commit: [2]) that is replaced by this API is to split this function into two parts: 1. A `new` function that returns a partially initialized `Mutex`, 2. An `init` function that requires the `Mutex` to be pinned and that fully initializes the `Mutex`. Both of these functions have to be marked `unsafe`, since a call to `new` needs to be accompanied with a call to `init`, otherwise using the `Mutex` could result in UB. And because calling `init` twice also is not safe. While `Mutex` initialization cannot fail, other structs might also have to allocate memory, which would result in conditional successful initialization requiring even more manual accommodation work. Combine this with the problem of pin-projections -- the way of accessing fields of a pinned struct -- which also have an `unsafe` API, pinned initialization is riddled with `unsafe` resulting in very poor ergonomics. Not only that, but also having to call two functions possibly multiple lines apart makes it very easy to forget it outright or during refactoring. Here is an example of the current way of initializing a struct with two synchronization primitives (see [3] for the full example): struct SharedState { state_changed: CondVar, inner: Mutex, } impl SharedState { fn try_new() -> Result> { let mut state = Pin::from(UniqueArc::try_new(Self { // SAFETY: `condvar_init!` is called below. state_changed: unsafe { CondVar::new() }, // SAFETY: `mutex_init!` is called below. inner: unsafe { Mutex::new(SharedStateInner { token_count: 0 }) }, })?); // SAFETY: `state_changed` is pinned when `state` is. let pinned = unsafe { state.as_mut().map_unchecked_mut(|s| &mut s.state_changed) }; kernel::condvar_init!(pinned, "SharedState::state_changed"); // SAFETY: `inner` is pinned when `state` is. let pinned = unsafe { state.as_mut().map_unchecked_mut(|s| &mut s.inner) }; kernel::mutex_init!(pinned, "SharedState::inner"); Ok(state.into()) } } The pin-init API of this patch solves this issue by providing a comprehensive solution comprised of macros and traits. Here is the example from above using the pin-init API: #[pin_data] struct SharedState { #[pin] state_changed: CondVar, #[pin] inner: Mutex, } impl SharedState { fn new() -> impl PinInit { pin_init!(Self { state_changed <- new_condvar!("SharedState::state_changed"), inner <- new_mutex!( SharedStateInner { token_count: 0 }, "SharedState::inner", ), }) } } Notably the way the macro is used here requires no `unsafe` and thus comes with the usual Rust promise of safe code not introducing any memory violations. Additionally it is now up to the caller of `new()` to decide the memory location of the `SharedState`. They can choose at the moment `Arc`, `Box` or the stack. -- The API has the following architecture: 1. Initializer traits `PinInit` and `Init` that act like closures. 2. Macros to create these initializer traits safely. 3. Functions to allow manually writing initializers. The initializers (an `impl PinInit`) receive a raw pointer pointing to uninitialized memory and their job is to fully initialize a `T` at that location. If initialization fails, they return an error (`E`) by value. This way of initializing cannot be safely exposed to the user, since it relies upon these properties outside of the control of the trait: - the memory location (slot) needs to be valid memory, - if initialization fails, the slot should not be read from, - the value in the slot should be pinned, so it cannot move and the memory cannot be deallocated until the value is dropped. This is why using an initializer is facilitated by another trait that ensures these requirements. These initializers can be created manually by just supplying a closure that fulfills the same safety requirements as `PinInit`. But this is an `unsafe` operation. To allow safe initializer creation, the `pin_init!` is provided along with three other variants: `try_pin_init!`, `try_init!` and `init!`. These take a modified struct initializer as a parameter and generate a closure that initializes the fields in sequence. The macros take great care in upholding the safety requirements: - A shadowed struct type is used as the return type of the closure instead of `()`. This is to prevent early returns, as these would prevent full initialization. - To ensure every field is only initialized once, a normal struct initializer is placed in unreachable code. The type checker will emit errors if a field is missing or specified multiple times. - When initializing a field fails, the whole initializer will fail and automatically drop fields that have been initialized earlier. - Only the correct initializer type is allowed for unpinned fields. You cannot use a `impl PinInit` to initialize a structurally not pinned field. To ensure the last point, an additional macro `#[pin_data]` is needed. This macro annotates the struct itself and the user specifies structurally pinned and not pinned fields. Because dropping a pinned struct is also not allowed to break the pinning invariants, another macro attribute `#[pinned_drop]` is needed. This macro is introduced in a following commit. These two macros also have mechanisms to ensure the overall safety of the API. Additionally, they utilize a combined proc-macro, declarative macro design: first a proc-macro enables the outer attribute syntax `#[...]` and does some important pre-parsing. Notably this prepares the generics such that the declarative macro can handle them using token trees. Then the actual parsing of the structure and the emission of code is handled by a declarative macro. For pin-projections the crates `pin-project` [4] and `pin-project-lite` [5] had been considered, but were ultimately rejected: - `pin-project` depends on `syn` [6] which is a very big dependency, around 50k lines of code. - `pin-project-lite` is a more reasonable 5k lines of code, but contains a very complex declarative macro to parse generics. On top of that it would require modification that would need to be maintained independently. Link: https://rust-for-linux.com/the-safe-pinned-initialization-problem [1] Link: https://github.com/Rust-for-Linux/linux/tree/0a04dc4ddd671efb87eef54dde0fb38e9074f4be [2] Link: https://github.com/Rust-for-Linux/linux/blob/f509ede33fc10a07eba3da14aa00302bd4b5dddd/samples/rust/rust_miscdev.rs [3] Link: https://crates.io/crates/pin-project [4] Link: https://crates.io/crates/pin-project-lite [5] Link: https://crates.io/crates/syn [6] Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Wedson Almeida Filho Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-7-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 187 +++++++++++++++++++++++++++++++++ rust/kernel/init/__internal.rs | 33 ++++++ rust/kernel/lib.rs | 6 ++ scripts/Makefile.build | 2 +- 4 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/init.rs create mode 100644 rust/kernel/init/__internal.rs diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs new file mode 100644 index 00000000000000..d041f0daf71e91 --- /dev/null +++ b/rust/kernel/init.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! API to safely and fallibly initialize pinned `struct`s using in-place constructors. +//! +//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack +//! overflow. +//! +//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential +//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. +//! +//! # Overview +//! +//! To initialize a `struct` with an in-place constructor you will need two things: +//! - an in-place constructor, +//! - a memory location that can hold your `struct`. +//! +//! To get an in-place constructor there are generally two options: +//! - a custom function/macro returning an in-place constructor provided by someone else, +//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. +//! +//! Aside from pinned initialization, this API also supports in-place construction without pinning, +//! the macros/types/functions are generally named like the pinned variants without the `pin` +//! prefix. +//! +//! [`sync`]: kernel::sync +//! [pinning]: https://doc.rust-lang.org/std/pin/index.html +//! [structurally pinned fields]: +//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field +//! [`Arc`]: crate::sync::Arc +//! [`impl PinInit`]: PinInit +//! [`impl PinInit`]: PinInit +//! [`impl Init`]: Init +//! [`Opaque`]: kernel::types::Opaque +//! [`pin_data`]: ::macros::pin_data +//! [`UniqueArc`]: kernel::sync::UniqueArc +//! [`Box`]: alloc::boxed::Box + +use core::{convert::Infallible, marker::PhantomData, mem::MaybeUninit}; + +#[doc(hidden)] +pub mod __internal; + +/// A pin-initializer for the type `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box`], [`Arc`], [`UniqueArc`]. +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this type you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. +/// +/// The [`PinInit::__pinned_init`] function +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +/// [`Arc`]: crate::sync::Arc +/// [`Arc::pin_init`]: crate::sync::Arc::pin_init +/// [`UniqueArc`]: kernel::sync::UniqueArc +/// [`Box`]: alloc::boxed::Box +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait PinInit: Sized { + /// Initializes `slot`. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + /// - `slot` will not move until it is dropped, i.e. it will be pinned. + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; +} + +/// An initializer for `T`. +/// +/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can +/// be [`Box`], [`Arc`], [`UniqueArc`]. Because [`PinInit`] is a super trait, you can +/// use every function that takes it as well. +/// +/// Also see the [module description](self). +/// +/// # Safety +/// +/// When implementing this type you will need to take great care. Also there are probably very few +/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. +/// +/// The [`Init::__init`] function +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +/// +/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same +/// code as `__init`. +/// +/// Contrary to its supertype [`PinInit`] the caller is allowed to +/// move the pointee after initialization. +/// +/// [`Arc`]: crate::sync::Arc +/// [`UniqueArc`]: kernel::sync::UniqueArc +/// [`Box`]: alloc::boxed::Box +#[must_use = "An initializer must be used in order to create its value."] +pub unsafe trait Init: Sized { + /// Initializes `slot`. + /// + /// # Safety + /// + /// - `slot` is a valid pointer to uninitialized memory. + /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to + /// deallocate. + unsafe fn __init(self, slot: *mut T) -> Result<(), E>; +} + +// SAFETY: Every in-place initializer can also be used as a pin-initializer. +unsafe impl PinInit for I +where + I: Init, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not + // require `slot` to not move after init. + unsafe { self.__init(slot) } + } +} + +/// Creates a new [`PinInit`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - may assume that the `slot` does not move if `T: !Unpin`, +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn pin_init_from_closure( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl PinInit { + __internal::InitClosure(f, PhantomData) +} + +/// Creates a new [`Init`] from the given closure. +/// +/// # Safety +/// +/// The closure: +/// - returns `Ok(())` if it initialized every field of `slot`, +/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: +/// - `slot` can be deallocated without UB occurring, +/// - `slot` does not need to be dropped, +/// - `slot` is not partially initialized. +/// - the `slot` may move after initialization. +/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. +#[inline] +pub const unsafe fn init_from_closure( + f: impl FnOnce(*mut T) -> Result<(), E>, +) -> impl Init { + __internal::InitClosure(f, PhantomData) +} + +/// An initializer that leaves the memory uninitialized. +/// +/// The initializer is a no-op. The `slot` memory is not changed. +#[inline] +pub fn uninit() -> impl Init, E> { + // SAFETY: The memory is allowed to be uninitialized. + unsafe { init_from_closure(|_| Ok(())) } +} + +// SAFETY: Every type can be initialized by-value. +unsafe impl Init for T { + unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { + unsafe { slot.write(self) }; + Ok(()) + } +} diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs new file mode 100644 index 00000000000000..08cbb533343878 --- /dev/null +++ b/rust/kernel/init/__internal.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! This module contains API-internal items for pin-init. +//! +//! These items must not be used outside of +//! - `kernel/init.rs` +//! - `macros/pin_data.rs` +//! - `macros/pinned_drop.rs` + +use super::*; + +/// See the [nomicon] for what subtyping is. See also [this table]. +/// +/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html +/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns +type Invariant = PhantomData *mut T>; + +/// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this +/// type, since the closure needs to fulfill the same safety requirement as the +/// `__pinned_init`/`__init` functions. +pub(crate) struct InitClosure(pub(crate) F, pub(crate) Invariant<(E, T)>); + +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__init` invariants. +unsafe impl Init for InitClosure +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 518559a0767ed2..821bd067151ce7 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,7 @@ #![feature(coerce_unsized)] #![feature(core_ffi_c)] #![feature(dispatch_from_dyn)] +#![feature(explicit_generic_args_with_impl_trait)] #![feature(generic_associated_types)] #![feature(new_uninit)] #![feature(pin_macro)] @@ -27,11 +28,16 @@ #[cfg(not(CONFIG_RUST))] compile_error!("Missing kernel configuration for conditional compilation"); +#[allow(unused_extern_crates)] +// Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). +extern crate self as kernel; + #[cfg(not(test))] #[cfg(not(testlib))] mod allocator; mod build_assert; pub mod error; +pub mod init; pub mod prelude; pub mod print; mod static_assert; diff --git a/scripts/Makefile.build b/scripts/Makefile.build index da70f68ba9e447..9f94fc83f08652 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := core_ffi_c,new_uninit,pin_macro +rust_allowed_features := core_ffi_c,explicit_generic_args_with_impl_trait,new_uninit,pin_macro rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ From 695729083bb9fe1f6a9b1a397164c44ab0476c2d Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:25:51 +0000 Subject: [PATCH 408/635] rust: init: add initialization macros Add the following initializer macros: - `#[pin_data]` to annotate structurally pinned fields of structs, needed for `pin_init!` and `try_pin_init!` to select the correct initializer of fields. - `pin_init!` create a pin-initializer for a struct with the `Infallible` error type. - `try_pin_init!` create a pin-initializer for a struct with a custom error type (`kernel::error::Error` is the default). - `init!` create an in-place-initializer for a struct with the `Infallible` error type. - `try_init!` create an in-place-initializer for a struct with a custom error type (`kernel::error::Error` is the default). Also add their needed internal helper traits and structs. Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-8-y86-dev@protonmail.com [ Fixed three typos. ] Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 807 ++++++++++++++++++++++++++++++++- rust/kernel/init/__internal.rs | 130 ++++++ rust/kernel/init/macros.rs | 707 +++++++++++++++++++++++++++++ rust/macros/lib.rs | 29 ++ rust/macros/pin_data.rs | 79 ++++ rust/macros/quote.rs | 2 - 6 files changed, 1747 insertions(+), 7 deletions(-) create mode 100644 rust/kernel/init/macros.rs create mode 100644 rust/macros/pin_data.rs diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index d041f0daf71e91..ecef0376d7260a 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -14,7 +14,8 @@ //! - an in-place constructor, //! - a memory location that can hold your `struct`. //! -//! To get an in-place constructor there are generally two options: +//! To get an in-place constructor there are generally three options: +//! - directly creating an in-place constructor using the [`pin_init!`] macro, //! - a custom function/macro returning an in-place constructor provided by someone else, //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. //! @@ -22,6 +23,87 @@ //! the macros/types/functions are generally named like the pinned variants without the `pin` //! prefix. //! +//! # Examples +//! +//! ## Using the [`pin_init!`] macro +//! +//! If you want to use [`PinInit`], then you will have to annotate your `struct` with +//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for +//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via +//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is +//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. +//! +//! ```rust +//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +//! use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! # use core::pin::Pin; +//! #[pin_data] +//! struct Foo { +//! #[pin] +//! a: Mutex, +//! b: u32, +//! } +//! +//! let foo = pin_init!(Foo { +//! a <- new_mutex!(42, "Foo::a"), +//! b: 24, +//! }); +//! ``` +//! +//! `foo` now is of the type [`impl PinInit`]. We can now use any smart pointer that we like +//! (or just the stack) to actually initialize a `Foo`: +//! +//! ```rust +//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +//! # use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! # use core::pin::Pin; +//! # #[pin_data] +//! # struct Foo { +//! # #[pin] +//! # a: Mutex, +//! # b: u32, +//! # } +//! # let foo = pin_init!(Foo { +//! # a <- new_mutex!(42, "Foo::a"), +//! # b: 24, +//! # }); +//! let foo: Result>> = Box::pin_init(foo); +//! ``` +//! +//! For more information see the [`pin_init!`] macro. +//! +//! ## Using a custom function/macro that returns an initializer +//! +//! Many types from the kernel supply a function/macro that returns an initializer, because the +//! above method only works for types where you can access the fields. +//! +//! ```rust +//! # use kernel::{new_mutex, sync::{Arc, Mutex}}; +//! let mtx: Result>> = Arc::pin_init(new_mutex!(42, "example::mtx")); +//! ``` +//! +//! To declare an init macro/function you just return an [`impl PinInit`]: +//! +//! ```rust +//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +//! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; +//! #[pin_data] +//! struct DriverData { +//! #[pin] +//! status: Mutex, +//! buffer: Box<[u8; 1_000_000]>, +//! } +//! +//! impl DriverData { +//! fn new() -> impl PinInit { +//! try_pin_init!(Self { +//! status <- new_mutex!(0, "DriverData::status"), +//! buffer: Box::init(kernel::init::zeroed())?, +//! }) +//! } +//! } +//! ``` +//! //! [`sync`]: kernel::sync //! [pinning]: https://doc.rust-lang.org/std/pin/index.html //! [structurally pinned fields]: @@ -33,12 +115,729 @@ //! [`Opaque`]: kernel::types::Opaque //! [`pin_data`]: ::macros::pin_data //! [`UniqueArc`]: kernel::sync::UniqueArc -//! [`Box`]: alloc::boxed::Box -use core::{convert::Infallible, marker::PhantomData, mem::MaybeUninit}; +use alloc::boxed::Box; +use core::{cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, ptr}; #[doc(hidden)] pub mod __internal; +#[doc(hidden)] +pub mod macros; + +/// Construct an in-place, pinned initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use +/// [`try_pin_init!`]. +/// +/// The syntax is almost identical to that of a normal `struct` initializer: +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, macros::pin_data, init::*}; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// a: usize, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// # fn demo() -> impl PinInit { +/// let a = 42; +/// +/// let initializer = pin_init!(Foo { +/// a, +/// b: Bar { +/// x: 64, +/// }, +/// }); +/// # initializer } +/// # Box::pin_init(demo()).unwrap(); +/// ``` +/// +/// Arbitrary Rust expressions can be used to set the value of a variable. +/// +/// The fields are initialized in the order that they appear in the initializer. So it is possible +/// to read already initialized fields using raw pointers. +/// +/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the +/// initializer. +/// +/// # Init-functions +/// +/// When working with this API it is often desired to let others construct your types without +/// giving access to all fields. This is where you would normally write a plain function `new` +/// that would return a new instance of your type. With this API that is also possible. +/// However, there are a few extra things to keep in mind. +/// +/// To create an initializer function, simply declare it like this: +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, prelude::*, init::*}; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// impl Foo { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// a: 42, +/// b: Bar { +/// x: 64, +/// }, +/// }) +/// } +/// } +/// ``` +/// +/// Users of `Foo` can now create it like this: +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, macros::pin_data, init::*}; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// let foo = Box::pin_init(Foo::new()); +/// ``` +/// +/// They can also easily embed it into their own `struct`s: +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, macros::pin_data, init::*}; +/// # use core::pin::Pin; +/// # #[pin_data] +/// # struct Foo { +/// # a: usize, +/// # b: Bar, +/// # } +/// # #[pin_data] +/// # struct Bar { +/// # x: u32, +/// # } +/// # impl Foo { +/// # fn new() -> impl PinInit { +/// # pin_init!(Self { +/// # a: 42, +/// # b: Bar { +/// # x: 64, +/// # }, +/// # }) +/// # } +/// # } +/// #[pin_data] +/// struct FooContainer { +/// #[pin] +/// foo1: Foo, +/// #[pin] +/// foo2: Foo, +/// other: u32, +/// } +/// +/// impl FooContainer { +/// fn new(other: u32) -> impl PinInit { +/// pin_init!(Self { +/// foo1 <- Foo::new(), +/// foo2 <- Foo::new(), +/// other, +/// }) +/// } +/// } +/// ``` +/// +/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. +/// This signifies that the given field is initialized in-place. As with `struct` initializers, just +/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. +/// +/// # Syntax +/// +/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with +/// the following modifications is expected: +/// - Fields that you want to initialize in-place have to use `<-` instead of `:`. +/// - In front of the initializer you can write `&this in` to have access to a [`NonNull`] +/// pointer named `this` inside of the initializer. +/// +/// For instance: +/// +/// ```rust +/// # use kernel::pin_init; +/// # use macros::pin_data; +/// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; +/// #[pin_data] +/// struct Buf { +/// // `ptr` points into `buf`. +/// ptr: *mut u8, +/// buf: [u8; 64], +/// #[pin] +/// pin: PhantomPinned, +/// } +/// pin_init!(&this in Buf { +/// buf: [0; 64], +/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, +/// pin: PhantomPinned, +/// }); +/// ``` +/// +/// [`try_pin_init!`]: kernel::try_pin_init +/// [`NonNull`]: core::ptr::NonNull +/// [`Error`]: kernel::error::Error +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `__internal` inside of `init/__internal.rs`. +#[macro_export] +macro_rules! pin_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_pin_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error(::core::convert::Infallible), + ) + }; +} + +/// Construct an in-place, fallible pinned initializer for `struct`s. +/// +/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. +/// +/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop +/// initialization and return the error. +/// +/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when +/// initialization fails, the memory can be safely deallocated without any further modifications. +/// +/// This macro defaults the error to [`Error`]. +/// +/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` +/// after the `struct` initializer to specify the error type you want to use. +/// +/// # Examples +/// +/// ```rust +/// # #![feature(new_uninit)] +/// use kernel::{init::{self, PinInit}, error::Error}; +/// #[pin_data] +/// struct BigBuf { +/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// ptr: *mut u8, +/// } +/// +/// impl BigBuf { +/// fn new() -> impl PinInit { +/// try_pin_init!(Self { +/// big: Box::init(init::zeroed())?, +/// small: [0; 1024 * 1024], +/// ptr: core::ptr::null_mut(), +/// }? Error) +/// } +/// } +/// ``` +/// +/// [`Error`]: kernel::error::Error +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `__internal` inside of `init/__internal.rs`. +#[macro_export] +macro_rules! try_pin_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_pin_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)? ), + @fields($($fields)*), + @error($crate::error::Error), + ) + }; + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + $crate::try_pin_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)? ), + @fields($($fields)*), + @error($err), + ) + }; + ( + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + ) => {{ + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return + // type and shadow it later when we insert the arbitrary user code. That way there will be + // no possibility of returning without `unsafe`. + struct __InitOk; + // Get the pin data from the supplied type. + let data = unsafe { + use $crate::init::__internal::HasPinData; + $t$(::<$($generics),*>)?::__pin_data() + }; + // Ensure that `data` really is of type `PinData` and help with type inference: + let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( + data, + move |slot| { + { + // Shadow the structure so it cannot be used to return early. + struct __InitOk; + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? + // Initialize every field. + $crate::try_pin_init!(init_slot: + @data(data), + @slot(slot), + @munch_fields($($fields)*,), + ); + // We use unreachable code to ensure that all fields have been mentioned exactly + // once, this struct initializer will still be type-checked and complain with a + // very natural error message if a field is forgotten/mentioned more than once. + #[allow(unreachable_code, clippy::diverging_sub_expression)] + if false { + $crate::try_pin_init!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + } + // Forget all guards, since initialization was a success. + $crate::try_pin_init!(forget_guards: + @munch_fields($($fields)*,), + ); + } + Ok(__InitOk) + } + ); + let init = move |slot| -> ::core::result::Result<(), $err> { + init(slot).map(|__InitOk| ()) + }; + let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; + init + }}; + (init_slot: + @data($data:ident), + @slot($slot:ident), + @munch_fields($(,)?), + ) => { + // Endpoint of munching, no fields are left. + }; + (init_slot: + @data($data:ident), + @slot($slot:ident), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let $field = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. + unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; + // Create the drop guard. + // + // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. + // + // SAFETY: We forget the guard later when initialization has succeeded. + let $field = &unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::try_pin_init!(init_slot: + @data($data), + @slot($slot), + @munch_fields($($rest)*), + ); + }; + (init_slot: + @data($data:ident), + @slot($slot:ident), + // Direct value init, this is safe for every field. + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + $(let $field = $val;)? + // Initialize the field. + // + // SAFETY: The memory at `slot` is uninitialized. + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + // Create the drop guard: + // + // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. + // + // SAFETY: We forget the guard later when initialization has succeeded. + let $field = &unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::try_pin_init!(init_slot: + @data($data), + @slot($slot), + @munch_fields($($rest)*), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields($(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. + // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to + // get the correct type inference here: + unsafe { + ::core::ptr::write($slot, $t { + $($acc)* + }); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::try_pin_init!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::try_pin_init!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; + (forget_guards: + @munch_fields($(,)?), + ) => { + // Munching finished. + }; + (forget_guards: + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + unsafe { $crate::init::__internal::DropGuard::forget($field) }; + + $crate::try_pin_init!(forget_guards: + @munch_fields($($rest)*), + ); + }; + (forget_guards: + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + unsafe { $crate::init::__internal::DropGuard::forget($field) }; + + $crate::try_pin_init!(forget_guards: + @munch_fields($($rest)*), + ); + }; +} + +/// Construct an in-place initializer for `struct`s. +/// +/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use +/// [`try_init!`]. +/// +/// The syntax is identical to [`pin_init!`] and its safety caveats also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// This initializer is for initializing data in-place that might later be moved. If you want to +/// pin-initialize, use [`pin_init!`]. +/// +/// [`Error`]: kernel::error::Error +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `__internal` inside of `init/__internal.rs`. +#[macro_export] +macro_rules! init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error(::core::convert::Infallible), + ) + } +} + +/// Construct an in-place fallible initializer for `struct`s. +/// +/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use +/// [`init!`]. +/// +/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, +/// append `? $type` after the `struct` initializer. +/// The safety caveats from [`try_pin_init!`] also apply: +/// - `unsafe` code must guarantee either full initialization or return an error and allow +/// deallocation of the memory. +/// - the fields are initialized in the order given in the initializer. +/// - no references to fields are allowed to be created inside of the initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::{init::PinInit, error::Error, InPlaceInit}; +/// struct BigBuf { +/// big: Box<[u8; 1024 * 1024 * 1024]>, +/// small: [u8; 1024 * 1024], +/// } +/// +/// impl BigBuf { +/// fn new() -> impl Init { +/// try_init!(Self { +/// big: Box::init(zeroed())?, +/// small: [0; 1024 * 1024], +/// }? Error) +/// } +/// } +/// ``` +/// +/// [`Error`]: kernel::error::Error +// For a detailed example of how this macro works, see the module documentation of the hidden +// module `__internal` inside of `init/__internal.rs`. +#[macro_export] +macro_rules! try_init { + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }) => { + $crate::try_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($crate::error::Error), + ) + }; + ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + $($fields:tt)* + }? $err:ty) => { + $crate::try_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + ) + }; + ( + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + ) => {{ + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return + // type and shadow it later when we insert the arbitrary user code. That way there will be + // no possibility of returning without `unsafe`. + struct __InitOk; + // Get the init data from the supplied type. + let data = unsafe { + use $crate::init::__internal::HasInitData; + $t$(::<$($generics),*>)?::__init_data() + }; + // Ensure that `data` really is of type `InitData` and help with type inference: + let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( + data, + move |slot| { + { + // Shadow the structure so it cannot be used to return early. + struct __InitOk; + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? + // Initialize every field. + $crate::try_init!(init_slot: + @slot(slot), + @munch_fields($($fields)*,), + ); + // We use unreachable code to ensure that all fields have been mentioned exactly + // once, this struct initializer will still be type-checked and complain with a + // very natural error message if a field is forgotten/mentioned more than once. + #[allow(unreachable_code, clippy::diverging_sub_expression)] + if false { + $crate::try_init!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + } + // Forget all guards, since initialization was a success. + $crate::try_init!(forget_guards: + @munch_fields($($fields)*,), + ); + } + Ok(__InitOk) + } + ); + let init = move |slot| -> ::core::result::Result<(), $err> { + init(slot).map(|__InitOk| ()) + }; + let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; + init + }}; + (init_slot: + @slot($slot:ident), + @munch_fields( $(,)?), + ) => { + // Endpoint of munching, no fields are left. + }; + (init_slot: + @slot($slot:ident), + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let $field = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + unsafe { + $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; + } + // Create the drop guard. + // + // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. + // + // SAFETY: We forget the guard later when initialization has succeeded. + let $field = &unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::try_init!(init_slot: + @slot($slot), + @munch_fields($($rest)*), + ); + }; + (init_slot: + @slot($slot:ident), + // Direct value init. + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + $(let $field = $val;)? + // Call the initializer. + // + // SAFETY: The memory at `slot` is uninitialized. + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + // Create the drop guard. + // + // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. + // + // SAFETY: We forget the guard later when initialization has succeeded. + let $field = &unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::try_init!(init_slot: + @slot($slot), + @munch_fields($($rest)*), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields( $(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. + // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to + // get the correct type inference here: + unsafe { + ::core::ptr::write($slot, $t { + $($acc)* + }); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::try_init!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)*$field: ::core::panic!(),), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::try_init!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)*$field: ::core::panic!(),), + ); + }; + (forget_guards: + @munch_fields($(,)?), + ) => { + // Munching finished. + }; + (forget_guards: + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + unsafe { $crate::init::__internal::DropGuard::forget($field) }; + + $crate::try_init!(forget_guards: + @munch_fields($($rest)*), + ); + }; + (forget_guards: + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + unsafe { $crate::init::__internal::DropGuard::forget($field) }; + + $crate::try_init!(forget_guards: + @munch_fields($($rest)*), + ); + }; +} /// A pin-initializer for the type `T`. /// @@ -63,7 +862,6 @@ pub mod __internal; /// [`Arc`]: crate::sync::Arc /// [`Arc::pin_init`]: crate::sync::Arc::pin_init /// [`UniqueArc`]: kernel::sync::UniqueArc -/// [`Box`]: alloc::boxed::Box #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait PinInit: Sized { /// Initializes `slot`. @@ -106,7 +904,6 @@ pub unsafe trait PinInit: Sized { /// /// [`Arc`]: crate::sync::Arc /// [`UniqueArc`]: kernel::sync::UniqueArc -/// [`Box`]: alloc::boxed::Box #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait Init: Sized { /// Initializes `slot`. diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 08cbb533343878..a3389a68429644 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -31,3 +31,133 @@ where (self.0)(slot) } } + +/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate +/// the pin projections within the initializers. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasPinData { + type PinData: PinData; + + unsafe fn __pin_data() -> Self::PinData; +} + +/// Marker trait for pinning data of structs. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait PinData: Copy { + type Datee: ?Sized + HasPinData; + + /// Type inference helper function. + fn make_closure(self, f: F) -> F + where + F: FnOnce(*mut Self::Datee) -> Result, + { + f + } +} + +/// This trait is automatically implemented for every type. It aims to provide the same type +/// inference help as `HasPinData`. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait HasInitData { + type InitData: InitData; + + unsafe fn __init_data() -> Self::InitData; +} + +/// Same function as `PinData`, but for arbitrary data. +/// +/// # Safety +/// +/// Only the `init` module is allowed to use this trait. +pub unsafe trait InitData: Copy { + type Datee: ?Sized + HasInitData; + + /// Type inference helper function. + fn make_closure(self, f: F) -> F + where + F: FnOnce(*mut Self::Datee) -> Result, + { + f + } +} + +pub struct AllData(PhantomData) -> Box>); + +impl Clone for AllData { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for AllData {} + +unsafe impl InitData for AllData { + type Datee = T; +} + +unsafe impl HasInitData for T { + type InitData = AllData; + + unsafe fn __init_data() -> Self::InitData { + AllData(PhantomData) + } +} + +/// When a value of this type is dropped, it drops a `T`. +/// +/// Can be forgotten to prevent the drop. +pub struct DropGuard { + ptr: *mut T, + do_drop: Cell, +} + +impl DropGuard { + /// Creates a new [`DropGuard`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer. + /// + /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`: + /// - has not been dropped, + /// - is not accessible by any other means, + /// - will not be dropped by any other means. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + Self { + ptr, + do_drop: Cell::new(true), + } + } + + /// Prevents this guard from dropping the supplied pointer. + /// + /// # Safety + /// + /// This function is unsafe in order to prevent safe code from forgetting this guard. It should + /// only be called by the macros in this module. + #[inline] + pub unsafe fn forget(&self) { + self.do_drop.set(false); + } +} + +impl Drop for DropGuard { + #[inline] + fn drop(&mut self) { + if self.do_drop.get() { + // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function + // ensuring that this operation is safe. + unsafe { ptr::drop_in_place(self.ptr) } + } + } +} diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs new file mode 100644 index 00000000000000..9ef02f5411a6b5 --- /dev/null +++ b/rust/kernel/init/macros.rs @@ -0,0 +1,707 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! This module provides the macros that actually implement the proc-macros `pin_data` and +//! `pinned_drop`. +//! +//! These macros should never be called directly, since they expect their input to be +//! in a certain format which is internal. Use the proc-macros instead. +//! +//! This architecture has been chosen because the kernel does not yet have access to `syn` which +//! would make matters a lot easier for implementing these as proc-macros. +//! +//! # Macro expansion example +//! +//! This section is intended for readers trying to understand the macros in this module and the +//! `pin_init!` macros from `init.rs`. +//! +//! We will look at the following example: +//! +//! ```rust +//! # use kernel::init::*; +//! #[pin_data] +//! #[repr(C)] +//! struct Bar { +//! #[pin] +//! t: T, +//! pub x: usize, +//! } +//! +//! impl Bar { +//! fn new(t: T) -> impl PinInit { +//! pin_init!(Self { t, x: 0 }) +//! } +//! } +//! ``` +//! +//! This example includes the most common and important features of the pin-init API. +//! +//! Below you can find individual section about the different macro invocations. Here are some +//! general things we need to take into account when designing macros: +//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()` +//! this ensures that the correct item is used, since users could define their own `mod core {}` +//! and then their own `panic!` inside to execute arbitrary code inside of our macro. +//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied +//! expressions inside of an `unsafe` block in the macro, because this would allow users to do +//! `unsafe` operations without an associated `unsafe` block. +//! +//! ## `#[pin_data]` on `Bar` +//! +//! This macro is used to specify which fields are structurally pinned and which fields are not. It +//! is placed on the struct definition and allows `#[pin]` to be placed on the fields. +//! +//! Here is the definition of `Bar` from our example: +//! +//! ```rust +//! # use kernel::init::*; +//! #[pin_data] +//! #[repr(C)] +//! struct Bar { +//! t: T, +//! pub x: usize, +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust +//! // Firstly the normal definition of the struct, attributes are preserved: +//! #[repr(C)] +//! struct Bar { +//! t: T, +//! pub x: usize, +//! } +//! // Then an anonymous constant is defined, this is because we do not want any code to access the +//! // types that we define inside: +//! const _: () = { +//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics, +//! // since we need to implement access functions for each field and thus need to know its +//! // type. +//! struct __ThePinData { +//! __phantom: ::core::marker::PhantomData) -> Bar>, +//! } +//! // We implement `Copy` for the pin-data struct, since all functions it defines will take +//! // `self` by value. +//! impl ::core::clone::Clone for __ThePinData { +//! fn clone(&self) -> Self { +//! *self +//! } +//! } +//! impl ::core::marker::Copy for __ThePinData {} +//! // For every field of `Bar`, the pin-data struct will define a function with the same name +//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the +//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field +//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`). +//! #[allow(dead_code)] +//! impl __ThePinData { +//! unsafe fn t( +//! self, +//! slot: *mut T, +//! init: impl ::kernel::init::Init, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::kernel::init::Init::__init(init, slot) } +//! } +//! pub unsafe fn x( +//! self, +//! slot: *mut usize, +//! init: impl ::kernel::init::Init, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::kernel::init::Init::__init(init, slot) } +//! } +//! } +//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct +//! // that we constructed beforehand. +//! unsafe impl ::kernel::init::__internal::HasPinData for Bar { +//! type PinData = __ThePinData; +//! unsafe fn __pin_data() -> Self::PinData { +//! __ThePinData { +//! __phantom: ::core::marker::PhantomData, +//! } +//! } +//! } +//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data +//! // struct. This is important to ensure that no user can implement a rouge `__pin_data` +//! // function without using `unsafe`. +//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData { +//! type Datee = Bar; +//! } +//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is +//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned +//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our +//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist +//! // for two reasons: +//! // - `__phantom`: every generic must be used, since we cannot really know which generics +//! // are used, we declere all and then use everything here once. +//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant +//! // over it. The lifetime is needed to work around the limitation that trait bounds must +//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is +//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler +//! // into accepting these bounds regardless. +//! #[allow(dead_code)] +//! struct __Unpin<'__pin, T> { +//! __phantom_pin: ::core::marker::PhantomData &'__pin ()>, +//! __phantom: ::core::marker::PhantomData) -> Bar>, +//! } +//! #[doc(hidden)] +//! impl<'__pin, T> +//! ::core::marker::Unpin for Bar where __Unpin<'__pin, T>: ::core::marker::Unpin {} +//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users +//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to +//! // UB with only safe code, so we disallow this by giving a trait implementation error using +//! // a direct impl and a blanket implementation. +//! trait MustNotImplDrop {} +//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do +//! // (normally people want to know if a type has any kind of drop glue at all, here we want +//! // to know if it has any kind of custom drop glue, which is exactly what this bound does). +//! #[allow(drop_bounds)] +//! impl MustNotImplDrop for T {} +//! impl MustNotImplDrop for Bar {} +//! }; +//! ``` +//! +//! ## `pin_init!` in `impl Bar` +//! +//! This macro creates an pin-initializer for the given struct. It requires that the struct is +//! annotated by `#[pin_data]`. +//! +//! Here is the impl on `Bar` defining the new function: +//! +//! ```rust +//! impl Bar { +//! fn new(t: T) -> impl PinInit { +//! pin_init!(Self { t, x: 0 }) +//! } +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust +//! impl Bar { +//! fn new(t: T) -> impl PinInit { +//! { +//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok` +//! // return type and shadow it later when we insert the arbitrary user code. That way +//! // there will be no possibility of returning without `unsafe`. +//! struct __InitOk; +//! // Get the pin-data type from the initialized type. +//! // - the function is unsafe, hence the unsafe block +//! // - we `use` the `HasPinData` trait in the block, it is only available in that +//! // scope. +//! let data = unsafe { +//! use ::kernel::init::__internal::HasPinData; +//! Self::__pin_data() +//! }; +//! // Use `data` to help with type inference, the closure supplied will have the type +//! // `FnOnce(*mut Self) -> Result<__InitOk, Infallible>`. +//! let init = ::kernel::init::__internal::PinData::make_closure::< +//! _, +//! __InitOk, +//! ::core::convert::Infallible, +//! >(data, move |slot| { +//! { +//! // Shadow the structure so it cannot be used to return early. If a user +//! // tries to write `return Ok(__InitOk)`, then they get a type error, since +//! // that will refer to this struct instead of the one defined above. +//! struct __InitOk; +//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. +//! unsafe { ::core::ptr::write(&raw mut (*slot).t, t) }; +//! // Since initialization could fail later (not in this case, since the error +//! // type is `Infallible`) we will need to drop this field if it fails. This +//! // `DropGuard` will drop the field when it gets dropped and has not yet +//! // been forgotten. We make a reference to it, so users cannot `mem::forget` +//! // it from the initializer, since the name is the same as the field. +//! let t = &unsafe { +//! ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).t) +//! }; +//! // Expansion of `x: 0,`: +//! // Since this can be an arbitrary expression we cannot place it inside of +//! // the `unsafe` block, so we bind it here. +//! let x = 0; +//! unsafe { ::core::ptr::write(&raw mut (*slot).x, x) }; +//! let x = &unsafe { +//! ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).x) +//! }; +//! +//! // Here we use the type checker to ensuer that every field has been +//! // initialized exactly once, since this is `if false` it will never get +//! // executed, but still type-checked. +//! // Additionally we abuse `slot` to automatically infer the correct type for +//! // the struct. This is also another check that every field is accessible +//! // from this scope. +//! #[allow(unreachable_code, clippy::diverging_sub_expression)] +//! if false { +//! unsafe { +//! ::core::ptr::write( +//! slot, +//! Self { +//! // We only care about typecheck finding every field here, +//! // the expression does not matter, just conjure one using +//! // `panic!()`: +//! t: ::core::panic!(), +//! x: ::core::panic!(), +//! }, +//! ); +//! }; +//! } +//! // Since initialization has successfully completed, we can now forget the +//! // guards. +//! unsafe { ::kernel::init::__internal::DropGuard::forget(t) }; +//! unsafe { ::kernel::init::__internal::DropGuard::forget(x) }; +//! } +//! // We leave the scope above and gain access to the previously shadowed +//! // `__InitOk` that we need to return. +//! Ok(__InitOk) +//! }); +//! // Change the return type of the closure. +//! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { +//! init(slot).map(|__InitOk| ()) +//! }; +//! // Construct the initializer. +//! let init = unsafe { +//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) +//! }; +//! init +//! } +//! } +//! } +//! ``` + +/// This macro first parses the struct definition such that it separates pinned and not pinned +/// fields. Afterwards it declares the struct and implement the `PinData` trait safely. +#[doc(hidden)] +#[macro_export] +macro_rules! __pin_data { + // Proc-macro entry point, this is supplied by the proc-macro pre-parsing. + (parse_input: + @args($($pinned_drop:ident)?), + @sig( + $(#[$($struct_attr:tt)*])* + $vis:vis struct $name:ident + $(where $($whr:tt)*)? + ), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @body({ $($fields:tt)* }), + ) => { + // We now use token munching to iterate through all of the fields. While doing this we + // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user + // wants these to be structurally pinned. The rest of the fields are the + // 'not pinned fields'. Additionally we collect all fields, since we need them in the right + // order to declare the struct. + // + // In this call we also put some explaining comments for the parameters. + $crate::__pin_data!(find_pinned_fields: + // Attributes on the struct itself, these will just be propagated to be put onto the + // struct definition. + @struct_attrs($(#[$($struct_attr)*])*), + // The visibility of the struct. + @vis($vis), + // The name of the struct. + @name($name), + // The 'impl generics', the generics that will need to be specified on the struct inside + // of an `impl<$ty_generics>` block. + @impl_generics($($impl_generics)*), + // The 'ty generics', the generics that will need to be specified on the impl blocks. + @ty_generics($($ty_generics)*), + // The where clause of any impl block and the declaration. + @where($($($whr)*)?), + // The remaining fields tokens that need to be processed. + // We add a `,` at the end to ensure correct parsing. + @fields_munch($($fields)* ,), + // The pinned fields. + @pinned(), + // The not pinned fields. + @not_pinned(), + // All fields. + @fields(), + // The accumulator containing all attributes already parsed. + @accum(), + // Contains `yes` or `` to indicate if `#[pin]` was found on the current field. + @is_pinned(), + // The proc-macro argument, this should be `PinnedDrop` or ``. + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // We found a PhantomPinned field, this should generally be pinned! + @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + // This field is not pinned. + @is_pinned(), + @pinned_drop($($pinned_drop:ident)?), + ) => { + ::core::compile_error!(concat!( + "The field `", + stringify!($field), + "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.", + )); + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), + @not_pinned($($not_pinned)*), + @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // We reached the field declaration. + @fields_munch($field:ident : $type:ty, $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + // This field is pinned. + @is_pinned(yes), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)* $($accum)* $field: $type,), + @not_pinned($($not_pinned)*), + @fields($($fields)* $($accum)* $field: $type,), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // We reached the field declaration. + @fields_munch($field:ident : $type:ty, $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + // This field is not pinned. + @is_pinned(), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)* $($accum)* $field: $type,), + @fields($($fields)* $($accum)* $field: $type,), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // We found the `#[pin]` attr. + @fields_munch(#[pin] $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + @is_pinned($($is_pinned:ident)?), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + // We do not include `#[pin]` in the list of attributes, since it is not actually an + // attribute that is defined somewhere. + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + @fields($($fields)*), + @accum($($accum)*), + // Set this to `yes`. + @is_pinned(yes), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // We reached the field declaration with visibility, for simplicity we only munch the + // visibility and put it into `$accum`. + @fields_munch($fvis:vis $field:ident $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + @is_pinned($($is_pinned:ident)?), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @fields_munch($field $($rest)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + @fields($($fields)*), + @accum($($accum)* $fvis), + @is_pinned($($is_pinned)?), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // Some other attribute, just put it into `$accum`. + @fields_munch(#[$($attr:tt)*] $($rest:tt)*), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum($($accum:tt)*), + @is_pinned($($is_pinned:ident)?), + @pinned_drop($($pinned_drop:ident)?), + ) => { + $crate::__pin_data!(find_pinned_fields: + @struct_attrs($($struct_attrs)*), + @vis($vis), + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @fields_munch($($rest)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + @fields($($fields)*), + @accum($($accum)* #[$($attr)*]), + @is_pinned($($is_pinned)?), + @pinned_drop($($pinned_drop)?), + ); + }; + (find_pinned_fields: + @struct_attrs($($struct_attrs:tt)*), + @vis($vis:vis), + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + // We reached the end of the fields, plus an optional additional comma, since we added one + // before and the user is also allowed to put a trailing comma. + @fields_munch($(,)?), + @pinned($($pinned:tt)*), + @not_pinned($($not_pinned:tt)*), + @fields($($fields:tt)*), + @accum(), + @is_pinned(), + @pinned_drop($($pinned_drop:ident)?), + ) => { + // Declare the struct with all fields in the correct order. + $($struct_attrs)* + $vis struct $name <$($impl_generics)*> + where $($whr)* + { + $($fields)* + } + + // We put the rest into this const item, because it then will not be accessible to anything + // outside. + const _: () = { + // We declare this struct which will host all of the projection function for our type. + // it will be invariant over all generic parameters which are inherited from the + // struct. + $vis struct __ThePinData<$($impl_generics)*> + where $($whr)* + { + __phantom: ::core::marker::PhantomData< + fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> + >, + } + + impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*> + where $($whr)* + { + fn clone(&self) -> Self { *self } + } + + impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*> + where $($whr)* + {} + + // Make all projection functions. + $crate::__pin_data!(make_pin_data: + @pin_data(__ThePinData), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @pinned($($pinned)*), + @not_pinned($($not_pinned)*), + ); + + // SAFETY: We have added the correct projection functions above to `__ThePinData` and + // we also use the least restrictive generics possible. + unsafe impl<$($impl_generics)*> + $crate::init::__internal::HasPinData for $name<$($ty_generics)*> + where $($whr)* + { + type PinData = __ThePinData<$($ty_generics)*>; + + unsafe fn __pin_data() -> Self::PinData { + __ThePinData { __phantom: ::core::marker::PhantomData } + } + } + + unsafe impl<$($impl_generics)*> + $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*> + where $($whr)* + { + type Datee = $name<$($ty_generics)*>; + } + + // This struct will be used for the unpin analysis. Since only structurally pinned + // fields are relevant whether the struct should implement `Unpin`. + #[allow(dead_code)] + struct __Unpin <'__pin, $($impl_generics)*> + where $($whr)* + { + __phantom_pin: ::core::marker::PhantomData &'__pin ()>, + __phantom: ::core::marker::PhantomData< + fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> + >, + // Only the pinned fields. + $($pinned)* + } + + #[doc(hidden)] + impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*> + where + __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin, + $($whr)* + {} + + // We need to disallow normal `Drop` implementation, the exact behavior depends on + // whether `PinnedDrop` was specified as the parameter. + $crate::__pin_data!(drop_prevention: + @name($name), + @impl_generics($($impl_generics)*), + @ty_generics($($ty_generics)*), + @where($($whr)*), + @pinned_drop($($pinned_drop)?), + ); + }; + }; + // When no `PinnedDrop` was specified, then we have to prevent implementing drop. + (drop_prevention: + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned_drop(), + ) => { + // We prevent this by creating a trait that will be implemented for all types implementing + // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, + // if it also implements `Drop` + trait MustNotImplDrop {} + #[allow(drop_bounds)] + impl MustNotImplDrop for T {} + impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> + where $($whr)* {} + }; + // If some other parameter was specified, we emit a readable error. + (drop_prevention: + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned_drop($($rest:tt)*), + ) => { + compile_error!( + "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.", + stringify!($($rest)*), + ); + }; + (make_pin_data: + @pin_data($pin_data:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), + @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), + ) => { + // For every field, we create a projection function according to its projection type. If a + // field is structurally pinned, then it must be initialized via `PinInit`, if it is not + // structurally pinned, then it can be initialized via `Init`. + // + // The functions are `unsafe` to prevent accidentally calling them. + #[allow(dead_code)] + impl<$($impl_generics)*> $pin_data<$($ty_generics)*> + where $($whr)* + { + $( + $pvis unsafe fn $p_field( + self, + slot: *mut $p_type, + init: impl $crate::init::PinInit<$p_type, E>, + ) -> ::core::result::Result<(), E> { + unsafe { $crate::init::PinInit::__pinned_init(init, slot) } + } + )* + $( + $fvis unsafe fn $field( + self, + slot: *mut $type, + init: impl $crate::init::Init<$type, E>, + ) -> ::core::result::Result<(), E> { + unsafe { $crate::init::Init::__init(init, slot) } + } + )* + } + }; +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 82b520f024dd7d..6d84f2fd0d7710 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -7,6 +7,7 @@ mod quote; mod concat_idents; mod helpers; mod module; +mod pin_data; mod vtable; use proc_macro::TokenStream; @@ -168,3 +169,31 @@ pub fn vtable(attr: TokenStream, ts: TokenStream) -> TokenStream { pub fn concat_idents(ts: TokenStream) -> TokenStream { concat_idents::concat_idents(ts) } + +/// Used to specify the pinning information of the fields of a struct. +/// +/// This is somewhat similar in purpose as +/// [pin-project-lite](https://crates.io/crates/pin-project-lite). +/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each +/// field you want to structurally pin. +/// +/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, +/// then `#[pin]` directs the type of initializer that is required. +/// +/// # Examples +/// +/// ```rust,ignore +/// #[pin_data] +/// struct DriverData { +/// #[pin] +/// queue: Mutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// } +/// ``` +/// +/// [`pin_init!`]: ../kernel/macro.pin_init.html +// ^ cannot use direct link, since `kernel` is not a dependency of `macros`. +#[proc_macro_attribute] +pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { + pin_data::pin_data(inner, item) +} diff --git a/rust/macros/pin_data.rs b/rust/macros/pin_data.rs new file mode 100644 index 00000000000000..954149d771813c --- /dev/null +++ b/rust/macros/pin_data.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro::{Punct, Spacing, TokenStream, TokenTree}; + +pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { + // This proc-macro only does some pre-parsing and then delegates the actual parsing to + // `kernel::__pin_data!`. + // + // In here we only collect the generics, since parsing them in declarative macros is very + // elaborate. We also do not need to analyse their structure, we only need to collect them. + + // `impl_generics`, the declared generics with their bounds. + let mut impl_generics = vec![]; + // Only the names of the generics, without any bounds. + let mut ty_generics = vec![]; + // Tokens not related to the generics e.g. the `impl` token. + let mut rest = vec![]; + // The current level of `<`. + let mut nesting = 0; + let mut toks = input.into_iter(); + // If we are at the beginning of a generic parameter. + let mut at_start = true; + for tt in &mut toks { + match tt.clone() { + TokenTree::Punct(p) if p.as_char() == '<' => { + if nesting >= 1 { + impl_generics.push(tt); + } + nesting += 1; + } + TokenTree::Punct(p) if p.as_char() == '>' => { + if nesting == 0 { + break; + } else { + nesting -= 1; + if nesting >= 1 { + impl_generics.push(tt); + } + if nesting == 0 { + break; + } + } + } + tt => { + if nesting == 1 { + match &tt { + TokenTree::Ident(i) if i.to_string() == "const" => {} + TokenTree::Ident(_) if at_start => { + ty_generics.push(tt.clone()); + ty_generics.push(TokenTree::Punct(Punct::new(',', Spacing::Alone))); + at_start = false; + } + TokenTree::Punct(p) if p.as_char() == ',' => at_start = true, + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { + ty_generics.push(tt.clone()); + } + _ => {} + } + } + if nesting >= 1 { + impl_generics.push(tt); + } else if nesting == 0 { + rest.push(tt); + } + } + } + } + rest.extend(toks); + // This should be the body of the struct `{...}`. + let last = rest.pop(); + quote!(::kernel::__pin_data! { + parse_input: + @args(#args), + @sig(#(#rest)*), + @impl_generics(#(#impl_generics)*), + @ty_generics(#(#ty_generics)*), + @body(#last), + }) +} diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs index 94a6277182ee6b..c8e08b3c1e4cfb 100644 --- a/rust/macros/quote.rs +++ b/rust/macros/quote.rs @@ -38,7 +38,6 @@ impl ToTokens for TokenStream { /// This is a similar to the /// [`quote_spanned!`](https://docs.rs/quote/latest/quote/macro.quote_spanned.html) macro from the /// `quote` crate but provides only just enough functionality needed by the current `macros` crate. -#[allow(unused_macros)] macro_rules! quote_spanned { ($span:expr => $($tt:tt)*) => { #[allow(clippy::vec_init_then_push)] @@ -137,7 +136,6 @@ macro_rules! quote_spanned { /// `macros` crate. /// /// [`Span::mixed_site()`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.mixed_site -#[allow(unused_macros)] macro_rules! quote { ($($tt:tt)*) => { quote_spanned!(::proc_macro::Span::mixed_site() => $($tt)*) From fb5eab771a7a3d7ee5b7382be7f721b83fbe521c Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:25:56 +0000 Subject: [PATCH 409/635] rust: init/sync: add `InPlaceInit` trait to pin-initialize smart pointers The `InPlaceInit` trait that provides two functions, for initializing using `PinInit` and `Init`. It is implemented by `Arc`, `UniqueArc` and `Box`. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-9-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 128 ++++++++++++++++++++++++++++++++++++---- rust/kernel/sync/arc.rs | 24 ++++++++ 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index ecef0376d7260a..6499cf5c9c2090 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -114,10 +114,16 @@ //! [`impl Init`]: Init //! [`Opaque`]: kernel::types::Opaque //! [`pin_data`]: ::macros::pin_data -//! [`UniqueArc`]: kernel::sync::UniqueArc +use crate::{ + error::{self, Error}, + sync::UniqueArc, +}; use alloc::boxed::Box; -use core::{cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, ptr}; +use core::{ + alloc::AllocError, cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, + pin::Pin, ptr, +}; #[doc(hidden)] pub mod __internal; @@ -309,7 +315,6 @@ pub mod macros; /// /// [`try_pin_init!`]: kernel::try_pin_init /// [`NonNull`]: core::ptr::NonNull -/// [`Error`]: kernel::error::Error // For a detailed example of how this macro works, see the module documentation of the hidden // module `__internal` inside of `init/__internal.rs`. #[macro_export] @@ -363,8 +368,6 @@ macro_rules! pin_init { /// } /// } /// ``` -/// -/// [`Error`]: kernel::error::Error // For a detailed example of how this macro works, see the module documentation of the hidden // module `__internal` inside of `init/__internal.rs`. #[macro_export] @@ -586,8 +589,6 @@ macro_rules! try_pin_init { /// /// This initializer is for initializing data in-place that might later be moved. If you want to /// pin-initialize, use [`pin_init!`]. -/// -/// [`Error`]: kernel::error::Error // For a detailed example of how this macro works, see the module documentation of the hidden // module `__internal` inside of `init/__internal.rs`. #[macro_export] @@ -635,8 +636,6 @@ macro_rules! init { /// } /// } /// ``` -/// -/// [`Error`]: kernel::error::Error // For a detailed example of how this macro works, see the module documentation of the hidden // module `__internal` inside of `init/__internal.rs`. #[macro_export] @@ -842,7 +841,8 @@ macro_rules! try_init { /// A pin-initializer for the type `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`]. +/// be [`Box`], [`Arc`], [`UniqueArc`]. Use the [`InPlaceInit::pin_init`] function of a +/// smart pointer like [`Arc`] on this. /// /// Also see the [module description](self). /// @@ -861,7 +861,6 @@ macro_rules! try_init { /// /// [`Arc`]: crate::sync::Arc /// [`Arc::pin_init`]: crate::sync::Arc::pin_init -/// [`UniqueArc`]: kernel::sync::UniqueArc #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait PinInit: Sized { /// Initializes `slot`. @@ -878,7 +877,8 @@ pub unsafe trait PinInit: Sized { /// An initializer for `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`]. Because [`PinInit`] is a super trait, you can +/// be [`Box`], [`Arc`], [`UniqueArc`]. Use the [`InPlaceInit::init`] function of a smart +/// pointer like [`Arc`] on this. Because [`PinInit`] is a super trait, you can /// use every function that takes it as well. /// /// Also see the [module description](self). @@ -903,7 +903,6 @@ pub unsafe trait PinInit: Sized { /// move the pointee after initialization. /// /// [`Arc`]: crate::sync::Arc -/// [`UniqueArc`]: kernel::sync::UniqueArc #[must_use = "An initializer must be used in order to create its value."] pub unsafe trait Init: Sized { /// Initializes `slot`. @@ -982,3 +981,106 @@ unsafe impl Init for T { Ok(()) } } + +/// Smart pointer that can initialize memory in-place. +pub trait InPlaceInit: Sized { + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From; + + /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this + /// type. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + fn pin_init(init: impl PinInit) -> error::Result> + where + Error: From, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + }; + Self::try_pin_init(init) + } + + /// Use the given initializer to in-place initialize a `T`. + fn try_init(init: impl Init) -> Result + where + E: From; + + /// Use the given initializer to in-place initialize a `T`. + fn init(init: impl Init) -> error::Result + where + Error: From, + { + // SAFETY: We delegate to `init` and only change the error type. + let init = unsafe { + init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) + }; + Self::try_init(init) + } +} + +impl InPlaceInit for Box { + #[inline] + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From, + { + let mut this = Box::try_new_uninit()?; + let slot = this.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }.into()) + } + + #[inline] + fn try_init(init: impl Init) -> Result + where + E: From, + { + let mut this = Box::try_new_uninit()?; + let slot = this.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }) + } +} + +impl InPlaceInit for UniqueArc { + #[inline] + fn try_pin_init(init: impl PinInit) -> Result, E> + where + E: From, + { + let mut this = UniqueArc::try_new_uninit()?; + let slot = this.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid and will not be moved, because we pin it later. + unsafe { init.__pinned_init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }.into()) + } + + #[inline] + fn try_init(init: impl Init) -> Result + where + E: From, + { + let mut this = UniqueArc::try_new_uninit()?; + let slot = this.as_mut_ptr(); + // SAFETY: When init errors/panics, slot will get deallocated but not dropped, + // slot is valid. + unsafe { init.__init(slot)? }; + // SAFETY: All fields have been initialized. + Ok(unsafe { this.assume_init() }) + } +} diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 34d0e7cbe62e59..b45769a29541d1 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -17,6 +17,8 @@ use crate::{ bindings, + error::{self, Error}, + init::{InPlaceInit, Init, PinInit}, types::{ForeignOwnable, Opaque}, }; use alloc::boxed::Box; @@ -166,6 +168,28 @@ impl Arc { // `Arc` object. Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// If `T: !Unpin` it will not be able to move afterwards. + #[inline] + pub fn pin_init(init: impl PinInit) -> error::Result + where + Error: From, + { + UniqueArc::pin_init(init).map(|u| u.into()) + } + + /// Use the given initializer to in-place initialize a `T`. + /// + /// This is equivalent to [`pin_init`], since an [`Arc`] is always pinned. + #[inline] + pub fn init(init: impl Init) -> error::Result + where + Error: From, + { + UniqueArc::init(init).map(|u| u.into()) + } } impl Arc { From 229a83176e8aec8fd8aef44217abf1f7147934ef Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:01 +0000 Subject: [PATCH 410/635] rust: init: add `PinnedDrop` trait and macros The `PinnedDrop` trait that facilitates destruction of pinned types. It has to be implemented via the `#[pinned_drop]` macro, since the `drop` function should not be called by normal code, only by other destructors. It also only works on structs that are annotated with `#[pin_data(PinnedDrop)]`. Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-10-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 111 ++++++++++++++ rust/kernel/init/__internal.rs | 15 ++ rust/kernel/init/macros.rs | 264 +++++++++++++++++++++++++++++++++ rust/macros/lib.rs | 49 ++++++ rust/macros/pinned_drop.rs | 49 ++++++ 5 files changed, 488 insertions(+) create mode 100644 rust/macros/pinned_drop.rs diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 6499cf5c9c2090..37e8159df24d91 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -104,6 +104,78 @@ //! } //! ``` //! +//! ## Manual creation of an initializer +//! +//! Often when working with primitives the previous approaches are not sufficient. That is where +//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a +//! [`impl PinInit`] directly from a closure. Of course you have to ensure that the closure +//! actually does the initialization in the correct way. Here are the things to look out for +//! (we are calling the parameter to the closure `slot`): +//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so +//! `slot` now contains a valid bit pattern for the type `T`, +//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so +//! you need to take care to clean up anything if your initialization fails mid-way, +//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of +//! `slot` gets called. +//! +//! ```rust +//! use kernel::{prelude::*, init}; +//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; +//! # mod bindings { +//! # pub struct foo; +//! # pub unsafe fn init_foo(_ptr: *mut foo) {} +//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} +//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } +//! # } +//! /// # Invariants +//! /// +//! /// `foo` is always initialized +//! #[pin_data(PinnedDrop)] +//! pub struct RawFoo { +//! #[pin] +//! foo: Opaque, +//! #[pin] +//! _p: PhantomPinned, +//! } +//! +//! impl RawFoo { +//! pub fn new(flags: u32) -> impl PinInit { +//! // SAFETY: +//! // - when the closure returns `Ok(())`, then it has successfully initialized and +//! // enabled `foo`, +//! // - when it returns `Err(e)`, then it has cleaned up before +//! unsafe { +//! init::pin_init_from_closure(move |slot: *mut Self| { +//! // `slot` contains uninit memory, avoid creating a reference. +//! let foo = addr_of_mut!((*slot).foo); +//! +//! // Initialize the `foo` +//! bindings::init_foo(Opaque::raw_get(foo)); +//! +//! // Try to enable it. +//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); +//! if err != 0 { +//! // Enabling has failed, first clean up the foo and then return the error. +//! bindings::destroy_foo(Opaque::raw_get(foo)); +//! return Err(Error::from_kernel_errno(err)); +//! } +//! +//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. +//! Ok(()) +//! }) +//! } +//! } +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for RawFoo { +//! fn drop(self: Pin<&mut Self>) { +//! // SAFETY: Since `foo` is initialized, destroying is safe. +//! unsafe { bindings::destroy_foo(self.foo.get()) }; +//! } +//! } +//! ``` +//! //! [`sync`]: kernel::sync //! [pinning]: https://doc.rust-lang.org/std/pin/index.html //! [structurally pinned fields]: @@ -1084,3 +1156,42 @@ impl InPlaceInit for UniqueArc { Ok(unsafe { this.assume_init() }) } } + +/// Trait facilitating pinned destruction. +/// +/// Use [`pinned_drop`] to implement this trait safely: +/// +/// ```rust +/// # use kernel::sync::Mutex; +/// use kernel::macros::pinned_drop; +/// use core::pin::Pin; +/// #[pin_data(PinnedDrop)] +/// struct Foo { +/// #[pin] +/// mtx: Mutex, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for Foo { +/// fn drop(self: Pin<&mut Self>) { +/// pr_info!("Foo is being dropped!"); +/// } +/// } +/// ``` +/// +/// # Safety +/// +/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. +/// +/// [`pinned_drop`]: kernel::macros::pinned_drop +pub unsafe trait PinnedDrop: __internal::HasPinData { + /// Executes the pinned destructor of this type. + /// + /// While this function is marked safe, it is actually unsafe to call it manually. For this + /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code + /// and thus prevents this function from being called where it should not. + /// + /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute + /// automatically. + fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); +} diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index a3389a68429644..774cb620afa777 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -161,3 +161,18 @@ impl Drop for DropGuard { } } } + +/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely +/// created struct. This is needed, because the `drop` function is safe, but should not be called +/// manually. +pub struct OnlyCallFromDrop(()); + +impl OnlyCallFromDrop { + /// # Safety + /// + /// This function should only be called from the [`Drop::drop`] function and only be used to + /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type. + pub unsafe fn new() -> Self { + Self(()) + } +} diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index 9ef02f5411a6b5..541cfad1d8beed 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -31,6 +31,26 @@ //! pin_init!(Self { t, x: 0 }) //! } //! } +//! +//! #[pin_data(PinnedDrop)] +//! struct Foo { +//! a: usize, +//! #[pin] +//! b: Bar, +//! } +//! +//! #[pinned_drop] +//! impl PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>) { +//! println!("{self:p} is getting dropped."); +//! } +//! } +//! +//! let a = 42; +//! let initializer = pin_init!(Foo { +//! a, +//! b <- Bar::new(36), +//! }); //! ``` //! //! This example includes the most common and important features of the pin-init API. @@ -155,6 +175,14 @@ //! #[allow(drop_bounds)] //! impl MustNotImplDrop for T {} //! impl MustNotImplDrop for Bar {} +//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to +//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed +//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. +//! #[allow(non_camel_case_types)] +//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} +//! impl +//! UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} +//! impl UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar {} //! }; //! ``` //! @@ -265,6 +293,210 @@ //! } //! } //! ``` +//! +//! ## `#[pin_data]` on `Foo` +//! +//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the +//! differences/new things in the expansion of the `Foo` definition: +//! +//! ```rust +//! #[pin_data(PinnedDrop)] +//! struct Foo { +//! a: usize, +//! #[pin] +//! b: Bar, +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust +//! struct Foo { +//! a: usize, +//! b: Bar, +//! } +//! const _: () = { +//! struct __ThePinData { +//! __phantom: ::core::marker::PhantomData Foo>, +//! } +//! impl ::core::clone::Clone for __ThePinData { +//! fn clone(&self) -> Self { +//! *self +//! } +//! } +//! impl ::core::marker::Copy for __ThePinData {} +//! #[allow(dead_code)] +//! impl __ThePinData { +//! unsafe fn b( +//! self, +//! slot: *mut Bar, +//! // Note that this is `PinInit` instead of `Init`, this is because `b` is +//! // structurally pinned, as marked by the `#[pin]` attribute. +//! init: impl ::kernel::init::PinInit, E>, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } +//! } +//! unsafe fn a( +//! self, +//! slot: *mut usize, +//! init: impl ::kernel::init::Init, +//! ) -> ::core::result::Result<(), E> { +//! unsafe { ::kernel::init::Init::__init(init, slot) } +//! } +//! } +//! unsafe impl ::kernel::init::__internal::HasPinData for Foo { +//! type PinData = __ThePinData; +//! unsafe fn __pin_data() -> Self::PinData { +//! __ThePinData { +//! __phantom: ::core::marker::PhantomData, +//! } +//! } +//! } +//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData { +//! type Datee = Foo; +//! } +//! #[allow(dead_code)] +//! struct __Unpin<'__pin> { +//! __phantom_pin: ::core::marker::PhantomData &'__pin ()>, +//! __phantom: ::core::marker::PhantomData Foo>, +//! // Since this field is `#[pin]`, it is listed here. +//! b: Bar, +//! } +//! #[doc(hidden)] +//! impl<'__pin> ::core::marker::Unpin for Foo where __Unpin<'__pin>: ::core::marker::Unpin {} +//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to +//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like +//! // before, instead we implement it here and delegate to `PinnedDrop`. +//! impl ::core::ops::Drop for Foo { +//! fn drop(&mut self) { +//! // Since we are getting dropped, no one else has a reference to `self` and thus we +//! // can assume that we never move. +//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; +//! // Create the unsafe token that proves that we are inside of a destructor, this +//! // type is only allowed to be created in a destructor. +//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() }; +//! ::kernel::init::PinnedDrop::drop(pinned, token); +//! } +//! } +//! }; +//! ``` +//! +//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo` +//! +//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an +//! extra parameter that should not be used at all. The macro hides that parameter. +//! +//! Here is the `PinnedDrop` impl for `Foo`: +//! +//! ```rust +//! #[pinned_drop] +//! impl PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>) { +//! println!("{self:p} is getting dropped."); +//! } +//! } +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust +//! // `unsafe`, full path and the token parameter are added, everything else stays the same. +//! unsafe impl ::kernel::init::PinnedDrop for Foo { +//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { +//! println!("{self:p} is getting dropped."); +//! } +//! } +//! ``` +//! +//! ## `pin_init!` on `Foo` +//! +//! Since we already took a look at `pin_init!` on `Bar`, this section will only explain the +//! differences/new things in the expansion of `pin_init!` on `Foo`: +//! +//! ```rust +//! let a = 42; +//! let initializer = pin_init!(Foo { +//! a, +//! b <- Bar::new(36), +//! }); +//! ``` +//! +//! This expands to the following code: +//! +//! ```rust +//! let a = 42; +//! let initializer = { +//! struct __InitOk; +//! let data = unsafe { +//! use ::kernel::init::__internal::HasPinData; +//! Foo::__pin_data() +//! }; +//! let init = ::kernel::init::__internal::PinData::make_closure::< +//! _, +//! __InitOk, +//! ::core::convert::Infallible, +//! >(data, move |slot| { +//! { +//! struct __InitOk; +//! unsafe { ::core::ptr::write(&raw mut (*slot).a, a) }; +//! let a = &unsafe { ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).a) }; +//! let b = Bar::new(36); +//! // Here we use `data` to access the correct field and require that `b` is of type +//! // `PinInit, Infallible>`. +//! unsafe { data.b(&raw mut (*slot).b, b)? }; +//! let b = &unsafe { ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).b) }; +//! +//! #[allow(unreachable_code, clippy::diverging_sub_expression)] +//! if false { +//! unsafe { +//! ::core::ptr::write( +//! slot, +//! Foo { +//! a: ::core::panic!(), +//! b: ::core::panic!(), +//! }, +//! ); +//! }; +//! } +//! unsafe { ::kernel::init::__internal::DropGuard::forget(a) }; +//! unsafe { ::kernel::init::__internal::DropGuard::forget(b) }; +//! } +//! Ok(__InitOk) +//! }); +//! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { +//! init(slot).map(|__InitOk| ()) +//! }; +//! let init = unsafe { +//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) +//! }; +//! init +//! }; +//! ``` + +/// Creates a `unsafe impl<...> PinnedDrop for $type` block. +/// +/// See [`PinnedDrop`] for more information. +#[doc(hidden)] +#[macro_export] +macro_rules! __pinned_drop { + ( + @impl_sig($($impl_sig:tt)*), + @impl_body( + $(#[$($attr:tt)*])* + fn drop($($sig:tt)*) { + $($inner:tt)* + } + ), + ) => { + unsafe $($impl_sig)* { + // Inherit all attributes and the type/ident tokens for the signature. + $(#[$($attr)*])* + fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) { + $($inner)* + } + } + } +} /// This macro first parses the struct definition such that it separates pinned and not pinned /// fields. Afterwards it declares the struct and implement the `PinData` trait safely. @@ -653,6 +885,38 @@ macro_rules! __pin_data { impl MustNotImplDrop for T {} impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> where $($whr)* {} + // We also take care to prevent users from writing a useless `PinnedDrop` implementation. + // They might implement `PinnedDrop` correctly for the struct, but forget to give + // `PinnedDrop` as the parameter to `#[pin_data]`. + #[allow(non_camel_case_types)] + trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} + impl + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} + impl<$($impl_generics)*> + UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> + where $($whr)* {} + }; + // When `PinnedDrop` was specified we just implement `Drop` and delegate. + (drop_prevention: + @name($name:ident), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @where($($whr:tt)*), + @pinned_drop(PinnedDrop), + ) => { + impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*> + where $($whr)* + { + fn drop(&mut self) { + // SAFETY: Since this is a destructor, `self` will not move after this function + // terminates, since it is inaccessible. + let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; + // SAFETY: Since this is a drop function, we can create this token to call the + // pinned destructor of this type. + let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() }; + $crate::init::PinnedDrop::drop(pinned, token); + } + } }; // If some other parameter was specified, we emit a readable error. (drop_prevention: diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 6d84f2fd0d7710..3fc74cb4ea1903 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -8,6 +8,7 @@ mod concat_idents; mod helpers; mod module; mod pin_data; +mod pinned_drop; mod vtable; use proc_macro::TokenStream; @@ -180,6 +181,10 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`, /// then `#[pin]` directs the type of initializer that is required. /// +/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this +/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with +/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care. +/// /// # Examples /// /// ```rust,ignore @@ -191,9 +196,53 @@ pub fn concat_idents(ts: TokenStream) -> TokenStream { /// } /// ``` /// +/// ```rust,ignore +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: Mutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut Info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +/// /// [`pin_init!`]: ../kernel/macro.pin_init.html // ^ cannot use direct link, since `kernel` is not a dependency of `macros`. #[proc_macro_attribute] pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { pin_data::pin_data(inner, item) } + +/// Used to implement `PinnedDrop` safely. +/// +/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`. +/// +/// # Examples +/// +/// ```rust,ignore +/// #[pin_data(PinnedDrop)] +/// struct DriverData { +/// #[pin] +/// queue: Mutex>, +/// buf: Box<[u8; 1024 * 1024]>, +/// raw_info: *mut Info, +/// } +/// +/// #[pinned_drop] +/// impl PinnedDrop for DriverData { +/// fn drop(self: Pin<&mut Self>) { +/// unsafe { bindings::destroy_info(self.raw_info) }; +/// } +/// } +/// ``` +#[proc_macro_attribute] +pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { + pinned_drop::pinned_drop(args, input) +} diff --git a/rust/macros/pinned_drop.rs b/rust/macros/pinned_drop.rs new file mode 100644 index 00000000000000..88fb72b2066047 --- /dev/null +++ b/rust/macros/pinned_drop.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use proc_macro::{TokenStream, TokenTree}; + +pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream { + let mut toks = input.into_iter().collect::>(); + assert!(!toks.is_empty()); + // Ensure that we have an `impl` item. + assert!(matches!(&toks[0], TokenTree::Ident(i) if i.to_string() == "impl")); + // Ensure that we are implementing `PinnedDrop`. + let mut nesting: usize = 0; + let mut pinned_drop_idx = None; + for (i, tt) in toks.iter().enumerate() { + match tt { + TokenTree::Punct(p) if p.as_char() == '<' => { + nesting += 1; + } + TokenTree::Punct(p) if p.as_char() == '>' => { + nesting = nesting.checked_sub(1).unwrap(); + continue; + } + _ => {} + } + if i >= 1 && nesting == 0 { + // Found the end of the generics, this should be `PinnedDrop`. + assert!( + matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"), + "expected 'PinnedDrop', found: '{:?}'", + tt + ); + pinned_drop_idx = Some(i); + break; + } + } + let idx = pinned_drop_idx + .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`.")); + // Fully qualify the `PinnedDrop`, as to avoid any tampering. + toks.splice(idx..idx, quote!(::kernel::init::)); + // Take the `{}` body and call the declarative macro. + if let Some(TokenTree::Group(last)) = toks.pop() { + let last = last.stream(); + quote!(::kernel::__pinned_drop! { + @impl_sig(#(#toks)*), + @impl_body(#last), + }) + } else { + TokenStream::from_iter(toks) + } +} From 95f4f810fbebbd1aa8880185682c479c3beca21e Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:07 +0000 Subject: [PATCH 411/635] rust: init: add `stack_pin_init!` macro The `stack_pin_init!` macro allows pin-initializing a value on the stack. It accepts a `impl PinInit` to initialize a `T`. It allows propagating any errors via `?` or handling it normally via `match`. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20230408122429.1103522-11-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 140 +++++++++++++++++++++++++++++++-- rust/kernel/init/__internal.rs | 57 ++++++++++++++ 2 files changed, 191 insertions(+), 6 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 37e8159df24d91..99751375e7c81e 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -12,7 +12,8 @@ //! //! To initialize a `struct` with an in-place constructor you will need two things: //! - an in-place constructor, -//! - a memory location that can hold your `struct`. +//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc`], +//! [`UniqueArc`], [`Box`] or any other smart pointer that implements [`InPlaceInit`]). //! //! To get an in-place constructor there are generally three options: //! - directly creating an in-place constructor using the [`pin_init!`] macro, @@ -180,6 +181,7 @@ //! [pinning]: https://doc.rust-lang.org/std/pin/index.html //! [structurally pinned fields]: //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field +//! [stack]: crate::stack_pin_init //! [`Arc`]: crate::sync::Arc //! [`impl PinInit`]: PinInit //! [`impl PinInit`]: PinInit @@ -202,6 +204,132 @@ pub mod __internal; #[doc(hidden)] pub mod macros; +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; +/// # use macros::pin_data; +/// # use core::pin::Pin; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: Mutex, +/// b: Bar, +/// } +/// +/// #[pin_data] +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_pin_init!(let foo = pin_init!(Foo { +/// a <- new_mutex!(42), +/// b: Bar { +/// x: 64, +/// }, +/// })); +/// let foo: Pin<&mut Foo> = foo; +/// pr_info!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error +/// type, then use [`stack_try_pin_init!`]. +#[macro_export] +macro_rules! stack_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = match $crate::init::__internal::StackInit::init($var, val) { + Ok(res) => res, + Err(x) => { + let x: ::core::convert::Infallible = x; + match x {} + } + }; + }; +} + +/// Initialize and pin a type directly on the stack. +/// +/// # Examples +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; +/// # use macros::pin_data; +/// # use core::{alloc::AllocError, pin::Pin}; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: Mutex, +/// b: Box, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Result, AllocError> = pin_init!(Foo { +/// a <- new_mutex!(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// })); +/// let foo = foo.unwrap(); +/// pr_info!("a: {}", &*foo.a.lock()); +/// ``` +/// +/// ```rust +/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; +/// # use macros::pin_data; +/// # use core::{alloc::AllocError, pin::Pin}; +/// #[pin_data] +/// struct Foo { +/// #[pin] +/// a: Mutex, +/// b: Box, +/// } +/// +/// struct Bar { +/// x: u32, +/// } +/// +/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { +/// a <- new_mutex!(42), +/// b: Box::try_new(Bar { +/// x: 64, +/// })?, +/// })); +/// pr_info!("a: {}", &*foo.a.lock()); +/// # Ok::<_, AllocError>(()) +/// ``` +/// +/// # Syntax +/// +/// A normal `let` binding with optional type annotation. The expression is expected to implement +/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the +/// `=` will propagate this error. +#[macro_export] +macro_rules! stack_try_pin_init { + (let $var:ident $(: $t:ty)? = $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::init::__internal::StackInit::init($var, val); + }; + (let $var:ident $(: $t:ty)? =? $val:expr) => { + let val = $val; + let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); + let mut $var = $crate::init::__internal::StackInit::init($var, val)?; + }; +} + /// Construct an in-place, pinned initializer for `struct`s. /// /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use @@ -913,8 +1041,8 @@ macro_rules! try_init { /// A pin-initializer for the type `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`]. Use the [`InPlaceInit::pin_init`] function of a -/// smart pointer like [`Arc`] on this. +/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the +/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc`] on this. /// /// Also see the [module description](self). /// @@ -949,9 +1077,9 @@ pub unsafe trait PinInit: Sized { /// An initializer for `T`. /// /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can -/// be [`Box`], [`Arc`], [`UniqueArc`]. Use the [`InPlaceInit::init`] function of a smart -/// pointer like [`Arc`] on this. Because [`PinInit`] is a super trait, you can -/// use every function that takes it as well. +/// be [`Box`], [`Arc`], [`UniqueArc`] or even the stack (see [`stack_pin_init!`]). Use the +/// [`InPlaceInit::init`] function of a smart pointer like [`Arc`] on this. Because +/// [`PinInit`] is a super trait, you can use every function that takes it as well. /// /// Also see the [module description](self). /// diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 774cb620afa777..44751fb62b51a2 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -112,6 +112,63 @@ unsafe impl HasInitData for T { } } +/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive. +/// +/// # Invariants +/// +/// If `self.is_init` is true, then `self.value` is initialized. +/// +/// [`stack_pin_init`]: kernel::stack_pin_init +pub struct StackInit { + value: MaybeUninit, + is_init: bool, +} + +impl Drop for StackInit { + #[inline] + fn drop(&mut self) { + if self.is_init { + // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is + // true, `self.value` is initialized. + unsafe { self.value.assume_init_drop() }; + } + } +} + +impl StackInit { + /// Creates a new [`StackInit`] that is uninitialized. Use [`stack_pin_init`] instead of this + /// primitive. + /// + /// [`stack_pin_init`]: kernel::stack_pin_init + #[inline] + pub fn uninit() -> Self { + Self { + value: MaybeUninit::uninit(), + is_init: false, + } + } + + /// Initializes the contents and returns the result. + #[inline] + pub fn init(self: Pin<&mut Self>, init: impl PinInit) -> Result, E> { + // SAFETY: We never move out of `this`. + let this = unsafe { Pin::into_inner_unchecked(self) }; + // The value is currently initialized, so it needs to be dropped before we can reuse + // the memory (this is a safety guarantee of `Pin`). + if this.is_init { + this.is_init = false; + // SAFETY: `this.is_init` was true and therefore `this.value` is initialized. + unsafe { this.value.assume_init_drop() }; + } + // SAFETY: The memory slot is valid and this type ensures that it will stay pinned. + unsafe { init.__pinned_init(this.value.as_mut_ptr())? }; + // INVARIANT: `this.value` is initialized above. + this.is_init = true; + // SAFETY: The slot is now pinned, since we will never give access to `&mut T`. + Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) }) + } +} + /// When a value of this type is dropped, it drops a `T`. /// /// Can be forgotten to prevent the drop. From 4325523528a1486638b5cf25d9e0eb12a49591f5 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:12 +0000 Subject: [PATCH 412/635] rust: init: add `Zeroable` trait and `init::zeroed` function Add the `Zeroable` trait which marks types that can be initialized by writing `0x00` to every byte of the type. Also add the `init::zeroed` function that creates an initializer for a `Zeroable` type that writes `0x00` to every byte. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-12-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 97 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 99751375e7c81e..ffd539e2f5ef2e 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -195,8 +195,14 @@ use crate::{ }; use alloc::boxed::Box; use core::{ - alloc::AllocError, cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, - pin::Pin, ptr, + alloc::AllocError, + cell::Cell, + convert::Infallible, + marker::PhantomData, + mem::MaybeUninit, + num::*, + pin::Pin, + ptr::{self, NonNull}, }; #[doc(hidden)] @@ -1323,3 +1329,90 @@ pub unsafe trait PinnedDrop: __internal::HasPinData { /// automatically. fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); } + +/// Marker trait for types that can be initialized by writing just zeroes. +/// +/// # Safety +/// +/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, +/// this is not UB: +/// +/// ```rust,ignore +/// let val: Self = unsafe { core::mem::zeroed() }; +/// ``` +pub unsafe trait Zeroable {} + +/// Create a new zeroed T. +/// +/// The returned initializer will write `0x00` to every byte of the given `slot`. +#[inline] +pub fn zeroed() -> impl Init { + // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` + // and because we write all zeroes, the memory is initialized. + unsafe { + init_from_closure(|slot: *mut T| { + slot.write_bytes(0, 1); + Ok(()) + }) + } +} + +macro_rules! impl_zeroable { + ($($({$($generics:tt)*})? $t:ty, )*) => { + $(unsafe impl$($($generics)*)? Zeroable for $t {})* + }; +} + +impl_zeroable! { + // SAFETY: All primitives that are allowed to be zero. + bool, + char, + u8, u16, u32, u64, u128, usize, + i8, i16, i32, i64, i128, isize, + f32, f64, + + // SAFETY: These are ZSTs, there is nothing to zero. + {} PhantomData, core::marker::PhantomPinned, Infallible, (), + + // SAFETY: Type is allowed to take any value, including all zeros. + {} MaybeUninit, + + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). + Option, Option, Option, Option, + Option, Option, + Option, Option, Option, Option, + Option, Option, + + // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). + // + // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. + {} Option>, + {} Option>, + + // SAFETY: `null` pointer is valid. + // + // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be + // null. + // + // When `Pointee` gets stabilized, we could use + // `T: ?Sized where ::Metadata: Zeroable` + {} *mut T, {} *const T, + + // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be + // zero. + {} *mut [T], {} *const [T], *mut str, *const str, + + // SAFETY: `T` is `Zeroable`. + {} [T; N], {} Wrapping, +} + +macro_rules! impl_tuple_zeroable { + ($(,)?) => {}; + ($first:ident, $($t:ident),* $(,)?) => { + // SAFETY: All elements are zeroable and padding can be zero. + unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} + impl_tuple_zeroable!($($t),* ,); + } +} + +impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); From 5973717070a44b42d66f890927bf266358c80695 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:17 +0000 Subject: [PATCH 413/635] rust: prelude: add `pin-init` API items to prelude Add `pin-init` API macros and traits to the prelude. Signed-off-by: Benno Lossin Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-13-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/prelude.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index 0bc1c97e5604c6..fcdc511d2ce837 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -18,7 +18,7 @@ pub use core::pin::Pin; pub use alloc::{boxed::Box, vec::Vec}; #[doc(no_inline)] -pub use macros::{module, vtable}; +pub use macros::{module, pin_data, pinned_drop, vtable}; pub use super::build_assert; @@ -27,8 +27,12 @@ pub use super::build_assert; pub use super::dbg; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; +pub use super::{init, pin_init, try_init, try_pin_init}; + pub use super::static_assert; pub use super::error::{code::*, Error, Result}; pub use super::{str::CStr, ThisModule}; + +pub use super::init::{InPlaceInit, Init, PinInit}; From 5a07ccdcbc62265f271b564d7c36a8673ca0b772 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:22 +0000 Subject: [PATCH 414/635] rust: types: add `Opaque::ffi_init` This function allows to easily initialize `Opaque` with the pin-init API. `Opaque::ffi_init` takes a closure and returns a pin-initializer. This pin-initiailizer calls the given closure with a pointer to the inner `T`. Co-developed-by: Gary Guo Signed-off-by: Gary Guo Signed-off-by: Benno Lossin Reviewed-by: Andreas Hindborg Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20230408122429.1103522-14-y86-dev@protonmail.com [ Fixed typo. ] Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 9 +++++++++ rust/kernel/types.rs | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index ffd539e2f5ef2e..a1298c8bbda073 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -177,6 +177,14 @@ //! } //! ``` //! +//! For the special case where initializing a field is a single FFI-function call that cannot fail, +//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single +//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination +//! with [`pin_init!`]. +//! +//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside +//! the `kernel` crate. The [`sync`] module is a good starting point. +//! //! [`sync`]: kernel::sync //! [pinning]: https://doc.rust-lang.org/std/pin/index.html //! [structurally pinned fields]: @@ -187,6 +195,7 @@ //! [`impl PinInit`]: PinInit //! [`impl Init`]: Init //! [`Opaque`]: kernel::types::Opaque +//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init //! [`pin_data`]: ::macros::pin_data use crate::{ diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index ff2b2fac951d69..a4b1e3778da7d8 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -2,6 +2,7 @@ //! Kernel types. +use crate::init::{self, PinInit}; use alloc::boxed::Box; use core::{ cell::UnsafeCell, @@ -234,6 +235,25 @@ impl Opaque { Self(MaybeUninit::uninit()) } + /// Creates a pin-initializer from the given initializer closure. + /// + /// The returned initializer calls the given closure with the pointer to the inner `T` of this + /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. + /// + /// This function is safe, because the `T` inside of an `Opaque` is allowed to be + /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs + /// to verify at that point that the inner value is valid. + pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit { + // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully + // initialize the `T`. + unsafe { + init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { + init_func(Self::raw_get(slot)); + Ok(()) + }) + } + } + /// Returns a raw pointer to the opaque data. pub fn get(&self) -> *mut T { UnsafeCell::raw_get(self.0.as_ptr()) From 0e567a836b9c729f93860fe35c00cce7b62ba0c0 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:27 +0000 Subject: [PATCH 415/635] rust: sync: reduce stack usage of `UniqueArc::try_new_uninit` `UniqueArc::try_new_uninit` calls `Arc::try_new(MaybeUninit::uninit())`. This results in the uninitialized memory being placed on the stack, which may be arbitrarily large due to the generic `T` and thus could cause a stack overflow for large types. Change the implementation to use the pin-init API which enables in-place initialization. In particular it avoids having to first construct and then move the uninitialized memory from the stack into the final location. Signed-off-by: Benno Lossin Reviewed-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-15-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/lib.rs | 1 - rust/kernel/sync/arc.rs | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 821bd067151ce7..2d7606135ef61c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -28,7 +28,6 @@ #[cfg(not(CONFIG_RUST))] compile_error!("Missing kernel configuration for conditional compilation"); -#[allow(unused_extern_crates)] // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). extern crate self as kernel; diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index b45769a29541d1..de36d55e81d868 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -18,7 +18,8 @@ use crate::{ bindings, error::{self, Error}, - init::{InPlaceInit, Init, PinInit}, + init::{self, InPlaceInit, Init, PinInit}, + try_init, types::{ForeignOwnable, Opaque}, }; use alloc::boxed::Box; @@ -31,6 +32,7 @@ use core::{ pin::Pin, ptr::NonNull, }; +use macros::pin_data; mod std_vendor; @@ -125,6 +127,7 @@ pub struct Arc { _p: PhantomData>, } +#[pin_data] #[repr(C)] struct ArcInner { refcount: Opaque, @@ -505,9 +508,16 @@ impl UniqueArc { /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. pub fn try_new_uninit() -> Result>, AllocError> { - Ok(UniqueArc::> { + // INVARIANT: The refcount is initialised to a non-zero value. + let inner = Box::try_init::(try_init!(ArcInner { + // SAFETY: There are no safety requirements for this FFI call. + refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), + data <- init::uninit::(), + }? AllocError))?; + Ok(UniqueArc { // INVARIANT: The newly-created object has a ref-count of 1. - inner: Arc::try_new(MaybeUninit::uninit())?, + // SAFETY: The pointer from the `Box` is valid. + inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, }) } } From 13130673027981b64cde1c57a81e7c038f17e2c5 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sat, 8 Apr 2023 12:26:33 +0000 Subject: [PATCH 416/635] rust: sync: add functions for initializing `UniqueArc>` Add two functions `init_with` and `pin_init_with` to `UniqueArc>` to initialize the memory of already allocated `UniqueArc`s. This is useful when you want to allocate memory check some condition inside of a context where allocation is forbidden and then conditionally initialize an object. Signed-off-by: Benno Lossin Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Andreas Hindborg Link: https://lore.kernel.org/r/20230408122429.1103522-16-y86-dev@protonmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/sync/arc.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index de36d55e81d868..e6d20624246529 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -544,6 +544,30 @@ impl UniqueArc> { inner: unsafe { Arc::from_inner(inner.cast()) }, } } + + /// Initialize `self` using the given initializer. + pub fn init_with(mut self, init: impl Init) -> core::result::Result, E> { + // SAFETY: The supplied pointer is valid for initialization. + match unsafe { init.__init(self.as_mut_ptr()) } { + // SAFETY: Initialization completed successfully. + Ok(()) => Ok(unsafe { self.assume_init() }), + Err(err) => Err(err), + } + } + + /// Pin-initialize `self` using the given pin-initializer. + pub fn pin_init_with( + mut self, + init: impl PinInit, + ) -> core::result::Result>, E> { + // SAFETY: The supplied pointer is valid for initialization and we will later pin the value + // to ensure it does not move. + match unsafe { init.__pinned_init(self.as_mut_ptr()) } { + // SAFETY: Initialization completed successfully. + Ok(()) => Ok(unsafe { self.assume_init() }.into()), + Err(err) => Err(err), + } + } } impl From> for Pin> { From 2df309d224afdfd0d8204e43495fa1ab9d71009c Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Thu, 13 Apr 2023 10:02:17 +0000 Subject: [PATCH 417/635] rust: init: broaden the blanket impl of `Init` This makes it possible to use `T` as a `impl Init` for every error type `E` instead of just `Infallible`. Signed-off-by: Benno Lossin Reviewed-by: Gary Guo Reviewed-by: Martin Rodriguez Reboredo Link: https://lore.kernel.org/r/20230413100157.740697-1-benno.lossin@proton.me Signed-off-by: Miguel Ojeda --- rust/kernel/init.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index a1298c8bbda073..4ebfb08dab1154 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1190,8 +1190,8 @@ pub fn uninit() -> impl Init, E> { } // SAFETY: Every type can be initialized by-value. -unsafe impl Init for T { - unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { +unsafe impl Init for T { + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { unsafe { slot.write(self) }; Ok(()) } From d7d4ef07aebff9c04a7111b5a6c7e0ec936d7141 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Thu, 13 Apr 2023 00:18:22 +0200 Subject: [PATCH 418/635] MAINTAINERS: add Benno Lossin as Rust reviewer Benno has been involved with the Rust for Linux project for the better part of a year now. He has been working on solving the safe pinned initialization problem [1], which resulted in the pin-init API patch series [2] that allows to reduce the need for `unsafe` code in the kernel. He is also working on the field projection RFC for Rust [3] to bring pin-init as a language feature. His expertise with the language will be very useful to have around in the future if Rust grows within the kernel, thus add him to the `RUST` entry as reviewer. Link: https://rust-for-linux.com/the-safe-pinned-initialization-problem [1] Link: https://lore.kernel.org/rust-for-linux/20230408122429.1103522-1-y86-dev@protonmail.com/ [2] Link: https://github.com/rust-lang/rfcs/pull/3318 [3] Cc: Benno Lossin Reviewed-by: Alex Gaynor Link: https://lore.kernel.org/r/20230412221823.830135-1-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index c6545eb541047e..131266706b677f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18226,6 +18226,7 @@ M: Wedson Almeida Filho R: Boqun Feng R: Gary Guo R: Björn Roy Baron +R: Benno Lossin L: rust-for-linux@vger.kernel.org S: Supported W: https://github.com/Rust-for-Linux/linux From 21912cb0f5296f75fa63b508c03f91288f475405 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:31 -0300 Subject: [PATCH 419/635] rust: sync: introduce `LockClassKey` It is a wrapper around C's `lock_class_key`, which is used by the synchronisation primitives that are checked with lockdep. This is in preparation for introducing Rust abstractions for these primitives. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Waiman Long Reviewed-by: Martin Rodriguez Reboredo Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Wedson Almeida Filho Reviewed-by: Gary Guo Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20230411054543.21278-1-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/sync.rs | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 33da23e3076d2f..541d235ffbebe5 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -5,6 +5,51 @@ //! This module contains the kernel APIs related to synchronisation that have been ported or //! wrapped for usage by Rust code in the kernel. +use crate::types::Opaque; + mod arc; pub use arc::{Arc, ArcBorrow, UniqueArc}; + +/// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. +#[repr(transparent)] +pub struct LockClassKey(Opaque); + +// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and +// provides its own synchronization. +unsafe impl Sync for LockClassKey {} + +impl LockClassKey { + /// Creates a new lock class key. + pub const fn new() -> Self { + Self(Opaque::uninit()) + } + + #[allow(dead_code)] + pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { + self.0.get() + } +} + +/// Defines a new static lock class and returns a pointer to it. +#[doc(hidden)] +#[macro_export] +macro_rules! static_lock_class { + () => {{ + static CLASS: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new(); + &CLASS + }}; +} + +/// Returns the given string, if one is provided, otherwise generates one based on the source code +/// location. +#[doc(hidden)] +#[macro_export] +macro_rules! optional_name { + () => { + $crate::c_str!(::core::concat!(::core::file!(), ":", ::core::line!())) + }; + ($name:literal) => { + $crate::c_str!($name) + }; +} From 6cddabfde31bdbd5f4f1eb7a41f72dad63f5b5dd Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:32 -0300 Subject: [PATCH 420/635] rust: sync: introduce `Lock` and `Guard` They are generic Rust implementations of a lock and a lock guard that contain code that is common to all locks. Different backends will be introduced in subsequent commits. Reviewed-by: Martin Rodriguez Reboredo Suggested-by: Gary Guo Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20230411054543.21278-2-wedsonaf@gmail.com [ Fixed typo. ] Signed-off-by: Miguel Ojeda --- rust/kernel/sync.rs | 2 +- rust/kernel/sync/lock.rs | 162 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/sync/lock.rs diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 541d235ffbebe5..81b0998eaa18b6 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -8,6 +8,7 @@ use crate::types::Opaque; mod arc; +pub mod lock; pub use arc::{Arc, ArcBorrow, UniqueArc}; @@ -25,7 +26,6 @@ impl LockClassKey { Self(Opaque::uninit()) } - #[allow(dead_code)] pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { self.0.get() } diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs new file mode 100644 index 00000000000000..f2dbdc496c235f --- /dev/null +++ b/rust/kernel/sync/lock.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic kernel lock and guard. +//! +//! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes, +//! spinlocks, raw spinlocks) to be provided with minimal effort. + +use super::LockClassKey; +use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; +use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; +use macros::pin_data; + +/// The "backend" of a lock. +/// +/// It is the actual implementation of the lock, without the need to repeat patterns used in all +/// locks. +/// +/// # Safety +/// +/// - Implementers must ensure that only one thread/CPU may access the protected data once the lock +/// is owned, that is, between calls to `lock` and `unlock`. +pub unsafe trait Backend { + /// The state required by the lock. + type State; + + /// The state required to be kept between lock and unlock. + type GuardState; + + /// Initialises the lock. + /// + /// # Safety + /// + /// `ptr` must be valid for write for the duration of the call, while `name` and `key` must + /// remain valid for read indefinitely. + unsafe fn init( + ptr: *mut Self::State, + name: *const core::ffi::c_char, + key: *mut bindings::lock_class_key, + ); + + /// Acquires the lock, making the caller its owner. + /// + /// # Safety + /// + /// Callers must ensure that [`Backend::init`] has been previously called. + #[must_use] + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState; + + /// Releases the lock, giving up its ownership. + /// + /// # Safety + /// + /// It must only be called by the current owner of the lock. + unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState); +} + +/// A mutual exclusion primitive. +/// +/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock backend +/// specified as the generic parameter `B`. +#[pin_data] +pub struct Lock { + /// The kernel lock object. + #[pin] + state: Opaque, + + /// Some locks are known to be self-referential (e.g., mutexes), while others are architecture + /// or config defined (e.g., spinlocks). So we conservatively require them to be pinned in case + /// some architecture uses self-references now or in the future. + #[pin] + _pin: PhantomPinned, + + /// The data protected by the lock. + data: UnsafeCell, +} + +// SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can. +unsafe impl Send for Lock {} + +// SAFETY: `Lock` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl Sync for Lock {} + +impl Lock { + /// Constructs a new lock initialiser. + #[allow(clippy::new_ret_no_self)] + pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit { + pin_init!(Self { + data: UnsafeCell::new(t), + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }) + } +} + +impl Lock { + /// Acquires the lock and gives the caller access to the data protected by it. + pub fn lock(&self) -> Guard<'_, T, B> { + // SAFETY: The constructor of the type calls `init`, so the existence of the object proves + // that `init` was called. + let state = unsafe { B::lock(self.state.get()) }; + // SAFETY: The lock was just acquired. + unsafe { Guard::new(self, state) } + } +} + +/// A lock guard. +/// +/// Allows mutual exclusion primitives that implement the `Backend` trait to automatically unlock +/// when a guard goes out of scope. It also provides a safe and convenient way to access the data +/// protected by the lock. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, T: ?Sized, B: Backend> { + pub(crate) lock: &'a Lock, + pub(crate) state: B::GuardState, + _not_send: PhantomData<*mut ()>, +} + +// SAFETY: `Guard` is sync when the data protected by the lock is also sync. +unsafe impl Sync for Guard<'_, T, B> {} + +impl core::ops::Deref for Guard<'_, T, B> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &*self.lock.data.get() } + } +} + +impl core::ops::DerefMut for Guard<'_, T, B> { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &mut *self.lock.data.get() } + } +} + +impl Drop for Guard<'_, T, B> { + fn drop(&mut self) { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { B::unlock(self.lock.state.get(), &self.state) }; + } +} + +impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { + /// Constructs a new immutable lock guard. + /// + /// # Safety + /// + /// The caller must ensure that it owns the lock. + pub(crate) unsafe fn new(lock: &'a Lock, state: B::GuardState) -> Self { + Self { + lock, + state, + _not_send: PhantomData, + } + } +} From 50b24086500908bfafd84fb777c5eaec0c3f2e7d Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:33 -0300 Subject: [PATCH 421/635] rust: lock: introduce `Mutex` This is the `struct mutex` lock backend and allows Rust code to use the kernel mutex idiomatically. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Waiman Long Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20230411054543.21278-3-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/helpers.c | 7 ++ rust/kernel/sync.rs | 1 + rust/kernel/sync/lock.rs | 2 + rust/kernel/sync/lock/mutex.rs | 118 +++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 rust/kernel/sync/lock/mutex.rs diff --git a/rust/helpers.c b/rust/helpers.c index 04b9be46e88778..86af099d2d6671 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -22,6 +22,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -29,6 +30,12 @@ __noreturn void rust_helper_BUG(void) } EXPORT_SYMBOL_GPL(rust_helper_BUG); +void rust_helper_mutex_lock(struct mutex *lock) +{ + mutex_lock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_mutex_lock); + refcount_t rust_helper_REFCOUNT_INIT(int n) { return (refcount_t)REFCOUNT_INIT(n); diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 81b0998eaa18b6..693f0b7f4e4fd7 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -11,6 +11,7 @@ mod arc; pub mod lock; pub use arc::{Arc, ArcBorrow, UniqueArc}; +pub use lock::mutex::Mutex; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index f2dbdc496c235f..2ea512a8dfab2d 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -10,6 +10,8 @@ use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; +pub mod mutex; + /// The "backend" of a lock. /// /// It is the actual implementation of the lock, without the need to repeat patterns used in all diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs new file mode 100644 index 00000000000000..923472f04af4bd --- /dev/null +++ b/rust/kernel/sync/lock/mutex.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel mutex. +//! +//! This module allows Rust code to use the kernel's `struct mutex`. + +use crate::bindings; + +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + +/// A mutual exclusion primitive. +/// +/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, +/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is +/// unlocked, at which point another thread will be allowed to wake up and make progress. +/// +/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts. +/// +/// Instances of [`Mutex`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](crate::pin_init) and [`new_mutex`] macros. +/// +/// # Examples +/// +/// The following example shows how to declare, allocate and initialise a struct (`Example`) that +/// contains an inner struct (`Inner`) that is protected by a mutex. +/// +/// ``` +/// use kernel::{init::InPlaceInit, init::PinInit, new_mutex, pin_init, sync::Mutex}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// c: u32, +/// #[pin] +/// d: Mutex, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c: 10, +/// d <- new_mutex!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example`. +/// let e = Box::pin_init(Example::new())?; +/// assert_eq!(e.c, 10); +/// assert_eq!(e.d.lock().a, 20); +/// assert_eq!(e.d.lock().b, 30); +/// ``` +/// +/// The following example shows how to use interior mutability to modify the contents of a struct +/// protected by a mutex despite only having a shared reference: +/// +/// ``` +/// use kernel::sync::Mutex; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn example(m: &Mutex) { +/// let mut guard = m.lock(); +/// guard.a += 10; +/// guard.b += 20; +/// } +/// ``` +/// +/// [`struct mutex`]: ../../../../include/linux/mutex.h +pub type Mutex = super::Lock; + +/// A kernel `struct mutex` lock backend. +pub struct MutexBackend; + +// SAFETY: The underlying kernel `struct mutex` object ensures mutual exclusion. +unsafe impl super::Backend for MutexBackend { + type State = bindings::mutex; + type GuardState = (); + + unsafe fn init( + ptr: *mut Self::State, + name: *const core::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__mutex_init(ptr, name, key) } + } + + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::mutex_lock(ptr) }; + } + + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the mutex. + unsafe { bindings::mutex_unlock(ptr) }; + } +} From 09a7e303a44c3fa79ab1f6770e621b5f436b59bb Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Wed, 19 Apr 2023 14:44:26 -0300 Subject: [PATCH 422/635] rust: lock: introduce `SpinLock` This is the `spinlock_t` lock backend and allows Rust code to use the kernel spinlock idiomatically. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Waiman Long Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20230419174426.132207-1-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/helpers.c | 24 +++++++ rust/kernel/sync.rs | 2 +- rust/kernel/sync/lock.rs | 1 + rust/kernel/sync/lock/spinlock.rs | 116 ++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/sync/lock/spinlock.rs diff --git a/rust/helpers.c b/rust/helpers.c index 86af099d2d6671..446e3cfdb935c1 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -23,6 +23,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -36,6 +37,29 @@ void rust_helper_mutex_lock(struct mutex *lock) } EXPORT_SYMBOL_GPL(rust_helper_mutex_lock); +void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_SPINLOCK + __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); +#else + spin_lock_init(lock); +#endif +} +EXPORT_SYMBOL_GPL(rust_helper___spin_lock_init); + +void rust_helper_spin_lock(spinlock_t *lock) +{ + spin_lock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_lock); + +void rust_helper_spin_unlock(spinlock_t *lock) +{ + spin_unlock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); + refcount_t rust_helper_REFCOUNT_INIT(int n) { return (refcount_t)REFCOUNT_INIT(n); diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 693f0b7f4e4fd7..c997ff7e951e5b 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -11,7 +11,7 @@ mod arc; pub mod lock; pub use arc::{Arc, ArcBorrow, UniqueArc}; -pub use lock::mutex::Mutex; +pub use lock::{mutex::Mutex, spinlock::SpinLock}; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 2ea512a8dfab2d..3c166e08f3d4c5 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -11,6 +11,7 @@ use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; pub mod mutex; +pub mod spinlock; /// The "backend" of a lock. /// diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs new file mode 100644 index 00000000000000..a52d20fc975543 --- /dev/null +++ b/rust/kernel/sync/lock/spinlock.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel spinlock. +//! +//! This module allows Rust code to use the kernel's `spinlock_t`. + +use crate::bindings; + +/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_spinlock { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::SpinLock::new( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + +/// A spinlock. +/// +/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only +/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is +/// unlocked, at which point another CPU will be allowed to make progress. +/// +/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](crate::pin_init) and [`new_spinlock`] macros. +/// +/// # Examples +/// +/// The following example shows how to declare, allocate and initialise a struct (`Example`) that +/// contains an inner struct (`Inner`) that is protected by a spinlock. +/// +/// ``` +/// use kernel::{init::InPlaceInit, init::PinInit, new_spinlock, pin_init, sync::SpinLock}; +/// +/// struct Inner { +/// a: u32, +/// b: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// c: u32, +/// #[pin] +/// d: SpinLock, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit { +/// pin_init!(Self { +/// c: 10, +/// d <- new_spinlock!(Inner { a: 20, b: 30 }), +/// }) +/// } +/// } +/// +/// // Allocate a boxed `Example`. +/// let e = Box::pin_init(Example::new())?; +/// assert_eq!(e.c, 10); +/// assert_eq!(e.d.lock().a, 20); +/// assert_eq!(e.d.lock().b, 30); +/// ``` +/// +/// The following example shows how to use interior mutability to modify the contents of a struct +/// protected by a spinlock despite only having a shared reference: +/// +/// ``` +/// use kernel::sync::SpinLock; +/// +/// struct Example { +/// a: u32, +/// b: u32, +/// } +/// +/// fn example(m: &SpinLock) { +/// let mut guard = m.lock(); +/// guard.a += 10; +/// guard.b += 20; +/// } +/// ``` +/// +/// [`spinlock_t`]: ../../../../include/linux/spinlock.h +pub type SpinLock = super::Lock; + +/// A kernel `spinlock_t` lock backend. +pub struct SpinLockBackend; + +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. +unsafe impl super::Backend for SpinLockBackend { + type State = bindings::spinlock_t; + type GuardState = (); + + unsafe fn init( + ptr: *mut Self::State, + name: *const core::ffi::c_char, + key: *mut bindings::lock_class_key, + ) { + // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and + // `key` are valid for read indefinitely. + unsafe { bindings::__spin_lock_init(ptr, name, key) } + } + + unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { + // SAFETY: The safety requirements of this function ensure that `ptr` points to valid + // memory, and that it has been initialised before. + unsafe { bindings::spin_lock(ptr) } + } + + unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { + // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the + // caller is the owner of the mutex. + unsafe { bindings::spin_unlock(ptr) } + } +} From f6dff5d1b26a4c2e018d1154f9803c864d54c406 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:38 -0300 Subject: [PATCH 423/635] rust: introduce `ARef` This is an owned reference to an object that is always ref-counted. This is meant to be used in wrappers for C types that have their own ref counting functions, for example, tasks, files, inodes, dentries, etc. Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Reviewed-by: Gary Guo Link: https://lore.kernel.org/r/20230411054543.21278-8-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/types.rs | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index a4b1e3778da7d8..29db59d6119a9b 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,8 +6,10 @@ use crate::init::{self, PinInit}; use alloc::boxed::Box; use core::{ cell::UnsafeCell, + marker::PhantomData, mem::MaybeUninit, ops::{Deref, DerefMut}, + ptr::NonNull, }; /// Used to transfer ownership to and from foreign (non-Rust) languages. @@ -268,6 +270,111 @@ impl Opaque { } } +/// Types that are _always_ reference counted. +/// +/// It allows such types to define their own custom ref increment and decrement functions. +/// Additionally, it allows users to convert from a shared reference `&T` to an owned reference +/// [`ARef`]. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. For +/// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted +/// instances of a type. +/// +/// # Safety +/// +/// Implementers must ensure that increments to the reference count keep the object alive in memory +/// at least until matching decrements are performed. +/// +/// Implementers must also ensure that all instances are reference-counted. (Otherwise they +/// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object +/// alive.) +pub unsafe trait AlwaysRefCounted { + /// Increments the reference count on the object. + fn inc_ref(&self); + + /// Decrements the reference count on the object. + /// + /// Frees the object when the count reaches zero. + /// + /// # Safety + /// + /// Callers must ensure that there was a previous matching increment to the reference count, + /// and that the object is no longer used after its reference count is decremented (as it may + /// result in the object being freed), unless the caller owns another increment on the refcount + /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls + /// [`AlwaysRefCounted::dec_ref`] once). + unsafe fn dec_ref(obj: NonNull); +} + +/// An owned reference to an always-reference-counted object. +/// +/// The object's reference count is automatically decremented when an instance of [`ARef`] is +/// dropped. It is also automatically incremented when a new instance is created via +/// [`ARef::clone`]. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In +/// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. +pub struct ARef { + ptr: NonNull, + _p: PhantomData, +} + +impl ARef { + /// Creates a new instance of [`ARef`]. + /// + /// It takes over an increment of the reference count on the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the reference count was incremented at least once, and that they + /// are properly relinquishing one increment. That is, if there is only one increment, callers + /// must not use the underlying object anymore -- it is only safe to do so via the newly + /// created [`ARef`]. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // increment on the refcount. + Self { + ptr, + _p: PhantomData, + } + } +} + +impl Clone for ARef { + fn clone(&self) -> Self { + self.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(self.ptr) } + } +} + +impl Deref for ARef { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl From<&T> for ARef { + fn from(b: &T) -> Self { + b.inc_ref(); + // SAFETY: We just incremented the refcount above. + unsafe { Self::from_raw(NonNull::from(b)) } + } +} + +impl Drop for ARef { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to + // decrement. + unsafe { T::dec_ref(self.ptr) }; + } +} + /// A sum type that always holds either a value of type `L` or `R`. pub enum Either { /// Constructs an instance of [`Either`] containing a value of type `L`. From 32a49e2a26fef84ccce96656ab3331f2b09cd88d Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:39 -0300 Subject: [PATCH 424/635] rust: add basic `Task` It is an abstraction for C's `struct task_struct`. It implements `AlwaysRefCounted`, so the refcount of the wrapped object is managed safely on the Rust side. Cc: Ingo Molnar Cc: Peter Zijlstra Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20230411054543.21278-9-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 19 +++++++++ rust/kernel/lib.rs | 1 + rust/kernel/task.rs | 75 +++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 rust/kernel/task.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 75d85bd6c59211..03656a44a83f45 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -8,6 +8,7 @@ #include #include +#include /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; diff --git a/rust/helpers.c b/rust/helpers.c index 446e3cfdb935c1..6e5b2c953d367d 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -24,6 +24,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -60,6 +61,12 @@ void rust_helper_spin_unlock(spinlock_t *lock) } EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); +int rust_helper_signal_pending(struct task_struct *t) +{ + return signal_pending(t); +} +EXPORT_SYMBOL_GPL(rust_helper_signal_pending); + refcount_t rust_helper_REFCOUNT_INIT(int n) { return (refcount_t)REFCOUNT_INIT(n); @@ -96,6 +103,18 @@ long rust_helper_PTR_ERR(__force const void *ptr) } EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); +void rust_helper_get_task_struct(struct task_struct *t) +{ + get_task_struct(t); +} +EXPORT_SYMBOL_GPL(rust_helper_get_task_struct); + +void rust_helper_put_task_struct(struct task_struct *t) +{ + put_task_struct(t); +} +EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 2d7606135ef61c..ee27e10da479b2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -44,6 +44,7 @@ mod static_assert; pub mod std_vendor; pub mod str; pub mod sync; +pub mod task; pub mod types; #[doc(hidden)] diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs new file mode 100644 index 00000000000000..d70cad13195601 --- /dev/null +++ b/rust/kernel/task.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Tasks (threads and processes). +//! +//! C header: [`include/linux/sched.h`](../../../../include/linux/sched.h). + +use crate::{bindings, types::Opaque}; +use core::ptr; + +/// Wraps the kernel's `struct task_struct`. +/// +/// # Invariants +/// +/// All instances are valid tasks created by the C portion of the kernel. +/// +/// Instances of this type are always ref-counted, that is, a call to `get_task_struct` ensures +/// that the allocation remains valid at least until the matching call to `put_task_struct`. +#[repr(transparent)] +pub struct Task(pub(crate) Opaque); + +// SAFETY: It's OK to access `Task` through references from other threads because we're either +// accessing properties that don't change (e.g., `pid`, `group_leader`) or that are properly +// synchronised by C code (e.g., `signal_pending`). +unsafe impl Sync for Task {} + +/// The type of process identifiers (PIDs). +type Pid = bindings::pid_t; + +impl Task { + /// Returns the group leader of the given task. + pub fn group_leader(&self) -> &Task { + // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always + // have a valid group_leader. + let ptr = unsafe { *ptr::addr_of!((*self.0.get()).group_leader) }; + + // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, + // and given that a task has a reference to its group leader, we know it must be valid for + // the lifetime of the returned task reference. + unsafe { &*ptr.cast() } + } + + /// Returns the PID of the given task. + pub fn pid(&self) -> Pid { + // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always + // have a valid pid. + unsafe { *ptr::addr_of!((*self.0.get()).pid) } + } + + /// Determines whether the given task has pending signals. + pub fn signal_pending(&self) -> bool { + // SAFETY: By the type invariant, we know that `self.0` is valid. + unsafe { bindings::signal_pending(self.0.get()) != 0 } + } + + /// Wakes up the task. + pub fn wake_up(&self) { + // SAFETY: By the type invariant, we know that `self.0.get()` is non-null and valid. + // And `wake_up_process` is safe to be called for any valid task, even if the task is + // running. + unsafe { bindings::wake_up_process(self.0.get()) }; + } +} + +// SAFETY: The type invariants guarantee that `Task` is always ref-counted. +unsafe impl crate::types::AlwaysRefCounted for Task { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference means that the refcount is nonzero. + unsafe { bindings::get_task_struct(self.0.get()) }; + } + + unsafe fn dec_ref(obj: ptr::NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is nonzero. + unsafe { bindings::put_task_struct(obj.cast().as_ptr()) } + } +} From 9d938604d66789f038f5592b6b350fd4a895dcd3 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:40 -0300 Subject: [PATCH 425/635] rust: introduce `current` This allows Rust code to get a reference to the current task without having to increment the refcount, but still guaranteeing memory safety. Cc: Ingo Molnar Cc: Peter Zijlstra Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/r/20230411054543.21278-10-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/helpers.c | 6 ++++ rust/kernel/prelude.rs | 2 ++ rust/kernel/task.rs | 82 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/rust/helpers.c b/rust/helpers.c index 6e5b2c953d367d..2cb7b82053d818 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -103,6 +103,12 @@ long rust_helper_PTR_ERR(__force const void *ptr) } EXPORT_SYMBOL_GPL(rust_helper_PTR_ERR); +struct task_struct *rust_helper_get_current(void) +{ + return current; +} +EXPORT_SYMBOL_GPL(rust_helper_get_current); + void rust_helper_get_task_struct(struct task_struct *t) { get_task_struct(t); diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index fcdc511d2ce837..c28587d68ebc1a 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -36,3 +36,5 @@ pub use super::error::{code::*, Error, Result}; pub use super::{str::CStr, ThisModule}; pub use super::init::{InPlaceInit, Init, PinInit}; + +pub use super::current; diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index d70cad13195601..526d29a0ae2786 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -5,7 +5,17 @@ //! C header: [`include/linux/sched.h`](../../../../include/linux/sched.h). use crate::{bindings, types::Opaque}; -use core::ptr; +use core::{marker::PhantomData, ops::Deref, ptr}; + +/// Returns the currently running task. +#[macro_export] +macro_rules! current { + () => { + // SAFETY: Deref + addr-of below create a temporary `TaskRef` that cannot outlive the + // caller. + unsafe { &*$crate::task::Task::current() } + }; +} /// Wraps the kernel's `struct task_struct`. /// @@ -15,6 +25,42 @@ use core::ptr; /// /// Instances of this type are always ref-counted, that is, a call to `get_task_struct` ensures /// that the allocation remains valid at least until the matching call to `put_task_struct`. +/// +/// # Examples +/// +/// The following is an example of getting the PID of the current thread with zero additional cost +/// when compared to the C version: +/// +/// ``` +/// let pid = current!().pid(); +/// ``` +/// +/// Getting the PID of the current process, also zero additional cost: +/// +/// ``` +/// let pid = current!().group_leader().pid(); +/// ``` +/// +/// Getting the current task and storing it in some struct. The reference count is automatically +/// incremented when creating `State` and decremented when it is dropped: +/// +/// ``` +/// use kernel::{task::Task, types::ARef}; +/// +/// struct State { +/// creator: ARef, +/// index: u32, +/// } +/// +/// impl State { +/// fn new() -> Self { +/// Self { +/// creator: current!().into(), +/// index: 0, +/// } +/// } +/// } +/// ``` #[repr(transparent)] pub struct Task(pub(crate) Opaque); @@ -27,6 +73,40 @@ unsafe impl Sync for Task {} type Pid = bindings::pid_t; impl Task { + /// Returns a task reference for the currently executing task/thread. + /// + /// The recommended way to get the current task/thread is to use the + /// [`current`](crate::current) macro because it is safe. + /// + /// # Safety + /// + /// Callers must ensure that the returned object doesn't outlive the current task/thread. + pub unsafe fn current() -> impl Deref { + struct TaskRef<'a> { + task: &'a Task, + _not_send: PhantomData<*mut ()>, + } + + impl Deref for TaskRef<'_> { + type Target = Task; + + fn deref(&self) -> &Self::Target { + self.task + } + } + + // SAFETY: Just an FFI call with no additional safety requirements. + let ptr = unsafe { bindings::get_current() }; + + TaskRef { + // SAFETY: If the current thread is still running, the current task is valid. Given + // that `TaskRef` is not `Send`, we know it cannot be transferred to another thread + // (where it could potentially outlive the caller). + task: unsafe { &*ptr.cast() }, + _not_send: PhantomData, + } + } + /// Returns the group leader of the given task. pub fn group_leader(&self) -> &Task { // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always From a57ce62c6e3240f1421e75032a8d7d62b73bdbea Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Tue, 11 Apr 2023 02:45:43 -0300 Subject: [PATCH 426/635] rust: sync: introduce `LockedBy` This allows us to have data protected by a lock despite not being wrapped by it. Access is granted by providing evidence that the lock is held by the caller. Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Reviewed-by: Benno Lossin Link: https://lore.kernel.org/r/20230411054543.21278-13-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/kernel/sync.rs | 2 + rust/kernel/sync/lock.rs | 2 +- rust/kernel/sync/locked_by.rs | 156 ++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/sync/locked_by.rs diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index c997ff7e951e5b..5393b0d7661908 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -9,9 +9,11 @@ use crate::types::Opaque; mod arc; pub mod lock; +mod locked_by; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use lock::{mutex::Mutex, spinlock::SpinLock}; +pub use locked_by::LockedBy; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 3c166e08f3d4c5..46e3328b67a69f 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -74,7 +74,7 @@ pub struct Lock { _pin: PhantomPinned, /// The data protected by the lock. - data: UnsafeCell, + pub(crate) data: UnsafeCell, } // SAFETY: `Lock` can be transferred across thread boundaries iff the data it protects can. diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs new file mode 100644 index 00000000000000..b17ee5cd98f3eb --- /dev/null +++ b/rust/kernel/sync/locked_by.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A wrapper for data protected by a lock that does not wrap it. + +use super::{lock::Backend, lock::Lock}; +use crate::build_assert; +use core::{cell::UnsafeCell, mem::size_of, ptr}; + +/// Allows access to some data to be serialised by a lock that does not wrap it. +/// +/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., +/// [`super::Mutex`] or [`super::SpinLock`]. [`LockedBy`] is meant for cases when this is not +/// possible. For example, if a container has a lock and some data in the contained elements needs +/// to be protected by the same lock. +/// +/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it +/// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence +/// refers to the wrong instance of the lock. +/// +/// # Examples +/// +/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an +/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in +/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at +/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also +/// locked; we enforce at run time that the right `InnerDirectory` is locked. +/// +/// ``` +/// use kernel::sync::{LockedBy, Mutex}; +/// +/// struct InnerFile { +/// bytes_used: u64, +/// } +/// +/// struct File { +/// _ino: u32, +/// inner: LockedBy, +/// } +/// +/// struct InnerDirectory { +/// /// The sum of the bytes used by all files. +/// bytes_used: u64, +/// _files: Vec, +/// } +/// +/// struct Directory { +/// _ino: u32, +/// inner: Mutex, +/// } +/// +/// /// Prints `bytes_used` from both the directory and file. +/// fn print_bytes_used(dir: &Directory, file: &File) { +/// let guard = dir.inner.lock(); +/// let inner_file = file.inner.access(&guard); +/// pr_info!("{} {}", guard.bytes_used, inner_file.bytes_used); +/// } +/// +/// /// Increments `bytes_used` for both the directory and file. +/// fn inc_bytes_used(dir: &Directory, file: &File) { +/// let mut guard = dir.inner.lock(); +/// guard.bytes_used += 10; +/// +/// let file_inner = file.inner.access_mut(&mut guard); +/// file_inner.bytes_used += 10; +/// } +/// +/// /// Creates a new file. +/// fn new_file(ino: u32, dir: &Directory) -> File { +/// File { +/// _ino: ino, +/// inner: LockedBy::new(&dir.inner, InnerFile { bytes_used: 0 }), +/// } +/// } +/// ``` +pub struct LockedBy { + owner: *const U, + data: UnsafeCell, +} + +// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can. +unsafe impl Send for LockedBy {} + +// SAFETY: `LockedBy` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl Sync for LockedBy {} + +impl LockedBy { + /// Constructs a new instance of [`LockedBy`]. + /// + /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure + /// that the right owner is being used to access the protected data. If the owner is freed, the + /// data becomes inaccessible; if another instance of the owner is allocated *on the same + /// memory location*, the data becomes accessible again: none of this affects memory safety + /// because in any case at most one thread (or CPU) can access the protected data at a time. + pub fn new(owner: &Lock, data: T) -> Self { + build_assert!( + size_of::>() > 0, + "The lock type cannot be a ZST because it may be impossible to distinguish instances" + ); + Self { + owner: owner.data.get(), + data: UnsafeCell::new(data), + } + } +} + +impl LockedBy { + /// Returns a reference to the protected data when the caller provides evidence (via a + /// reference) that the owner is locked. + /// + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&U` that matches + /// the data protected by the lock without actually holding it. + /// + /// # Panics + /// + /// Panics if `owner` is different from the data protected by the lock used in + /// [`new`](LockedBy::new). + pub fn access<'a>(&'a self, owner: &'a U) -> &'a T { + build_assert!( + size_of::() > 0, + "`U` cannot be a ZST because `owner` wouldn't be unique" + ); + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that the owner is locked. + unsafe { &*self.data.get() } + } + + /// Returns a mutable reference to the protected data when the caller provides evidence (via a + /// mutable owner) that the owner is locked mutably. + /// + /// `U` cannot be a zero-sized type (ZST) because there are ways to get an `&mut U` that + /// matches the data protected by the lock without actually holding it. + /// + /// Showing a mutable reference to the owner is sufficient because we know no other references + /// can exist to it. + /// + /// # Panics + /// + /// Panics if `owner` is different from the data protected by the lock used in + /// [`new`](LockedBy::new). + pub fn access_mut<'a>(&'a self, owner: &'a mut U) -> &'a mut T { + build_assert!( + size_of::() > 0, + "`U` cannot be a ZST because `owner` wouldn't be unique" + ); + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that there is only one reference to the owner. + unsafe { &mut *self.data.get() } + } +} From f20e590ce0d5af36335262a4a727d0f1fe7b43fb Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Mon, 27 Mar 2023 15:43:08 -0300 Subject: [PATCH 427/635] rust: lock: add `Guard::do_unlocked` It releases the lock, executes some function provided by the caller, then reacquires the lock. This is preparation for the implementation of condvars, which will sleep after between unlocking and relocking. We need an explicit `relock` method for primitives like `SpinLock` that have an irqsave variant: we use the guard state to determine if the lock was originally acquired with the regular `lock` function or `lock_irqsave`. Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Link: https://lore.kernel.org/rust-for-linux/20230412121431.41627-1-wedsonaf@gmail.com/ [ Removed the irqsave bits as discussed. ] Signed-off-by: Miguel Ojeda --- rust/kernel/sync/lock.rs | 29 ++++++++++++++++++++++++++++- rust/kernel/sync/lock/spinlock.rs | 3 ++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 46e3328b67a69f..802465d1fa54bd 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -6,7 +6,7 @@ //! spinlocks, raw spinlocks) to be provided with minimal effort. use super::LockClassKey; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; +use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; @@ -22,6 +22,8 @@ pub mod spinlock; /// /// - Implementers must ensure that only one thread/CPU may access the protected data once the lock /// is owned, that is, between calls to `lock` and `unlock`. +/// - Implementers must also ensure that `relock` uses the same locking method as the original +/// lock operation. pub unsafe trait Backend { /// The state required by the lock. type State; @@ -55,6 +57,17 @@ pub unsafe trait Backend { /// /// It must only be called by the current owner of the lock. unsafe fn unlock(ptr: *mut Self::State, guard_state: &Self::GuardState); + + /// Reacquires the lock, making the caller its owner. + /// + /// # Safety + /// + /// Callers must ensure that `guard_state` comes from a previous call to [`Backend::lock`] (or + /// variant) that has been unlocked with [`Backend::unlock`] and will be relocked now. + unsafe fn relock(ptr: *mut Self::State, guard_state: &mut Self::GuardState) { + // SAFETY: The safety requirements ensure that the lock is initialised. + *guard_state = unsafe { Self::lock(ptr) }; + } } /// A mutual exclusion primitive. @@ -126,6 +139,20 @@ pub struct Guard<'a, T: ?Sized, B: Backend> { // SAFETY: `Guard` is sync when the data protected by the lock is also sync. unsafe impl Sync for Guard<'_, T, B> {} +impl Guard<'_, T, B> { + #[allow(dead_code)] + pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce()) { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { B::unlock(self.lock.state.get(), &self.state) }; + + // SAFETY: The lock was just unlocked above and is being relocked now. + let _relock = + ScopeGuard::new(|| unsafe { B::relock(self.lock.state.get(), &mut self.state) }); + + cb(); + } +} + impl core::ops::Deref for Guard<'_, T, B> { type Target = T; diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index a52d20fc975543..979b56464a4e9f 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -87,7 +87,8 @@ pub type SpinLock = super::Lock; /// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend; -// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. +// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the +// default implementation that always calls the same locking method. unsafe impl super::Backend for SpinLockBackend { type State = bindings::spinlock_t; type GuardState = (); From 46c8876b8514c565ef4a291f290527105836b5cb Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Sun, 26 Mar 2023 00:57:38 -0300 Subject: [PATCH 428/635] rust: sync: introduce `CondVar` This is the traditional condition variable or monitor synchronisation primitive. It is implemented with C's `wait_queue_head_t`. It allows users to release a lock and go to sleep while guaranteeing that notifications won't be missed. This is achieved by enqueuing a wait entry before releasing the lock. Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Waiman Long Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Wedson Almeida Filho Reviewed-by: Alice Ryhl Link: https://lore.kernel.org/r/20230411054543.21278-12-wedsonaf@gmail.com Signed-off-by: Miguel Ojeda --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 7 ++ rust/kernel/sync.rs | 2 + rust/kernel/sync/condvar.rs | 174 ++++++++++++++++++++++++++++++++ rust/kernel/sync/lock.rs | 1 - 5 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/sync/condvar.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 03656a44a83f45..50e7a76d545509 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -8,6 +8,7 @@ #include #include +#include #include /* `bindgen` gets confused at certain things. */ diff --git a/rust/helpers.c b/rust/helpers.c index 2cb7b82053d818..81e80261d597c6 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -25,6 +25,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -61,6 +62,12 @@ void rust_helper_spin_unlock(spinlock_t *lock) } EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); +void rust_helper_init_wait(struct wait_queue_entry *wq_entry) +{ + init_wait(wq_entry); +} +EXPORT_SYMBOL_GPL(rust_helper_init_wait); + int rust_helper_signal_pending(struct task_struct *t) { return signal_pending(t); diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 5393b0d7661908..d219ee518eff15 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -8,10 +8,12 @@ use crate::types::Opaque; mod arc; +mod condvar; pub mod lock; mod locked_by; pub use arc::{Arc, ArcBorrow, UniqueArc}; +pub use condvar::CondVar; pub use lock::{mutex::Mutex, spinlock::SpinLock}; pub use locked_by::LockedBy; diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs new file mode 100644 index 00000000000000..ed353399c4e56f --- /dev/null +++ b/rust/kernel/sync/condvar.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A condition variable. +//! +//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition +//! variable. + +use super::{lock::Backend, lock::Guard, LockClassKey}; +use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; +use core::marker::PhantomPinned; +use macros::pin_data; + +/// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_condvar { + ($($name:literal)?) => { + $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + +/// A conditional variable. +/// +/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to +/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And +/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or +/// [`CondVar::notify_all`]) or because the thread received a signal. It may also wake up +/// spuriously. +/// +/// Instances of [`CondVar`] need a lock class and to be pinned. The recommended way to create such +/// instances is with the [`pin_init`](crate::pin_init) and [`new_condvar`] macros. +/// +/// # Examples +/// +/// The following is an example of using a condvar with a mutex: +/// +/// ``` +/// use kernel::sync::{CondVar, Mutex}; +/// use kernel::{new_condvar, new_mutex}; +/// +/// #[pin_data] +/// pub struct Example { +/// #[pin] +/// value: Mutex, +/// +/// #[pin] +/// value_changed: CondVar, +/// } +/// +/// /// Waits for `e.value` to become `v`. +/// fn wait_for_value(e: &Example, v: u32) { +/// let mut guard = e.value.lock(); +/// while *guard != v { +/// e.value_changed.wait_uninterruptible(&mut guard); +/// } +/// } +/// +/// /// Increments `e.value` and notifies all potential waiters. +/// fn increment(e: &Example) { +/// *e.value.lock() += 1; +/// e.value_changed.notify_all(); +/// } +/// +/// /// Allocates a new boxed `Example`. +/// fn new_example() -> Result>> { +/// Box::pin_init(pin_init!(Example { +/// value <- new_mutex!(0), +/// value_changed <- new_condvar!(), +/// })) +/// } +/// ``` +/// +/// [`struct wait_queue_head`]: ../../../include/linux/wait.h +#[pin_data] +pub struct CondVar { + #[pin] + pub(crate) wait_list: Opaque, + + /// A condvar needs to be pinned because it contains a [`struct list_head`] that is + /// self-referential, so it cannot be safely moved once it is initialised. + #[pin] + _pin: PhantomPinned, +} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread. +#[allow(clippy::non_send_fields_in_send_ty)] +unsafe impl Send for CondVar {} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads +// concurrently. +unsafe impl Sync for CondVar {} + +impl CondVar { + /// Constructs a new condvar initialiser. + #[allow(clippy::new_ret_no_self)] + pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit { + pin_init!(Self { + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + wait_list <- Opaque::ffi_init(|slot| unsafe { + bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr()) + }), + }) + } + + fn wait_internal(&self, wait_state: u32, guard: &mut Guard<'_, T, B>) { + let wait = Opaque::::uninit(); + + // SAFETY: `wait` points to valid memory. + unsafe { bindings::init_wait(wait.get()) }; + + // SAFETY: Both `wait` and `wait_list` point to valid memory. + unsafe { + bindings::prepare_to_wait_exclusive(self.wait_list.get(), wait.get(), wait_state as _) + }; + + // SAFETY: No arguments, switches to another thread. + guard.do_unlocked(|| unsafe { bindings::schedule() }); + + // SAFETY: Both `wait` and `wait_list` point to valid memory. + unsafe { bindings::finish_wait(self.wait_list.get(), wait.get()) }; + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by + /// [`CondVar::notify_one`] or [`CondVar::notify_all`], or when the thread receives a signal. + /// It may also wake up spuriously. + /// + /// Returns whether there is a signal pending. + #[must_use = "wait returns if a signal is pending, so the caller must check the return value"] + pub fn wait(&self, guard: &mut Guard<'_, T, B>) -> bool { + self.wait_internal(bindings::TASK_INTERRUPTIBLE, guard); + crate::current!().signal_pending() + } + + /// Releases the lock and waits for a notification in uninterruptible mode. + /// + /// Similar to [`CondVar::wait`], except that the wait is not interruptible. That is, the + /// thread won't wake up due to signals. It may, however, wake up supirously. + pub fn wait_uninterruptible(&self, guard: &mut Guard<'_, T, B>) { + self.wait_internal(bindings::TASK_UNINTERRUPTIBLE, guard) + } + + /// Calls the kernel function to notify the appropriate number of threads with the given flags. + fn notify(&self, count: i32, flags: u32) { + // SAFETY: `wait_list` points to valid memory. + unsafe { + bindings::__wake_up( + self.wait_list.get(), + bindings::TASK_NORMAL, + count, + flags as _, + ) + }; + } + + /// Wakes a single waiter up, if any. + /// + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost + /// completely (as opposed to automatically waking up the next waiter). + pub fn notify_one(&self) { + self.notify(1, 0); + } + + /// Wakes all waiters up, if any. + /// + /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost + /// completely (as opposed to automatically waking up the next waiter). + pub fn notify_all(&self) { + self.notify(0, 0); + } +} diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 802465d1fa54bd..a2216325632dff 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -140,7 +140,6 @@ pub struct Guard<'a, T: ?Sized, B: Backend> { unsafe impl Sync for Guard<'_, T, B> {} impl Guard<'_, T, B> { - #[allow(dead_code)] pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce()) { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; From f17ee4663cd83c2c393b7a882c552e5e6c5824d9 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 3 Apr 2023 18:33:52 +0900 Subject: [PATCH 429/635] rust: uapi: Add UAPI crate This crate mirrors the `bindings` crate, but will contain only UAPI bindings. Unlike the bindings crate, drivers may directly use this crate if they have to interface with userspace. Initially, just bind the generic ioctl stuff. In the future, we would also like to add additional checks to ensure that all types exposed by this crate satisfy UAPI-safety guarantees (that is, they are safely castable to/from a "bag of bits"). [ Miguel: added support for the `rustdoc` and `rusttest` targets, since otherwise they fail, and we want to keep them working. ] Reviewed-by: Martin Rodriguez Reboredo Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230329-rust-uapi-v2-1-bca5fb4d4a12@asahilina.net Signed-off-by: Miguel Ojeda --- rust/.gitignore | 1 + rust/Makefile | 28 +++++++++++++++++++++++----- rust/kernel/lib.rs | 1 + rust/uapi/lib.rs | 27 +++++++++++++++++++++++++++ rust/uapi/uapi_helper.h | 9 +++++++++ 5 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 rust/uapi/lib.rs create mode 100644 rust/uapi/uapi_helper.h diff --git a/rust/.gitignore b/rust/.gitignore index 168cb26a31b999..21552992b401fa 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -2,6 +2,7 @@ bindings_generated.rs bindings_helpers_generated.rs +uapi_generated.rs exports_*_generated.h doc/ test/ diff --git a/rust/Makefile b/rust/Makefile index aef85e9e8eeb8d..7c9d9f11aec505 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -16,6 +16,9 @@ obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \ exports_kernel_generated.h +always-$(CONFIG_RUST) += uapi/uapi_generated.rs +obj-$(CONFIG_RUST) += uapi.o + ifdef CONFIG_RUST_BUILD_ASSERT_ALLOW obj-$(CONFIG_RUST) += build_error.o else @@ -113,7 +116,7 @@ rustdoc-alloc: $(src)/alloc/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE rustdoc-kernel: private rustc_target_flags = --extern alloc \ --extern build_error --extern macros=$(objtree)/$(obj)/libmacros.so \ - --extern bindings + --extern bindings --extern uapi rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ $(obj)/bindings.o FORCE @@ -141,6 +144,9 @@ rusttestlib-macros: $(src)/macros/lib.rs rusttest-prepare FORCE rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE $(call if_changed,rustc_test_library) +rusttestlib-uapi: $(src)/uapi/lib.rs rusttest-prepare FORCE + $(call if_changed,rustc_test_library) + quiet_cmd_rustdoc_test = RUSTDOC T $< cmd_rustdoc_test = \ OBJTREE=$(abspath $(objtree)) \ @@ -223,9 +229,10 @@ rusttest-macros: $(src)/macros/lib.rs rusttest-prepare FORCE $(call if_changed,rustdoc_test) rusttest-kernel: private rustc_target_flags = --extern alloc \ - --extern build_error --extern macros --extern bindings + --extern build_error --extern macros --extern bindings --extern uapi rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ - rusttestlib-build_error rusttestlib-macros rusttestlib-bindings FORCE + rusttestlib-build_error rusttestlib-macros rusttestlib-bindings \ + rusttestlib-uapi FORCE $(call if_changed,rustc_test) $(call if_changed,rustc_test_library) @@ -302,6 +309,12 @@ $(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ $(src)/bindgen_parameters FORCE $(call if_changed_dep,bindgen) +$(obj)/uapi/uapi_generated.rs: private bindgen_target_flags = \ + $(shell grep -v '^#\|^$$' $(srctree)/$(src)/bindgen_parameters) +$(obj)/uapi/uapi_generated.rs: $(src)/uapi/uapi_helper.h \ + $(src)/bindgen_parameters FORCE + $(call if_changed_dep,bindgen) + # See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn # with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here # given it is `libclang`; but for consistency, future Clang changes and/or @@ -402,10 +415,15 @@ $(obj)/bindings.o: $(src)/bindings/lib.rs \ $(obj)/bindings/bindings_helpers_generated.rs FORCE $(call if_changed_dep,rustc_library) +$(obj)/uapi.o: $(src)/uapi/lib.rs \ + $(obj)/compiler_builtins.o \ + $(obj)/uapi/uapi_generated.rs FORCE + $(call if_changed_dep,rustc_library) + $(obj)/kernel.o: private rustc_target_flags = --extern alloc \ - --extern build_error --extern macros --extern bindings + --extern build_error --extern macros --extern bindings --extern uapi $(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o $(obj)/build_error.o \ - $(obj)/libmacros.so $(obj)/bindings.o FORCE + $(obj)/libmacros.so $(obj)/bindings.o $(obj)/uapi.o FORCE $(call if_changed_dep,rustc_library) endif # CONFIG_RUST diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index ee27e10da479b2..dd3b0d69fdd158 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -50,6 +50,7 @@ pub mod types; #[doc(hidden)] pub use bindings; pub use macros; +pub use uapi; #[doc(hidden)] pub use build_error::build_error; diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs new file mode 100644 index 00000000000000..29f69f3a52de4e --- /dev/null +++ b/rust/uapi/lib.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! UAPI Bindings. +//! +//! Contains the bindings generated by `bindgen` for UAPI interfaces. +//! +//! This crate may be used directly by drivers that need to interact with +//! userspace APIs. + +#![no_std] +#![feature(core_ffi_c)] +// See . +#![cfg_attr(test, allow(deref_nullptr))] +#![cfg_attr(test, allow(unaligned_references))] +#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] +#![allow( + clippy::all, + missing_docs, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + improper_ctypes, + unreachable_pub, + unsafe_op_in_unsafe_fn +)] + +include!(concat!(env!("OBJTREE"), "/rust/uapi/uapi_generated.rs")); diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h new file mode 100644 index 00000000000000..301f5207f0238d --- /dev/null +++ b/rust/uapi/uapi_helper.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Header that contains the headers for which Rust UAPI bindings + * will be automatically generated by `bindgen`. + * + * Sorted alphabetically. + */ + +#include From aae52daef1e28fbbc9ae9033c670a03c1454039d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 3 Apr 2023 18:33:53 +0900 Subject: [PATCH 430/635] rust: ioctl: Add ioctl number manipulation functions Add simple 1:1 wrappers of the C ioctl number manipulation functions. Since these are macros we cannot bindgen them directly, and since they should be usable in const context we cannot use helper wrappers, so we'll have to reimplement them in Rust. Thankfully, the C headers do declare defines for the relevant bitfield positions, so we don't need to duplicate that. Signed-off-by: Asahi Lina Link: https://lore.kernel.org/r/20230329-rust-uapi-v2-2-bca5fb4d4a12@asahilina.net [ Moved the `#![allow(non_snake_case)]` to the usual place. ] Signed-off-by: Miguel Ojeda --- rust/kernel/ioctl.rs | 72 ++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 2 files changed, 73 insertions(+) create mode 100644 rust/kernel/ioctl.rs diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs new file mode 100644 index 00000000000000..c49e1a8d3fd075 --- /dev/null +++ b/rust/kernel/ioctl.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! ioctl() number definitions +//! +//! C header: [`include/asm-generic/ioctl.h`](../../../../include/asm-generic/ioctl.h) + +#![allow(non_snake_case)] + +use crate::build_assert; + +/// Build an ioctl number, analogous to the C macro of the same name. +#[inline(always)] +const fn _IOC(dir: u32, ty: u32, nr: u32, size: usize) -> u32 { + build_assert!(dir <= uapi::_IOC_DIRMASK); + build_assert!(ty <= uapi::_IOC_TYPEMASK); + build_assert!(nr <= uapi::_IOC_NRMASK); + build_assert!(size <= (uapi::_IOC_SIZEMASK as usize)); + + (dir << uapi::_IOC_DIRSHIFT) + | (ty << uapi::_IOC_TYPESHIFT) + | (nr << uapi::_IOC_NRSHIFT) + | ((size as u32) << uapi::_IOC_SIZESHIFT) +} + +/// Build an ioctl number for an argumentless ioctl. +#[inline(always)] +pub const fn _IO(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_NONE, ty, nr, 0) +} + +/// Build an ioctl number for an read-only ioctl. +#[inline(always)] +pub const fn _IOR(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::()) +} + +/// Build an ioctl number for an write-only ioctl. +#[inline(always)] +pub const fn _IOW(ty: u32, nr: u32) -> u32 { + _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::()) +} + +/// Build an ioctl number for a read-write ioctl. +#[inline(always)] +pub const fn _IOWR(ty: u32, nr: u32) -> u32 { + _IOC( + uapi::_IOC_READ | uapi::_IOC_WRITE, + ty, + nr, + core::mem::size_of::(), + ) +} + +/// Get the ioctl direction from an ioctl number. +pub const fn _IOC_DIR(nr: u32) -> u32 { + (nr >> uapi::_IOC_DIRSHIFT) & uapi::_IOC_DIRMASK +} + +/// Get the ioctl type from an ioctl number. +pub const fn _IOC_TYPE(nr: u32) -> u32 { + (nr >> uapi::_IOC_TYPESHIFT) & uapi::_IOC_TYPEMASK +} + +/// Get the ioctl number from an ioctl number. +pub const fn _IOC_NR(nr: u32) -> u32 { + (nr >> uapi::_IOC_NRSHIFT) & uapi::_IOC_NRMASK +} + +/// Get the ioctl size from an ioctl number. +pub const fn _IOC_SIZE(nr: u32) -> usize { + ((nr >> uapi::_IOC_SIZESHIFT) & uapi::_IOC_SIZEMASK) as usize +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index dd3b0d69fdd158..676995d4e46038 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -37,6 +37,7 @@ mod allocator; mod build_assert; pub mod error; pub mod init; +pub mod ioctl; pub mod prelude; pub mod print; mod static_assert; From 2c1790dedb8234986d3d084c6f26f58b10fd41c9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Wed, 25 Jan 2023 16:37:37 +0000 Subject: [PATCH 431/635] arm64: rust: Enable Rust support for AArch64 This commit provides the build flags for Rust for AArch64. The core Rust support already in the kernel does the rest. The Rust samples have been tested with this commit. [jcunliffe: Arm specific parts taken from Miguel's upstream tree] Signed-off-by: Miguel Ojeda Co-developed-by: Jamie Cunliffe Signed-off-by: Jamie Cunliffe Signed-off-by: Vincenzo Palazzo Reviewed-by: Vincenzo Palazzo Reviewed-by: Gary Guo --- Documentation/rust/arch-support.rst | 1 + arch/arm64/Kconfig | 1 + scripts/generate_rust_target.rs | 13 ++++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst index b91e9ef4d0c21e..9b022af2f649c4 100644 --- a/Documentation/rust/arch-support.rst +++ b/Documentation/rust/arch-support.rst @@ -15,6 +15,7 @@ support corresponds to ``S`` values in the ``MAINTAINERS`` file. ============ ================ ============================================== Architecture Level of support Constraints ============ ================ ============================================== +``arm64`` Maintained None. ``um`` Maintained ``x86_64`` only. ``x86`` Maintained ``x86_64`` only. ============ ================ ============================================== diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1023e896d46b89..72048a3bdaa5c7 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -211,6 +211,7 @@ config ARM64 select HAVE_FUNCTION_ARG_ACCESS_API select MMU_GATHER_RCU_TABLE_FREE select HAVE_RSEQ + select HAVE_RUST select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select HAVE_KPROBES diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs index 3c6cbe2b278d30..fe0e4ba54492ea 100644 --- a/scripts/generate_rust_target.rs +++ b/scripts/generate_rust_target.rs @@ -148,7 +148,18 @@ fn main() { let mut ts = TargetSpec::new(); // `llvm-target`s are taken from `scripts/Makefile.clang`. - if cfg.has("X86_64") { + if cfg.has("ARM64") { + ts.push("arch", "aarch64"); + ts.push( + "data-layout", + "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128", + ); + ts.push("disable-redzone", true); + ts.push("features", "+strict-align,+neon,+fp-armv8"); + ts.push("llvm-target", "aarch64-linux-gnu"); + ts.push("max-atomic-width", 128); + ts.push("target-pointer-width", "64"); + } else if cfg.has("X86_64") { ts.push("arch", "x86_64"); ts.push( "data-layout", From f9caa83b01acf004bb4ee48dea267cf22fc40ca3 Mon Sep 17 00:00:00 2001 From: Jamie Cunliffe Date: Wed, 25 Jan 2023 16:37:38 +0000 Subject: [PATCH 432/635] arm64: rust: Enable PAC support for Rust. Enable the PAC ret and BTI options in the Rust build flags to match the options that are used when building C. Signed-off-by: Jamie Cunliffe Reviewed-by: Vincenzo Palazzo --- arch/arm64/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 2d49aea0ff67a8..3fc31f436837dc 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -65,9 +65,11 @@ endif ifeq ($(CONFIG_ARM64_BTI_KERNEL),y) KBUILD_CFLAGS += -mbranch-protection=pac-ret+bti + KBUILD_RUSTFLAGS += -Z branch-protection=bti,pac-ret,leaf else ifeq ($(CONFIG_ARM64_PTR_AUTH_KERNEL),y) ifeq ($(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET),y) KBUILD_CFLAGS += -mbranch-protection=pac-ret + KBUILD_RUSTFLAGS += -Z branch-protection=pac-ret,leaf else KBUILD_CFLAGS += -msign-return-address=non-leaf endif From 5a0a4dbfb657d7d0e5ae878e5a5df93bb9e31481 Mon Sep 17 00:00:00 2001 From: Jamie Cunliffe Date: Wed, 25 Jan 2023 16:37:39 +0000 Subject: [PATCH 433/635] arm64: rust: Disable neon and fp target features. Disable the neon and fp target features to avoid fp & simd registers. The use of fp-armv8 will cause a warning from rustc about an unknown feature that is specified. The target feature is still passed through to LLVM, this behaviour is documented as part of the warning. This will be fixed in a future version of the rustc toolchain. Signed-off-by: Jamie Cunliffe Reviewed-by: Vincenzo Palazzo --- arch/arm64/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 3fc31f436837dc..e548b0c055ffe4 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -41,6 +41,8 @@ KBUILD_CFLAGS += -mgeneral-regs-only \ KBUILD_CFLAGS += $(call cc-disable-warning, psabi) KBUILD_AFLAGS += $(compat_vdso) +KBUILD_RUSTFLAGS += -C target-feature="-neon,-fp-armv8" + KBUILD_CFLAGS += $(call cc-option,-mabi=lp64) KBUILD_AFLAGS += $(call cc-option,-mabi=lp64) From 2c2c013d8aee15362b65ee05775cf172edf529b8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Oct 2022 20:56:03 +0900 Subject: [PATCH 434/635] rust: time: New module for timekeeping functions This module is intended to contain functions related to kernel timekeeping and time. Initially, this just wraps ktime_get() and ktime_get_boottime() and returns them as core::time::Duration instances. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 3 ++- rust/kernel/lib.rs | 1 + rust/kernel/time.rs | 23 +++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/time.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 50e7a76d545509..3faad774b89ca0 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -6,10 +6,11 @@ * Sorted alphabetically. */ -#include +#include #include #include #include +#include /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 676995d4e46038..5282db94b0a8b4 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -46,6 +46,7 @@ pub mod std_vendor; pub mod str; pub mod sync; pub mod task; +pub mod time; pub mod types; #[doc(hidden)] diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs new file mode 100644 index 00000000000000..881f45492268ba --- /dev/null +++ b/rust/kernel/time.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Timekeeping functions. +//! +//! C header: [`include/linux/ktime.h`](../../../../include/linux/ktime.h) +//! C header: [`include/linux/timekeeping.h`](../../../../include/linux/timekeeping.h) + +use crate::bindings; +use core::time::Duration; + +/// Returns the kernel time elapsed since boot, excluding time spent sleeping, as a [`Duration`]. +pub fn ktime_get() -> Duration { + Duration::from_nanos(unsafe { bindings::ktime_get() }.try_into().unwrap()) +} + +/// Returns the kernel time elapsed since boot, including time spent sleeping, as a [`Duration`]. +pub fn ktime_get_boottime() -> Duration { + Duration::from_nanos( + unsafe { bindings::ktime_get_with_offset(bindings::tk_offsets_TK_OFFS_BOOT) } + .try_into() + .unwrap(), + ) +} From 373eae7cf8f1f0bc2fea236ce275ad8ad0483fa4 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 11 Jan 2023 19:54:58 +0900 Subject: [PATCH 435/635] rust: xarray: Add an abstraction for XArray The XArray is an abstract data type which behaves like a very large array of pointers. Add a Rust abstraction for this data type. The initial implementation uses explicit locking on get operations and returns a guard which blocks mutation, ensuring that the referenced object remains alive. To avoid excessive serialization, users are expected to use an inner type that can be efficiently cloned (such as Arc), and eagerly clone and drop the guard to unblock other users after a lookup. Future variants may support using RCU instead to avoid mutex locking. This abstraction also introduces a reservation mechanism, which can be used by alloc-capable XArrays to reserve a free slot without immediately filling it, and then do so at a later time. If the reservation is dropped without being filled, the slot is freed again for other users, which eliminates the need for explicit cleanup code. Signed-off-by: Asahi Lina --- Hi everyone! This abstraction is part of the set of dependencies for the drm/asahi Apple M1/M2 GPU driver. The branch at [1] contains the full series of patches rebased on upstream leading to the complete driver, for reference on how it is intended to be used. Thank you everyone who helped review this on GitHub [2]! I hope I didn't forget any CCs... Note that I dropped the convenience `Deref` impl for `Guard`, since I couldn't figure out how to do it safely. Suggestions welcome, or we can leave it for a future improvement ^^ [1] https://github.com/AsahiLinux/linux/tree/gpu/rebase-20230224 [2] https://github.com/Rust-for-Linux/linux/pull/952 Changes in v3: - Updated to the error v2/v3 series API. - Renamed `err` to `ret` for consistency with the other instance. - Link to v2: https://lore.kernel.org/r/20230224-rust-xarray-v2-1-4eeb0134944c@asahilina.net Changes in v2: - Added Pin requirement for all XArray operations, to close a soundness hole due to the lock in the XArray (locks are not safe to move while locked). Creation does not require pinning in place, since the lock cannot be acquired at that point. - Added safety note to Drop impl about why we don't need to do the lock unlock dance to ensure soundness in case of a dropped lock guard. - Downstream drm/asahi driver was also rebased on this version to prove it works (previously it was still on a pre-v1 version). - This still depends on the Error series (v1). v2 of that will need a trivial rename of Error::from_kernel_errno -> Error::from_errno. If this version of XArray ends up looking good, I'll send a trivial v4 of XArray with the rename, after sending the v2 of the Error series. - Link to v1: https://lore.kernel.org/r/20230224-rust-xarray-v1-1-80f0904ce5d3@asahilina.net --- rust/bindings/bindings_helper.h | 17 ++ rust/helpers.c | 37 ++++ rust/kernel/lib.rs | 1 + rust/kernel/xarray.rs | 302 ++++++++++++++++++++++++++++++++ 4 files changed, 357 insertions(+) create mode 100644 rust/kernel/xarray.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 3faad774b89ca0..f84e7e2c9d891e 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -11,7 +11,24 @@ #include #include #include +#include /* `bindgen` gets confused at certain things. */ const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO; + +const gfp_t BINDINGS_XA_FLAGS_LOCK_IRQ = XA_FLAGS_LOCK_IRQ; +const gfp_t BINDINGS_XA_FLAGS_LOCK_BH = XA_FLAGS_LOCK_BH; +const gfp_t BINDINGS_XA_FLAGS_TRACK_FREE = XA_FLAGS_TRACK_FREE; +const gfp_t BINDINGS_XA_FLAGS_ZERO_BUSY = XA_FLAGS_ZERO_BUSY; +const gfp_t BINDINGS_XA_FLAGS_ALLOC_WRAPPED = XA_FLAGS_ALLOC_WRAPPED; +const gfp_t BINDINGS_XA_FLAGS_ACCOUNT = XA_FLAGS_ACCOUNT; +const gfp_t BINDINGS_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC; +const gfp_t BINDINGS_XA_FLAGS_ALLOC1 = XA_FLAGS_ALLOC1; + +const xa_mark_t BINDINGS_XA_MARK_0 = XA_MARK_0; +const xa_mark_t BINDINGS_XA_MARK_1 = XA_MARK_1; +const xa_mark_t BINDINGS_XA_MARK_2 = XA_MARK_2; +const xa_mark_t BINDINGS_XA_PRESENT = XA_PRESENT; +const xa_mark_t BINDINGS_XA_MARK_MAX = XA_MARK_MAX; +const xa_mark_t BINDINGS_XA_FREE_MARK = XA_FREE_MARK; diff --git a/rust/helpers.c b/rust/helpers.c index 81e80261d597c6..7c7661ed53eabf 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -26,6 +26,7 @@ #include #include #include +#include __noreturn void rust_helper_BUG(void) { @@ -128,6 +129,42 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); +void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags) +{ + xa_init_flags(xa, flags); +} +EXPORT_SYMBOL_GPL(rust_helper_xa_init_flags); + +bool rust_helper_xa_empty(struct xarray *xa) +{ + return xa_empty(xa); +} +EXPORT_SYMBOL_GPL(rust_helper_xa_empty); + +int rust_helper_xa_alloc(struct xarray *xa, u32 *id, void *entry, struct xa_limit limit, gfp_t gfp) +{ + return xa_alloc(xa, id, entry, limit, gfp); +} +EXPORT_SYMBOL_GPL(rust_helper_xa_alloc); + +void rust_helper_xa_lock(struct xarray *xa) +{ + xa_lock(xa); +} +EXPORT_SYMBOL_GPL(rust_helper_xa_lock); + +void rust_helper_xa_unlock(struct xarray *xa) +{ + xa_unlock(xa); +} +EXPORT_SYMBOL_GPL(rust_helper_xa_unlock); + +int rust_helper_xa_err(void *entry) +{ + return xa_err(entry); +} +EXPORT_SYMBOL_GPL(rust_helper_xa_err); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 5282db94b0a8b4..7b364354a3870e 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -48,6 +48,7 @@ pub mod sync; pub mod task; pub mod time; pub mod types; +pub mod xarray; #[doc(hidden)] pub use bindings; diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs new file mode 100644 index 00000000000000..365a3288a78752 --- /dev/null +++ b/rust/kernel/xarray.rs @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! XArray abstraction. +//! +//! C header: [`include/linux/xarray.h`](../../include/linux/xarray.h) + +use crate::{ + bindings, + error::{Error, Result}, + types::{ForeignOwnable, Opaque, ScopeGuard}, +}; +use core::{ + marker::{PhantomData, PhantomPinned}, + pin::Pin, + ptr::NonNull, +}; + +/// Flags passed to `XArray::new` to configure the `XArray`. +type Flags = bindings::gfp_t; + +/// Flag values passed to `XArray::new` to configure the `XArray`. +pub mod flags { + /// Use IRQ-safe locking. + pub const LOCK_IRQ: super::Flags = bindings::BINDINGS_XA_FLAGS_LOCK_IRQ; + /// Use softirq-safe locking. + pub const LOCK_BH: super::Flags = bindings::BINDINGS_XA_FLAGS_LOCK_BH; + /// Track which entries are free (distinct from None). + pub const TRACK_FREE: super::Flags = bindings::BINDINGS_XA_FLAGS_TRACK_FREE; + /// Initialize array index 0 as busy. + pub const ZERO_BUSY: super::Flags = bindings::BINDINGS_XA_FLAGS_ZERO_BUSY; + /// Use GFP_ACCOUNT for internal memory allocations. + pub const ACCOUNT: super::Flags = bindings::BINDINGS_XA_FLAGS_ACCOUNT; + /// Create an allocating `XArray` starting at index 0. + pub const ALLOC: super::Flags = bindings::BINDINGS_XA_FLAGS_ALLOC; + /// Create an allocating `XArray` starting at index 1. + pub const ALLOC1: super::Flags = bindings::BINDINGS_XA_FLAGS_ALLOC1; +} + +/// Wrapper for a value owned by the `XArray` which holds the `XArray` lock until dropped. +pub struct Guard<'a, T: ForeignOwnable>(NonNull, Pin<&'a XArray>); + +impl<'a, T: ForeignOwnable> Guard<'a, T> { + /// Borrow the underlying value wrapped by the `Guard`. + /// + /// Returns a `T::Borrowed` type for the owned `ForeignOwnable` type. + pub fn borrow(&self) -> T::Borrowed<'_> { + // SAFETY: The value is owned by the `XArray`, the lifetime it is borrowed for must not + // outlive the `XArray` itself, nor the Guard that holds the lock ensuring the value + // remains in the `XArray`. + unsafe { T::borrow(self.0.as_ptr() as _) } + } +} + +impl<'a, T: ForeignOwnable> Drop for Guard<'a, T> { + fn drop(&mut self) { + // SAFETY: The XArray we have a reference to owns the C xarray object. + unsafe { bindings::xa_unlock(self.1.xa.get()) }; + } +} + +/// Represents a reserved slot in an `XArray`, which does not yet have a value but has an assigned +/// index and may not be allocated by any other user. If the Reservation is dropped without +/// being filled, the entry is marked as available again. +/// +/// Users must ensure that reserved slots are not filled by other mechanisms, or otherwise their +/// contents may be dropped and replaced (which will print a warning). +pub struct Reservation<'a, T: ForeignOwnable>(Pin<&'a XArray>, usize, PhantomData); + +impl<'a, T: ForeignOwnable> Reservation<'a, T> { + /// Stores a value into the reserved slot. + pub fn store(self, value: T) -> Result { + if self.0.replace(self.1, value)?.is_some() { + crate::pr_err!("XArray: Reservation stored but the entry already had data!\n"); + // Consider it a success anyway, not much we can do + } + let index = self.1; + // The reservation is now fulfilled, so do not run our destructor. + core::mem::forget(self); + Ok(index) + } + + /// Returns the index of this reservation. + pub fn index(&self) -> usize { + self.1 + } +} + +impl<'a, T: ForeignOwnable> Drop for Reservation<'a, T> { + fn drop(&mut self) { + if self.0.remove(self.1).is_some() { + crate::pr_err!("XArray: Reservation dropped but the entry was not empty!\n"); + } + } +} + +/// An array which efficiently maps sparse integer indices to owned objects. +/// +/// This is similar to a `Vec>`, but more efficient when there are holes in the +/// index space, and can be efficiently grown. +/// +/// This structure is expected to often be used with an inner type that can either be efficiently +/// cloned, such as an `Arc`. +pub struct XArray { + xa: Opaque, + _p: PhantomData, + _q: PhantomPinned, +} + +impl XArray { + /// Creates a new `XArray` with the given flags. + pub fn new(flags: Flags) -> XArray { + let xa = Opaque::uninit(); + + // SAFETY: We have just created `xa`. This data structure does not require + // pinning. + unsafe { bindings::xa_init_flags(xa.get(), flags) }; + + // INVARIANT: Initialize the `XArray` with a valid `xa`. + XArray { + xa, + _p: PhantomData, + _q: PhantomPinned, + } + } + + /// Replaces an entry with a new value, returning the old value (if any). + pub fn replace(self: Pin<&Self>, index: usize, value: T) -> Result> { + let new = value.into_foreign(); + // SAFETY: `new` just came from into_foreign(), and we dismiss this guard if + // the xa_store operation succeeds and takes ownership of the pointer. + let guard = ScopeGuard::new(|| unsafe { + T::from_foreign(new); + }); + + // SAFETY: `self.xa` is always valid by the type invariant, and we are storing + // a `T::into_foreign()` result which upholds the later invariants. + let old = unsafe { + bindings::xa_store( + self.xa.get(), + index.try_into()?, + new as *mut _, + bindings::GFP_KERNEL, + ) + }; + + let ret = unsafe { bindings::xa_err(old) }; + if ret != 0 { + Err(Error::from_errno(ret)) + } else if old.is_null() { + guard.dismiss(); + Ok(None) + } else { + guard.dismiss(); + // SAFETY: The old value must have been stored by either this function or + // `alloc_limits_opt`, both of which ensure non-NULL entries are valid + // ForeignOwnable pointers. + Ok(Some(unsafe { T::from_foreign(old) })) + } + } + + /// Replaces an entry with a new value, dropping the old value (if any). + pub fn set(self: Pin<&Self>, index: usize, value: T) -> Result { + self.replace(index, value)?; + Ok(()) + } + + /// Looks up and returns a reference to an entry in the array, returning a `Guard` if it + /// exists. + /// + /// This guard blocks all other actions on the `XArray`. Callers are expected to drop the + /// `Guard` eagerly to avoid blocking other users, such as by taking a clone of the value. + pub fn get(self: Pin<&Self>, index: usize) -> Option> { + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_lock(self.xa.get()) }; + + // SAFETY: `self.xa` is always valid by the type invariant. + let guard = ScopeGuard::new(|| unsafe { bindings::xa_unlock(self.xa.get()) }); + + // SAFETY: `self.xa` is always valid by the type invariant. + let p = unsafe { bindings::xa_load(self.xa.get(), index.try_into().ok()?) }; + + NonNull::new(p as *mut T).map(|p| { + guard.dismiss(); + Guard(p, self) + }) + } + + /// Removes and returns an entry, returning it if it existed. + pub fn remove(self: Pin<&Self>, index: usize) -> Option { + let p = unsafe { bindings::xa_erase(self.xa.get(), index.try_into().ok()?) }; + if p.is_null() { + None + } else { + Some(unsafe { T::from_foreign(p) }) + } + } + + /// Allocates a new index in the array, optionally storing a new value into it, with + /// configurable bounds for the index range to allocate from. + /// + /// If `value` is `None`, then the index is reserved from further allocation but remains + /// free for storing a value into it. + fn alloc_limits_opt(self: Pin<&Self>, value: Option, min: u32, max: u32) -> Result { + let new = value.map_or(core::ptr::null(), |a| a.into_foreign()); + let mut id: u32 = 0; + + // SAFETY: `self.xa` is always valid by the type invariant. If this succeeds, it + // takes ownership of the passed `T` (if any). If it fails, we must drop the + // `T` again. + let ret = unsafe { + bindings::xa_alloc( + self.xa.get(), + &mut id, + new as *mut _, + bindings::xa_limit { min, max }, + bindings::GFP_KERNEL, + ) + }; + + if ret < 0 { + // Make sure to drop the value we failed to store + if !new.is_null() { + // SAFETY: If `new` is not NULL, it came from the `ForeignOwnable` we got + // from the caller. + unsafe { T::from_foreign(new) }; + } + Err(Error::from_errno(ret)) + } else { + Ok(id as usize) + } + } + + /// Allocates a new index in the array, storing a new value into it, with configurable + /// bounds for the index range to allocate from. + pub fn alloc_limits(self: Pin<&Self>, value: T, min: u32, max: u32) -> Result { + self.alloc_limits_opt(Some(value), min, max) + } + + /// Allocates a new index in the array, storing a new value into it. + pub fn alloc(self: Pin<&Self>, value: T) -> Result { + self.alloc_limits(value, 0, u32::MAX) + } + + /// Reserves a new index in the array within configurable bounds for the index. + /// + /// Returns a `Reservation` object, which can then be used to store a value at this index or + /// otherwise free it for reuse. + pub fn reserve_limits(self: Pin<&Self>, min: u32, max: u32) -> Result> { + Ok(Reservation( + self, + self.alloc_limits_opt(None, min, max)?, + PhantomData, + )) + } + + /// Reserves a new index in the array. + /// + /// Returns a `Reservation` object, which can then be used to store a value at this index or + /// otherwise free it for reuse. + pub fn reserve(self: Pin<&Self>) -> Result> { + Ok(Reservation( + self, + self.alloc_limits_opt(None, 0, u32::MAX)?, + PhantomData, + )) + } +} + +impl Drop for XArray { + fn drop(&mut self) { + // SAFETY: `self.xa` is valid by the type invariant, and as we have the only reference to + // the `XArray` we can safely iterate its contents and drop everything. + unsafe { + let mut index: core::ffi::c_ulong = 0; + let mut entry = bindings::xa_find( + self.xa.get(), + &mut index, + core::ffi::c_ulong::MAX, + bindings::BINDINGS_XA_PRESENT, + ); + while !entry.is_null() { + T::from_foreign(entry); + entry = bindings::xa_find_after( + self.xa.get(), + &mut index, + core::ffi::c_ulong::MAX, + bindings::BINDINGS_XA_PRESENT, + ); + } + + // Locked locks are not safe to drop. Normally we would want to try_lock()/unlock() here + // for safety or something similar, but in this case xa_destroy() is guaranteed to + // acquire the lock anyway. This will deadlock if a lock guard was improperly dropped, + // but that is not UB, so it's sufficient for soundness purposes. + bindings::xa_destroy(self.xa.get()); + } + } +} + +// SAFETY: XArray is thread-safe and all mutation operations are internally locked. +unsafe impl Send for XArray {} +unsafe impl Sync for XArray {} From 0ac04d11393186365dfd5cb4ba9a2568923b86cb Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 5 Feb 2023 20:29:01 +0900 Subject: [PATCH 436/635] rust: Add a Sealed trait Some traits exposed by the kernel crate may not be intended to be implemented by downstream modules. Add a Sealed trait to allow avoiding this using the sealed trait pattern. Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 7b364354a3870e..efda00a1a3160a 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -58,6 +58,11 @@ pub use uapi; #[doc(hidden)] pub use build_error::build_error; +pub(crate) mod private { + #[allow(unreachable_pub)] + pub trait Sealed {} +} + /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; From 56ddbbf61c8d3b5809287d5daf5cc2c0b2946523 Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Thu, 18 Nov 2021 00:30:04 +0000 Subject: [PATCH 437/635] rust: device: Add an abstraction for devices Add a RawDevice trait which can be implemented by any type representing a device class (such as a PlatformDevice), and a Device type which represents an owned reference to a generic struct device. Lina: Rewrote commit message, dropped the Amba bits, and squashed in simple changes to the core RawDevice/Device code from latter commits in rust-for-linux/rust. Also include the rust_helper_dev_get_drvdata helper which will be needed by consumers later on. Co-developed-by: Miguel Ojeda Signed-off-by: Miguel Ojeda Signed-off-by: Wedson Almeida Filho Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 13 +++++ rust/kernel/device.rs | 97 +++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 4 files changed, 113 insertions(+) create mode 100644 rust/kernel/device.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index f84e7e2c9d891e..0f798314436fee 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -6,6 +6,7 @@ * Sorted alphabetically. */ +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 7c7661ed53eabf..f6b69a9668572b 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -165,6 +166,18 @@ int rust_helper_xa_err(void *entry) } EXPORT_SYMBOL_GPL(rust_helper_xa_err); +void *rust_helper_dev_get_drvdata(struct device *dev) +{ + return dev_get_drvdata(dev); +} +EXPORT_SYMBOL_GPL(rust_helper_dev_get_drvdata); + +const char *rust_helper_dev_name(const struct device *dev) +{ + return dev_name(dev); +} +EXPORT_SYMBOL_GPL(rust_helper_dev_name); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs new file mode 100644 index 00000000000000..e57da622d81736 --- /dev/null +++ b/rust/kernel/device.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic devices that are part of the kernel's driver model. +//! +//! C header: [`include/linux/device.h`](../../../../include/linux/device.h) + +use crate::{bindings, str::CStr}; + +/// A raw device. +/// +/// # Safety +/// +/// Implementers must ensure that the `*mut device` returned by [`RawDevice::raw_device`] is +/// related to `self`, that is, actions on it will affect `self`. For example, if one calls +/// `get_device`, then the refcount on the device represented by `self` will be incremented. +/// +/// Additionally, implementers must ensure that the device is never renamed. Commit a5462516aa99 +/// ("driver-core: document restrictions on device_rename()") has details on why `device_rename` +/// should not be used. +pub unsafe trait RawDevice { + /// Returns the raw `struct device` related to `self`. + fn raw_device(&self) -> *mut bindings::device; + + /// Returns the name of the device. + fn name(&self) -> &CStr { + let ptr = self.raw_device(); + + // SAFETY: `ptr` is valid because `self` keeps it alive. + let name = unsafe { bindings::dev_name(ptr) }; + + // SAFETY: The name of the device remains valid while it is alive (because the device is + // never renamed, per the safety requirement of this trait). This is guaranteed to be the + // case because the reference to `self` outlives the one of the returned `CStr` (enforced + // by the compiler because of their lifetimes). + unsafe { CStr::from_char_ptr(name) } + } +} + +/// A ref-counted device. +/// +/// # Invariants +/// +/// `ptr` is valid, non-null, and has a non-zero reference count. One of the references is owned by +/// `self`, and will be decremented when `self` is dropped. +pub struct Device { + pub(crate) ptr: *mut bindings::device, +} + +// SAFETY: `Device` only holds a pointer to a C device, which is safe to be used from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` only holds a pointer to a C device, references to which are safe to be used +// from any thread. +unsafe impl Sync for Device {} + +impl Device { + /// Creates a new device instance. + /// + /// # Safety + /// + /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count. + pub unsafe fn new(ptr: *mut bindings::device) -> Self { + // SAFETY: By the safety requirements, ptr is valid and its refcounted will be incremented. + unsafe { bindings::get_device(ptr) }; + // INVARIANT: The safety requirements satisfy all but one invariant, which is that `self` + // owns a reference. This is satisfied by the call to `get_device` above. + Self { ptr } + } + + /// Creates a new device instance from an existing [`RawDevice`] instance. + pub fn from_dev(dev: &dyn RawDevice) -> Self { + // SAFETY: The requirements are satisfied by the existence of `RawDevice` and its safety + // requirements. + unsafe { Self::new(dev.raw_device()) } + } +} + +// SAFETY: The device returned by `raw_device` is the one for which we hold a reference. +unsafe impl RawDevice for Device { + fn raw_device(&self) -> *mut bindings::device { + self.ptr + } +} + +impl Drop for Device { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // relinquish it now. + unsafe { bindings::put_device(self.ptr) }; + } +} + +impl Clone for Device { + fn clone(&self) -> Self { + Device::from_dev(self) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index efda00a1a3160a..e3fe1485d9dd18 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -34,7 +34,9 @@ extern crate self as kernel; #[cfg(not(test))] #[cfg(not(testlib))] mod allocator; + mod build_assert; +pub mod device; pub mod error; pub mod init; pub mod ioctl; From 0b70b0c8073b334c067c1a2c5cde3761d48dbfd2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 7 Sep 2022 17:39:44 +0900 Subject: [PATCH 438/635] rust: io_pgtable: Add io_pgtable abstraction The io_pgtable subsystem implements page table management for various IOMMU page table formats. This abstraction allows Rust drivers for devices with an embedded MMU to use this shared code. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/kernel/io_pgtable.rs | 347 ++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 3 files changed, 349 insertions(+) create mode 100644 rust/kernel/io_pgtable.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 0f798314436fee..25556e0d48dc85 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -7,6 +7,7 @@ */ #include +#include #include #include #include diff --git a/rust/kernel/io_pgtable.rs b/rust/kernel/io_pgtable.rs new file mode 100644 index 00000000000000..ecd0f9e77946bc --- /dev/null +++ b/rust/kernel/io_pgtable.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IOMMU page table management +//! +//! C header: [`include/io-pgtable.h`](../../../../include/io-pgtable.h) + +use crate::{ + bindings, device, + error::{code::*, to_result, Result}, + types::{ForeignOwnable, ScopeGuard}, +}; + +use core::marker::PhantomData; +use core::mem; +use core::num::NonZeroU64; + +/// Protection flags used with IOMMU mappings. +pub mod prot { + /// Read access. + pub const READ: u32 = bindings::IOMMU_READ; + /// Write access. + pub const WRITE: u32 = bindings::IOMMU_WRITE; + /// Request cache coherency. + pub const CACHE: u32 = bindings::IOMMU_CACHE; + /// Request no-execute permission. + pub const NOEXEC: u32 = bindings::IOMMU_NOEXEC; + /// MMIO peripheral mapping. + pub const MMIO: u32 = bindings::IOMMU_MMIO; + /// Privileged mapping. + pub const PRIV: u32 = bindings::IOMMU_PRIV; +} + +/// Represents a requested io_pgtable configuration. +pub struct Config { + /// Quirk bitmask (type-specific). + pub quirks: usize, + /// Valid page sizes, as a bitmask of powers of two. + pub pgsize_bitmap: usize, + /// Input address space size in bits. + pub ias: usize, + /// Output address space size in bits. + pub oas: usize, + /// IOMMU uses coherent accesses for page table walks. + pub coherent_walk: bool, +} + +/// IOMMU callbacks for TLB and page table management. +/// +/// Users must implement this trait to perform the TLB flush actions for this IOMMU, if +/// required. +pub trait FlushOps { + /// User-specified type owned by the IOPagetable that will be passed to TLB operations. + type Data: ForeignOwnable + Send + Sync; + + /// Synchronously invalidate the entire TLB context. + fn tlb_flush_all(data: ::Borrowed<'_>); + + /// Synchronously invalidate all intermediate TLB state (sometimes referred to as the "walk + /// cache") for a virtual address range. + fn tlb_flush_walk( + data: ::Borrowed<'_>, + iova: usize, + size: usize, + granule: usize, + ); + + /// Optional callback to queue up leaf TLB invalidation for a single page. + /// + /// IOMMUs that cannot batch TLB invalidation operations efficiently will typically issue + /// them here, but others may decide to update the iommu_iotlb_gather structure and defer + /// the invalidation until iommu_iotlb_sync() instead. + /// + /// TODO: Implement the gather argument for batching. + fn tlb_add_page( + data: ::Borrowed<'_>, + iova: usize, + granule: usize, + ); +} + +/// Inner page table info shared across all table types. +/// # Invariants +/// +/// - [`self.ops`] is valid and non-null. +/// - [`self.cfg`] is valid and non-null. +#[doc(hidden)] +pub struct IoPageTableInner { + ops: *mut bindings::io_pgtable_ops, + cfg: bindings::io_pgtable_cfg, + data: *mut core::ffi::c_void, +} + +/// Helper trait to get the config type for a single page table type from the union. +pub trait GetConfig { + /// Returns the specific output configuration for this page table type. + fn cfg(iopt: &impl IoPageTable) -> &Self + where + Self: Sized; +} + +/// A generic IOMMU page table +pub trait IoPageTable: crate::private::Sealed { + #[doc(hidden)] + const FLUSH_OPS: bindings::iommu_flush_ops; + + #[doc(hidden)] + fn new_fmt( + dev: &dyn device::RawDevice, + format: u32, + config: Config, + data: T::Data, + ) -> Result { + let ptr = data.into_foreign() as *mut _; + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr) }; + }); + + let mut raw_cfg = bindings::io_pgtable_cfg { + quirks: config.quirks.try_into()?, + pgsize_bitmap: config.pgsize_bitmap.try_into()?, + ias: config.ias.try_into()?, + oas: config.oas.try_into()?, + coherent_walk: config.coherent_walk, + tlb: &Self::FLUSH_OPS, + iommu_dev: dev.raw_device(), + __bindgen_anon_1: unsafe { mem::zeroed() }, + }; + + let ops = unsafe { + bindings::alloc_io_pgtable_ops(format as bindings::io_pgtable_fmt, &mut raw_cfg, ptr) + }; + + if ops.is_null() { + return Err(EINVAL); + } + + guard.dismiss(); + Ok(IoPageTableInner { + ops, + cfg: raw_cfg, + data: ptr, + }) + } + + /// Map a range of pages. + fn map_pages( + &mut self, + iova: usize, + paddr: usize, + pgsize: usize, + pgcount: usize, + prot: u32, + ) -> Result { + let mut mapped: usize = 0; + + to_result(unsafe { + (*self.inner_mut().ops).map_pages.unwrap()( + self.inner_mut().ops, + iova as u64, + paddr as u64, + pgsize, + pgcount, + prot as i32, + bindings::GFP_KERNEL, + &mut mapped, + ) + })?; + + Ok(mapped) + } + + /// Unmap a range of pages. + fn unmap_pages( + &mut self, + iova: usize, + pgsize: usize, + pgcount: usize, + // TODO: gather: *mut iommu_iotlb_gather, + ) -> usize { + unsafe { + (*self.inner_mut().ops).unmap_pages.unwrap()( + self.inner_mut().ops, + iova as u64, + pgsize, + pgcount, + core::ptr::null_mut(), + ) + } + } + + /// Translate an IOVA to the corresponding physical address, if mapped. + fn iova_to_phys(&mut self, iova: usize) -> Option { + NonZeroU64::new(unsafe { + (*self.inner_mut().ops).iova_to_phys.unwrap()(self.inner_mut().ops, iova as u64) + }) + } + + #[doc(hidden)] + fn inner_mut(&mut self) -> &mut IoPageTableInner; + + #[doc(hidden)] + fn inner(&self) -> &IoPageTableInner; + + #[doc(hidden)] + fn raw_cfg(&self) -> &bindings::io_pgtable_cfg { + &self.inner().cfg + } +} + +unsafe impl Send for IoPageTableInner {} +unsafe impl Sync for IoPageTableInner {} + +unsafe extern "C" fn tlb_flush_all_callback(cookie: *mut core::ffi::c_void) { + T::tlb_flush_all(unsafe { T::Data::borrow(cookie) }); +} + +unsafe extern "C" fn tlb_flush_walk_callback( + iova: core::ffi::c_ulong, + size: usize, + granule: usize, + cookie: *mut core::ffi::c_void, +) { + T::tlb_flush_walk( + unsafe { T::Data::borrow(cookie) }, + iova as usize, + size, + granule, + ); +} + +unsafe extern "C" fn tlb_add_page_callback( + _gather: *mut bindings::iommu_iotlb_gather, + iova: core::ffi::c_ulong, + granule: usize, + cookie: *mut core::ffi::c_void, +) { + T::tlb_add_page(unsafe { T::Data::borrow(cookie) }, iova as usize, granule); +} + +macro_rules! iopt_cfg { + ($name:ident, $field:ident, $type:ident) => { + /// An IOMMU page table configuration for a specific kind of pagetable. + pub type $name = bindings::$type; + + impl GetConfig for $name { + fn cfg(iopt: &impl IoPageTable) -> &$name { + unsafe { &iopt.raw_cfg().__bindgen_anon_1.$field } + } + } + }; +} + +impl GetConfig for () { + fn cfg(_iopt: &impl IoPageTable) -> &() { + &() + } +} + +macro_rules! iopt_type { + ($type:ident, $cfg:ty, $fmt:ident) => { + /// Represents an IOPagetable of this type. + pub struct $type(IoPageTableInner, PhantomData); + + impl $type { + /// Creates a new IOPagetable implementation of this type. + pub fn new(dev: &dyn device::RawDevice, config: Config, data: T::Data) -> Result { + Ok(Self( + ::new_fmt::(dev, bindings::$fmt, config, data)?, + PhantomData, + )) + } + + /// Get the configuration for this IOPagetable. + pub fn cfg(&self) -> &$cfg { + <$cfg as GetConfig>::cfg(self) + } + } + + impl crate::private::Sealed for $type {} + + impl IoPageTable for $type { + const FLUSH_OPS: bindings::iommu_flush_ops = bindings::iommu_flush_ops { + tlb_flush_all: Some(tlb_flush_all_callback::), + tlb_flush_walk: Some(tlb_flush_walk_callback::), + tlb_add_page: Some(tlb_add_page_callback::), + }; + + fn inner(&self) -> &IoPageTableInner { + &self.0 + } + + fn inner_mut(&mut self) -> &mut IoPageTableInner { + &mut self.0 + } + } + + impl Drop for $type { + fn drop(&mut self) { + // SAFETY: The pointer is valid by the type invariant. + unsafe { bindings::free_io_pgtable_ops(self.0.ops) }; + + // Free context data. + // + // SAFETY: This matches the call to `into_foreign` from `new` in the success case. + unsafe { T::Data::from_foreign(self.0.data) }; + } + } + }; +} + +// Ew... +iopt_cfg!( + ARMLPAES1Cfg, + arm_lpae_s1_cfg, + io_pgtable_cfg__bindgen_ty_1__bindgen_ty_1 +); +iopt_cfg!( + ARMLPAES2Cfg, + arm_lpae_s2_cfg, + io_pgtable_cfg__bindgen_ty_1__bindgen_ty_2 +); +iopt_cfg!( + ARMv7SCfg, + arm_v7s_cfg, + io_pgtable_cfg__bindgen_ty_1__bindgen_ty_3 +); +iopt_cfg!( + ARMMaliLPAECfg, + arm_mali_lpae_cfg, + io_pgtable_cfg__bindgen_ty_1__bindgen_ty_4 +); +iopt_cfg!( + AppleDARTCfg, + apple_dart_cfg, + io_pgtable_cfg__bindgen_ty_1__bindgen_ty_5 +); + +iopt_type!(ARM32LPAES1, ARMLPAES1Cfg, io_pgtable_fmt_ARM_32_LPAE_S1); +iopt_type!(ARM32LPAES2, ARMLPAES2Cfg, io_pgtable_fmt_ARM_32_LPAE_S2); +iopt_type!(ARM64LPAES1, ARMLPAES1Cfg, io_pgtable_fmt_ARM_64_LPAE_S1); +iopt_type!(ARM64LPAES2, ARMLPAES2Cfg, io_pgtable_fmt_ARM_64_LPAE_S2); +iopt_type!(ARMv7S, ARMv7SCfg, io_pgtable_fmt_ARM_V7S); +iopt_type!(ARMMaliLPAE, ARMMaliLPAECfg, io_pgtable_fmt_ARM_MALI_LPAE); +iopt_type!(AMDIOMMUV1, (), io_pgtable_fmt_AMD_IOMMU_V1); +iopt_type!(AppleDART, AppleDARTCfg, io_pgtable_fmt_APPLE_DART); +iopt_type!(AppleDART2, AppleDARTCfg, io_pgtable_fmt_APPLE_DART2); diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e3fe1485d9dd18..c7123e46354100 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -39,6 +39,7 @@ mod build_assert; pub mod device; pub mod error; pub mod init; +pub mod io_pgtable; pub mod ioctl; pub mod prelude; pub mod print; From d2e71f1c62e9502a0b4436f03c5f95cdb5da38ba Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 18 Aug 2022 02:13:54 +0900 Subject: [PATCH 439/635] rust: soc: apple: rtkit: Add Apple RTKit abstraction RTKit is Apple's proprietary real-time operating system framework, used across many subdevices on Apple Silicon platforms including NVMe, system management, GPU, etc. Add Rust abstractions for this subsystem, so that it can be used by upcoming Rust drivers. Note: Although ARM64 support is not yet merged, this can be built on amd64 with CONFIG_COMPILE_TEST=y. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/kernel/lib.rs | 1 + rust/kernel/soc/apple/mod.rs | 6 + rust/kernel/soc/apple/rtkit.rs | 257 ++++++++++++++++++++++++++++++++ rust/kernel/soc/mod.rs | 5 + 5 files changed, 270 insertions(+) create mode 100644 rust/kernel/soc/apple/mod.rs create mode 100644 rust/kernel/soc/apple/rtkit.rs create mode 100644 rust/kernel/soc/mod.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 25556e0d48dc85..fe1dd60d6561d2 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index c7123e46354100..810646b82aa246 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -43,6 +43,7 @@ pub mod io_pgtable; pub mod ioctl; pub mod prelude; pub mod print; +pub mod soc; mod static_assert; #[doc(hidden)] pub mod std_vendor; diff --git a/rust/kernel/soc/apple/mod.rs b/rust/kernel/soc/apple/mod.rs new file mode 100644 index 00000000000000..dd69db63677dd6 --- /dev/null +++ b/rust/kernel/soc/apple/mod.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple SoC drivers + +#[cfg(CONFIG_APPLE_RTKIT = "y")] +pub mod rtkit; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs new file mode 100644 index 00000000000000..3b7ef301cc77a1 --- /dev/null +++ b/rust/kernel/soc/apple/rtkit.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple RTKit coprocessors. +//! +//! C header: [`include/linux/soc/apple/rtkit.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + bindings, device, + error::{code::*, from_err_ptr, from_result, to_result, Result}, + str::CStr, + types::{ForeignOwnable, ScopeGuard}, +}; + +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::ptr; +use macros::vtable; + +/// Trait to represent allocatable buffers for the RTKit core. +/// +/// Users must implement this trait for their own representation of those allocations. +pub trait Buffer { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn iova(&self) -> Result; + + /// Returns a mutable byte slice of the buffer contents, or an + /// error if unavailable. + fn buf(&mut self) -> Result<&mut [u8]>; +} + +/// Callback operations for an RTKit client. +#[vtable] +pub trait Operations { + /// Arbitrary user context type. + type Data: ForeignOwnable + Send + Sync; + + /// Type representing an allocated buffer for RTKit. + type Buffer: Buffer; + + /// Called when RTKit crashes. + fn crashed(_data: ::Borrowed<'_>) {} + + /// Called when a message was received on a non-system endpoint. Called in non-IRQ context. + fn recv_message( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) { + } + + /// Called in IRQ context when a message was received on a non-system endpoint. + /// + /// Must return `true` if the message is handled, or `false` to process it in + /// the handling thread. + fn recv_message_early( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) -> bool { + false + } + + /// Allocate a buffer for use by RTKit. + fn shmem_alloc( + _data: ::Borrowed<'_>, + _size: usize, + ) -> Result { + Err(EINVAL) + } + + /// Map an existing buffer used by RTKit at a device-specified virtual address. + fn shmem_map( + _data: ::Borrowed<'_>, + _iova: usize, + _size: usize, + ) -> Result { + Err(EINVAL) + } +} + +/// Represents `struct apple_rtkit *`. +/// +/// # Invariants +/// +/// The rtk pointer is valid. +/// The data pointer is a valid pointer from T::Data::into_foreign(). +pub struct RtKit { + rtk: *mut bindings::apple_rtkit, + data: *mut core::ffi::c_void, + _p: PhantomData, +} + +unsafe extern "C" fn crashed_callback(cookie: *mut core::ffi::c_void) { + T::crashed(unsafe { T::Data::borrow(cookie) }); +} + +unsafe extern "C" fn recv_message_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) { + T::recv_message(unsafe { T::Data::borrow(cookie) }, endpoint, message); +} + +unsafe extern "C" fn recv_message_early_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) -> bool { + T::recv_message_early(unsafe { T::Data::borrow(cookie) }, endpoint, message) +} + +unsafe extern "C" fn shmem_setup_callback( + cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) -> core::ffi::c_int { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + + from_result(|| { + let mut buf = if bfr_mut.iova != 0 { + bfr_mut.is_mapped = true; + T::shmem_map( + // SAFETY: `cookie` came from a previous call to `into_foreign`. + unsafe { T::Data::borrow(cookie) }, + bfr_mut.iova as usize, + bfr_mut.size, + )? + } else { + bfr_mut.is_mapped = false; + // SAFETY: `cookie` came from a previous call to `into_foreign`. + T::shmem_alloc(unsafe { T::Data::borrow(cookie) }, bfr_mut.size)? + }; + + let iova = buf.iova()?; + let slice = buf.buf()?; + + if slice.len() < bfr_mut.size { + return Err(ENOMEM); + } + + bfr_mut.iova = iova as u64; + bfr_mut.buffer = slice.as_mut_ptr() as *mut _; + + // Now box the returned buffer type and stash it in the private pointer of the + // `apple_rtkit_shmem` struct for safekeeping. + let boxed = Box::try_new(buf)?; + bfr_mut.private = Box::into_raw(boxed) as *mut _; + Ok(0) + }) +} + +unsafe extern "C" fn shmem_destroy_callback( + _cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) { + let bfr_mut = unsafe { &mut *bfr }; + // SAFETY: Per shmem_setup_callback, this has to be a pointer to a Buffer if it is set. + if !bfr_mut.private.is_null() { + unsafe { + core::mem::drop(Box::from_raw(bfr_mut.private as *mut T::Buffer)); + } + bfr_mut.private = core::ptr::null_mut(); + } +} + +impl RtKit { + const VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + crashed: Some(crashed_callback::), + recv_message: Some(recv_message_callback::), + recv_message_early: Some(recv_message_early_callback::), + shmem_setup: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_setup_callback::) + } else { + None + }, + shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_destroy_callback::) + } else { + None + }, + }; + + /// Creates a new RTKit client for a given device and optional mailbox name or index. + pub fn new( + dev: &dyn device::RawDevice, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + let ptr = data.into_foreign() as *mut _; + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr) }; + }); + // SAFETY: This just calls the C init function. + let rtk = unsafe { + from_err_ptr(bindings::apple_rtkit_init( + dev.raw_device(), + ptr, + match mbox_name { + Some(s) => s.as_char_ptr(), + None => ptr::null(), + }, + mbox_idx.try_into()?, + &Self::VTABLE, + )) + }?; + + guard.dismiss(); + // INVARIANT: `rtk` and `data` are valid here. + Ok(Self { + rtk, + data: ptr, + _p: PhantomData, + }) + } + + /// Boots (wakes up) the RTKit coprocessor. + pub fn boot(&mut self) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) + } + + /// Starts a non-system endpoint. + pub fn start_endpoint(&mut self, endpoint: u8) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_start_ep(self.rtk, endpoint) }) + } + + /// Sends a message to a given endpoint. + pub fn send_message(&mut self, endpoint: u8, message: u64) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { + bindings::apple_rtkit_send_message(self.rtk, endpoint, message, ptr::null_mut(), false) + }) + } +} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Sync for RtKit {} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Send for RtKit {} + +impl Drop for RtKit { + fn drop(&mut self) { + // SAFETY: The pointer is valid by the type invariant. + unsafe { bindings::apple_rtkit_free(self.rtk) }; + + // Free context data. + // + // SAFETY: This matches the call to `into_foreign` from `new` in the success case. + unsafe { T::Data::from_foreign(self.data) }; + } +} diff --git a/rust/kernel/soc/mod.rs b/rust/kernel/soc/mod.rs new file mode 100644 index 00000000000000..e3024042e74f0d --- /dev/null +++ b/rust/kernel/soc/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC drivers + +pub mod apple; From 7a8515b3623cb1077fb43735c7bb435bc5cb8d3c Mon Sep 17 00:00:00 2001 From: Wedson Almeida Filho Date: Thu, 1 Apr 2021 15:47:57 +0100 Subject: [PATCH 440/635] rust: Add `container_of` and `offset_of` macros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Rust counterparts to these C macros. `container_of` is useful for C struct subtyping, to recover the original pointer to the container structure. `offset_of` is useful for struct-relative addressing. Lina: Rewrote commit message and squashed in later changes from rust-for-linux/linux, attributed below. Co-authored-by: Miguel Ojeda Signed-off-by: Miguel Ojeda Co-authored:by: Léo Lanteri Thauvin Signed-off-by: Léo Lanteri Thauvin Co-authored-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 810646b82aa246..a05b3fc49ffccc 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -112,3 +112,66 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! { // instead of `!`. See . loop {} } + +/// Calculates the offset of a field from the beginning of the struct it belongs to. +/// +/// # Examples +/// +/// ``` +/// # use kernel::prelude::*; +/// # use kernel::offset_of; +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// assert_eq!(offset_of!(Test, b), 8); +/// ``` +#[macro_export] +macro_rules! offset_of { + ($type:ty, $($f:tt)*) => {{ + let tmp = core::mem::MaybeUninit::<$type>::uninit(); + let outer = tmp.as_ptr(); + // To avoid warnings when nesting `unsafe` blocks. + #[allow(unused_unsafe)] + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let inner = unsafe { core::ptr::addr_of!((*outer).$($f)*) } as *const u8; + // To avoid warnings when nesting `unsafe` blocks. + #[allow(unused_unsafe)] + // SAFETY: The two pointers are within the same allocation block. + unsafe { inner.offset_from(outer as *const u8) } + }} +} + +/// Produces a pointer to an object from a pointer to one of its fields. +/// +/// # Safety +/// +/// Callers must ensure that the pointer to the field is in fact a pointer to the specified field, +/// as opposed to a pointer to another object of the same type. If this condition is not met, +/// any dereference of the resulting pointer is UB. +/// +/// # Examples +/// +/// ``` +/// # use kernel::container_of; +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// let test = Test { a: 10, b: 20 }; +/// let b_ptr = &test.b; +/// let test_alias = container_of!(b_ptr, Test, b); +/// assert!(core::ptr::eq(&test, test_alias)); +/// ``` +#[macro_export] +macro_rules! container_of { + ($ptr:expr, $type:ty, $($f:tt)*) => {{ + let ptr = $ptr as *const _ as *const u8; + let offset = $crate::offset_of!($type, $($f)*); + ptr.wrapping_offset(-offset) as *const $type + }} +} From e10678b8ad8c3fae772b3ed1fd3b8d65cb983568 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 16:23:27 +0900 Subject: [PATCH 441/635] *RFL import: kernel::types::Bool Commit reference: 3dfc5ebff103 --- rust/kernel/types.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 29db59d6119a9b..c12a3bf1803bd6 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -383,3 +383,84 @@ pub enum Either { /// Constructs an instance of [`Either`] containing a value of type `R`. Right(R), } + +/// A trait for boolean types. +/// +/// This is meant to be used in type states to allow boolean constraints in implementation blocks. +/// In the example below, the implementation containing `MyType::set_value` could _not_ be +/// constrained to type states containing `Writable = true` if `Writable` were a constant instead +/// of a type. +/// +/// # Safety +/// +/// No additional implementations of [`Bool`] should be provided, as [`True`] and [`False`] are +/// already provided. +/// +/// # Examples +/// +/// ``` +/// # use kernel::{Bool, False, True}; +/// use core::marker::PhantomData; +/// +/// // Type state specifies whether the type is writable. +/// trait MyTypeState { +/// type Writable: Bool; +/// } +/// +/// // In state S1, the type is writable. +/// struct S1; +/// impl MyTypeState for S1 { +/// type Writable = True; +/// } +/// +/// // In state S2, the type is not writable. +/// struct S2; +/// impl MyTypeState for S2 { +/// type Writable = False; +/// } +/// +/// struct MyType { +/// value: u32, +/// _p: PhantomData, +/// } +/// +/// impl MyType { +/// fn new(value: u32) -> Self { +/// Self { +/// value, +/// _p: PhantomData, +/// } +/// } +/// } +/// +/// // This implementation block only applies if the type state is writable. +/// impl MyType +/// where +/// T: MyTypeState, +/// { +/// fn set_value(&mut self, v: u32) { +/// self.value = v; +/// } +/// } +/// +/// let mut x = MyType::::new(10); +/// let mut y = MyType::::new(20); +/// +/// x.set_value(30); +/// +/// // The code below fails to compile because `S2` is not writable. +/// // y.set_value(40); +/// ``` +pub unsafe trait Bool {} + +/// Represents the `true` value for types with [`Bool`] bound. +pub struct True; + +// SAFETY: This is one of the only two implementations of `Bool`. +unsafe impl Bool for True {} + +/// Represents the `false` value for types wth [`Bool`] bound. +pub struct False; + +// SAFETY: This is one of the only two implementations of `Bool`. +unsafe impl Bool for False {} From 15d53d2a0021728a17bc8917f03172b9285cdeb5 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 15:51:49 +0900 Subject: [PATCH 442/635] *RFL import: kernel::io_buffer Commit reference: 3dfc5ebff103 --- rust/kernel/io_buffer.rs | 153 +++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 2 files changed, 154 insertions(+) create mode 100644 rust/kernel/io_buffer.rs diff --git a/rust/kernel/io_buffer.rs b/rust/kernel/io_buffer.rs new file mode 100644 index 00000000000000..d5a258a5ff8ffd --- /dev/null +++ b/rust/kernel/io_buffer.rs @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Buffers used in IO. + +use crate::error::Result; +use alloc::vec::Vec; +use core::mem::{size_of, MaybeUninit}; + +/// Represents a buffer to be read from during IO. +pub trait IoBufferReader { + /// Returns the number of bytes left to be read from the io buffer. + /// + /// Note that even reading less than this number of bytes may fail. + fn len(&self) -> usize; + + /// Returns `true` if no data is available in the io buffer. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Reads raw data from the io buffer into a raw kernel buffer. + /// + /// # Safety + /// + /// The output buffer must be valid. + unsafe fn read_raw(&mut self, out: *mut u8, len: usize) -> Result; + + /// Reads all data remaining in the io buffer. + /// + /// Returns `EFAULT` if the address does not currently point to mapped, readable memory. + fn read_all(&mut self) -> Result> { + let mut data = Vec::::new(); + data.try_resize(self.len(), 0)?; + + // SAFETY: The output buffer is valid as we just allocated it. + unsafe { self.read_raw(data.as_mut_ptr(), data.len())? }; + Ok(data) + } + + /// Reads a byte slice from the io buffer. + /// + /// Returns `EFAULT` if the byte slice is bigger than the remaining size of the user slice or + /// if the address does not currently point to mapped, readable memory. + fn read_slice(&mut self, data: &mut [u8]) -> Result { + // SAFETY: The output buffer is valid as it's coming from a live reference. + unsafe { self.read_raw(data.as_mut_ptr(), data.len()) } + } + + /// Reads the contents of a plain old data (POD) type from the io buffer. + fn read(&mut self) -> Result { + let mut out = MaybeUninit::::uninit(); + // SAFETY: The buffer is valid as it was just allocated. + unsafe { self.read_raw(out.as_mut_ptr() as _, size_of::()) }?; + // SAFETY: We just initialised the data. + Ok(unsafe { out.assume_init() }) + } +} + +/// Represents a buffer to be written to during IO. +pub trait IoBufferWriter { + /// Returns the number of bytes left to be written into the io buffer. + /// + /// Note that even writing less than this number of bytes may fail. + fn len(&self) -> usize; + + /// Returns `true` if the io buffer cannot hold any additional data. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Writes zeroes to the io buffer. + /// + /// Differently from the other write functions, `clear` will zero as much as it can and update + /// the writer internal state to reflect this. It will, however, return an error if it cannot + /// clear `len` bytes. + /// + /// For example, if a caller requests that 100 bytes be cleared but a segfault happens after + /// 20 bytes, then EFAULT is returned and the writer is advanced by 20 bytes. + fn clear(&mut self, len: usize) -> Result; + + /// Writes a byte slice into the io buffer. + /// + /// Returns `EFAULT` if the byte slice is bigger than the remaining size of the io buffer or if + /// the address does not currently point to mapped, writable memory. + fn write_slice(&mut self, data: &[u8]) -> Result { + // SAFETY: The input buffer is valid as it's coming from a live reference. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to the io buffer from a raw kernel buffer. + /// + /// # Safety + /// + /// The input buffer must be valid. + unsafe fn write_raw(&mut self, data: *const u8, len: usize) -> Result; + + /// Writes the contents of the given data into the io buffer. + fn write(&mut self, data: &T) -> Result { + // SAFETY: The input buffer is valid as it's coming from a live + // reference to a type that implements `WritableToBytes`. + unsafe { self.write_raw(data as *const T as _, size_of::()) } + } +} + +/// Specifies that a type is safely readable from byte slices. +/// +/// Not all types can be safely read from byte slices; examples from +/// include `bool` +/// that must be either `0` or `1`, and `char` that cannot be a surrogate or above `char::MAX`. +/// +/// # Safety +/// +/// Implementers must ensure that the type is made up only of types that can be safely read from +/// arbitrary byte sequences (e.g., `u32`, `u64`, etc.). +pub unsafe trait ReadableFromBytes {} + +// SAFETY: All bit patterns are acceptable values of the types below. +unsafe impl ReadableFromBytes for u8 {} +unsafe impl ReadableFromBytes for u16 {} +unsafe impl ReadableFromBytes for u32 {} +unsafe impl ReadableFromBytes for u64 {} +unsafe impl ReadableFromBytes for usize {} +unsafe impl ReadableFromBytes for i8 {} +unsafe impl ReadableFromBytes for i16 {} +unsafe impl ReadableFromBytes for i32 {} +unsafe impl ReadableFromBytes for i64 {} +unsafe impl ReadableFromBytes for isize {} + +/// Specifies that a type is safely writable to byte slices. +/// +/// This means that we don't read undefined values (which leads to UB) in preparation for writing +/// to the byte slice. It also ensures that no potentially sensitive information is leaked into the +/// byte slices. +/// +/// # Safety +/// +/// A type must not include padding bytes and must be fully initialised to safely implement +/// [`WritableToBytes`] (i.e., it doesn't contain [`MaybeUninit`] fields). A composition of +/// writable types in a structure is not necessarily writable because it may result in padding +/// bytes. +pub unsafe trait WritableToBytes {} + +// SAFETY: Initialised instances of the following types have no uninitialised portions. +unsafe impl WritableToBytes for u8 {} +unsafe impl WritableToBytes for u16 {} +unsafe impl WritableToBytes for u32 {} +unsafe impl WritableToBytes for u64 {} +unsafe impl WritableToBytes for usize {} +unsafe impl WritableToBytes for i8 {} +unsafe impl WritableToBytes for i16 {} +unsafe impl WritableToBytes for i32 {} +unsafe impl WritableToBytes for i64 {} +unsafe impl WritableToBytes for isize {} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index a05b3fc49ffccc..6c1a69d5a32640 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -39,6 +39,7 @@ mod build_assert; pub mod device; pub mod error; pub mod init; +pub mod io_buffer; pub mod io_pgtable; pub mod ioctl; pub mod prelude; From b9c71a64f4d82dfeefde8de339df3c961bb147fe Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 15:54:30 +0900 Subject: [PATCH 443/635] *RFL import: kernel::user_ptr Commit reference: 3dfc5ebff103 --- rust/helpers.c | 18 +++++ rust/kernel/lib.rs | 1 + rust/kernel/user_ptr.rs | 175 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 rust/kernel/user_ptr.rs diff --git a/rust/helpers.c b/rust/helpers.c index f6b69a9668572b..abe43f7ccd7aa4 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -178,6 +178,24 @@ const char *rust_helper_dev_name(const struct device *dev) } EXPORT_SYMBOL_GPL(rust_helper_dev_name); +unsigned long rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return copy_from_user(to, from, n); +} +EXPORT_SYMBOL_GPL(rust_helper_copy_from_user); + +unsigned long rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return copy_to_user(to, from, n); +} +EXPORT_SYMBOL_GPL(rust_helper_copy_to_user); + +unsigned long rust_helper_clear_user(void __user *to, unsigned long n) +{ + return clear_user(to, n); +} +EXPORT_SYMBOL_GPL(rust_helper_clear_user); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 6c1a69d5a32640..41b6d77b7f3cc5 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -53,6 +53,7 @@ pub mod sync; pub mod task; pub mod time; pub mod types; +pub mod user_ptr; pub mod xarray; #[doc(hidden)] diff --git a/rust/kernel/user_ptr.rs b/rust/kernel/user_ptr.rs new file mode 100644 index 00000000000000..084535675c4a31 --- /dev/null +++ b/rust/kernel/user_ptr.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! User pointers. +//! +//! C header: [`include/linux/uaccess.h`](../../../../include/linux/uaccess.h) + +use crate::{ + bindings, + error::code::*, + error::Result, + io_buffer::{IoBufferReader, IoBufferWriter}, +}; +use alloc::vec::Vec; + +/// A reference to an area in userspace memory, which can be either +/// read-only or read-write. +/// +/// All methods on this struct are safe: invalid pointers return +/// `EFAULT`. Concurrent access, *including data races to/from userspace +/// memory*, is permitted, because fundamentally another userspace +/// thread/process could always be modifying memory at the same time +/// (in the same way that userspace Rust's [`std::io`] permits data races +/// with the contents of files on disk). In the presence of a race, the +/// exact byte values read/written are unspecified but the operation is +/// well-defined. Kernelspace code should validate its copy of data +/// after completing a read, and not expect that multiple reads of the +/// same address will return the same value. +/// +/// All APIs enforce the invariant that a given byte of memory from userspace +/// may only be read once. By preventing double-fetches we avoid TOCTOU +/// vulnerabilities. This is accomplished by taking `self` by value to prevent +/// obtaining multiple readers on a given [`UserSlicePtr`], and the readers +/// only permitting forward reads. +/// +/// Constructing a [`UserSlicePtr`] performs no checks on the provided +/// address and length, it can safely be constructed inside a kernel thread +/// with no current userspace process. Reads and writes wrap the kernel APIs +/// `copy_from_user` and `copy_to_user`, which check the memory map of the +/// current process and enforce that the address range is within the user +/// range (no additional calls to `access_ok` are needed). +/// +/// [`std::io`]: https://doc.rust-lang.org/std/io/index.html +pub struct UserSlicePtr(*mut core::ffi::c_void, usize); + +impl UserSlicePtr { + /// Constructs a user slice from a raw pointer and a length in bytes. + /// + /// # Safety + /// + /// Callers must be careful to avoid time-of-check-time-of-use + /// (TOCTOU) issues. The simplest way is to create a single instance of + /// [`UserSlicePtr`] per user memory block as it reads each byte at + /// most once. + pub unsafe fn new(ptr: *mut core::ffi::c_void, length: usize) -> Self { + UserSlicePtr(ptr, length) + } + + /// Reads the entirety of the user slice. + /// + /// Returns `EFAULT` if the address does not currently point to + /// mapped, readable memory. + pub fn read_all(self) -> Result> { + self.reader().read_all() + } + + /// Constructs a [`UserSlicePtrReader`]. + pub fn reader(self) -> UserSlicePtrReader { + UserSlicePtrReader(self.0, self.1) + } + + /// Writes the provided slice into the user slice. + /// + /// Returns `EFAULT` if the address does not currently point to + /// mapped, writable memory (in which case some data from before the + /// fault may be written), or `data` is larger than the user slice + /// (in which case no data is written). + pub fn write_all(self, data: &[u8]) -> Result { + self.writer().write_slice(data) + } + + /// Constructs a [`UserSlicePtrWriter`]. + pub fn writer(self) -> UserSlicePtrWriter { + UserSlicePtrWriter(self.0, self.1) + } + + /// Constructs both a [`UserSlicePtrReader`] and a [`UserSlicePtrWriter`]. + pub fn reader_writer(self) -> (UserSlicePtrReader, UserSlicePtrWriter) { + ( + UserSlicePtrReader(self.0, self.1), + UserSlicePtrWriter(self.0, self.1), + ) + } +} + +/// A reader for [`UserSlicePtr`]. +/// +/// Used to incrementally read from the user slice. +pub struct UserSlicePtrReader(*mut core::ffi::c_void, usize); + +impl IoBufferReader for UserSlicePtrReader { + /// Returns the number of bytes left to be read from this. + /// + /// Note that even reading less than this number of bytes may fail. + fn len(&self) -> usize { + self.1 + } + + /// Reads raw data from the user slice into a raw kernel buffer. + /// + /// # Safety + /// + /// The output buffer must be valid. + unsafe fn read_raw(&mut self, out: *mut u8, len: usize) -> Result { + if len > self.1 || len > u32::MAX as usize { + return Err(EFAULT); + } + let res = unsafe { bindings::copy_from_user(out as _, self.0, len as _) }; + if res != 0 { + return Err(EFAULT); + } + // Since this is not a pointer to a valid object in our program, + // we cannot use `add`, which has C-style rules for defined + // behavior. + self.0 = self.0.wrapping_add(len); + self.1 -= len; + Ok(()) + } +} + +/// A writer for [`UserSlicePtr`]. +/// +/// Used to incrementally write into the user slice. +pub struct UserSlicePtrWriter(*mut core::ffi::c_void, usize); + +impl IoBufferWriter for UserSlicePtrWriter { + fn len(&self) -> usize { + self.1 + } + + fn clear(&mut self, mut len: usize) -> Result { + let mut ret = Ok(()); + if len > self.1 { + ret = Err(EFAULT); + len = self.1; + } + + // SAFETY: The buffer will be validated by `clear_user`. We ensure that `len` is within + // bounds in the check above. + let left = unsafe { bindings::clear_user(self.0, len as _) } as usize; + if left != 0 { + ret = Err(EFAULT); + len -= left; + } + + self.0 = self.0.wrapping_add(len); + self.1 -= len; + ret + } + + unsafe fn write_raw(&mut self, data: *const u8, len: usize) -> Result { + if len > self.1 || len > u32::MAX as usize { + return Err(EFAULT); + } + let res = unsafe { bindings::copy_to_user(self.0, data as _, len as _) }; + if res != 0 { + return Err(EFAULT); + } + // Since this is not a pointer to a valid object in our program, + // we cannot use `add`, which has C-style rules for defined + // behavior. + self.0 = self.0.wrapping_add(len); + self.1 -= len; + Ok(()) + } +} From 20448aded9e19771065793d6a0841f2f6f478a77 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 16:29:46 +0900 Subject: [PATCH 444/635] MISSING SIGNOFFS: rust: Add PAGE_SIZE constant to kernel crate Computed based on the PAGE_SHIFT macro from C. // Co-authored-by: Adam Bratschi-Kaye // Co-authored-by: Miguel Ojeda // Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 41b6d77b7f3cc5..1d02fef1a49897 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -69,6 +69,11 @@ pub(crate) mod private { pub trait Sealed {} } +/// Page size defined in terms of the `PAGE_SHIFT` macro from C. +/// +/// [`PAGE_SHIFT`]: ../../../include/asm-generic/page.h +pub const PAGE_SIZE: usize = 1 << bindings::PAGE_SHIFT; + /// Prefix to appear before log messages printed from within the `kernel` crate. const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; From 1d8ab2fa392beb3133d9e0342246827edd27aac8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 16:32:20 +0900 Subject: [PATCH 445/635] rust: Enable const_mut_refs feature for the kernel crate Needed by the rust-for-linux/rust module_param module. Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 1d02fef1a49897..eeb6426a4b998b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -14,6 +14,7 @@ #![no_std] #![feature(allocator_api)] #![feature(coerce_unsized)] +#![feature(const_mut_refs)] #![feature(core_ffi_c)] #![feature(dispatch_from_dyn)] #![feature(explicit_generic_args_with_impl_trait)] From 9635fe484168d7db121fea6f9dd19d117b28b031 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 16:33:34 +0900 Subject: [PATCH 446/635] *RFL import: kernel::module_param Commit reference: 3dfc5ebff103 --- rust/kernel/lib.rs | 1 + rust/kernel/module_param.rs | 499 ++++++++++++++++++++++++++++++++++++ 2 files changed, 500 insertions(+) create mode 100644 rust/kernel/module_param.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index eeb6426a4b998b..1aac54deaae790 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -43,6 +43,7 @@ pub mod init; pub mod io_buffer; pub mod io_pgtable; pub mod ioctl; +pub mod module_param; pub mod prelude; pub mod print; pub mod soc; diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 00000000000000..276c1c0b639012 --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h) + +use crate::error::{code::*, from_result}; +use crate::str::{CStr, Formatter}; +use core::fmt::Write; + +/// Types that can be used for module parameters. +/// +/// Note that displaying the type in `sysfs` will fail if +/// [`alloc::string::ToString::to_string`] (as implemented through the +/// [`core::fmt::Display`] trait) writes more than [`PAGE_SIZE`] +/// bytes (including an additional null terminator). +/// +/// [`PAGE_SIZE`]: `crate::PAGE_SIZE` +pub trait ModuleParam: core::fmt::Display + core::marker::Sized { + /// The `ModuleParam` will be used by the kernel module through this type. + /// + /// This may differ from `Self` if, for example, `Self` needs to track + /// ownership without exposing it or allocate extra space for other possible + /// parameter values. See [`StringParam`] or [`ArrayParam`] for examples. + type Value: ?Sized; + + /// Whether the parameter is allowed to be set without an argument. + /// + /// Setting this to `true` allows the parameter to be passed without an + /// argument (e.g. just `module.param` instead of `module.param=foo`). + const NOARG_ALLOWED: bool; + + /// Convert a parameter argument into the parameter value. + /// + /// `None` should be returned when parsing of the argument fails. + /// `arg == None` indicates that the parameter was passed without an + /// argument. If `NOARG_ALLOWED` is set to `false` then `arg` is guaranteed + /// to always be `Some(_)`. + /// + /// Parameters passed at boot time will be set before [`kmalloc`] is + /// available (even if the module is loaded at a later time). However, in + /// this case, the argument buffer will be valid for the entire lifetime of + /// the kernel. So implementations of this method which need to allocate + /// should first check that the allocator is available (with + /// [`crate::bindings::slab_is_available`]) and when it is not available + /// provide an alternative implementation which doesn't allocate. In cases + /// where the allocator is not available it is safe to save references to + /// `arg` in `Self`, but in other cases a copy should be made. + /// + /// [`kmalloc`]: ../../../include/linux/slab.h + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option; + + /// Get the current value of the parameter for use in the kernel module. + /// + /// This function should not be used directly. Instead use the wrapper + /// `read` which will be generated by [`macros::module`]. + fn value(&self) -> &Self::Value; + + /// Set the module parameter from a string. + /// + /// Used to set the parameter value when loading the module or when set + /// through `sysfs`. + /// + /// # Safety + /// + /// If `val` is non-null then it must point to a valid null-terminated + /// string. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn set_param( + val: *const core::ffi::c_char, + param: *const crate::bindings::kernel_param, + ) -> core::ffi::c_int { + let arg = if val.is_null() { + None + } else { + Some(unsafe { CStr::from_char_ptr(val).as_bytes() }) + }; + match Self::try_from_param_arg(arg) { + Some(new_value) => { + let old_value = unsafe { (*param).__bindgen_anon_1.arg as *mut Self }; + let _ = unsafe { core::ptr::replace(old_value, new_value) }; + 0 + } + None => EINVAL.to_errno(), + } + } + + /// Write a string representation of the current parameter value to `buf`. + /// + /// Used for displaying the current parameter value in `sysfs`. + /// + /// # Safety + /// + /// `buf` must be a buffer of length at least `kernel::PAGE_SIZE` that is + /// writeable. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn get_param( + buf: *mut core::ffi::c_char, + param: *const crate::bindings::kernel_param, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C contracts guarantees that the buffer is at least `PAGE_SIZE` bytes. + let mut f = unsafe { Formatter::from_buffer(buf.cast(), crate::PAGE_SIZE) }; + unsafe { write!(f, "{}\0", *((*param).__bindgen_anon_1.arg as *mut Self)) }?; + Ok(f.bytes_written().try_into()?) + }) + } + + /// Drop the parameter. + /// + /// Called when unloading a module. + /// + /// # Safety + /// + /// The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn free(arg: *mut core::ffi::c_void) { + unsafe { core::ptr::drop_in_place(arg as *mut Self) }; + } +} + +/// Trait for parsing integers. +/// +/// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtol +/// [`kstrtoul()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtoul +trait ParseInt: Sized { + fn from_str_radix(src: &str, radix: u32) -> Result; + fn checked_neg(self) -> Option; + + fn from_str_unsigned(src: &str) -> Result { + let (radix, digits) = if let Some(n) = src.strip_prefix("0x") { + (16, n) + } else if let Some(n) = src.strip_prefix("0X") { + (16, n) + } else if let Some(n) = src.strip_prefix("0o") { + (8, n) + } else if let Some(n) = src.strip_prefix("0O") { + (8, n) + } else if let Some(n) = src.strip_prefix("0b") { + (2, n) + } else if let Some(n) = src.strip_prefix("0B") { + (2, n) + } else if src.starts_with('0') { + (8, src) + } else { + (10, src) + }; + Self::from_str_radix(digits, radix) + } + + fn from_str(src: &str) -> Option { + match src.bytes().next() { + None => None, + Some(b'-') => Self::from_str_unsigned(&src[1..]).ok()?.checked_neg(), + Some(b'+') => Some(Self::from_str_unsigned(&src[1..]).ok()?), + Some(_) => Some(Self::from_str_unsigned(src).ok()?), + } + } +} + +macro_rules! impl_parse_int { + ($ty:ident) => { + impl ParseInt for $ty { + fn from_str_radix(src: &str, radix: u32) -> Result { + $ty::from_str_radix(src, radix) + } + + fn checked_neg(self) -> Option { + self.checked_neg() + } + } + }; +} + +impl_parse_int!(i8); +impl_parse_int!(u8); +impl_parse_int!(i16); +impl_parse_int!(u16); +impl_parse_int!(i32); +impl_parse_int!(u32); +impl_parse_int!(i64); +impl_parse_int!(u64); +impl_parse_int!(isize); +impl_parse_int!(usize); + +macro_rules! impl_module_param { + ($ty:ident) => { + impl ModuleParam for $ty { + type Value = $ty; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + let bytes = arg?; + let utf8 = core::str::from_utf8(bytes).ok()?; + <$ty as crate::module_param::ParseInt>::from_str(utf8) + } + + fn value(&self) -> &Self::Value { + self + } + } + }; +} + +#[doc(hidden)] +#[macro_export] +/// Generate a static [`kernel_param_ops`](../../../include/linux/moduleparam.h) struct. +/// +/// # Examples +/// +/// ```ignore +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) => { + $crate::make_param_ops!( + #[doc=""] + $ops, + $ty + ); + }; + ($(#[$meta:meta])* $ops:ident, $ty:ty) => { + $(#[$meta])* + /// + /// Static [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// struct generated by [`make_param_ops`]. + pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops { + flags: if <$ty as $crate::module_param::ModuleParam>::NOARG_ALLOWED { + $crate::bindings::KERNEL_PARAM_OPS_FL_NOARG + } else { + 0 + }, + set: Some(<$ty as $crate::module_param::ModuleParam>::set_param), + get: Some(<$ty as $crate::module_param::ModuleParam>::get_param), + free: Some(<$ty as $crate::module_param::ModuleParam>::free), + }; + }; +} + +impl_module_param!(i8); +impl_module_param!(u8); +impl_module_param!(i16); +impl_module_param!(u16); +impl_module_param!(i32); +impl_module_param!(u32); +impl_module_param!(i64); +impl_module_param!(u64); +impl_module_param!(isize); +impl_module_param!(usize); + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i8`]. + PARAM_OPS_I8, + i8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u8`]. + PARAM_OPS_U8, + u8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i16`]. + PARAM_OPS_I16, + i16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u16`]. + PARAM_OPS_U16, + u16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i32`]. + PARAM_OPS_I32, + i32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u32`]. + PARAM_OPS_U32, + u32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i64`]. + PARAM_OPS_I64, + i64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u64`]. + PARAM_OPS_U64, + u64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`isize`]. + PARAM_OPS_ISIZE, + isize +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`usize`]. + PARAM_OPS_USIZE, + usize +); + +impl ModuleParam for bool { + type Value = bool; + + const NOARG_ALLOWED: bool = true; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + match arg { + None => Some(true), + Some(b"y") | Some(b"Y") | Some(b"1") | Some(b"true") => Some(true), + Some(b"n") | Some(b"N") | Some(b"0") | Some(b"false") => Some(false), + _ => None, + } + } + + fn value(&self) -> &Self::Value { + self + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`bool`]. + PARAM_OPS_BOOL, + bool +); + +/// An array of at __most__ `N` values. +/// +/// # Invariant +/// +/// The first `self.used` elements of `self.values` are initialized. +pub struct ArrayParam { + values: [core::mem::MaybeUninit; N], + used: usize, +} + +impl ArrayParam { + fn values(&self) -> &[T] { + // SAFETY: The invariant maintained by `ArrayParam` allows us to cast + // the first `self.used` elements to `T`. + unsafe { + &*(&self.values[0..self.used] as *const [core::mem::MaybeUninit] as *const [T]) + } + } +} + +impl ArrayParam { + const fn new() -> Self { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + ArrayParam { + values: [core::mem::MaybeUninit::uninit(); N], + used: 0, + } + } + + const fn push(&mut self, val: T) { + if self.used < N { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + self.values[self.used] = core::mem::MaybeUninit::new(val); + self.used += 1; + } + } + + /// Create an instance of `ArrayParam` initialized with `vals`. + /// + /// This function is only meant to be used in the [`module::module`] macro. + pub const fn create(vals: &[T]) -> Self { + let mut result = ArrayParam::new(); + let mut i = 0; + while i < vals.len() { + result.push(vals[i]); + i += 1; + } + result + } +} + +impl core::fmt::Display for ArrayParam { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + for val in self.values() { + write!(f, "{},", val)?; + } + Ok(()) + } +} + +impl ModuleParam + for ArrayParam +{ + type Value = [T]; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + arg.and_then(|args| { + let mut result = Self::new(); + for arg in args.split(|b| *b == b',') { + result.push(T::try_from_param_arg(Some(arg))?); + } + Some(result) + }) + } + + fn value(&self) -> &Self::Value { + self.values() + } +} + +/// A C-style string parameter. +/// +/// The Rust version of the [`charp`] parameter. This type is meant to be +/// used by the [`macros::module`] macro, not handled directly. Instead use the +/// `read` method generated by that macro. +/// +/// [`charp`]: ../../../include/linux/moduleparam.h +pub enum StringParam { + /// A borrowed parameter value. + /// + /// Either the default value (which is static in the module) or borrowed + /// from the original argument buffer used to set the value. + Ref(&'static [u8]), + + /// A value that was allocated when the parameter was set. + /// + /// The value needs to be freed when the parameter is reset or the module is + /// unloaded. + Owned(alloc::vec::Vec), +} + +impl StringParam { + fn bytes(&self) -> &[u8] { + match self { + StringParam::Ref(bytes) => *bytes, + StringParam::Owned(vec) => &vec[..], + } + } +} + +impl core::fmt::Display for StringParam { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let bytes = self.bytes(); + match core::str::from_utf8(bytes) { + Ok(utf8) => write!(f, "{}", utf8), + Err(_) => write!(f, "{:?}", bytes), + } + } +} + +impl ModuleParam for StringParam { + type Value = [u8]; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option { + // SAFETY: It is always safe to call [`slab_is_available`](../../../include/linux/slab.h). + let slab_available = unsafe { crate::bindings::slab_is_available() }; + arg.and_then(|arg| { + if slab_available { + let mut vec = alloc::vec::Vec::new(); + vec.try_extend_from_slice(arg).ok()?; + Some(StringParam::Owned(vec)) + } else { + Some(StringParam::Ref(arg)) + } + }) + } + + fn value(&self) -> &Self::Value { + self.bytes() + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`StringParam`]. + PARAM_OPS_STR, + StringParam +); From 1933b5be976f5b3469b6108cea9026925ecfa5cf Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sat, 24 Dec 2022 19:56:17 +0100 Subject: [PATCH 447/635] rust: upgrade to Rust 1.66.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgrade the Rust version from 1.62.0 to 1.66.0. The overwhelming majority of the commit is about upgrading our `alloc` fork to the new version from upstream [1]. License has not changed [2][3] (there were changes in the `COPYRIGHT` file, but unrelated to `alloc`). As in the previous version upgrades (done out of tree so far), upgrading `alloc` requires checking that our small additions (`try_*`) still match their original (non`-try_*`) versions. With this version upgrade, the following unstable Rust features were stabilized: `bench_black_box` (1.66.0), `const_ptr_offset_from` (1.65.0), `core_ffi_c` (1.64.0) and `generic_associated_types` (1.65.0). Thus remove them. This also implies that only two unstable features remain allowed for non-`rust/` code: `allocator_api` and `const_refs_to_cell`. There are some new Clippy warnings that we are triggering (i.e. introduced since 1.62.0), as well as a few others that were not triggered before, thus allow them in this commit and clean up or remove them as needed later on: - `borrow_deref_ref` (new in 1.63.0). - `explicit_auto_deref` (new in 1.64.0). - `bool_to_int_with_if` (new in 1.65.0). - `needless_borrow`. - `type_complexity`. - `unnecessary_cast` (allowed only on `CONFIG_ARM`). Furthermore, `rustdoc` lint `broken_intra_doc_links` is triggering on links pointing to `macro_export` `macro_rules` defined in the same module (i.e. appearing in the crate root). However, even if the warning appears, the link still gets generated like in previous versions, thus it is a bit confusing. An issue has been opened upstream [4], and it appears that the link still being generated was a compatibility measure, thus we will need to adapt to the new behavior (done in the next patch). In addition, there is an added `#[const_trait]` attribute in `RawDeviceId`, due to 1.66.0's PR "Require `#[const_trait]` on `Trait` for `impl const Trait`") [5]. Finally, the `-Aunused-imports` was added for compiling `core`. This was fixed upstream for 1.67.0 in PR "Fix warning when libcore is compiled with no_fp_fmt_parse" [6], and prevented for the future for that `cfg` in PR "core: ensure `no_fp_fmt_parse builds` are warning-free" [7]. Reviewed-by: Björn Roy Baron Reviewed-by: Martin Rodriguez Reboredo Tested-by: Martin Rodriguez Reboredo Reviewed-by: Gary Guo Reviewed-by: Vincenzo Palazzo Reviewed-by: Alice Ferrazzi Tested-by: Alice Ferrazzi Reviewed-by: Neal Gompa Tested-by: Neal Gompa Link: https://github.com/Rust-for-Linux/linux/pull/947 Link: https://github.com/rust-lang/rust/tree/1.66.0/library/alloc/src [1] Link: https://github.com/rust-lang/rust/blob/1.66.0/library/alloc/Cargo.toml#L4 [2] Link: https://github.com/rust-lang/rust/blob/1.66.0/COPYRIGHT [3] Link: https://github.com/rust-lang/rust/issues/106142 [4] Link: https://github.com/rust-lang/rust/pull/100982 [5] Link: https://github.com/rust-lang/rust/pull/105434 [6] Link: https://github.com/rust-lang/rust/pull/105811 [7] Signed-off-by: Miguel Ojeda --- Documentation/process/changes.rst | 2 +- rust/Makefile | 2 +- rust/alloc/alloc.rs | 26 +- rust/alloc/boxed.rs | 435 ++++++++++++++++++++++++++++-- rust/alloc/collections/mod.rs | 3 + rust/alloc/lib.rs | 41 ++- rust/alloc/raw_vec.rs | 14 +- rust/alloc/slice.rs | 100 ++----- rust/alloc/vec/drain.rs | 75 +++++- rust/alloc/vec/drain_filter.rs | 60 ++++- rust/alloc/vec/into_iter.rs | 78 ++++-- rust/alloc/vec/is_zero.rs | 73 ++++- rust/alloc/vec/mod.rs | 351 ++++++++++++++++++------ rust/alloc/vec/spec_extend.rs | 4 +- rust/bindings/lib.rs | 1 - rust/kernel/build_assert.rs | 1 + rust/kernel/lib.rs | 3 - rust/kernel/module_param.rs | 1 + rust/kernel/std_vendor.rs | 1 + rust/uapi/lib.rs | 1 - scripts/Makefile.build | 2 +- scripts/min-tool-version.sh | 2 +- 22 files changed, 1019 insertions(+), 257 deletions(-) diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst index ef540865ad22e3..68596479847931 100644 --- a/Documentation/process/changes.rst +++ b/Documentation/process/changes.rst @@ -31,7 +31,7 @@ you probably needn't concern yourself with pcmciautils. ====================== =============== ======================================== GNU C 5.1 gcc --version Clang/LLVM (optional) 11.0.0 clang --version -Rust (optional) 1.62.0 rustc --version +Rust (optional) 1.66.0 rustc --version bindgen (optional) 0.56.0 bindgen --version GNU make 3.82 make --version bash 4.2 bash --version diff --git a/rust/Makefile b/rust/Makefile index 7c9d9f11aec505..2328977cd5791f 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -392,7 +392,7 @@ endif $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub $(obj)/core.o: private rustc_objcopy = $(foreach sym,$(redirect-intrinsics),--redefine-sym $(sym)=__rust$(sym)) -$(obj)/core.o: private rustc_target_flags = $(core-cfgs) +$(obj)/core.o: private rustc_target_flags = $(core-cfgs) -Aunused-imports $(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs scripts/target.json FORCE $(call if_changed_dep,rustc_library) diff --git a/rust/alloc/alloc.rs b/rust/alloc/alloc.rs index ca224a54177077..0142178370e9af 100644 --- a/rust/alloc/alloc.rs +++ b/rust/alloc/alloc.rs @@ -27,16 +27,23 @@ extern "Rust" { // (the code expanding that attribute macro generates those functions), or to call // the default implementations in libstd (`__rdl_alloc` etc. in `library/std/src/alloc.rs`) // otherwise. - // The rustc fork of LLVM also special-cases these function names to be able to optimize them + // The rustc fork of LLVM 14 and earlier also special-cases these function names to be able to optimize them // like `malloc`, `realloc`, and `free`, respectively. #[rustc_allocator] - #[rustc_allocator_nounwind] + #[cfg_attr(not(bootstrap), rustc_nounwind)] + #[cfg_attr(bootstrap, rustc_allocator_nounwind)] fn __rust_alloc(size: usize, align: usize) -> *mut u8; - #[rustc_allocator_nounwind] + #[rustc_deallocator] + #[cfg_attr(not(bootstrap), rustc_nounwind)] + #[cfg_attr(bootstrap, rustc_allocator_nounwind)] fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize); - #[rustc_allocator_nounwind] + #[rustc_reallocator] + #[cfg_attr(not(bootstrap), rustc_nounwind)] + #[cfg_attr(bootstrap, rustc_allocator_nounwind)] fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8; - #[rustc_allocator_nounwind] + #[rustc_allocator_zeroed] + #[cfg_attr(not(bootstrap), rustc_nounwind)] + #[cfg_attr(bootstrap, rustc_allocator_nounwind)] fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8; } @@ -72,11 +79,14 @@ pub use std::alloc::Global; /// # Examples /// /// ``` -/// use std::alloc::{alloc, dealloc, Layout}; +/// use std::alloc::{alloc, dealloc, handle_alloc_error, Layout}; /// /// unsafe { /// let layout = Layout::new::(); /// let ptr = alloc(layout); +/// if ptr.is_null() { +/// handle_alloc_error(layout); +/// } /// /// *(ptr as *mut u16) = 42; /// assert_eq!(*(ptr as *mut u16), 42); @@ -400,13 +410,13 @@ pub mod __alloc_error_handler { // if there is no `#[alloc_error_handler]` #[rustc_std_internal_symbol] - pub unsafe extern "C-unwind" fn __rdl_oom(size: usize, _align: usize) -> ! { + pub unsafe fn __rdl_oom(size: usize, _align: usize) -> ! { panic!("memory allocation of {size} bytes failed") } // if there is an `#[alloc_error_handler]` #[rustc_std_internal_symbol] - pub unsafe extern "C-unwind" fn __rg_oom(size: usize, align: usize) -> ! { + pub unsafe fn __rg_oom(size: usize, align: usize) -> ! { let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; extern "Rust" { #[lang = "oom"] diff --git a/rust/alloc/boxed.rs b/rust/alloc/boxed.rs index dcfe87b14f3a64..8126774ae03cdd 100644 --- a/rust/alloc/boxed.rs +++ b/rust/alloc/boxed.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -//! A pointer type for heap allocation. +//! The `Box` type for heap allocation. //! //! [`Box`], casually referred to as a 'box', provides the simplest form of //! heap allocation in Rust. Boxes provide ownership for this allocation, and @@ -124,7 +124,21 @@ //! definition is just using `T*` can lead to undefined behavior, as //! described in [rust-lang/unsafe-code-guidelines#198][ucg#198]. //! +//! # Considerations for unsafe code +//! +//! **Warning: This section is not normative and is subject to change, possibly +//! being relaxed in the future! It is a simplified summary of the rules +//! currently implemented in the compiler.** +//! +//! The aliasing rules for `Box` are the same as for `&mut T`. `Box` +//! asserts uniqueness over its content. Using raw pointers derived from a box +//! after that box has been mutated through, moved or borrowed as `&mut T` +//! is not allowed. For more guidance on working with box from unsafe code, see +//! [rust-lang/unsafe-code-guidelines#326][ucg#326]. +//! +//! //! [ucg#198]: https://github.com/rust-lang/unsafe-code-guidelines/issues/198 +//! [ucg#326]: https://github.com/rust-lang/unsafe-code-guidelines/issues/326 //! [dereferencing]: core::ops::Deref //! [`Box::::from_raw(value)`]: Box::from_raw //! [`Global`]: crate::alloc::Global @@ -139,6 +153,7 @@ use core::async_iter::AsyncIterator; use core::borrow; use core::cmp::Ordering; use core::convert::{From, TryFrom}; +use core::error::Error; use core::fmt; use core::future::Future; use core::hash::{Hash, Hasher}; @@ -163,6 +178,8 @@ use crate::raw_vec::RawVec; #[cfg(not(no_global_oom_handling))] use crate::str::from_boxed_utf8_unchecked; #[cfg(not(no_global_oom_handling))] +use crate::string::String; +#[cfg(not(no_global_oom_handling))] use crate::vec::Vec; #[cfg(not(no_thin))] @@ -196,12 +213,13 @@ impl Box { /// ``` /// let five = Box::new(5); /// ``` - #[cfg(not(no_global_oom_handling))] + #[cfg(all(not(no_global_oom_handling)))] #[inline(always)] #[stable(feature = "rust1", since = "1.0.0")] #[must_use] pub fn new(x: T) -> Self { - box x + #[rustc_box] + Box::new(x) } /// Constructs a new box with uninitialized contents. @@ -256,14 +274,21 @@ impl Box { Self::new_zeroed_in(Global) } - /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then /// `x` will be pinned in memory and unable to be moved. + /// + /// Constructing and pinning of the `Box` can also be done in two steps: `Box::pin(x)` + /// does the same as [Box::into_pin]\([Box::new]\(x)). Consider using + /// [`into_pin`](Box::into_pin) if you already have a `Box`, or if you want to + /// construct a (pinned) `Box` in a different way than with [`Box::new`]. #[cfg(not(no_global_oom_handling))] #[stable(feature = "pin", since = "1.33.0")] #[must_use] #[inline(always)] pub fn pin(x: T) -> Pin> { - (box x).into() + (#[rustc_box] + Box::new(x)) + .into() } /// Allocates memory on the heap then places `x` into it, @@ -543,8 +568,13 @@ impl Box { unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } } - /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then + /// Constructs a new `Pin>`. If `T` does not implement [`Unpin`], then /// `x` will be pinned in memory and unable to be moved. + /// + /// Constructing and pinning of the `Box` can also be done in two steps: `Box::pin_in(x, alloc)` + /// does the same as [Box::into_pin]\([Box::new_in]\(x, alloc)). Consider using + /// [`into_pin`](Box::into_pin) if you already have a `Box`, or if you want to + /// construct a (pinned) `Box` in a different way than with [`Box::new_in`]. #[cfg(not(no_global_oom_handling))] #[unstable(feature = "allocator_api", issue = "32838")] #[rustc_const_unstable(feature = "const_box", issue = "92521")] @@ -926,6 +956,7 @@ impl Box { /// [`Layout`]: crate::Layout #[stable(feature = "box_raw", since = "1.4.0")] #[inline] + #[must_use = "call `drop(from_raw(ptr))` if you intend to drop the `Box`"] pub unsafe fn from_raw(raw: *mut T) -> Self { unsafe { Self::from_raw_in(raw, Global) } } @@ -1160,19 +1191,44 @@ impl Box { unsafe { &mut *mem::ManuallyDrop::new(b).0.as_ptr() } } - /// Converts a `Box` into a `Pin>` + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*boxed` will be pinned in memory and unable to be moved. /// /// This conversion does not allocate on the heap and happens in place. /// /// This is also available via [`From`]. - #[unstable(feature = "box_into_pin", issue = "62370")] + /// + /// Constructing and pinning a `Box` with Box::into_pin([Box::new]\(x)) + /// can also be written more concisely using [Box::pin]\(x). + /// This `into_pin` method is useful if you already have a `Box`, or you are + /// constructing a (pinned) `Box` in a different way than with [`Box::new`]. + /// + /// # Notes + /// + /// It's not recommended that crates add an impl like `From> for Pin`, + /// as it'll introduce an ambiguity when calling `Pin::from`. + /// A demonstration of such a poor impl is shown below. + /// + /// ```compile_fail + /// # use std::pin::Pin; + /// struct Foo; // A type defined in this crate. + /// impl From> for Pin { + /// fn from(_: Box<()>) -> Pin { + /// Pin::new(Foo) + /// } + /// } + /// + /// let foo = Box::new(()); + /// let bar = Pin::from(foo); + /// ``` + #[stable(feature = "box_into_pin", since = "1.63.0")] #[rustc_const_unstable(feature = "const_box", issue = "92521")] pub const fn into_pin(boxed: Self) -> Pin where A: 'static, { // It's not possible to move or replace the insides of a `Pin>` - // when `T: !Unpin`, so it's safe to pin it directly without any + // when `T: !Unpin`, so it's safe to pin it directly without any // additional requirements. unsafe { Pin::new_unchecked(boxed) } } @@ -1190,7 +1246,8 @@ unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Box { impl Default for Box { /// Creates a `Box`, with the `Default` value for T. fn default() -> Self { - box T::default() + #[rustc_box] + Box::new(T::default()) } } @@ -1408,9 +1465,17 @@ impl const From> for Pin> where A: 'static, { - /// Converts a `Box` into a `Pin>` + /// Converts a `Box` into a `Pin>`. If `T` does not implement [`Unpin`], then + /// `*boxed` will be pinned in memory and unable to be moved. /// /// This conversion does not allocate on the heap and happens in place. + /// + /// This is also available via [`Box::into_pin`]. + /// + /// Constructing and pinning a `Box` with >>::from([Box::new]\(x)) + /// can also be written more concisely using [Box::pin]\(x). + /// This `From` implementation is useful if you already have a `Box`, or you are + /// constructing a (pinned) `Box` in a different way than with [`Box::new`]. fn from(boxed: Box) -> Self { Box::into_pin(boxed) } @@ -1422,7 +1487,7 @@ impl From<&[T]> for Box<[T]> { /// Converts a `&[T]` into a `Box<[T]>` /// /// This conversion allocates on the heap - /// and performs a copy of `slice`. + /// and performs a copy of `slice` and its contents. /// /// # Examples /// ```rust @@ -1554,10 +1619,27 @@ impl From<[T; N]> for Box<[T]> { /// println!("{boxed:?}"); /// ``` fn from(array: [T; N]) -> Box<[T]> { - box array + #[rustc_box] + Box::new(array) } } +/// Casts a boxed slice to a boxed array. +/// +/// # Safety +/// +/// `boxed_slice.len()` must be exactly `N`. +unsafe fn boxed_slice_as_array_unchecked( + boxed_slice: Box<[T], A>, +) -> Box<[T; N], A> { + debug_assert_eq!(boxed_slice.len(), N); + + let (ptr, alloc) = Box::into_raw_with_allocator(boxed_slice); + // SAFETY: Pointer and allocator came from an existing box, + // and our safety condition requires that the length is exactly `N` + unsafe { Box::from_raw_in(ptr as *mut [T; N], alloc) } +} + #[stable(feature = "boxed_slice_try_from", since = "1.43.0")] impl TryFrom> for Box<[T; N]> { type Error = Box<[T]>; @@ -1573,13 +1655,46 @@ impl TryFrom> for Box<[T; N]> { /// `boxed_slice.len()` does not equal `N`. fn try_from(boxed_slice: Box<[T]>) -> Result { if boxed_slice.len() == N { - Ok(unsafe { Box::from_raw(Box::into_raw(boxed_slice) as *mut [T; N]) }) + Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) }) } else { Err(boxed_slice) } } } +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")] +impl TryFrom> for Box<[T; N]> { + type Error = Vec; + + /// Attempts to convert a `Vec` into a `Box<[T; N]>`. + /// + /// Like [`Vec::into_boxed_slice`], this is in-place if `vec.capacity() == N`, + /// but will require a reallocation otherwise. + /// + /// # Errors + /// + /// Returns the original `Vec` in the `Err` variant if + /// `boxed_slice.len()` does not equal `N`. + /// + /// # Examples + /// + /// This can be used with [`vec!`] to create an array on the heap: + /// + /// ``` + /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap(); + /// assert_eq!(state.len(), 100); + /// ``` + fn try_from(vec: Vec) -> Result { + if vec.len() == N { + let boxed_slice = vec.into_boxed_slice(); + Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) }) + } else { + Err(vec) + } + } +} + impl Box { /// Attempt to downcast the box to a concrete type. /// @@ -1973,8 +2088,7 @@ impl AsMut for Box { * could have a method to project a Pin from it. */ #[stable(feature = "pin", since = "1.33.0")] -#[rustc_const_unstable(feature = "const_box", issue = "92521")] -impl const Unpin for Box where A: 'static {} +impl Unpin for Box where A: 'static {} #[unstable(feature = "generator_trait", issue = "43122")] impl + Unpin, R, A: Allocator> Generator for Box @@ -2026,3 +2140,292 @@ impl AsyncIterator for Box { (**self).size_hint() } } + +impl dyn Error { + #[inline] + #[stable(feature = "error_downcast", since = "1.3.0")] + #[rustc_allow_incoherent_impl] + /// Attempts to downcast the box to a concrete type. + pub fn downcast(self: Box) -> Result, Box> { + if self.is::() { + unsafe { + let raw: *mut dyn Error = Box::into_raw(self); + Ok(Box::from_raw(raw as *mut T)) + } + } else { + Err(self) + } + } +} + +impl dyn Error + Send { + #[inline] + #[stable(feature = "error_downcast", since = "1.3.0")] + #[rustc_allow_incoherent_impl] + /// Attempts to downcast the box to a concrete type. + pub fn downcast(self: Box) -> Result, Box> { + let err: Box = self; + ::downcast(err).map_err(|s| unsafe { + // Reapply the `Send` marker. + mem::transmute::, Box>(s) + }) + } +} + +impl dyn Error + Send + Sync { + #[inline] + #[stable(feature = "error_downcast", since = "1.3.0")] + #[rustc_allow_incoherent_impl] + /// Attempts to downcast the box to a concrete type. + pub fn downcast(self: Box) -> Result, Box> { + let err: Box = self; + ::downcast(err).map_err(|s| unsafe { + // Reapply the `Send + Sync` marker. + mem::transmute::, Box>(s) + }) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, E: Error + 'a> From for Box { + /// Converts a type of [`Error`] into a box of dyn [`Error`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::fmt; + /// use std::mem; + /// + /// #[derive(Debug)] + /// struct AnError; + /// + /// impl fmt::Display for AnError { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "An error") + /// } + /// } + /// + /// impl Error for AnError {} + /// + /// let an_error = AnError; + /// assert!(0 == mem::size_of_val(&an_error)); + /// let a_boxed_error = Box::::from(an_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: E) -> Box { + Box::new(err) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, E: Error + Send + Sync + 'a> From for Box { + /// Converts a type of [`Error`] + [`Send`] + [`Sync`] into a box of + /// dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::fmt; + /// use std::mem; + /// + /// #[derive(Debug)] + /// struct AnError; + /// + /// impl fmt::Display for AnError { + /// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + /// write!(f, "An error") + /// } + /// } + /// + /// impl Error for AnError {} + /// + /// unsafe impl Send for AnError {} + /// + /// unsafe impl Sync for AnError {} + /// + /// let an_error = AnError; + /// assert!(0 == mem::size_of_val(&an_error)); + /// let a_boxed_error = Box::::from(an_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: E) -> Box { + Box::new(err) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl From for Box { + /// Converts a [`String`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_string_error = "a string error".to_string(); + /// let a_boxed_error = Box::::from(a_string_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + #[inline] + fn from(err: String) -> Box { + struct StringError(String); + + impl Error for StringError { + #[allow(deprecated)] + fn description(&self) -> &str { + &self.0 + } + } + + impl fmt::Display for StringError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } + } + + // Purposefully skip printing "StringError(..)" + impl fmt::Debug for StringError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } + } + + Box::new(StringError(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "string_box_error", since = "1.6.0")] +impl From for Box { + /// Converts a [`String`] into a box of dyn [`Error`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_string_error = "a string error".to_string(); + /// let a_boxed_error = Box::::from(a_string_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(str_err: String) -> Box { + let err1: Box = From::from(str_err); + let err2: Box = err1; + err2 + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a> From<&str> for Box { + /// Converts a [`str`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// [`str`]: prim@str + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_str_error = "a str error"; + /// let a_boxed_error = Box::::from(a_str_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + #[inline] + fn from(err: &str) -> Box { + From::from(String::from(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "string_box_error", since = "1.6.0")] +impl From<&str> for Box { + /// Converts a [`str`] into a box of dyn [`Error`]. + /// + /// [`str`]: prim@str + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// + /// let a_str_error = "a str error"; + /// let a_boxed_error = Box::::from(a_str_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: &str) -> Box { + From::from(String::from(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_box_error", since = "1.22.0")] +impl<'a, 'b> From> for Box { + /// Converts a [`Cow`] into a box of dyn [`Error`] + [`Send`] + [`Sync`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// use std::borrow::Cow; + /// + /// let a_cow_str_error = Cow::from("a str error"); + /// let a_boxed_error = Box::::from(a_cow_str_error); + /// assert!( + /// mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: Cow<'b, str>) -> Box { + From::from(String::from(err)) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_box_error", since = "1.22.0")] +impl<'a> From> for Box { + /// Converts a [`Cow`] into a box of dyn [`Error`]. + /// + /// # Examples + /// + /// ``` + /// use std::error::Error; + /// use std::mem; + /// use std::borrow::Cow; + /// + /// let a_cow_str_error = Cow::from("a str error"); + /// let a_boxed_error = Box::::from(a_cow_str_error); + /// assert!(mem::size_of::>() == mem::size_of_val(&a_boxed_error)) + /// ``` + fn from(err: Cow<'a, str>) -> Box { + From::from(String::from(err)) + } +} + +#[stable(feature = "box_error", since = "1.8.0")] +impl core::error::Error for Box { + #[allow(deprecated, deprecated_in_future)] + fn description(&self) -> &str { + core::error::Error::description(&**self) + } + + #[allow(deprecated)] + fn cause(&self) -> Option<&dyn core::error::Error> { + core::error::Error::cause(&**self) + } + + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + core::error::Error::source(&**self) + } +} diff --git a/rust/alloc/collections/mod.rs b/rust/alloc/collections/mod.rs index 1eec265b28f807..da61544124168c 100644 --- a/rust/alloc/collections/mod.rs +++ b/rust/alloc/collections/mod.rs @@ -154,3 +154,6 @@ trait SpecExtend { /// Extends `self` with the contents of the given iterator. fn spec_extend(&mut self, iter: I); } + +#[stable(feature = "try_reserve", since = "1.57.0")] +impl core::error::Error for TryReserveError {} diff --git a/rust/alloc/lib.rs b/rust/alloc/lib.rs index 3aebf83c9967ea..0eeee6ccfdf4a3 100644 --- a/rust/alloc/lib.rs +++ b/rust/alloc/lib.rs @@ -58,10 +58,6 @@ //! [`Rc`]: rc //! [`RefCell`]: core::cell -// To run liballoc tests without x.py without ending up with two copies of liballoc, Miri needs to be -// able to "empty" this crate. See . -// rustc itself never sets the feature, so this line has no affect there. -#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] #![allow(unused_attributes)] #![stable(feature = "alloc", since = "1.36.0")] #![doc( @@ -75,10 +71,16 @@ any(not(feature = "miri-test-libstd"), test, doctest), no_global_oom_handling, not(no_global_oom_handling), + not(no_rc), + not(no_sync), target_has_atomic = "ptr" ))] #![no_std] #![needs_allocator] +// To run liballoc tests without x.py without ending up with two copies of liballoc, Miri needs to be +// able to "empty" this crate. See . +// rustc itself never sets the feature, so this line has no affect there. +#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] // // Lints: #![deny(unsafe_op_in_unsafe_fn)] @@ -88,10 +90,10 @@ #![allow(explicit_outlives_requirements)] // // Library features: -#![cfg_attr(not(no_global_oom_handling), feature(alloc_c_string))] #![feature(alloc_layout_extra)] #![feature(allocator_api)] #![feature(array_chunks)] +#![feature(array_into_iter_constructors)] #![feature(array_methods)] #![feature(array_windows)] #![feature(assert_matches)] @@ -99,7 +101,7 @@ #![feature(coerce_unsized)] #![cfg_attr(not(no_global_oom_handling), feature(const_alloc_error))] #![feature(const_box)] -#![cfg_attr(not(no_global_oom_handling), feature(const_btree_new))] +#![cfg_attr(not(no_global_oom_handling), feature(const_btree_len))] #![cfg_attr(not(no_borrow), feature(const_cow_is_borrowed))] #![feature(const_convert)] #![feature(const_size_of_val)] @@ -108,13 +110,14 @@ #![feature(const_maybe_uninit_write)] #![feature(const_maybe_uninit_as_mut_ptr)] #![feature(const_refs_to_cell)] -#![feature(core_c_str)] #![feature(core_intrinsics)] -#![feature(core_ffi_c)] #![feature(const_eval_select)] #![feature(const_pin)] +#![feature(const_waker)] #![feature(cstr_from_bytes_until_nul)] #![feature(dispatch_from_dyn)] +#![feature(error_generic_member_access)] +#![feature(error_in_core)] #![feature(exact_size_is_empty)] #![feature(extend_one)] #![feature(fmt_internals)] @@ -122,16 +125,24 @@ #![feature(hasher_prefixfree_extras)] #![feature(inplace_iteration)] #![feature(iter_advance_by)] +#![feature(iter_next_chunk)] #![feature(layout_for_ptr)] #![feature(maybe_uninit_slice)] +#![feature(maybe_uninit_uninit_array)] +#![feature(maybe_uninit_uninit_array_transpose)] #![cfg_attr(test, feature(new_uninit))] #![feature(nonnull_slice_from_raw_parts)] #![feature(pattern)] +#![feature(pointer_byte_offsets)] +#![feature(provide_any)] #![feature(ptr_internals)] #![feature(ptr_metadata)] #![feature(ptr_sub_ptr)] #![feature(receiver_trait)] +#![feature(saturating_int_impl)] #![feature(set_ptr_value)] +#![feature(sized_type_properties)] +#![feature(slice_from_ptr_range)] #![feature(slice_group_by)] #![feature(slice_ptr_get)] #![feature(slice_ptr_len)] @@ -144,12 +155,13 @@ #![feature(unchecked_math)] #![feature(unicode_internals)] #![feature(unsize)] +#![feature(utf8_chunks)] +#![feature(std_internals)] // // Language features: #![feature(allocator_internals)] #![feature(allow_internal_unstable)] #![feature(associated_type_bounds)] -#![feature(box_syntax)] #![feature(cfg_sanitize)] #![feature(const_deref)] #![feature(const_mut_refs)] @@ -163,19 +175,20 @@ #![cfg_attr(not(test), feature(generator_trait))] #![feature(hashmap_internals)] #![feature(lang_items)] -#![feature(let_else)] #![feature(min_specialization)] #![feature(negative_impls)] #![feature(never_type)] -#![feature(nll)] // Not necessary, but here to test the `nll` feature. #![feature(rustc_allow_const_fn_unstable)] #![feature(rustc_attrs)] +#![feature(pointer_is_aligned)] #![feature(slice_internals)] #![feature(staged_api)] +#![feature(stmt_expr_attributes)] #![cfg_attr(test, feature(test))] #![feature(unboxed_closures)] #![feature(unsized_fn_params)] #![feature(c_unwind)] +#![feature(with_negative_coherence)] // // Rustdoc features: #![feature(doc_cfg)] @@ -218,7 +231,7 @@ mod boxed { #[cfg(not(no_borrow))] pub mod borrow; pub mod collections; -#[cfg(not(no_global_oom_handling))] +#[cfg(all(not(no_rc), not(no_sync), not(no_global_oom_handling)))] pub mod ffi; #[cfg(not(no_fmt))] pub mod fmt; @@ -230,9 +243,9 @@ pub mod str; #[cfg(not(no_string))] pub mod string; #[cfg(not(no_sync))] -#[cfg(target_has_atomic = "ptr")] +#[cfg(all(not(no_rc), target_has_atomic = "ptr"))] pub mod sync; -#[cfg(all(not(no_global_oom_handling), target_has_atomic = "ptr"))] +#[cfg(all(not(no_global_oom_handling), not(no_rc), not(no_sync), target_has_atomic = "ptr"))] pub mod task; #[cfg(test)] mod tests; diff --git a/rust/alloc/raw_vec.rs b/rust/alloc/raw_vec.rs index eb77db5def55d0..f02faff67a80b7 100644 --- a/rust/alloc/raw_vec.rs +++ b/rust/alloc/raw_vec.rs @@ -5,7 +5,7 @@ use core::alloc::LayoutError; use core::cmp; use core::intrinsics; -use core::mem::{self, ManuallyDrop, MaybeUninit}; +use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; use core::ops::Drop; use core::ptr::{self, NonNull, Unique}; use core::slice; @@ -177,7 +177,7 @@ impl RawVec { #[cfg(not(no_global_oom_handling))] fn allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Self { // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. - if mem::size_of::() == 0 || capacity == 0 { + if T::IS_ZST || capacity == 0 { Self::new_in(alloc) } else { // We avoid `unwrap_or_else` here because it bloats the amount of @@ -212,7 +212,7 @@ impl RawVec { fn try_allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Result { // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. - if mem::size_of::() == 0 || capacity == 0 { + if T::IS_ZST || capacity == 0 { return Ok(Self::new_in(alloc)); } @@ -262,7 +262,7 @@ impl RawVec { /// This will always be `usize::MAX` if `T` is zero-sized. #[inline(always)] pub fn capacity(&self) -> usize { - if mem::size_of::() == 0 { usize::MAX } else { self.cap } + if T::IS_ZST { usize::MAX } else { self.cap } } /// Returns a shared reference to the allocator backing this `RawVec`. @@ -271,7 +271,7 @@ impl RawVec { } fn current_memory(&self) -> Option<(NonNull, Layout)> { - if mem::size_of::() == 0 || self.cap == 0 { + if T::IS_ZST || self.cap == 0 { None } else { // We have an allocated chunk of memory, so we can bypass runtime @@ -419,7 +419,7 @@ impl RawVec { // This is ensured by the calling contexts. debug_assert!(additional > 0); - if mem::size_of::() == 0 { + if T::IS_ZST { // Since we return a capacity of `usize::MAX` when `elem_size` is // 0, getting to here necessarily means the `RawVec` is overfull. return Err(CapacityOverflow.into()); @@ -445,7 +445,7 @@ impl RawVec { // `grow_amortized`, but this method is usually instantiated less often so // it's less critical. fn grow_exact(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { - if mem::size_of::() == 0 { + if T::IS_ZST { // Since we return a capacity of `usize::MAX` when the type size is // 0, getting to here necessarily means the `RawVec` is overfull. return Err(CapacityOverflow.into()); diff --git a/rust/alloc/slice.rs b/rust/alloc/slice.rs index e444e97fa14538..467935cf9a06db 100644 --- a/rust/alloc/slice.rs +++ b/rust/alloc/slice.rs @@ -1,84 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -//! A dynamically-sized view into a contiguous sequence, `[T]`. +//! Utilities for the slice primitive type. //! //! *[See also the slice primitive type](slice).* //! -//! Slices are a view into a block of memory represented as a pointer and a -//! length. +//! Most of the structs in this module are iterator types which can only be created +//! using a certain function. For example, `slice.iter()` yields an [`Iter`]. //! -//! ``` -//! // slicing a Vec -//! let vec = vec![1, 2, 3]; -//! let int_slice = &vec[..]; -//! // coercing an array to a slice -//! let str_slice: &[&str] = &["one", "two", "three"]; -//! ``` -//! -//! Slices are either mutable or shared. The shared slice type is `&[T]`, -//! while the mutable slice type is `&mut [T]`, where `T` represents the element -//! type. For example, you can mutate the block of memory that a mutable slice -//! points to: -//! -//! ``` -//! let x = &mut [1, 2, 3]; -//! x[1] = 7; -//! assert_eq!(x, &[1, 7, 3]); -//! ``` -//! -//! Here are some of the things this module contains: -//! -//! ## Structs -//! -//! There are several structs that are useful for slices, such as [`Iter`], which -//! represents iteration over a slice. -//! -//! ## Trait Implementations -//! -//! There are several implementations of common traits for slices. Some examples -//! include: -//! -//! * [`Clone`] -//! * [`Eq`], [`Ord`] - for slices whose element type are [`Eq`] or [`Ord`]. -//! * [`Hash`] - for slices whose element type is [`Hash`]. -//! -//! ## Iteration -//! -//! The slices implement `IntoIterator`. The iterator yields references to the -//! slice elements. -//! -//! ``` -//! let numbers = &[0, 1, 2]; -//! for n in numbers { -//! println!("{n} is a number!"); -//! } -//! ``` -//! -//! The mutable slice yields mutable references to the elements: -//! -//! ``` -//! let mut scores = [7, 8, 9]; -//! for score in &mut scores[..] { -//! *score += 1; -//! } -//! ``` -//! -//! This iterator yields mutable references to the slice's elements, so while -//! the element type of the slice is `i32`, the element type of the iterator is -//! `&mut i32`. -//! -//! * [`.iter`] and [`.iter_mut`] are the explicit methods to return the default -//! iterators. -//! * Further methods that return iterators are [`.split`], [`.splitn`], -//! [`.chunks`], [`.windows`] and more. -//! -//! [`Hash`]: core::hash::Hash -//! [`.iter`]: slice::iter -//! [`.iter_mut`]: slice::iter_mut -//! [`.split`]: slice::split -//! [`.splitn`]: slice::splitn -//! [`.chunks`]: slice::chunks -//! [`.windows`]: slice::windows +//! A few functions are provided to create a slice from a value reference +//! or from a raw pointer. #![stable(feature = "rust1", since = "1.0.0")] // Many of the usings in this module are only used in the test configuration. // It's cleaner to just turn off the unused_imports warning than to fix them. @@ -88,9 +18,7 @@ use core::borrow::{Borrow, BorrowMut}; #[cfg(not(no_global_oom_handling))] use core::cmp::Ordering::{self, Less}; #[cfg(not(no_global_oom_handling))] -use core::mem; -#[cfg(not(no_global_oom_handling))] -use core::mem::size_of; +use core::mem::{self, SizedTypeProperties}; #[cfg(not(no_global_oom_handling))] use core::ptr; @@ -116,6 +44,8 @@ pub use core::slice::EscapeAscii; pub use core::slice::SliceIndex; #[stable(feature = "from_ref", since = "1.28.0")] pub use core::slice::{from_mut, from_ref}; +#[unstable(feature = "slice_from_ptr_range", issue = "89792")] +pub use core::slice::{from_mut_ptr_range, from_ptr_range}; #[stable(feature = "rust1", since = "1.0.0")] pub use core::slice::{from_raw_parts, from_raw_parts_mut}; #[stable(feature = "rust1", since = "1.0.0")] @@ -275,7 +205,7 @@ impl [T] { where T: Ord, { - merge_sort(self, |a, b| a.lt(b)); + merge_sort(self, T::lt); } /// Sorts the slice with a comparator function. @@ -836,14 +766,14 @@ impl> Join<&[T]> for [V] { //////////////////////////////////////////////////////////////////////////////// #[stable(feature = "rust1", since = "1.0.0")] -impl Borrow<[T]> for Vec { +impl Borrow<[T]> for Vec { fn borrow(&self) -> &[T] { &self[..] } } #[stable(feature = "rust1", since = "1.0.0")] -impl BorrowMut<[T]> for Vec { +impl BorrowMut<[T]> for Vec { fn borrow_mut(&mut self) -> &mut [T] { &mut self[..] } @@ -1024,7 +954,7 @@ where // Consume the greater side. // If equal, prefer the right run to maintain stability. unsafe { - let to_copy = if is_less(&*right.offset(-1), &*left.offset(-1)) { + let to_copy = if is_less(&*right.sub(1), &*left.sub(1)) { decrement_and_get(left) } else { decrement_and_get(right) @@ -1038,12 +968,12 @@ where unsafe fn get_and_increment(ptr: &mut *mut T) -> *mut T { let old = *ptr; - *ptr = unsafe { ptr.offset(1) }; + *ptr = unsafe { ptr.add(1) }; old } unsafe fn decrement_and_get(ptr: &mut *mut T) -> *mut T { - *ptr = unsafe { ptr.offset(-1) }; + *ptr = unsafe { ptr.sub(1) }; *ptr } @@ -1088,7 +1018,7 @@ where const MIN_RUN: usize = 10; // Sorting has no meaningful behavior on zero-sized types. - if size_of::() == 0 { + if T::IS_ZST { return; } diff --git a/rust/alloc/vec/drain.rs b/rust/alloc/vec/drain.rs index b6a5f98e4fcd95..3594ad890c3dca 100644 --- a/rust/alloc/vec/drain.rs +++ b/rust/alloc/vec/drain.rs @@ -3,7 +3,7 @@ use crate::alloc::{Allocator, Global}; use core::fmt; use core::iter::{FusedIterator, TrustedLen}; -use core::mem; +use core::mem::{self, ManuallyDrop, SizedTypeProperties}; use core::ptr::{self, NonNull}; use core::slice::{self}; @@ -67,6 +67,77 @@ impl<'a, T, A: Allocator> Drain<'a, T, A> { pub fn allocator(&self) -> &A { unsafe { self.vec.as_ref().allocator() } } + + /// Keep unyielded elements in the source `Vec`. + /// + /// # Examples + /// + /// ``` + /// #![feature(drain_keep_rest)] + /// + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// + /// assert_eq!(drain.next().unwrap(), 'a'); + /// + /// // This call keeps 'b' and 'c' in the vec. + /// drain.keep_rest(); + /// + /// // If we wouldn't call `keep_rest()`, + /// // `vec` would be empty. + /// assert_eq!(vec, ['b', 'c']); + /// ``` + #[unstable(feature = "drain_keep_rest", issue = "101122")] + pub fn keep_rest(self) { + // At this moment layout looks like this: + // + // [head] [yielded by next] [unyielded] [yielded by next_back] [tail] + // ^-- start \_________/-- unyielded_len \____/-- self.tail_len + // ^-- unyielded_ptr ^-- tail + // + // Normally `Drop` impl would drop [unyielded] and then move [tail] to the `start`. + // Here we want to + // 1. Move [unyielded] to `start` + // 2. Move [tail] to a new start at `start + len(unyielded)` + // 3. Update length of the original vec to `len(head) + len(unyielded) + len(tail)` + // a. In case of ZST, this is the only thing we want to do + // 4. Do *not* drop self, as everything is put in a consistent state already, there is nothing to do + let mut this = ManuallyDrop::new(self); + + unsafe { + let source_vec = this.vec.as_mut(); + + let start = source_vec.len(); + let tail = this.tail_start; + + let unyielded_len = this.iter.len(); + let unyielded_ptr = this.iter.as_slice().as_ptr(); + + // ZSTs have no identity, so we don't need to move them around. + let needs_move = mem::size_of::() != 0; + + if needs_move { + let start_ptr = source_vec.as_mut_ptr().add(start); + + // memmove back unyielded elements + if unyielded_ptr != start_ptr { + let src = unyielded_ptr; + let dst = start_ptr; + + ptr::copy(src, dst, unyielded_len); + } + + // memmove back untouched tail + if tail != (start + unyielded_len) { + let src = source_vec.as_ptr().add(tail); + let dst = start_ptr.add(unyielded_len); + ptr::copy(src, dst, this.tail_len); + } + } + + source_vec.set_len(start + unyielded_len + this.tail_len); + } + } } #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] @@ -133,7 +204,7 @@ impl Drop for Drain<'_, T, A> { let mut vec = self.vec; - if mem::size_of::() == 0 { + if T::IS_ZST { // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. unsafe { diff --git a/rust/alloc/vec/drain_filter.rs b/rust/alloc/vec/drain_filter.rs index b04fce041622f3..4b019220657dfc 100644 --- a/rust/alloc/vec/drain_filter.rs +++ b/rust/alloc/vec/drain_filter.rs @@ -1,8 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::alloc::{Allocator, Global}; -use core::ptr::{self}; -use core::slice::{self}; +use core::mem::{self, ManuallyDrop}; +use core::ptr; +use core::slice; use super::Vec; @@ -56,6 +57,61 @@ where pub fn allocator(&self) -> &A { self.vec.allocator() } + + /// Keep unyielded elements in the source `Vec`. + /// + /// # Examples + /// + /// ``` + /// #![feature(drain_filter)] + /// #![feature(drain_keep_rest)] + /// + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain_filter(|_| true); + /// + /// assert_eq!(drain.next().unwrap(), 'a'); + /// + /// // This call keeps 'b' and 'c' in the vec. + /// drain.keep_rest(); + /// + /// // If we wouldn't call `keep_rest()`, + /// // `vec` would be empty. + /// assert_eq!(vec, ['b', 'c']); + /// ``` + #[unstable(feature = "drain_keep_rest", issue = "101122")] + pub fn keep_rest(self) { + // At this moment layout looks like this: + // + // _____________________/-- old_len + // / \ + // [kept] [yielded] [tail] + // \_______/ ^-- idx + // \-- del + // + // Normally `Drop` impl would drop [tail] (via .for_each(drop), ie still calling `pred`) + // + // 1. Move [tail] after [kept] + // 2. Update length of the original vec to `old_len - del` + // a. In case of ZST, this is the only thing we want to do + // 3. Do *not* drop self, as everything is put in a consistent state already, there is nothing to do + let mut this = ManuallyDrop::new(self); + + unsafe { + // ZSTs have no identity, so we don't need to move them around. + let needs_move = mem::size_of::() != 0; + + if needs_move && this.idx < this.old_len && this.del > 0 { + let ptr = this.vec.as_mut_ptr(); + let src = ptr.add(this.idx); + let dst = src.sub(this.del); + let tail_len = this.old_len - this.idx; + src.copy_to(dst, tail_len); + } + + let new_len = this.old_len - this.del; + this.vec.set_len(new_len); + } + } } #[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] diff --git a/rust/alloc/vec/into_iter.rs b/rust/alloc/vec/into_iter.rs index f7a50e76691e80..a8a2a8b66bfddf 100644 --- a/rust/alloc/vec/into_iter.rs +++ b/rust/alloc/vec/into_iter.rs @@ -4,13 +4,13 @@ use super::AsVecIntoIter; use crate::alloc::{Allocator, Global}; use crate::raw_vec::RawVec; +use core::array; use core::fmt; -use core::intrinsics::arith_offset; use core::iter::{ FusedIterator, InPlaceIterable, SourceIter, TrustedLen, TrustedRandomAccessNoCoerce, }; use core::marker::PhantomData; -use core::mem::{self, ManuallyDrop}; +use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; #[cfg(not(no_global_oom_handling))] use core::ops::Deref; use core::ptr::{self, NonNull}; @@ -97,13 +97,16 @@ impl IntoIter { } /// Drops remaining elements and relinquishes the backing allocation. + /// This method guarantees it won't panic before relinquishing + /// the backing allocation. /// /// This is roughly equivalent to the following, but more efficient /// /// ``` /// # let mut into_iter = Vec::::with_capacity(10).into_iter(); + /// let mut into_iter = std::mem::replace(&mut into_iter, Vec::new().into_iter()); /// (&mut into_iter).for_each(core::mem::drop); - /// unsafe { core::ptr::write(&mut into_iter, Vec::new().into_iter()); } + /// std::mem::forget(into_iter); /// ``` /// /// This method is used by in-place iteration, refer to the vec::in_place_collect @@ -120,6 +123,8 @@ impl IntoIter { self.ptr = self.buf.as_ptr(); self.end = self.buf.as_ptr(); + // Dropping the remaining elements can panic, so this needs to be + // done only after updating the other fields. unsafe { ptr::drop_in_place(remaining); } @@ -150,19 +155,19 @@ impl Iterator for IntoIter { #[inline] fn next(&mut self) -> Option { - if self.ptr as *const _ == self.end { + if self.ptr == self.end { None - } else if mem::size_of::() == 0 { + } else if T::IS_ZST { // purposefully don't use 'ptr.offset' because for // vectors with 0-size elements this would return the // same pointer. - self.ptr = unsafe { arith_offset(self.ptr as *const i8, 1) as *mut T }; + self.ptr = self.ptr.wrapping_byte_add(1); // Make up a value of this ZST. Some(unsafe { mem::zeroed() }) } else { let old = self.ptr; - self.ptr = unsafe { self.ptr.offset(1) }; + self.ptr = unsafe { self.ptr.add(1) }; Some(unsafe { ptr::read(old) }) } @@ -170,7 +175,7 @@ impl Iterator for IntoIter { #[inline] fn size_hint(&self) -> (usize, Option) { - let exact = if mem::size_of::() == 0 { + let exact = if T::IS_ZST { self.end.addr().wrapping_sub(self.ptr.addr()) } else { unsafe { self.end.sub_ptr(self.ptr) } @@ -182,11 +187,11 @@ impl Iterator for IntoIter { fn advance_by(&mut self, n: usize) -> Result<(), usize> { let step_size = self.len().min(n); let to_drop = ptr::slice_from_raw_parts_mut(self.ptr as *mut T, step_size); - if mem::size_of::() == 0 { + if T::IS_ZST { // SAFETY: due to unchecked casts of unsigned amounts to signed offsets the wraparound // effectively results in unsigned pointers representing positions 0..usize::MAX, // which is valid for ZSTs. - self.ptr = unsafe { arith_offset(self.ptr as *const i8, step_size as isize) as *mut T } + self.ptr = self.ptr.wrapping_byte_add(step_size); } else { // SAFETY: the min() above ensures that step_size is in bounds self.ptr = unsafe { self.ptr.add(step_size) }; @@ -206,6 +211,43 @@ impl Iterator for IntoIter { self.len() } + #[inline] + fn next_chunk(&mut self) -> Result<[T; N], core::array::IntoIter> { + let mut raw_ary = MaybeUninit::uninit_array(); + + let len = self.len(); + + if T::IS_ZST { + if len < N { + self.forget_remaining_elements(); + // Safety: ZSTs can be conjured ex nihilo, only the amount has to be correct + return Err(unsafe { array::IntoIter::new_unchecked(raw_ary, 0..len) }); + } + + self.ptr = self.ptr.wrapping_byte_add(N); + // Safety: ditto + return Ok(unsafe { raw_ary.transpose().assume_init() }); + } + + if len < N { + // Safety: `len` indicates that this many elements are available and we just checked that + // it fits into the array. + unsafe { + ptr::copy_nonoverlapping(self.ptr, raw_ary.as_mut_ptr() as *mut T, len); + self.forget_remaining_elements(); + return Err(array::IntoIter::new_unchecked(raw_ary, 0..len)); + } + } + + // Safety: `len` is larger than the array size. Copy a fixed amount here to fully initialize + // the array. + return unsafe { + ptr::copy_nonoverlapping(self.ptr, raw_ary.as_mut_ptr() as *mut T, N); + self.ptr = self.ptr.add(N); + Ok(raw_ary.transpose().assume_init()) + }; + } + unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> Self::Item where Self: TrustedRandomAccessNoCoerce, @@ -219,7 +261,7 @@ impl Iterator for IntoIter { // that `T: Copy` so reading elements from the buffer doesn't invalidate // them for `Drop`. unsafe { - if mem::size_of::() == 0 { mem::zeroed() } else { ptr::read(self.ptr.add(i)) } + if T::IS_ZST { mem::zeroed() } else { ptr::read(self.ptr.add(i)) } } } } @@ -230,14 +272,14 @@ impl DoubleEndedIterator for IntoIter { fn next_back(&mut self) -> Option { if self.end == self.ptr { None - } else if mem::size_of::() == 0 { + } else if T::IS_ZST { // See above for why 'ptr.offset' isn't used - self.end = unsafe { arith_offset(self.end as *const i8, -1) as *mut T }; + self.end = self.end.wrapping_byte_sub(1); // Make up a value of this ZST. Some(unsafe { mem::zeroed() }) } else { - self.end = unsafe { self.end.offset(-1) }; + self.end = unsafe { self.end.sub(1) }; Some(unsafe { ptr::read(self.end) }) } @@ -246,14 +288,12 @@ impl DoubleEndedIterator for IntoIter { #[inline] fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { let step_size = self.len().min(n); - if mem::size_of::() == 0 { + if T::IS_ZST { // SAFETY: same as for advance_by() - self.end = unsafe { - arith_offset(self.end as *const i8, step_size.wrapping_neg() as isize) as *mut T - } + self.end = self.end.wrapping_byte_sub(step_size); } else { // SAFETY: same as for advance_by() - self.end = unsafe { self.end.offset(step_size.wrapping_neg() as isize) }; + self.end = unsafe { self.end.sub(step_size) }; } let to_drop = ptr::slice_from_raw_parts_mut(self.end as *mut T, step_size); // SAFETY: same as for advance_by() diff --git a/rust/alloc/vec/is_zero.rs b/rust/alloc/vec/is_zero.rs index 377f3d17277755..426bb2c9f6ff5f 100644 --- a/rust/alloc/vec/is_zero.rs +++ b/rust/alloc/vec/is_zero.rs @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +use core::num::{Saturating, Wrapping}; use crate::boxed::Box; @@ -19,12 +20,14 @@ macro_rules! impl_is_zero { }; } +impl_is_zero!(i8, |x| x == 0); // It is needed to impl for arrays and tuples of i8. impl_is_zero!(i16, |x| x == 0); impl_is_zero!(i32, |x| x == 0); impl_is_zero!(i64, |x| x == 0); impl_is_zero!(i128, |x| x == 0); impl_is_zero!(isize, |x| x == 0); +impl_is_zero!(u8, |x| x == 0); // It is needed to impl for arrays and tuples of u8. impl_is_zero!(u16, |x| x == 0); impl_is_zero!(u32, |x| x == 0); impl_is_zero!(u64, |x| x == 0); @@ -56,15 +59,41 @@ unsafe impl IsZero for [T; N] { fn is_zero(&self) -> bool { // Because this is generated as a runtime check, it's not obvious that // it's worth doing if the array is really long. The threshold here - // is largely arbitrary, but was picked because as of 2022-05-01 LLVM - // can const-fold the check in `vec![[0; 32]; n]` but not in - // `vec![[0; 64]; n]`: https://godbolt.org/z/WTzjzfs5b + // is largely arbitrary, but was picked because as of 2022-07-01 LLVM + // fails to const-fold the check in `vec![[1; 32]; n]` + // See https://github.com/rust-lang/rust/pull/97581#issuecomment-1166628022 // Feel free to tweak if you have better evidence. - N <= 32 && self.iter().all(IsZero::is_zero) + N <= 16 && self.iter().all(IsZero::is_zero) } } +// This is recursive macro. +macro_rules! impl_for_tuples { + // Stopper + () => { + // No use for implementing for empty tuple because it is ZST. + }; + ($first_arg:ident $(,$rest:ident)*) => { + unsafe impl <$first_arg: IsZero, $($rest: IsZero,)*> IsZero for ($first_arg, $($rest,)*){ + #[inline] + fn is_zero(&self) -> bool{ + // Destructure tuple to N references + // Rust allows to hide generic params by local variable names. + #[allow(non_snake_case)] + let ($first_arg, $($rest,)*) = self; + + $first_arg.is_zero() + $( && $rest.is_zero() )* + } + } + + impl_for_tuples!($($rest),*); + } +} + +impl_for_tuples!(A, B, C, D, E, F, G, H); + // `Option<&T>` and `Option>` are guaranteed to represent `None` as null. // For fat pointers, the bytes that would be the pointer metadata in the `Some` // variant are padding in the `None` variant, so ignoring them and @@ -118,3 +147,39 @@ impl_is_zero_option_of_nonzero!( NonZeroUsize, NonZeroIsize, ); + +unsafe impl IsZero for Wrapping { + #[inline] + fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +unsafe impl IsZero for Saturating { + #[inline] + fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +macro_rules! impl_for_optional_bool { + ($($t:ty,)+) => {$( + unsafe impl IsZero for $t { + #[inline] + fn is_zero(&self) -> bool { + // SAFETY: This is *not* a stable layout guarantee, but + // inside `core` we're allowed to rely on the current rustc + // behaviour that options of bools will be one byte with + // no padding, so long as they're nested less than 254 deep. + let raw: u8 = unsafe { core::mem::transmute(*self) }; + raw == 0 + } + } + )+}; +} +impl_for_optional_bool! { + Option, + Option>, + Option>>, + // Could go further, but not worth the metadata overhead +} diff --git a/rust/alloc/vec/mod.rs b/rust/alloc/vec/mod.rs index fe4fff5064bcd0..3308724352cc4f 100644 --- a/rust/alloc/vec/mod.rs +++ b/rust/alloc/vec/mod.rs @@ -61,12 +61,12 @@ use core::cmp::Ordering; use core::convert::TryFrom; use core::fmt; use core::hash::{Hash, Hasher}; -use core::intrinsics::{arith_offset, assume}; +use core::intrinsics::assume; use core::iter; #[cfg(not(no_global_oom_handling))] use core::iter::FromIterator; use core::marker::PhantomData; -use core::mem::{self, ManuallyDrop, MaybeUninit}; +use core::mem::{self, ManuallyDrop, MaybeUninit, SizedTypeProperties}; use core::ops::{self, Index, IndexMut, Range, RangeBounds}; use core::ptr::{self, NonNull}; use core::slice::{self, SliceIndex}; @@ -127,7 +127,7 @@ use self::set_len_on_drop::SetLenOnDrop; mod set_len_on_drop; #[cfg(not(no_global_oom_handling))] -use self::in_place_drop::InPlaceDrop; +use self::in_place_drop::{InPlaceDrop, InPlaceDstBufDrop}; #[cfg(not(no_global_oom_handling))] mod in_place_drop; @@ -428,17 +428,25 @@ impl Vec { Vec { buf: RawVec::NEW, len: 0 } } - /// Constructs a new, empty `Vec` with the specified capacity. + /// Constructs a new, empty `Vec` with at least the specified capacity. /// - /// The vector will be able to hold exactly `capacity` elements without - /// reallocating. If `capacity` is 0, the vector will not allocate. + /// The vector will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the vector will not allocate. /// /// It is important to note that although the returned vector has the - /// *capacity* specified, the vector will have a zero *length*. For an - /// explanation of the difference between length and capacity, see + /// minimum *capacity* specified, the vector will have a zero *length*. For + /// an explanation of the difference between length and capacity, see /// *[Capacity and reallocation]*. /// + /// If it is important to know the exact allocated capacity of a `Vec`, + /// always use the [`capacity`] method after construction. + /// + /// For `Vec` where `T` is a zero-sized type, there will be no allocation + /// and the capacity will always be `usize::MAX`. + /// /// [Capacity and reallocation]: #capacity-and-reallocation + /// [`capacity`]: Vec::capacity /// /// # Panics /// @@ -451,19 +459,24 @@ impl Vec { /// /// // The vector contains no items, even though it has capacity for more /// assert_eq!(vec.len(), 0); - /// assert_eq!(vec.capacity(), 10); + /// assert!(vec.capacity() >= 10); /// /// // These are all done without reallocating... /// for i in 0..10 { /// vec.push(i); /// } /// assert_eq!(vec.len(), 10); - /// assert_eq!(vec.capacity(), 10); + /// assert!(vec.capacity() >= 10); /// /// // ...but this may make the vector reallocate /// vec.push(11); /// assert_eq!(vec.len(), 11); /// assert!(vec.capacity() >= 11); + /// + /// // A vector of a zero-sized type will always over-allocate, since no + /// // allocation is necessary + /// let vec_units = Vec::<()>::with_capacity(10); + /// assert_eq!(vec_units.capacity(), usize::MAX); /// ``` #[cfg(not(no_global_oom_handling))] #[inline] @@ -473,17 +486,25 @@ impl Vec { Self::with_capacity_in(capacity, Global) } - /// Tries to construct a new, empty `Vec` with the specified capacity. + /// Tries to construct a new, empty `Vec` with at least the specified capacity. /// - /// The vector will be able to hold exactly `capacity` elements without - /// reallocating. If `capacity` is 0, the vector will not allocate. + /// The vector will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the vector will not allocate. /// /// It is important to note that although the returned vector has the - /// *capacity* specified, the vector will have a zero *length*. For an - /// explanation of the difference between length and capacity, see + /// minimum *capacity* specified, the vector will have a zero *length*. For + /// an explanation of the difference between length and capacity, see /// *[Capacity and reallocation]*. /// + /// If it is important to know the exact allocated capacity of a `Vec`, + /// always use the [`capacity`] method after construction. + /// + /// For `Vec` where `T` is a zero-sized type, there will be no allocation + /// and the capacity will always be `usize::MAX`. + /// /// [Capacity and reallocation]: #capacity-and-reallocation + /// [`capacity`]: Vec::capacity /// /// # Examples /// @@ -492,22 +513,24 @@ impl Vec { /// /// // The vector contains no items, even though it has capacity for more /// assert_eq!(vec.len(), 0); - /// assert_eq!(vec.capacity(), 10); + /// assert!(vec.capacity() >= 10); /// /// // These are all done without reallocating... /// for i in 0..10 { /// vec.push(i); /// } /// assert_eq!(vec.len(), 10); - /// assert_eq!(vec.capacity(), 10); + /// assert!(vec.capacity() >= 10); /// /// // ...but this may make the vector reallocate /// vec.push(11); /// assert_eq!(vec.len(), 11); /// assert!(vec.capacity() >= 11); /// - /// let mut result = Vec::try_with_capacity(usize::MAX); - /// assert!(result.is_err()); + /// // A vector of a zero-sized type will always over-allocate, since no + /// // allocation is necessary + /// let vec_units = Vec::<()>::try_with_capacity(10).unwrap(); + /// assert_eq!(vec_units.capacity(), usize::MAX); /// ``` #[inline] #[stable(feature = "kernel", since = "1.0.0")] @@ -515,15 +538,13 @@ impl Vec { Self::try_with_capacity_in(capacity, Global) } - /// Creates a `Vec` directly from the raw components of another vector. + /// Creates a `Vec` directly from a pointer, a capacity, and a length. /// /// # Safety /// /// This is highly unsafe, due to the number of invariants that aren't /// checked: /// - /// * `ptr` needs to have been previously allocated via [`String`]/`Vec` - /// (at least, it's highly likely to be incorrect if it wasn't). /// * `T` needs to have the same alignment as what `ptr` was allocated with. /// (`T` having a less strict alignment is not sufficient, the alignment really /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be @@ -532,6 +553,14 @@ impl Vec { /// to be the same size as the pointer was allocated with. (Because similar to /// alignment, [`dealloc`] must be called with the same layout `size`.) /// * `length` needs to be less than or equal to `capacity`. + /// * The first `length` values must be properly initialized values of type `T`. + /// * `capacity` needs to be the capacity that the pointer was allocated with. + /// * The allocated size in bytes must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// These requirements are always upheld by any `ptr` that has been allocated + /// via `Vec`. Other allocation sources are allowed if the invariants are + /// upheld. /// /// Violating these may cause problems like corrupting the allocator's /// internal data structures. For example it is normally **not** safe @@ -574,8 +603,8 @@ impl Vec { /// /// unsafe { /// // Overwrite memory with 4, 5, 6 - /// for i in 0..len as isize { - /// ptr::write(p.offset(i), 4 + i); + /// for i in 0..len { + /// ptr::write(p.add(i), 4 + i); /// } /// /// // Put everything back together into a Vec @@ -583,6 +612,32 @@ impl Vec { /// assert_eq!(rebuilt, [4, 5, 6]); /// } /// ``` + /// + /// Using memory that was allocated elsewhere: + /// + /// ```rust + /// #![feature(allocator_api)] + /// + /// use std::alloc::{AllocError, Allocator, Global, Layout}; + /// + /// fn main() { + /// let layout = Layout::array::(16).expect("overflow cannot happen"); + /// + /// let vec = unsafe { + /// let mem = match Global.allocate(layout) { + /// Ok(mem) => mem.cast::().as_ptr(), + /// Err(AllocError) => return, + /// }; + /// + /// mem.write(1_000_000); + /// + /// Vec::from_raw_parts_in(mem, 1, 16, Global) + /// }; + /// + /// assert_eq!(vec, &[1_000_000]); + /// assert_eq!(vec.capacity(), 16); + /// } + /// ``` #[inline] #[stable(feature = "rust1", since = "1.0.0")] pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { @@ -611,18 +666,26 @@ impl Vec { Vec { buf: RawVec::new_in(alloc), len: 0 } } - /// Constructs a new, empty `Vec` with the specified capacity with the provided - /// allocator. + /// Constructs a new, empty `Vec` with at least the specified capacity + /// with the provided allocator. /// - /// The vector will be able to hold exactly `capacity` elements without - /// reallocating. If `capacity` is 0, the vector will not allocate. + /// The vector will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the vector will not allocate. /// /// It is important to note that although the returned vector has the - /// *capacity* specified, the vector will have a zero *length*. For an - /// explanation of the difference between length and capacity, see + /// minimum *capacity* specified, the vector will have a zero *length*. For + /// an explanation of the difference between length and capacity, see /// *[Capacity and reallocation]*. /// + /// If it is important to know the exact allocated capacity of a `Vec`, + /// always use the [`capacity`] method after construction. + /// + /// For `Vec` where `T` is a zero-sized type, there will be no allocation + /// and the capacity will always be `usize::MAX`. + /// /// [Capacity and reallocation]: #capacity-and-reallocation + /// [`capacity`]: Vec::capacity /// /// # Panics /// @@ -652,6 +715,11 @@ impl Vec { /// vec.push(11); /// assert_eq!(vec.len(), 11); /// assert!(vec.capacity() >= 11); + /// + /// // A vector of a zero-sized type will always over-allocate, since no + /// // allocation is necessary + /// let vec_units = Vec::<(), System>::with_capacity_in(10, System); + /// assert_eq!(vec_units.capacity(), usize::MAX); /// ``` #[cfg(not(no_global_oom_handling))] #[inline] @@ -660,18 +728,26 @@ impl Vec { Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 } } - /// Tries to construct a new, empty `Vec` with the specified capacity + /// Tries to construct a new, empty `Vec` with at least the specified capacity /// with the provided allocator. /// - /// The vector will be able to hold exactly `capacity` elements without - /// reallocating. If `capacity` is 0, the vector will not allocate. + /// The vector will be able to hold at least `capacity` elements without + /// reallocating. This method is allowed to allocate for more elements than + /// `capacity`. If `capacity` is 0, the vector will not allocate. /// /// It is important to note that although the returned vector has the - /// *capacity* specified, the vector will have a zero *length*. For an - /// explanation of the difference between length and capacity, see + /// minimum *capacity* specified, the vector will have a zero *length*. For + /// an explanation of the difference between length and capacity, see /// *[Capacity and reallocation]*. /// + /// If it is important to know the exact allocated capacity of a `Vec`, + /// always use the [`capacity`] method after construction. + /// + /// For `Vec` where `T` is a zero-sized type, there will be no allocation + /// and the capacity will always be `usize::MAX`. + /// /// [Capacity and reallocation]: #capacity-and-reallocation + /// [`capacity`]: Vec::capacity /// /// # Examples /// @@ -698,8 +774,10 @@ impl Vec { /// assert_eq!(vec.len(), 11); /// assert!(vec.capacity() >= 11); /// - /// let mut result = Vec::try_with_capacity_in(usize::MAX, System); - /// assert!(result.is_err()); + /// // A vector of a zero-sized type will always over-allocate, since no + /// // allocation is necessary + /// let vec_units = Vec::<(), System>::try_with_capacity_in(10, System).unwrap(); + /// assert_eq!(vec_units.capacity(), usize::MAX); /// ``` #[inline] #[stable(feature = "kernel", since = "1.0.0")] @@ -707,21 +785,30 @@ impl Vec { Ok(Vec { buf: RawVec::try_with_capacity_in(capacity, alloc)?, len: 0 }) } - /// Creates a `Vec` directly from the raw components of another vector. + /// Creates a `Vec` directly from a pointer, a capacity, a length, + /// and an allocator. /// /// # Safety /// /// This is highly unsafe, due to the number of invariants that aren't /// checked: /// - /// * `ptr` needs to have been previously allocated via [`String`]/`Vec` - /// (at least, it's highly likely to be incorrect if it wasn't). - /// * `T` needs to have the same size and alignment as what `ptr` was allocated with. + /// * `T` needs to have the same alignment as what `ptr` was allocated with. /// (`T` having a less strict alignment is not sufficient, the alignment really /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be /// allocated and deallocated with the same layout.) + /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs + /// to be the same size as the pointer was allocated with. (Because similar to + /// alignment, [`dealloc`] must be called with the same layout `size`.) /// * `length` needs to be less than or equal to `capacity`. - /// * `capacity` needs to be the capacity that the pointer was allocated with. + /// * The first `length` values must be properly initialized values of type `T`. + /// * `capacity` needs to [*fit*] the layout size that the pointer was allocated with. + /// * The allocated size in bytes must be no larger than `isize::MAX`. + /// See the safety documentation of [`pointer::offset`]. + /// + /// These requirements are always upheld by any `ptr` that has been allocated + /// via `Vec`. Other allocation sources are allowed if the invariants are + /// upheld. /// /// Violating these may cause problems like corrupting the allocator's /// internal data structures. For example it is **not** safe @@ -739,6 +826,7 @@ impl Vec { /// /// [`String`]: crate::string::String /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// [*fit*]: crate::alloc::Allocator#memory-fitting /// /// # Examples /// @@ -768,8 +856,8 @@ impl Vec { /// /// unsafe { /// // Overwrite memory with 4, 5, 6 - /// for i in 0..len as isize { - /// ptr::write(p.offset(i), 4 + i); + /// for i in 0..len { + /// ptr::write(p.add(i), 4 + i); /// } /// /// // Put everything back together into a Vec @@ -777,6 +865,29 @@ impl Vec { /// assert_eq!(rebuilt, [4, 5, 6]); /// } /// ``` + /// + /// Using memory that was allocated elsewhere: + /// + /// ```rust + /// use std::alloc::{alloc, Layout}; + /// + /// fn main() { + /// let layout = Layout::array::(16).expect("overflow cannot happen"); + /// let vec = unsafe { + /// let mem = alloc(layout).cast::(); + /// if mem.is_null() { + /// return; + /// } + /// + /// mem.write(1_000_000); + /// + /// Vec::from_raw_parts(mem, 1, 16) + /// }; + /// + /// assert_eq!(vec, &[1_000_000]); + /// assert_eq!(vec.capacity(), 16); + /// } + /// ``` #[inline] #[unstable(feature = "allocator_api", issue = "32838")] pub unsafe fn from_raw_parts_in(ptr: *mut T, length: usize, capacity: usize, alloc: A) -> Self { @@ -869,13 +980,14 @@ impl Vec { (ptr, len, capacity, alloc) } - /// Returns the number of elements the vector can hold without + /// Returns the total number of elements the vector can hold without /// reallocating. /// /// # Examples /// /// ``` - /// let vec: Vec = Vec::with_capacity(10); + /// let mut vec: Vec = Vec::with_capacity(10); + /// vec.push(42); /// assert_eq!(vec.capacity(), 10); /// ``` #[inline] @@ -885,10 +997,10 @@ impl Vec { } /// Reserves capacity for at least `additional` more elements to be inserted - /// in the given `Vec`. The collection may reserve more space to avoid - /// frequent reallocations. After calling `reserve`, capacity will be - /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. + /// in the given `Vec`. The collection may reserve more space to + /// speculatively avoid frequent reallocations. After calling `reserve`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if capacity is already sufficient. /// /// # Panics /// @@ -907,10 +1019,12 @@ impl Vec { self.buf.reserve(self.len, additional); } - /// Reserves the minimum capacity for exactly `additional` more elements to - /// be inserted in the given `Vec`. After calling `reserve_exact`, - /// capacity will be greater than or equal to `self.len() + additional`. - /// Does nothing if the capacity is already sufficient. + /// Reserves the minimum capacity for at least `additional` more elements to + /// be inserted in the given `Vec`. Unlike [`reserve`], this will not + /// deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional`. Does nothing if the capacity is already + /// sufficient. /// /// Note that the allocator may give the collection more space than it /// requests. Therefore, capacity can not be relied upon to be precisely @@ -936,10 +1050,11 @@ impl Vec { } /// Tries to reserve capacity for at least `additional` more elements to be inserted - /// in the given `Vec`. The collection may reserve more space to avoid + /// in the given `Vec`. The collection may reserve more space to speculatively avoid /// frequent reallocations. After calling `try_reserve`, capacity will be - /// greater than or equal to `self.len() + additional`. Does nothing if - /// capacity is already sufficient. + /// greater than or equal to `self.len() + additional` if it returns + /// `Ok(())`. Does nothing if capacity is already sufficient. This method + /// preserves the contents even if an error occurs. /// /// # Errors /// @@ -971,10 +1086,11 @@ impl Vec { self.buf.try_reserve(self.len, additional) } - /// Tries to reserve the minimum capacity for exactly `additional` - /// elements to be inserted in the given `Vec`. After calling - /// `try_reserve_exact`, capacity will be greater than or equal to - /// `self.len() + additional` if it returns `Ok(())`. + /// Tries to reserve the minimum capacity for at least `additional` + /// elements to be inserted in the given `Vec`. Unlike [`try_reserve`], + /// this will not deliberately over-allocate to speculatively avoid frequent + /// allocations. After calling `try_reserve_exact`, capacity will be greater + /// than or equal to `self.len() + additional` if it returns `Ok(())`. /// Does nothing if the capacity is already sufficient. /// /// Note that the allocator may give the collection more space than it @@ -1199,7 +1315,8 @@ impl Vec { self } - /// Returns a raw pointer to the vector's buffer. + /// Returns a raw pointer to the vector's buffer, or a dangling raw pointer + /// valid for zero sized reads if the vector didn't allocate. /// /// The caller must ensure that the vector outlives the pointer this /// function returns, or else it will end up pointing to garbage. @@ -1236,7 +1353,8 @@ impl Vec { ptr } - /// Returns an unsafe mutable pointer to the vector's buffer. + /// Returns an unsafe mutable pointer to the vector's buffer, or a dangling + /// raw pointer valid for zero sized reads if the vector didn't allocate. /// /// The caller must ensure that the vector outlives the pointer this /// function returns, or else it will end up pointing to garbage. @@ -1440,9 +1558,6 @@ impl Vec { } let len = self.len(); - if index > len { - assert_failed(index, len); - } // space for the new element if len == self.buf.capacity() { @@ -1454,9 +1569,15 @@ impl Vec { // The spot to put the new value { let p = self.as_mut_ptr().add(index); - // Shift everything over to make space. (Duplicating the - // `index`th element into two consecutive places.) - ptr::copy(p, p.offset(1), len - index); + if index < len { + // Shift everything over to make space. (Duplicating the + // `index`th element into two consecutive places.) + ptr::copy(p, p.add(1), len - index); + } else if index == len { + // No elements need shifting. + } else { + assert_failed(index, len); + } // Write it in, overwriting the first copy of the `index`th // element. ptr::write(p, element); @@ -1513,7 +1634,7 @@ impl Vec { ret = ptr::read(ptr); // Shift everything down to fill in that spot. - ptr::copy(ptr.offset(1), ptr, len - index - 1); + ptr::copy(ptr.add(1), ptr, len - index - 1); } self.set_len(len - 1); ret @@ -1562,11 +1683,11 @@ impl Vec { /// /// ``` /// let mut vec = vec![1, 2, 3, 4]; - /// vec.retain_mut(|x| if *x > 3 { - /// false - /// } else { + /// vec.retain_mut(|x| if *x <= 3 { /// *x += 1; /// true + /// } else { + /// false /// }); /// assert_eq!(vec, [2, 3, 4]); /// ``` @@ -1854,6 +1975,51 @@ impl Vec { Ok(()) } + /// Appends an element if there is sufficient spare capacity, otherwise an error is returned + /// with the element. + /// + /// Unlike [`push`] this method will not reallocate when there's insufficient capacity. + /// The caller should use [`reserve`] or [`try_reserve`] to ensure that there is enough capacity. + /// + /// [`push`]: Vec::push + /// [`reserve`]: Vec::reserve + /// [`try_reserve`]: Vec::try_reserve + /// + /// # Examples + /// + /// A manual, panic-free alternative to [`FromIterator`]: + /// + /// ``` + /// #![feature(vec_push_within_capacity)] + /// + /// use std::collections::TryReserveError; + /// fn from_iter_fallible(iter: impl Iterator) -> Result, TryReserveError> { + /// let mut vec = Vec::new(); + /// for value in iter { + /// if let Err(value) = vec.push_within_capacity(value) { + /// vec.try_reserve(1)?; + /// // this cannot fail, the previous line either returned or added at least 1 free slot + /// let _ = vec.push_within_capacity(value); + /// } + /// } + /// Ok(vec) + /// } + /// assert_eq!(from_iter_fallible(0..100), Ok(Vec::from_iter(0..100))); + /// ``` + #[inline] + #[unstable(feature = "vec_push_within_capacity", issue = "100486")] + pub fn push_within_capacity(&mut self, value: T) -> Result<(), T> { + if self.len == self.buf.capacity() { + return Err(value); + } + unsafe { + let end = self.as_mut_ptr().add(self.len); + ptr::write(end, value); + self.len += 1; + } + Ok(()) + } + /// Removes the last element from a vector and returns it, or [`None`] if it /// is empty. /// @@ -1886,7 +2052,7 @@ impl Vec { /// /// # Panics /// - /// Panics if the number of elements in the vector overflows a `usize`. + /// Panics if the new capacity exceeds `isize::MAX` bytes. /// /// # Examples /// @@ -1980,9 +2146,7 @@ impl Vec { unsafe { // set self.vec length's to start, to be safe in case Drain is leaked self.set_len(start); - // Use the borrow in the IterMut to indicate borrowing behavior of the - // whole Drain iterator (like &mut T). - let range_slice = slice::from_raw_parts_mut(self.as_mut_ptr().add(start), end - start); + let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); Drain { tail_start: end, tail_len: len - end, @@ -2174,7 +2338,6 @@ impl Vec { /// static_ref[0] += 1; /// assert_eq!(static_ref, &[2, 2, 3]); /// ``` - #[cfg(not(no_global_oom_handling))] #[stable(feature = "vec_leak", since = "1.47.0")] #[inline] pub fn leak<'a>(self) -> &'a mut [T] @@ -2501,7 +2664,7 @@ impl Vec<[T; N], A> { #[unstable(feature = "slice_flatten", issue = "95629")] pub fn into_flattened(self) -> Vec { let (ptr, len, cap, alloc) = self.into_raw_parts_with_alloc(); - let (new_len, new_cap) = if mem::size_of::() == 0 { + let (new_len, new_cap) = if T::IS_ZST { (len.checked_mul(N).expect("vec len overflow"), usize::MAX) } else { // SAFETY: @@ -2563,7 +2726,7 @@ impl Vec { // Write all elements except the last one for _ in 1..n { ptr::write(ptr, value.next()); - ptr = ptr.offset(1); + ptr = ptr.add(1); // Increment the length in every step in case next() panics local_len.increment_len(1); } @@ -2689,7 +2852,7 @@ impl ExtendFromWithinSpec for Vec { // SAFETY: // - Both pointers are created from unique slice references (`&mut [_]`) // so they are valid and do not overlap. - // - Elements are :Copy so it's OK to to copy them, without doing + // - Elements are :Copy so it's OK to copy them, without doing // anything with the original values // - `count` is equal to the len of `source`, so source is valid for // `count` reads @@ -2712,6 +2875,7 @@ impl ExtendFromWithinSpec for Vec { impl ops::Deref for Vec { type Target = [T]; + #[inline] fn deref(&self) -> &[T] { unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } } @@ -2719,6 +2883,7 @@ impl ops::Deref for Vec { #[stable(feature = "rust1", since = "1.0.0")] impl ops::DerefMut for Vec { + #[inline] fn deref_mut(&mut self) -> &mut [T] { unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } } @@ -2845,10 +3010,13 @@ impl IntoIterator for Vec { /// /// ``` /// let v = vec!["a".to_string(), "b".to_string()]; - /// for s in v.into_iter() { - /// // s has type String, not &String - /// println!("{s}"); - /// } + /// let mut v_iter = v.into_iter(); + /// + /// let first_element: Option = v_iter.next(); + /// + /// assert_eq!(first_element, Some("a".to_string())); + /// assert_eq!(v_iter.next(), Some("b".to_string())); + /// assert_eq!(v_iter.next(), None); /// ``` #[inline] fn into_iter(self) -> IntoIter { @@ -2856,8 +3024,8 @@ impl IntoIterator for Vec { let mut me = ManuallyDrop::new(self); let alloc = ManuallyDrop::new(ptr::read(me.allocator())); let begin = me.as_mut_ptr(); - let end = if mem::size_of::() == 0 { - arith_offset(begin as *const i8, me.len() as isize) as *const T + let end = if T::IS_ZST { + begin.wrapping_byte_add(me.len()) } else { begin.add(me.len()) as *const T }; @@ -3135,6 +3303,8 @@ unsafe impl<#[may_dangle] T, A: Allocator> Drop for Vec { #[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] impl const Default for Vec { /// Creates an empty `Vec`. + /// + /// The vector will not allocate until elements are pushed onto it. fn default() -> Vec { Vec::new() } @@ -3227,12 +3397,15 @@ impl From<[T; N]> for Vec { /// ``` #[cfg(not(test))] fn from(s: [T; N]) -> Vec { - <[T]>::into_vec(box s) + <[T]>::into_vec( + #[rustc_box] + Box::new(s), + ) } #[cfg(test)] fn from(s: [T; N]) -> Vec { - crate::slice::into_vec(box s) + crate::slice::into_vec(Box::new(s)) } } diff --git a/rust/alloc/vec/spec_extend.rs b/rust/alloc/vec/spec_extend.rs index 5ce2d00991bcdc..9621120fc1c949 100644 --- a/rust/alloc/vec/spec_extend.rs +++ b/rust/alloc/vec/spec_extend.rs @@ -59,7 +59,7 @@ where let mut local_len = SetLenOnDrop::new(&mut self.len); iterator.for_each(move |element| { ptr::write(ptr, element); - ptr = ptr.offset(1); + ptr = ptr.add(1); // Since the loop executes user code which can panic we have to bump the pointer // after each step. // NB can't overflow since we would have had to alloc the address space @@ -97,7 +97,7 @@ where let mut local_len = SetLenOnDrop::new(&mut self.len); iterator.for_each(move |element| { ptr::write(ptr, element); - ptr = ptr.offset(1); + ptr = ptr.add(1); // Since the loop executes user code which can panic we have to bump the pointer // after each step. // NB can't overflow since we would have had to alloc the address space diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 7b246454e00918..9bcbea04dac305 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -9,7 +9,6 @@ //! using this crate. #![no_std] -#![feature(core_ffi_c)] // See . #![cfg_attr(test, allow(deref_nullptr))] #![cfg_attr(test, allow(unaligned_references))] diff --git a/rust/kernel/build_assert.rs b/rust/kernel/build_assert.rs index 659542393c0901..97098082721437 100644 --- a/rust/kernel/build_assert.rs +++ b/rust/kernel/build_assert.rs @@ -67,6 +67,7 @@ macro_rules! build_error { /// assert!(n > 1); // Run-time check /// } /// ``` +#[allow(rustdoc::broken_intra_doc_links)] #[macro_export] macro_rules! build_assert { ($cond:expr $(,)?) => {{ diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 1aac54deaae790..18d94cfa51cd34 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -15,10 +15,7 @@ #![feature(allocator_api)] #![feature(coerce_unsized)] #![feature(const_mut_refs)] -#![feature(core_ffi_c)] #![feature(dispatch_from_dyn)] -#![feature(explicit_generic_args_with_impl_trait)] -#![feature(generic_associated_types)] #![feature(new_uninit)] #![feature(pin_macro)] #![feature(receiver_trait)] diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs index 276c1c0b639012..b597abf0abba4e 100644 --- a/rust/kernel/module_param.rs +++ b/rust/kernel/module_param.rs @@ -451,6 +451,7 @@ pub enum StringParam { impl StringParam { fn bytes(&self) -> &[u8] { match self { + #[allow(clippy::explicit_auto_deref)] StringParam::Ref(bytes) => *bytes, StringParam::Owned(vec) => &vec[..], } diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index b3e68b24a8c674..2e864354cac1cc 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -137,6 +137,7 @@ /// [`std::dbg`]: https://doc.rust-lang.org/std/macro.dbg.html /// [`eprintln`]: https://doc.rust-lang.org/std/macro.eprintln.html /// [`printk`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html +#[allow(rustdoc::broken_intra_doc_links)] #[macro_export] macro_rules! dbg { // NOTE: We cannot use `concat!` to make a static string as a format argument diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index 29f69f3a52de4e..0caad902ba40a2 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -8,7 +8,6 @@ //! userspace APIs. #![no_std] -#![feature(core_ffi_c)] // See . #![cfg_attr(test, allow(deref_nullptr))] #![cfg_attr(test, allow(unaligned_references))] diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 9f94fc83f08652..f9d228a87f9a5f 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := core_ffi_c,explicit_generic_args_with_impl_trait,new_uninit,pin_macro +rust_allowed_features := const_refs_to_cell,new_uninit,pin_macro rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 20d483ec6f5fca..a69f29184c8ad8 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -27,7 +27,7 @@ llvm) fi ;; rustc) - echo 1.62.0 + echo 1.66.0 ;; bindgen) echo 0.56.0 From 283f5cdb2f5d97f021b2d38baccd1c3f1b6362fa Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 18:03:44 +0900 Subject: [PATCH 448/635] rust: Add `name` argument to Module::init() Co-developed-by: Wedson Almeida Filho Signed-off-by: Wedson Almeida Filho Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 2 +- rust/macros/module.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 18d94cfa51cd34..e0571e54c8eb31 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -86,7 +86,7 @@ pub trait Module: Sized + Sync { /// should do. /// /// Equivalent to the `module_init` macro in the C API. - fn init(module: &'static ThisModule) -> error::Result; + fn init(name: &'static crate::str::CStr, module: &'static ThisModule) -> error::Result; } /// Equivalent to `THIS_MODULE` in the C API. diff --git a/rust/macros/module.rs b/rust/macros/module.rs index fb1244f8c2e694..7678c7e041f0c3 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -270,7 +270,7 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { }} fn __init() -> core::ffi::c_int {{ - match <{type_} as kernel::Module>::init(&THIS_MODULE) {{ + match <{type_} as kernel::Module>::init(kernel::c_str!(\"{name}\"), &THIS_MODULE) {{ Ok(m) => {{ unsafe {{ __MOD = Some(m); From f97f5ce32b18311cb3dc8541c37907d9b61747f7 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 18:04:55 +0900 Subject: [PATCH 449/635] *RFL import: kernel::driver Commit reference: 3dfc5ebff103 --- rust/kernel/driver.rs | 443 ++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 3 + 2 files changed, 446 insertions(+) create mode 100644 rust/kernel/driver.rs diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs new file mode 100644 index 00000000000000..70e1697db50231 --- /dev/null +++ b/rust/kernel/driver.rs @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generic support for drivers of different buses (e.g., PCI, Platform, Amba, etc.). +//! +//! Each bus/subsystem is expected to implement [`DriverOps`], which allows drivers to register +//! using the [`Registration`] class. + +use crate::{error::code::*, error::Result, str::CStr, sync::Arc, ThisModule}; +use alloc::boxed::Box; +use core::{cell::UnsafeCell, marker::PhantomData, ops::Deref, pin::Pin}; + +/// A subsystem (e.g., PCI, Platform, Amba, etc.) that allows drivers to be written for it. +pub trait DriverOps { + /// The type that holds information about the registration. This is typically a struct defined + /// by the C portion of the kernel. + type RegType: Default; + + /// Registers a driver. + /// + /// # Safety + /// + /// `reg` must point to valid, initialised, and writable memory. It may be modified by this + /// function to hold registration state. + /// + /// On success, `reg` must remain pinned and valid until the matching call to + /// [`DriverOps::unregister`]. + unsafe fn register( + reg: *mut Self::RegType, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result; + + /// Unregisters a driver previously registered with [`DriverOps::register`]. + /// + /// # Safety + /// + /// `reg` must point to valid writable memory, initialised by a previous successful call to + /// [`DriverOps::register`]. + unsafe fn unregister(reg: *mut Self::RegType); +} + +/// The registration of a driver. +pub struct Registration { + is_registered: bool, + concrete_reg: UnsafeCell, +} + +// SAFETY: `Registration` has no fields or methods accessible via `&Registration`, so it is safe to +// share references to it with multiple threads as nothing can be done. +unsafe impl Sync for Registration {} + +impl Registration { + /// Creates a new instance of the registration object. + pub fn new() -> Self { + Self { + is_registered: false, + concrete_reg: UnsafeCell::new(T::RegType::default()), + } + } + + /// Allocates a pinned registration object and registers it. + /// + /// Returns a pinned heap-allocated representation of the registration. + pub fn new_pinned(name: &'static CStr, module: &'static ThisModule) -> Result>> { + let mut reg = Pin::from(Box::try_new(Self::new())?); + reg.as_mut().register(name, module)?; + Ok(reg) + } + + /// Registers a driver with its subsystem. + /// + /// It must be pinned because the memory block that represents the registration is potentially + /// self-referential. + pub fn register( + self: Pin<&mut Self>, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: We never move out of `this`. + let this = unsafe { self.get_unchecked_mut() }; + if this.is_registered { + // Already registered. + return Err(EINVAL); + } + + // SAFETY: `concrete_reg` was initialised via its default constructor. It is only freed + // after `Self::drop` is called, which first calls `T::unregister`. + unsafe { T::register(this.concrete_reg.get(), name, module) }?; + + this.is_registered = true; + Ok(()) + } +} + +impl Default for Registration { + fn default() -> Self { + Self::new() + } +} + +impl Drop for Registration { + fn drop(&mut self) { + if self.is_registered { + // SAFETY: This path only runs if a previous call to `T::register` completed + // successfully. + unsafe { T::unregister(self.concrete_reg.get()) }; + } + } +} + +/// Conversion from a device id to a raw device id. +/// +/// This is meant to be implemented by buses/subsystems so that they can use [`IdTable`] to +/// guarantee (at compile-time) zero-termination of device id tables provided by drivers. +/// +/// # Safety +/// +/// Implementers must ensure that: +/// - [`RawDeviceId::ZERO`] is actually a zeroed-out version of the raw device id. +/// - [`RawDeviceId::to_rawid`] stores `offset` in the context/data field of the raw device id so +/// that buses can recover the pointer to the data. +#[const_trait] +pub unsafe trait RawDeviceId { + /// The raw type that holds the device id. + /// + /// Id tables created from [`Self`] are going to hold this type in its zero-terminated array. + type RawType: Copy; + + /// A zeroed-out representation of the raw device id. + /// + /// Id tables created from [`Self`] use [`Self::ZERO`] as the sentinel to indicate the end of + /// the table. + const ZERO: Self::RawType; + + /// Converts an id into a raw id. + /// + /// `offset` is the offset from the memory location where the raw device id is stored to the + /// location where its associated context information is stored. Implementations must store + /// this in the appropriate context/data field of the raw type. + fn to_rawid(&self, offset: isize) -> Self::RawType; +} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + ids: [T::RawType; N], + sentinel: T::RawType, + id_infos: [Option; N], +} + +impl IdArray { + /// Creates a new instance of the array. + /// + /// The contents are derived from the given identifiers and context information. + pub const fn new(ids: [T; N], infos: [Option; N]) -> Self + where + T: ~const RawDeviceId + Copy, + { + let mut array = Self { + ids: [T::ZERO; N], + sentinel: T::ZERO, + id_infos: infos, + }; + let mut i = 0usize; + while i < N { + // SAFETY: Both pointers are within `array` (or one byte beyond), consequently they are + // derived from the same allocated object. We are using a `u8` pointer, whose size 1, + // so the pointers are necessarily 1-byte aligned. + let offset = unsafe { + (&array.id_infos[i] as *const _ as *const u8) + .offset_from(&array.ids[i] as *const _ as _) + }; + array.ids[i] = ids[i].to_rawid(offset); + i += 1; + } + array + } + + /// Returns an `IdTable` backed by `self`. + /// + /// This is used to essentially erase the array size. + pub const fn as_table(&self) -> IdTable<'_, T, U> { + IdTable { + first: &self.ids[0], + _p: PhantomData, + } + } +} + +/// A device id table. +/// +/// The table is guaranteed to be zero-terminated and to be followed by an array of context data of +/// type `Option`. +pub struct IdTable<'a, T: RawDeviceId, U> { + first: &'a T::RawType, + _p: PhantomData<&'a U>, +} + +impl const AsRef for IdTable<'_, T, U> { + fn as_ref(&self) -> &T::RawType { + self.first + } +} + +/// Counts the number of parenthesis-delimited, comma-separated items. +/// +/// # Examples +/// +/// ``` +/// # use kernel::count_paren_items; +/// +/// assert_eq!(0, count_paren_items!()); +/// assert_eq!(1, count_paren_items!((A))); +/// assert_eq!(1, count_paren_items!((A),)); +/// assert_eq!(2, count_paren_items!((A), (B))); +/// assert_eq!(2, count_paren_items!((A), (B),)); +/// assert_eq!(3, count_paren_items!((A), (B), (C))); +/// assert_eq!(3, count_paren_items!((A), (B), (C),)); +/// ``` +#[macro_export] +macro_rules! count_paren_items { + (($($item:tt)*), $($remaining:tt)*) => { 1 + $crate::count_paren_items!($($remaining)*) }; + (($($item:tt)*)) => { 1 }; + () => { 0 }; +} + +/// Converts a comma-separated list of pairs into an array with the first element. That is, it +/// discards the second element of the pair. +/// +/// Additionally, it automatically introduces a type if the first element is warpped in curly +/// braces, for example, if it's `{v: 10}`, it becomes `X { v: 10 }`; this is to avoid repeating +/// the type. +/// +/// # Examples +/// +/// ``` +/// # use kernel::first_item; +/// +/// #[derive(PartialEq, Debug)] +/// struct X { +/// v: u32, +/// } +/// +/// assert_eq!([] as [X; 0], first_item!(X, )); +/// assert_eq!([X { v: 10 }], first_item!(X, ({ v: 10 }, Y))); +/// assert_eq!([X { v: 10 }], first_item!(X, ({ v: 10 }, Y),)); +/// assert_eq!([X { v: 10 }], first_item!(X, (X { v: 10 }, Y))); +/// assert_eq!([X { v: 10 }], first_item!(X, (X { v: 10 }, Y),)); +/// assert_eq!([X { v: 10 }, X { v: 20 }], first_item!(X, ({ v: 10 }, Y), ({ v: 20 }, Y))); +/// assert_eq!([X { v: 10 }, X { v: 20 }], first_item!(X, ({ v: 10 }, Y), ({ v: 20 }, Y),)); +/// assert_eq!([X { v: 10 }, X { v: 20 }], first_item!(X, (X { v: 10 }, Y), (X { v: 20 }, Y))); +/// assert_eq!([X { v: 10 }, X { v: 20 }], first_item!(X, (X { v: 10 }, Y), (X { v: 20 }, Y),)); +/// assert_eq!([X { v: 10 }, X { v: 20 }, X { v: 30 }], +/// first_item!(X, ({ v: 10 }, Y), ({ v: 20 }, Y), ({v: 30}, Y))); +/// assert_eq!([X { v: 10 }, X { v: 20 }, X { v: 30 }], +/// first_item!(X, ({ v: 10 }, Y), ({ v: 20 }, Y), ({v: 30}, Y),)); +/// assert_eq!([X { v: 10 }, X { v: 20 }, X { v: 30 }], +/// first_item!(X, (X { v: 10 }, Y), (X { v: 20 }, Y), (X {v: 30}, Y))); +/// assert_eq!([X { v: 10 }, X { v: 20 }, X { v: 30 }], +/// first_item!(X, (X { v: 10 }, Y), (X { v: 20 }, Y), (X {v: 30}, Y),)); +/// ``` +#[macro_export] +macro_rules! first_item { + ($id_type:ty, $(({$($first:tt)*}, $second:expr)),* $(,)?) => { + { + type IdType = $id_type; + [$(IdType{$($first)*},)*] + } + }; + ($id_type:ty, $(($first:expr, $second:expr)),* $(,)?) => { [$($first,)*] }; +} + +/// Converts a comma-separated list of pairs into an array with the second element. That is, it +/// discards the first element of the pair. +/// +/// # Examples +/// +/// ``` +/// # use kernel::second_item; +/// +/// assert_eq!([] as [u32; 0], second_item!()); +/// assert_eq!([10u32], second_item!((X, 10u32))); +/// assert_eq!([10u32], second_item!((X, 10u32),)); +/// assert_eq!([10u32], second_item!(({ X }, 10u32))); +/// assert_eq!([10u32], second_item!(({ X }, 10u32),)); +/// assert_eq!([10u32, 20], second_item!((X, 10u32), (X, 20))); +/// assert_eq!([10u32, 20], second_item!((X, 10u32), (X, 20),)); +/// assert_eq!([10u32, 20], second_item!(({ X }, 10u32), ({ X }, 20))); +/// assert_eq!([10u32, 20], second_item!(({ X }, 10u32), ({ X }, 20),)); +/// assert_eq!([10u32, 20, 30], second_item!((X, 10u32), (X, 20), (X, 30))); +/// assert_eq!([10u32, 20, 30], second_item!((X, 10u32), (X, 20), (X, 30),)); +/// assert_eq!([10u32, 20, 30], second_item!(({ X }, 10u32), ({ X }, 20), ({ X }, 30))); +/// assert_eq!([10u32, 20, 30], second_item!(({ X }, 10u32), ({ X }, 20), ({ X }, 30),)); +/// ``` +#[macro_export] +macro_rules! second_item { + ($(({$($first:tt)*}, $second:expr)),* $(,)?) => { [$($second,)*] }; + ($(($first:expr, $second:expr)),* $(,)?) => { [$($second,)*] }; +} + +/// Defines a new constant [`IdArray`] with a concise syntax. +/// +/// It is meant to be used by buses and subsystems to create a similar macro with their device id +/// type already specified, i.e., with fewer parameters to the end user. +/// +/// # Examples +/// +// TODO: Exported but not usable by kernel modules (requires `const_trait_impl`). +/// ```ignore +/// #![feature(const_trait_impl)] +/// # use kernel::{define_id_array, driver::RawDeviceId}; +/// +/// #[derive(Copy, Clone)] +/// struct Id(u32); +/// +/// // SAFETY: `ZERO` is all zeroes and `to_rawid` stores `offset` as the second element of the raw +/// // device id pair. +/// unsafe impl const RawDeviceId for Id { +/// type RawType = (u64, isize); +/// const ZERO: Self::RawType = (0, 0); +/// fn to_rawid(&self, offset: isize) -> Self::RawType { +/// (self.0 as u64 + 1, offset) +/// } +/// } +/// +/// define_id_array!(A1, Id, (), []); +/// define_id_array!(A2, Id, &'static [u8], [(Id(10), None)]); +/// define_id_array!(A3, Id, &'static [u8], [(Id(10), Some(b"id1")), ]); +/// define_id_array!(A4, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), Some(b"id2"))]); +/// define_id_array!(A5, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), Some(b"id2")), ]); +/// define_id_array!(A6, Id, &'static [u8], [(Id(10), None), (Id(20), Some(b"id2")), ]); +/// define_id_array!(A7, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), None), ]); +/// define_id_array!(A8, Id, &'static [u8], [(Id(10), None), (Id(20), None), ]); +/// ``` +#[macro_export] +macro_rules! define_id_array { + ($table_name:ident, $id_type:ty, $data_type:ty, [ $($t:tt)* ]) => { + const $table_name: + $crate::driver::IdArray<$id_type, $data_type, { $crate::count_paren_items!($($t)*) }> = + $crate::driver::IdArray::new( + $crate::first_item!($id_type, $($t)*), $crate::second_item!($($t)*)); + }; +} + +/// Defines a new constant [`IdTable`] with a concise syntax. +/// +/// It is meant to be used by buses and subsystems to create a similar macro with their device id +/// type already specified, i.e., with fewer parameters to the end user. +/// +/// # Examples +/// +// TODO: Exported but not usable by kernel modules (requires `const_trait_impl`). +/// ```ignore +/// #![feature(const_trait_impl)] +/// # use kernel::{define_id_table, driver::RawDeviceId}; +/// +/// #[derive(Copy, Clone)] +/// struct Id(u32); +/// +/// // SAFETY: `ZERO` is all zeroes and `to_rawid` stores `offset` as the second element of the raw +/// // device id pair. +/// unsafe impl const RawDeviceId for Id { +/// type RawType = (u64, isize); +/// const ZERO: Self::RawType = (0, 0); +/// fn to_rawid(&self, offset: isize) -> Self::RawType { +/// (self.0 as u64 + 1, offset) +/// } +/// } +/// +/// define_id_table!(T1, Id, &'static [u8], [(Id(10), None)]); +/// define_id_table!(T2, Id, &'static [u8], [(Id(10), Some(b"id1")), ]); +/// define_id_table!(T3, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), Some(b"id2"))]); +/// define_id_table!(T4, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), Some(b"id2")), ]); +/// define_id_table!(T5, Id, &'static [u8], [(Id(10), None), (Id(20), Some(b"id2")), ]); +/// define_id_table!(T6, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), None), ]); +/// define_id_table!(T7, Id, &'static [u8], [(Id(10), None), (Id(20), None), ]); +/// ``` +#[macro_export] +macro_rules! define_id_table { + ($table_name:ident, $id_type:ty, $data_type:ty, [ $($t:tt)* ]) => { + const $table_name: Option<$crate::driver::IdTable<'static, $id_type, $data_type>> = { + $crate::define_id_array!(ARRAY, $id_type, $data_type, [ $($t)* ]); + Some(ARRAY.as_table()) + }; + }; +} + +/// Custom code within device removal. +pub trait DeviceRemoval { + /// Cleans resources up when the device is removed. + /// + /// This is called when a device is removed and offers implementers the chance to run some code + /// that cleans state up. + fn device_remove(&self); +} + +impl DeviceRemoval for () { + fn device_remove(&self) {} +} + +impl DeviceRemoval for Arc { + fn device_remove(&self) { + self.deref().device_remove(); + } +} + +impl DeviceRemoval for Box { + fn device_remove(&self) { + self.deref().device_remove(); + } +} + +/// A kernel module that only registers the given driver on init. +/// +/// This is a helper struct to make it easier to define single-functionality modules, in this case, +/// modules that offer a single driver. +pub struct Module { + _driver: Pin>>, +} + +impl crate::Module for Module { + fn init(name: &'static CStr, module: &'static ThisModule) -> Result { + Ok(Self { + _driver: Registration::new_pinned(name, module)?, + }) + } +} + +/// Declares a kernel module that exposes a single driver. +/// +/// It is meant to be used as a helper by other subsystems so they can more easily expose their own +/// macros. +#[macro_export] +macro_rules! module_driver { + (<$gen_type:ident>, $driver_ops:ty, { type: $type:ty, $($f:tt)* }) => { + type Ops<$gen_type> = $driver_ops; + type ModuleType = $crate::driver::Module>; + $crate::prelude::module! { + type: ModuleType, + $($f)* + } + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e0571e54c8eb31..864ec983cbc5ac 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -15,6 +15,8 @@ #![feature(allocator_api)] #![feature(coerce_unsized)] #![feature(const_mut_refs)] +#![feature(const_refs_to_cell)] +#![feature(const_trait_impl)] #![feature(dispatch_from_dyn)] #![feature(new_uninit)] #![feature(pin_macro)] @@ -35,6 +37,7 @@ mod allocator; mod build_assert; pub mod device; +pub mod driver; pub mod error; pub mod init; pub mod io_buffer; From 7f9a3511f572ef46504824c03cb8ed3cbdae851a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 18:05:48 +0900 Subject: [PATCH 450/635] *RFL import: The rest of kernel::device (minus clk stuff) Commit reference: 3dfc5ebff103 Signed-off-by: Asahi Lina --- rust/kernel/device.rs | 426 ++++++++++++++++++++++++++++++++++++++++- rust/kernel/prelude.rs | 2 + 2 files changed, 427 insertions(+), 1 deletion(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index e57da622d81736..5a7d82aca6d17d 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -4,7 +4,24 @@ //! //! C header: [`include/linux/device.h`](../../../../include/linux/device.h) -use crate::{bindings, str::CStr}; +use crate::{ + bindings, + error::Result, + macros::pin_data, + pin_init, pr_crit, + str::CStr, + sync::{lock::mutex, lock::Guard, LockClassKey, Mutex, UniqueArc}, +}; +use core::{ + fmt, + ops::{Deref, DerefMut}, + pin::Pin, +}; + +use crate::init::InPlaceInit; + +#[cfg(CONFIG_PRINTK)] +use crate::c_str; /// A raw device. /// @@ -34,6 +51,110 @@ pub unsafe trait RawDevice { // by the compiler because of their lifetimes). unsafe { CStr::from_char_ptr(name) } } + + /// Prints an emergency-level message (level 0) prefixed with device information. + /// + /// More details are available from [`dev_emerg`]. + /// + /// [`dev_emerg`]: crate::dev_emerg + fn pr_emerg(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_EMERG, args) }; + } + + /// Prints an alert-level message (level 1) prefixed with device information. + /// + /// More details are available from [`dev_alert`]. + /// + /// [`dev_alert`]: crate::dev_alert + fn pr_alert(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ALERT, args) }; + } + + /// Prints a critical-level message (level 2) prefixed with device information. + /// + /// More details are available from [`dev_crit`]. + /// + /// [`dev_crit`]: crate::dev_crit + fn pr_crit(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_CRIT, args) }; + } + + /// Prints an error-level message (level 3) prefixed with device information. + /// + /// More details are available from [`dev_err`]. + /// + /// [`dev_err`]: crate::dev_err + fn pr_err(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_ERR, args) }; + } + + /// Prints a warning-level message (level 4) prefixed with device information. + /// + /// More details are available from [`dev_warn`]. + /// + /// [`dev_warn`]: crate::dev_warn + fn pr_warn(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_WARNING, args) }; + } + + /// Prints a notice-level message (level 5) prefixed with device information. + /// + /// More details are available from [`dev_notice`]. + /// + /// [`dev_notice`]: crate::dev_notice + fn pr_notice(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_NOTICE, args) }; + } + + /// Prints an info-level message (level 6) prefixed with device information. + /// + /// More details are available from [`dev_info`]. + /// + /// [`dev_info`]: crate::dev_info + fn pr_info(&self, args: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_INFO, args) }; + } + + /// Prints a debug-level message (level 7) prefixed with device information. + /// + /// More details are available from [`dev_dbg`]. + /// + /// [`dev_dbg`]: crate::dev_dbg + fn pr_dbg(&self, args: fmt::Arguments<'_>) { + if cfg!(debug_assertions) { + // SAFETY: `klevel` is null-terminated, uses one of the kernel constants. + unsafe { self.printk(bindings::KERN_DEBUG, args) }; + } + } + + /// Prints the provided message to the console. + /// + /// # Safety + /// + /// Callers must ensure that `klevel` is null-terminated; in particular, one of the + /// `KERN_*`constants, for example, `KERN_CRIT`, `KERN_ALERT`, etc. + #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] + unsafe fn printk(&self, klevel: &[u8], msg: fmt::Arguments<'_>) { + // SAFETY: `klevel` is null-terminated and one of the kernel constants. `self.raw_device` + // is valid because `self` is valid. The "%pA" format string expects a pointer to + // `fmt::Arguments`, which is what we're passing as the last argument. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_dev_printk( + klevel as *const _ as *const core::ffi::c_char, + self.raw_device(), + c_str!("%pA").as_char_ptr(), + &msg as *const _ as *const core::ffi::c_void, + ) + }; + } } /// A ref-counted device. @@ -95,3 +216,306 @@ impl Clone for Device { Device::from_dev(self) } } + +/// Device data. +/// +/// When a device is removed (for whatever reason, for example, because the device was unplugged or +/// because the user decided to unbind the driver), the driver is given a chance to clean its state +/// up, and all io resources should ideally not be used anymore. +/// +/// However, the device data is reference-counted because other subsystems hold pointers to it. So +/// some device state must be freed and not used anymore, while others must remain accessible. +/// +/// This struct separates the device data into three categories: +/// 1. Registrations: are destroyed when the device is removed, but before the io resources +/// become inaccessible. +/// 2. Io resources: are available until the device is removed. +/// 3. General data: remain available as long as the ref count is nonzero. +/// +/// This struct implements the `DeviceRemoval` trait so that it can clean resources up even if not +/// explicitly called by the device drivers. +#[pin_data] +pub struct Data { + #[pin] + registrations: Mutex, + resources: U, + general: V, +} + +/// Safely creates an new reference-counted instance of [`Data`]. +#[doc(hidden)] +#[macro_export] +macro_rules! new_device_data { + ($reg:expr, $res:expr, $gen:expr, $name:literal) => {{ + static CLASS1: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new(); + let regs = $reg; + let res = $res; + let gen = $gen; + let name = $crate::c_str!($name); + $crate::device::Data::try_new(regs, res, gen, name, &CLASS1) + }}; +} + +impl Data { + /// Creates a new instance of `Data`. + /// + /// It is recommended that the [`new_device_data`] macro be used as it automatically creates + /// the lock classes. + pub fn try_new( + registrations: T, + resources: U, + general: V, + name: &'static CStr, + key1: &'static LockClassKey, + ) -> Result>> { + let ret = UniqueArc::pin_init(pin_init!(Self { + registrations <- Mutex::new(registrations, name, key1), + resources, + general, + }))?; + Ok(ret) + } + + /// Returns the resources if they're still available. + pub fn resources(&self) -> Option<&U> { + Some(&self.resources) + } + + /// Returns the locked registrations if they're still available. + pub fn registrations(&self) -> Option> { + Some(self.registrations.lock()) + } +} + +impl crate::driver::DeviceRemoval for Data { + fn device_remove(&self) { + pr_crit!("Device removal not properly implemented!\n"); + } +} + +impl Deref for Data { + type Target = V; + + fn deref(&self) -> &V { + &self.general + } +} + +impl DerefMut for Data { + fn deref_mut(&mut self) -> &mut V { + &mut self.general + } +} + +#[doc(hidden)] +#[macro_export] +macro_rules! dev_printk { + ($method:ident, $dev:expr, $($f:tt)*) => { + { + // We have an explicity `use` statement here so that callers of this macro are not + // required to explicitly use the `RawDevice` trait to use its functions. + use $crate::device::RawDevice; + ($dev).$method(core::format_args!($($f)*)); + } + } +} + +/// Prints an emergency-level message (level 0) prefixed with device information. +/// +/// This level should be used if the system is unusable. +/// +/// Equivalent to the kernel's `dev_emerg` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_emerg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_emerg { + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); } +} + +/// Prints an alert-level message (level 1) prefixed with device information. +/// +/// This level should be used if action must be taken immediately. +/// +/// Equivalent to the kernel's `dev_alert` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_alert!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_alert { + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); } +} + +/// Prints a critical-level message (level 2) prefixed with device information. +/// +/// This level should be used in critical conditions. +/// +/// Equivalent to the kernel's `dev_crit` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_crit!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_crit { + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); } +} + +/// Prints an error-level message (level 3) prefixed with device information. +/// +/// This level should be used in error conditions. +/// +/// Equivalent to the kernel's `dev_err` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_err!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_err { + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); } +} + +/// Prints a warning-level message (level 4) prefixed with device information. +/// +/// This level should be used in warning conditions. +/// +/// Equivalent to the kernel's `dev_warn` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_warn!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_warn { + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); } +} + +/// Prints a notice-level message (level 5) prefixed with device information. +/// +/// This level should be used in normal but significant conditions. +/// +/// Equivalent to the kernel's `dev_notice` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_notice!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_notice { + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); } +} + +/// Prints an info-level message (level 6) prefixed with device information. +/// +/// This level should be used for informational messages. +/// +/// Equivalent to the kernel's `dev_info` macro. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_info!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_info { + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); } +} + +/// Prints a debug-level message (level 7) prefixed with device information. +/// +/// This level should be used for debug messages. +/// +/// Equivalent to the kernel's `dev_dbg` macro, except that it doesn't support dynamic debug yet. +/// +/// Mimics the interface of [`std::print!`]. More information about the syntax is available from +/// [`core::fmt`] and [`alloc::format!`]. +/// +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// # use kernel::device::Device; +/// +/// fn example(dev: &Device) { +/// dev_dbg!(dev, "hello {}\n", "there"); +/// } +/// ``` +#[macro_export] +macro_rules! dev_dbg { + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); } +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index c28587d68ebc1a..e62b4d0045a9c2 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -25,6 +25,8 @@ pub use super::build_assert; // `super::std_vendor` is hidden, which makes the macro inline for some reason. #[doc(no_inline)] pub use super::dbg; +pub use super::fmt; +pub use super::{dev_alert, dev_crit, dev_dbg, dev_emerg, dev_err, dev_info, dev_notice, dev_warn}; pub use super::{pr_alert, pr_crit, pr_debug, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; pub use super::{init, pin_init, try_init, try_pin_init}; From 61f15e866529148c87d259db6a3360536f681d60 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 20:20:17 +0900 Subject: [PATCH 451/635] *RFL import: kernel::io_mem Commit reference: 3dfc5ebff103 --- rust/helpers.c | 116 ++++++++++++++++++ rust/kernel/io_mem.rs | 278 ++++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 3 files changed, 395 insertions(+) create mode 100644 rust/kernel/io_mem.rs diff --git a/rust/helpers.c b/rust/helpers.c index abe43f7ccd7aa4..24a93a9261a6a7 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -196,6 +196,122 @@ unsigned long rust_helper_clear_user(void __user *to, unsigned long n) } EXPORT_SYMBOL_GPL(rust_helper_clear_user); +void __iomem *rust_helper_ioremap(resource_size_t offset, unsigned long size) +{ + return ioremap(offset, size); +} +EXPORT_SYMBOL_GPL(rust_helper_ioremap); + +u8 rust_helper_readb(const volatile void __iomem *addr) +{ + return readb(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readb); + +u16 rust_helper_readw(const volatile void __iomem *addr) +{ + return readw(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readw); + +u32 rust_helper_readl(const volatile void __iomem *addr) +{ + return readl(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readl); + +#ifdef CONFIG_64BIT +u64 rust_helper_readq(const volatile void __iomem *addr) +{ + return readq(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readq); +#endif + +void rust_helper_writeb(u8 value, volatile void __iomem *addr) +{ + writeb(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writeb); + +void rust_helper_writew(u16 value, volatile void __iomem *addr) +{ + writew(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writew); + +void rust_helper_writel(u32 value, volatile void __iomem *addr) +{ + writel(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writel); + +#ifdef CONFIG_64BIT +void rust_helper_writeq(u64 value, volatile void __iomem *addr) +{ + writeq(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writeq); +#endif + +u8 rust_helper_readb_relaxed(const volatile void __iomem *addr) +{ + return readb_relaxed(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readb_relaxed); + +u16 rust_helper_readw_relaxed(const volatile void __iomem *addr) +{ + return readw_relaxed(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readw_relaxed); + +u32 rust_helper_readl_relaxed(const volatile void __iomem *addr) +{ + return readl_relaxed(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readl_relaxed); + +#ifdef CONFIG_64BIT +u64 rust_helper_readq_relaxed(const volatile void __iomem *addr) +{ + return readq_relaxed(addr); +} +EXPORT_SYMBOL_GPL(rust_helper_readq_relaxed); +#endif + +void rust_helper_writeb_relaxed(u8 value, volatile void __iomem *addr) +{ + writeb_relaxed(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writeb_relaxed); + +void rust_helper_writew_relaxed(u16 value, volatile void __iomem *addr) +{ + writew_relaxed(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writew_relaxed); + +void rust_helper_writel_relaxed(u32 value, volatile void __iomem *addr) +{ + writel_relaxed(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writel_relaxed); + +#ifdef CONFIG_64BIT +void rust_helper_writeq_relaxed(u64 value, volatile void __iomem *addr) +{ + writeq_relaxed(value, addr); +} +EXPORT_SYMBOL_GPL(rust_helper_writeq_relaxed); +#endif + +void rust_helper_memcpy_fromio(void *to, const volatile void __iomem *from, long count) +{ + memcpy_fromio(to, from, count); +} +EXPORT_SYMBOL_GPL(rust_helper_memcpy_fromio); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/io_mem.rs b/rust/kernel/io_mem.rs new file mode 100644 index 00000000000000..0c646ddcf7ea09 --- /dev/null +++ b/rust/kernel/io_mem.rs @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Memory-mapped IO. +//! +//! C header: [`include/asm-generic/io.h`](../../../../include/asm-generic/io.h) + +#![allow(dead_code)] + +use crate::{bindings, error::code::*, error::Result}; +use core::convert::TryInto; + +/// Represents a memory resource. +pub struct Resource { + offset: bindings::resource_size_t, + size: bindings::resource_size_t, +} + +impl Resource { + pub(crate) fn new( + start: bindings::resource_size_t, + end: bindings::resource_size_t, + ) -> Option { + if start == 0 { + return None; + } + Some(Self { + offset: start, + size: end.checked_sub(start)?.checked_add(1)?, + }) + } +} + +/// Represents a memory block of at least `SIZE` bytes. +/// +/// # Invariants +/// +/// `ptr` is a non-null and valid address of at least `SIZE` bytes and returned by an `ioremap` +/// variant. `ptr` is also 8-byte aligned. +/// +/// # Examples +/// +/// ``` +/// # use kernel::prelude::*; +/// use kernel::io_mem::{IoMem, Resource}; +/// +/// fn test(res: Resource) -> Result { +/// // Create an io mem block of at least 100 bytes. +/// // SAFETY: No DMA operations are initiated through `mem`. +/// let mem = unsafe { IoMem::<100>::try_new(res) }?; +/// +/// // Read one byte from offset 10. +/// let v = mem.readb(10); +/// +/// // Write value to offset 20. +/// mem.writeb(v, 20); +/// +/// Ok(()) +/// } +/// ``` +pub struct IoMem { + ptr: usize, +} + +macro_rules! define_read { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => { + /// Reads IO data from the given offset known, at compile time. + /// + /// If the offset is not known at compile time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, offset: usize) -> $type_name { + Self::check_offset::<$type_name>(offset); + let ptr = self.ptr.wrapping_add(offset); + // SAFETY: The type invariants guarantee that `ptr` is a valid pointer. The check above + // guarantees that the code won't build if `offset` makes the read go out of bounds + // (including the type size). + unsafe { bindings::$name(ptr as _) } + } + + /// Reads IO data from the given offset. + /// + /// It fails if/when the offset (plus the type size) is out of bounds. + $(#[$attr])* + pub fn $try_name(&self, offset: usize) -> Result<$type_name> { + if !Self::offset_ok::<$type_name>(offset) { + return Err(EINVAL); + } + let ptr = self.ptr.wrapping_add(offset); + // SAFETY: The type invariants guarantee that `ptr` is a valid pointer. The check above + // returns an error if `offset` would make the read go out of bounds (including the + // type size). + Ok(unsafe { bindings::$name(ptr as _) }) + } + }; +} + +macro_rules! define_write { + ($(#[$attr:meta])* $name:ident, $try_name:ident, $type_name:ty) => { + /// Writes IO data to the given offset, known at compile time. + /// + /// If the offset is not known at compile time, the build will fail. + $(#[$attr])* + #[inline] + pub fn $name(&self, value: $type_name, offset: usize) { + Self::check_offset::<$type_name>(offset); + let ptr = self.ptr.wrapping_add(offset); + // SAFETY: The type invariants guarantee that `ptr` is a valid pointer. The check above + // guarantees that the code won't link if `offset` makes the write go out of bounds + // (including the type size). + unsafe { bindings::$name(value, ptr as _) } + } + + /// Writes IO data to the given offset. + /// + /// It fails if/when the offset (plus the type size) is out of bounds. + $(#[$attr])* + pub fn $try_name(&self, value: $type_name, offset: usize) -> Result { + if !Self::offset_ok::<$type_name>(offset) { + return Err(EINVAL); + } + let ptr = self.ptr.wrapping_add(offset); + // SAFETY: The type invariants guarantee that `ptr` is a valid pointer. The check above + // returns an error if `offset` would make the write go out of bounds (including the + // type size). + unsafe { bindings::$name(value, ptr as _) }; + Ok(()) + } + }; +} + +impl IoMem { + /// Tries to create a new instance of a memory block. + /// + /// The resource described by `res` is mapped into the CPU's address space so that it can be + /// accessed directly. It is also consumed by this function so that it can't be mapped again + /// to a different address. + /// + /// # Safety + /// + /// Callers must ensure that either (a) the resulting interface cannot be used to initiate DMA + /// operations, or (b) that DMA operations initiated via the returned interface use DMA handles + /// allocated through the `dma` module. + pub unsafe fn try_new(res: Resource) -> Result { + // Check that the resource has at least `SIZE` bytes in it. + if res.size < SIZE.try_into()? { + return Err(EINVAL); + } + + // To be able to check pointers at compile time based only on offsets, we need to guarantee + // that the base pointer is minimally aligned. So we conservatively expect at least 8 bytes. + if res.offset % 8 != 0 { + crate::pr_err!("Physical address is not 64-bit aligned: {:x}", res.offset); + return Err(EDOM); + } + + // Try to map the resource. + // SAFETY: Just mapping the memory range. + let addr = unsafe { bindings::ioremap(res.offset, res.size as _) }; + if addr.is_null() { + Err(ENOMEM) + } else { + // INVARIANT: `addr` is non-null and was returned by `ioremap`, so it is valid. It is + // also 8-byte aligned because we checked it above. + Ok(Self { ptr: addr as usize }) + } + } + + #[inline] + const fn offset_ok(offset: usize) -> bool { + let type_size = core::mem::size_of::(); + if let Some(end) = offset.checked_add(type_size) { + end <= SIZE && offset % type_size == 0 + } else { + false + } + } + + fn offset_ok_of_val(offset: usize, value: &T) -> bool { + let value_size = core::mem::size_of_val(value); + let value_alignment = core::mem::align_of_val(value); + if let Some(end) = offset.checked_add(value_size) { + end <= SIZE && offset % value_alignment == 0 + } else { + false + } + } + + #[inline] + const fn check_offset(offset: usize) { + crate::build_assert!(Self::offset_ok::(offset), "IoMem offset overflow"); + } + + /// Copy memory block from an i/o memory by filling the specified buffer with it. + /// + /// # Examples + /// ``` + /// use kernel::io_mem::{self, IoMem, Resource}; + /// + /// fn test(res: Resource) -> Result { + /// // Create an i/o memory block of at least 100 bytes. + /// let mem = unsafe { IoMem::<100>::try_new(res) }?; + /// + /// let mut buffer: [u8; 32] = [0; 32]; + /// + /// // Memcpy 16 bytes from an offset 10 of i/o memory block into the buffer. + /// mem.try_memcpy_fromio(&mut buffer[..16], 10)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn try_memcpy_fromio(&self, buffer: &mut [u8], offset: usize) -> Result { + if !Self::offset_ok_of_val(offset, buffer) { + return Err(EINVAL); + } + + let ptr = self.ptr.wrapping_add(offset); + + // SAFETY: + // - The type invariants guarantee that `ptr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `offset_ok_of_val()`. + unsafe { + bindings::memcpy_fromio( + buffer.as_mut_ptr() as *mut _, + ptr as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + + define_read!(readb, try_readb, u8); + define_read!(readw, try_readw, u16); + define_read!(readl, try_readl, u32); + define_read!( + #[cfg(CONFIG_64BIT)] + readq, + try_readq, + u64 + ); + + define_read!(readb_relaxed, try_readb_relaxed, u8); + define_read!(readw_relaxed, try_readw_relaxed, u16); + define_read!(readl_relaxed, try_readl_relaxed, u32); + define_read!( + #[cfg(CONFIG_64BIT)] + readq_relaxed, + try_readq_relaxed, + u64 + ); + + define_write!(writeb, try_writeb, u8); + define_write!(writew, try_writew, u16); + define_write!(writel, try_writel, u32); + define_write!( + #[cfg(CONFIG_64BIT)] + writeq, + try_writeq, + u64 + ); + + define_write!(writeb_relaxed, try_writeb_relaxed, u8); + define_write!(writew_relaxed, try_writew_relaxed, u16); + define_write!(writel_relaxed, try_writel_relaxed, u32); + define_write!( + #[cfg(CONFIG_64BIT)] + writeq_relaxed, + try_writeq_relaxed, + u64 + ); +} + +impl Drop for IoMem { + fn drop(&mut self) { + // SAFETY: By the type invariant, `self.ptr` is a value returned by a previous successful + // call to `ioremap`. + unsafe { bindings::iounmap(self.ptr as _) }; + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 864ec983cbc5ac..e445a102c6359c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -41,6 +41,7 @@ pub mod driver; pub mod error; pub mod init; pub mod io_buffer; +pub mod io_mem; pub mod io_pgtable; pub mod ioctl; pub mod module_param; From 2e56b7d0c84d7a5b70f607e0fb84f816ed442250 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 20:23:17 +0900 Subject: [PATCH 452/635] *RFL import: kernel::of Commit reference: 3dfc5ebff103 --- rust/kernel/lib.rs | 1 + rust/kernel/of.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 rust/kernel/of.rs diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e445a102c6359c..7d6639ece56522 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -45,6 +45,7 @@ pub mod io_mem; pub mod io_pgtable; pub mod ioctl; pub mod module_param; +pub mod of; pub mod prelude; pub mod print; pub mod soc; diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs new file mode 100644 index 00000000000000..cdcd8324433769 --- /dev/null +++ b/rust/kernel/of.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Devicetree and Open Firmware abstractions. +//! +//! C header: [`include/linux/of_*.h`](../../../../include/linux/of_*.h) + +use crate::{bindings, driver, str::BStr}; + +/// An open firmware device id. +#[derive(Clone, Copy)] +pub enum DeviceId { + /// An open firmware device id where only a compatible string is specified. + Compatible(&'static BStr), +} + +/// Defines a const open firmware device id table that also carries per-entry data/context/info. +/// +/// The name of the const is `OF_DEVICE_ID_TABLE`, which is what buses are expected to name their +/// open firmware tables. +/// +/// # Examples +/// +/// ``` +/// # use kernel::define_of_id_table; +/// use kernel::of; +/// +/// define_of_id_table! {u32, [ +/// (of::DeviceId::Compatible(b"test-device1,test-device2"), Some(0xff)), +/// (of::DeviceId::Compatible(b"test-device3"), None), +/// ]}; +/// ``` +#[macro_export] +macro_rules! define_of_id_table { + ($data_type:ty, $($t:tt)*) => { + $crate::define_id_table!(OF_DEVICE_ID_TABLE, $crate::of::DeviceId, $data_type, $($t)*); + }; +} + +// SAFETY: `ZERO` is all zeroed-out and `to_rawid` stores `offset` in `of_device_id::data`. +unsafe impl const driver::RawDeviceId for DeviceId { + type RawType = bindings::of_device_id; + const ZERO: Self::RawType = bindings::of_device_id { + name: [0; 32], + type_: [0; 32], + compatible: [0; 128], + data: core::ptr::null(), + }; + + fn to_rawid(&self, offset: isize) -> Self::RawType { + let DeviceId::Compatible(compatible) = self; + let mut id = Self::ZERO; + let mut i = 0; + while i < compatible.len() { + // If `compatible` does not fit in `id.compatible`, an "index out of bounds" build time + // error will be triggered. + id.compatible[i] = compatible[i] as _; + i += 1; + } + id.compatible[i] = b'\0' as _; + id.data = offset as _; + id + } +} From 37c9e20324e18d0d62e8151c80835f6ce01be27a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 20:26:46 +0900 Subject: [PATCH 453/635] *RFL import: kernel::platform Commit reference: 3dfc5ebff103 --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 25 ++++ rust/kernel/lib.rs | 2 + rust/kernel/platform.rs | 225 ++++++++++++++++++++++++++++++++ 4 files changed, 253 insertions(+) create mode 100644 rust/kernel/platform.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index fe1dd60d6561d2..a6a5685c7c2e84 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 24a93a9261a6a7..b034a95ac4c0b8 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -312,6 +315,28 @@ void rust_helper_memcpy_fromio(void *to, const volatile void __iomem *from, long } EXPORT_SYMBOL_GPL(rust_helper_memcpy_fromio); +void * +rust_helper_platform_get_drvdata(const struct platform_device *pdev) +{ + return platform_get_drvdata(pdev); +} +EXPORT_SYMBOL_GPL(rust_helper_platform_get_drvdata); + +void +rust_helper_platform_set_drvdata(struct platform_device *pdev, + void *data) +{ + platform_set_drvdata(pdev, data); +} +EXPORT_SYMBOL_GPL(rust_helper_platform_set_drvdata); + +const struct of_device_id *rust_helper_of_match_device( + const struct of_device_id *matches, const struct device *dev) +{ + return of_match_device(matches, dev); +} +EXPORT_SYMBOL_GPL(rust_helper_of_match_device); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 7d6639ece56522..745db3507b4dd0 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -13,6 +13,7 @@ #![no_std] #![feature(allocator_api)] +#![feature(associated_type_defaults)] #![feature(coerce_unsized)] #![feature(const_mut_refs)] #![feature(const_refs_to_cell)] @@ -46,6 +47,7 @@ pub mod io_pgtable; pub mod ioctl; pub mod module_param; pub mod of; +pub mod platform; pub mod prelude; pub mod print; pub mod soc; diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs new file mode 100644 index 00000000000000..54c788bf1f8cd6 --- /dev/null +++ b/rust/kernel/platform.rs @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Platform devices and drivers. +//! +//! Also called `platdev`, `pdev`. +//! +//! C header: [`include/linux/platform_device.h`](../../../../include/linux/platform_device.h) + +use crate::{ + bindings, + device::{self, RawDevice}, + driver, + error::{from_kernel_result, to_result, Result}, + of, + str::CStr, + types::ForeignOwnable, + ThisModule, +}; + +/// A registration of a platform driver. +pub type Registration = driver::Registration>; + +/// An adapter for the registration of platform drivers. +pub struct Adapter(T); + +impl driver::DriverOps for Adapter { + type RegType = bindings::platform_driver; + + unsafe fn register( + reg: *mut bindings::platform_driver, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + // SAFETY: By the safety requirements of this function (defined in the trait definition), + // `reg` is non-null and valid. + let pdrv = unsafe { &mut *reg }; + + pdrv.driver.name = name.as_char_ptr(); + pdrv.probe = Some(Self::probe_callback); + pdrv.remove = Some(Self::remove_callback); + if let Some(t) = T::OF_DEVICE_ID_TABLE { + pdrv.driver.of_match_table = t.as_ref(); + } + // SAFETY: + // - `pdrv` lives at least until the call to `platform_driver_unregister()` returns. + // - `name` pointer has static lifetime. + // - `module.0` lives at least as long as the module. + // - `probe()` and `remove()` are static functions. + // - `of_match_table` is either a raw pointer with static lifetime, + // as guaranteed by the [`driver::IdTable`] type, or null. + to_result(unsafe { bindings::__platform_driver_register(reg, module.0) }) + } + + unsafe fn unregister(reg: *mut bindings::platform_driver) { + // SAFETY: By the safety requirements of this function (defined in the trait definition), + // `reg` was passed (and updated) by a previous successful call to + // `platform_driver_register`. + unsafe { bindings::platform_driver_unregister(reg) }; + } +} + +impl Adapter { + fn get_id_info(dev: &Device) -> Option<&'static T::IdInfo> { + let table = T::OF_DEVICE_ID_TABLE?; + + // SAFETY: `table` has static lifetime, so it is valid for read. `dev` is guaranteed to be + // valid while it's alive, so is the raw device returned by it. + let id = unsafe { bindings::of_match_device(table.as_ref(), dev.raw_device()) }; + if id.is_null() { + return None; + } + + // SAFETY: `id` is a pointer within the static table, so it's always valid. + let offset = unsafe { (*id).data }; + if offset.is_null() { + return None; + } + + // SAFETY: The offset comes from a previous call to `offset_from` in `IdArray::new`, which + // guarantees that the resulting pointer is within the table. + let ptr = unsafe { + id.cast::() + .offset(offset as _) + .cast::>() + }; + + // SAFETY: The id table has a static lifetime, so `ptr` is guaranteed to be valid for read. + #[allow(clippy::needless_borrow)] + unsafe { + (&*ptr).as_ref() + } + } + + extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: `pdev` is valid by the contract with the C code. `dev` is alive only for the + // duration of this call, so it is guaranteed to remain alive for the lifetime of + // `pdev`. + let mut dev = unsafe { Device::from_ptr(pdev) }; + let info = Self::get_id_info(&dev); + let data = T::probe(&mut dev, info)?; + // SAFETY: `pdev` is guaranteed to be a valid, non-null pointer. + unsafe { bindings::platform_set_drvdata(pdev, data.into_foreign() as _) }; + Ok(0) + }) + } + + extern "C" fn remove_callback(pdev: *mut bindings::platform_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: `pdev` is guaranteed to be a valid, non-null pointer. + let ptr = unsafe { bindings::platform_get_drvdata(pdev) }; + // SAFETY: + // - we allocated this pointer using `T::Data::into_foreign`, + // so it is safe to turn back into a `T::Data`. + // - the allocation happened in `probe`, no-one freed the memory, + // `remove` is the canonical kernel location to free driver data. so OK + // to convert the pointer back to a Rust structure here. + let data = unsafe { T::Data::from_foreign(ptr) }; + let ret = T::remove(&data); + ::device_remove(&data); + ret?; + Ok(0) + }) + } +} + +/// A platform driver. +pub trait Driver { + /// Data stored on device by driver. + /// + /// Corresponds to the data set or retrieved via the kernel's + /// `platform_{set,get}_drvdata()` functions. + /// + /// Require that `Data` implements `ForeignOwnable`. We guarantee to + /// never move the underlying wrapped data structure. This allows + type Data: ForeignOwnable + Send + Sync + driver::DeviceRemoval = (); + + /// The type holding information about each device id supported by the driver. + type IdInfo: 'static = (); + + /// The table of device ids supported by the driver. + const OF_DEVICE_ID_TABLE: Option> = None; + + /// Platform driver probe. + /// + /// Called when a new platform device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe(dev: &mut Device, id_info: Option<&Self::IdInfo>) -> Result; + + /// Platform driver remove. + /// + /// Called when a platform device is removed. + /// Implementers should prepare the device for complete removal here. + fn remove(_data: &Self::Data) -> Result { + Ok(()) + } +} + +/// A platform device. +/// +/// # Invariants +/// +/// The field `ptr` is non-null and valid for the lifetime of the object. +pub struct Device { + ptr: *mut bindings::platform_device, +} + +impl Device { + /// Creates a new device from the given pointer. + /// + /// # Safety + /// + /// `ptr` must be non-null and valid. It must remain valid for the lifetime of the returned + /// instance. + unsafe fn from_ptr(ptr: *mut bindings::platform_device) -> Self { + // INVARIANT: The safety requirements of the function ensure the lifetime invariant. + Self { ptr } + } + + /// Returns id of the platform device. + pub fn id(&self) -> i32 { + // SAFETY: By the type invariants, we know that `self.ptr` is non-null and valid. + unsafe { (*self.ptr).id } + } +} + +// SAFETY: The device returned by `raw_device` is the raw platform device. +unsafe impl device::RawDevice for Device { + fn raw_device(&self) -> *mut bindings::device { + // SAFETY: By the type invariants, we know that `self.ptr` is non-null and valid. + unsafe { &mut (*self.ptr).dev } + } +} + +/// Declares a kernel module that exposes a single platform driver. +/// +/// # Examples +/// +/// ```ignore +/// # use kernel::{platform, define_of_id_table, module_platform_driver}; +/// # +/// struct MyDriver; +/// impl platform::Driver for MyDriver { +/// // [...] +/// # fn probe(_dev: &mut platform::Device, _id_info: Option<&Self::IdInfo>) -> Result { +/// # Ok(()) +/// # } +/// # define_of_id_table! {(), [ +/// # (of::DeviceId::Compatible(b"brcm,bcm2835-rng"), None), +/// # ]} +/// } +/// +/// module_platform_driver! { +/// type: MyDriver, +/// name: "module_name", +/// author: "Author name", +/// license: "GPL", +/// } +/// ``` +#[macro_export] +macro_rules! module_platform_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::platform::Adapter, { $($f)* }); + }; +} From fcb2f51835dccaf2b4e1ec6feff017bef99a1141 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 20:29:27 +0900 Subject: [PATCH 454/635] *RFL import: kernel::delay Commit reference: 3dfc5ebff103 --- rust/bindings/bindings_helper.h | 1 + rust/kernel/delay.rs | 104 ++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 3 files changed, 107 insertions(+) create mode 100644 rust/kernel/delay.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index a6a5685c7c2e84..7aaa595794cb0d 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -6,6 +6,7 @@ * Sorted alphabetically. */ +#include #include #include #include diff --git a/rust/kernel/delay.rs b/rust/kernel/delay.rs new file mode 100644 index 00000000000000..1e987fa659419b --- /dev/null +++ b/rust/kernel/delay.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Delay functions for operations like sleeping. +//! +//! C header: [`include/linux/delay.h`](../../../../include/linux/delay.h) + +use crate::bindings; +use core::{cmp::min, time::Duration}; + +const MILLIS_PER_SEC: u64 = 1_000; + +fn coarse_sleep_conversion(duration: Duration) -> core::ffi::c_uint { + let milli_as_nanos = Duration::MILLISECOND.subsec_nanos(); + + // Rounds the nanosecond component of `duration` up to the nearest millisecond. + let nanos_as_millis = duration.subsec_nanos().wrapping_add(milli_as_nanos - 1) / milli_as_nanos; + + // Saturates the second component of `duration` to `c_uint::MAX`. + let seconds_as_millis = min( + duration.as_secs().saturating_mul(MILLIS_PER_SEC), + u64::from(core::ffi::c_uint::MAX), + ) as core::ffi::c_uint; + + seconds_as_millis.saturating_add(nanos_as_millis) +} + +/// Sleeps safely even with waitqueue interruptions. +/// +/// This function forwards the call to the C side `msleep` function. As a result, +/// `duration` will be rounded up to the nearest millisecond if granularity less +/// than a millisecond is provided. Any [`Duration`] that exceeds +/// [`c_uint::MAX`][core::ffi::c_uint::MAX] in milliseconds is saturated. +/// +/// # Examples +/// +// Keep these in sync with `test_coarse_sleep_examples`. +/// ``` +/// # use core::time::Duration; +/// # use kernel::delay::coarse_sleep; +/// coarse_sleep(Duration::ZERO); // Equivalent to `msleep(0)`. +/// coarse_sleep(Duration::from_nanos(1)); // Equivalent to `msleep(1)`. +/// +/// coarse_sleep(Duration::from_nanos(1_000_000)); // Equivalent to `msleep(1)`. +/// coarse_sleep(Duration::from_nanos(1_000_001)); // Equivalent to `msleep(2)`. +/// coarse_sleep(Duration::from_nanos(1_999_999)); // Equivalent to `msleep(2)`. +/// +/// coarse_sleep(Duration::from_millis(1)); // Equivalent to `msleep(1)`. +/// coarse_sleep(Duration::from_millis(2)); // Equivalent to `msleep(2)`. +/// +/// coarse_sleep(Duration::from_secs(1)); // Equivalent to `msleep(1000)`. +/// coarse_sleep(Duration::new(1, 1)); // Equivalent to `msleep(1001)`. +/// coarse_sleep(Duration::new(1, 2)); // Equivalent to `msleep(1001)`. +/// ``` +pub fn coarse_sleep(duration: Duration) { + // SAFETY: `msleep` is safe for all values of its argument. + unsafe { bindings::msleep(coarse_sleep_conversion(duration)) } +} + +#[cfg(test)] +mod tests { + use super::{coarse_sleep_conversion, MILLIS_PER_SEC}; + use core::time::Duration; + + #[test] + fn test_coarse_sleep_examples() { + // Keep these in sync with `coarse_sleep`'s `# Examples` section. + + assert_eq!(coarse_sleep_conversion(Duration::ZERO), 0); + assert_eq!(coarse_sleep_conversion(Duration::from_nanos(1)), 1); + + assert_eq!(coarse_sleep_conversion(Duration::from_nanos(1_000_000)), 1); + assert_eq!(coarse_sleep_conversion(Duration::from_nanos(1_000_001)), 2); + assert_eq!(coarse_sleep_conversion(Duration::from_nanos(1_999_999)), 2); + + assert_eq!(coarse_sleep_conversion(Duration::from_millis(1)), 1); + assert_eq!(coarse_sleep_conversion(Duration::from_millis(2)), 2); + + assert_eq!(coarse_sleep_conversion(Duration::from_secs(1)), 1000); + assert_eq!(coarse_sleep_conversion(Duration::new(1, 1)), 1001); + assert_eq!(coarse_sleep_conversion(Duration::new(1, 2)), 1001); + } + + #[test] + fn test_coarse_sleep_saturation() { + assert!( + coarse_sleep_conversion(Duration::new( + core::ffi::c_uint::MAX as u64 / MILLIS_PER_SEC, + 0 + )) < core::ffi::c_uint::MAX + ); + assert_eq!( + coarse_sleep_conversion(Duration::new( + core::ffi::c_uint::MAX as u64 / MILLIS_PER_SEC, + 999_999_999 + )), + core::ffi::c_uint::MAX + ); + + assert_eq!( + coarse_sleep_conversion(Duration::MAX), + core::ffi::c_uint::MAX + ); + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 745db3507b4dd0..b78e8b96edefb5 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -19,6 +19,7 @@ #![feature(const_refs_to_cell)] #![feature(const_trait_impl)] #![feature(dispatch_from_dyn)] +#![feature(duration_constants)] #![feature(new_uninit)] #![feature(pin_macro)] #![feature(receiver_trait)] @@ -37,6 +38,7 @@ extern crate self as kernel; mod allocator; mod build_assert; +pub mod delay; pub mod device; pub mod driver; pub mod error; From 9e496b356ee8e25f9bee9258491aa6ae3a4f1ddf Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 22 Oct 2022 00:10:30 +0900 Subject: [PATCH 455/635] rust: of: Add OF node abstraction This abstraction enables Rust drivers to walk Device Tree nodes and query their properties. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 3 + rust/helpers.c | 14 + rust/kernel/device.rs | 9 +- rust/kernel/of.rs | 461 +++++++++++++++++++++++++++++++- 4 files changed, 485 insertions(+), 2 deletions(-) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 7aaa595794cb0d..26b7666ab603d6 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -10,6 +10,9 @@ #include #include #include +#include +#include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index b034a95ac4c0b8..1dbf00cebccc63 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -337,6 +337,20 @@ const struct of_device_id *rust_helper_of_match_device( } EXPORT_SYMBOL_GPL(rust_helper_of_match_device); +bool rust_helper_of_node_is_root(const struct device_node *np) +{ + return of_node_is_root(np); +} +EXPORT_SYMBOL_GPL(rust_helper_of_node_is_root); + +struct device_node *rust_helper_of_parse_phandle(const struct device_node *np, + const char *phandle_name, + int index) +{ + return of_parse_phandle(np, phandle_name, index); +} +EXPORT_SYMBOL_GPL(rust_helper_of_parse_phandle); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 5a7d82aca6d17d..14c551df0a41b2 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -8,7 +8,7 @@ use crate::{ bindings, error::Result, macros::pin_data, - pin_init, pr_crit, + of, pin_init, pr_crit, str::CStr, sync::{lock::mutex, lock::Guard, LockClassKey, Mutex, UniqueArc}, }; @@ -52,6 +52,13 @@ pub unsafe trait RawDevice { unsafe { CStr::from_char_ptr(name) } } + /// Gets the OpenFirmware node attached to this device + fn of_node(&self) -> Option { + let ptr = self.raw_device(); + + unsafe { of::Node::get_from_raw((*ptr).of_node) } + } + /// Prints an emergency-level message (level 0) prefixed with device information. /// /// More details are available from [`dev_emerg`]. diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index cdcd8324433769..615b2183a085c5 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -4,7 +4,19 @@ //! //! C header: [`include/linux/of_*.h`](../../../../include/linux/of_*.h) -use crate::{bindings, driver, str::BStr}; +// Note: Most OF functions turn into inline dummies with CONFIG_OF(_*) disabled. +// We have to either add config conditionals to helpers.c or here; let's do it +// here for now. In the future, once bindgen can auto-generate static inline +// helpers, this can go away if desired. + +use core::marker::PhantomData; +use core::num::NonZeroU32; + +use crate::{ + bindings, driver, + prelude::*, + str::{BStr, CStr}, +}; /// An open firmware device id. #[derive(Clone, Copy)] @@ -61,3 +73,450 @@ unsafe impl const driver::RawDeviceId for DeviceId { id } } + +/// Type alias for an OF phandle +pub type PHandle = bindings::phandle; + +/// An OF device tree node. +/// +/// # Invariants +/// +/// `raw_node` points to a valid OF node, and we hold a reference to it. +pub struct Node { + raw_node: *mut bindings::device_node, +} + +#[allow(dead_code)] +impl Node { + /// Creates a `Node` from a raw C pointer. The pointer must be owned (the caller + /// gives up its reference). If the pointer is NULL, returns None. + pub(crate) unsafe fn from_raw(raw_node: *mut bindings::device_node) -> Option { + if raw_node.is_null() { + None + } else { + // INVARIANT: `raw_node` is valid per the above contract, and non-null per the + // above check. + Some(Node { raw_node }) + } + } + + /// Creates a `Node` from a raw C pointer. The pointer must be borrowed (the caller + /// retains its reference, which must be valid for the duration of the call). If the + /// pointer is NULL, returns None. + pub(crate) unsafe fn get_from_raw(raw_node: *mut bindings::device_node) -> Option { + // SAFETY: `raw_node` is valid or NULL per the above contract. `of_node_get` can handle + // NULL. + unsafe { + #[cfg(CONFIG_OF_DYNAMIC)] + bindings::of_node_get(raw_node); + Node::from_raw(raw_node) + } + } + + /// Returns a reference to the underlying C `device_node` structure. + fn node(&self) -> &bindings::device_node { + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { &*self.raw_node } + } + + /// Returns the name of the node. + pub fn name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().name) } + } + + /// Returns the phandle for this node. + pub fn phandle(&self) -> PHandle { + self.node().phandle + } + + /// Returns the full name (with address) for this node. + pub fn full_name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().full_name) } + } + + /// Returns `true` if the node is the root node. + pub fn is_root(&self) -> bool { + unsafe { bindings::of_node_is_root(self.raw_node) } + } + + /// Returns the parent node, if any. + pub fn parent(&self) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and `of_get_parent()` takes a + // new reference to the parent (or returns NULL). + unsafe { + Node::from_raw(bindings::of_get_parent(self.raw_node)) + } + } + + /// Returns an iterator over the node's children. + // TODO: use type alias for return type once type_alias_impl_trait is stable + pub fn children( + &self, + ) -> NodeIterator<'_, impl Fn(*mut bindings::device_node) -> *mut bindings::device_node + '_> + { + #[cfg(not(CONFIG_OF))] + { + NodeIterator::new(|_prev| core::ptr::null_mut()) + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and the lifetime of the `NodeIterator` + // does not exceed the lifetime of the `Node` so it can borrow its reference. + NodeIterator::new(|prev| unsafe { bindings::of_get_next_child(self.raw_node, prev) }) + } + + /// Find a child by its name and return it, or None if not found. + #[allow(unused_variables)] + pub fn get_child_by_name(&self, name: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + Node::from_raw(bindings::of_get_child_by_name( + self.raw_node, + name.as_char_ptr(), + )) + } + } + + /// Checks whether the node is compatible with the given compatible string. + /// + /// Returns `None` if there is no match, or `Some` if there is, with the value + /// representing as match score (higher values for more specific compatible matches). + #[allow(unused_variables)] + pub fn is_compatible(&self, compatible: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + let ret = 0; + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. + let ret = + unsafe { bindings::of_device_is_compatible(self.raw_node, compatible.as_char_ptr()) }; + + NonZeroU32::new(ret.try_into().ok()?) + } + + /// Parse a phandle property and return the Node referenced at a given index, if any. + /// + /// Used only for phandle properties with no arguments. + #[allow(unused_variables)] + pub fn parse_phandle(&self, name: &CStr, index: usize) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. `of_parse_phandle` returns an + // owned reference. + unsafe { + Node::from_raw(bindings::of_parse_phandle( + self.raw_node, + name.as_char_ptr(), + index.try_into().ok()?, + )) + } + } + + #[allow(unused_variables)] + /// Look up a node property by name, returning a `Property` object if found. + pub fn find_property(&self, propname: &CStr) -> Option> { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. The property structure + // returned borrows the reference to the owning node, and so has the same + // lifetime. + unsafe { + Property::from_raw(bindings::of_find_property( + self.raw_node, + propname.as_char_ptr(), + core::ptr::null_mut(), + )) + } + } + + /// Look up a mandatory node property by name, and decode it into a value type. + /// + /// Returns `Err(ENOENT)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_property<'a, T: TryFrom>>(&'a self, propname: &CStr) -> Result + where + crate::error::Error: From<>>::Error>, + { + Ok(self.find_property(propname).ok_or(ENOENT)?.try_into()?) + } + + /// Look up an optional node property by name, and decode it into a value type. + /// + /// Returns `Ok(None)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_opt_property<'a, T: TryFrom>>( + &'a self, + propname: &CStr, + ) -> Result> + where + crate::error::Error: From<>>::Error>, + { + self.find_property(propname) + .map_or(Ok(None), |p| Ok(Some(p.try_into()?))) + } +} + +/// A property attached to a device tree `Node`. +/// +/// # Invariants +/// +/// `raw` must be valid and point to a property that outlives the lifetime of this object. +#[derive(Copy, Clone)] +pub struct Property<'a> { + raw: *mut bindings::property, + _p: PhantomData<&'a Node>, +} + +impl<'a> Property<'a> { + #[cfg(CONFIG_OF)] + /// Create a `Property` object from a raw C pointer. Returns `None` if NULL. + /// + /// The passed pointer must be valid and outlive the lifetime argument, or NULL. + unsafe fn from_raw(raw: *mut bindings::property) -> Option> { + if raw.is_null() { + None + } else { + Some(Property { + raw, + _p: PhantomData, + }) + } + } + + /// Returns the name of the property as a `CStr`. + pub fn name(&self) -> &CStr { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the `CStr` does not + // outlive it. + unsafe { CStr::from_char_ptr((*self.raw).name) } + } + + /// Returns the name of the property as a `&[u8]`. + pub fn value(&self) -> &[u8] { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the slice does not + // outlive it. + unsafe { core::slice::from_raw_parts((*self.raw).value as *const u8, self.len()) } + } + + /// Returns the length of the property in bytes. + pub fn len(&self) -> usize { + // SAFETY: `raw` is valid per the type invariant. + unsafe { (*self.raw).length.try_into().unwrap() } + } + + /// Returns true if the property is empty (zero-length), which typically represents boolean true. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// A trait that represents a value decodable from a property with a fixed unit size. +/// +/// This allows us to auto-derive property decode implementations for `Vec`. +pub trait PropertyUnit: Sized { + /// The size in bytes of a single data unit. + const UNIT_SIZE: usize; + + /// Decode this data unit from a byte slice. The passed slice will have a length of `UNIT_SIZE`. + fn from_bytes(data: &[u8]) -> Result; +} + +// This doesn't work... +// impl<'a, T: PropertyUnit> TryFrom> for T { +// type Error = Error; +// +// fn try_from(p: Property<'_>) -> core::result::Result { +// if p.value().len() != T::UNIT_SIZE { +// Err(EINVAL) +// } else { +// Ok(T::from_bytes(p.value())?) +// } +// } +// } + +impl<'a, T: PropertyUnit> TryFrom> for Vec { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result, Self::Error> { + if p.len() % T::UNIT_SIZE != 0 { + return Err(EINVAL); + } + + let mut v = Vec::new(); + let val = p.value(); + for off in (0..p.len()).step_by(T::UNIT_SIZE) { + v.try_push(T::from_bytes(&val[off..off + T::UNIT_SIZE])?)?; + } + Ok(v) + } +} + +macro_rules! prop_int_type ( + ($type:ty) => { + impl<'a> TryFrom> for $type { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result<$type, Self::Error> { + Ok(<$type>::from_be_bytes(p.value().try_into().or(Err(EINVAL))?)) + } + } + + impl PropertyUnit for $type { + const UNIT_SIZE: usize = <$type>::BITS as usize / 8; + + fn from_bytes(data: &[u8]) -> Result { + Ok(<$type>::from_be_bytes(data.try_into().or(Err(EINVAL))?)) + } + } + } +); + +prop_int_type!(u8); +prop_int_type!(u16); +prop_int_type!(u32); +prop_int_type!(u64); +prop_int_type!(i8); +prop_int_type!(i16); +prop_int_type!(i32); +prop_int_type!(i64); + +/// An iterator across a collection of Node objects. +/// +/// # Invariants +/// +/// `cur` must be NULL or a valid node owned reference. If NULL, it represents either the first +/// or last position of the iterator. +/// +/// If `done` is true, `cur` must be NULL. +/// +/// fn_next must be a callback that iterates from one node to the next, and it must not capture +/// values that exceed the lifetime of the iterator. It must return owned references and also +/// take owned references. +pub struct NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + cur: *mut bindings::device_node, + done: bool, + fn_next: T, + _p: PhantomData<&'a T>, +} + +impl<'a, T> NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn new(next: T) -> NodeIterator<'a, T> { + // INVARIANT: `cur` is initialized to NULL to represent the initial state. + NodeIterator { + cur: core::ptr::null_mut(), + done: false, + fn_next: next, + _p: PhantomData, + } + } +} + +impl<'a, T> Iterator for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + type Item = Node; + + fn next(&mut self) -> Option { + if self.done { + None + } else { + // INVARIANT: if the new `cur` is NULL, then the iterator has reached its end and we + // set `done` to `true`. + self.cur = (self.fn_next)(self.cur); + self.done = self.cur.is_null(); + // SAFETY: `fn_next` must return an owned reference per the iterator contract. + // The iterator itself is considered to own this reference, so we take another one. + unsafe { Node::get_from_raw(self.cur) } + } + } +} + +// Drop impl to ensure we drop the current node being iterated on, if any. +impl<'a, T> Drop for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn drop(&mut self) { + // SAFETY: `cur` is valid or NULL, and `of_node_put()` can handle NULL. + #[cfg(CONFIG_OF_DYNAMIC)] + unsafe { + bindings::of_node_put(self.cur) + }; + } +} + +/// Returns the root node of the OF device tree (if any). +pub fn root() -> Option { + unsafe { Node::get_from_raw(bindings::of_root) } +} + +/// Returns the /chosen node of the OF device tree (if any). +pub fn chosen() -> Option { + unsafe { Node::get_from_raw(bindings::of_chosen) } +} + +/// Returns the /aliases node of the OF device tree (if any). +pub fn aliases() -> Option { + unsafe { Node::get_from_raw(bindings::of_aliases) } +} + +/// Returns the system stdout node of the OF device tree (if any). +pub fn stdout() -> Option { + unsafe { Node::get_from_raw(bindings::of_stdout) } +} + +#[allow(unused_variables)] +/// Looks up a node in the device tree by phandle. +pub fn find_node_by_phandle(handle: PHandle) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + unsafe { + #[allow(dead_code)] + Node::from_raw(bindings::of_find_node_by_phandle(handle)) + } +} + +impl Clone for Node { + fn clone(&self) -> Node { + // SAFETY: `raw_node` is valid and non-NULL per the type invariant, + // so this can never return None. + unsafe { Node::get_from_raw(self.raw_node).unwrap() } + } +} + +impl Drop for Node { + fn drop(&mut self) { + #[cfg(CONFIG_OF_DYNAMIC)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + bindings::of_node_put(self.raw_node) + }; + } +} From 8bd9c2384bee046ad3276fcd24cc6d9e0cdbf88e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 8 Dec 2022 15:57:50 +0900 Subject: [PATCH 456/635] rust: driver,of: Support passing ID tables to modpost for alias generation In order for modpost to work and correctly generate module aliases from device ID tables, it needs those tables to exist as global symbols with a specific name. Additionally, modpost checks the size of the symbol, so it cannot contain trailing data. To support this, split IdArrayIds out of IdArray. The former contains just the IDs. Then split out the device table definition macro from the macro that defines the device table for a given bus driver, and add another macro to declare a device table as a module device table. Drivers can now define their ID table once, and then specify that it should be used for both the driver and the module: // Generic OF Device ID table. kernel::define_of_id_table! {ASAHI_ID_TABLE, &'static hw::HwConfig, [ (of::DeviceId::Compatible(b"apple,agx-t8103"), Some(&hw::t8103::HWCONFIG)), (of::DeviceId::Compatible(b"apple,agx-t8112"), Some(&hw::t8112::HWCONFIG)), // ... ]} /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiDriver { /// Data associated with each hardware ID. type IdInfo = &'static hw::HwConfig; // Assign the above OF ID table to this driver. kernel::driver_of_id_table!(ASAHI_ID_TABLE); // ... } // Export the OF ID table as a module ID table, to make modpost/autoloading work. kernel::module_of_id_table!(MOD_TABLE, ASAHI_ID_TABLE); Signed-off-by: Asahi Lina --- rust/kernel/driver.rs | 98 ++++++++++++++++++++++++++++--------------- rust/kernel/of.rs | 40 ++++++++++++++---- 2 files changed, 97 insertions(+), 41 deletions(-) diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index 70e1697db50231..aa1441ae809b73 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -140,11 +140,20 @@ pub unsafe trait RawDeviceId { fn to_rawid(&self, offset: isize) -> Self::RawType; } -/// A zero-terminated device id array, followed by context data. +/// A zero-terminated device id array. +#[derive(Copy, Clone)] #[repr(C)] -pub struct IdArray { +pub struct IdArrayIds { ids: [T::RawType; N], sentinel: T::RawType, +} + +unsafe impl Sync for IdArrayIds {} + +/// A zero-terminated device id array, followed by context data. +#[repr(C)] +pub struct IdArray { + ids: IdArrayIds, id_infos: [Option; N], } @@ -155,10 +164,13 @@ impl IdArray { pub const fn new(ids: [T; N], infos: [Option; N]) -> Self where T: ~const RawDeviceId + Copy, + T::RawType: Copy + Clone, { let mut array = Self { - ids: [T::ZERO; N], - sentinel: T::ZERO, + ids: IdArrayIds { + ids: [T::ZERO; N], + sentinel: T::ZERO, + }, id_infos: infos, }; let mut i = 0usize; @@ -168,9 +180,9 @@ impl IdArray { // so the pointers are necessarily 1-byte aligned. let offset = unsafe { (&array.id_infos[i] as *const _ as *const u8) - .offset_from(&array.ids[i] as *const _ as _) + .offset_from(&array.ids.ids[i] as *const _ as _) }; - array.ids[i] = ids[i].to_rawid(offset); + array.ids.ids[i] = ids[i].to_rawid(offset); i += 1; } array @@ -181,10 +193,23 @@ impl IdArray { /// This is used to essentially erase the array size. pub const fn as_table(&self) -> IdTable<'_, T, U> { IdTable { - first: &self.ids[0], + first: &self.ids.ids[0], _p: PhantomData, } } + + /// Returns the number of items in the ID table. + pub const fn count(&self) -> usize { + self.ids.ids.len() + } + + /// Returns the inner IdArrayIds array, without the context data. + pub const fn as_ids(&self) -> IdArrayIds + where + T: ~const RawDeviceId + Copy, + { + self.ids + } } /// A device id table. @@ -331,6 +356,11 @@ macro_rules! second_item { /// define_id_array!(A6, Id, &'static [u8], [(Id(10), None), (Id(20), Some(b"id2")), ]); /// define_id_array!(A7, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), None), ]); /// define_id_array!(A8, Id, &'static [u8], [(Id(10), None), (Id(20), None), ]); +/// +/// // Within a bus driver: +/// driver_id_table!(BUS_ID_TABLE, Id, &'static [u8], A1); +/// // At the top level: +/// module_id_table!(MODULE_ID_TABLE, "mybus", Id, A1); /// ``` #[macro_export] macro_rules! define_id_array { @@ -342,7 +372,7 @@ macro_rules! define_id_array { }; } -/// Defines a new constant [`IdTable`] with a concise syntax. +/// Declares an [`IdArray`] as an [`IdTable`] for a bus driver with a concise syntax. /// /// It is meant to be used by buses and subsystems to create a similar macro with their device id /// type already specified, i.e., with fewer parameters to the end user. @@ -352,36 +382,38 @@ macro_rules! define_id_array { // TODO: Exported but not usable by kernel modules (requires `const_trait_impl`). /// ```ignore /// #![feature(const_trait_impl)] -/// # use kernel::{define_id_table, driver::RawDeviceId}; +/// # use kernel::{driver_id_table}; + +/// driver_id_table!(BUS_ID_TABLE, Id, &'static [u8], MY_ID_ARRAY); +/// ``` +#[macro_export] +macro_rules! driver_id_table { + ($table_name:ident, $id_type:ty, $data_type:ty, $target:expr) => { + const $table_name: Option<$crate::driver::IdTable<'static, $id_type, $data_type>> = + Some($target.as_table()); + }; +} + +/// Declares an [`IdArray`] as a module-level ID tablewith a concise syntax. /// -/// #[derive(Copy, Clone)] -/// struct Id(u32); +/// It is meant to be used by buses and subsystems to create a similar macro with their device id +/// type already specified, i.e., with fewer parameters to the end user. /// -/// // SAFETY: `ZERO` is all zeroes and `to_rawid` stores `offset` as the second element of the raw -/// // device id pair. -/// unsafe impl const RawDeviceId for Id { -/// type RawType = (u64, isize); -/// const ZERO: Self::RawType = (0, 0); -/// fn to_rawid(&self, offset: isize) -> Self::RawType { -/// (self.0 as u64 + 1, offset) -/// } -/// } +/// # Examples /// -/// define_id_table!(T1, Id, &'static [u8], [(Id(10), None)]); -/// define_id_table!(T2, Id, &'static [u8], [(Id(10), Some(b"id1")), ]); -/// define_id_table!(T3, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), Some(b"id2"))]); -/// define_id_table!(T4, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), Some(b"id2")), ]); -/// define_id_table!(T5, Id, &'static [u8], [(Id(10), None), (Id(20), Some(b"id2")), ]); -/// define_id_table!(T6, Id, &'static [u8], [(Id(10), Some(b"id1")), (Id(20), None), ]); -/// define_id_table!(T7, Id, &'static [u8], [(Id(10), None), (Id(20), None), ]); +// TODO: Exported but not usable by kernel modules (requires `const_trait_impl`). +/// ```ignore +/// #![feature(const_trait_impl)] +/// # use kernel::{driver_id_table}; + +/// driver_id_table!(BUS_ID_TABLE, Id, &'static [u8], MY_ID_ARRAY); /// ``` #[macro_export] -macro_rules! define_id_table { - ($table_name:ident, $id_type:ty, $data_type:ty, [ $($t:tt)* ]) => { - const $table_name: Option<$crate::driver::IdTable<'static, $id_type, $data_type>> = { - $crate::define_id_array!(ARRAY, $id_type, $data_type, [ $($t)* ]); - Some(ARRAY.as_table()) - }; +macro_rules! module_id_table { + ($item_name:ident, $table_type:literal, $id_type:ty, $table_name:ident) => { + #[export_name = concat!("__mod_", $table_type, "__", stringify!($table_name), "_device_table")] + static $item_name: $crate::driver::IdArrayIds<$id_type, { $table_name.count() }> = + $table_name.as_ids(); }; } diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 615b2183a085c5..a27621b57fbbc3 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -27,24 +27,48 @@ pub enum DeviceId { /// Defines a const open firmware device id table that also carries per-entry data/context/info. /// -/// The name of the const is `OF_DEVICE_ID_TABLE`, which is what buses are expected to name their -/// open firmware tables. -/// -/// # Examples +/// # Example /// /// ``` -/// # use kernel::define_of_id_table; +/// # use kernel::{define_of_id_table, module_of_id_table, driver_of_id_table}; /// use kernel::of; /// -/// define_of_id_table! {u32, [ +/// define_of_id_table! {MY_ID_TABLE, u32, [ /// (of::DeviceId::Compatible(b"test-device1,test-device2"), Some(0xff)), /// (of::DeviceId::Compatible(b"test-device3"), None), /// ]}; +/// +/// module_of_id_table!(MOD_TABLE, ASAHI_ID_TABLE); +/// +/// // Within the `Driver` implementation: +/// driver_of_id_table!(MY_ID_TABLE); /// ``` #[macro_export] macro_rules! define_of_id_table { - ($data_type:ty, $($t:tt)*) => { - $crate::define_id_table!(OF_DEVICE_ID_TABLE, $crate::of::DeviceId, $data_type, $($t)*); + ($name:ident, $data_type:ty, $($t:tt)*) => { + $crate::define_id_array!($name, $crate::of::DeviceId, $data_type, $($t)*); + }; +} + +/// Convenience macro to declare which device ID table to use for a bus driver. +#[macro_export] +macro_rules! driver_of_id_table { + ($name:expr) => { + $crate::driver_id_table!( + OF_DEVICE_ID_TABLE, + $crate::of::DeviceId, + Self::IdInfo, + $name + ); + }; +} + +/// Declare a device ID table as a module-level table. This creates the necessary module alias +/// entries to enable module autoloading. +#[macro_export] +macro_rules! module_of_id_table { + ($item_name:ident, $table_name:ident) => { + $crate::module_id_table!($item_name, "of", $crate::of::DeviceId, $table_name); }; } From d0927e59a2dc6227ba94289d2480fc242f2bbcfd Mon Sep 17 00:00:00 2001 From: Maciej Falkowski Date: Tue, 8 Feb 2022 16:31:13 +0100 Subject: [PATCH 457/635] rust: platform: add `ioremap_resource` and `get_resource` methods This patch adds a logic similar to `devm_platform_ioremap_resource` function adding: - `IoResource` enumerated type that groups the `IORESOURCE_*` macros. - `get_resource()` method that is a binding of `platform_get_resource` - `ioremap_resource` that is newly written method similar to `devm_platform_ioremap_resource`. Lina: Removed `bit` dependency and rebased Co-developed-by: Asahi Lina Signed-off-by: Maciej Falkowski --- rust/kernel/io_mem.rs | 6 +++++ rust/kernel/platform.rs | 60 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/rust/kernel/io_mem.rs b/rust/kernel/io_mem.rs index 0c646ddcf7ea09..1083e6f4aec08f 100644 --- a/rust/kernel/io_mem.rs +++ b/rust/kernel/io_mem.rs @@ -9,6 +9,12 @@ use crate::{bindings, error::code::*, error::Result}; use core::convert::TryInto; +/// The type of `Resource`. +pub enum IoResource { + /// i/o memory + Mem = bindings::IORESOURCE_MEM as _, +} + /// Represents a memory resource. pub struct Resource { offset: bindings::resource_size_t, diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 54c788bf1f8cd6..b961bb295060db 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -10,7 +10,8 @@ use crate::{ bindings, device::{self, RawDevice}, driver, - error::{from_kernel_result, to_result, Result}, + error::{code::*, from_result, to_result, Result}, + io_mem::{IoMem, IoResource, Resource}, of, str::CStr, types::ForeignOwnable, @@ -163,6 +164,7 @@ pub trait Driver { /// The field `ptr` is non-null and valid for the lifetime of the object. pub struct Device { ptr: *mut bindings::platform_device, + used_resource: u64, } impl Device { @@ -174,7 +176,10 @@ impl Device { /// instance. unsafe fn from_ptr(ptr: *mut bindings::platform_device) -> Self { // INVARIANT: The safety requirements of the function ensure the lifetime invariant. - Self { ptr } + Self { + ptr, + used_resource: 0, + } } /// Returns id of the platform device. @@ -182,6 +187,57 @@ impl Device { // SAFETY: By the type invariants, we know that `self.ptr` is non-null and valid. unsafe { (*self.ptr).id } } + + /// Gets a system resources of a platform device. + pub fn get_resource(&mut self, rtype: IoResource, num: usize) -> Result { + // SAFETY: `self.ptr` is valid by the type invariant. + let res = unsafe { bindings::platform_get_resource(self.ptr, rtype as _, num as _) }; + if res.is_null() { + return Err(EINVAL); + } + + // Get the position of the found resource in the array. + // SAFETY: + // - `self.ptr` is valid by the type invariant. + // - `res` is a displaced pointer to one of the array's elements, + // and `resource` is its base pointer. + let index = unsafe { res.offset_from((*self.ptr).resource) } as usize; + + // Make sure that the index does not exceed the 64-bit mask. + assert!(index < 64); + + if self.used_resource >> index & 1 == 1 { + return Err(EBUSY); + } + self.used_resource |= 1 << index; + + // SAFETY: The pointer `res` is returned from `bindings::platform_get_resource` + // above and checked if it is not a NULL. + unsafe { Resource::new((*res).start, (*res).end) }.ok_or(EINVAL) + } + + /// Ioremaps resources of a platform device. + /// + /// # Safety + /// + /// Callers must ensure that either (a) the resulting interface cannot be used to initiate DMA + /// operations, or (b) that DMA operations initiated via the returned interface use DMA handles + /// allocated through the `dma` module. + pub unsafe fn ioremap_resource( + &mut self, + index: usize, + ) -> Result> { + let mask = self.used_resource; + let res = self.get_resource(IoResource::Mem, index)?; + + // SAFETY: Valid by the safety contract. + let iomem = unsafe { IoMem::::try_new(res) }; + // If remapping fails, the given resource won't be used, so restore the old mask. + if iomem.is_err() { + self.used_resource = mask; + } + iomem + } } // SAFETY: The device returned by `raw_device` is the raw platform device. From 7b559ecc00e0abd52b48f62571279c029614d483 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 7 Sep 2022 17:25:18 +0900 Subject: [PATCH 458/635] rust: kernel: platform: Add Device.set_dma_masks() Allows drivers to configure the DMA masks for a device. Implemented here, not in device, because it requires a mutable platform device reference this way (device::Device is a safely clonable reference). Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 7 +++++++ rust/kernel/platform.rs | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 26b7666ab603d6..614e5b88cac3a7 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 1dbf00cebccc63..e1080b225ab3f4 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -351,6 +352,12 @@ struct device_node *rust_helper_of_parse_phandle(const struct device_node *np, } EXPORT_SYMBOL_GPL(rust_helper_of_parse_phandle); +int rust_helper_dma_set_mask_and_coherent(struct device *dev, u64 mask) +{ + return dma_set_mask_and_coherent(dev, mask); +} +EXPORT_SYMBOL_GPL(rust_helper_dma_set_mask_and_coherent); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index b961bb295060db..71979520fe46c5 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -188,6 +188,11 @@ impl Device { unsafe { (*self.ptr).id } } + /// Sets the DMA masks (normal and coherent) for a platform device. + pub fn set_dma_masks(&mut self, mask: u64) -> Result { + to_result(unsafe { bindings::dma_set_mask_and_coherent(&mut (*self.ptr).dev, mask) }) + } + /// Gets a system resources of a platform device. pub fn get_resource(&mut self, rtype: IoResource, num: usize) -> Result { // SAFETY: `self.ptr` is valid by the type invariant. From 9f785676fc40493b48188181e9834015da8b8a3e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 21 Sep 2022 14:50:56 +0900 Subject: [PATCH 459/635] rust: Add ioremap_np support to io_mem & friends Apple SoCs require non-posted mappings for MMIO, and this is automatically handled by devm_ioremap_resource() and friends via a resource flag. Implement the same logic in kernel::io_mem, so it can work the same way. Signed-off-by: Asahi Lina --- rust/helpers.c | 6 ++++++ rust/kernel/io_mem.rs | 10 +++++++++- rust/kernel/platform.rs | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/rust/helpers.c b/rust/helpers.c index e1080b225ab3f4..2e78c61a11c5a1 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -206,6 +206,12 @@ void __iomem *rust_helper_ioremap(resource_size_t offset, unsigned long size) } EXPORT_SYMBOL_GPL(rust_helper_ioremap); +void __iomem *rust_helper_ioremap_np(resource_size_t offset, unsigned long size) +{ + return ioremap_np(offset, size); +} +EXPORT_SYMBOL_GPL(rust_helper_ioremap_np); + u8 rust_helper_readb(const volatile void __iomem *addr) { return readb(addr); diff --git a/rust/kernel/io_mem.rs b/rust/kernel/io_mem.rs index 1083e6f4aec08f..5bb8800b04f59b 100644 --- a/rust/kernel/io_mem.rs +++ b/rust/kernel/io_mem.rs @@ -19,12 +19,14 @@ pub enum IoResource { pub struct Resource { offset: bindings::resource_size_t, size: bindings::resource_size_t, + flags: core::ffi::c_ulong, } impl Resource { pub(crate) fn new( start: bindings::resource_size_t, end: bindings::resource_size_t, + flags: core::ffi::c_ulong, ) -> Option { if start == 0 { return None; @@ -32,6 +34,7 @@ impl Resource { Some(Self { offset: start, size: end.checked_sub(start)?.checked_add(1)?, + flags, }) } } @@ -161,7 +164,12 @@ impl IoMem { // Try to map the resource. // SAFETY: Just mapping the memory range. - let addr = unsafe { bindings::ioremap(res.offset, res.size as _) }; + let addr = if res.flags & (bindings::IORESOURCE_MEM_NONPOSTED as core::ffi::c_ulong) != 0 { + unsafe { bindings::ioremap_np(res.offset, res.size as _) } + } else { + unsafe { bindings::ioremap(res.offset, res.size as _) } + }; + if addr.is_null() { Err(ENOMEM) } else { diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 71979520fe46c5..d42ae1cbaf9e21 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -218,7 +218,7 @@ impl Device { // SAFETY: The pointer `res` is returned from `bindings::platform_get_resource` // above and checked if it is not a NULL. - unsafe { Resource::new((*res).start, (*res).end) }.ok_or(EINVAL) + unsafe { Resource::new((*res).start, (*res).end, (*res).flags) }.ok_or(EINVAL) } /// Ioremaps resources of a platform device. From ff9bef6725d35800dfe85b4c8138b97f75ff2081 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 21:28:58 +0900 Subject: [PATCH 460/635] *RFL import: macros::module params functionality & deps Commit reference: 3dfc5ebff103 --- rust/macros/helpers.rs | 20 +++ rust/macros/module.rs | 346 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 363 insertions(+), 3 deletions(-) diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index b2bdd4d8c958af..99f9de209c8451 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -56,6 +56,10 @@ pub(crate) fn expect_string_ascii(it: &mut token_stream::IntoIter) -> String { string } +pub(crate) fn expect_literal(it: &mut token_stream::IntoIter) -> String { + try_literal(it).expect("Expected Literal") +} + pub(crate) fn expect_group(it: &mut token_stream::IntoIter) -> Group { if let TokenTree::Group(group) = it.next().expect("Reached end of token stream for Group") { group @@ -69,3 +73,19 @@ pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { panic!("Expected end"); } } + +pub(crate) fn get_literal(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let literal = expect_literal(it); + assert_eq!(expect_punct(it), ','); + literal +} + +pub(crate) fn get_string(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let string = expect_string(it); + assert_eq!(expect_punct(it), ','); + string +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs index 7678c7e041f0c3..234787a0eecd07 100644 --- a/rust/macros/module.rs +++ b/rust/macros/module.rs @@ -1,9 +1,41 @@ // SPDX-License-Identifier: GPL-2.0 use crate::helpers::*; -use proc_macro::{token_stream, Delimiter, Literal, TokenStream, TokenTree}; +use proc_macro::{token_stream, Delimiter, Group, Literal, TokenStream, TokenTree}; use std::fmt::Write; +#[derive(Clone, PartialEq)] +enum ParamType { + Ident(String), + Array { vals: String, max_length: usize }, +} + +fn expect_array_fields(it: &mut token_stream::IntoIter) -> ParamType { + assert_eq!(expect_punct(it), '<'); + let vals = expect_ident(it); + assert_eq!(expect_punct(it), ','); + let max_length_str = expect_literal(it); + let max_length = max_length_str + .parse::() + .expect("Expected usize length"); + assert_eq!(expect_punct(it), '>'); + ParamType::Array { vals, max_length } +} + +fn expect_type(it: &mut token_stream::IntoIter) -> ParamType { + if let TokenTree::Ident(ident) = it + .next() + .expect("Reached end of token stream for param type") + { + match ident.to_string().as_ref() { + "ArrayParam" => expect_array_fields(it), + _ => ParamType::Ident(ident.to_string()), + } + } else { + panic!("Expected Param Type") + } +} + fn expect_string_array(it: &mut token_stream::IntoIter) -> Vec { let group = expect_group(it); assert_eq!(group.delimiter(), Delimiter::Bracket); @@ -87,6 +119,113 @@ impl<'a> ModInfoBuilder<'a> { self.emit_only_builtin(field, content); self.emit_only_loadable(field, content); } + + fn emit_param(&mut self, field: &str, param: &str, content: &str) { + let content = format!("{param}:{content}", param = param, content = content); + self.emit(field, &content); + } +} + +fn permissions_are_readonly(perms: &str) -> bool { + let (radix, digits) = if let Some(n) = perms.strip_prefix("0x") { + (16, n) + } else if let Some(n) = perms.strip_prefix("0o") { + (8, n) + } else if let Some(n) = perms.strip_prefix("0b") { + (2, n) + } else { + (10, perms) + }; + match u32::from_str_radix(digits, radix) { + Ok(perms) => perms & 0o222 == 0, + Err(_) => false, + } +} + +fn param_ops_path(param_type: &str) -> &'static str { + match param_type { + "bool" => "kernel::module_param::PARAM_OPS_BOOL", + "i8" => "kernel::module_param::PARAM_OPS_I8", + "u8" => "kernel::module_param::PARAM_OPS_U8", + "i16" => "kernel::module_param::PARAM_OPS_I16", + "u16" => "kernel::module_param::PARAM_OPS_U16", + "i32" => "kernel::module_param::PARAM_OPS_I32", + "u32" => "kernel::module_param::PARAM_OPS_U32", + "i64" => "kernel::module_param::PARAM_OPS_I64", + "u64" => "kernel::module_param::PARAM_OPS_U64", + "isize" => "kernel::module_param::PARAM_OPS_ISIZE", + "usize" => "kernel::module_param::PARAM_OPS_USIZE", + "str" => "kernel::module_param::PARAM_OPS_STR", + t => panic!("Unrecognized type {}", t), + } +} + +#[allow(clippy::type_complexity)] +fn try_simple_param_val( + param_type: &str, +) -> Box Option> { + match param_type { + "bool" => Box::new(try_ident), + "str" => Box::new(|param_it| { + try_string(param_it) + .map(|s| format!("kernel::module_param::StringParam::Ref(b\"{}\")", s)) + }), + _ => Box::new(try_literal), + } +} + +fn get_default(param_type: &ParamType, param_it: &mut token_stream::IntoIter) -> String { + let try_param_val = match param_type { + ParamType::Ident(ref param_type) + | ParamType::Array { + vals: ref param_type, + max_length: _, + } => try_simple_param_val(param_type), + }; + assert_eq!(expect_ident(param_it), "default"); + assert_eq!(expect_punct(param_it), ':'); + let default = match param_type { + ParamType::Ident(_) => try_param_val(param_it).expect("Expected default param value"), + ParamType::Array { + vals: _, + max_length: _, + } => { + let group = expect_group(param_it); + assert_eq!(group.delimiter(), Delimiter::Bracket); + let mut default_vals = Vec::new(); + let mut it = group.stream().into_iter(); + + while let Some(default_val) = try_param_val(&mut it) { + default_vals.push(default_val); + match it.next() { + Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','), + None => break, + _ => panic!("Expected ',' or end of array default values"), + } + } + + let mut default_array = "kernel::module_param::ArrayParam::create(&[".to_string(); + default_array.push_str( + &default_vals + .iter() + .map(|val| val.to_string()) + .collect::>() + .join(","), + ); + default_array.push_str("])"); + default_array + } + }; + assert_eq!(expect_punct(param_it), ','); + default +} + +fn generated_array_ops_name(vals: &str, max_length: usize) -> String { + format!( + "__generated_array_ops_{vals}_{max_length}", + vals = vals, + max_length = max_length + ) } #[derive(Debug, Default)] @@ -97,14 +236,22 @@ struct ModuleInfo { author: Option, description: Option, alias: Option>, + params: Option, } impl ModuleInfo { fn parse(it: &mut token_stream::IntoIter) -> Self { let mut info = ModuleInfo::default(); - const EXPECTED_KEYS: &[&str] = - &["type", "name", "author", "description", "license", "alias"]; + const EXPECTED_KEYS: &[&str] = &[ + "type", + "name", + "author", + "description", + "license", + "alias", + "params", + ]; const REQUIRED_KEYS: &[&str] = &["type", "name", "license"]; let mut seen_keys = Vec::new(); @@ -131,6 +278,7 @@ impl ModuleInfo { "description" => info.description = Some(expect_string(it)), "license" => info.license = expect_string_ascii(it), "alias" => info.alias = Some(expect_string_array(it)), + "params" => info.params = Some(expect_group(it)), _ => panic!( "Unknown key \"{}\". Valid keys are: {:?}.", key, EXPECTED_KEYS @@ -192,6 +340,195 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable"); modinfo.emit_only_builtin("file", &file); + let mut array_types_to_generate = Vec::new(); + if let Some(params) = info.params { + assert_eq!(params.delimiter(), Delimiter::Brace); + + let mut it = params.stream().into_iter(); + + loop { + let param_name = match it.next() { + Some(TokenTree::Ident(ident)) => ident.to_string(), + Some(_) => panic!("Expected Ident or end"), + None => break, + }; + + assert_eq!(expect_punct(&mut it), ':'); + let param_type = expect_type(&mut it); + let group = expect_group(&mut it); + assert_eq!(expect_punct(&mut it), ','); + + assert_eq!(group.delimiter(), Delimiter::Brace); + + let mut param_it = group.stream().into_iter(); + let param_default = get_default(¶m_type, &mut param_it); + let param_permissions = get_literal(&mut param_it, "permissions"); + let param_description = get_string(&mut param_it, "description"); + expect_end(&mut param_it); + + // TODO: More primitive types. + // TODO: Other kinds: unsafes, etc. + let (param_kernel_type, ops): (String, _) = match param_type { + ParamType::Ident(ref param_type) => ( + param_type.to_string(), + param_ops_path(param_type).to_string(), + ), + ParamType::Array { + ref vals, + max_length, + } => { + array_types_to_generate.push((vals.clone(), max_length)); + ( + format!("__rust_array_param_{}_{}", vals, max_length), + generated_array_ops_name(vals, max_length), + ) + } + }; + + modinfo.emit_param("parmtype", ¶m_name, ¶m_kernel_type); + modinfo.emit_param("parm", ¶m_name, ¶m_description); + let param_type_internal = match param_type { + ParamType::Ident(ref param_type) => match param_type.as_ref() { + "str" => "kernel::module_param::StringParam".to_string(), + other => other.to_string(), + }, + ParamType::Array { + ref vals, + max_length, + } => format!( + "kernel::module_param::ArrayParam<{vals}, {max_length}>", + vals = vals, + max_length = max_length + ), + }; + let read_func = if permissions_are_readonly(¶m_permissions) { + format!( + " + fn read(&self) + -> &<{param_type_internal} as kernel::module_param::ModuleParam>::Value {{ + // SAFETY: Parameters do not need to be locked because they are + // read only or sysfs is not enabled. + unsafe {{ + <{param_type_internal} as kernel::module_param::ModuleParam>::value( + &__{name}_{param_name}_value + ) + }} + }} + ", + name = info.name, + param_name = param_name, + param_type_internal = param_type_internal, + ) + } else { + format!( + " + fn read<'lck>(&self, lock: &'lck kernel::KParamGuard) + -> &'lck <{param_type_internal} as kernel::module_param::ModuleParam>::Value {{ + // SAFETY: Parameters are locked by `KParamGuard`. + unsafe {{ + <{param_type_internal} as kernel::module_param::ModuleParam>::value( + &__{name}_{param_name}_value + ) + }} + }} + ", + name = info.name, + param_name = param_name, + param_type_internal = param_type_internal, + ) + }; + let kparam = format!( + " + kernel::bindings::kernel_param__bindgen_ty_1 {{ + arg: unsafe {{ &__{name}_{param_name}_value }} + as *const _ as *mut core::ffi::c_void, + }}, + ", + name = info.name, + param_name = param_name, + ); + write!( + modinfo.buffer, + " + static mut __{name}_{param_name}_value: {param_type_internal} = {param_default}; + + struct __{name}_{param_name}; + + impl __{name}_{param_name} {{ {read_func} }} + + const {param_name}: __{name}_{param_name} = __{name}_{param_name}; + + // Note: the C macro that generates the static structs for the `__param` section + // asks for them to be `aligned(sizeof(void *))`. However, that was put in place + // in 2003 in commit 38d5b085d2a0 (\"[PATCH] Fix over-alignment problem on x86-64\") + // to undo GCC over-alignment of static structs of >32 bytes. It seems that is + // not the case anymore, so we simplify to a transparent representation here + // in the expectation that it is not needed anymore. + // TODO: Revisit this to confirm the above comment and remove it if it happened. + #[repr(transparent)] + struct __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param); + + unsafe impl Sync for __{name}_{param_name}_RacyKernelParam {{ + }} + + #[cfg(not(MODULE))] + const __{name}_{param_name}_name: *const core::ffi::c_char = + b\"{name}.{param_name}\\0\" as *const _ as *const core::ffi::c_char; + + #[cfg(MODULE)] + const __{name}_{param_name}_name: *const core::ffi::c_char = + b\"{param_name}\\0\" as *const _ as *const core::ffi::c_char; + + #[link_section = \"__param\"] + #[used] + static __{name}_{param_name}_struct: __{name}_{param_name}_RacyKernelParam = + __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param {{ + name: __{name}_{param_name}_name, + // SAFETY: `__this_module` is constructed by the kernel at load time + // and will not be freed until the module is unloaded. + #[cfg(MODULE)] + mod_: unsafe {{ &kernel::bindings::__this_module as *const _ as *mut _ }}, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: unsafe {{ &{ops} }} as *const kernel::bindings::kernel_param_ops, + perm: {permissions}, + level: -1, + flags: 0, + __bindgen_anon_1: {kparam} + }}); + ", + name = info.name, + param_type_internal = param_type_internal, + read_func = read_func, + param_default = param_default, + param_name = param_name, + ops = ops, + permissions = param_permissions, + kparam = kparam, + ) + .unwrap(); + } + } + + let mut generated_array_types = String::new(); + + for (vals, max_length) in array_types_to_generate { + let ops_name = generated_array_ops_name(&vals, max_length); + write!( + generated_array_types, + " + kernel::make_param_ops!( + {ops_name}, + kernel::module_param::ArrayParam<{vals}, {{ {max_length} }}> + ); + ", + ops_name = ops_name, + vals = vals, + max_length = max_length, + ) + .unwrap(); + } + format!( " /// The module name. @@ -291,10 +628,13 @@ pub(crate) fn module(ts: TokenStream) -> TokenStream { }} {modinfo} + + {generated_array_types} ", type_ = info.type_, name = info.name, modinfo = modinfo.buffer, + generated_array_types = generated_array_types, initcall_section = ".initcall6.init" ) .parse() From 55c6f7dd98126e42399a63b579086bea71c6dfbe Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 21:34:36 +0900 Subject: [PATCH 461/635] *RFL import: Rest of kernel::error::code::* Commit reference: 3dfc5ebff103 --- rust/kernel/error.rs | 111 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 5f4114b30b94c7..3800ea0757bce2 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -58,6 +58,117 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + declare_err!(EDEADLK, "Resource deadlock would occur"); + declare_err!(ENAMETOOLONG, "File name too long"); + declare_err!(ENOLCK, "No record locks available"); + declare_err!( + ENOSYS, + "Invalid system call number.", + "", + "This error code is special: arch syscall entry code will return", + "[`ENOSYS`] if users try to call a syscall that doesn't exist.", + "To keep failures of syscalls that really do exist distinguishable from", + "failures due to attempts to use a nonexistent syscall, syscall", + "implementations should refrain from returning [`ENOSYS`]." + ); + declare_err!(ENOTEMPTY, "Directory not empty."); + declare_err!(ELOOP, "Too many symbolic links encountered."); + declare_err!(EWOULDBLOCK, "Operation would block."); + declare_err!(ENOMSG, "No message of desired type."); + declare_err!(EIDRM, "Identifier removed."); + declare_err!(ECHRNG, "Channel number out of range."); + declare_err!(EL2NSYNC, "Level 2 not synchronized."); + declare_err!(EL3HLT, "Level 3 halted."); + declare_err!(EL3RST, "Level 3 reset."); + declare_err!(ELNRNG, "Link number out of range."); + declare_err!(EUNATCH, "Protocol driver not attached."); + declare_err!(ENOCSI, "No CSI structure available."); + declare_err!(EL2HLT, "Level 2 halted."); + declare_err!(EBADE, "Invalid exchange."); + declare_err!(EBADR, "Invalid request descriptor."); + declare_err!(EXFULL, "Exchange full."); + declare_err!(ENOANO, "No anode."); + declare_err!(EBADRQC, "Invalid request code."); + declare_err!(EBADSLT, "Invalid slot."); + declare_err!(EDEADLOCK, "Resource deadlock would occur."); + declare_err!(EBFONT, "Bad font file format."); + declare_err!(ENOSTR, "Device not a stream."); + declare_err!(ENODATA, "No data available."); + declare_err!(ETIME, "Timer expired."); + declare_err!(ENOSR, "Out of streams resources."); + declare_err!(ENONET, "Machine is not on the network."); + declare_err!(ENOPKG, "Package not installed."); + declare_err!(EREMOTE, "Object is remote."); + declare_err!(ENOLINK, "Link has been severed."); + declare_err!(EADV, "Advertise error."); + declare_err!(ESRMNT, "Srmount error."); + declare_err!(ECOMM, "Communication error on send."); + declare_err!(EPROTO, "Protocol error."); + declare_err!(EMULTIHOP, "Multihop attempted."); + declare_err!(EDOTDOT, "RFS specific error."); + declare_err!(EBADMSG, "Not a data message."); + declare_err!(EOVERFLOW, "Value too large for defined data type."); + declare_err!(ENOTUNIQ, "Name not unique on network."); + declare_err!(EBADFD, "File descriptor in bad state."); + declare_err!(EREMCHG, "Remote address changed."); + declare_err!(ELIBACC, "Can not access a needed shared library."); + declare_err!(ELIBBAD, "Accessing a corrupted shared library."); + declare_err!(ELIBSCN, ".lib section in a.out corrupted."); + declare_err!(ELIBMAX, "Attempting to link in too many shared libraries."); + declare_err!(ELIBEXEC, "Cannot exec a shared library directly."); + declare_err!(EILSEQ, "Illegal byte sequence."); + declare_err!(ERESTART, "Interrupted system call should be restarted."); + declare_err!(ESTRPIPE, "Streams pipe error."); + declare_err!(EUSERS, "Too many users."); + declare_err!(ENOTSOCK, "Socket operation on non-socket."); + declare_err!(EDESTADDRREQ, "Destination address required."); + declare_err!(EMSGSIZE, "Message too long."); + declare_err!(EPROTOTYPE, "Protocol wrong type for socket."); + declare_err!(ENOPROTOOPT, "Protocol not available."); + declare_err!(EPROTONOSUPPORT, "Protocol not supported."); + declare_err!(ESOCKTNOSUPPORT, "Socket type not supported."); + declare_err!(EOPNOTSUPP, "Operation not supported on transport endpoint."); + declare_err!(EPFNOSUPPORT, "Protocol family not supported."); + declare_err!(EAFNOSUPPORT, "Address family not supported by protocol."); + declare_err!(EADDRINUSE, "Address already in use."); + declare_err!(EADDRNOTAVAIL, "Cannot assign requested address."); + declare_err!(ENETDOWN, "Network is down."); + declare_err!(ENETUNREACH, "Network is unreachable."); + declare_err!(ENETRESET, "Network dropped connection because of reset."); + declare_err!(ECONNABORTED, "Software caused connection abort."); + declare_err!(ECONNRESET, "Connection reset by peer."); + declare_err!(ENOBUFS, "No buffer space available."); + declare_err!(EISCONN, "Transport endpoint is already connected."); + declare_err!(ENOTCONN, "Transport endpoint is not connected."); + declare_err!(ESHUTDOWN, "Cannot send after transport endpoint shutdown."); + declare_err!(ETOOMANYREFS, "Too many references: cannot splice."); + declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECONNREFUSED, "Connection refused."); + declare_err!(EHOSTDOWN, "Host is down."); + declare_err!(EHOSTUNREACH, "No route to host."); + declare_err!(EALREADY, "Operation already in progress."); + declare_err!(EINPROGRESS, "Operation now in progress."); + declare_err!(ESTALE, "Stale file handle."); + declare_err!(EUCLEAN, "Structure needs cleaning."); + declare_err!(ENOTNAM, "Not a XENIX named type file."); + declare_err!(ENAVAIL, "No XENIX semaphores available."); + declare_err!(EISNAM, "Is a named type file."); + declare_err!(EREMOTEIO, "Remote I/O error."); + declare_err!(EDQUOT, "Quota exceeded."); + declare_err!(ENOMEDIUM, "No medium found."); + declare_err!(EMEDIUMTYPE, "Wrong medium type."); + declare_err!(ECANCELED, "Operation Canceled."); + declare_err!(ENOKEY, "Required key not available."); + declare_err!(EKEYEXPIRED, "Key has expired."); + declare_err!(EKEYREVOKED, "Key has been revoked."); + declare_err!(EKEYREJECTED, "Key was rejected by service."); + declare_err!(EOWNERDEAD, "Owner died.", "", "For robust mutexes."); + declare_err!(ENOTRECOVERABLE, "State not recoverable."); + declare_err!(ERFKILL, "Operation not possible due to RF-kill."); + declare_err!(EHWPOISON, "Memory page has hardware error."); + declare_err!(ERESTARTSYS, "Restart the system call."); + declare_err!(ENOTSUPP, "Operation is not supported."); + declare_err!(ENOPARAM, "Parameter not supported."); } /// Generic integer kernel error. From 03aba9ed480ef8ed86e97306c87ff4e55c266702 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 7 Sep 2022 17:41:06 +0900 Subject: [PATCH 462/635] rust: bindings: Add resource_size wrapper TODO: This isn't abstracted properly yet Signed-off-by: Asahi Lina --- rust/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/helpers.c b/rust/helpers.c index 2e78c61a11c5a1..dc3d3dee6cf279 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -364,6 +364,12 @@ int rust_helper_dma_set_mask_and_coherent(struct device *dev, u64 mask) } EXPORT_SYMBOL_GPL(rust_helper_dma_set_mask_and_coherent); +resource_size_t rust_helper_resource_size(const struct resource *res) +{ + return resource_size(res); +} +EXPORT_SYMBOL_GPL(rust_helper_resource_size); + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust From 457e753d374dde85bf42f406fb2ff7b9af6e6b2c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 21:45:19 +0900 Subject: [PATCH 463/635] rust: Allow feature allocator_api Signed-off-by: Asahi Lina --- scripts/Makefile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index f9d228a87f9a5f..c3e2e2f257afdf 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := const_refs_to_cell,new_uninit,pin_macro +rust_allowed_features := allocator_api,const_refs_to_cell,new_uninit,pin_macro rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ From 010cde24f0d9d8595bdbbf1f1c12d7944d905573 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 21:47:25 +0900 Subject: [PATCH 464/635] *RFL import: kernel::KParamGuard & friends Commit reference: 3dfc5ebff103 --- rust/kernel/lib.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index b78e8b96edefb5..41b8185a339e33 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -115,6 +115,43 @@ impl ThisModule { pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { ThisModule(ptr) } + + /// Locks the module parameters to access them. + /// + /// Returns a [`KParamGuard`] that will release the lock when dropped. + pub fn kernel_param_lock(&self) -> KParamGuard<'_> { + // SAFETY: `kernel_param_lock` will check if the pointer is null and + // use the built-in mutex in that case. + #[cfg(CONFIG_SYSFS)] + unsafe { + bindings::kernel_param_lock(self.0) + } + + KParamGuard { + #[cfg(CONFIG_SYSFS)] + this_module: self, + phantom: core::marker::PhantomData, + } + } +} + +/// Scoped lock on the kernel parameters of [`ThisModule`]. +/// +/// Lock will be released when this struct is dropped. +pub struct KParamGuard<'a> { + #[cfg(CONFIG_SYSFS)] + this_module: &'a ThisModule, + phantom: core::marker::PhantomData<&'a ()>, +} + +#[cfg(CONFIG_SYSFS)] +impl<'a> Drop for KParamGuard<'a> { + fn drop(&mut self) { + // SAFETY: `kernel_param_lock` will check if the pointer is null and + // use the built-in mutex in that case. The existence of `self` + // guarantees that the lock is held. + unsafe { bindings::kernel_param_unlock(self.this_module.0) } + } } #[cfg(not(any(testlib, test)))] From fbb7a4446d9c2f5429ba2ff33fcd8c5088044f8c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 16 Feb 2023 21:48:32 +0900 Subject: [PATCH 465/635] *RFL import: kernel::error::Error Debug impl & dependencies Commit reference: 3dfc5ebff103 --- rust/helpers.c | 7 +++++++ rust/kernel/error.rs | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/rust/helpers.c b/rust/helpers.c index dc3d3dee6cf279..d101bde427352e 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -134,6 +135,12 @@ void rust_helper_put_task_struct(struct task_struct *t) } EXPORT_SYMBOL_GPL(rust_helper_put_task_struct); +const char *rust_helper_errname(int err) +{ + return errname(err); +} +EXPORT_SYMBOL_GPL(rust_helper_errname); + void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags) { xa_init_flags(xa, flags); diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 3800ea0757bce2..20bf887d909ee9 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -4,12 +4,15 @@ //! //! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h) +use crate::str::CStr; + use alloc::{ alloc::{AllocError, LayoutError}, collections::TryReserveError, }; use core::convert::From; +use core::fmt; use core::num::TryFromIntError; use core::str::Utf8Error; @@ -224,6 +227,42 @@ impl Error { // SAFETY: self.0 is a valid error due to its invariant. unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } } + + /// Returns a string representing the error, if one exists. + #[cfg(not(testlib))] + pub fn name(&self) -> Option<&'static CStr> { + // SAFETY: Just an FFI call, there are no extra safety requirements. + let ptr = unsafe { bindings::errname(-self.0) }; + if ptr.is_null() { + None + } else { + // SAFETY: The string returned by `errname` is static and `NUL`-terminated. + Some(unsafe { CStr::from_char_ptr(ptr) }) + } + } + + /// Returns a string representing the error, if one exists. + /// + /// When `testlib` is configured, this always returns `None` to avoid the dependency on a + /// kernel function so that tests that use this (e.g., by calling [`Result::unwrap`]) can still + /// run in userspace. + #[cfg(testlib)] + pub fn name(&self) -> Option<&'static CStr> { + None + } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.name() { + // Print out number if no name can be found. + None => f.debug_tuple("Error").field(&-self.0).finish(), + // SAFETY: These strings are ASCII-only. + Some(name) => f + .debug_tuple(unsafe { core::str::from_utf8_unchecked(name) }) + .finish(), + } + } } impl From for Error { From 23a0680e103d61769ec3bee280cc62d5ab52d3e7 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 18 Aug 2022 02:07:35 +0900 Subject: [PATCH 466/635] iommu/io-pgtable: Add Apple UAT variant format Apple Silicon SoCs (M1, M2, etc.) have a GPU with an ARM64 firmware coprocessor. The firmware and the GPU share page tables in the standard ARM64 format (the firmware literally sets the base as its TTBR0/1 registers). TTBR0 covers the low half of the address space and is intended to be per-GPU-VM (GPU user mappings and kernel-managed buffers), while TTBR1 covers the upper half and is global (firmware code, data, management structures shared with the AP, and a few GPU-accessible data structures). In typical Apple fashion, the permissions are interpreted differently from traditional ARM PTEs. By default, firmware mappings use Apple SPRR permission remapping. The firmware only uses that for its own code/data/MMIO mappings, and those pages are not accessible by the GPU hardware. We never need to touch/manage these mappings, so this patch does not support them. When a specific bit is set in the PTEs, permissions switch to a different scheme which supports various combinations of firmware/GPU access. This is the mode intended to be used by AP GPU drivers, and what we implement here. The prot bits are interpreted as follows: - IOMMU_READ and IOMMU_WRITE have the usual meaning. - IOMMU_PRIV creates firmware-only mappings (no GPU access) - IOMMU_NOEXEC creates GPU-only structures (no FW access) - Otherwise structures are accessible by both GPU and FW - IOMMU_MMIO creates Device mappings for firmware - IOMMU_CACHE creates Normal-NC mappings for firmware (cache-coherent from the point of view of the AP, but slower) - Otherwise creates Normal mappings for firmware (this requires manual cache management on the firmware side, as it is not coherent with the SoC fabric) GPU-only mappings (textures/etc) are expected to use IOMMU_CACHE and are seemingly coherent with the CPU (or otherwise the firmware/GPU already issue the required cache management operations when correctly configured). There is a GPU-RO/FW-RW mode, but it is not currently implemented (it doesn't seem to be very useful for the driver). There seems to be no real noexec control (i.e. for shaders) on the GPU side. All of these mappings are implicitly noexec for the firmware. Drivers are expected to fully manage per-user (TTBR0) page tables, but ownership of shared kernel (TTBR1) page tables is shared between the firmware and the AP OS. We handle this by simply using a smaller IAS to drop down one level of page tables, so the driver can install a PTE in the top-level (firmware-initialized) page table directly and just add an offset to the VAs passed into the io_pgtable code. This avoids having to have any special handling for this here. The firmware-relevant data structures are small, so we do not expect to ever require more VA space than one top-level PTE covers (IAS=36 for the next level, 64 GiB). Only 16K page mode is supported. The coprocessor MMU supports huge pages as usual for ARM64, but the GPU MMU does not, so we do not enable them. Signed-off-by: Asahi Lina --- drivers/iommu/io-pgtable-arm.c | 101 +++++++++++++++++++++++++++++++-- drivers/iommu/io-pgtable.c | 1 + include/linux/io-pgtable.h | 6 ++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 72dcdd468cf30d..bbcbf30a064bbe 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -130,6 +130,15 @@ #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL +#define APPLE_UAT_MEMATTR_PRIV (((arm_lpae_iopte)0x0) << 2) +#define APPLE_UAT_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) +#define APPLE_UAT_MEMATTR_SHARED (((arm_lpae_iopte)0x2) << 2) +#define APPLE_UAT_GPU_ACCESS (((arm_lpae_iopte)1) << 55) +#define APPLE_UAT_UXN (((arm_lpae_iopte)1) << 54) +#define APPLE_UAT_PXN (((arm_lpae_iopte)1) << 53) +#define APPLE_UAT_AP1 (((arm_lpae_iopte)1) << 7) +#define APPLE_UAT_AP0 (((arm_lpae_iopte)1) << 6) + /* IOPTE accessors */ #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) @@ -402,7 +411,42 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, { arm_lpae_iopte pte; - if (data->iop.fmt == ARM_64_LPAE_S1 || + if (data->iop.fmt == APPLE_UAT) { + /* + * This bit enables GPU access and the particular permission + * rules that follow. Without it, access is firmware-only and + * permissions follow the firmware's Apple SPRR configuration. + */ + pte = APPLE_UAT_GPU_ACCESS; + if (prot & IOMMU_PRIV) { + /* Firmware structures */ + pte |= APPLE_UAT_AP0; + if (prot & IOMMU_WRITE) { + /* Firmware RW */ + pte |= APPLE_UAT_UXN; + } else if (!(prot & IOMMU_READ)) { + /* No access */ + pte |= APPLE_UAT_PXN; + } + } else if (prot & IOMMU_NOEXEC) { + /* GPU structures (no FW access) */ + pte |= APPLE_UAT_AP1 | ARM_LPAE_PTE_nG; + if (!(prot & IOMMU_READ)) { + pte |= APPLE_UAT_PXN; + if (!(prot & IOMMU_WRITE)) + pte |= APPLE_UAT_UXN; + } else if (prot & IOMMU_WRITE) { + pte |= APPLE_UAT_UXN; + } + } else { + pte |= ARM_LPAE_PTE_nG; + /* GPU structures (also FW accessible) */ + if (prot & IOMMU_WRITE) + pte |= APPLE_UAT_UXN; + if (prot & IOMMU_READ) + pte |= APPLE_UAT_PXN; + } + } else if (data->iop.fmt == ARM_64_LPAE_S1 || data->iop.fmt == ARM_32_LPAE_S1) { pte = ARM_LPAE_PTE_nG; if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) @@ -421,7 +465,14 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, * Note that this logic is structured to accommodate Mali LPAE * having stage-1-like attributes but stage-2-like permissions. */ - if (data->iop.fmt == ARM_64_LPAE_S2 || + if (data->iop.fmt == APPLE_UAT) { + if (prot & IOMMU_MMIO) + pte |= APPLE_UAT_MEMATTR_DEV; + else if (prot & IOMMU_CACHE) + pte |= APPLE_UAT_MEMATTR_SHARED; + else + pte |= APPLE_UAT_MEMATTR_PRIV; + } else if (data->iop.fmt == ARM_64_LPAE_S2 || data->iop.fmt == ARM_32_LPAE_S2) { if (prot & IOMMU_MMIO) pte |= ARM_LPAE_PTE_MEMATTR_DEV; @@ -444,12 +495,14 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, * "outside the GPU" (i.e. either the Inner or System domain in CPU * terms, depending on coherency). */ - if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) + if (data->iop.fmt == APPLE_UAT) + pte |= ARM_LPAE_PTE_SH_NS; + else if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) pte |= ARM_LPAE_PTE_SH_IS; else pte |= ARM_LPAE_PTE_SH_OS; - if (prot & IOMMU_NOEXEC) + if (prot & IOMMU_NOEXEC && data->iop.fmt != APPLE_UAT) pte |= ARM_LPAE_PTE_XN; if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) @@ -1079,6 +1132,41 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) return NULL; } +static struct io_pgtable * +apple_uat_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct arm_lpae_io_pgtable *data; + + /* No quirks for UAT (hopefully) */ + if (cfg->quirks) + return NULL; + + if (cfg->ias > 48 || cfg->oas > 42) + return NULL; + + cfg->pgsize_bitmap &= SZ_16K; + + data = arm_lpae_alloc_pgtable(cfg); + if (!data) + return NULL; + + /* UAT needs full 16K aligned pages for the pgd */ + data->pgd = __arm_lpae_alloc_pages(SZ_16K, GFP_KERNEL, cfg); + if (!data->pgd) + goto out_free_data; + + /* Ensure the empty pgd is visible before the TTBAT can be written */ + wmb(); + + cfg->apple_uat_cfg.ttbr = virt_to_phys(data->pgd); + + return &data->iop; + +out_free_data: + kfree(data); + return NULL; +} + struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { .alloc = arm_64_lpae_alloc_pgtable_s1, .free = arm_lpae_free_pgtable, @@ -1104,6 +1192,11 @@ struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = { .free = arm_lpae_free_pgtable, }; +struct io_pgtable_init_fns io_pgtable_apple_uat_init_fns = { + .alloc = apple_uat_alloc_pgtable, + .free = arm_lpae_free_pgtable, +}; + #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST static struct io_pgtable_cfg *cfg_cookie __initdata; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index b843fcd365d286..faec53e22388cf 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -20,6 +20,7 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, [ARM_MALI_LPAE] = &io_pgtable_arm_mali_lpae_init_fns, + [APPLE_UAT] = &io_pgtable_apple_uat_init_fns, #endif #ifdef CONFIG_IOMMU_IO_PGTABLE_DART [APPLE_DART] = &io_pgtable_apple_dart_init_fns, diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 1b7a44b35616c7..41842c4f76494d 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -19,6 +19,7 @@ enum io_pgtable_fmt { AMD_IOMMU_V2, APPLE_DART, APPLE_DART2, + APPLE_UAT, IO_PGTABLE_NUM_FMTS, }; @@ -144,6 +145,10 @@ struct io_pgtable_cfg { u64 ttbr[4]; u32 n_ttbrs; } apple_dart_cfg; + + struct { + u64 ttbr; + } apple_uat_cfg; }; }; @@ -258,5 +263,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns; extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns; extern struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns; +extern struct io_pgtable_init_fns io_pgtable_apple_uat_init_fns; #endif /* __IO_PGTABLE_H */ From 7cff1133ba8e43d99c8428cbf2f0d796a546d2e0 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 17 Feb 2023 00:21:35 +0900 Subject: [PATCH 467/635] rust: io_pgtable: Add the Apple UAT format abstraction Signed-off-by: Asahi Lina --- rust/kernel/io_pgtable.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/kernel/io_pgtable.rs b/rust/kernel/io_pgtable.rs index ecd0f9e77946bc..4c90adefcd2442 100644 --- a/rust/kernel/io_pgtable.rs +++ b/rust/kernel/io_pgtable.rs @@ -335,6 +335,11 @@ iopt_cfg!( apple_dart_cfg, io_pgtable_cfg__bindgen_ty_1__bindgen_ty_5 ); +iopt_cfg!( + AppleUATCfg, + apple_uat_cfg, + io_pgtable_cfg__bindgen_ty_1__bindgen_ty_6 +); iopt_type!(ARM32LPAES1, ARMLPAES1Cfg, io_pgtable_fmt_ARM_32_LPAE_S1); iopt_type!(ARM32LPAES2, ARMLPAES2Cfg, io_pgtable_fmt_ARM_32_LPAE_S2); @@ -345,3 +350,4 @@ iopt_type!(ARMMaliLPAE, ARMMaliLPAECfg, io_pgtable_fmt_ARM_MALI_LPAE); iopt_type!(AMDIOMMUV1, (), io_pgtable_fmt_AMD_IOMMU_V1); iopt_type!(AppleDART, AppleDARTCfg, io_pgtable_fmt_APPLE_DART); iopt_type!(AppleDART2, AppleDARTCfg, io_pgtable_fmt_APPLE_DART2); +iopt_type!(AppleUAT, AppleUATCfg, io_pgtable_fmt_APPLE_UAT); From 1a8ae47cb37d469ec2bb6825706cd70656846d1c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 5 Feb 2023 20:42:15 +0900 Subject: [PATCH 468/635] rust: drm: ioctl: Add DRM ioctl abstraction DRM drivers need to be able to declare which driver-specific ioctls they support. This abstraction adds the required types and a helper macro to generate the ioctl definition inside the DRM driver. Note that this macro is not usable until further bits of the abstraction are in place (but it will not fail to compile on its own, if not called). Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/kernel/drm/ioctl.rs | 160 ++++++++++++++++++++++++++++++++ rust/kernel/drm/mod.rs | 5 + rust/kernel/lib.rs | 2 + rust/uapi/uapi_helper.h | 1 + 5 files changed, 169 insertions(+) create mode 100644 rust/kernel/drm/ioctl.rs create mode 100644 rust/kernel/drm/mod.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 614e5b88cac3a7..cb8ae62f68c3f6 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -6,6 +6,7 @@ * Sorted alphabetically. */ +#include #include #include #include diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs new file mode 100644 index 00000000000000..3f6a93cf009c51 --- /dev/null +++ b/rust/kernel/drm/ioctl.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +#![allow(non_snake_case)] + +//! DRM IOCTL definitions. +//! +//! C header: [`include/linux/drm/drm_ioctl.h`](../../../../include/linux/drm/drm_ioctl.h) + +use crate::ioctl; + +const BASE: u32 = bindings::DRM_IOCTL_BASE as u32; + +/// Construct a DRM ioctl number with no argument. +#[inline(always)] +pub const fn IO(nr: u32) -> u32 { + ioctl::_IO(BASE, nr) +} + +/// Construct a DRM ioctl number with a read-only argument. +#[inline(always)] +pub const fn IOR(nr: u32) -> u32 { + ioctl::_IOR::(BASE, nr) +} + +/// Construct a DRM ioctl number with a write-only argument. +#[inline(always)] +pub const fn IOW(nr: u32) -> u32 { + ioctl::_IOW::(BASE, nr) +} + +/// Construct a DRM ioctl number with a read-write argument. +#[inline(always)] +pub const fn IOWR(nr: u32) -> u32 { + ioctl::_IOWR::(BASE, nr) +} + +/// Descriptor type for DRM ioctls. Use the `declare_drm_ioctls!{}` macro to construct them. +pub type DrmIoctlDescriptor = bindings::drm_ioctl_desc; + +/// This is for ioctl which are used for rendering, and require that the file descriptor is either +/// for a render node, or if it’s a legacy/primary node, then it must be authenticated. +pub const AUTH: u32 = bindings::drm_ioctl_flags_DRM_AUTH; + +/// This must be set for any ioctl which can change the modeset or display state. Userspace must +/// call the ioctl through a primary node, while it is the active master. +/// +/// Note that read-only modeset ioctl can also be called by unauthenticated clients, or when a +/// master is not the currently active one. +pub const MASTER: u32 = bindings::drm_ioctl_flags_DRM_MASTER; + +/// Anything that could potentially wreak a master file descriptor needs to have this flag set. +/// +/// Current that’s only for the SETMASTER and DROPMASTER ioctl, which e.g. logind can call to force +/// a non-behaving master (display compositor) into compliance. +/// +/// This is equivalent to callers with the SYSADMIN capability. +pub const ROOT_ONLY: u32 = bindings::drm_ioctl_flags_DRM_ROOT_ONLY; + +/// Whether drm_ioctl_desc.func should be called with the DRM BKL held or not. Enforced as the +/// default for all modern drivers, hence there should never be a need to set this flag. +/// +/// Do not use anywhere else than for the VBLANK_WAIT IOCTL, which is the only legacy IOCTL which +/// needs this. +pub const UNLOCKED: u32 = bindings::drm_ioctl_flags_DRM_UNLOCKED; + +/// This is used for all ioctl needed for rendering only, for drivers which support render nodes. +/// This should be all new render drivers, and hence it should be always set for any ioctl with +/// `AUTH` set. Note though that read-only query ioctl might have this set, but have not set +/// DRM_AUTH because they do not require authentication. +pub const RENDER_ALLOW: u32 = bindings::drm_ioctl_flags_DRM_RENDER_ALLOW; + +/// Internal structures used by the [`declare_drm_ioctls!{}`] macro. Do not use directly. +#[doc(hidden)] +pub mod internal { + pub use bindings::drm_device; + pub use bindings::drm_file; + pub use bindings::drm_ioctl_desc; +} + +/// Declare the DRM ioctls for a driver. +/// +/// Each entry in the list should have the form: +/// +/// `(ioctl_number, argument_type, flags, user_callback),` +/// +/// `argument_type` is the type name within the `bindings` crate. +/// `user_callback` should have the following prototype: +/// +/// ``` +/// fn foo(device: &kernel::drm::device::Device, +/// data: &mut bindings::argument_type, +/// file: &kernel::drm::file::File, +/// ) +/// ``` +/// where `Self` is the drm::drv::Driver implementation these ioctls are being declared within. +/// +/// # Examples +/// +/// ``` +/// kernel::declare_drm_ioctls! { +/// (FOO_GET_PARAM, drm_foo_get_param, ioctl::RENDER_ALLOW, my_get_param_handler), +/// } +/// ``` +/// +#[macro_export] +macro_rules! declare_drm_ioctls { + ( $(($cmd:ident, $struct:ident, $flags:expr, $func:expr)),* $(,)? ) => { + const IOCTLS: &'static [$crate::drm::ioctl::DrmIoctlDescriptor] = { + use $crate::uapi::*; + const _:() = { + let i: u32 = $crate::uapi::DRM_COMMAND_BASE; + // Assert that all the IOCTLs are in the right order and there are no gaps, + // and that the sizeof of the specified type is correct. + $( + let cmd: u32 = $crate::macros::concat_idents!(DRM_IOCTL_, $cmd); + ::core::assert!(i == $crate::ioctl::_IOC_NR(cmd)); + ::core::assert!(core::mem::size_of::<$crate::uapi::$struct>() == $crate::ioctl::_IOC_SIZE(cmd)); + let i: u32 = i + 1; + )* + }; + + let ioctls = &[$( + $crate::drm::ioctl::internal::drm_ioctl_desc { + cmd: $crate::macros::concat_idents!(DRM_IOCTL_, $cmd) as u32, + func: { + #[allow(non_snake_case)] + unsafe extern "C" fn $cmd( + raw_dev: *mut $crate::drm::ioctl::internal::drm_device, + raw_data: *mut ::core::ffi::c_void, + raw_file_priv: *mut $crate::drm::ioctl::internal::drm_file, + ) -> core::ffi::c_int { + // SAFETY: We never drop this, and the DRM core ensures the device lives + // while callbacks are being called. + // + // FIXME: Currently there is nothing enforcing that the types of the + // dev/file match the current driver these ioctls are being declared + // for, and it's not clear how to enforce this within the type system. + let dev = ::core::mem::ManuallyDrop::new(unsafe { + $crate::drm::device::Device::from_raw(raw_dev) + }); + // SAFETY: This is just the ioctl argument, which hopefully has the right type + // (we've done our best checking the size). + let data = unsafe { &mut *(raw_data as *mut $crate::uapi::$struct) }; + // SAFETY: This is just the DRM file structure + let file = unsafe { $crate::drm::file::File::from_raw(raw_file_priv) }; + + match $func(&*dev, data, &file) { + Err(e) => e.to_errno(), + Ok(i) => i.try_into().unwrap_or(ERANGE.to_errno()), + } + } + Some($cmd) + }, + flags: $flags, + name: $crate::c_str!(::core::stringify!($cmd)).as_char_ptr(), + } + ),*]; + ioctls + }; + }; +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs new file mode 100644 index 00000000000000..9ec6d7cbcaf331 --- /dev/null +++ b/rust/kernel/drm/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM subsystem abstractions. + +pub mod ioctl; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 41b8185a339e33..f736d066c54505 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -41,6 +41,8 @@ mod build_assert; pub mod delay; pub mod device; pub mod driver; +#[cfg(CONFIG_DRM = "y")] +pub mod drm; pub mod error; pub mod init; pub mod io_buffer; diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 301f5207f0238d..ab8fb904c30aa5 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,3 +7,4 @@ */ #include +#include From 2566057a9ba20fc0f46567d3298a9666cb0f31bf Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 5 Feb 2023 20:53:47 +0900 Subject: [PATCH 469/635] rust: drm: Add Device and Driver abstractions Add the initial abstractions for DRM drivers and devices. These go together in one commit since they are fairly tightly coupled types. A few things have been stubbed out, to be implemented as further bits of the DRM subsystem are introduced. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 3 + rust/kernel/drm/device.rs | 76 +++++++ rust/kernel/drm/drv.rs | 339 ++++++++++++++++++++++++++++++++ rust/kernel/drm/mod.rs | 2 + 4 files changed, 420 insertions(+) create mode 100644 rust/kernel/drm/device.rs create mode 100644 rust/kernel/drm/drv.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index cb8ae62f68c3f6..7982f642f21b4d 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -6,10 +6,13 @@ * Sorted alphabetically. */ +#include +#include #include #include #include #include +#include #include #include #include diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs new file mode 100644 index 00000000000000..6007f941137a2c --- /dev/null +++ b/rust/kernel/drm/device.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM device. +//! +//! C header: [`include/linux/drm/drm_device.h`](../../../../include/linux/drm/drm_device.h) + +use crate::{bindings, device, drm, types::ForeignOwnable}; +use core::marker::PhantomData; + +/// Represents a reference to a DRM device. The device is reference-counted and is guaranteed to +/// not be dropped while this object is alive. +pub struct Device { + // Type invariant: ptr must be a valid and initialized drm_device, + // and this value must either own a reference to it or the caller + // must ensure that it is never dropped if the reference is borrowed. + pub(super) ptr: *mut bindings::drm_device, + _p: PhantomData, +} + +impl Device { + // Not intended to be called externally, except via declare_drm_ioctls!() + #[doc(hidden)] + pub unsafe fn from_raw(raw: *mut bindings::drm_device) -> Device { + Device { + ptr: raw, + _p: PhantomData, + } + } + + #[allow(dead_code)] + pub(crate) fn raw(&self) -> *const bindings::drm_device { + self.ptr + } + + pub(crate) fn raw_mut(&mut self) -> *mut bindings::drm_device { + self.ptr + } + + /// Returns a borrowed reference to the user data associated with this Device. + pub fn data(&self) -> ::Borrowed<'_> { + unsafe { T::Data::borrow((*self.ptr).dev_private) } + } +} + +impl Drop for Device { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to + // relinquish it now. + unsafe { bindings::drm_dev_put(self.ptr) }; + } +} + +impl Clone for Device { + fn clone(&self) -> Self { + // SAFETY: We get a new reference and then create a new owning object from the raw pointer + unsafe { + bindings::drm_dev_get(self.ptr); + Device::from_raw(self.ptr) + } + } +} + +// SAFETY: `Device` only holds a pointer to a C device, which is safe to be used from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` only holds a pointer to a C device, references to which are safe to be used +// from any thread. +unsafe impl Sync for Device {} + +// Make drm::Device work for dev_info!() and friends +unsafe impl device::RawDevice for Device { + fn raw_device(&self) -> *mut bindings::device { + // SAFETY: ptr must be valid per the type invariant + unsafe { (*self.ptr).dev } + } +} diff --git a/rust/kernel/drm/drv.rs b/rust/kernel/drm/drv.rs new file mode 100644 index 00000000000000..82c90b6e9a2c21 --- /dev/null +++ b/rust/kernel/drm/drv.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM driver core. +//! +//! C header: [`include/linux/drm/drm_drv.h`](../../../../include/linux/drm/drm_drv.h) + +use crate::{ + bindings, device, drm, + error::code::*, + error::from_err_ptr, + error::{Error, Result}, + prelude::*, + private::Sealed, + str::CStr, + types::ForeignOwnable, + ThisModule, +}; +use core::{ + marker::{PhantomData, PhantomPinned}, + pin::Pin, +}; +use macros::vtable; + +/// Driver use the GEM memory manager. This should be set for all modern drivers. +pub const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports mode setting interfaces (KMS). +pub const FEAT_MODESET: u32 = bindings::drm_driver_feature_DRIVER_MODESET; +/// Driver supports dedicated render nodes. +pub const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; +/// Driver supports the full atomic modesetting userspace API. +/// +/// Drivers which only use atomic internally, but do not support the full userspace API (e.g. not +/// all properties converted to atomic, or multi-plane updates are not guaranteed to be tear-free) +/// should not set this flag. +pub const FEAT_ATOMIC: u32 = bindings::drm_driver_feature_DRIVER_ATOMIC; +/// Driver supports DRM sync objects for explicit synchronization of command submission. +pub const FEAT_SYNCOBJ: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ; +/// Driver supports the timeline flavor of DRM sync objects for explicit synchronization of command +/// submission. +pub const FEAT_SYNCOBJ_TIMELINE: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ_TIMELINE; + +/// Information data for a DRM Driver. +pub struct DriverInfo { + /// Driver major version. + pub major: i32, + /// Driver minor version. + pub minor: i32, + /// Driver patchlevel version. + pub patchlevel: i32, + /// Driver name. + pub name: &'static CStr, + /// Driver description. + pub desc: &'static CStr, + /// Driver date. + pub date: &'static CStr, +} + +/// Internal memory management operation set, normally created by memory managers (e.g. GEM). +/// +/// See `kernel::drm::gem` and `kernel::drm::gem::shmem`. +pub struct AllocOps { + pub(crate) gem_create_object: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + size: usize, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) prime_handle_to_fd: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + file_priv: *mut bindings::drm_file, + handle: u32, + flags: u32, + prime_fd: *mut core::ffi::c_int, + ) -> core::ffi::c_int, + >, + pub(crate) prime_fd_to_handle: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + file_priv: *mut bindings::drm_file, + prime_fd: core::ffi::c_int, + handle: *mut u32, + ) -> core::ffi::c_int, + >, + pub(crate) gem_prime_import: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + dma_buf: *mut bindings::dma_buf, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) gem_prime_import_sg_table: Option< + unsafe extern "C" fn( + dev: *mut bindings::drm_device, + attach: *mut bindings::dma_buf_attachment, + sgt: *mut bindings::sg_table, + ) -> *mut bindings::drm_gem_object, + >, + pub(crate) gem_prime_mmap: Option< + unsafe extern "C" fn( + obj: *mut bindings::drm_gem_object, + vma: *mut bindings::vm_area_struct, + ) -> core::ffi::c_int, + >, + pub(crate) dumb_create: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + args: *mut bindings::drm_mode_create_dumb, + ) -> core::ffi::c_int, + >, + pub(crate) dumb_map_offset: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + handle: u32, + offset: *mut u64, + ) -> core::ffi::c_int, + >, + pub(crate) dumb_destroy: Option< + unsafe extern "C" fn( + file_priv: *mut bindings::drm_file, + dev: *mut bindings::drm_device, + handle: u32, + ) -> core::ffi::c_int, + >, +} + +/// Trait for memory manager implementations. Implemented internally. +pub trait AllocImpl: Sealed { + /// The C callback operations for this memory manager. + const ALLOC_OPS: AllocOps; +} + +/// A DRM driver implementation. +#[vtable] +pub trait Driver { + /// Context data associated with the DRM driver + /// + /// Determines the type of the context data passed to each of the methods of the trait. + type Data: ForeignOwnable + Sync + Send; + + /// The type used to manage memory for this driver. + /// + /// Should be either `drm::gem::Object` or `drm::gem::shmem::Object`. + type Object: AllocImpl; + + /// Driver metadata + const INFO: DriverInfo; + + /// Feature flags + const FEATURES: u32; + + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. + const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; +} + +/// A registration of a DRM device +/// +/// # Invariants: +/// +/// drm is always a valid pointer to an allocated drm_device +pub struct Registration { + drm: drm::device::Device, + registered: bool, + fops: bindings::file_operations, + vtable: Pin>, + _p: PhantomData, + _pin: PhantomPinned, +} + +#[cfg(CONFIG_DRM_LEGACY)] +macro_rules! drm_legacy_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::drm_driver { + $( $field: $val ),*, + firstopen: None, + preclose: None, + dma_ioctl: None, + dma_quiescent: None, + context_dtor: None, + irq_handler: None, + irq_preinstall: None, + irq_postinstall: None, + irq_uninstall: None, + get_vblank_counter: None, + enable_vblank: None, + disable_vblank: None, + dev_priv_size: 0, + } + } +} + +#[cfg(not(CONFIG_DRM_LEGACY))] +macro_rules! drm_legacy_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::drm_driver { + $( $field: $val ),* + } + } +} + +/// Registers a DRM device with the rest of the kernel. +/// +/// It automatically picks up THIS_MODULE. +#[allow(clippy::crate_in_macro_def)] +#[macro_export] +macro_rules! drm_device_register { + ($reg:expr, $data:expr, $flags:expr $(,)?) => {{ + $crate::drm::drv::Registration::register($reg, $data, $flags, &crate::THIS_MODULE) + }}; +} + +impl Registration { + const VTABLE: bindings::drm_driver = drm_legacy_fields! { + load: None, + open: None, // TODO: File abstraction + postclose: None, // TODO: File abstraction + lastclose: None, + unload: None, + release: None, + master_set: None, + master_drop: None, + debugfs_init: None, + gem_create_object: T::Object::ALLOC_OPS.gem_create_object, + prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, + prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, + gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, + gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, + gem_prime_mmap: T::Object::ALLOC_OPS.gem_prime_mmap, + dumb_create: T::Object::ALLOC_OPS.dumb_create, + dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, + dumb_destroy: T::Object::ALLOC_OPS.dumb_destroy, + + major: T::INFO.major, + minor: T::INFO.minor, + patchlevel: T::INFO.patchlevel, + name: T::INFO.name.as_char_ptr() as *mut _, + desc: T::INFO.desc.as_char_ptr() as *mut _, + date: T::INFO.date.as_char_ptr() as *mut _, + + driver_features: T::FEATURES, + ioctls: T::IOCTLS.as_ptr(), + num_ioctls: T::IOCTLS.len() as i32, + fops: core::ptr::null_mut(), + }; + + /// Creates a new [`Registration`] but does not register it yet. + /// + /// It is allowed to move. + pub fn new(parent: &dyn device::RawDevice) -> Result { + let vtable = Pin::new(Box::try_new(Self::VTABLE)?); + let raw_drm = unsafe { bindings::drm_dev_alloc(&*vtable, parent.raw_device()) }; + let raw_drm = from_err_ptr(raw_drm)?; + + // The reference count is one, and now we take ownership of that reference as a + // drm::device::Device. + let drm = unsafe { drm::device::Device::from_raw(raw_drm) }; + + Ok(Self { + drm, + registered: false, + vtable, + fops: Default::default(), // TODO: GEM abstraction + _pin: PhantomPinned, + _p: PhantomData, + }) + } + + /// Registers a DRM device with the rest of the kernel. + /// + /// Users are encouraged to use the [`drm_device_register!()`] macro because it automatically + /// picks up the current module. + pub fn register( + self: Pin<&mut Self>, + data: T::Data, + flags: usize, + module: &'static ThisModule, + ) -> Result { + if self.registered { + // Already registered. + return Err(EINVAL); + } + + // SAFETY: We never move out of `this`. + let this = unsafe { self.get_unchecked_mut() }; + let data_pointer = ::into_foreign(data); + // SAFETY: `drm` is valid per the type invariant + unsafe { + (*this.drm.raw_mut()).dev_private = data_pointer as *mut _; + } + + this.fops.owner = module.0; + this.vtable.fops = &this.fops; + + // SAFETY: The device is now initialized and ready to be registered. + let ret = unsafe { bindings::drm_dev_register(this.drm.raw_mut(), flags as u64) }; + if ret < 0 { + // SAFETY: `data_pointer` was returned by `into_foreign` above. + unsafe { T::Data::from_foreign(data_pointer) }; + return Err(Error::from_errno(ret)); + } + + this.registered = true; + Ok(()) + } + + /// Returns a reference to the `Device` instance for this registration. + pub fn device(&self) -> &drm::device::Device { + &self.drm + } +} + +// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between threads +// or CPUs, so it is safe to share it. +unsafe impl Sync for Registration {} + +// SAFETY: Registration with and unregistration from the drm subsystem can happen from any thread. +// Additionally, `T::Data` (which is dropped during unregistration) is `Send`, so it is ok to move +// `Registration` to different threads. +#[allow(clippy::non_send_fields_in_send_ty)] +unsafe impl Send for Registration {} + +impl Drop for Registration { + /// Removes the registration from the kernel if it has completed successfully before. + fn drop(&mut self) { + if self.registered { + // Get a pointer to the data stored in device before destroying it. + // SAFETY: `drm` is valid per the type invariant + let data_pointer = unsafe { (*self.drm.raw_mut()).dev_private }; + + // SAFETY: Since `registered` is true, `self.drm` is both valid and registered. + unsafe { bindings::drm_dev_unregister(self.drm.raw_mut()) }; + + // Free data as well. + // SAFETY: `data_pointer` was returned by `into_foreign` during registration. + unsafe { ::from_foreign(data_pointer) }; + } + } +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 9ec6d7cbcaf331..69376b3c6db99f 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -2,4 +2,6 @@ //! DRM subsystem abstractions. +pub mod device; +pub mod drv; pub mod ioctl; From 97aed2a0dd7de2742792b55badeb75916d23a401 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 5 Feb 2023 20:58:29 +0900 Subject: [PATCH 470/635] rust: drm: file: Add File abstraction A DRM File is the DRM counterpart to a kernel file structure, representing an open DRM file descriptor. Add a Rust abstraction to allow drivers to implement their own File types that implement the DriverFile trait. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/kernel/drm/drv.rs | 7 +- rust/kernel/drm/file.rs | 111 ++++++++++++++++++++++++++++++++ rust/kernel/drm/mod.rs | 1 + 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 rust/kernel/drm/file.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 7982f642f21b4d..5ae0befac4bba9 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/rust/kernel/drm/drv.rs b/rust/kernel/drm/drv.rs index 82c90b6e9a2c21..e41244ad60fa6f 100644 --- a/rust/kernel/drm/drv.rs +++ b/rust/kernel/drm/drv.rs @@ -144,6 +144,9 @@ pub trait Driver { /// Should be either `drm::gem::Object` or `drm::gem::shmem::Object`. type Object: AllocImpl; + /// The type used to represent a DRM File (client) + type File: drm::file::DriverFile; + /// Driver metadata const INFO: DriverInfo; @@ -213,8 +216,8 @@ macro_rules! drm_device_register { impl Registration { const VTABLE: bindings::drm_driver = drm_legacy_fields! { load: None, - open: None, // TODO: File abstraction - postclose: None, // TODO: File abstraction + open: Some(drm::file::open_callback::), + postclose: Some(drm::file::postclose_callback::), lastclose: None, unload: None, release: None, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs new file mode 100644 index 00000000000000..eb01f5833206d1 --- /dev/null +++ b/rust/kernel/drm/file.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM File objects. +//! +//! C header: [`include/linux/drm/drm_file.h`](../../../../include/linux/drm/drm_file.h) + +use crate::{bindings, drm, error::Result}; +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::pin::Pin; + +/// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). +pub trait DriverFile { + /// The parent `Driver` implementation for this `DriverFile`. + type Driver: drm::drv::Driver; + + /// Open a new file (called when a client opens the DRM device). + fn open(device: &drm::device::Device) -> Result>>; +} + +/// An open DRM File. +/// +/// # Invariants +/// `raw` is a valid pointer to a `drm_file` struct. +#[repr(transparent)] +pub struct File { + raw: *mut bindings::drm_file, + _p: PhantomData, +} + +pub(super) unsafe extern "C" fn open_callback( + raw_dev: *mut bindings::drm_device, + raw_file: *mut bindings::drm_file, +) -> core::ffi::c_int { + let drm = core::mem::ManuallyDrop::new(unsafe { drm::device::Device::from_raw(raw_dev) }); + // SAFETY: This reference won't escape this function + let file = unsafe { &mut *raw_file }; + + let inner = match T::open(&drm) { + Err(e) => { + return e.to_errno(); + } + Ok(i) => i, + }; + + // SAFETY: This pointer is treated as pinned, and the Drop guarantee is upheld below. + file.driver_priv = Box::into_raw(unsafe { Pin::into_inner_unchecked(inner) }) as *mut _; + + 0 +} + +pub(super) unsafe extern "C" fn postclose_callback( + _dev: *mut bindings::drm_device, + raw_file: *mut bindings::drm_file, +) { + // SAFETY: This reference won't escape this function + let file = unsafe { &*raw_file }; + + // Drop the DriverFile + unsafe { Box::from_raw(file.driver_priv as *mut T) }; +} + +impl File { + // Not intended to be called externally, except via declare_drm_ioctls!() + #[doc(hidden)] + pub unsafe fn from_raw(raw_file: *mut bindings::drm_file) -> File { + File { + raw: raw_file, + _p: PhantomData, + } + } + + #[allow(dead_code)] + /// Return the raw pointer to the underlying `drm_file`. + pub(super) fn raw(&self) -> *const bindings::drm_file { + self.raw + } + + /// Return an immutable reference to the raw `drm_file` structure. + pub(super) fn file(&self) -> &bindings::drm_file { + unsafe { &*self.raw } + } + + /// Return a pinned reference to the driver file structure. + pub fn inner(&self) -> Pin<&T> { + unsafe { Pin::new_unchecked(&*(self.file().driver_priv as *const T)) } + } +} + +impl crate::private::Sealed for File {} + +/// Generic trait to allow users that don't care about driver specifics to accept any File. +/// +/// # Safety +/// Must only be implemented for File and return the pointer, following the normal invariants +/// of that type. +pub unsafe trait GenericFile: crate::private::Sealed { + /// Returns the raw const pointer to the `struct drm_file` + fn raw(&self) -> *const bindings::drm_file; + /// Returns the raw mut pointer to the `struct drm_file` + fn raw_mut(&mut self) -> *mut bindings::drm_file; +} + +unsafe impl GenericFile for File { + fn raw(&self) -> *const bindings::drm_file { + self.raw + } + fn raw_mut(&mut self) -> *mut bindings::drm_file { + self.raw + } +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 69376b3c6db99f..a767942d0b5220 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -4,4 +4,5 @@ pub mod device; pub mod drv; +pub mod file; pub mod ioctl; From dd2db744e25b5e530de0a8859d92d9ff0cfbb30e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 5 Feb 2023 21:12:31 +0900 Subject: [PATCH 471/635] rust: drm: gem: Add GEM object abstraction The DRM GEM subsystem is the DRM memory management subsystem used by most modern drivers. Add a Rust abstraction to allow Rust DRM driver implementations to use it. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 23 ++ rust/kernel/drm/drv.rs | 4 +- rust/kernel/drm/gem/mod.rs | 371 ++++++++++++++++++++++++++++++++ rust/kernel/drm/mod.rs | 1 + 5 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 rust/kernel/drm/gem/mod.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 5ae0befac4bba9..69b4bebef3f0c8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index d101bde427352e..f7af1fe4077876 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -18,6 +18,7 @@ * accidentally exposed. */ +#include #include #include #include @@ -377,6 +378,28 @@ resource_size_t rust_helper_resource_size(const struct resource *res) } EXPORT_SYMBOL_GPL(rust_helper_resource_size); +#ifdef CONFIG_DRM + +void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) +{ + drm_gem_object_get(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_object_get); + +void rust_helper_drm_gem_object_put(struct drm_gem_object *obj) +{ + drm_gem_object_put(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_object_put); + +__u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) +{ + return drm_vma_node_offset_addr(node); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_vma_node_offset_addr); + +#endif + /* * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type * as the Rust `usize` type, so we can use it in contexts where Rust diff --git a/rust/kernel/drm/drv.rs b/rust/kernel/drm/drv.rs index e41244ad60fa6f..c9a2f68605a95f 100644 --- a/rust/kernel/drm/drv.rs +++ b/rust/kernel/drm/drv.rs @@ -126,7 +126,7 @@ pub struct AllocOps { } /// Trait for memory manager implementations. Implemented internally. -pub trait AllocImpl: Sealed { +pub trait AllocImpl: Sealed + drm::gem::IntoGEMObject { /// The C callback operations for this memory manager. const ALLOC_OPS: AllocOps; } @@ -263,7 +263,7 @@ impl Registration { drm, registered: false, vtable, - fops: Default::default(), // TODO: GEM abstraction + fops: drm::gem::create_fops(), _pin: PhantomPinned, _p: PhantomData, }) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs new file mode 100644 index 00000000000000..c8756c171253ff --- /dev/null +++ b/rust/kernel/drm/gem/mod.rs @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM GEM API +//! +//! C header: [`include/linux/drm/drm_gem.h`](../../../../include/linux/drm/drm_gem.h) + +use alloc::boxed::Box; + +use crate::{ + bindings, + drm::{device, drv, file}, + error::{to_result, Result}, + prelude::*, +}; +use core::{mem, mem::ManuallyDrop, ops::Deref, ops::DerefMut}; + +/// GEM object functions, which must be implemented by drivers. +pub trait BaseDriverObject: Sync + Send + Sized { + /// Create a new driver data object for a GEM object of a given size. + fn new(dev: &device::Device, size: usize) -> Result; + + /// Open a new handle to an existing object, associated with a File. + fn open( + _obj: &<::Driver as drv::Driver>::Object, + _file: &file::File<<::Driver as drv::Driver>::File>, + ) -> Result { + Ok(()) + } + + /// Close a handle to an existing object, associated with a File. + fn close( + _obj: &<::Driver as drv::Driver>::Object, + _file: &file::File<<::Driver as drv::Driver>::File>, + ) { + } +} + +/// Trait that represents a GEM object subtype +pub trait IntoGEMObject: Sized + crate::private::Sealed { + /// Owning driver for this type + type Driver: drv::Driver; + + /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as + /// this owning object is valid. + fn gem_obj(&self) -> &bindings::drm_gem_object; + + /// Converts a pointer to a `drm_gem_object` into a pointer to this type. + fn from_gem_obj(obj: *mut bindings::drm_gem_object) -> *mut Self; +} + +/// Trait which must be implemented by drivers using base GEM objects. +pub trait DriverObject: BaseDriverObject> { + /// Parent `Driver` for this object. + type Driver: drv::Driver; +} + +unsafe extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: All of our objects are Object. + let this = crate::container_of!(obj, Object, obj) as *mut Object; + + // SAFETY: The pointer we got has to be valid + unsafe { bindings::drm_gem_object_release(obj) }; + + // SAFETY: All of our objects are allocated via Box<>, and we're in the + // free callback which guarantees this object has zero remaining references, + // so we can drop it + unsafe { Box::from_raw(this) }; +} + +unsafe extern "C" fn open_callback, U: BaseObject>( + raw_obj: *mut bindings::drm_gem_object, + raw_file: *mut bindings::drm_file, +) -> core::ffi::c_int { + // SAFETY: The pointer we got has to be valid. + let file = unsafe { + file::File::<<::Driver as drv::Driver>::File>::from_raw(raw_file) + }; + let obj = + <<::Driver as drv::Driver>::Object as IntoGEMObject>::from_gem_obj( + raw_obj, + ); + + // SAFETY: from_gem_obj() returns a valid pointer as long as the type is + // correct and the raw_obj we got is valid. + match T::open(unsafe { &*obj }, &file) { + Err(e) => e.to_errno(), + Ok(()) => 0, + } +} + +unsafe extern "C" fn close_callback, U: BaseObject>( + raw_obj: *mut bindings::drm_gem_object, + raw_file: *mut bindings::drm_file, +) { + // SAFETY: The pointer we got has to be valid. + let file = unsafe { + file::File::<<::Driver as drv::Driver>::File>::from_raw(raw_file) + }; + let obj = + <<::Driver as drv::Driver>::Object as IntoGEMObject>::from_gem_obj( + raw_obj, + ); + + // SAFETY: from_gem_obj() returns a valid pointer as long as the type is + // correct and the raw_obj we got is valid. + T::close(unsafe { &*obj }, &file); +} + +impl IntoGEMObject for Object { + type Driver = T::Driver; + + fn gem_obj(&self) -> &bindings::drm_gem_object { + &self.obj + } + + fn from_gem_obj(obj: *mut bindings::drm_gem_object) -> *mut Object { + crate::container_of!(obj, Object, obj) as *mut Object + } +} + +/// Base operations shared by all GEM object classes +pub trait BaseObject: IntoGEMObject { + /// Returns the size of the object in bytes. + fn size(&self) -> usize { + self.gem_obj().size + } + + /// Creates a new reference to the object. + fn reference(&self) -> ObjectRef { + // SAFETY: Having a reference to an Object implies holding a GEM reference + unsafe { + bindings::drm_gem_object_get(self.gem_obj() as *const _ as *mut _); + } + ObjectRef { + ptr: self as *const _, + } + } + + /// Creates a new handle for the object associated with a given `File` + /// (or returns an existing one). + fn create_handle( + &self, + file: &file::File<<::Driver as drv::Driver>::File>, + ) -> Result { + let mut handle: u32 = 0; + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_handle_create( + file.raw() as *mut _, + self.gem_obj() as *const _ as *mut _, + &mut handle, + ) + })?; + Ok(handle) + } + + /// Looks up an object by its handle for a given `File`. + fn lookup_handle( + file: &file::File<<::Driver as drv::Driver>::File>, + handle: u32, + ) -> Result> { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_gem_object_lookup(file.raw() as *mut _, handle) }; + + if ptr.is_null() { + Err(ENOENT) + } else { + Ok(ObjectRef { + ptr: ptr as *const _, + }) + } + } + + /// Creates an mmap offset to map the object from userspace. + fn create_mmap_offset(&self) -> Result { + // SAFETY: The arguments are valid per the type invariant. + to_result(unsafe { + // TODO: is this threadsafe? + bindings::drm_gem_create_mmap_offset(self.gem_obj() as *const _ as *mut _) + })?; + Ok(unsafe { + bindings::drm_vma_node_offset_addr(&self.gem_obj().vma_node as *const _ as *mut _) + }) + } +} + +impl BaseObject for T {} + +/// A base GEM object. +#[repr(C)] +pub struct Object { + obj: bindings::drm_gem_object, + // The DRM core ensures the Device exists as long as its objects exist, so we don't need to + // manage the reference count here. + dev: ManuallyDrop>, + inner: T, +} + +impl Object { + /// The size of this object's structure. + pub const SIZE: usize = mem::size_of::(); + + const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(free_callback::), + open: Some(open_callback::>), + close: Some(close_callback::>), + print_info: None, + export: None, + pin: None, + unpin: None, + get_sg_table: None, + vmap: None, + vunmap: None, + mmap: None, + vm_ops: core::ptr::null_mut(), + }; + + /// Create a new GEM object. + pub fn new(dev: &device::Device, size: usize) -> Result> { + let mut obj: Box = Box::try_new(Self { + // SAFETY: This struct is expected to be zero-initialized + obj: unsafe { mem::zeroed() }, + // SAFETY: The drm subsystem guarantees that the drm_device will live as long as + // the GEM object lives, so we can conjure a reference out of thin air. + dev: ManuallyDrop::new(unsafe { device::Device::from_raw(dev.ptr) }), + inner: T::new(dev, size)?, + })?; + + obj.obj.funcs = &Self::OBJECT_FUNCS; + to_result(unsafe { + bindings::drm_gem_object_init(dev.raw() as *mut _, &mut obj.obj, size) + })?; + + let obj_ref = UniqueObjectRef { + ptr: Box::leak(obj), + }; + + Ok(obj_ref) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + &self.dev + } +} + +impl crate::private::Sealed for Object {} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl drv::AllocImpl for Object { + const ALLOC_OPS: drv::AllocOps = drv::AllocOps { + gem_create_object: None, + prime_handle_to_fd: Some(bindings::drm_gem_prime_handle_to_fd), + prime_fd_to_handle: Some(bindings::drm_gem_prime_fd_to_handle), + gem_prime_import: None, + gem_prime_import_sg_table: None, + gem_prime_mmap: Some(bindings::drm_gem_prime_mmap), + dumb_create: None, + dumb_map_offset: None, + dumb_destroy: None, + }; +} + +/// A reference-counted shared reference to a base GEM object. +pub struct ObjectRef { + // Invariant: the pointer is valid and initialized, and this ObjectRef owns a reference to it. + ptr: *const T, +} + +/// SAFETY: GEM object references are safe to share between threads. +unsafe impl Send for ObjectRef {} +unsafe impl Sync for ObjectRef {} + +impl Clone for ObjectRef { + fn clone(&self) -> Self { + self.reference() + } +} + +impl Drop for ObjectRef { + fn drop(&mut self) { + // SAFETY: Having an ObjectRef implies holding a GEM reference. + // The free callback will take care of deallocation. + unsafe { + bindings::drm_gem_object_put((*self.ptr).gem_obj() as *const _ as *mut _); + } + } +} + +impl Deref for ObjectRef { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The pointer is valid per the invariant + unsafe { &*self.ptr } + } +} + +/// A unique reference to a base GEM object. +pub struct UniqueObjectRef { + // Invariant: the pointer is valid and initialized, and this ObjectRef owns the only reference + // to it. + ptr: *mut T, +} + +impl UniqueObjectRef { + /// Downgrade this reference to a shared reference. + pub fn into_ref(self) -> ObjectRef { + let ptr = self.ptr as *const _; + core::mem::forget(self); + + ObjectRef { ptr } + } +} + +impl Drop for UniqueObjectRef { + fn drop(&mut self) { + // SAFETY: Having a UniqueObjectRef implies holding a GEM + // reference. The free callback will take care of deallocation. + unsafe { + bindings::drm_gem_object_put((*self.ptr).gem_obj() as *const _ as *mut _); + } + } +} + +impl Deref for UniqueObjectRef { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The pointer is valid per the invariant + unsafe { &*self.ptr } + } +} + +impl DerefMut for UniqueObjectRef { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The pointer is valid per the invariant + unsafe { &mut *self.ptr } + } +} + +pub(super) fn create_fops() -> bindings::file_operations { + bindings::file_operations { + owner: core::ptr::null_mut(), + open: Some(bindings::drm_open), + release: Some(bindings::drm_release), + unlocked_ioctl: Some(bindings::drm_ioctl), + #[cfg(CONFIG_COMPAT)] + compat_ioctl: Some(bindings::drm_compat_ioctl), + #[cfg(not(CONFIG_COMPAT))] + compat_ioctl: None, + poll: Some(bindings::drm_poll), + read: Some(bindings::drm_read), + llseek: Some(bindings::noop_llseek), + mmap: Some(bindings::drm_gem_mmap), + ..Default::default() + } +} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index a767942d0b5220..c44760a1332fa1 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -5,4 +5,5 @@ pub mod device; pub mod drv; pub mod file; +pub mod gem; pub mod ioctl; From 88fdb95383ebb562f2943cc453a143315fe34235 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 7 Sep 2022 17:34:35 +0900 Subject: [PATCH 472/635] drm/gem-shmem: Export VM ops functions There doesn't seem to be a way for the Rust bindings to get a compile-time constant reference to drm_gem_shmem_vm_ops, so we need to duplicate that structure in Rust... this isn't nice... Signed-off-by: Asahi Lina --- drivers/gpu/drm/drm_gem_shmem_helper.c | 9 ++++++--- include/drm/drm_gem_shmem_helper.h | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 2b2163c8138ef4..4d3a54434c2862 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -534,7 +534,7 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); -static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) +vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; @@ -563,8 +563,9 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) return ret; } +EXPORT_SYMBOL_GPL(drm_gem_shmem_fault); -static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) +void drm_gem_shmem_vm_open(struct vm_area_struct *vma) { struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); @@ -585,8 +586,9 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) drm_gem_vm_open(vma); } +EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_open); -static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) +void drm_gem_shmem_vm_close(struct vm_area_struct *vma) { struct drm_gem_object *obj = vma->vm_private_data; struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); @@ -594,6 +596,7 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) drm_gem_shmem_put_pages(shmem); drm_gem_vm_close(vma); } +EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_close); const struct vm_operations_struct drm_gem_shmem_vm_ops = { .fault = drm_gem_shmem_fault, diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index a2201b2488c56c..b9f349b3ed76c2 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -138,6 +138,9 @@ void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); extern const struct vm_operations_struct drm_gem_shmem_vm_ops; +vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf); +void drm_gem_shmem_vm_open(struct vm_area_struct *vma); +void drm_gem_shmem_vm_close(struct vm_area_struct *vma); /* * GEM object functions From 90f73f2907a4fbd97d8f203fd4bd9f65577aac68 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sun, 5 Feb 2023 21:18:25 +0900 Subject: [PATCH 473/635] rust: drm: gem: shmem: Add DRM shmem helper abstraction The DRM shmem helper includes common code useful for drivers which allocate GEM objects as anonymous shmem. Add a Rust abstraction for this. Drivers can choose the raw GEM implementation or the shmem layer, depending on their needs. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 2 + rust/helpers.c | 67 ++++++ rust/kernel/drm/gem/mod.rs | 3 + rust/kernel/drm/gem/shmem.rs | 383 ++++++++++++++++++++++++++++++++ 4 files changed, 455 insertions(+) create mode 100644 rust/kernel/drm/gem/shmem.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 69b4bebef3f0c8..5145afefa5f886 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -10,12 +10,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index f7af1fe4077876..e9c9d2d20e0c1b 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -19,6 +19,7 @@ */ #include +#include #include #include #include @@ -378,6 +379,18 @@ resource_size_t rust_helper_resource_size(const struct resource *res) } EXPORT_SYMBOL_GPL(rust_helper_resource_size); +dma_addr_t rust_helper_sg_dma_address(const struct scatterlist *sg) +{ + return sg_dma_address(sg); +} +EXPORT_SYMBOL_GPL(rust_helper_sg_dma_address); + +int rust_helper_sg_dma_len(const struct scatterlist *sg) +{ + return sg_dma_len(sg); +} +EXPORT_SYMBOL_GPL(rust_helper_sg_dma_len); + #ifdef CONFIG_DRM void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) @@ -398,6 +411,60 @@ __u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) } EXPORT_SYMBOL_GPL(rust_helper_drm_vma_node_offset_addr); +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER + +void rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_free); + +void rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_print_info); + +int rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_pin); + +void rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_unpin); + +struct sg_table *rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_get_sg_table); + +int rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_vmap); + +void rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_vunmap); + +int rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_gem_shmem_object_mmap); + +#endif #endif /* diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index c8756c171253ff..1805fa123f033e 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -4,6 +4,9 @@ //! //! C header: [`include/linux/drm/drm_gem.h`](../../../../include/linux/drm/drm_gem.h) +#[cfg(CONFIG_DRM_GEM_SHMEM_HELPER = "y")] +pub mod shmem; + use alloc::boxed::Box; use crate::{ diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 00000000000000..dafc08d1eecb99 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](../../../../include/linux/drm/drm_gem_shmem_helper.h) + +use crate::drm::{device, drv, gem}; +use crate::{ + error::{from_err_ptr, to_result}, + prelude::*, +}; +use core::{ + marker::PhantomData, + mem, + mem::{ManuallyDrop, MaybeUninit}, + ops::{Deref, DerefMut}, + ptr::addr_of_mut, + slice, +}; + +use gem::BaseObject; + +/// Trait which must be implemented by drivers using shmem-backed GEM objects. +pub trait DriverObject: gem::BaseDriverObject> { + /// Parent `Driver` for this object. + type Driver: drv::Driver; +} + +// FIXME: This is terrible and I don't know how to avoid it +#[cfg(CONFIG_NUMA)] +macro_rules! vm_numa_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::vm_operations_struct { + $( $field: $val ),*, + set_policy: None, + get_policy: None, + } + } +} + +#[cfg(not(CONFIG_NUMA))] +macro_rules! vm_numa_fields { + ( $($field:ident: $val:expr),* $(,)? ) => { + bindings::vm_operations_struct { + $( $field: $val ),* + } + } +} + +const SHMEM_VM_OPS: bindings::vm_operations_struct = vm_numa_fields! { + open: Some(bindings::drm_gem_shmem_vm_open), + close: Some(bindings::drm_gem_shmem_vm_close), + may_split: None, + mremap: None, + mprotect: None, + fault: Some(bindings::drm_gem_shmem_fault), + huge_fault: None, + map_pages: None, + pagesize: None, + page_mkwrite: None, + pfn_mkwrite: None, + access: None, + name: None, + find_special_page: None, +}; + +/// A shmem-backed GEM object. +#[repr(C)] +pub struct Object { + obj: bindings::drm_gem_shmem_object, + // The DRM core ensures the Device exists as long as its objects exist, so we don't need to + // manage the reference count here. + dev: ManuallyDrop>, + inner: T, +} + +unsafe extern "C" fn gem_create_object( + raw_dev: *mut bindings::drm_device, + size: usize, +) -> *mut bindings::drm_gem_object { + // SAFETY: GEM ensures the device lives as long as its objects live, + // so we can conjure up a reference from thin air and never drop it. + let dev = ManuallyDrop::new(unsafe { device::Device::from_raw(raw_dev) }); + + let inner = match T::new(&*dev, size) { + Ok(v) => v, + Err(e) => return e.to_ptr(), + }; + + let p = unsafe { + bindings::krealloc( + core::ptr::null(), + Object::::SIZE, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + ) as *mut Object + }; + + if p.is_null() { + return ENOMEM.to_ptr(); + } + + // SAFETY: p is valid as long as the alloc succeeded + unsafe { + addr_of_mut!((*p).dev).write(dev); + addr_of_mut!((*p).inner).write(inner); + } + + // SAFETY: drm_gem_shmem_object is safe to zero-init, and + // the rest of Object has been initialized + let new: &mut Object = unsafe { &mut *(p as *mut _) }; + + new.obj.base.funcs = &Object::::VTABLE; + &mut new.obj.base +} + +unsafe extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: All of our objects are Object. + let shmem = crate::container_of!(obj, bindings::drm_gem_shmem_object, base) + as *mut bindings::drm_gem_shmem_object; + let p = crate::container_of!(shmem, Object, obj) as *mut Object; + + // SAFETY: p is never used after this + unsafe { + core::ptr::drop_in_place(&mut (*p).inner); + } + + // SAFETY: This pointer has to be valid, since p is valid + unsafe { + bindings::drm_gem_shmem_free(&mut (*p).obj); + } +} + +impl Object { + /// The size of this object's structure. + const SIZE: usize = mem::size_of::(); + + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(free_callback::), + open: Some(super::open_callback::>), + close: Some(super::close_callback::>), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: None, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + vm_ops: &SHMEM_VM_OPS, + }; + + // SAFETY: Must only be used with DRM functions that are thread-safe + unsafe fn mut_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + &self.obj as *const _ as *mut _ + } + + /// Create a new shmem-backed DRM object of the given size. + pub fn new(dev: &device::Device, size: usize) -> Result> { + // SAFETY: This function can be called as long as the ALLOC_OPS are set properly + // for this driver, and the gem_create_object is called. + let p = unsafe { bindings::drm_gem_shmem_create(dev.raw() as *mut _, size) }; + let p = crate::container_of!(p, Object, obj) as *mut _; + + // SAFETY: The gem_create_object callback ensures this is a valid Object, + // so we can take a unique reference to it. + let obj_ref = gem::UniqueObjectRef { ptr: p }; + + Ok(obj_ref) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + &self.dev + } + + /// Creates (if necessary) and returns a scatter-gather table of DMA pages for this object. + /// + /// This will pin the object in memory. + pub fn sg_table(&self) -> Result> { + // SAFETY: drm_gem_shmem_get_pages_sgt is thread-safe. + let sgt = from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.mut_shmem()) })?; + + Ok(SGTable { + sgt, + _owner: self.reference(), + }) + } + + /// Creates and returns a virtual kernel memory mapping for this object. + pub fn vmap(&self) -> Result> { + let mut map: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: drm_gem_shmem_vmap is thread-safe + to_result(unsafe { bindings::drm_gem_shmem_vmap(self.mut_shmem(), map.as_mut_ptr()) })?; + + // SAFETY: if drm_gem_shmem_vmap did not fail, map is initialized now + let map = unsafe { map.assume_init() }; + + Ok(VMap { + map, + owner: self.reference(), + }) + } + + /// Set the write-combine flag for this object. + /// + /// Should be called before any mappings are made. + pub fn set_wc(&mut self, map_wc: bool) { + unsafe { (*self.mut_shmem()).map_wc = map_wc }; + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl crate::private::Sealed for Object {} + +impl gem::IntoGEMObject for Object { + type Driver = T::Driver; + + fn gem_obj(&self) -> &bindings::drm_gem_object { + &self.obj.base + } + + fn from_gem_obj(obj: *mut bindings::drm_gem_object) -> *mut Object { + let shmem = crate::container_of!(obj, bindings::drm_gem_shmem_object, base) + as *mut bindings::drm_gem_shmem_object; + crate::container_of!(shmem, Object, obj) as *mut Object + } +} + +impl drv::AllocImpl for Object { + const ALLOC_OPS: drv::AllocOps = drv::AllocOps { + gem_create_object: Some(gem_create_object::), + prime_handle_to_fd: Some(bindings::drm_gem_prime_handle_to_fd), + prime_fd_to_handle: Some(bindings::drm_gem_prime_fd_to_handle), + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + gem_prime_mmap: Some(bindings::drm_gem_prime_mmap), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + dumb_destroy: None, + }; +} + +/// A virtual mapping for a shmem-backed GEM object in kernel address space. +pub struct VMap { + map: bindings::iosys_map, + owner: gem::ObjectRef>, +} + +impl VMap { + /// Returns a const raw pointer to the start of the mapping. + pub fn as_ptr(&self) -> *const core::ffi::c_void { + // SAFETY: The shmem helpers always return non-iomem maps + unsafe { self.map.__bindgen_anon_1.vaddr } + } + + /// Returns a mutable raw pointer to the start of the mapping. + pub fn as_mut_ptr(&mut self) -> *mut core::ffi::c_void { + // SAFETY: The shmem helpers always return non-iomem maps + unsafe { self.map.__bindgen_anon_1.vaddr } + } + + /// Returns a byte slice view of the mapping. + pub fn as_slice(&self) -> &[u8] { + // SAFETY: The vmap maps valid memory up to the owner size + unsafe { slice::from_raw_parts(self.as_ptr() as *const u8, self.owner.size()) } + } + + /// Returns mutable a byte slice view of the mapping. + pub fn as_mut_slice(&mut self) -> &mut [u8] { + // SAFETY: The vmap maps valid memory up to the owner size + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr() as *mut u8, self.owner.size()) } + } + + /// Borrows a reference to the object that owns this virtual mapping. + pub fn owner(&self) -> &gem::ObjectRef> { + &self.owner + } +} + +impl Drop for VMap { + fn drop(&mut self) { + // SAFETY: This function is thread-safe + unsafe { + bindings::drm_gem_shmem_vunmap(self.owner.mut_shmem(), &mut self.map); + } + } +} + +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Send for VMap {} +unsafe impl Sync for VMap {} + +/// A single scatter-gather entry, representing a span of pages in the device's DMA address space. +/// +/// For devices not behind a standalone IOMMU, this corresponds to physical addresses. +#[repr(transparent)] +pub struct SGEntry(bindings::scatterlist); + +impl SGEntry { + /// Returns the starting DMA address of this span + pub fn dma_address(&self) -> usize { + (unsafe { bindings::sg_dma_address(&self.0) }) as usize + } + + /// Returns the length of this span in bytes + pub fn dma_len(&self) -> usize { + (unsafe { bindings::sg_dma_len(&self.0) }) as usize + } +} + +/// A scatter-gather table of DMA address spans for a GEM shmem object. +/// +/// # Invariants +/// `sgt` must be a valid pointer to the `sg_table`, which must correspond to the owned +/// object in `_owner` (which ensures it remains valid). +pub struct SGTable { + sgt: *const bindings::sg_table, + _owner: gem::ObjectRef>, +} + +impl SGTable { + /// Returns an iterator through the SGTable's entries + pub fn iter(&'_ self) -> SGTableIter<'_> { + SGTableIter { + left: unsafe { (*self.sgt).nents } as usize, + sg: unsafe { (*self.sgt).sgl }, + _p: PhantomData, + } + } +} + +impl<'a, T: DriverObject> IntoIterator for &'a SGTable { + type Item = &'a SGEntry; + type IntoIter = SGTableIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// SAFETY: `sg_table` objects are safe to send across threads. +unsafe impl Send for SGTable {} +unsafe impl Sync for SGTable {} + +/// An iterator through `SGTable` entries. +/// +/// # Invariants +/// `sg` must be a valid pointer to the scatterlist, which must outlive our lifetime. +pub struct SGTableIter<'a> { + sg: *mut bindings::scatterlist, + left: usize, + _p: PhantomData<&'a ()>, +} + +impl<'a> Iterator for SGTableIter<'a> { + type Item = &'a SGEntry; + + fn next(&mut self) -> Option { + if self.left == 0 { + None + } else { + let sg = self.sg; + self.sg = unsafe { bindings::sg_next(self.sg) }; + self.left -= 1; + Some(unsafe { &(*(sg as *const SGEntry)) }) + } + } +} From 4654f6bdac9fe528ba3e4e50ec7a20fc5990f3ad Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 16:50:51 +0900 Subject: [PATCH 474/635] rust: drm: mm: Add DRM MM Range Allocator abstraction drm_mm provides a simple range allocator, useful for managing virtual address ranges. Add a Rust abstraction to expose this module to Rust drivers. Signed-off-by: Asahi Lina --- rust/kernel/drm/mm.rs | 309 +++++++++++++++++++++++++++++++++++++++++ rust/kernel/drm/mod.rs | 1 + 2 files changed, 310 insertions(+) create mode 100644 rust/kernel/drm/mm.rs diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs new file mode 100644 index 00000000000000..97e948becd6e1e --- /dev/null +++ b/rust/kernel/drm/mm.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM MM range allocator +//! +//! C header: [`include/linux/drm/drm_mm.h`](../../../../include/linux/drm/drm_mm.h) + +use crate::{ + bindings, + error::{to_result, Result}, + str::CStr, + sync::{Arc, LockClassKey, Mutex, UniqueArc}, + types::Opaque, +}; + +use crate::init::InPlaceInit; +use alloc::boxed::Box; + +use core::{ + marker::{PhantomData, PhantomPinned}, + ops::Deref, + pin::Pin, +}; + +/// Type alias representing a DRM MM node. +pub type Node = Pin>>; + +/// Trait which must be implemented by the inner allocator state type provided by the user. +pub trait AllocInner { + /// Notification that a node was dropped from the allocator. + fn drop_object(&mut self, _start: u64, _size: u64, _color: usize, _object: &mut T) {} +} + +impl AllocInner for () {} + +/// Wrapper type for a `struct drm_mm` plus user AllocInner object. +/// +/// # Invariants +/// The `drm_mm` struct is valid and initialized. +struct MmInner, T>(Opaque, A, PhantomData); + +/// Represents a single allocated node in the MM allocator +pub struct NodeData, T> { + node: bindings::drm_mm_node, + mm: Arc>>, + valid: bool, + /// A drm_mm_node needs to be pinned because nodes reference each other in a linked list. + _pin: PhantomPinned, + inner: T, +} + +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Send> Send for NodeData {} +unsafe impl, T: Sync> Sync for NodeData {} + +/// Available MM node insertion modes +#[repr(u32)] +pub enum InsertMode { + /// Search for the smallest hole (within the search range) that fits the desired node. + /// + /// Allocates the node from the bottom of the found hole. + Best = bindings::drm_mm_insert_mode_DRM_MM_INSERT_BEST, + + /// Search for the lowest hole (address closest to 0, within the search range) that fits the + /// desired node. + /// + /// Allocates the node from the bottom of the found hole. + Low = bindings::drm_mm_insert_mode_DRM_MM_INSERT_LOW, + + /// Search for the highest hole (address closest to U64_MAX, within the search range) that fits + /// the desired node. + /// + /// Allocates the node from the top of the found hole. The specified alignment for the node is + /// applied to the base of the node (`Node.start()`). + High = bindings::drm_mm_insert_mode_DRM_MM_INSERT_HIGH, + + /// Search for the most recently evicted hole (within the search range) that fits the desired + /// node. This is appropriate for use immediately after performing an eviction scan and removing + /// the selected nodes to form a hole. + /// + /// Allocates the node from the bottom of the found hole. + Evict = bindings::drm_mm_insert_mode_DRM_MM_INSERT_EVICT, +} + +/// A clonable, interlocked reference to the allocator state. +/// +/// This is useful to perform actions on the user-supplied `AllocInner` type given just a Node, +/// without immediately taking the lock. +#[derive(Clone)] +pub struct InnerRef, T>(Arc>>); + +impl, T> InnerRef { + /// Operate on the user `AllocInner` implementation, taking the lock. + pub fn with(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.0.lock(); + cb(&mut l.1) + } +} + +impl, T> NodeData { + /// Returns the color of the node (an opaque value) + pub fn color(&self) -> usize { + self.node.color as usize + } + + /// Returns the start address of the node + pub fn start(&self) -> u64 { + self.node.start + } + + /// Returns the size of the node in bytes + pub fn size(&self) -> u64 { + self.node.size + } + + /// Operate on the user `AllocInner` implementation associated with this node's allocator. + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.mm.lock(); + cb(&mut l.1) + } + + /// Return a clonable, detached reference to the allocator inner data. + pub fn alloc_ref(&self) -> InnerRef { + InnerRef(self.mm.clone()) + } + + /// Return a mutable reference to the inner data. + pub fn inner_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: This is okay because inner is not structural + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl, T> Deref for NodeData { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl, T> Drop for NodeData { + fn drop(&mut self) { + if self.valid { + let mut guard = self.mm.lock(); + + // Inform the user allocator that a node is being dropped. + guard + .1 + .drop_object(self.start(), self.size(), self.color(), &mut self.inner); + // SAFETY: The MM lock is still taken, so we can safely remove the node. + unsafe { bindings::drm_mm_remove_node(&mut self.node) }; + } + } +} + +/// An instance of a DRM MM range allocator. +pub struct Allocator, T> { + mm: Arc>>, + _p: PhantomData, +} + +impl, T> Allocator { + /// Create a new range allocator for the given start and size range of addresses. + /// + /// The user may optionally provide an inner object representing allocator state, which will + /// be protected by the same lock. If not required, `()` can be used. + pub fn new( + start: u64, + size: u64, + inner: A, + name: &'static CStr, + lock_key: &'static LockClassKey, + ) -> Result> { + // SAFETY: We call `Mutex::init_lock` below. + let mm = UniqueArc::pin_init(Mutex::new( + MmInner(Opaque::uninit(), inner, PhantomData), + name, + lock_key, + ))?; + + unsafe { + // SAFETY: The Opaque instance provides a valid pointer, and it is initialized after + // this call. + bindings::drm_mm_init(mm.lock().0.get(), start, size); + } + + Ok(Allocator { + mm: mm.into(), + _p: PhantomData, + }) + } + + /// Insert a new node into the allocator of a given size. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node(&mut self, node: T, size: u64) -> Result> { + self.insert_node_generic(node, size, 0, 0, InsertMode::Best) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, and insertion mode. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node_generic( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + mode: InsertMode, + ) -> Result> { + self.insert_node_in_range(node, size, alignment, color, 0, u64::MAX, mode) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, insertion mode, and sub-range to allocate from. + /// + /// `node` is the user `T` type data to store into the node. + #[allow(clippy::too_many_arguments)] + pub fn insert_node_in_range( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + start: u64, + end: u64, + mode: InsertMode, + ) -> Result> { + let mut mm_node = Box::try_new(NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + })?; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { + bindings::drm_mm_insert_node_in_range( + guard.0.get(), + &mut mm_node.node, + size, + alignment, + color as core::ffi::c_ulong, + start, + end, + mode as u32, + ) + })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Insert a node into the allocator at a fixed start address. + /// + /// `node` is the user `T` type data to store into the node. + pub fn reserve_node( + &mut self, + node: T, + start: u64, + size: u64, + color: usize, + ) -> Result> { + let mut mm_node = Box::try_new(NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + })?; + + mm_node.node.start = start; + mm_node.node.size = size; + mm_node.node.color = color as core::ffi::c_ulong; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { bindings::drm_mm_reserve_node(guard.0.get(), &mut mm_node.node) })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Operate on the inner user type `A`, taking the allocator lock + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut guard = self.mm.lock(); + cb(&mut guard.1) + } +} + +impl, T> Drop for MmInner { + fn drop(&mut self) { + // SAFETY: If the MmInner is dropped then all nodes are gone (since they hold references), + // so it is safe to tear down the allocator. + unsafe { + bindings::drm_mm_takedown(self.0.get()); + } + } +} + +// MmInner is safely Send if the AllocInner user type is Send. +unsafe impl, T> Send for MmInner {} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index c44760a1332fa1..73fab2dee3af95 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -7,3 +7,4 @@ pub mod drv; pub mod file; pub mod gem; pub mod ioctl; +pub mod mm; From 29b0bf0eed1a1011e729d34508c82aa3fb406500 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 16:56:21 +0900 Subject: [PATCH 475/635] rust: dma_fence: Add DMA Fence abstraction DMA fences are the internal synchronization primitive used for DMA operations like GPU rendering, video en/decoding, etc. Add an abstraction to allow Rust drivers to interact with this subsystem. Note: This uses a raw spinlock living next to the fence, since we do not interact with it other than for initialization. TODO: Expose this to the user at some point with a safe abstraction. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 4 +- rust/helpers.c | 36 +++ rust/kernel/dma_fence.rs | 532 ++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 2 + 4 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 rust/kernel/dma_fence.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 5145afefa5f886..624665ca6060d0 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -14,10 +14,12 @@ #include #include #include +#include +#include #include #include -#include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index e9c9d2d20e0c1b..2c6a9f5182a95f 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -391,6 +393,40 @@ int rust_helper_sg_dma_len(const struct scatterlist *sg) } EXPORT_SYMBOL_GPL(rust_helper_sg_dma_len); +#ifdef CONFIG_DMA_SHARED_BUFFER + +void rust_helper_dma_fence_get(struct dma_fence *fence) +{ + dma_fence_get(fence); +} +EXPORT_SYMBOL_GPL(rust_helper_dma_fence_get); + +void rust_helper_dma_fence_put(struct dma_fence *fence) +{ + dma_fence_put(fence); +} +EXPORT_SYMBOL_GPL(rust_helper_dma_fence_put); + +struct dma_fence_chain *rust_helper_dma_fence_chain_alloc(void) +{ + return dma_fence_chain_alloc(); +} +EXPORT_SYMBOL_GPL(rust_helper_dma_fence_chain_alloc); + +void rust_helper_dma_fence_chain_free(struct dma_fence_chain *chain) +{ + dma_fence_chain_free(chain); +} +EXPORT_SYMBOL_GPL(rust_helper_dma_fence_chain_free); + +void rust_helper_dma_fence_set_error(struct dma_fence *fence, int error) +{ + dma_fence_set_error(fence, error); +} +EXPORT_SYMBOL_GPL(rust_helper_dma_fence_set_error); + +#endif + #ifdef CONFIG_DRM void rust_helper_drm_gem_object_get(struct drm_gem_object *obj) diff --git a/rust/kernel/dma_fence.rs b/rust/kernel/dma_fence.rs new file mode 100644 index 00000000000000..b67a67ed2d8a1f --- /dev/null +++ b/rust/kernel/dma_fence.rs @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA fence abstraction. +//! +//! C header: [`include/linux/dma_fence.h`](../../include/linux/dma_fence.h) + +use crate::{ + bindings, + error::{to_result, Result}, + prelude::*, + sync::LockClassKey, + types::Opaque, +}; +use core::fmt::Write; +use core::ops::{Deref, DerefMut}; +use core::ptr::addr_of_mut; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// Any kind of DMA Fence Object +/// +/// # Invariants +/// raw() returns a valid pointer to a dma_fence and we own a reference to it. +pub trait RawDmaFence: crate::private::Sealed { + /// Returns the raw `struct dma_fence` pointer. + fn raw(&self) -> *mut bindings::dma_fence; + + /// Returns the raw `struct dma_fence` pointer and consumes the object. + /// + /// The caller is responsible for dropping the reference. + fn into_raw(self) -> *mut bindings::dma_fence + where + Self: Sized, + { + let ptr = self.raw(); + core::mem::forget(self); + ptr + } + + /// Advances this fence to the chain node which will signal this sequence number. + /// If no sequence number is provided, this returns `self` again. + fn chain_find_seqno(self, seqno: u64) -> Result + where + Self: Sized, + { + let mut ptr = self.into_raw(); + + // SAFETY: This will safely fail if this DmaFence is not a chain. + // `ptr` is valid per the type invariant. + let ret = unsafe { bindings::dma_fence_chain_find_seqno(&mut ptr, seqno) }; + + if ret != 0 { + // SAFETY: This is either an owned reference or NULL, dma_fence_put can handle both. + unsafe { bindings::dma_fence_put(ptr) }; + Err(Error::from_errno(ret)) + } else if ptr.is_null() { + Err(EINVAL) // When can this happen? + } else { + // SAFETY: ptr is valid and non-NULL as checked above. + Ok(unsafe { Fence::from_raw(ptr) }) + } + } + + /// Signal completion of this fence + fn signal(&self) -> Result { + to_result(unsafe { bindings::dma_fence_signal(self.raw()) }) + } + + /// Set the error flag on this fence + fn set_error(&self, err: Error) { + unsafe { bindings::dma_fence_set_error(self.raw(), err.to_errno()) }; + } +} + +/// A generic DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +pub struct Fence { + ptr: *mut bindings::dma_fence, +} + +impl Fence { + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// The caller must own a reference to the dma_fence, which is transferred to the new object. + pub(crate) unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Fence { + Fence { ptr } + } + + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// Takes a borrowed reference to the dma_fence, and increments the reference count. + pub(crate) unsafe fn get_raw(ptr: *mut bindings::dma_fence) -> Fence { + // SAFETY: Pointer is valid per the safety contract + unsafe { bindings::dma_fence_get(ptr) }; + Fence { ptr } + } + + /// Create a new Fence object from a RawDmaFence. + pub fn from_fence(fence: &dyn RawDmaFence) -> Fence { + // SAFETY: Pointer is valid per the RawDmaFence contract + unsafe { Self::get_raw(fence.raw()) } + } +} + +impl crate::private::Sealed for Fence {} + +impl RawDmaFence for Fence { + fn raw(&self) -> *mut bindings::dma_fence { + self.ptr + } +} + +impl Drop for Fence { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::dma_fence_put(self.ptr) }; + } +} + +impl Clone for Fence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.ptr); + Self::from_raw(self.ptr) + } + } +} + +unsafe impl Sync for Fence {} +unsafe impl Send for Fence {} + +/// Trait which must be implemented by driver-specific fence objects. +#[vtable] +pub trait FenceOps: Sized + Send + Sync { + /// True if this dma_fence implementation uses 64bit seqno, false otherwise. + const USE_64BIT_SEQNO: bool; + + /// Returns the driver name. This is a callback to allow drivers to compute the name at + /// runtime, without having it to store permanently for each fence, or build a cache of + /// some sort. + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Return the name of the context this fence belongs to. This is a callback to allow drivers + /// to compute the name at runtime, without having it to store permanently for each fence, or + /// build a cache of some sort. + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Enable software signaling of fence. + fn enable_signaling(self: &FenceObject) -> bool { + false + } + + /// Peek whether the fence is signaled, as a fastpath optimization for e.g. dma_fence_wait() or + /// dma_fence_add_callback(). + fn signaled(self: &FenceObject) -> bool { + false + } + + /// Callback to fill in free-form debug info specific to this fence, like the sequence number. + fn fence_value_str(self: &FenceObject, _output: &mut dyn Write) {} + + /// Fills in the current value of the timeline as a string, like the sequence number. Note that + /// the specific fence passed to this function should not matter, drivers should only use it to + /// look up the corresponding timeline structures. + fn timeline_value_str(self: &FenceObject, _output: &mut dyn Write) {} +} + +unsafe extern "C" fn get_driver_name_cb( + fence: *mut bindings::dma_fence, +) -> *const core::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_driver_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn get_timeline_name_cb( + fence: *mut bindings::dma_fence, +) -> *const core::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_timeline_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn enable_signaling_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::enable_signaling(unsafe { &mut *p }) +} + +unsafe extern "C" fn signaled_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::signaled(unsafe { &mut *p }) +} + +unsafe extern "C" fn release_cb(fence: *mut bindings::dma_fence) { + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: p is never used after this + unsafe { + core::ptr::drop_in_place(&mut (*p).inner); + } + + // SAFETY: All of our fences are allocated using kmalloc, so this is safe. + unsafe { bindings::dma_fence_free(fence) }; +} + +unsafe extern "C" fn fence_value_str_cb( + fence: *mut bindings::dma_fence, + string: *mut core::ffi::c_char, + size: core::ffi::c_int, +) { + let size: usize = size.try_into().unwrap_or(0); + + if size == 0 { + return; + } + + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: The caller is responsible for the validity of string/size + let mut f = unsafe { crate::str::Formatter::from_buffer(string as *mut _, size) }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::fence_value_str(unsafe { &mut *p }, &mut f); + let _ = f.write_str("\0"); + + // SAFETY: `size` is at least 1 per the check above + unsafe { *string.add(size - 1) = 0 }; +} + +unsafe extern "C" fn timeline_value_str_cb( + fence: *mut bindings::dma_fence, + string: *mut core::ffi::c_char, + size: core::ffi::c_int, +) { + let size: usize = size.try_into().unwrap_or(0); + + if size == 0 { + return; + } + + // SAFETY: All of our fences are FenceObject. + let p = crate::container_of!(fence, FenceObject, fence) as *mut FenceObject; + + // SAFETY: The caller is responsible for the validity of string/size + let mut f = unsafe { crate::str::Formatter::from_buffer(string as *mut _, size) }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::timeline_value_str(unsafe { &mut *p }, &mut f); + let _ = f.write_str("\0"); + + // SAFETY: `size` is at least 1 per the check above + unsafe { *string.add(size - 1) = 0 }; +} + +// Allow FenceObject to be used as a self argument, for ergonomics +impl core::ops::Receiver for FenceObject {} + +/// A driver-specific DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +#[repr(C)] +pub struct FenceObject { + fence: bindings::dma_fence, + lock: Opaque, + inner: T, +} + +impl FenceObject { + const SIZE: usize = core::mem::size_of::(); + + const VTABLE: bindings::dma_fence_ops = bindings::dma_fence_ops { + use_64bit_seqno: T::USE_64BIT_SEQNO, + get_driver_name: Some(get_driver_name_cb::), + get_timeline_name: Some(get_timeline_name_cb::), + enable_signaling: if T::HAS_ENABLE_SIGNALING { + Some(enable_signaling_cb::) + } else { + None + }, + signaled: if T::HAS_SIGNALED { + Some(signaled_cb::) + } else { + None + }, + wait: None, // Deprecated + release: Some(release_cb::), + fence_value_str: if T::HAS_FENCE_VALUE_STR { + Some(fence_value_str_cb::) + } else { + None + }, + timeline_value_str: if T::HAS_TIMELINE_VALUE_STR { + Some(timeline_value_str_cb::) + } else { + None + }, + }; +} + +impl Deref for FenceObject { + type Target = T; + + fn deref(&self) -> &T { + &self.inner + } +} + +impl DerefMut for FenceObject { + fn deref_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +impl crate::private::Sealed for FenceObject {} +impl RawDmaFence for FenceObject { + fn raw(&self) -> *mut bindings::dma_fence { + &self.fence as *const _ as *mut _ + } +} + +/// A unique reference to a driver-specific fence object +pub struct UniqueFence(*mut FenceObject); + +impl Deref for UniqueFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + unsafe { &*self.0 } + } +} + +impl DerefMut for UniqueFence { + fn deref_mut(&mut self) -> &mut FenceObject { + unsafe { &mut *self.0 } + } +} + +impl crate::private::Sealed for UniqueFence {} +impl RawDmaFence for UniqueFence { + fn raw(&self) -> *mut bindings::dma_fence { + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl From> for UserFence { + fn from(value: UniqueFence) -> Self { + let ptr = value.0; + core::mem::forget(value); + + UserFence(ptr) + } +} + +impl Drop for UniqueFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +unsafe impl Sync for UniqueFence {} +unsafe impl Send for UniqueFence {} + +/// A shared reference to a driver-specific fence object +pub struct UserFence(*mut FenceObject); + +impl Deref for UserFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + unsafe { &*self.0 } + } +} + +impl Clone for UserFence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.raw()); + Self(self.0) + } + } +} + +impl crate::private::Sealed for UserFence {} +impl RawDmaFence for UserFence { + fn raw(&self) -> *mut bindings::dma_fence { + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl Drop for UserFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +unsafe impl Sync for UserFence {} +unsafe impl Send for UserFence {} + +/// An array of fence contexts, out of which fences can be created. +pub struct FenceContexts { + start: u64, + count: u32, + seqnos: Vec, + lock_name: &'static CStr, + lock_key: &'static LockClassKey, +} + +impl FenceContexts { + /// Create a new set of fence contexts. + pub fn new( + count: u32, + name: &'static CStr, + key: &'static LockClassKey, + ) -> Result { + let mut seqnos: Vec = Vec::new(); + + seqnos.try_reserve(count as usize)?; + + for _ in 0..count { + seqnos.try_push(Default::default())?; + } + + let start = unsafe { bindings::dma_fence_context_alloc(count as core::ffi::c_uint) }; + + Ok(FenceContexts { + start, + count, + seqnos, + lock_name: name, + lock_key: key, + }) + } + + /// Create a new fence in a given context index. + pub fn new_fence(&self, context: u32, inner: T) -> Result> { + if context > self.count { + return Err(EINVAL); + } + + let p = unsafe { + bindings::krealloc( + core::ptr::null_mut(), + FenceObject::::SIZE, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + ) as *mut FenceObject + }; + + if p.is_null() { + return Err(ENOMEM); + } + + let seqno = self.seqnos[context as usize].fetch_add(1, Ordering::Relaxed); + + // SAFETY: The pointer is valid, so pointers to members are too. + // After this, all fields are initialized. + unsafe { + addr_of_mut!((*p).inner).write(inner); + bindings::__spin_lock_init( + addr_of_mut!((*p).lock) as *mut _, + self.lock_name.as_char_ptr(), + self.lock_key.as_ptr(), + ); + bindings::dma_fence_init( + addr_of_mut!((*p).fence), + &FenceObject::::VTABLE, + addr_of_mut!((*p).lock) as *mut _, + self.start + context as u64, + seqno, + ); + }; + + Ok(UniqueFence(p)) + } +} + +/// A DMA Fence Chain Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence_chain which we own. +pub struct FenceChain { + ptr: *mut bindings::dma_fence_chain, +} + +impl FenceChain { + /// Create a new DmaFenceChain object. + pub fn new() -> Result { + // SAFETY: This function is safe to call and takes no arguments. + let ptr = unsafe { bindings::dma_fence_chain_alloc() }; + + if ptr.is_null() { + Err(ENOMEM) + } else { + Ok(FenceChain { ptr }) + } + } + + /// Convert the DmaFenceChain into the underlying raw pointer. + /// + /// This assumes the caller will take ownership of the object. + pub(crate) fn into_raw(self) -> *mut bindings::dma_fence_chain { + let ptr = self.ptr; + core::mem::forget(self); + ptr + } +} + +impl Drop for FenceChain { + fn drop(&mut self) { + // SAFETY: We own this dma_fence_chain. + unsafe { bindings::dma_fence_chain_free(self.ptr) }; + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index f736d066c54505..84f89e3845dafb 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -40,6 +40,8 @@ mod allocator; mod build_assert; pub mod delay; pub mod device; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] +pub mod dma_fence; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; From 7f7a8652765a811b4ec43bca582608dc1d51dcd2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 16:59:20 +0900 Subject: [PATCH 476/635] rust: drm: syncobj: Add DRM Sync Object abstraction DRM Sync Objects are a container for a DMA fence, and can be waited on signaled, exported, and imported from userspace. Add a Rust abstraction so Rust DRM drivers can support this functionality. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 19 ++++++++ rust/kernel/drm/mod.rs | 1 + rust/kernel/drm/syncobj.rs | 77 +++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 rust/kernel/drm/syncobj.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 624665ca6060d0..9ed48416f77735 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 2c6a9f5182a95f..7b4321a22f06f7 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -447,6 +448,24 @@ __u64 rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) } EXPORT_SYMBOL_GPL(rust_helper_drm_vma_node_offset_addr); +void rust_helper_drm_syncobj_get(struct drm_syncobj *obj) +{ + drm_syncobj_get(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_syncobj_get); + +void rust_helper_drm_syncobj_put(struct drm_syncobj *obj) +{ + drm_syncobj_put(obj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_syncobj_put); + +struct dma_fence *rust_helper_drm_syncobj_fence_get(struct drm_syncobj *syncobj) +{ + return drm_syncobj_fence_get(syncobj); +} +EXPORT_SYMBOL_GPL(rust_helper_drm_syncobj_fence_get); + #ifdef CONFIG_DRM_GEM_SHMEM_HELPER void rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 73fab2dee3af95..dae98826edfd1d 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -8,3 +8,4 @@ pub mod file; pub mod gem; pub mod ioctl; pub mod mm; +pub mod syncobj; diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs new file mode 100644 index 00000000000000..10eed05eb27ad5 --- /dev/null +++ b/rust/kernel/drm/syncobj.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/linux/drm/drm_syncobj.h`](../../../../include/linux/drm/drm_syncobj.h) + +use crate::{bindings, dma_fence::*, drm, error::Result, prelude::*}; + +/// A DRM Sync Object +/// +/// # Invariants +/// ptr is a valid pointer to a drm_syncobj and we own a reference to it. +pub struct SyncObj { + ptr: *mut bindings::drm_syncobj, +} + +impl SyncObj { + /// Looks up a sync object by its handle for a given `File`. + pub fn lookup_handle(file: &impl drm::file::GenericFile, handle: u32) -> Result { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_syncobj_find(file.raw() as *mut _, handle) }; + + if ptr.is_null() { + Err(ENOENT) + } else { + Ok(SyncObj { ptr }) + } + } + + /// Returns the DMA fence associated with this sync object, if any. + pub fn fence_get(&self) -> Option { + let fence = unsafe { bindings::drm_syncobj_fence_get(self.ptr) }; + if fence.is_null() { + None + } else { + // SAFETY: The pointer is non-NULL and drm_syncobj_fence_get acquired an + // additional reference. + Some(unsafe { Fence::from_raw(fence) }) + } + } + + /// Replaces the DMA fence with a new one, or removes it if fence is None. + pub fn replace_fence(&self, fence: Option<&Fence>) { + unsafe { + bindings::drm_syncobj_replace_fence( + self.ptr, + fence.map_or(core::ptr::null_mut(), |a| a.raw()), + ) + }; + } + + /// Adds a new timeline point to the syncobj. + pub fn add_point(&self, chain: FenceChain, fence: &Fence, point: u64) { + // SAFETY: All arguments should be valid per the respective type invariants. + // This takes over the FenceChain ownership. + unsafe { bindings::drm_syncobj_add_point(self.ptr, chain.into_raw(), fence.raw(), point) }; + } +} + +impl Drop for SyncObj { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::drm_syncobj_put(self.ptr) }; + } +} + +impl Clone for SyncObj { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { bindings::drm_syncobj_get(self.ptr) }; + SyncObj { ptr: self.ptr } + } +} + +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Sync for SyncObj {} +unsafe impl Send for SyncObj {} From 0f9b490f0ee4dd6c8f312a9682c432c56dd6d935 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 17:03:15 +0900 Subject: [PATCH 477/635] drm/sched: Add can_run_job callback Some hardware may require more complex resource utilization accounting than the simple job count supported by drm_sched internally. Add a can_run_job callback to allow drivers to implement more logic before deciding whether to run a GPU job. Signed-off-by: Asahi Lina --- drivers/gpu/drm/scheduler/sched_main.c | 10 ++++++++++ include/drm/gpu_scheduler.h | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 1e08cc5a17029e..3cf591946c53dd 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -996,6 +996,16 @@ static int drm_sched_main(void *param) if (!entity) continue; + if (sched->ops->can_run_job) { + sched_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue)); + if (!sched_job) { + complete_all(&entity->entity_idle); + continue; + } + if (!sched->ops->can_run_job(sched_job)) + continue; + } + sched_job = drm_sched_entity_pop_job(entity); if (!sched_job) { diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 9935d1e2ff69a3..6d558538fc626b 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -389,6 +389,14 @@ struct drm_sched_backend_ops { struct dma_fence *(*prepare_job)(struct drm_sched_job *sched_job, struct drm_sched_entity *s_entity); + /** + * @can_run_job: Called before job execution to check whether the + * hardware is free enough to run the job. This can be used to + * implement more complex hardware resource policies than the + * hw_submission limit. + */ + bool (*can_run_job)(struct drm_sched_job *sched_job); + /** * @run_job: Called to execute the job once all of the dependencies * have been resolved. This may be called multiple times, if From 8fdbfe94a601ce3c03cbee63bc8888d2ac9938ea Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 17:08:36 +0900 Subject: [PATCH 478/635] rust: drm: sched: Add GPU scheduler abstraction The GPU scheduler manages scheduling GPU jobs and dependencies between them. This Rust abstraction allows Rust DRM drivers to use this functionality. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 6 + rust/kernel/drm/mod.rs | 1 + rust/kernel/drm/sched.rs | 358 ++++++++++++++++++++++++++++++++ 4 files changed, 366 insertions(+) create mode 100644 rust/kernel/drm/sched.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 9ed48416f77735..a752ca815b7e63 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 7b4321a22f06f7..0983d09cc8abc7 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -394,6 +394,12 @@ int rust_helper_sg_dma_len(const struct scatterlist *sg) } EXPORT_SYMBOL_GPL(rust_helper_sg_dma_len); +unsigned long rust_helper_msecs_to_jiffies(const unsigned int m) +{ + return msecs_to_jiffies(m); +} +EXPORT_SYMBOL_GPL(rust_helper_msecs_to_jiffies); + #ifdef CONFIG_DMA_SHARED_BUFFER void rust_helper_dma_fence_get(struct dma_fence *fence) diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index dae98826edfd1d..b1f182453ec1dc 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -8,4 +8,5 @@ pub mod file; pub mod gem; pub mod ioctl; pub mod mm; +pub mod sched; pub mod syncobj; diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs new file mode 100644 index 00000000000000..a5275cc1617921 --- /dev/null +++ b/rust/kernel/drm/sched.rs @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Scheduler +//! +//! C header: [`include/linux/drm/gpu_scheduler.h`](../../../../include/linux/drm/gpu_scheduler.h) + +use crate::{ + bindings, device, + dma_fence::*, + error::{to_result, Result}, + prelude::*, + sync::{Arc, UniqueArc}, +}; +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::ptr::addr_of_mut; + +/// Scheduler status after timeout recovery +#[repr(u32)] +pub enum Status { + /// Device recovered from the timeout and can execute jobs again + Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NOMINAL, + /// Device is no longer available + NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, +} + +/// Scheduler priorities +#[repr(i32)] +pub enum Priority { + /// Low userspace priority + Min = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_MIN, + /// Normal userspace priority + Normal = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_NORMAL, + /// High userspace priority + High = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_HIGH, + /// Kernel priority (highest) + Kernel = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_KERNEL, +} + +/// Trait to be implemented by driver job objects. +pub trait JobImpl: Sized { + /// Called when the scheduler is considering scheduling this job next, to get another Fence + /// for this job to block on. Once it returns None, run() may be called. + fn prepare(_job: &mut Job) -> Option { + None // Equivalent to NULL function pointer + } + + /// Called before job execution to check whether the hardware is free enough to run the job. + /// This can be used to implement more complex hardware resource policies than the hw_submission + /// limit. + fn can_run(_job: &mut Job) -> bool { + true + } + + /// Called to execute the job once all of the dependencies have been resolved. This may be + /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides + /// to try it again. + fn run(job: &mut Job) -> Result>; + + /// Called when a job has taken too long to execute, to trigger GPU recovery. + /// + /// This method is called in a workqueue context. + fn timed_out(job: &mut Job) -> Status; +} + +unsafe extern "C" fn prepare_job_cb( + sched_job: *mut bindings::drm_sched_job, + _s_entity: *mut bindings::drm_sched_entity, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = crate::container_of!(sched_job, Job, job) as *mut Job; + + match T::prepare(unsafe { &mut *p }) { + None => core::ptr::null_mut(), + Some(fence) => fence.into_raw(), + } +} + +unsafe extern "C" fn run_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = crate::container_of!(sched_job, Job, job) as *mut Job; + + match T::run(unsafe { &mut *p }) { + Err(e) => e.to_ptr(), + Ok(None) => core::ptr::null_mut(), + Ok(Some(fence)) => fence.into_raw(), + } +} + +unsafe extern "C" fn can_run_job_cb(sched_job: *mut bindings::drm_sched_job) -> bool { + // SAFETY: All of our jobs are Job. + let p = crate::container_of!(sched_job, Job, job) as *mut Job; + + T::can_run(unsafe { &mut *p }) +} + +unsafe extern "C" fn timedout_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> bindings::drm_gpu_sched_stat { + // SAFETY: All of our jobs are Job. + let p = crate::container_of!(sched_job, Job, job) as *mut Job; + + T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat +} + +unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = crate::container_of!(sched_job, Job, job) as *mut Job; + + // Convert the job back to a Box and drop it + // SAFETY: All of our Jobs are created inside a box. + unsafe { Box::from_raw(p) }; +} + +/// A DRM scheduler job. +pub struct Job { + job: bindings::drm_sched_job, + inner: T, +} + +impl Deref for Job { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Job { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Drop for Job { + fn drop(&mut self) { + // SAFETY: At this point the job has either been submitted and this is being called from + // `free_job_cb` above, or it hasn't and it is safe to call `drm_sched_job_cleanup`. + unsafe { bindings::drm_sched_job_cleanup(&mut self.job) }; + } +} + +/// A pending DRM scheduler job (not yet armed) +pub struct PendingJob<'a, T: JobImpl>(Box>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> PendingJob<'a, T> { + /// Add a fence as a dependency to the job + pub fn add_dependency(&mut self, fence: Fence) -> Result { + to_result(unsafe { + bindings::drm_sched_job_add_dependency(&mut self.0.job, fence.into_raw()) + }) + } + + /// Arm the job to make it ready for execution + pub fn arm(mut self) -> ArmedJob<'a, T> { + unsafe { bindings::drm_sched_job_arm(&mut self.0.job) }; + ArmedJob(self.0, PhantomData) + } +} + +impl<'a, T: JobImpl> Deref for PendingJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for PendingJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// An armed DRM scheduler job (not yet submitted) +pub struct ArmedJob<'a, T: JobImpl>(Box>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> ArmedJob<'a, T> { + /// Returns the job fences + pub fn fences(&self) -> JobFences<'_> { + JobFences(unsafe { &mut *self.0.job.s_fence }) + } + + /// Push the job for execution into the scheduler + pub fn push(self) { + // After this point, the job is submitted and owned by the scheduler + let ptr = match self { + ArmedJob(job, _) => Box::>::into_raw(job), + }; + + // SAFETY: We are passing in ownership of a valid Box raw pointer. + unsafe { bindings::drm_sched_entity_push_job(addr_of_mut!((*ptr).job)) }; + } +} +impl<'a, T: JobImpl> Deref for ArmedJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for ArmedJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Reference to the bundle of fences attached to a DRM scheduler job +pub struct JobFences<'a>(&'a mut bindings::drm_sched_fence); + +impl<'a> JobFences<'a> { + /// Returns a new reference to the job scheduled fence. + pub fn scheduled(&mut self) -> Fence { + unsafe { Fence::get_raw(&mut self.0.scheduled) } + } + + /// Returns a new reference to the job finished fence. + pub fn finished(&mut self) -> Fence { + unsafe { Fence::get_raw(&mut self.0.finished) } + } +} + +struct EntityInner { + entity: bindings::drm_sched_entity, + // TODO: Allow users to share guilty flag between entities + sched: Arc>, + guilty: bindings::atomic_t, + _p: PhantomData, +} + +impl Drop for EntityInner { + fn drop(&mut self) { + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for EntityInner {} +unsafe impl Send for EntityInner {} + +/// A DRM scheduler entity. +pub struct Entity(Pin>>); + +impl Entity { + /// Create a new scheduler entity. + pub fn new(sched: &Scheduler, priority: Priority) -> Result { + let mut entity: Box>> = Box::try_new_zeroed()?; + + let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + + // SAFETY: The Box is allocated above and valid. + unsafe { + bindings::drm_sched_entity_init( + addr_of_mut!((*entity.as_mut_ptr()).entity), + priority as _, + &mut sched_ptr, + 1, + addr_of_mut!((*entity.as_mut_ptr()).guilty), + ) + }; + + // SAFETY: The Box is allocated above and valid. + unsafe { addr_of_mut!((*entity.as_mut_ptr()).sched).write(sched.0.clone()) }; + + // SAFETY: entity is now initialized. + Ok(Self(Pin::from(unsafe { entity.assume_init() }))) + } + + /// Create a new job on this entity. + /// + /// The entity must outlive the pending job until it transitions into the submitted state, + /// after which the scheduler owns it. + pub fn new_job(&self, inner: T) -> Result> { + let mut job: Box>> = Box::try_new_zeroed()?; + + // SAFETY: We hold a reference to the entity (which is a valid pointer), + // and the job object was just allocated above. + to_result(unsafe { + bindings::drm_sched_job_init( + addr_of_mut!((*job.as_mut_ptr()).job), + &self.0.as_ref().get_ref().entity as *const _ as *mut _, + core::ptr::null_mut(), + ) + })?; + + // SAFETY: The Box pointer is valid, and this initializes the inner member. + unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + + // SAFETY: All fields of the Job are now initialized. + Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) + } +} + +/// DRM scheduler inner data +pub struct SchedulerInner { + sched: bindings::drm_gpu_scheduler, + _p: PhantomData, +} + +impl Drop for SchedulerInner { + fn drop(&mut self) { + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of + // freeing all in-progress jobs. + unsafe { bindings::drm_sched_fini(&mut self.sched) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +unsafe impl Send for SchedulerInner {} + +/// A DRM Scheduler +pub struct Scheduler(Arc>); + +impl Scheduler { + const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { + prepare_job: Some(prepare_job_cb::), + can_run_job: Some(can_run_job_cb::), + run_job: Some(run_job_cb::), + timedout_job: Some(timedout_job_cb::), + free_job: Some(free_job_cb::), + }; + /// Creates a new DRM Scheduler object + // TODO: Shared timeout workqueues & scores + pub fn new( + device: &impl device::RawDevice, + hw_submission: u32, + hang_limit: u32, + timeout_ms: usize, + name: &'static CStr, + ) -> Result> { + let mut sched: UniqueArc>> = UniqueArc::try_new_uninit()?; + + // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + &Self::OPS, + hw_submission, + hang_limit, + bindings::msecs_to_jiffies(timeout_ms.try_into()?).try_into()?, + core::ptr::null_mut(), + core::ptr::null_mut(), + name.as_char_ptr(), + device.raw_device(), + ) + })?; + + // SAFETY: All fields of SchedulerInner are now initialized. + Ok(Scheduler(unsafe { sched.assume_init() }.into())) + } +} From d0acf471efc9f61611fb163ab274aff34bda9b34 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 10 Feb 2023 15:31:42 +0900 Subject: [PATCH 479/635] drm/gem: Add a flag to control whether objects can be exported Drivers may want to support driver-private objects, which cannot be shared. This allows them to share a single lock and enables other optimizations. Add an `exportable` field to drm_gem_object, which blocks PRIME export if set to false. It is initialized to true in drm_gem_private_object_init. Signed-off-by: Asahi Lina --- drivers/gpu/drm/drm_gem.c | 1 + drivers/gpu/drm/drm_prime.c | 5 +++++ include/drm/drm_gem.h | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index a5d392f7e11f6a..e47ceecb706ea1 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -166,6 +166,7 @@ void drm_gem_private_object_init(struct drm_device *dev, drm_vma_node_reset(&obj->vma_node); INIT_LIST_HEAD(&obj->lru_node); + obj->exportable = true; } EXPORT_SYMBOL(drm_gem_private_object_init); diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index f924b8b4ab6b34..9d2dd982580efa 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -391,6 +391,11 @@ static struct dma_buf *export_and_register_object(struct drm_device *dev, return dmabuf; } + if (!obj->exportable) { + dmabuf = ERR_PTR(-EINVAL); + return dmabuf; + } + if (obj->funcs && obj->funcs->export) dmabuf = obj->funcs->export(obj, flags); else diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h index f1f00fc2dba614..120b5a2e662549 100644 --- a/include/drm/drm_gem.h +++ b/include/drm/drm_gem.h @@ -361,6 +361,14 @@ struct drm_gem_object { * The current LRU list that the GEM object is on. */ struct drm_gem_lru *lru; + + /** + * @exportable: + * + * Whether this GEM object can be exported via the drm_gem_object_funcs->export + * callback. Defaults to true. + */ + bool exportable; }; /** From 3423e153a384cebb7b728855813cc85e2685823d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 11 Feb 2023 16:03:02 +0900 Subject: [PATCH 480/635] rust: drm: gem: Add set_exportable() method This allows drivers to control whether a given GEM object is allowed to be exported via PRIME to other drivers. --- rust/kernel/drm/gem/mod.rs | 13 +++++++++++++ rust/kernel/drm/gem/shmem.rs | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 1805fa123f033e..68018289556984 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -47,6 +47,10 @@ pub trait IntoGEMObject: Sized + crate::private::Sealed { /// this owning object is valid. fn gem_obj(&self) -> &bindings::drm_gem_object; + /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as + /// this owning object is valid. + fn mut_gem_obj(&mut self) -> &mut bindings::drm_gem_object; + /// Converts a pointer to a `drm_gem_object` into a pointer to this type. fn from_gem_obj(obj: *mut bindings::drm_gem_object) -> *mut Self; } @@ -116,6 +120,10 @@ impl IntoGEMObject for Object { &self.obj } + fn mut_gem_obj(&mut self) -> &mut bindings::drm_gem_object { + &mut self.obj + } + fn from_gem_obj(obj: *mut bindings::drm_gem_object) -> *mut Object { crate::container_of!(obj, Object, obj) as *mut Object } @@ -128,6 +136,11 @@ pub trait BaseObject: IntoGEMObject { self.gem_obj().size } + /// Sets the exportable flag, which controls whether the object can be exported via PRIME. + fn set_exportable(&mut self, exportable: bool) { + self.mut_gem_obj().exportable = exportable; + } + /// Creates a new reference to the object. fn reference(&self) -> ObjectRef { // SAFETY: Having a reference to an Object implies holding a GEM reference diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index dafc08d1eecb99..fe5f976b484a5f 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -234,6 +234,10 @@ impl gem::IntoGEMObject for Object { &self.obj.base } + fn mut_gem_obj(&mut self) -> &mut bindings::drm_gem_object { + &mut self.obj.base + } + fn from_gem_obj(obj: *mut bindings::drm_gem_object) -> *mut Object { let shmem = crate::container_of!(obj, bindings::drm_gem_shmem_object, base) as *mut bindings::drm_gem_shmem_object; From 64f75e8c5dae17d782b980ed5dbac1e636984cff Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 17 Feb 2023 00:20:39 +0900 Subject: [PATCH 481/635] drm/asahi: Add the Asahi driver UAPI Signed-off-by: Asahi Lina --- include/uapi/drm/asahi_drm.h | 562 +++++++++++++++++++++++++++++++++++ 1 file changed, 562 insertions(+) create mode 100644 include/uapi/drm/asahi_drm.h diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h new file mode 100644 index 00000000000000..01f57a20d97401 --- /dev/null +++ b/include/uapi/drm/asahi_drm.h @@ -0,0 +1,562 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright (C) The Asahi Linux Contributors + * + * Based on asahi_drm.h which is + * + * Copyright © 2014-2018 Broadcom + * Copyright © 2019 Collabora ltd. + */ +#ifndef _ASAHI_DRM_H_ +#define _ASAHI_DRM_H_ + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +#define DRM_ASAHI_UNSTABLE_UABI_VERSION 10008 + +#define DRM_ASAHI_GET_PARAMS 0x00 +#define DRM_ASAHI_VM_CREATE 0x01 +#define DRM_ASAHI_VM_DESTROY 0x02 +#define DRM_ASAHI_GEM_CREATE 0x03 +#define DRM_ASAHI_GEM_MMAP_OFFSET 0x04 +#define DRM_ASAHI_GEM_BIND 0x05 +#define DRM_ASAHI_QUEUE_CREATE 0x06 +#define DRM_ASAHI_QUEUE_DESTROY 0x07 +#define DRM_ASAHI_SUBMIT 0x08 +#define DRM_ASAHI_GET_TIME 0x09 + +#define DRM_ASAHI_MAX_CLUSTERS 32 + +struct drm_asahi_params_global { + __u32 unstable_uabi_version; + __u32 pad0; + + __u64 feat_compat; + __u64 feat_incompat; + + __u32 gpu_generation; + __u32 gpu_variant; + __u32 gpu_revision; + __u32 chip_id; + + __u32 num_dies; + __u32 num_clusters_total; + __u32 num_cores_per_cluster; + __u32 num_frags_per_cluster; + __u32 num_gps_per_cluster; + __u32 num_cores_total_active; + __u64 core_masks[DRM_ASAHI_MAX_CLUSTERS]; + + __u32 vm_page_size; + __u32 pad1; + __u64 vm_user_start; + __u64 vm_user_end; + __u64 vm_shader_start; + __u64 vm_shader_end; + + __u32 max_syncs_per_submission; + __u32 max_commands_per_submission; + __u32 max_commands_in_flight; + __u32 max_attachments; + + __u32 timer_frequency_hz; + __u32 min_frequency_khz; + __u32 max_frequency_khz; + __u32 max_power_mw; + + __u32 result_render_size; + __u32 result_compute_size; +}; + +/* +enum drm_asahi_feat_compat { +}; +*/ + +enum drm_asahi_feat_incompat { + DRM_ASAHI_FEAT_MANDATORY_ZS_COMPRESSION = (1UL) << 0, +}; + +struct drm_asahi_get_params { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @param: Parameter group to fetch (MBZ) */ + __u32 param_group; + + /** @pad: MBZ */ + __u32 pad; + + /** @value: User pointer to write parameter struct */ + __u64 pointer; + + /** @value: Size of user buffer, max size supported on return */ + __u64 size; +}; + +struct drm_asahi_vm_create { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @value: Returned VM ID */ + __u32 vm_id; + + /** @pad: MBZ */ + __u32 pad; +}; + +struct drm_asahi_vm_destroy { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @value: VM ID to be destroyed */ + __u32 vm_id; + + /** @pad: MBZ */ + __u32 pad; +}; + +#define ASAHI_GEM_WRITEBACK (1L << 0) +#define ASAHI_GEM_VM_PRIVATE (1L << 1) + +struct drm_asahi_gem_create { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @size: Size of the BO */ + __u64 size; + + /** @flags: BO creation flags */ + __u32 flags; + + /** @handle: VM ID to assign to the BO, if ASAHI_GEM_VM_PRIVATE is set. */ + __u32 vm_id; + + /** @handle: Returned GEM handle for the BO */ + __u32 handle; +}; + +struct drm_asahi_gem_mmap_offset { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @handle: Handle for the object being mapped. */ + __u32 handle; + + /** @flags: Must be zero */ + __u32 flags; + + /** @offset: The fake offset to use for subsequent mmap call */ + __u64 offset; +}; + +enum drm_asahi_bind_op { + ASAHI_BIND_OP_BIND = 0, + ASAHI_BIND_OP_UNBIND = 1, + ASAHI_BIND_OP_UNBIND_ALL = 2, +}; + +#define ASAHI_BIND_READ (1L << 0) +#define ASAHI_BIND_WRITE (1L << 1) + +struct drm_asahi_gem_bind { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @obj: Bind operation */ + __u32 op; + + /** @flags: One or more of ASAHI_BIND_* */ + __u32 flags; + + /** @obj: GEM object to bind */ + __u32 handle; + + /** @vm_id: The ID of the VM to bind to */ + __u32 vm_id; + + /** @offset: Offset into the object */ + __u64 offset; + + /** @range: Number of bytes from the object to bind to addr */ + __u64 range; + + /** @addr: Address to bind to */ + __u64 addr; +}; + +enum drm_asahi_cmd_type { + DRM_ASAHI_CMD_RENDER = 0, + DRM_ASAHI_CMD_BLIT = 1, + DRM_ASAHI_CMD_COMPUTE = 2, +}; + +/* Note: this is an enum so that it can be resolved by Rust bindgen. */ +enum drm_asahi_queue_cap { + DRM_ASAHI_QUEUE_CAP_RENDER = (1UL << DRM_ASAHI_CMD_RENDER), + DRM_ASAHI_QUEUE_CAP_BLIT = (1UL << DRM_ASAHI_CMD_BLIT), + DRM_ASAHI_QUEUE_CAP_COMPUTE = (1UL << DRM_ASAHI_CMD_COMPUTE), +}; + +struct drm_asahi_queue_create { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @flags: MBZ */ + __u32 flags; + + /** @vm_id: The ID of the VM this queue is bound to */ + __u32 vm_id; + + /** @type: Bitmask of DRM_ASAHI_QUEUE_CAP_* */ + __u32 queue_caps; + + /** @priority: Queue priority, 0-3 */ + __u32 priority; + + /** @queue_id: The returned queue ID */ + __u32 queue_id; +}; + +struct drm_asahi_queue_destroy { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @queue_id: The queue ID to be destroyed */ + __u32 queue_id; +}; + +enum drm_asahi_sync_type { + DRM_ASAHI_SYNC_SYNCOBJ = 0, + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, +}; + +struct drm_asahi_sync { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @sync_type: One of drm_asahi_sync_type */ + __u32 sync_type; + + /** @handle: The sync object handle */ + __u32 handle; + + /** @timeline_value: Timeline value for timeline sync objects */ + __u64 timeline_value; +}; + +enum drm_asahi_subqueue { + DRM_ASAHI_SUBQUEUE_RENDER = 0, /* Also blit */ + DRM_ASAHI_SUBQUEUE_COMPUTE = 1, + DRM_ASAHI_SUBQUEUE_COUNT = 2, +}; + +#define DRM_ASAHI_BARRIER_NONE ~(0U) + +struct drm_asahi_command { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @type: One of drm_asahi_cmd_type */ + __u32 cmd_type; + + /** @flags: Flags for command submission */ + __u32 flags; + + /** @cmdbuf: Pointer to the appropriate command buffer structure */ + __u64 cmd_buffer; + + /** @cmdbuf: Size of the command buffer structure */ + __u64 cmd_buffer_size; + + /** @cmdbuf: Offset into the result BO to return information about this command */ + __u64 result_offset; + + /** @cmdbuf: Size of the result data structure */ + __u64 result_size; + + /** @barriers: Array of command indices per subqueue to wait on */ + __u32 barriers[DRM_ASAHI_SUBQUEUE_COUNT]; +}; + +struct drm_asahi_submit { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @in_syncs: An optional array of drm_asahi_sync to wait on before starting this job. */ + __u64 in_syncs; + + /** @in_syncs: An optional array of drm_asahi_sync objects to signal upon completion. */ + __u64 out_syncs; + + /** @commands: Pointer to the drm_asahi_command array of commands to submit. */ + __u64 commands; + + /** @flags: Flags for command submission (MBZ) */ + __u32 flags; + + /** @queue_id: The queue ID to be submitted to */ + __u32 queue_id; + + /** @result_handle: An optional BO handle to place result data in */ + __u32 result_handle; + + /** @in_sync_count: Number of sync objects to wait on before starting this job. */ + __u32 in_sync_count; + + /** @in_sync_count: Number of sync objects to signal upon completion of this job. */ + __u32 out_sync_count; + + /** @pad: Number of commands to be submitted */ + __u32 command_count; +}; + +struct drm_asahi_attachment { + /** @pointer: Base address of the attachment */ + __u64 pointer; + /** @size: Size of the attachment in bytes */ + __u64 size; + /** @order: Power of 2 exponent related to attachment size (?) */ + __u32 order; + /** @flags: MBZ */ + __u32 flags; +}; + +#define ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES (1UL << 0) +#define ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S (1UL << 1) +#define ASAHI_RENDER_MEMORYLESS_RTS_USED (1UL << 2) +#define ASAHI_RENDER_PROCESS_EMPTY_TILES (1UL << 3) +#define ASAHI_RENDER_NO_VERTEX_CLUSTERING (1UL << 4) +#define ASAHI_RENDER_MSAA_ZS (1UL << 5) + +struct drm_asahi_cmd_render { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + __u64 flags; + + __u64 encoder_ptr; + + __u64 vertex_attachments; + __u64 fragment_attachments; + __u32 vertex_attachment_count; + __u32 fragment_attachment_count; + + __u64 depth_buffer_load; + __u64 depth_buffer_store; + __u64 depth_buffer_partial; + __u64 depth_meta_buffer_load; + __u64 depth_meta_buffer_store; + __u64 depth_meta_buffer_partial; + + __u64 stencil_buffer_load; + __u64 stencil_buffer_store; + __u64 stencil_buffer_partial; + __u64 stencil_meta_buffer_load; + __u64 stencil_meta_buffer_store; + __u64 stencil_meta_buffer_partial; + + __u64 scissor_array; + __u64 depth_bias_array; + __u64 visibility_result_buffer; + + __u64 zls_ctrl; + __u64 ppp_multisamplectl; + __u32 ppp_ctrl; + + __u32 fb_width; + __u32 fb_height; + + __u32 utile_width; + __u32 utile_height; + + __u32 samples; + __u32 layers; + + __u32 encoder_id; + __u32 cmd_ta_id; + __u32 cmd_3d_id; + + __u32 sample_size; + __u32 tib_blocks; + __u32 iogpu_unk_214; + + __u32 merge_upper_x; + __u32 merge_upper_y; + + __u32 load_pipeline; + __u32 load_pipeline_bind; + + __u32 store_pipeline; + __u32 store_pipeline_bind; + + __u32 partial_reload_pipeline; + __u32 partial_reload_pipeline_bind; + + __u32 partial_store_pipeline; + __u32 partial_store_pipeline_bind; + + __u32 depth_dimensions; + __u32 isp_bgobjdepth; + __u32 isp_bgobjvals; + +}; + +struct drm_asahi_cmd_compute { + __u64 flags; + + __u64 encoder_ptr; + __u64 encoder_end; + + __u64 attachments; + __u32 attachment_count; + __u32 pad; + + __u64 buffer_descriptor; + + __u32 buffer_descriptor_size; /* ? */ + __u32 ctx_switch_prog; + + __u32 encoder_id; + __u32 cmd_id; + + __u32 iogpu_unk_40; + __u32 iogpu_unk_44; +}; + +enum drm_asahi_status { + DRM_ASAHI_STATUS_PENDING = 0, + DRM_ASAHI_STATUS_COMPLETE, + DRM_ASAHI_STATUS_UNKNOWN_ERROR, + DRM_ASAHI_STATUS_TIMEOUT, + DRM_ASAHI_STATUS_FAULT, + DRM_ASAHI_STATUS_KILLED, + DRM_ASAHI_STATUS_NO_DEVICE, +}; + +enum drm_asahi_fault { + DRM_ASAHI_FAULT_NONE = 0, + DRM_ASAHI_FAULT_UNKNOWN, + DRM_ASAHI_FAULT_UNMAPPED, + DRM_ASAHI_FAULT_AF_FAULT, + DRM_ASAHI_FAULT_WRITE_ONLY, + DRM_ASAHI_FAULT_READ_ONLY, + DRM_ASAHI_FAULT_NO_ACCESS, +}; + +struct drm_asahi_result_info { + /** @status: One of enum drm_asahi_status */ + __u32 status; + + /** @reason: One of drm_asahi_fault_type */ + __u32 fault_type; + + /** @unit: Unit number, hardware dependent */ + __u32 unit; + + /** @sideband: Sideband information, hardware dependent */ + __u32 sideband; + + /** @level: Page table level at which the fault occurred, hardware dependent */ + __u8 level; + + /** @read: Fault was a read */ + __u8 is_read; + + /** @pad: MBZ */ + __u16 pad; + + /** @unk_5: Extra bits, hardware dependent */ + __u32 extra; + + /** @address: Fault address, cache line aligned */ + __u64 address; +}; + +#define DRM_ASAHI_RESULT_RENDER_TVB_GROW_OVF (1UL << 0) +#define DRM_ASAHI_RESULT_RENDER_TVB_GROW_MIN (1UL << 1) +#define DRM_ASAHI_RESULT_RENDER_TVB_OVERFLOWED (1UL << 2) + +struct drm_asahi_result_render { + /** @address: Common result information */ + struct drm_asahi_result_info info; + + /** @flags: Zero or more of of DRM_ASAHI_RESULT_RENDER_* */ + __u64 flags; + + /** @vertex_ts_start: Timestamp of the start of vertex processing */ + __u64 vertex_ts_start; + + /** @vertex_ts_end: Timestamp of the end of vertex processing */ + __u64 vertex_ts_end; + + /** @fragment_ts_start: Timestamp of the start of fragment processing */ + __u64 fragment_ts_start; + + /** @fragment_ts_end: Timestamp of the end of fragment processing */ + __u64 fragment_ts_end; + + /** @tvb_size_bytes: TVB size at the start of this render */ + __u64 tvb_size_bytes; + + /** @tvb_usage_bytes: Total TVB usage in bytes for this render */ + __u64 tvb_usage_bytes; + + /** @num_tvb_overflows: Number of TVB overflows that occurred for this render */ + __u32 num_tvb_overflows; +}; + +struct drm_asahi_result_compute { + /** @address: Common result information */ + struct drm_asahi_result_info info; + + /** @flags: Zero or more of of DRM_ASAHI_RESULT_COMPUTE_* */ + __u64 flags; + + /** @ts_start: Timestamp of the start of this compute command */ + __u64 ts_start; + + /** @vertex_ts_end: Timestamp of the end of this compute command */ + __u64 ts_end; +}; + +struct drm_asahi_get_time { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @flags: MBZ. */ + __u64 flags; + + /** @tv_sec: On return, seconds part of a point in time */ + __s64 tv_sec; + + /** @tv_nsec: On return, nanoseconds part of a point in time */ + __s64 tv_nsec; + + /** @gpu_timestamp: On return, the GPU timestamp at that point in time */ + __u64 gpu_timestamp; +}; + +/* Note: this is an enum so that it can be resolved by Rust bindgen. */ +enum { + DRM_IOCTL_ASAHI_GET_PARAMS = DRM_IOWR(DRM_COMMAND_BASE + DRM_ASAHI_GET_PARAMS, struct drm_asahi_get_params), + DRM_IOCTL_ASAHI_VM_CREATE = DRM_IOWR(DRM_COMMAND_BASE + DRM_ASAHI_VM_CREATE, struct drm_asahi_vm_create), + DRM_IOCTL_ASAHI_VM_DESTROY = DRM_IOW(DRM_COMMAND_BASE + DRM_ASAHI_VM_DESTROY, struct drm_asahi_vm_destroy), + DRM_IOCTL_ASAHI_GEM_CREATE = DRM_IOWR(DRM_COMMAND_BASE + DRM_ASAHI_GEM_CREATE, struct drm_asahi_gem_create), + DRM_IOCTL_ASAHI_GEM_MMAP_OFFSET = DRM_IOWR(DRM_COMMAND_BASE + DRM_ASAHI_GEM_MMAP_OFFSET, struct drm_asahi_gem_mmap_offset), + DRM_IOCTL_ASAHI_GEM_BIND = DRM_IOW(DRM_COMMAND_BASE + DRM_ASAHI_GEM_BIND, struct drm_asahi_gem_bind), + DRM_IOCTL_ASAHI_QUEUE_CREATE = DRM_IOWR(DRM_COMMAND_BASE + DRM_ASAHI_QUEUE_CREATE, struct drm_asahi_queue_create), + DRM_IOCTL_ASAHI_QUEUE_DESTROY = DRM_IOW(DRM_COMMAND_BASE + DRM_ASAHI_QUEUE_DESTROY, struct drm_asahi_queue_destroy), + DRM_IOCTL_ASAHI_SUBMIT = DRM_IOW(DRM_COMMAND_BASE + DRM_ASAHI_SUBMIT, struct drm_asahi_submit), + DRM_IOCTL_ASAHI_GET_TIME = DRM_IOWR(DRM_COMMAND_BASE + DRM_ASAHI_GET_TIME, struct drm_asahi_get_time), +}; + +#if defined(__cplusplus) +} +#endif + +#endif /* _ASAHI_DRM_H_ */ From a57d7f8c4bff4b4b83d1d6567226e5945781fcc4 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 17 Feb 2023 00:20:55 +0900 Subject: [PATCH 482/635] rust: bindings: Bind the Asahi DRM UAPI Signed-off-by: Asahi Lina --- rust/uapi/uapi_helper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index ab8fb904c30aa5..61aef6399763a2 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,4 +7,5 @@ */ #include +#include #include From b6151fc84e725afbd826b4466be59031684df432 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 17 Feb 2023 00:28:27 +0900 Subject: [PATCH 483/635] rust: macros: Add versions macro Signed-off-by: Asahi Lina --- rust/macros/lib.rs | 7 + rust/macros/versions.rs | 289 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+) create mode 100644 rust/macros/versions.rs diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 3fc74cb4ea1903..e22915b9c45454 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -9,6 +9,7 @@ mod helpers; mod module; mod pin_data; mod pinned_drop; +mod versions; mod vtable; use proc_macro::TokenStream; @@ -77,6 +78,12 @@ pub fn module(ts: TokenStream) -> TokenStream { module::module(ts) } +/// Declares multiple variants of a structure or impl code +#[proc_macro_attribute] +pub fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + versions::versions(attr, item) +} + /// Declares or implements a vtable trait. /// /// Linux's use of pure vtables is very close to Rust traits, but they differ diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs new file mode 100644 index 00000000000000..6dfdfd7d25c55d --- /dev/null +++ b/rust/macros/versions.rs @@ -0,0 +1,289 @@ +use proc_macro::{Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; + +//use crate::helpers::expect_punct; + +fn expect_group(it: &mut impl Iterator) -> Group { + if let Some(TokenTree::Group(group)) = it.next() { + group + } else { + panic!("Expected Group") + } +} + +fn expect_punct(it: &mut impl Iterator) -> String { + if let Some(TokenTree::Punct(punct)) = it.next() { + punct.to_string() + } else { + panic!("Expected Group") + } +} + +fn drop_until_punct(it: &mut impl Iterator, delimiter: &str) { + let mut depth: isize = 0; + for token in it.by_ref() { + if let TokenTree::Punct(punct) = token { + match punct.as_char() { + '<' => { + depth += 1; + } + '>' => { + depth -= 1; + } + _ => { + if depth == 0 && delimiter.contains(&punct.to_string()) { + break; + } + } + } + } + } +} + +struct VersionConfig { + fields: &'static [&'static str], + enums: &'static [&'static [&'static str]], + versions: &'static [&'static [&'static str]], +} + +static AGX_VERSIONS: VersionConfig = VersionConfig { + fields: &["G", "V"], + enums: &[&["G13", "G14"], &["V12_3", "V12_4", "V13_0B4", "V13_2"]], + versions: &[ + &["G13", "V12_3"], + &["G14", "V12_4"], + // &["G13", "V13_0B4"], + // &["G14", "V13_0B4"], + &["G13", "V13_2"], + &["G14", "V13_2"], + ], +}; + +fn check_version( + config: &VersionConfig, + ver: &[usize], + it: &mut impl Iterator, +) -> bool { + let first = it.next().unwrap(); + let val: bool = match &first { + TokenTree::Group(group) => check_version(config, ver, &mut group.stream().into_iter()), + TokenTree::Ident(ident) => { + let key = config + .fields + .iter() + .position(|&r| r == ident.to_string()) + .unwrap_or_else(|| panic!("Unknown field {}", ident)); + let mut operator = expect_punct(it); + let mut rhs_token = it.next().unwrap(); + if let TokenTree::Punct(punct) = &rhs_token { + operator.extend(std::iter::once(punct.as_char())); + rhs_token = it.next().unwrap(); + } + let rhs_name = if let TokenTree::Ident(ident) = &rhs_token { + ident.to_string() + } else { + panic!("Unexpected token {}", ident) + }; + + let rhs = config.enums[key] + .iter() + .position(|&r| r == rhs_name) + .unwrap_or_else(|| panic!("Unknown value for {}:{}", ident, rhs_name)); + let lhs = ver[key]; + + match operator.as_str() { + "==" => lhs == rhs, + "!=" => lhs != rhs, + ">" => lhs > rhs, + ">=" => lhs >= rhs, + "<" => lhs < rhs, + "<=" => lhs <= rhs, + _ => panic!("Unknown operator {}", operator), + } + } + _ => { + panic!("Unknown token {}", first) + } + }; + + let boolop = it.next(); + match boolop { + Some(TokenTree::Punct(punct)) => { + let right = expect_punct(it); + if right != punct.to_string() { + panic!("Unexpected op {}{}", punct, right); + } + match punct.as_char() { + '&' => val && check_version(config, ver, it), + '|' => val || check_version(config, ver, it), + _ => panic!("Unexpected op {}{}", right, right), + } + } + Some(a) => panic!("Unexpected op {}", a), + None => val, + } +} + +fn filter_versions( + config: &VersionConfig, + tag: &str, + ver: &[usize], + tree: impl IntoIterator, + is_struct: bool, +) -> Vec { + let mut out = Vec::::new(); + let mut it = tree.into_iter(); + + while let Some(token) = it.next() { + let mut tail: Option = None; + match &token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + let group = expect_group(&mut it); + let mut grp_it = group.stream().into_iter(); + let attr = grp_it.next().unwrap(); + match attr { + TokenTree::Ident(ident) if ident.to_string() == "ver" => { + if check_version(config, ver, &mut grp_it) { + } else if is_struct { + drop_until_punct(&mut it, ","); + } else { + let first = it.next().unwrap(); + match &first { + TokenTree::Group(_) => (), + _ => { + drop_until_punct(&mut it, ",;"); + } + } + } + } + _ => { + out.push(token.clone()); + out.push(TokenTree::Group(group.clone())); + } + } + continue; + } + TokenTree::Punct(punct) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Punct(punct)) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Ident(idtag)) if idtag.to_string() == "ver" => { + let ident = match out.pop() { + Some(TokenTree::Ident(ident)) => ident, + a => panic!("$ver not following ident: {:?}", a), + }; + let name = ident.to_string() + tag; + let new_ident = Ident::new(name.as_str(), ident.span()); + out.push(TokenTree::Ident(new_ident)); + continue; + } + Some(a) => { + out.push(token.clone()); + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + out.push(token.clone()); + } + } + } + Some(a) => { + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + continue; + } + } + } + _ => { + tail = Some(token); + } + } + match &tail { + Some(TokenTree::Group(group)) => { + let new_body = + filter_versions(config, tag, ver, &mut group.stream().into_iter(), is_struct); + let mut stream = TokenStream::new(); + stream.extend(new_body); + let mut filtered_group = Group::new(group.delimiter(), stream); + filtered_group.set_span(group.span()); + out.push(TokenTree::Group(filtered_group)); + } + Some(token) => { + out.push(token.clone()); + } + None => {} + } + } + + out +} + +pub(crate) fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + let config = match attr.to_string().as_str() { + "AGX" => &AGX_VERSIONS, + _ => panic!("Unknown version group {}", attr), + }; + + let mut it = item.into_iter(); + let mut out = TokenStream::new(); + let mut body: Vec = Vec::new(); + let mut is_struct = false; + + while let Some(token) = it.next() { + match token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + body.push(TokenTree::Punct(punct)); + body.push(it.next().unwrap()); + } + TokenTree::Ident(ident) + if ["struct", "enum", "union", "const", "type"] + .contains(&ident.to_string().as_str()) => + { + is_struct = ident.to_string() != "const"; + body.push(TokenTree::Ident(ident)); + body.push(it.next().unwrap()); + // This isn't valid syntax in a struct definition, so add it for the user + body.push(TokenTree::Punct(Punct::new(':', Spacing::Joint))); + body.push(TokenTree::Punct(Punct::new(':', Spacing::Alone))); + body.push(TokenTree::Ident(Ident::new("ver", Span::call_site()))); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "impl" => { + body.push(TokenTree::Ident(ident)); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "fn" => { + body.push(TokenTree::Ident(ident)); + break; + } + _ => { + body.push(token); + } + } + } + + body.extend(it); + + for ver in config.versions { + let tag = ver.join(""); + let mut ver_num = Vec::::new(); + for (i, comp) in ver.iter().enumerate() { + let idx = config.enums[i].iter().position(|&r| r == *comp).unwrap(); + ver_num.push(idx); + } + out.extend(filter_versions( + config, + &tag, + &ver_num, + body.clone(), + is_struct, + )); + } + + out +} From fc54ef9fb234ac6eb308506bcec5676c484da6bf Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 17 Feb 2023 00:31:51 +0900 Subject: [PATCH 484/635] drm/asahi: Add the Asahi driver for Apple AGX GPUs Signed-off-by: Asahi Lina --- drivers/gpu/drm/Kconfig | 2 + drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/asahi/Kconfig | 40 + drivers/gpu/drm/asahi/Makefile | 3 + drivers/gpu/drm/asahi/alloc.rs | 1046 ++++++++++++++++++++ drivers/gpu/drm/asahi/asahi.rs | 56 ++ drivers/gpu/drm/asahi/buffer.rs | 694 +++++++++++++ drivers/gpu/drm/asahi/channel.rs | 530 ++++++++++ drivers/gpu/drm/asahi/debug.rs | 129 +++ drivers/gpu/drm/asahi/driver.rs | 166 ++++ drivers/gpu/drm/asahi/event.rs | 232 +++++ drivers/gpu/drm/asahi/file.rs | 758 ++++++++++++++ drivers/gpu/drm/asahi/float.rs | 381 +++++++ drivers/gpu/drm/asahi/fw/buffer.rs | 170 ++++ drivers/gpu/drm/asahi/fw/channels.rs | 385 ++++++++ drivers/gpu/drm/asahi/fw/compute.rs | 107 ++ drivers/gpu/drm/asahi/fw/event.rs | 100 ++ drivers/gpu/drm/asahi/fw/fragment.rs | 282 ++++++ drivers/gpu/drm/asahi/fw/initdata.rs | 1264 ++++++++++++++++++++++++ drivers/gpu/drm/asahi/fw/job.rs | 56 ++ drivers/gpu/drm/asahi/fw/microseq.rs | 384 +++++++ drivers/gpu/drm/asahi/fw/mod.rs | 15 + drivers/gpu/drm/asahi/fw/types.rs | 233 +++++ drivers/gpu/drm/asahi/fw/vertex.rs | 186 ++++ drivers/gpu/drm/asahi/fw/workqueue.rs | 168 ++++ drivers/gpu/drm/asahi/gem.rs | 301 ++++++ drivers/gpu/drm/asahi/gpu.rs | 1093 ++++++++++++++++++++ drivers/gpu/drm/asahi/hw/mod.rs | 522 ++++++++++ drivers/gpu/drm/asahi/hw/t600x.rs | 140 +++ drivers/gpu/drm/asahi/hw/t8103.rs | 80 ++ drivers/gpu/drm/asahi/hw/t8112.rs | 82 ++ drivers/gpu/drm/asahi/initdata.rs | 777 +++++++++++++++ drivers/gpu/drm/asahi/mem.rs | 133 +++ drivers/gpu/drm/asahi/microseq.rs | 61 ++ drivers/gpu/drm/asahi/mmu.rs | 1249 +++++++++++++++++++++++ drivers/gpu/drm/asahi/object.rs | 704 +++++++++++++ drivers/gpu/drm/asahi/place.rs | 343 +++++++ drivers/gpu/drm/asahi/queue/common.rs | 52 + drivers/gpu/drm/asahi/queue/compute.rs | 373 +++++++ drivers/gpu/drm/asahi/queue/mod.rs | 717 ++++++++++++++ drivers/gpu/drm/asahi/queue/render.rs | 1181 ++++++++++++++++++++++ drivers/gpu/drm/asahi/regs.rs | 387 ++++++++ drivers/gpu/drm/asahi/slotalloc.rs | 292 ++++++ drivers/gpu/drm/asahi/util.rs | 44 + drivers/gpu/drm/asahi/workqueue.rs | 853 ++++++++++++++++ 45 files changed, 16772 insertions(+) create mode 100644 drivers/gpu/drm/asahi/Kconfig create mode 100644 drivers/gpu/drm/asahi/Makefile create mode 100644 drivers/gpu/drm/asahi/alloc.rs create mode 100644 drivers/gpu/drm/asahi/asahi.rs create mode 100644 drivers/gpu/drm/asahi/buffer.rs create mode 100644 drivers/gpu/drm/asahi/channel.rs create mode 100644 drivers/gpu/drm/asahi/debug.rs create mode 100644 drivers/gpu/drm/asahi/driver.rs create mode 100644 drivers/gpu/drm/asahi/event.rs create mode 100644 drivers/gpu/drm/asahi/file.rs create mode 100644 drivers/gpu/drm/asahi/float.rs create mode 100644 drivers/gpu/drm/asahi/fw/buffer.rs create mode 100644 drivers/gpu/drm/asahi/fw/channels.rs create mode 100644 drivers/gpu/drm/asahi/fw/compute.rs create mode 100644 drivers/gpu/drm/asahi/fw/event.rs create mode 100644 drivers/gpu/drm/asahi/fw/fragment.rs create mode 100644 drivers/gpu/drm/asahi/fw/initdata.rs create mode 100644 drivers/gpu/drm/asahi/fw/job.rs create mode 100644 drivers/gpu/drm/asahi/fw/microseq.rs create mode 100644 drivers/gpu/drm/asahi/fw/mod.rs create mode 100644 drivers/gpu/drm/asahi/fw/types.rs create mode 100644 drivers/gpu/drm/asahi/fw/vertex.rs create mode 100644 drivers/gpu/drm/asahi/fw/workqueue.rs create mode 100644 drivers/gpu/drm/asahi/gem.rs create mode 100644 drivers/gpu/drm/asahi/gpu.rs create mode 100644 drivers/gpu/drm/asahi/hw/mod.rs create mode 100644 drivers/gpu/drm/asahi/hw/t600x.rs create mode 100644 drivers/gpu/drm/asahi/hw/t8103.rs create mode 100644 drivers/gpu/drm/asahi/hw/t8112.rs create mode 100644 drivers/gpu/drm/asahi/initdata.rs create mode 100644 drivers/gpu/drm/asahi/mem.rs create mode 100644 drivers/gpu/drm/asahi/microseq.rs create mode 100644 drivers/gpu/drm/asahi/mmu.rs create mode 100644 drivers/gpu/drm/asahi/object.rs create mode 100644 drivers/gpu/drm/asahi/place.rs create mode 100644 drivers/gpu/drm/asahi/queue/common.rs create mode 100644 drivers/gpu/drm/asahi/queue/compute.rs create mode 100644 drivers/gpu/drm/asahi/queue/mod.rs create mode 100644 drivers/gpu/drm/asahi/queue/render.rs create mode 100644 drivers/gpu/drm/asahi/regs.rs create mode 100644 drivers/gpu/drm/asahi/slotalloc.rs create mode 100644 drivers/gpu/drm/asahi/util.rs create mode 100644 drivers/gpu/drm/asahi/workqueue.rs diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index dc0f94f02a82ed..ab20df64de71ca 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -273,6 +273,8 @@ config DRM_VKMS If M is selected the module will be called vkms. +source "drivers/gpu/drm/asahi/Kconfig" + source "drivers/gpu/drm/exynos/Kconfig" source "drivers/gpu/drm/rockchip/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index ab4460fcd63f6b..f144501d9c60f6 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -190,3 +190,4 @@ obj-y += gud/ obj-$(CONFIG_DRM_HYPERV) += hyperv/ obj-y += solomon/ obj-$(CONFIG_DRM_SPRD) += sprd/ +obj-$(CONFIG_DRM_ASAHI) += asahi/ diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig new file mode 100644 index 00000000000000..4ce4a773db5d43 --- /dev/null +++ b/drivers/gpu/drm/asahi/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +config RUST_DRM_SCHED + bool + select DRM_SCHED + +config RUST_DRM_GEM_SHMEM_HELPER + bool + select DRM_GEM_SHMEM_HELPER + +config RUST_APPLE_RTKIT + bool + select APPLE_RTKIT + select APPLE_MBOX + +config DRM_ASAHI + tristate "Asahi (DRM support for Apple AGX GPUs)" + depends on RUST + depends on DRM + depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on MMU + select RUST_DRM_SCHED + select IOMMU_SUPPORT + select IOMMU_IO_PGTABLE_LPAE + select RUST_DRM_GEM_SHMEM_HELPER + select RUST_APPLE_RTKIT + help + DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + +config DRM_ASAHI_DEBUG_ALLOCATOR + bool "Use debug allocator" + depends on DRM_ASAHI + help + Use an alternate, simpler allocator which significantly reduces + performance, but can help find firmware- or GPU-side memory safety + issues. However, it can also trigger firmware bugs more easily, + so expect GPU crashes. + + Say N unless you are debugging firmware structures or porting to a + new firmware version. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile new file mode 100644 index 00000000000000..e6724866798760 --- /dev/null +++ b/drivers/gpu/drm/asahi/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DRM_ASAHI) += asahi.o diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs new file mode 100644 index 00000000000000..5397da5f64e183 --- /dev/null +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -0,0 +1,1046 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU kernel object allocator. +//! +//! This kernel driver needs to manage a large number of GPU objects, in both firmware/kernel +//! address space and user address space. This module implements a simple grow-only heap allocator +//! based on the DRM MM range allocator, and a debug allocator that allocates each object as a +//! separate GEM object. +//! +//! Allocations may optionally have debugging enabled, which adds preambles that store metadata +//! about the allocation. This is useful for live debugging using the hypervisor or postmortem +//! debugging with a GPU memory snapshot, since it makes it easier to identify use-after-free and +//! caching issues. + +use kernel::{c_str, drm::mm, error::Result, prelude::*, str::CString, sync::LockClassKey}; + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::fw::types::Zeroed; +use crate::mmu; +use crate::object::{GpuArray, GpuObject, GpuOnlyArray, GpuStruct, GpuWeakPointer}; + +use core::cmp::Ordering; +use core::fmt; +use core::fmt::{Debug, Formatter}; +use core::marker::PhantomData; +use core::mem; +use core::mem::MaybeUninit; +use core::ptr::NonNull; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Alloc; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = HeapAllocator; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = HeapAllocation; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = SimpleAllocator; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = SimpleAllocation; + +/// Represents a raw allocation (without any type information). +pub(crate) trait RawAllocation { + /// Returns the CPU-side pointer (if CPU mapping is enabled) as a byte non-null pointer. + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the size of the allocation in bytes. + fn size(&self) -> usize; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// Represents a typed allocation. +pub(crate) trait Allocation: Debug { + /// Returns the typed CPU-side pointer (if CPU mapping is enabled). + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the size of the allocation in bytes. + fn size(&self) -> usize; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// A generic typed allocation wrapping a RawAllocation. +/// +/// This is currently the only Allocation implementation, since it is shared by all allocators. +pub(crate) struct GenericAlloc { + alloc: U, + alloc_size: usize, + debug_offset: usize, + padding: usize, + _p: PhantomData, +} + +impl Allocation for GenericAlloc { + fn ptr(&self) -> Option> { + self.alloc + .ptr() + .map(|p| unsafe { NonNull::new_unchecked(p.as_ptr().add(self.debug_offset) as *mut T) }) + } + fn gpu_ptr(&self) -> u64 { + self.alloc.gpu_ptr() + self.debug_offset as u64 + } + fn size(&self) -> usize { + self.alloc_size + } + fn device(&self) -> &AsahiDevice { + self.alloc.device() + } +} + +impl Debug for GenericAlloc { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::>()) + .field("ptr", &format_args!("{:?}", self.ptr())) + .field("gpu_ptr", &format_args!("{:#X?}", self.gpu_ptr())) + .field("size", &format_args!("{:#X?}", self.size())) + .finish() + } +} + +/// Debugging data associated with an allocation, when debugging is enabled. +#[repr(C)] +struct AllocDebugData { + state: u32, + _pad: u32, + size: u64, + base_gpuva: u64, + obj_gpuva: u64, + name: [u8; 0x20], +} + +/// Magic flag indicating a live allocation. +const STATE_LIVE: u32 = 0x4556494c; +/// Magic flag indicating a freed allocation. +const STATE_DEAD: u32 = 0x44414544; + +/// Marker byte to identify when firmware/GPU write beyond the end of an allocation. +const GUARD_MARKER: u8 = 0x93; + +impl Drop for GenericAlloc { + fn drop(&mut self) { + let debug_len = mem::size_of::(); + if self.debug_offset >= debug_len { + if let Some(p) = self.alloc.ptr() { + unsafe { + let p = p.as_ptr().add(self.debug_offset - debug_len); + (p as *mut u32).write(STATE_DEAD); + } + } + } + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = self.ptr() { + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + } + } + if self.padding != 0 { + if let Some(p) = self.ptr() { + let guard = unsafe { + core::slice::from_raw_parts( + (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + self.padding, + ) + }; + if let Some(first_err) = guard.iter().position(|&r| r != GUARD_MARKER) { + let last_err = guard + .iter() + .rev() + .position(|&r| r != GUARD_MARKER) + .unwrap_or(0); + dev_warn!( + self.device(), + "Allocator: Corruption after object of type {} at {:#x}:{:#x} + {:#x}..{:#x}\n", + core::any::type_name::(), + self.gpu_ptr(), + self.size(), + first_err, + self.padding - last_err + ); + } + } + } + } +} + +static_assert!(mem::size_of::() == 0x40); + +/// A trait representing an allocator. +pub(crate) trait Allocator { + /// The raw allocation type used by this allocator. + type Raw: RawAllocation; + // TODO: Needs associated_type_defaults + // type Allocation = GenericAlloc; + + /// Returns the `AsahiDevice` associated with this allocator. + fn device(&self) -> &AsahiDevice; + /// Returns whether CPU-side mapping is enabled. + fn cpu_maps(&self) -> bool; + /// Returns the minimum alignment for allocations. + fn min_align(&self) -> usize; + /// Allocate an object of the given size in bytes with the given alignment. + fn alloc(&mut self, size: usize, align: usize) -> Result; + + /// Returns a tuple of (count, size) of how much garbage (freed but not yet reusable objects) + /// exists in this allocator. Optional. + fn garbage(&self) -> (usize, usize) { + (0, 0) + } + /// Collect garbage for this allocator, up to the given object count. Optional. + fn collect_garbage(&mut self, _count: usize) {} + + /// Allocate a new GpuStruct object. See [`GpuObject::new`]. + #[inline(never)] + fn new_object( + &mut self, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result>> { + GpuObject::>::new(self.alloc_object()?, inner, callback) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_boxed`]. + #[inline(never)] + fn new_boxed( + &mut self, + inner: Box, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result>> { + GpuObject::>::new_boxed(self.alloc_object()?, inner, callback) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_inplace`]. + #[inline(never)] + fn new_inplace( + &mut self, + inner: T, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result>> { + GpuObject::>::new_inplace( + self.alloc_object()?, + inner, + callback, + ) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_default`]. + #[inline(never)] + fn new_default( + &mut self, + ) -> Result>> + where + for<'a> ::Raw<'a>: Default + Zeroed, + { + GpuObject::>::new_default(self.alloc_object()?) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_prealloc`]. + #[inline(never)] + fn new_prealloc( + &mut self, + inner_cb: impl FnOnce(GpuWeakPointer) -> Result>, + raw_cb: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result>> { + GpuObject::>::new_prealloc( + self.alloc_object()?, + inner_cb, + raw_cb, + ) + } + + /// Allocate a generic buffer of the given size and alignment, applying the debug features if + /// enabled to tag it and detect overflows. + fn alloc_generic( + &mut self, + size: usize, + align: usize, + ) -> Result> { + let padding = if debug_enabled(DebugFlags::DetectOverflows) { + size + } else { + 0 + }; + + let ret: GenericAlloc = + if self.cpu_maps() && debug_enabled(debug::DebugFlags::DebugAllocations) { + let debug_align = self.min_align().max(align); + let debug_len = mem::size_of::(); + let debug_offset = (debug_len * 2 + debug_align - 1) & !(debug_align - 1); + + let alloc = self.alloc(size + debug_offset + padding, align)?; + + let mut debug = AllocDebugData { + state: STATE_LIVE, + _pad: 0, + size: size as u64, + base_gpuva: alloc.gpu_ptr(), + obj_gpuva: alloc.gpu_ptr() + debug_offset as u64, + name: [0; 0x20], + }; + + let name = core::any::type_name::().as_bytes(); + let len = name.len().min(debug.name.len() - 1); + debug.name[..len].copy_from_slice(&name[..len]); + + if let Some(p) = alloc.ptr() { + unsafe { + let p = p.as_ptr(); + p.write_bytes(0x42, debug_offset - 2 * debug_len); + let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; + let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + prev.copy_from(cur, 1); + cur.copy_from(&debug, 1); + }; + } + + GenericAlloc { + alloc, + alloc_size: size, + debug_offset, + padding, + _p: PhantomData, + } + } else { + GenericAlloc { + alloc: self.alloc(size + padding, align)?, + alloc_size: size, + debug_offset: 0, + padding, + _p: PhantomData, + } + }; + + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = ret.ptr() { + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + } + } + + if padding != 0 { + if let Some(p) = ret.ptr() { + unsafe { + (p.as_ptr() as *mut u8) + .add(ret.size()) + .write_bytes(GUARD_MARKER, padding); + } + } + } + + Ok(ret) + } + + /// Allocate an object of a given type, without actually initializing the allocation. + /// + /// This is useful to directly call [`GpuObject::new_*`], without borrowing a reference to the + /// allocator for the entire duration (e.g. if further allocations need to happen inside the + /// callbacks). + fn alloc_object(&mut self) -> Result> { + let size = mem::size_of::>(); + let align = mem::align_of::>(); + + self.alloc_generic(size, align) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align)?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuOnlyArray` of a given type and length. + fn array_gpuonly( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align)?; + GpuOnlyArray::>::new(alloc, count) + } +} + +/// A simple allocation backed by a separate GEM object. +/// +/// # Invariants +/// `ptr` is either None or a valid, non-null pointer to the CPU view of the object. +/// `gpu_ptr` is the GPU-side VA of the object. +pub(crate) struct SimpleAllocation { + dev: AsahiDevice, + ptr: Option>, + gpu_ptr: u64, + size: usize, + vm: mmu::Vm, + obj: crate::gem::ObjectRef, +} + +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to send across threads. +unsafe impl Send for SimpleAllocation {} +unsafe impl Sync for SimpleAllocation {} + +impl Drop for SimpleAllocation { + fn drop(&mut self) { + mod_dev_dbg!( + self.device(), + "SimpleAllocator: drop object @ {:#x}\n", + self.gpu_ptr() + ); + if debug_enabled(DebugFlags::FillAllocations) { + if let Ok(vmap) = self.obj.vmap() { + vmap.as_mut_slice().fill(0x42); + } + } + self.obj.drop_vm_mappings(self.vm.id()); + } +} + +impl RawAllocation for SimpleAllocation { + fn ptr(&self) -> Option> { + self.ptr + } + fn gpu_ptr(&self) -> u64 { + self.gpu_ptr + } + fn size(&self) -> usize { + self.size + } + + fn device(&self) -> &AsahiDevice { + &self.dev + } +} + +/// A simple allocator that allocates each object as its own GEM object, aligned to the end of a +/// page. +/// +/// This is very slow, but it has the advantage that over-reads by the firmware or GPU will fault on +/// the guard page after the allocation, which can be useful to validate that the firmware's or +/// GPU's idea of object size what we expect. +pub(crate) struct SimpleAllocator { + dev: AsahiDevice, + start: u64, + end: u64, + prot: u32, + vm: mmu::Vm, + min_align: usize, + cpu_maps: bool, +} + +impl SimpleAllocator { + /// Create a new `SimpleAllocator` for a given address range and `Vm`. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + start: u64, + end: u64, + min_align: usize, + prot: u32, + _block_size: usize, + mut cpu_maps: bool, + _name: fmt::Arguments<'_>, + _keep_garbage: bool, + ) -> Result { + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + Ok(SimpleAllocator { + dev: dev.clone(), + vm: vm.clone(), + start, + end, + prot, + min_align, + cpu_maps, + }) + } +} + +impl Allocator for SimpleAllocator { + type Raw = SimpleAllocation; + + fn device(&self) -> &AsahiDevice { + &self.dev + } + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + #[inline(never)] + fn alloc(&mut self, size: usize, align: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + let align = self.min_align.max(align); + let offset = (size_aligned - size) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new: size={:#x} size_al={:#x} al={:#x} off={:#x}\n", + size, + size_aligned, + align, + offset + ); + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + let p = obj.vmap()?.as_mut_ptr() as *mut u8; + if debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.as_mut_slice().fill(0xde); + } + let iova = obj.map_into_range( + &self.vm, + self.start, + self.end, + self.min_align.max(mmu::UAT_PGSZ) as u64, + self.prot, + true, + )?; + + let ptr = unsafe { p.add(offset) } as *mut u8; + let gpu_ptr = (iova + offset) as u64; + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new -> {:#?} / {:#?} | {:#x} / {:#x}\n", + p, + ptr, + iova, + gpu_ptr + ); + + Ok(SimpleAllocation { + dev: self.dev.clone(), + ptr: NonNull::new(ptr), + gpu_ptr, + size, + vm: self.vm.clone(), + obj, + }) + } +} + +/// Inner data for an allocation from the heap allocator. +/// +/// This is wrapped in an `mm::Node`. +pub(crate) struct HeapAllocationInner { + dev: AsahiDevice, + ptr: Option>, + real_size: usize, +} + +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to send across threads. +unsafe impl Send for HeapAllocationInner {} +unsafe impl Sync for HeapAllocationInner {} + +/// Outer view of a heap allocation. +/// +/// This uses an Option<> so we can move the internal `Node` into the garbage pool when it gets +/// dropped. +/// +/// # Invariants +/// The `Option` must always be `Some(...)` while this object is alive. +pub(crate) struct HeapAllocation(Option>); + +impl Drop for HeapAllocation { + fn drop(&mut self) { + let node = self.0.take().unwrap(); + let size = node.size(); + let alloc = node.alloc_ref(); + + alloc.with(|a| { + if let Some(garbage) = a.garbage.as_mut() { + if garbage.try_push(node).is_err() { + dev_err!( + &a.dev, + "HeapAllocation[{}]::drop: Failed to keep garbage\n", + &*a.name, + ); + } + a.total_garbage += size as usize; + None + } else { + // We need to ensure node survives this scope, since dropping it + // will try to take the mm lock and deadlock us + Some(node) + } + }); + } +} + +impl mm::AllocInner for HeapAllocatorInner { + fn drop_object( + &mut self, + start: u64, + _size: u64, + _color: usize, + obj: &mut HeapAllocationInner, + ) { + /* real_size == 0 means it's a guard node */ + if obj.real_size > 0 { + mod_dev_dbg!( + obj.dev, + "HeapAllocator[{}]: drop object @ {:#x} ({} bytes)\n", + &*self.name, + start, + obj.real_size, + ); + self.allocated -= obj.real_size; + } + } +} + +impl RawAllocation for HeapAllocation { + // SAFETY: This function must always return a valid pointer. + // Since the HeapAllocation contains a reference to the + // backing_objects array that contains the object backing this pointer, + // and objects are only ever added to it, this pointer is guaranteed to + // remain valid for the lifetime of the HeapAllocation. + fn ptr(&self) -> Option> { + self.0.as_ref().unwrap().ptr + } + // SAFETY: This function must always return a valid GPU pointer. + // See the explanation in ptr(). + fn gpu_ptr(&self) -> u64 { + self.0.as_ref().unwrap().start() + } + fn size(&self) -> usize { + self.0.as_ref().unwrap().size() as usize + } + fn device(&self) -> &AsahiDevice { + &self.0.as_ref().unwrap().dev + } +} + +/// Inner data for a heap allocator which uses the DRM MM range allocator to manage the heap. +/// +/// This is wrapped by an `mm::Allocator`. +struct HeapAllocatorInner { + dev: AsahiDevice, + allocated: usize, + backing_objects: Vec<(crate::gem::ObjectRef, u64)>, + garbage: Option>>, + total_garbage: usize, + name: CString, + vm_id: u64, +} + +/// A heap allocator which uses the DRM MM range allocator to manage its objects. +/// +/// The heap is composed of a series of GEM objects. This implementation only ever grows the heap, +/// never shrinks it. +pub(crate) struct HeapAllocator { + dev: AsahiDevice, + start: u64, + end: u64, + top: u64, + prot: u32, + vm: mmu::Vm, + min_align: usize, + block_size: usize, + cpu_maps: bool, + guard_nodes: Vec>, + mm: mm::Allocator, + name: CString, +} + +static LOCK_KEY: LockClassKey = LockClassKey::new(); + +impl HeapAllocator { + /// Create a new HeapAllocator for a given `Vm` and address range. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + start: u64, + end: u64, + min_align: usize, + prot: u32, + block_size: usize, + mut cpu_maps: bool, + name: fmt::Arguments<'_>, + keep_garbage: bool, + ) -> Result { + if !min_align.is_power_of_two() { + return Err(EINVAL); + } + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + + let name = CString::try_from_fmt(name)?; + + let inner = HeapAllocatorInner { + dev: dev.clone(), + allocated: 0, + backing_objects: Vec::new(), + // TODO: This clearly needs a try_clone() or similar + name: CString::try_from_fmt(fmt!("{}", &*name))?, + vm_id: vm.id(), + garbage: if keep_garbage { Some(Vec::new()) } else { None }, + total_garbage: 0, + }; + + let mm = mm::Allocator::new( + start, + end - start + 1, + inner, + c_str!("HeapAllocator"), + &LOCK_KEY, + )?; + + Ok(HeapAllocator { + dev: dev.clone(), + vm: vm.clone(), + start, + end, + top: start, + prot, + min_align, + block_size: block_size.max(min_align), + cpu_maps, + guard_nodes: Vec::new(), + mm, + name, + }) + } + + /// Add a new backing block of the given size to this heap. + /// + /// If CPU mapping is enabled, this also adds a guard node to the range allocator to ensure that + /// objects cannot straddle backing block boundaries, since we cannot easily create a contiguous + /// CPU VA mapping for them. This can create some fragmentation. If CPU mapping is disabled, we + /// skip the guard blocks, since the GPU view of the heap is always contiguous. + fn add_block(&mut self, size: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + if self.top.saturating_add(size_aligned as u64) >= self.end { + dev_err!( + &self.dev, + "HeapAllocator[{}]::add_block: Exhausted VA space\n", + &*self.name, + ); + } + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + if self.cpu_maps && debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.as_mut_slice().fill(0xde); + } + + let gpu_ptr = self.top; + if let Err(e) = obj.map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps) { + dev_err!( + &self.dev, + "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n", + &*self.name, + gpu_ptr, + e + ); + return Err(e); + } + + self.mm + .with_inner(|inner| inner.backing_objects.try_reserve(1))?; + + let mut new_top = self.top + size_aligned as u64; + if self.cpu_maps { + let guard = self.min_align.max(mmu::UAT_PGSZ); + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: Adding guard node {:#x}:{:#x}\n", + &*self.name, + new_top, + guard + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: 0, + }; + + let node = match self.mm.reserve_node(inner, new_top, guard as u64, 0) { + Ok(a) => a, + Err(a) => { + dev_err!( + &self.dev, + "HeapAllocator[{}]::add_block: Failed to reserve guard node {:#x}:{:#x}: {:?}\n", + &*self.name, + guard, + new_top, + a + ); + return Err(EIO); + } + }; + + self.guard_nodes.try_push(node)?; + + new_top += guard as u64; + } + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: top={:#x}\n", + &*self.name, + new_top + ); + + self.mm + .with_inner(|inner| inner.backing_objects.try_push((obj, gpu_ptr)))?; + + self.top = new_top; + + cls_dev_dbg!( + MemStats, + &self.dev, + "{} Heap: grow to {} bytes\n", + &*self.name, + self.top - self.start + ); + + Ok(()) + } + + /// Find the backing object index that backs a given GPU address. + fn find_obj(&mut self, addr: u64) -> Result { + self.mm.with_inner(|inner| { + inner + .backing_objects + .binary_search_by(|obj| { + let start = obj.1; + let end = obj.1 + obj.0.size() as u64; + if start > addr { + Ordering::Greater + } else if end <= addr { + Ordering::Less + } else { + Ordering::Equal + } + }) + .or(Err(ENOENT)) + }) + } +} + +impl Allocator for HeapAllocator { + type Raw = HeapAllocation; + + fn device(&self) -> &AsahiDevice { + &self.dev + } + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + fn alloc(&mut self, size: usize, align: usize) -> Result { + if align != 0 && !align.is_power_of_two() { + return Err(EINVAL); + } + let align = self.min_align.max(align); + let size_aligned = (size + align - 1) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::new: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: size, + }; + + let mut node = match self.mm.insert_node_generic( + inner, + size_aligned as u64, + align as u64, + 0, + mm::InsertMode::Best, + ) { + Ok(a) => a, + Err(a) => { + dev_err!( + &self.dev, + "HeapAllocator[{}]::new: Failed to insert node of size {:#x} / align {:#x}: {:?}\n", + &*self.name, size_aligned, align, a + ); + return Err(a); + } + }; + + self.mm.with_inner(|inner| inner.allocated += size); + + let mut new_object = false; + let start = node.start(); + let end = start + node.size(); + if end > self.top { + if start > self.top { + dev_warn!( + self.dev, + "HeapAllocator[{}]::alloc: top={:#x}, start={:#x}\n", + &*self.name, + self.top, + start + ); + } + let block_size = self.block_size.max((end - self.top) as usize); + self.add_block(block_size)?; + new_object = true; + } + assert!(end <= self.top); + + if self.cpu_maps { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: mapping to CPU\n", + &*self.name + ); + + let idx = if new_object { + None + } else { + Some(match self.find_obj(start) { + Ok(a) => a, + Err(_) => { + dev_warn!( + self.dev, + "HeapAllocator[{}]::alloc: Failed to find object at {:#x}\n", + &*self.name, + start + ); + return Err(EIO); + } + }) + }; + let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { + let idx = idx.unwrap_or(inner.backing_objects.len() - 1); + let obj = &mut inner.backing_objects[idx]; + let p = obj.0.vmap()?.as_mut_ptr() as *mut u8; + Ok((obj.1, obj.0.size(), p)) + })?; + assert!(obj_start <= start); + assert!(obj_start + obj_size as u64 >= end); + node.as_mut().inner_mut().ptr = + NonNull::new(unsafe { p.add((start - obj_start) as usize) }); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: CPU pointer = {:?}\n", + &*self.name, + node.ptr + ); + } + + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: Allocated {:#x} bytes @ {:#x}\n", + &*self.name, + end - start, + start + ); + + Ok(HeapAllocation(Some(node))) + } + + fn garbage(&self) -> (usize, usize) { + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_ref() { + (g.len(), inner.total_garbage) + } else { + (0, 0) + } + }) + } + + fn collect_garbage(&mut self, count: usize) { + // Take the garbage out of the inner block, so we can safely drop it without deadlocking + let mut garbage = Vec::new(); + + if garbage.try_reserve(count).is_err() { + dev_crit!( + self.dev, + "HeapAllocator[{}]:collect_garbage: failed to reserve space\n", + &*self.name, + ); + return; + } + + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_mut() { + for node in g.drain(0..count) { + inner.total_garbage -= node.size() as usize; + garbage + .try_push(node) + .expect("try_push() failed after reserve()"); + } + } + }); + } +} + +impl Drop for HeapAllocatorInner { + fn drop(&mut self) { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]: dropping allocator\n", + &*self.name + ); + if self.allocated > 0 { + // This should never happen + dev_crit!( + self.dev, + "HeapAllocator[{}]: dropping with {} bytes allocated\n", + &*self.name, + self.allocated + ); + } else { + for mut obj in self.backing_objects.drain(..) { + obj.0.drop_vm_mappings(self.vm_id); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs new file mode 100644 index 00000000000000..425e8d6ada70e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "1024"] +#![allow(dead_code)] +#![allow(unused_imports)] +#![allow(unused_variables)] + +//! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. + +mod alloc; +mod buffer; +mod channel; +mod debug; +mod driver; +mod event; +mod file; +mod float; +mod fw; +mod gem; +mod gpu; +mod hw; +mod initdata; +mod mem; +mod microseq; +mod mmu; +mod object; +mod place; +mod queue; +mod regs; +mod slotalloc; +mod util; +mod workqueue; + +use kernel::module_platform_driver; + +module_platform_driver! { + type: driver::AsahiDriver, + name: "asahi", + license: "Dual MIT/GPL", + params: { + debug_flags: u64 { + default: 0, + permissions: 0o644, + description: "Debug flags", + }, + fault_control: u32 { + default: 0, + permissions: 0, + description: "Fault control (0x0: hard faults, 0xb: macOS default)", + }, + initial_tvb_size: usize { + default: 0x8, + permissions: 0o644, + description: "Initial TVB size in blocks", + }, + }, +} diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs new file mode 100644 index 00000000000000..767ea161176f2d --- /dev/null +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -0,0 +1,694 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tiled Vertex Buffer management +//! +//! This module manages the Tiled Vertex Buffer, also known as the Parameter Buffer (in imgtec +//! parlance) or the tiler heap (on other architectures). This buffer holds transformed primitive +//! data between the vertex/tiling stage and the fragment stage. +//! +//! On AGX, the buffer is a heap of 128K blocks split into 32K pages (which must be aligned to a +//! multiple of 32K in VA space). The buffer can be shared between multiple render jobs, and each +//! will allocate pages from it during vertex processing and return them during fragment processing. +//! +//! If the buffer runs out of free pages, the vertex pass stops and a partial fragment pass occurs, +//! spilling the intermediate render target state to RAM (a partial render). This is all managed +//! transparently by the firmware. Since partial renders are less efficient, the kernel must grow +//! the heap in response to feedback from the firmware to avoid partial renders in the future. +//! Currently, we only ever grow the heap, and never shrink it. +//! +//! AGX also supports memoryless render targets, which can be used for intermediate results within +//! a render pass. To support partial renders, it seems the GPU/firmware has the ability to borrow +//! pages from the TVB buffer as a temporary render target buffer. Since this happens during a +//! partial render itself, if the buffer runs out of space, it requires synchronous growth in +//! response to a firmware interrupt. This is not currently supported, but may be in the future, +//! though it is unclear whether it is worth the effort. +//! +//! This module is also in charge of managing the temporary objects associated with a single render +//! pass, which includes the top-level tile array, the tail pointer cache, preemption buffers, and +//! other miscellaneous structures collectively managed as a "scene". +//! +//! To avoid runaway memory usage, there is a maximum size for buffers (at that point it's unlikely +//! that partial renders will incur much overhead over the buffer data access itself). This is +//! different depending on whether memoryless render targets are in use, and is currently hardcoded. +//! to the most common value used by macOS. + +use crate::debug::*; +use crate::fw::buffer; +use crate::fw::types::*; +use crate::util::*; +use crate::{alloc, fw, gpu, mmu, slotalloc}; +use crate::{box_in_place, place}; +use core::sync::atomic::Ordering; +use kernel::prelude::*; +use kernel::sync::{smutex::Mutex, Arc}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; + +/// There are 127 GPU/firmware-side buffer manager slots (yes, 127, not 128). +const NUM_BUFFERS: u32 = 127; + +/// Page size bits for buffer pages (32K). VAs must be aligned to this size. +pub(crate) const PAGE_SHIFT: usize = 15; +/// Page size for buffer pages. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_SHIFT; +/// Number of pages in a buffer block, which should be contiguous in VA space. +pub(crate) const PAGES_PER_BLOCK: usize = 4; +/// Size of a buffer block. +pub(crate) const BLOCK_SIZE: usize = PAGE_SIZE * PAGES_PER_BLOCK; + +/// Metadata about the tiling configuration for a scene. This is computed in the `render` module. +/// based on dimensions, tile size, and other info. +pub(crate) struct TileInfo { + /// Tile count in the X dimension. Tiles are always 32x32. + pub(crate) tiles_x: u32, + /// Tile count in the Y dimension. Tiles are always 32x32. + pub(crate) tiles_y: u32, + /// Total tile count. + pub(crate) tiles: u32, + /// Micro-tile width (16 or 32). + pub(crate) utile_width: u32, + /// Micro-tile height (16 or 32). + pub(crate) utile_height: u32, + // Macro-tiles in the X dimension. Always 4. + //pub(crate) mtiles_x: u32, + // Macro-tiles in the Y dimension. Always 4. + //pub(crate) mtiles_y: u32, + /// Tiles per macro-tile in the X dimension. + pub(crate) tiles_per_mtile_x: u32, + /// Tiles per macro-tile in the Y dimension. + pub(crate) tiles_per_mtile_y: u32, + // Total tiles per macro-tile. + //pub(crate) tiles_per_mtile: u32, + /// Micro-tiles per macro-tile in the X dimension. + pub(crate) utiles_per_mtile_x: u32, + /// Micro-tiles per macro-tile in the Y dimension. + pub(crate) utiles_per_mtile_y: u32, + // Total micro-tiles per macro-tile. + //pub(crate) utiles_per_mtile: u32, + /// Size of the top-level tilemap, in bytes (for all layers, one cluster). + pub(crate) tilemap_size: usize, + /// Size of the Tail Pointer Cache, in bytes (for all layers * clusters). + pub(crate) tpc_size: usize, + /// Number of blocks in the clustering meta buffer (for clustering). + pub(crate) meta1_blocks: u32, + /// Minimum number of TVB blocks for this render. + pub(crate) min_tvb_blocks: usize, + /// XXX: Allocation factor for cluster tilemaps and meta4. Always 2? + pub(crate) cluster_factor: usize, + /// Tiling parameter structure passed to firmware. + pub(crate) params: fw::vertex::raw::TilingParameters, +} + +/// A single scene, representing a render pass and its required buffers. +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Scene { + object: GpuObject, + slot: u32, + rebind: bool, + preempt2_off: usize, + preempt3_off: usize, + // Note: these are dead code only on some version variants. + // It's easier to do this than to propagate the version conditionals everywhere. + #[allow(dead_code)] + meta2_off: usize, + #[allow(dead_code)] + meta3_off: usize, + #[allow(dead_code)] + meta4_off: usize, +} + +#[versions(AGX)] +impl Scene::ver { + /// Returns true if the buffer was bound to a fresh manager slot, and therefore needs an init + /// command before a render. + pub(crate) fn rebind(&self) -> bool { + self.rebind + } + + /// Returns the buffer manager slot this scene's buffer was bound to. + pub(crate) fn slot(&self) -> u32 { + self.slot + } + + /// Returns the GPU pointer to the [`buffer::Scene::ver`]. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, buffer::Scene::ver> { + self.object.gpu_pointer() + } + + /// Returns the GPU weak pointer to the [`buffer::Scene::ver`]. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + self.object.weak_pointer() + } + + /// Returns the GPU weak pointer to the kernel-side temp buffer. + /// (purpose unknown...) + pub(crate) fn kernel_buffer_pointer(&self) -> GpuWeakPointer<[u8]> { + self.object.buffer.inner.lock().kernel_buffer.weak_pointer() + } + + /// Returns the GPU pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn buffer_pointer(&self) -> GpuPointer<'_, buffer::Info::ver> { + // We can't return the strong pointer directly since its lifetime crosses a lock, but we know + // its lifetime will be valid as long as &self since we hold a reference to the buffer, + // so just construct the strong pointer with the right lifetime here. + unsafe { self.weak_buffer_pointer().upgrade() } + } + + /// Returns the GPU weak pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn weak_buffer_pointer(&self) -> GpuWeakPointer { + self.object.buffer.inner.lock().info.weak_pointer() + } + + /// Returns the GPU pointer to the TVB heap metadata buffer. + pub(crate) fn tvb_heapmeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_pointer() + } + + /// Returns the GPU pointer to the top-level TVB tilemap buffer. + pub(crate) fn tvb_tilemap_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_tilemap.gpu_pointer() + } + + /// Returns the GPU pointer to the Tail Pointer Cache buffer. + pub(crate) fn tpc_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tpc.gpu_pointer() + } + + /// Returns the GPU pointer to the first preemption scratch buffer. + pub(crate) fn preempt_buf_1_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.preempt_buf.gpu_pointer() + } + + /// Returns the GPU pointer to the second preemption scratch buffer. + pub(crate) fn preempt_buf_2_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt2_off) + } + + /// Returns the GPU pointer to the third preemption scratch buffer. + pub(crate) fn preempt_buf_3_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt3_off) + } + + /// Returns the GPU pointer to the per-cluster tilemap buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn cluster_tilemaps_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.tilemaps.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering metadata 1 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_1_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering metadata 2 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_2_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta2_off)) + } + + /// Returns the GPU pointer to the clustering metadata 3 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_3_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta3_off)) + } + + /// Returns the GPU pointer to the clustering metadata 4 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_4_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta4_off)) + } + + /// Returns the GPU pointer to an unknown buffer with incrementing numbers. + pub(crate) fn seq_buf_pointer(&self) -> GpuPointer<'_, &'_ [u64]> { + self.object.seq_buf.gpu_pointer() + } + + /// Returns the number of TVB bytes used for this scene. + pub(crate) fn used_bytes(&self) -> usize { + self.object + .with(|raw, _inner| raw.total_page_count.load(Ordering::Relaxed) as usize * PAGE_SIZE) + } + + /// Returns whether the TVB overflowed while rendering this scene. + pub(crate) fn overflowed(&self) -> bool { + self.object.with(|raw, _inner| { + raw.total_page_count.load(Ordering::Relaxed) + > raw.pass_page_count.load(Ordering::Relaxed) + }) + } +} + +#[versions(AGX)] +impl Drop for Scene::ver { + fn drop(&mut self) { + let mut inner = self.object.buffer.inner.lock(); + assert_ne!(inner.active_scenes, 0); + inner.active_scenes -= 1; + + if inner.active_scenes == 0 { + mod_pr_debug!( + "Buffer: no scenes left, dropping slot {}", + inner.active_slot.take().unwrap().slot() + ); + inner.active_slot = None; + } + } +} + +/// Inner data for a single TVB buffer object. +#[versions(AGX)] +struct BufferInner { + info: GpuObject, + ualloc: Arc>, + ualloc_priv: Arc>, + blocks: Vec>, + max_blocks: usize, + max_blocks_nomemless: usize, + mgr: BufferManager, + active_scenes: usize, + active_slot: Option>, + last_token: Option, + tpc: Option>>, + kernel_buffer: GpuArray, + stats: GpuObject, + preempt1_size: usize, + preempt2_size: usize, + preempt3_size: usize, + num_clusters: usize, +} + +/// Locked and reference counted TVB buffer. +#[versions(AGX)] +pub(crate) struct Buffer { + inner: Arc>, +} + +#[versions(AGX)] +impl Buffer::ver { + /// Create a new Buffer for a given VM, given the per-VM allocators. + pub(crate) fn new( + gpu: &dyn gpu::GpuManager, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + mgr: &BufferManager, + ) -> Result { + // These are the typical max numbers on macOS. + // 8GB machines have this halved. + let max_size: usize = 862_322_688; // bytes + let max_size_nomemless = max_size / 3; + + let max_blocks = max_size / BLOCK_SIZE; + let max_blocks_nomemless = max_size_nomemless / BLOCK_SIZE; + let max_pages = max_blocks * PAGES_PER_BLOCK; + let max_pages_nomemless = max_blocks_nomemless * PAGES_PER_BLOCK; + + let num_clusters = gpu.get_dyncfg().id.num_clusters as usize; + let num_clusters_adj = if num_clusters > 1 { + num_clusters + 1 + } else { + 1 + }; + + let preempt1_size = num_clusters_adj * gpu.get_cfg().preempt1_size; + let preempt2_size = num_clusters_adj * gpu.get_cfg().preempt2_size; + let preempt3_size = num_clusters_adj * gpu.get_cfg().preempt3_size; + + let inner = box_in_place!(buffer::Info::ver { + block_ctl: alloc.shared.new_default::()?, + counter: alloc.shared.new_default::()?, + page_list: ualloc_priv.lock().array_empty(max_pages)?, + block_list: ualloc_priv.lock().array_empty(max_blocks * 2)?, + })?; + + let info = alloc.private.new_boxed(inner, |inner, ptr| { + Ok(place!( + ptr, + buffer::raw::Info::ver { + gpu_counter: 0x0, + unk_4: 0, + last_id: 0x0, + cur_id: -1, + unk_10: 0x0, + gpu_counter2: 0x0, + unk_18: 0x0, + #[ver(V < V13_0B4)] + unk_1c: 0x0, + page_list: inner.page_list.gpu_pointer(), + page_list_size: (4 * max_pages).try_into()?, + page_count: AtomicU32::new(0), + max_blocks: max_blocks.try_into()?, + block_count: AtomicU32::new(0), + unk_38: 0x0, + block_list: inner.block_list.gpu_pointer(), + block_ctl: inner.block_ctl.gpu_pointer(), + last_page: AtomicU32::new(0), + gpu_page_ptr1: 0x0, + gpu_page_ptr2: 0x0, + unk_58: 0x0, + block_size: BLOCK_SIZE as u32, + unk_60: U64(0x0), + counter: inner.counter.gpu_pointer(), + unk_70: 0x0, + unk_74: 0x0, + unk_78: 0x0, + unk_7c: 0x0, + unk_80: 0x1, + max_pages: max_pages.try_into()?, + max_pages_nomemless: max_pages_nomemless.try_into()?, + unk_8c: 0x0, + unk_90: Default::default(), + } + )) + })?; + + // Technically similar to Scene below, let's play it safe. + let kernel_buffer = alloc.shared.array_empty(0x40)?; + let stats = alloc + .shared + .new_object(Default::default(), |_inner| buffer::raw::Stats { + reset: AtomicU32::from(1), + ..Default::default() + })?; + + Ok(Buffer::ver { + inner: Arc::try_new(Mutex::new(BufferInner::ver { + info, + ualloc, + ualloc_priv, + blocks: Vec::new(), + max_blocks, + max_blocks_nomemless, + mgr: mgr.clone(), + active_scenes: 0, + active_slot: None, + last_token: None, + tpc: None, + kernel_buffer, + stats, + preempt1_size, + preempt2_size, + preempt3_size, + num_clusters, + }))?, + }) + } + + /// Returns the total block count allocated to this Buffer. + pub(crate) fn block_count(&self) -> u32 { + self.inner.lock().blocks.len() as u32 + } + + /// Returns the total size in bytes allocated to this Buffer. + pub(crate) fn size(&self) -> usize { + self.block_count() as usize * BLOCK_SIZE + } + + /// Automatically grow the Buffer based on feedback from the statistics. + pub(crate) fn auto_grow(&self) -> Result { + let inner = self.inner.lock(); + + let used_pages = inner.stats.with(|raw, _inner| { + let used = raw.max_pages.load(Ordering::Relaxed); + raw.reset.store(1, Ordering::Release); + used as usize + }); + + let need_blocks = div_ceil(used_pages * 2, PAGES_PER_BLOCK).min(inner.max_blocks_nomemless); + let want_blocks = div_ceil(used_pages * 3, PAGES_PER_BLOCK).min(inner.max_blocks_nomemless); + + let cur_count = inner.blocks.len(); + + if need_blocks <= cur_count { + Ok(false) + } else { + // Grow to 3x requested size (same logic as macOS) + core::mem::drop(inner); + self.ensure_blocks(want_blocks)?; + Ok(true) + } + } + + /// Ensure that the buffer has at least a certain minimum size in blocks. + pub(crate) fn ensure_blocks(&self, min_blocks: usize) -> Result { + let mut inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + if cur_count >= min_blocks { + return Ok(false); + } + if min_blocks > inner.max_blocks { + return Err(ENOMEM); + } + + let add_blocks = min_blocks - cur_count; + let new_count = min_blocks; + + let mut new_blocks: Vec> = Vec::new(); + + // Allocate the new blocks first, so if it fails they will be dropped + let mut ualloc = inner.ualloc.lock(); + for _i in 0..add_blocks { + new_blocks.try_push(ualloc.array_gpuonly(BLOCK_SIZE)?)?; + } + core::mem::drop(ualloc); + + // Then actually commit them + inner.blocks.try_reserve(add_blocks)?; + + for (i, block) in new_blocks.into_iter().enumerate() { + let page_num = (block.gpu_va().get() >> PAGE_SHIFT) as u32; + + inner + .blocks + .try_push(block) + .expect("try_push() failed after try_reserve()"); + inner.info.block_list[2 * (cur_count + i)] = page_num; + for j in 0..PAGES_PER_BLOCK { + inner.info.page_list[(cur_count + i) * PAGES_PER_BLOCK + j] = page_num + j as u32; + } + } + + inner.info.block_ctl.with(|raw, _inner| { + raw.total.store(new_count as u32, Ordering::SeqCst); + raw.wptr.store(new_count as u32, Ordering::SeqCst); + }); + + let page_count = (new_count * PAGES_PER_BLOCK) as u32; + inner.info.with(|raw, _inner| { + raw.page_count.store(page_count, Ordering::Relaxed); + raw.block_count.store(new_count as u32, Ordering::Relaxed); + raw.last_page.store(page_count - 1, Ordering::Relaxed); + }); + + Ok(true) + } + + /// Create a new [`Scene::ver`] (render pass) using this buffer. + pub(crate) fn new_scene( + &self, + alloc: &mut gpu::KernelAllocators, + tile_info: &TileInfo, + ) -> Result { + let mut inner = self.inner.lock(); + + let tilemap_size = tile_info.tilemap_size; + let tpc_size = tile_info.tpc_size; + + // TODO: what is this exactly? + mod_pr_debug!("Buffer: Allocating TVB buffers\n"); + + // This seems to be a list, with 4x2 bytes of headers and 8 bytes per entry. + // On single-cluster devices, the used length always seems to be 1. + // On M1 Ultra, it can grow and usually doesn't exceed 8 * cluster_factor + // entries. macOS allocates a whole 64K * 0x80 for this, so let's go with + // that to be safe... + let user_buffer = inner.ualloc.lock().array_empty(if inner.num_clusters > 1 { + 0x10080 + } else { + 0x80 + })?; + + let tvb_heapmeta = inner.ualloc.lock().array_empty(0x200)?; + let tvb_tilemap = inner.ualloc.lock().array_empty(tilemap_size)?; + + mod_pr_debug!("Buffer: Allocating misc buffers\n"); + let preempt_buf = inner + .ualloc + .lock() + .array_empty(inner.preempt1_size + inner.preempt2_size + inner.preempt3_size)?; + + let mut seq_buf = inner.ualloc.lock().array_empty(0x800)?; + for i in 1..0x400 { + seq_buf[i] = (i + 1) as u64; + } + + let tpc = match inner.tpc.as_ref() { + Some(buf) if buf.len() >= tpc_size => buf.clone(), + _ => { + // MacOS allocates this as shared GPU+FW, but + // priv seems to work and might be faster? + // Needs to be FW-writable anyway, so ualloc + // won't work. + let buf = Arc::try_new( + inner + .ualloc_priv + .lock() + .array_empty((tpc_size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK)?, + )?; + inner.tpc = Some(buf.clone()); + buf + } + }; + + // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) + let meta1_size = align(tile_info.meta1_blocks as usize * 0x44, 0x80); + // check + let meta2_size = align(0x190 * inner.num_clusters, 0x80); + let meta3_size = align(0x280 * inner.num_clusters, 0x80); + // Like user_buffer for single-cluster modes, 0x30 per cluster * the cluster + // factor. + let meta4_size = align(0x30 * inner.num_clusters * tile_info.cluster_factor, 0x80); + let meta_size = meta1_size + meta2_size + meta3_size + meta4_size; + + let clustering = if inner.num_clusters > 1 { + mod_pr_debug!("Buffer: Allocating clustering buffers\n"); + let tilemaps = inner + .ualloc + .lock() + .array_empty(inner.num_clusters * tilemap_size * tile_info.cluster_factor)?; + let meta = inner.ualloc.lock().array_empty(meta_size)?; + Some(buffer::ClusterBuffers { tilemaps, meta }) + } else { + None + }; + + let scene_inner = box_in_place!(buffer::Scene::ver { + user_buffer: user_buffer, + buffer: self.clone(), + tvb_heapmeta: tvb_heapmeta, + tvb_tilemap: tvb_tilemap, + tpc: tpc, + clustering: clustering, + preempt_buf: preempt_buf, + seq_buf: seq_buf, + })?; + + // Could be made strong, but we wind up with a deadlock if we try to grab the + // pointer through the inner.buffer path inside the closure. + let stats_pointer = inner.stats.weak_pointer(); + + // macOS allocates this as private. However, the firmware does not + // DC CIVAC this before reading it (like it does most other things), + // which causes odd cache incoherency bugs when combined with + // speculation on the firmware side (maybe). This doesn't happen + // on macOS because these structs are a circular pool that is mapped + // already initialized. Just mark this shared for now. + let scene = alloc.shared.new_boxed(scene_inner, |inner, ptr| { + Ok(place!( + ptr, + buffer::raw::Scene { + pass_page_count: AtomicU32::new(0), + unk_4: 0, + unk_8: U64(0), + unk_10: U64(0), + user_buffer: inner.user_buffer.gpu_pointer(), + unk_20: 0, + stats: stats_pointer, + total_page_count: AtomicU32::new(0), + unk_30: U64(0), + unk_38: U64(0), + } + )) + })?; + + let mut rebind = false; + + if inner.active_slot.is_none() { + assert_eq!(inner.active_scenes, 0); + + let slot = inner.mgr.0.get(inner.last_token)?; + rebind = slot.changed(); + + mod_pr_debug!("Buffer: assigning slot {} (rebind={})", slot.slot(), rebind); + + inner.last_token = Some(slot.token()); + inner.active_slot = Some(slot); + } + + inner.active_scenes += 1; + + Ok(Scene::ver { + object: scene, + slot: inner.active_slot.as_ref().unwrap().slot(), + rebind, + preempt2_off: inner.preempt1_size, + preempt3_off: inner.preempt1_size + inner.preempt2_size, + meta2_off: meta1_size, + meta3_off: meta1_size + meta2_size, + meta4_off: meta1_size + meta2_size + meta3_size, + }) + } + + /// Increment the buffer manager usage count. Should we done once we know the Scene is ready + /// to be committed and used in commands submitted to the GPU. + pub(crate) fn increment(&self) { + let inner = self.inner.lock(); + inner.info.counter.with(|raw, _inner| { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + // We have inner locked anyway, so this is not racy. + let v = raw.count.load(Ordering::Relaxed); + raw.count.store(v + 1, Ordering::Relaxed); + }); + } +} + +#[versions(AGX)] +impl Clone for Buffer::ver { + fn clone(&self) -> Self { + Buffer::ver { + inner: self.inner.clone(), + } + } +} + +/// The GPU-global buffer manager, used to allocate and release buffer slots from the pool. +pub(crate) struct BufferManager(slotalloc::SlotAllocator<()>); + +impl BufferManager { + pub(crate) fn new() -> Result { + Ok(BufferManager(slotalloc::SlotAllocator::new( + NUM_BUFFERS, + (), + |_inner, _slot| (), + )?)) + } +} + +impl Clone for BufferManager { + fn clone(&self) -> Self { + BufferManager(self.0.clone()) + } +} diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs new file mode 100644 index 00000000000000..03aeecc75da102 --- /dev/null +++ b/drivers/gpu/drm/asahi/channel.rs @@ -0,0 +1,530 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU ring buffer channels +//! +//! The GPU firmware use a set of ring buffer channels to receive commands from the driver and send +//! it notifications and status messages. +//! +//! These ring buffers mostly follow uniform conventions, so they share the same base +//! implementation. + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::fw::channels::*; +use crate::fw::initdata::{raw, ChannelRing}; +use crate::fw::types::*; +use crate::{event, gpu, mem}; +use core::time::Duration; +use kernel::{c_str, delay::coarse_sleep, prelude::*, sync::Arc, time}; + +pub(crate) use crate::fw::channels::PipeType; + +/// A receive (FW->driver) channel. +pub(crate) struct RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroed, +{ + ring: ChannelRing, + // FIXME: needs feature(generic_const_exprs) + //rptr: [u32; T::SUB_CHANNELS], + rptr: [u32; 6], + count: u32, +} + +impl RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroed, +{ + /// Allocates a new receive channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(RxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(T::SUB_CHANNELS * count)?, + }, + rptr: Default::default(), + count: count as u32, + }) + } + + /// Receives a message on the specified sub-channel index, optionally leaving in the ring + /// buffer. + /// + /// Returns None if the channel is empty. + fn get_or_peek(&mut self, index: usize, peek: bool) -> Option { + self.ring.state.with(|raw, _inner| { + let wptr = T::wptr(raw, index); + let rptr = &mut self.rptr[index]; + if wptr == *rptr { + None + } else { + let off = self.count as usize * index; + let msg = self.ring.ring[off + *rptr as usize]; + if !peek { + *rptr = (*rptr + 1) % self.count; + T::set_rptr(raw, index, *rptr); + } + Some(msg) + } + }) + } + + /// Receives a message on the specified sub-channel index, and dequeues it from the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn get(&mut self, index: usize) -> Option { + self.get_or_peek(index, false) + } + + /// Peeks a message on the specified sub-channel index, leaving it in the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn peek(&mut self, index: usize) -> Option { + self.get_or_peek(index, true) + } +} + +/// A transmit (driver->FW) channel. +pub(crate) struct TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroed, +{ + ring: ChannelRing, + wptr: u32, + count: u32, +} + +impl TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroed, +{ + /// Allocates a new cached transmit channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.private.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Allocates a new uncached transmit channel with a given message count. + pub(crate) fn new_uncached( + alloc: &mut gpu::KernelAllocators, + count: usize, + ) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Send a message to the ring, returning a cookie with the ring buffer position. + /// + /// This will poll/block if the ring is full, which we don't really expect to happen. + pub(crate) fn put(&mut self, msg: &U) -> u32 { + self.ring.state.with(|raw, _inner| { + let next_wptr = (self.wptr + 1) % self.count; + let mut rptr = T::rptr(raw); + if next_wptr == rptr { + pr_err!( + "TX ring buffer is full! Waiting... ({}, {})\n", + next_wptr, + rptr + ); + // TODO: block properly on incoming messages? + while next_wptr == rptr { + coarse_sleep(Duration::from_millis(8)); + rptr = T::rptr(raw); + } + } + self.ring.ring[self.wptr as usize] = *msg; + mem::sync(); + T::set_wptr(raw, next_wptr); + self.wptr = next_wptr; + }); + self.wptr + } + + /// Wait for a previously submitted message to be popped off of the ring by the GPU firmware. + /// + /// This busy-loops, and is intended to be used for rare cases when we need to block for + /// completion of a cache management or invalidation operation synchronously. + pub(crate) fn wait_for(&mut self, wptr: u32, timeout_ms: u64) -> Result { + let timeout = time::ktime_get() + Duration::from_millis(timeout_ms); + self.ring.state.with(|raw, _inner| { + while time::ktime_get() < timeout { + if T::rptr(raw) == wptr { + return Ok(()); + } + mem::sync(); + } + Err(ETIMEDOUT) + }) + } +} + +/// Device Control channel for global device management commands. +#[versions(AGX)] +pub(crate) struct DeviceControlChannel { + dev: AsahiDevice, + ch: TxChannel, +} + +#[versions(AGX)] +impl DeviceControlChannel::ver { + const COMMAND_TIMEOUT_MS: u64 = 100; + + /// Allocate a new Device Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(DeviceControlChannel::ver { + dev: dev.clone(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Device Control command. + pub(crate) fn send(&mut self, msg: &DeviceControlMsg::ver) -> u32 { + cls_dev_dbg!(DeviceControlCh, self.dev, "DeviceControl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Device Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Pipe channel to submit WorkQueue execution requests. +#[versions(AGX)] +pub(crate) struct PipeChannel { + dev: AsahiDevice, + ch: TxChannel, +} + +#[versions(AGX)] +impl PipeChannel::ver { + /// Allocate a new Pipe submission channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(PipeChannel::ver { + dev: dev.clone(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Pipe kick command to the firmware. + pub(crate) fn send(&mut self, msg: &PipeMsg::ver) { + cls_dev_dbg!(PipeCh, self.dev, "Pipe: {:?}\n", msg); + self.ch.put(msg); + } +} + +/// Firmware Control channel, used for secure cache flush requests. +pub(crate) struct FwCtlChannel { + dev: AsahiDevice, + ch: TxChannel, +} + +impl FwCtlChannel { + const COMMAND_TIMEOUT_MS: u64 = 100; + + /// Allocate a new Firmware Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwCtlChannel { + dev: dev.clone(), + ch: TxChannel::::new_uncached(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Firmware Control command to the firmware. + pub(crate) fn send(&mut self, msg: &FwCtlMsg) -> u32 { + cls_dev_dbg!(FwCtlCh, self.dev, "FwCtl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Firmware Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Event channel, used to notify the driver of command completions, GPU faults and errors, and +/// other events. +pub(crate) struct EventChannel { + dev: AsahiDevice, + ch: RxChannel, + mgr: Arc, + gpu: Option>, +} + +impl EventChannel { + /// Allocate a new Event channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + mgr: Arc, + ) -> Result { + Ok(EventChannel { + dev: dev.clone(), + ch: RxChannel::::new(alloc, 0x100)?, + mgr, + gpu: None, + }) + } + + /// Registers the managing `Gpu` instance that will handle events on this channel. + pub(crate) fn set_manager(&mut self, gpu: Arc) { + self.gpu = Some(gpu); + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new Event messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=EVENT_MAX => { + let msg = unsafe { msg.msg }; + + cls_dev_dbg!(EventCh, self.dev, "Event: {:?}\n", msg); + match msg { + EventMsg::Fault => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_fault(), + None => { + dev_crit!(self.dev, "EventChannel: No GPU manager available!\n") + } + }, + EventMsg::Timeout { + counter, + event_slot, + .. + } => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_timeout(counter, event_slot), + None => { + dev_crit!(self.dev, "EventChannel: No GPU manager available!\n") + } + }, + EventMsg::Flag { firing, .. } => { + for (i, flags) in firing.iter().enumerate() { + for j in 0..32 { + if flags & (1u32 << j) != 0 { + self.mgr.signal((i * 32 + j) as u32); + } + } + } + } + msg => { + dev_crit!(self.dev, "Unknown event message: {:?}\n", msg); + } + } + } + _ => { + dev_warn!(self.dev, "Unknown event message: {:?}\n", unsafe { + msg.raw + }); + } + } + } + } +} + +/// Firmware Log channel. This one is pretty special, since it has 6 sub-channels (for different log +/// levels), and it also uses a side buffer to actually hold the log messages, only passing around +/// pointers in the main buffer. +pub(crate) struct FwLogChannel { + dev: AsahiDevice, + ch: RxChannel, + payload_buf: GpuArray, +} + +impl FwLogChannel { + const RING_SIZE: usize = 0x100; + const BUF_SIZE: usize = 0x100; + + /// Allocate a new Firmware Log channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwLogChannel { + dev: dev.clone(), + ch: RxChannel::::new(alloc, Self::RING_SIZE)?, + payload_buf: alloc + .shared + .array_empty(Self::BUF_SIZE * FwLogChannelState::SUB_CHANNELS)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Returns the GPU pointers to the firmware log payload buffer. + pub(crate) fn get_buf(&self) -> GpuWeakPointer<[RawFwLogPayloadMsg]> { + self.payload_buf.weak_pointer() + } + + /// Polls for new log messages on all sub-rings. + pub(crate) fn poll(&mut self) { + for i in 0..=FwLogChannelState::SUB_CHANNELS - 1 { + while let Some(msg) = self.ch.peek(i) { + cls_dev_dbg!(FwLogCh, self.dev, "FwLog{}: {:?}\n", i, msg); + if msg.msg_type != 2 { + dev_warn!(self.dev, "Unknown FWLog{} message: {:?}\n", i, msg); + self.ch.get(i); + continue; + } + if msg.msg_index.0 as usize >= Self::BUF_SIZE { + dev_warn!( + self.dev, + "FWLog{} message index out of bounds: {:?}\n", + i, + msg + ); + self.ch.get(i); + continue; + } + let index = Self::BUF_SIZE * i + msg.msg_index.0 as usize; + let payload = &self.payload_buf.as_slice()[index]; + if payload.msg_type != 3 { + dev_warn!(self.dev, "Unknown FWLog{} payload: {:?}\n", i, payload); + self.ch.get(i); + continue; + } + let msg = if let Some(end) = payload.msg.iter().position(|&r| r == 0) { + CStr::from_bytes_with_nul(&(*payload.msg)[..end + 1]) + .unwrap_or(c_str!("cstr_err")) + } else { + dev_warn!( + self.dev, + "FWLog{} payload not NUL-terminated: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + }; + match i { + 0 => dev_dbg!(self.dev, "FWLog: {}\n", msg), + 1 => dev_info!(self.dev, "FWLog: {}\n", msg), + 2 => dev_notice!(self.dev, "FWLog: {}\n", msg), + 3 => dev_warn!(self.dev, "FWLog: {}\n", msg), + 4 => dev_err!(self.dev, "FWLog: {}\n", msg), + 5 => dev_crit!(self.dev, "FWLog: {}\n", msg), + _ => (), + }; + self.ch.get(i); + } + } + } +} + +pub(crate) struct KTraceChannel { + dev: AsahiDevice, + ch: RxChannel, +} + +/// KTrace channel, used to receive detailed execution trace markers from the firmware. +/// We currently disable this in initdata, so no messages are expected here at this time. +impl KTraceChannel { + /// Allocate a new KTrace channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(KTraceChannel { + dev: dev.clone(), + ch: RxChannel::::new(alloc, 0x200)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new KTrace messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + cls_dev_dbg!(KTraceCh, self.dev, "KTrace: {:?}\n", msg); + } + } +} + +/// Statistics channel, reporting power-related statistics to the driver. +/// Not really implemented other than debug logs yet... +#[versions(AGX)] +pub(crate) struct StatsChannel { + dev: AsahiDevice, + ch: RxChannel, +} + +#[versions(AGX)] +impl StatsChannel::ver { + /// Allocate a new Statistics channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(StatsChannel::ver { + dev: dev.clone(), + ch: RxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new statistics messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=STATS_MAX::ver => { + let msg = unsafe { msg.msg }; + cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + } + _ => { + pr_warn!("Unknown stats message: {:?}\n", unsafe { msg.raw }); + } + } + } + } +} diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs new file mode 100644 index 00000000000000..2f3a70e04cfd6b --- /dev/null +++ b/drivers/gpu/drm/asahi/debug.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(dead_code)] + +//! Debug enable/disable flags and convenience macros + +#[allow(unused_imports)] +pub(crate) use super::{cls_dev_dbg, cls_pr_debug, debug, mod_dev_dbg, mod_pr_debug}; +use core::sync::atomic::{AtomicU64, Ordering}; + +static DEBUG_FLAGS: AtomicU64 = AtomicU64::new(0); + +/// Debug flag bit indices +pub(crate) enum DebugFlags { + // 0-3: Memory-related debug + Mmu = 0, + Alloc = 1, + Gem = 2, + Object = 3, + + // 4-7: Firmware objects and resources + Event = 4, + Buffer = 5, + WorkQueue = 6, + + // 8-13: DRM interface, rendering, compute, GPU globals + Gpu = 8, + File = 9, + Queue = 10, + Render = 11, + Compute = 12, + + // 14-15: Misc stats + MemStats = 14, + TVBStats = 15, + + // 16-22: Channels + FwLogCh = 16, + KTraceCh = 17, + StatsCh = 18, + EventCh = 19, + PipeCh = 20, + DeviceControlCh = 21, + FwCtlCh = 22, + + // 32-35: Allocator debugging + FillAllocations = 32, + DebugAllocations = 33, + DetectOverflows = 34, + ForceCPUMaps = 35, + + // 36-: Behavior flags + ConservativeTlbi = 36, + KeepGpuPowered = 37, + WaitForPowerOff = 38, + NoGpuRecovery = 39, + DisableClustering = 40, + + // 48-: Misc + Debug0 = 48, + Debug1 = 49, + Debug2 = 50, + Debug3 = 51, + Debug4 = 52, + Debug5 = 53, + Debug6 = 54, + Debug7 = 55, +} + +/// Update the cached global debug flags from the module parameter +pub(crate) fn update_debug_flags() { + let flags = { + let lock = crate::THIS_MODULE.kernel_param_lock(); + *crate::debug_flags.read(&lock) + }; + + DEBUG_FLAGS.store(flags, Ordering::Relaxed); +} + +/// Check whether debug is enabled for a given flag +#[inline(always)] +pub(crate) fn debug_enabled(flag: DebugFlags) -> bool { + DEBUG_FLAGS.load(Ordering::Relaxed) & 1 << (flag as usize) != 0 +} + +/// Run some code only if debug is enabled for the calling module +#[macro_export] +macro_rules! debug { + ($($arg:tt)*) => { + if $crate::debug::debug_enabled(DEBUG_CLASS) { + $($arg)* + } + }; +} + +/// pr_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_pr_debug ( + ($($arg:tt)*) => ( + $crate::debug! { ::kernel::pr_info! ( $($arg)* ); } + ) +); + +/// dev_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_dev_dbg ( + ($($arg:tt)*) => ( + $crate::debug! { ::kernel::dev_info! ( $($arg)* ); } + ) +); + +/// pr_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_pr_debug ( + ($cls:ident, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::pr_info! ( $($arg)* ); + } + ) +); + +/// dev_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_dev_dbg ( + ($cls:ident, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::dev_info! ( $($arg)* ); + } + ) +); diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs new file mode 100644 index 00000000000000..d49d8b1934a4f5 --- /dev/null +++ b/drivers/gpu/drm/asahi/driver.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU driver implementation. + +use kernel::{ + c_str, device, drm, drm::drv, drm::ioctl, error::Result, of, platform, prelude::*, sync::Arc, +}; + +use crate::{debug, file, gem, gpu, hw, regs}; + +use kernel::device::RawDevice; +use kernel::macros::vtable; + +/// Driver metadata +const INFO: drv::DriverInfo = drv::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c_str!("asahi"), + desc: c_str!("Apple AGX Graphics"), + date: c_str!("20220831"), +}; + +/// Device data for the driver registration. +/// +/// Holds a reference to the top-level `GpuManager` object. +pub(crate) struct AsahiData { + pub(crate) dev: device::Device, + pub(crate) gpu: Arc, +} + +/// Convenience type alias for the `device::Data` type for this driver. +type DeviceData = device::Data, regs::Resources, AsahiData>; + +/// Empty struct representing this driver. +pub(crate) struct AsahiDriver; + +/// Convenience type alias for the DRM device type for this driver. +pub(crate) type AsahiDevice = kernel::drm::device::Device; + +/// DRM Driver implementation for `AsahiDriver`. +#[vtable] +impl drv::Driver for AsahiDriver { + /// Our `DeviceData` type, reference-counted + type Data = Arc; + /// Our `File` type. + type File = file::File; + /// Our `Object` type. + type Object = gem::Object; + + const INFO: drv::DriverInfo = INFO; + const FEATURES: u32 = + drv::FEAT_GEM | drv::FEAT_RENDER | drv::FEAT_SYNCOBJ | drv::FEAT_SYNCOBJ_TIMELINE; + + kernel::declare_drm_ioctls! { + (ASAHI_GET_PARAMS, drm_asahi_get_params, + ioctl::RENDER_ALLOW, file::File::get_params), + (ASAHI_VM_CREATE, drm_asahi_vm_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::vm_create), + (ASAHI_VM_DESTROY, drm_asahi_vm_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::vm_destroy), + (ASAHI_GEM_CREATE, drm_asahi_gem_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::gem_create), + (ASAHI_GEM_MMAP_OFFSET, drm_asahi_gem_mmap_offset, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::gem_mmap_offset), + (ASAHI_GEM_BIND, drm_asahi_gem_bind, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::gem_bind), + (ASAHI_QUEUE_CREATE, drm_asahi_queue_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::queue_create), + (ASAHI_QUEUE_DESTROY, drm_asahi_queue_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::queue_destroy), + (ASAHI_SUBMIT, drm_asahi_submit, + ioctl::AUTH | ioctl::RENDER_ALLOW, file::File::submit), + } +} + +// OF Device ID table. +kernel::define_of_id_table! {ASAHI_ID_TABLE, &'static hw::HwConfig, [ + (of::DeviceId::Compatible(b"apple,agx-t8103"), Some(&hw::t8103::HWCONFIG)), + (of::DeviceId::Compatible(b"apple,agx-t8112"), Some(&hw::t8112::HWCONFIG)), + (of::DeviceId::Compatible(b"apple,agx-t6000"), Some(&hw::t600x::HWCONFIG_T6000)), + (of::DeviceId::Compatible(b"apple,agx-t6001"), Some(&hw::t600x::HWCONFIG_T6001)), + (of::DeviceId::Compatible(b"apple,agx-t6002"), Some(&hw::t600x::HWCONFIG_T6002)), +]} + +/// Platform Driver implementation for `AsahiDriver`. +impl platform::Driver for AsahiDriver { + /// Our `DeviceData` type, reference-counted + type Data = Arc; + /// Data associated with each hardware ID. + type IdInfo = &'static hw::HwConfig; + + // Assign the above OF ID table to this driver. + kernel::driver_of_id_table!(ASAHI_ID_TABLE); + + /// Device probe function. + fn probe( + pdev: &mut platform::Device, + id_info: Option<&Self::IdInfo>, + ) -> Result> { + debug::update_debug_flags(); + + let dev = device::Device::from_dev(pdev); + + dev_info!(dev, "Probing...\n"); + + let cfg = id_info.ok_or(ENODEV)?; + + pdev.set_dma_masks((1 << cfg.uat_oas) - 1)?; + + let res = regs::Resources::new(pdev)?; + + // Initialize misc MMIO + res.init_mmio()?; + + // Start the coprocessor CPU, so UAT can initialize the handoff + res.start_cpu()?; + + let node = dev.of_node().ok_or(EIO)?; + let compat: Vec = node.get_property(c_str!("apple,firmware-compat"))?; + + let reg = drm::drv::Registration::::new(&dev)?; + let gpu = match (cfg.gpu_gen, compat.as_slice()) { + (hw::GpuGen::G13, &[12, 3, 0]) => { + gpu::GpuManagerG13V12_3::new(reg.device(), &res, cfg)? as Arc + } + (hw::GpuGen::G13, &[13, 2, 0]) => { + gpu::GpuManagerG13V13_2::new(reg.device(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, &[12, 4, 0]) => { + gpu::GpuManagerG14V12_4::new(reg.device(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, &[13, 2, 0]) => { + gpu::GpuManagerG14V13_2::new(reg.device(), &res, cfg)? as Arc + } + _ => { + dev_info!( + dev, + "Unsupported GPU/firmware combination ({:?}, {:?})\n", + cfg.gpu_gen, + compat + ); + return Err(ENODEV); + } + }; + + let data = + kernel::new_device_data!(reg, res, AsahiData { dev, gpu }, "Asahi::Registrations")?; + + let data = Arc::::from(data); + + data.gpu.init()?; + + kernel::drm_device_register!( + data.registrations().ok_or(ENXIO)?.as_pinned_mut(), + data.clone(), + 0 + )?; + + dev_info!(data.dev, "Probed!\n"); + Ok(data) + } +} + +// Export the OF ID table as a module ID table, to make modpost/autoloading work. +kernel::module_of_id_table!(MOD_TABLE, ASAHI_ID_TABLE); diff --git a/drivers/gpu/drm/asahi/event.rs b/drivers/gpu/drm/asahi/event.rs new file mode 100644 index 00000000000000..3ae18d507ddc7e --- /dev/null +++ b/drivers/gpu/drm/asahi/event.rs @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU event manager +//! +//! The GPU firmware manages work completion by using event objects (Apple calls them "stamps"), +//! which are monotonically incrementing counters. There are a fixed number of objects, and +//! they are managed with a `SlotAllocator`. +//! +//! This module manages the set of available events and lets users compute expected values. +//! It also manages signaling owners when the GPU firmware reports that an event fired. + +use crate::debug::*; +use crate::fw::types::*; +use crate::{gpu, slotalloc, workqueue}; +use core::cmp; +use core::sync::atomic::Ordering; +use kernel::prelude::*; +use kernel::sync::Arc; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Event; + +/// Number of events managed by the firmware. +const NUM_EVENTS: u32 = 128; + +/// Inner data associated with a given event slot. +pub(crate) struct EventInner { + /// CPU pointer to the driver notification event stamp + stamp: *const AtomicU32, + /// GPU pointer to the driver notification event stamp + gpu_stamp: GpuWeakPointer, + /// GPU pointer to the firmware-internal event stamp + gpu_fw_stamp: GpuWeakPointer, +} + +/// SAFETY: The event slots are safe to send across threads. +unsafe impl Send for EventInner {} + +/// Alias for an event token, which allows requesting the same event. +pub(crate) type Token = slotalloc::SlotToken; +/// Alias for an allocated `Event` that has a slot. +pub(crate) type Event = slotalloc::Guard; + +/// Represents a given stamp value for an event. +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +#[repr(transparent)] +pub(crate) struct EventValue(u32); + +impl EventValue { + /// Returns the counter portion of the value. + pub(crate) fn counter(&self) -> u32 { + self.0 >> 8 + } + + /// Returns the `EventValue` that succeeds this one. + pub(crate) fn next(&self) -> EventValue { + EventValue(self.0.wrapping_add(0x100)) + } + + /// Increments this `EventValue` in place. + pub(crate) fn increment(&mut self) { + self.0 = self.0.wrapping_add(0x100); + } + + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn add(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_add(val.checked_mul(0x100).expect("Adding too many events")); + } + + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn sub(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_sub(val.checked_mul(0x100).expect("Subtracting too many events")); + } + + /// Computes the delta between this event and another event. + pub(crate) fn delta(&self, other: &EventValue) -> i32 { + (self.0.wrapping_sub(other.0) as i32) >> 8 + } +} + +impl PartialOrd for EventValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for EventValue { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.delta(other).cmp(&0) + } +} + +impl EventInner { + /// Returns the GPU pointer to the driver notification stamp + pub(crate) fn stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_stamp + } + + /// Returns the GPU pointer to the firmware internal stamp + pub(crate) fn fw_stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_fw_stamp + } + + /// Fetches the current event value from shared memory + pub(crate) fn current(&self) -> EventValue { + // SAFETY: The pointer is always valid as constructed in + // EventManager below, and outside users cannot construct + // new EventInners, nor move or copy them, and Guards as + // returned by the SlotAllocator hold a reference to the + // SlotAllocator containing the EventManagerInner, which + // keeps the GpuObject the stamp is contained within alive. + EventValue(unsafe { &*self.stamp }.load(Ordering::Acquire)) + } +} + +impl slotalloc::SlotItem for EventInner { + type Data = EventManagerInner; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("EventManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the event manager, to be protected by the SlotAllocator lock. +pub(crate) struct EventManagerInner { + stamps: GpuArray, + fw_stamps: GpuArray, + // Note: Use dyn to avoid having to version this entire module. + owners: Vec>>, +} + +/// Top-level EventManager object. +pub(crate) struct EventManager { + alloc: slotalloc::SlotAllocator, +} + +impl EventManager { + /// Create a new EventManager. + #[inline(never)] + pub(crate) fn new(alloc: &mut gpu::KernelAllocators) -> Result { + let mut owners = Vec::new(); + for _i in 0..(NUM_EVENTS as usize) { + owners.try_push(None)?; + } + let inner = EventManagerInner { + stamps: alloc.shared.array_empty(NUM_EVENTS as usize)?, + fw_stamps: alloc.private.array_empty(NUM_EVENTS as usize)?, + owners, + }; + + Ok(EventManager { + alloc: slotalloc::SlotAllocator::new( + NUM_EVENTS, + inner, + |inner: &mut EventManagerInner, slot| EventInner { + stamp: &inner.stamps[slot as usize].0, + gpu_stamp: inner.stamps.weak_item_pointer(slot as usize), + gpu_fw_stamp: inner.fw_stamps.weak_item_pointer(slot as usize), + }, + )?, + }) + } + + /// Gets a free `Event`, optionally trying to reuse the last one allocated by this caller. + pub(crate) fn get( + &self, + token: Option, + owner: Arc, + ) -> Result { + let ev = self.alloc.get_inner(token, |inner, ev| { + mod_pr_debug!( + "EventManager: Registered owner {:p} on slot {}\n", + &*owner, + ev.slot() + ); + inner.owners[ev.slot() as usize] = Some(owner); + Ok(()) + })?; + Ok(ev) + } + + /// Signals an event by slot, indicating completion (of one or more commands). + pub(crate) fn signal(&self, slot: u32) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.signal(); + } + None => { + mod_pr_debug!("EventManager: Received event for empty slot {}\n", slot); + } + } + } + + /// Marks the owner of an event as having lost its work due to a GPU error. + pub(crate) fn mark_error(&self, slot: u32, wait_value: u32, error: workqueue::WorkError) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.mark_error(EventValue(wait_value), error); + } + None => { + pr_err!("Received error for empty slot {}\n", slot); + } + } + } + + /// Fail all commands, used when the GPU crashes. + pub(crate) fn fail_all(&self, error: workqueue::WorkError) { + let mut owners: Vec> = Vec::new(); + + self.alloc.with_inner(|inner| { + for wq in inner.owners.iter().filter_map(|o| o.as_ref()).cloned() { + if owners.try_push(wq).is_err() { + pr_err!("Failed to signal failure to WorkQueue\n"); + } + } + }); + + for wq in owners { + wq.fail_all(error); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs new file mode 100644 index 00000000000000..2e3d90585c0978 --- /dev/null +++ b/drivers/gpu/drm/asahi/file.rs @@ -0,0 +1,758 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! File implementation, which represents a single DRM client. +//! +//! This is in charge of managing the resources associated with one GPU client, including an +//! arbitrary number of submission queues and Vm objects, and reporting hardware/driver +//! information to userspace and accepting submissions. + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::{alloc, buffer, driver, gem, mmu, queue}; +use core::mem::MaybeUninit; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::gem::BaseObject; +use kernel::io_buffer::{IoBufferReader, IoBufferWriter}; +use kernel::prelude::*; +use kernel::sync::{smutex::Mutex, Arc}; +use kernel::user_ptr::UserSlicePtr; +use kernel::{bindings, dma_fence, drm, xarray}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::File; + +const MAX_SYNCS_PER_SUBMISSION: u32 = 64; +const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; +pub(crate) const MAX_COMMANDS_IN_FLIGHT: u32 = 1024; + +/// A client instance of an `mmu::Vm` address space. +struct Vm { + ualloc: Arc>, + ualloc_priv: Arc>, + vm: mmu::Vm, + dummy_obj: gem::ObjectRef, +} + +impl Drop for Vm { + fn drop(&mut self) { + // Mappings create a reference loop, make sure to break it. + self.dummy_obj.drop_vm_mappings(self.vm.id()); + } +} + +/// Sync object from userspace. +pub(crate) struct SyncItem { + pub(crate) syncobj: drm::syncobj::SyncObj, + pub(crate) fence: Option, + pub(crate) chain_fence: Option, + pub(crate) timeline_value: u64, +} + +impl SyncItem { + fn parse_one(file: &DrmFile, data: bindings::drm_asahi_sync, out: bool) -> Result { + if data.extensions != 0 { + return Err(EINVAL); + } + + match data.sync_type { + bindings::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { + if data.timeline_value != 0 { + return Err(EINVAL); + } + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + + Ok(SyncItem { + fence: if out { + None + } else { + Some(syncobj.fence_get().ok_or(EINVAL)?) + }, + syncobj, + chain_fence: None, + timeline_value: data.timeline_value, + }) + } + bindings::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + let fence = if out { + None + } else { + Some( + syncobj + .fence_get() + .ok_or(EINVAL)? + .chain_find_seqno(data.timeline_value)?, + ) + }; + + Ok(SyncItem { + fence, + syncobj, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, + timeline_value: data.timeline_value, + }) + } + _ => Err(EINVAL), + } + } + + fn parse_array(file: &DrmFile, ptr: u64, count: u32, out: bool) -> Result> { + let mut vec = Vec::try_with_capacity(count as usize)?; + + const STRIDE: usize = core::mem::size_of::(); + let size = STRIDE * count as usize; + + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = unsafe { UserSlicePtr::new(ptr as usize as *mut _, size).reader() }; + + for i in 0..count { + let mut sync: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: The size of `sync` is STRIDE + unsafe { reader.read_raw(sync.as_mut_ptr() as *mut u8, STRIDE)? }; + + // SAFETY: All bit patterns in the struct are valid + let sync = unsafe { sync.assume_init() }; + + vec.try_push(SyncItem::parse_one(file, sync, out)?)?; + } + + Ok(vec) + } +} + +/// State associated with a client. +pub(crate) struct File { + id: u64, + vms: xarray::XArray>, + queues: xarray::XArray>>>, +} + +/// Convenience type alias for our DRM `File` type. +pub(crate) type DrmFile = drm::file::File; + +/// Start address of the 32-bit USC address space. +const VM_SHADER_START: u64 = 0x11_00000000; +/// End address of the 32-bit USC address space. +const VM_SHADER_END: u64 = 0x11_ffffffff; +/// Start address of the general user mapping region. +const VM_USER_START: u64 = 0x20_00000000; +/// End address of the general user mapping region. +const VM_USER_END: u64 = 0x5f_ffffffff; + +/// Start address of the kernel-managed GPU-only mapping region. +const VM_DRV_GPU_START: u64 = 0x60_00000000; +/// End address of the kernel-managed GPU-only mapping region. +const VM_DRV_GPU_END: u64 = 0x60_ffffffff; +/// Start address of the kernel-managed GPU/FW shared mapping region. +const VM_DRV_GPUFW_START: u64 = 0x61_00000000; +/// End address of the kernel-managed GPU/FW shared mapping region. +const VM_DRV_GPUFW_END: u64 = 0x61_ffffffff; +/// Address of a special dummy page? +const VM_UNK_PAGE: u64 = 0x6f_ffff8000; + +impl drm::file::DriverFile for File { + type Driver = driver::AsahiDriver; + + /// Create a new `File` instance for a fresh client. + fn open(device: &AsahiDevice) -> Result>> { + debug::update_debug_flags(); + + let gpu = &device.data().gpu; + let id = gpu.ids().file.next(); + + mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); + Ok(Box::into_pin(Box::try_new(Self { + id, + vms: xarray::XArray::new(xarray::flags::ALLOC1), + queues: xarray::XArray::new(xarray::flags::ALLOC1), + })?)) + } +} + +impl File { + fn vms(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for vms. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.vms) } + } + + #[allow(clippy::type_complexity)] + fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { + // SAFETY: Structural pinned projection for queues. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.queues) } + } + + /// IOCTL: get_param: Get a driver parameter value. + pub(crate) fn get_params( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_get_params, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!(device, "[File {}]: IOCTL: get_params\n", file.inner().id); + + let gpu = &device.data().gpu; + + if data.extensions != 0 || data.param_group != 0 || data.pad != 0 { + return Err(EINVAL); + } + + let mut params = bindings::drm_asahi_params_global { + unstable_uabi_version: bindings::DRM_ASAHI_UNSTABLE_UABI_VERSION, + pad0: 0, + + feat_compat: gpu.get_cfg().gpu_feat_compat, + feat_incompat: gpu.get_cfg().gpu_feat_incompat, + + gpu_generation: gpu.get_dyncfg().id.gpu_gen as u32, + gpu_variant: gpu.get_dyncfg().id.gpu_variant as u32, + gpu_revision: gpu.get_dyncfg().id.gpu_rev as u32, + chip_id: gpu.get_cfg().chip_id, + + num_dies: gpu.get_dyncfg().id.max_dies, + num_clusters_total: gpu.get_dyncfg().id.num_clusters, + num_cores_per_cluster: gpu.get_dyncfg().id.num_cores, + num_frags_per_cluster: gpu.get_dyncfg().id.num_frags, + num_gps_per_cluster: gpu.get_dyncfg().id.num_gps, + num_cores_total_active: gpu.get_dyncfg().id.total_active_cores, + core_masks: [0; bindings::DRM_ASAHI_MAX_CLUSTERS as usize], + + vm_page_size: mmu::UAT_PGSZ as u32, + pad1: 0, + vm_user_start: VM_USER_START, + vm_user_end: VM_USER_END, + vm_shader_start: VM_SHADER_START, + vm_shader_end: VM_SHADER_END, + + max_syncs_per_submission: MAX_SYNCS_PER_SUBMISSION, + max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, + max_commands_in_flight: MAX_COMMANDS_IN_FLIGHT, + max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, + + timer_frequency_hz: gpu.get_cfg().base_clock_hz, + min_frequency_khz: gpu.get_dyncfg().pwr.min_frequency_khz(), + max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), + max_power_mw: gpu.get_dyncfg().pwr.max_power_mw, + + result_render_size: core::mem::size_of::() as u32, + result_compute_size: core::mem::size_of::() as u32, + }; + + for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).try_into()?; + } + + let size = + core::mem::size_of::().min(data.size.try_into()?); + + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + let mut params_writer = + unsafe { UserSlicePtr::new(data.pointer as usize as *mut _, size).writer() }; + + // SAFETY: `size` is at most the sizeof of `params` + unsafe { params_writer.write_raw(¶ms as *const _ as *const u8, size)? }; + + Ok(0) + } + + /// IOCTL: vm_create: Create a new `Vm`. + pub(crate) fn vm_create( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_vm_create, + file: &DrmFile, + ) -> Result { + if data.extensions != 0 { + return Err(EINVAL); + } + + let gpu = &device.data().gpu; + let file_id = file.inner().id; + let vm = gpu.new_vm(file_id)?; + + let resv = file.inner().vms().reserve()?; + let id: u32 = resv.index().try_into()?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM Create\n", file_id, id); + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating allocators\n", + file_id, + id + ); + let ualloc = Arc::try_new(Mutex::new(alloc::DefaultAllocator::new( + device, + &vm, + VM_DRV_GPU_START, + VM_DRV_GPU_END, + buffer::PAGE_SIZE, + mmu::PROT_GPU_SHARED_RW, + 512 * 1024, + true, + fmt!("File {} VM {} GPU Shared", file_id, id), + false, + )?))?; + let ualloc_priv = Arc::try_new(Mutex::new(alloc::DefaultAllocator::new( + device, + &vm, + VM_DRV_GPUFW_START, + VM_DRV_GPUFW_END, + buffer::PAGE_SIZE, + mmu::PROT_GPU_FW_PRIV_RW, + 64 * 1024, + true, + fmt!("File {} VM {} GPU FW Private", file_id, id), + false, + )?))?; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating dummy object\n", + file_id, + id + ); + let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?; + dummy_obj.vmap()?.as_mut_slice().fill(0); + dummy_obj.map_at(&vm, VM_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); + resv.store(Box::try_new(Vm { + ualloc, + ualloc_priv, + vm, + dummy_obj, + })?)?; + + data.vm_id = id; + + Ok(0) + } + + /// IOCTL: vm_destroy: Destroy a `Vm`. + pub(crate) fn vm_destroy( + _device: &AsahiDevice, + data: &mut bindings::drm_asahi_vm_destroy, + file: &DrmFile, + ) -> Result { + if data.extensions != 0 { + return Err(EINVAL); + } + + if file.inner().vms().remove(data.vm_id as usize).is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: gem_create: Create a new GEM object. + pub(crate) fn gem_create( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_gem_create, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x?}\n", + file.inner().id, + data.size + ); + + if data.extensions != 0 + || (data.flags & !(bindings::ASAHI_GEM_WRITEBACK | bindings::ASAHI_GEM_VM_PRIVATE)) != 0 + || (data.flags & bindings::ASAHI_GEM_VM_PRIVATE == 0 && data.vm_id != 0) + { + return Err(EINVAL); + } + + let vm_id = if data.flags & bindings::ASAHI_GEM_VM_PRIVATE != 0 { + Some( + file.inner() + .vms() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .borrow() + .vm + .id(), + ) + } else { + None + }; + + let bo = gem::new_object(device, data.size.try_into()?, data.flags, vm_id)?; + + let handle = bo.gem.create_handle(file)?; + data.handle = handle; + + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x} handle={:#x?}\n", + file.inner().id, + data.size, + data.handle + ); + + Ok(0) + } + + /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. + pub(crate) fn gem_mmap_offset( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_gem_mmap_offset, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_mmap_offset handle={:#x?}\n", + file.inner().id, + data.handle + ); + + if data.extensions != 0 || data.flags != 0 { + return Err(EINVAL); + } + + let bo = gem::lookup_handle(file, data.handle)?; + data.offset = bo.gem.create_mmap_offset()?; + Ok(0) + } + + /// IOCTL: gem_bind: Map or unmap a GEM object into a Vm. + pub(crate) fn gem_bind( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_gem_bind, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: gem_bind op={:?} handle={:#x?} flags={:#x?} {:#x?}:{:#x?} -> {:#x?}\n", + file.inner().id, + data.vm_id, + data.op, + data.handle, + data.flags, + data.offset, + data.range, + data.addr + ); + + if data.extensions != 0 { + return Err(EINVAL); + } + + match data.op { + bindings::drm_asahi_bind_op_ASAHI_BIND_OP_BIND => Self::do_gem_bind(device, data, file), + bindings::drm_asahi_bind_op_ASAHI_BIND_OP_UNBIND => Err(ENOTSUPP), + bindings::drm_asahi_bind_op_ASAHI_BIND_OP_UNBIND_ALL => { + Self::do_gem_unbind_all(device, data, file) + } + _ => Err(EINVAL), + } + } + + pub(crate) fn do_gem_bind( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_gem_bind, + file: &DrmFile, + ) -> Result { + if data.offset != 0 { + return Err(EINVAL); // Not supported yet + } + + if (data.addr | data.range) as usize & mmu::UAT_PGMSK != 0 { + return Err(EINVAL); // Must be page aligned + } + + if (data.flags & !(bindings::ASAHI_BIND_READ | bindings::ASAHI_BIND_WRITE)) != 0 { + return Err(EINVAL); + } + + let mut bo = gem::lookup_handle(file, data.handle)?; + + if data.range != bo.size().try_into()? { + return Err(EINVAL); // Not supported yet + } + + let start = data.addr; + let end = data.addr + data.range - 1; + + if (VM_SHADER_START..=VM_SHADER_END).contains(&start) { + if !(VM_SHADER_START..=VM_SHADER_END).contains(&end) { + return Err(EINVAL); // Invalid map range + } + } else if (VM_USER_START..=VM_USER_END).contains(&start) { + if !(VM_USER_START..=VM_USER_END).contains(&end) { + return Err(EINVAL); // Invalid map range + } + } else { + return Err(EINVAL); // Invalid map range + } + + // Just in case + if end >= VM_DRV_GPU_START { + return Err(EINVAL); + } + + let prot = if data.flags & bindings::ASAHI_BIND_READ != 0 { + if data.flags & bindings::ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_RW + } else { + mmu::PROT_GPU_SHARED_RO + } + } else if data.flags & bindings::ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_WO + } else { + return Err(EINVAL); // Must specify one of ASAHI_BIND_{READ,WRITE} + }; + + // Clone it immediately so we aren't holding the XArray lock + let vm = file + .inner() + .vms() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .borrow() + .vm + .clone(); + + bo.map_at(&vm, start, prot, true)?; + + Ok(0) + } + + pub(crate) fn do_gem_unbind_all( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_gem_bind, + file: &DrmFile, + ) -> Result { + if data.flags != 0 || data.offset != 0 || data.range != 0 || data.addr != 0 { + return Err(EINVAL); + } + + let mut bo = gem::lookup_handle(file, data.handle)?; + + if data.vm_id == 0 { + bo.drop_file_mappings(file.inner().id); + } else { + let vm_id = file + .inner() + .vms() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .borrow() + .vm + .id(); + bo.drop_vm_mappings(vm_id); + } + + Ok(0) + } + + /// IOCTL: queue_create: Create a new command submission queue of a given type. + pub(crate) fn queue_create( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_queue_create, + file: &DrmFile, + ) -> Result { + let file_id = file.inner().id; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating queue caps={:?} prio={:?} flags={:#x?}\n", + file_id, + data.vm_id, + data.queue_caps, + data.priority, + data.flags, + ); + + if data.extensions != 0 + || data.flags != 0 + || data.priority > 3 + || data.queue_caps == 0 + || (data.queue_caps + & !(bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER + | bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_BLIT + | bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE)) + != 0 + { + return Err(EINVAL); + } + + let resv = file.inner().queues().reserve()?; + let file_vm = file + .inner() + .vms() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)?; + let vm = file_vm.borrow().vm.clone(); + let ualloc = file_vm.borrow().ualloc.clone(); + let ualloc_priv = file_vm.borrow().ualloc_priv.clone(); + // Drop the vms lock eagerly + core::mem::drop(file_vm); + + let queue = + device + .data() + .gpu + .new_queue(vm, ualloc, ualloc_priv, data.priority, data.queue_caps)?; + + data.queue_id = resv.index().try_into()?; + resv.store(Arc::try_new(Mutex::new(queue))?)?; + + Ok(0) + } + + /// IOCTL: queue_destroy: Destroy a command submission queue. + pub(crate) fn queue_destroy( + _device: &AsahiDevice, + data: &mut bindings::drm_asahi_queue_destroy, + file: &DrmFile, + ) -> Result { + if data.extensions != 0 { + return Err(EINVAL); + } + + if file + .inner() + .queues() + .remove(data.queue_id as usize) + .is_none() + { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: submit: Submit GPU work to a command submission queue. + pub(crate) fn submit( + device: &AsahiDevice, + data: &mut bindings::drm_asahi_submit, + file: &DrmFile, + ) -> Result { + if data.extensions != 0 + || data.flags != 0 + || data.in_sync_count > MAX_SYNCS_PER_SUBMISSION + || data.out_sync_count > MAX_SYNCS_PER_SUBMISSION + || data.command_count > MAX_COMMANDS_PER_SUBMISSION + { + return Err(EINVAL); + } + + debug::update_debug_flags(); + + let gpu = &device.data().gpu; + gpu.update_globals(); + + // Upgrade to Arc to drop the XArray lock early + let queue: Arc>> = file + .inner() + .queues() + .get(data.queue_id.try_into()?) + .ok_or(ENOENT)? + .borrow() + .into(); + + let id = gpu.ids().submission.next(); + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit (submission ID: {})\n", + file.inner().id, + data.queue_id, + id + ); + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing in_syncs\n", + file.inner().id, + data.queue_id, + id + ); + let in_syncs = SyncItem::parse_array(file, data.in_syncs, data.in_sync_count, false)?; + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing out_syncs\n", + file.inner().id, + data.queue_id, + id + ); + let out_syncs = SyncItem::parse_array(file, data.out_syncs, data.out_sync_count, true)?; + + let result_buf = if data.result_handle != 0 { + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Looking up result_handle {}\n", + file.inner().id, + data.queue_id, + id, + data.result_handle + ); + Some(gem::lookup_handle(file, data.result_handle)?) + } else { + None + }; + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing commands\n", + file.inner().id, + data.queue_id, + id + ); + let mut commands = Vec::try_with_capacity(data.command_count as usize)?; + + const STRIDE: usize = core::mem::size_of::(); + let size = STRIDE * data.command_count as usize; + + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = + unsafe { UserSlicePtr::new(data.commands as usize as *mut _, size).reader() }; + + for i in 0..data.command_count { + let mut cmd: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: The size of `sync` is STRIDE + unsafe { reader.read_raw(cmd.as_mut_ptr() as *mut u8, STRIDE)? }; + + // SAFETY: All bit patterns in the struct are valid + commands.try_push(unsafe { cmd.assume_init() })?; + } + + let ret = queue + .lock() + .submit(id, in_syncs, out_syncs, result_buf, commands); + + match ret { + Err(ERESTARTSYS) => Err(ERESTARTSYS), + Err(e) => { + dev_info!( + device, + "[File {} Queue {}]: IOCTL: submit failed! (submission ID: {} err: {:?})\n", + file.inner().id, + data.queue_id, + id, + e + ); + Err(e) + } + Ok(_) => Ok(0), + } + } + + /// Returns the unique file ID for this `File`. + pub(crate) fn file_id(&self) -> u64 { + self.id + } +} + +impl Drop for File { + fn drop(&mut self) { + mod_pr_debug!("[File {}]: Closing...\n", self.id); + } +} diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs new file mode 100644 index 00000000000000..e73b4b628cf9f5 --- /dev/null +++ b/drivers/gpu/drm/asahi/float.rs @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Basic soft floating-point support +//! +//! The GPU firmware requires a large number of power-related configuration values, many of which +//! are IEEE 754 32-bit floating point values. These values change not only between GPU/SoC +//! variants, but also between specific hardware platforms using these SoCs, so they must be +//! derived from device tree properties. There are many redundant values computed from the same +//! inputs with simple add/sub/mul/div calculations, plus a few values that are actually specific +//! to each individual device depending on its binning and fused voltage configuration, so it +//! doesn't make sense to store the final values to be passed to the firmware in the device tree. +//! +//! Therefore, we need a way to perform floating-point calculations in the kernel. +//! +//! Using the actual FPU from kernel mode is asking for trouble, since there is no way to bound +//! the execution of FPU instructions to a controlled section of code without outright putting it +//! in its own compilation unit, which is quite painful for Rust. Since these calculations only +//! have to happen at initialization time and there is no need for performance, let's use a simple +//! software float implementation instead. +//! +//! This implementation makes no attempt to be fully IEEE754 compliant, but it's good enough and +//! gives bit-identical results to macOS in the vast majority of cases, with one or two exceptions +//! related to slightly non-compliant rounding. + +use core::ops; +use kernel::{of, prelude::*}; + +/// An IEEE754-compatible floating point number implemented in software. +#[derive(Default, Debug, Copy, Clone)] +pub(crate) struct F32(u32); + +#[derive(Default, Debug, Copy, Clone)] +struct F32U { + sign: bool, + exp: i32, + frac: i64, +} + +impl F32 { + /// Convert a raw 32-bit representation into an F32 + pub(crate) const fn from_bits(u: u32) -> F32 { + F32(u) + } + + // Convert a `f32` value into an F32 + // + // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. + #[doc(hidden)] + pub(crate) const fn from_f32(v: f32) -> F32 { + F32(unsafe { core::mem::transmute(v) }) + } + + // Convert an F32 into a `f32` value + // + // For testing only. + #[doc(hidden)] + #[cfg(test)] + pub(crate) fn to_f32(self) -> f32 { + f32::from_bits(self.0) + } + + const fn unpack(&self) -> F32U { + F32U { + sign: self.0 & (1 << 31) != 0, + exp: ((self.0 >> 23) & 0xff) as i32 - 127, + frac: (((self.0 & 0x7fffff) | 0x800000) as i64) << 9, + } + .norm() + } +} + +/// Safely construct an `F32` out of a constant floating-point value. +/// +/// This ensures that the conversion happens in const context, so no floating point operations are +/// emitted. +#[macro_export] +macro_rules! f32 { + ([$($val:expr),*]) => {{ + [$(f32!($val)),*] + }}; + ($val:expr) => {{ + const _K: $crate::float::F32 = $crate::float::F32::from_f32($val); + _K + }}; +} + +impl ops::Neg for F32 { + type Output = F32; + + fn neg(self) -> F32 { + F32(self.0 ^ (1 << 31)) + } +} + +impl ops::Add for F32 { + type Output = F32; + + fn add(self, rhs: F32) -> F32 { + self.unpack().add(rhs.unpack()).pack() + } +} + +impl ops::Sub for F32 { + type Output = F32; + + fn sub(self, rhs: F32) -> F32 { + self.unpack().add((-rhs).unpack()).pack() + } +} + +impl ops::Mul for F32 { + type Output = F32; + + fn mul(self, rhs: F32) -> F32 { + self.unpack().mul(rhs.unpack()).pack() + } +} + +impl ops::Div for F32 { + type Output = F32; + + fn div(self, rhs: F32) -> F32 { + self.unpack().div(rhs.unpack()).pack() + } +} + +macro_rules! from_ints { + ($u:ty, $i:ty) => { + impl From<$i> for F32 { + fn from(v: $i) -> F32 { + F32U::from_i64(v as i64).pack() + } + } + impl From<$u> for F32 { + fn from(v: $u) -> F32 { + F32U::from_u64(v as u64).pack() + } + } + }; +} + +from_ints!(u8, i8); +from_ints!(u16, i16); +from_ints!(u32, i32); +from_ints!(u64, i64); + +impl F32U { + const INFINITY: F32U = f32!(f32::INFINITY).unpack(); + const NEG_INFINITY: F32U = f32!(f32::NEG_INFINITY).unpack(); + + fn from_i64(v: i64) -> F32U { + F32U { + sign: v < 0, + exp: 32, + frac: v.abs(), + } + .norm() + } + + fn from_u64(mut v: u64) -> F32U { + let mut exp = 32; + if v >= (1 << 63) { + exp = 31; + v >>= 1; + } + F32U { + sign: false, + exp, + frac: v as i64, + } + .norm() + } + + fn shr(&mut self, shift: i32) { + if shift > 63 { + self.exp = 0; + self.frac = 0; + } else { + self.frac >>= shift; + } + } + + fn align(a: &mut F32U, b: &mut F32U) { + if a.exp > b.exp { + b.shr(a.exp - b.exp); + b.exp = a.exp; + } else { + a.shr(b.exp - a.exp); + a.exp = b.exp; + } + } + + fn mul(self, other: F32U) -> F32U { + F32U { + sign: self.sign != other.sign, + exp: self.exp + other.exp, + frac: ((self.frac >> 8) * (other.frac >> 8)) >> 16, + } + } + + fn div(self, other: F32U) -> F32U { + if other.frac == 0 || self.is_inf() { + if self.sign { + F32U::NEG_INFINITY + } else { + F32U::INFINITY + } + } else { + F32U { + sign: self.sign != other.sign, + exp: self.exp - other.exp, + frac: ((self.frac << 24) / (other.frac >> 8)), + } + } + } + + fn add(mut self, mut other: F32U) -> F32U { + F32U::align(&mut self, &mut other); + if self.sign == other.sign { + self.frac += other.frac; + } else { + self.frac -= other.frac; + } + if self.frac < 0 { + self.sign = !self.sign; + self.frac = -self.frac; + } + self + } + + const fn norm(mut self) -> F32U { + let lz = self.frac.leading_zeros() as i32; + if lz > 31 { + self.frac <<= lz - 31; + self.exp -= lz - 31; + } else if lz < 31 { + self.frac >>= 31 - lz; + self.exp += 31 - lz; + } + + if self.is_zero() { + return F32U { + sign: self.sign, + frac: 0, + exp: 0, + }; + } + self + } + + const fn is_zero(&self) -> bool { + self.frac == 0 || self.exp < -126 + } + + const fn is_inf(&self) -> bool { + self.exp > 127 + } + + const fn pack(mut self) -> F32 { + self = self.norm(); + if !self.is_zero() { + self.frac += 0x100; + self = self.norm(); + } + + if self.is_inf() { + if self.sign { + return f32!(f32::NEG_INFINITY); + } else { + return f32!(f32::INFINITY); + } + } else if self.is_zero() { + if self.sign { + return f32!(-0.0); + } else { + return f32!(0.0); + } + } + + F32(if self.sign { 1u32 << 31 } else { 0u32 } + | ((self.exp + 127) as u32) << 23 + | ((self.frac >> 9) & 0x7fffff) as u32) + } +} + +impl<'a> TryFrom> for F32 { + type Error = Error; + + fn try_from(p: of::Property<'_>) -> core::result::Result { + let bits: u32 = p.try_into()?; + Ok(F32::from_bits(bits)) + } +} + +impl of::PropertyUnit for F32 { + const UNIT_SIZE: usize = 4; + + fn from_bytes(data: &[u8]) -> Result { + Ok(F32::from_bits(::from_bytes(data)?)) + } +} + +// TODO: Make this an actual test and figure out how to make it run. +#[cfg(test)] +mod tests { + #[test] + fn test_all() { + fn add(a: f32, b: f32) { + println!( + "{} + {} = {} {}", + a, + b, + (F32::from_f32(a) + F32::from_f32(b)).to_f32(), + a + b + ); + } + fn sub(a: f32, b: f32) { + println!( + "{} - {} = {} {}", + a, + b, + (F32::from_f32(a) - F32::from_f32(b)).to_f32(), + a - b + ); + } + fn mul(a: f32, b: f32) { + println!( + "{} * {} = {} {}", + a, + b, + (F32::from_f32(a) * F32::from_f32(b)).to_f32(), + a * b + ); + } + fn div(a: f32, b: f32) { + println!( + "{} / {} = {} {}", + a, + b, + (F32::from_f32(a) / F32::from_f32(b)).to_f32(), + a / b + ); + } + + fn test(a: f32, b: f32) { + add(a, b); + sub(a, b); + mul(a, b); + div(a, b); + } + + test(1.123, 7.567); + test(1.123, 1.456); + test(7.567, 1.123); + test(1.123, -7.567); + test(1.123, -1.456); + test(7.567, -1.123); + test(-1.123, -7.567); + test(-1.123, -1.456); + test(-7.567, -1.123); + test(1000.123, 0.001); + test(1000.123, 0.0000001); + test(0.0012, 1000.123); + test(0.0000001, 1000.123); + test(0., 0.); + test(0., 1.); + test(1., 0.); + test(1., 1.); + test(2., f32::INFINITY); + test(2., f32::NEG_INFINITY); + test(f32::INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::MAX, 2.); + test(f32::MIN, 2.); + test(f32::MIN_POSITIVE, 2.); + test(2., f32::MAX); + test(2., f32::MIN); + test(2., f32::MIN_POSITIVE); + } +} diff --git a/drivers/gpu/drm/asahi/fw/buffer.rs b/drivers/gpu/drm/asahi/fw/buffer.rs new file mode 100644 index 00000000000000..a8a46787951895 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/buffer.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU tiled vertex buffer control firmware structures + +use super::types::*; +use super::workqueue; +use crate::{default_zeroed, no_debug, trivial_gpustruct}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct BlockControl { + pub(crate) total: AtomicU32, + pub(crate) wptr: AtomicU32, + pub(crate) unk: AtomicU32, + pub(crate) pad: Pad<0x34>, + } + default_zeroed!(BlockControl); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Counter { + pub(crate) count: AtomicU32, + __pad: Pad<0x3c>, + } + default_zeroed!(Counter); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct Stats { + pub(crate) max_pages: AtomicU32, + pub(crate) max_b: AtomicU32, + pub(crate) overflow_count: AtomicU32, + pub(crate) gpu_c: AtomicU32, + pub(crate) __pad0: Pad<0x10>, + pub(crate) reset: AtomicU32, + pub(crate) __pad1: Pad<0x1c>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Info<'a> { + pub(crate) gpu_counter: u32, + pub(crate) unk_4: u32, + pub(crate) last_id: i32, + pub(crate) cur_id: i32, + pub(crate) unk_10: u32, + pub(crate) gpu_counter2: u32, + pub(crate) unk_18: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_1c: u32, + + pub(crate) page_list: GpuPointer<'a, &'a [u32]>, + pub(crate) page_list_size: u32, + pub(crate) page_count: AtomicU32, + pub(crate) max_blocks: u32, + pub(crate) block_count: AtomicU32, + pub(crate) unk_38: u32, + pub(crate) block_list: GpuPointer<'a, &'a [u32]>, + pub(crate) block_ctl: GpuPointer<'a, super::BlockControl>, + pub(crate) last_page: AtomicU32, + pub(crate) gpu_page_ptr1: u32, + pub(crate) gpu_page_ptr2: u32, + pub(crate) unk_58: u32, + pub(crate) block_size: u32, + pub(crate) unk_60: U64, + pub(crate) counter: GpuPointer<'a, super::Counter>, + pub(crate) unk_70: u32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) max_pages: u32, + pub(crate) max_pages_nomemless: u32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: Array<0x30, u8>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Scene<'a> { + pub(crate) pass_page_count: AtomicU32, + pub(crate) unk_4: u32, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) user_buffer: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_20: u32, + pub(crate) stats: GpuWeakPointer, + pub(crate) total_page_count: AtomicU32, + pub(crate) unk_30: U64, // pad + pub(crate) unk_38: U64, // pad + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitBuffer<'a> { + pub(crate) tag: workqueue::CommandType, + pub(crate) vm_slot: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + pub(crate) block_count: u32, + pub(crate) buffer: GpuPointer<'a, super::Info::ver>, + pub(crate) stamp_value: EventValue, + } +} + +trivial_gpustruct!(BlockControl); +trivial_gpustruct!(Counter); +trivial_gpustruct!(Stats); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Info { + pub(crate) block_ctl: GpuObject, + pub(crate) counter: GpuObject, + pub(crate) page_list: GpuArray, + pub(crate) block_list: GpuArray, +} + +#[versions(AGX)] +impl GpuStruct for Info::ver { + type Raw<'a> = raw::Info::ver<'a>; +} + +pub(crate) struct ClusterBuffers { + pub(crate) tilemaps: GpuArray, + pub(crate) meta: GpuArray, +} + +#[versions(AGX)] +pub(crate) struct Scene { + pub(crate) user_buffer: GpuArray, + pub(crate) buffer: crate::buffer::Buffer::ver, + pub(crate) tvb_heapmeta: GpuArray, + pub(crate) tvb_tilemap: GpuArray, + pub(crate) tpc: Arc>, + pub(crate) clustering: Option, + pub(crate) preempt_buf: GpuArray, + pub(crate) seq_buf: GpuArray, +} + +#[versions(AGX)] +no_debug!(Scene::ver); + +#[versions(AGX)] +impl GpuStruct for Scene::ver { + type Raw<'a> = raw::Scene<'a>; +} + +#[versions(AGX)] +pub(crate) struct InitBuffer { + pub(crate) scene: Arc, +} + +#[versions(AGX)] +no_debug!(InitBuffer::ver); + +#[versions(AGX)] +impl workqueue::Command for InitBuffer::ver {} + +#[versions(AGX)] +impl GpuStruct for InitBuffer::ver { + type Raw<'a> = raw::InitBuffer::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs new file mode 100644 index 00000000000000..db5ac9a3ded5cd --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU communication channel firmware structures (ring buffers) + +use super::types::*; +use crate::default_zeroed; +use core::sync::atomic::Ordering; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0x1c>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, ChannelState<'a>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct FwCtlChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, FwCtlChannelState<'a>); +} + +pub(crate) trait RxChannelState: GpuStruct + Debug + Default +where + for<'a> ::Raw<'a>: Default + Zeroed, +{ + const SUB_CHANNELS: usize; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32; + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32); +} + +#[derive(Debug, Default)] +pub(crate) struct ChannelState {} + +impl GpuStruct for ChannelState { + type Raw<'a> = raw::ChannelState<'a>; +} + +impl RxChannelState for ChannelState { + const SUB_CHANNELS: usize = 1; + + fn wptr(raw: &Self::Raw<'_>, _index: usize) -> u32 { + raw.write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, _index: usize, rptr: u32) { + raw.read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwLogChannelState {} + +impl GpuStruct for FwLogChannelState { + type Raw<'a> = Array<6, raw::ChannelState<'a>>; +} + +impl RxChannelState for FwLogChannelState { + const SUB_CHANNELS: usize = 6; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32 { + raw[index].write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32) { + raw[index].read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwCtlChannelState {} + +impl GpuStruct for FwCtlChannelState { + type Raw<'a> = raw::FwCtlChannelState<'a>; +} + +pub(crate) trait TxChannelState: GpuStruct + Debug + Default { + fn rptr(raw: &Self::Raw<'_>) -> u32; + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32); +} + +impl TxChannelState for ChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +impl TxChannelState for FwCtlChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(u32)] +pub(crate) enum PipeType { + #[default] + Vertex = 0, + Fragment = 1, + Compute = 2, +} + +#[versions(AGX)] +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RunWorkQueueMsg { + pub(crate) pipe_type: PipeType, + pub(crate) work_queue: Option>, + pub(crate) wptr: u32, + pub(crate) event_slot: u32, + pub(crate) is_new: bool, + #[ver(V >= V13_2 && G >= G14)] + pub(crate) __pad: Pad<0x2b>, + #[ver(V < V13_2 || G < G14)] + pub(crate) __pad: Pad<0x1b>, +} + +#[versions(AGX)] +pub(crate) type PipeMsg = RunWorkQueueMsg::ver; + +#[versions(AGX)] +pub(crate) const DEVICECONTROL_SZ: usize = { + #[ver(V < V13_2 || G < G14)] + { + 0x2c + } + #[ver(V >= V13_2 && G >= G14)] + { + 0x3c + } +}; + +// TODO: clean up when arbitrary_enum_discriminant is stable +// https://github.com/rust-lang/rust/issues/60553 + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum DeviceControlMsg { + Unk00(Array), + Unk01(Array), + Unk02(Array), + Unk03(Array), + Unk04(Array), + Unk05(Array), + Unk06(Array), + Unk07(Array), + Unk08(Array), + Unk09(Array), + Unk0a(Array), + Unk0b(Array), + Unk0c(Array), + Unk0d(Array), + Unk0e(Array), + Unk0f(Array), + Unk10(Array), + Unk11(Array), + Unk12(Array), + Unk13(Array), + Unk14(Array), + Unk15(Array), + Unk16(Array), + DestroyContext { + unk_4: u32, + ctx_23: u8, + __pad0: Pad<3>, + unk_c: u32, + unk_10: u32, + ctx_0: u8, + ctx_1: u8, + ctx_4: u8, + __pad1: Pad<1>, + unk_18: u32, + gpu_context: Option>, + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x20 }>, + }, + Unk18(Array), + Initialize(Pad), +} + +#[versions(AGX)] +default_zeroed!(DeviceControlMsg::ver); + +#[derive(Copy, Clone, Default, Debug)] +#[repr(C)] +#[allow(dead_code)] +pub(crate) struct FwCtlMsg { + pub(crate) addr: U64, + pub(crate) unk_8: u32, + pub(crate) slot: u32, + pub(crate) page_count: u16, + pub(crate) unk_12: u16, +} + +pub(crate) const EVENT_SZ: usize = 0x34; + +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum EventMsg { + Fault, + Flag { + firing: [u32; 4], + unk_14: u16, + }, + Unk2(Array), + Unk3(Array), + Timeout { + counter: u32, + unk_8: u32, + event_slot: u32, + }, // Max discriminant: 0x4 +} + +pub(crate) const EVENT_MAX: u32 = 0x4; + +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawEventMsg { + pub(crate) raw: (u32, Array), + pub(crate) msg: EventMsg, +} + +default_zeroed!(RawEventMsg); + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogMsg { + pub(crate) msg_type: u32, + __pad0: u32, + pub(crate) msg_index: U64, + __pad1: Pad<0x28>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogPayloadMsg { + pub(crate) msg_type: u32, + pub(crate) seq_no: u32, + pub(crate) timestamp: U64, + pub(crate) msg: Array<0xc8, u8>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawKTraceMsg { + pub(crate) msg_type: u32, + pub(crate) timestamp: U64, + pub(crate) args: Array<4, U64>, + pub(crate) code: u8, + pub(crate) channel: u8, + __pad: Pad<1>, + pub(crate) thread: u8, + pub(crate) unk_flag: U64, +} + +#[versions(AGX)] +pub(crate) const STATS_SZ: usize = { + #[ver(V < V13_0B4)] + { + 0x2c + } + #[ver(V >= V13_0B4)] + { + 0x3c + } +}; + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum StatsMsg { + Power { + // 0x00 + __pad: Pad<0x18>, + power: U64, + }, + Unk1(Array<{ STATS_SZ::ver }, u8>), + PowerOn { + // 0x02 + off_time: U64, + }, + PowerOff { + // 0x03 + on_time: U64, + }, + Utilization { + // 0x04 + timestamp: U64, + util1: u32, + util2: u32, + util3: u32, + util4: u32, + }, + Unk5(Array<{ STATS_SZ::ver }, u8>), + Unk6(Array<{ STATS_SZ::ver }, u8>), + Unk7(Array<{ STATS_SZ::ver }, u8>), + Unk8(Array<{ STATS_SZ::ver }, u8>), + AvgPower { + // 0x09 + active_cs: U64, + unk2: u32, + unk3: u32, + unk4: u32, + avg_power: u32, + }, + Temperature { + // 0x0a + __pad: Pad<0x8>, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, + PowerState { + // 0x0b + timestamp: U64, + last_busy_ts: U64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + }, + FwBusy { + // 0x0c + timestamp: U64, + busy: u32, + }, + PState { + // 0x0d + __pad: Pad<0x8>, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + }, + TempSensor { + // 0x0e + __pad: Pad<0x4>, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, // Max discriminant: 0xe +} + +#[versions(AGX)] +pub(crate) const STATS_MAX: u32 = 0xe; + +#[versions(AGX)] +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawStatsMsg { + pub(crate) raw: (u32, Array<{ STATS_SZ::ver }, u8>), + pub(crate) msg: StatsMsg::ver, +} + +#[versions(AGX)] +default_zeroed!(RawStatsMsg::ver); diff --git a/drivers/gpu/drm/asahi/fw/compute.rs b/drivers/gpu/drm/asahi/fw/compute.rs new file mode 100644 index 00000000000000..0dbcd77c5e3e39 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/compute.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU compute job firmware structures + +use super::types::*; +use super::{event, job, workqueue}; +use crate::{microseq, mmu}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) encoder: U64, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf4: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf5: GpuPointer<'a, &'a [u8]>, + pub(crate) pipeline_base: U64, + pub(crate) unk_38: U64, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) compute_layout_addr: U64, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) iogpu_unk_40: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + #[ver(V >= V13_0B4)] + pub(crate) unk_0_0: u32, + pub(crate) unk_0: Array<0x24, u8>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) encoder_end: U64, + pub(crate) unk_34: Array<0x28, u8>, + #[ver(V < V13_0B4)] + pub(crate) unk_5c: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunCompute<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) unk_4: u32, + pub(crate) vm_slot: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) unk_pointee: Array<0x54, u8>, + pub(crate) job_params1: JobParameters1<'a>, + pub(crate) unk_b8: Array<0x11c, u8>, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) job_params2: JobParameters2::ver<'a>, + pub(crate) encoder_params: job::raw::EncoderParams<'a>, + pub(crate) meta: job::raw::JobMeta, + pub(crate) cur_ts: U64, + pub(crate) start_ts: Option>, + pub(crate) end_ts: Option>, + pub(crate) unk_2c0: u32, + pub(crate) unk_2c4: u32, + pub(crate) unk_2c8: u32, + pub(crate) unk_2cc: u32, + pub(crate) client_sequence: u8, + pub(crate) pad_2d1: Array<3, u8>, + pub(crate) unk_2d4: u32, + pub(crate) unk_2d8: u8, + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + #[ver(V >= V13_0B4)] + pub(crate) unk_2e1: Array<0x1c, u8>, + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: U32, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x10, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunCompute { + pub(crate) notifier: Arc>, + pub(crate) preempt_buf: GpuArray, + pub(crate) seq_buf: GpuArray, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, +} + +#[versions(AGX)] +impl GpuStruct for RunCompute::ver { + type Raw<'a> = raw::RunCompute::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunCompute::ver {} diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs new file mode 100644 index 00000000000000..fbf65ab6d97624 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU events control structures & stamps + +use super::types::*; +use crate::{default_zeroed, trivial_gpustruct}; +use core::sync::atomic::Ordering; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct LinkedListHead { + pub(crate) prev: Option>, + pub(crate) next: Option>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierList { + pub(crate) list_head: LinkedListHead, + pub(crate) unkptr_10: U64, + } + default_zeroed!(NotifierList); + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierState { + unk_14: u32, + unk_18: U64, + unk_20: u32, + vm_slot: u32, + has_vtx: u32, + pstamp_vtx: Array<4, U64>, + has_frag: u32, + pstamp_frag: Array<4, U64>, + has_comp: u32, + pstamp_comp: Array<4, U64>, + #[ver(G >= G14 && V < V13_0B4)] + unk_98_g14_0: Array<0x14, u8>, + in_list: u32, + list_head: LinkedListHead, + #[ver(G >= G14 && V < V13_0B4)] + unk_a8_g14_0: Pad<4>, + #[ver(V >= V13_0B4)] + pub(crate) unk_buf: Array<0x8, u8>, // Init to all-ff + } + + #[versions(AGX)] + impl Default for NotifierState::ver { + fn default() -> Self { + #[allow(unused_mut)] + let mut s: Self = unsafe { core::mem::zeroed() }; + #[ver(V >= V13_0B4)] + s.unk_buf = Array::new([0xff; 0x8]); + s + } + } + + #[derive(Debug)] + #[repr(transparent)] + pub(crate) struct Threshold(AtomicU64); + default_zeroed!(Threshold); + + impl Threshold { + pub(crate) fn increment(&self) { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + let v = self.0.load(Ordering::Relaxed); + self.0.store(v + 1, Ordering::Relaxed); + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Notifier<'a> { + pub(crate) threshold: GpuPointer<'a, super::Threshold>, + pub(crate) generation: AtomicU32, + pub(crate) cur_count: AtomicU32, + pub(crate) unk_10: AtomicU32, + pub(crate) state: NotifierState::ver, + } +} + +trivial_gpustruct!(Threshold); +trivial_gpustruct!(NotifierList); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Notifier { + pub(crate) threshold: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for Notifier::ver { + type Raw<'a> = raw::Notifier::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs new file mode 100644 index 00000000000000..20594233efe831 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU fragment job firmware structures + +use super::types::*; +use super::{event, job, workqueue}; +use crate::{buffer, fw, microseq, mmu}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct ClearPipelineBinding { + pub(crate) pipeline_bind: U64, + pub(crate) address: U64, + } + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct StorePipelineBinding { + pub(crate) unk_0: U64, + pub(crate) unk_8: u32, + pub(crate) pipeline_bind: u32, + pub(crate) unk_10: u32, + pub(crate) address: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c_padding: u32, + } + + impl StorePipelineBinding { + pub(crate) fn new(pipeline_bind: u32, address: u32) -> StorePipelineBinding { + StorePipelineBinding { + pipeline_bind, + address, + ..Default::default() + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ArrayAddr { + pub(crate) ptr: U64, + pub(crate) unk_padding: U64, + } + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct AuxFBInfo { + pub(crate) iogpu_unk_214: u32, + pub(crate) unk2: u32, + pub(crate) width: u32, + pub(crate) height: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk3: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) utile_config: u32, + pub(crate) unk_4: u32, + pub(crate) clear_pipeline: ClearPipelineBinding, + pub(crate) ppp_multisamplectl: U64, + pub(crate) scissor_array: U64, + pub(crate) depth_bias_array: U64, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) depth_dimensions: U64, + pub(crate) visibility_result_buffer: U64, + pub(crate) zls_ctrl: U64, + + #[ver(G >= G14)] + pub(crate) unk_58_g14_0: U64, + #[ver(G >= G14)] + pub(crate) unk_58_g14_8: U64, + + pub(crate) depth_buffer_ptr1: U64, + pub(crate) depth_buffer_ptr2: U64, + pub(crate) stencil_buffer_ptr1: U64, + pub(crate) stencil_buffer_ptr2: U64, + + #[ver(G >= G14)] + pub(crate) unk_68_g14_0: Array<0x20, u8>, + + pub(crate) unk_78: Array<0x4, U64>, + pub(crate) depth_meta_buffer_ptr1: U64, + pub(crate) unk_a0: U64, + pub(crate) depth_meta_buffer_ptr2: U64, + pub(crate) unk_b0: U64, + pub(crate) stencil_meta_buffer_ptr1: U64, + pub(crate) unk_c0: U64, + pub(crate) stencil_meta_buffer_ptr2: U64, + pub(crate) unk_d0: U64, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) mtile_stride_dwords: U64, + pub(crate) tvb_heapmeta_2: GpuPointer<'a, &'a [u8]>, + pub(crate) tile_config: U64, + pub(crate) aux_fb: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_108: Array<0x6, U64>, + pub(crate) pipeline_base: U64, + pub(crate) unk_140: U64, + pub(crate) unk_148: U64, + pub(crate) unk_150: U64, + pub(crate) unk_158: U64, + pub(crate) unk_160: U64, + + #[ver(G < G14)] + pub(crate) unk_168_padding: Array<0x1d8, u8>, + #[ver(G >= G14)] + pub(crate) unk_168_padding: Array<0x1a8, u8>, + #[ver(V < V13_0B4)] + pub(crate) __pad0: Pad<0x8>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2 { + pub(crate) store_pipeline_bind: u32, + pub(crate) store_pipeline_addr: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) merge_upper_x: F32, + pub(crate) merge_upper_y: F32, + pub(crate) unk_18: U64, + pub(crate) utiles_per_mtile_y: u16, + pub(crate) utiles_per_mtile_x: u16, + pub(crate) unk_24: u32, + pub(crate) tile_counts: u32, + pub(crate) iogpu_unk_212: u32, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) unk_40: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters3 { + pub(crate) unk_44_padding: Array<0xac, u8>, + pub(crate) depth_bias_array: ArrayAddr, + pub(crate) scissor_array: ArrayAddr, + pub(crate) visibility_result_buffer: U64, + pub(crate) unk_118: U64, + pub(crate) unk_120: Array<0x25, U64>, + pub(crate) unk_reload_pipeline: ClearPipelineBinding, + pub(crate) unk_258: U64, + pub(crate) unk_260: U64, + pub(crate) unk_268: U64, + pub(crate) unk_270: U64, + pub(crate) reload_pipeline: ClearPipelineBinding, + pub(crate) zls_ctrl: U64, + pub(crate) unk_290: U64, + pub(crate) depth_buffer_ptr1: U64, + pub(crate) unk_2a0: U64, + pub(crate) unk_2a8: U64, + pub(crate) depth_buffer_ptr2: U64, + pub(crate) depth_buffer_ptr3: U64, + pub(crate) depth_meta_buffer_ptr3: U64, + pub(crate) stencil_buffer_ptr1: U64, + pub(crate) unk_2d0: U64, + pub(crate) unk_2d8: U64, + pub(crate) stencil_buffer_ptr2: U64, + pub(crate) stencil_buffer_ptr3: U64, + pub(crate) stencil_meta_buffer_ptr3: U64, + pub(crate) unk_2f8: Array<2, U64>, + pub(crate) iogpu_unk_212: u32, + pub(crate) unk_30c: u32, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) unk_320_padding: Array<0x10, u8>, + pub(crate) unk_partial_store_pipeline: StorePipelineBinding, + pub(crate) partial_store_pipeline: StorePipelineBinding, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) iogpu_unk_49: u32, + pub(crate) unk_37c: u32, + pub(crate) unk_380: U64, + pub(crate) unk_388: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_390_0: U64, + + pub(crate) depth_dimensions: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunFragment<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) ppp_multisamplectl: U64, + pub(crate) samples: u32, + pub(crate) tiles_per_mtile_y: u16, + pub(crate) tiles_per_mtile_x: u16, + pub(crate) unk_50: U64, + pub(crate) unk_58: U64, + pub(crate) merge_upper_x: F32, + pub(crate) merge_upper_y: F32, + pub(crate) unk_68: U64, + pub(crate) tile_count: U64, + pub(crate) job_params1: JobParameters1::ver<'a>, + pub(crate) job_params2: JobParameters2, + pub(crate) job_params3: JobParameters3::ver, + pub(crate) unk_758_flag: u32, + pub(crate) unk_75c_flag: u32, + pub(crate) unk_buf: Array<0x110, u8>, + pub(crate) busy_flag: u32, + pub(crate) tvb_overflow_count: u32, + pub(crate) unk_878: u32, + pub(crate) encoder_params: job::raw::EncoderParams<'a>, + pub(crate) process_empty_tiles: u32, + pub(crate) no_clear_pipeline_textures: u32, + pub(crate) unk_param: u32, + pub(crate) unk_pointee: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) cur_ts: U64, + pub(crate) start_ts: Option>, + pub(crate) end_ts: Option>, + pub(crate) unk_914: u32, + pub(crate) unk_918: U64, + pub(crate) unk_920: u32, + pub(crate) client_sequence: u8, + pub(crate) pad_925: Array<3, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_928_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_928_4: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_928_d: Array<0x1b, u8>, + // Alignment - handled by allocator + //pad_928: [u8; 0x18], + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunFragment { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) aux_fb: GpuArray, + pub(crate) timestamps: Arc>, +} + +#[versions(AGX)] +impl GpuStruct for RunFragment::ver { + type Raw<'a> = raw::RunFragment::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunFragment::ver {} diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs new file mode 100644 index 00000000000000..44de0c1cccf35a --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -0,0 +1,1264 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{default_zeroed, no_debug, trivial_gpustruct}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannels { + pub(crate) vtx: ChannelRing, + pub(crate) frag: ChannelRing, + pub(crate) comp: ChannelRing, + } + #[versions(AGX)] + default_zeroed!(PipeChannels::ver); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + __pad4: Pad<0xc>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, i16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2T8112 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) t8112: HwDataShared2T8112, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + pub(crate) unk_510: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) unk_4c: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) unk_60: F32, + pub(crate) unk_64: F32, + pub(crate) unk_68: F32, + pub(crate) unk_6c: F32, + pub(crate) unk_70: F32, + pub(crate) unk_74: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) unk_80: F32, + pub(crate) unk_84: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: u32, + pub(crate) unk_b4: u32, + pub(crate) unk_b8: u32, + pub(crate) unk_bc: u32, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + #[derive(Default)] + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + + #[versions(AGX)] + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZone { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_10: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct HwDataA { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + #[ver(V >= V13_0B4)] + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) ppm_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + pub(crate) unk_coef_a1: Array<8, Array<8, F32>>, + pub(crate) unk_coef_a2: Array<8, Array<8, F32>>, + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_target_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZone::ver>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_e10_pad: Array<0x10, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_e10_0: HwDataA130Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + pub(crate) fast_die0_sensor_mask_2: U64, + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array<8, F32>>, + pub(crate) unk_coef_b2: Array<8, Array<8, F32>>, + pub(crate) pad_1048: Pad<0x5e4>, + pub(crate) fast_die0_sensor_mask_alt: U64, + #[ver(V < V13_0B4)] + pub(crate) fast_die0_sensor_present: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + #[ver(V >= V13_0B4)] + pub(crate) unk_pad1: Pad<0x20>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + pub(crate) unk_3d34: Array<0x38, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3d6c: Array<0x38, u8>, + } + #[versions(AGX)] + default_zeroed!(HwDataA::ver); + #[versions(AGX)] + no_debug!(HwDataA::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) size: u32, + pub(crate) range_size: u32, + pub(crate) readwrite: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataB { + #[ver(V < V13_0B4)] + pub(crate) unk_0: U64, + + pub(crate) unk_8: U64, + + #[ver(V < V13_0B4)] + pub(crate) unk_10: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) unkptr_38: U64, + pub(crate) pad_40: Pad<0x20>, + + #[ver(V < V13_0B4)] + pub(crate) yuv_matrices: Array<0xf, Array<3, Array<4, i16>>>, + + #[ver(V >= V13_0B4)] + pub(crate) yuv_matrices: Array<0x3f, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array<0x14, IOMapping>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_450_0: Array<0x68, u8>, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + pub(crate) pad_498: Pad<0x4>, + pub(crate) unk_49c: u32, + pub(crate) unk_4a0: u32, + pub(crate) unk_4a4: u32, + pub(crate) pad_4a8: Pad<0x4>, + pub(crate) unk_4ac: u32, + pub(crate) pad_4b0: Pad<0x8>, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: Array<0x4, u8>, + pub(crate) unk_4c0: u32, + pub(crate) unk_4c4: u32, + pub(crate) unk_4c8: u32, + pub(crate) unk_4cc: u32, + pub(crate) unk_4d0: u32, + pub(crate) unk_4d4: u32, + pub(crate) unk_4d8: Array<0x4, u8>, + pub(crate) unk_4dc: u32, + pub(crate) unk_4e0: U64, + pub(crate) unk_4e8: u32, + pub(crate) unk_4ec: u32, + pub(crate) unk_4f0: u32, + pub(crate) unk_4f4: u32, + pub(crate) unk_4f8: u32, + pub(crate) unk_4fc: u32, + pub(crate) unk_500: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_504_0: u32, + + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + pub(crate) unk_510: u32, + pub(crate) unk_514: u32, + pub(crate) unk_518: u32, + pub(crate) unk_51c: u32, + pub(crate) unk_520: u32, + pub(crate) unk_524: u32, + pub(crate) unk_528: u32, + pub(crate) unk_52c: u32, + pub(crate) unk_530: u32, + pub(crate) unk_534: u32, + pub(crate) unk_538: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_53c_0: u32, + + pub(crate) num_frags: u32, + pub(crate) unk_540: u32, + pub(crate) unk_544: u32, + pub(crate) unk_548: u32, + pub(crate) unk_54c: u32, + pub(crate) unk_550: u32, + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + #[ver(V < V13_0B4)] + pub(crate) num_pstates: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + #[ver(V < V13_0B4)] + pub(crate) min_sram_volt: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ab8: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_abc: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ac0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ac4_0: Array<0x1f0, u8>, + + pub(crate) pad_ac4: Pad<0x8>, + pub(crate) unk_acc: u32, + pub(crate) unk_ad0: u32, + pub(crate) pad_ad4: Pad<0x10>, + pub(crate) unk_ae4: Array<0x4, u32>, + pub(crate) pad_af4: Pad<0x4>, + pub(crate) unk_af8: u32, + pub(crate) pad_afc: Pad<0x8>, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: u32, + pub(crate) unk_b0c: u32, + pub(crate) unk_b10: u32, + pub(crate) pad_b14: Pad<0x8>, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u32, + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: u32, + pub(crate) unk_b30: u32, + pub(crate) unk_b34: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_4: u32, + + pub(crate) unk_b38: Array<0xc, u32>, + pub(crate) unk_b68: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b6c: Array<0xd0, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c3c: u32, + } + #[versions(AGX)] + default_zeroed!(HwDataB::ver); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct GpuQueueStatsVtx { + pub(crate) busy: u32, + pub(crate) unk_4: u32, + pub(crate) cur_cmdqueue: U64, + pub(crate) cur_count: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(GpuQueueStatsVtx); + + #[versions(AGX)] + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + pub(crate) unk_4: u32, + pub(crate) queues: Array<0x4, GpuQueueStatsVtx>, + pub(crate) unk_68: Array<0x8, u8>, + pub(crate) unk_70: u32, + pub(crate) unk_74: u32, + pub(crate) unk_timestamp: U64, + pub(crate) unk_80: Array<0x40, u8>, + } + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct GpuQueueStatsFrag { + pub(crate) busy: u32, + pub(crate) cur_cmdqueue: U64, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: Array<0x14, u8>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFrag { + pub(crate) unk_0: Array<0x18, u8>, + pub(crate) queues: Array<0x4, GpuQueueStatsFrag>, + pub(crate) unk_d0: Array<0x38, u8>, + pub(crate) tvb_overflows_1: u32, + pub(crate) tvb_overflows_2: u32, + pub(crate) unk_f8: u32, + pub(crate) unk_fc: u32, + pub(crate) cur_stamp_id: i32, + pub(crate) unk_104: Array<0x14, u8>, + pub(crate) unk_118: i32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) unk_128: u32, + pub(crate) unk_12c: u32, + pub(crate) unk_timestamp: U64, + pub(crate) unk_134: Array<0x8c, u8>, + } + #[versions(AGX)] + default_zeroed!(GpuStatsFrag::ver); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx::ver, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x5c4, u8>, + } + #[versions(AGX)] + default_zeroed!(GpuGlobalStatsVtx::ver); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFrag { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFrag::ver, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x580, u8>, + } + #[versions(AGX)] + default_zeroed!(GpuGlobalStatsFrag::ver); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + pub(crate) unk: Array<0x140, u8>, + } + default_zeroed!(GpuStatsComp); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RuntimeScratch { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + default_zeroed!(RuntimeScratch); + + pub(crate) type BufferMgrCtl = Array<4, u32>; + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct RuntimePointers<'a> { + pub(crate) pipes: Array<4, PipeChannels::ver>, + + pub(crate) device_control: + ChannelRing, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + pub(crate) __pad0: Pad<0x50>, + pub(crate) unk_160: U64, + pub(crate) unk_168: U64, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx::ver>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, + pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, + pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) hwdata_b_2: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) fwlog_buf: Option>, + pub(crate) unkptr_1b8: GpuPointer<'a, &'a [u8]>, + pub(crate) unkptr_1c0: GpuPointer<'a, &'a [u8]>, + pub(crate) unkptr_1c8: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: u32, + pub(crate) unk_1d8: Array<0x3c, u8>, + pub(crate) buffer_mgr_ctl: GpuPointer<'a, &'a [BufferMgrCtl]>, + pub(crate) buffer_mgr_ctl_2: GpuPointer<'a, &'a [BufferMgrCtl]>, + pub(crate) __pad1: Pad<0x5c>, + pub(crate) gpu_scratch: RuntimeScratch, + } + #[versions(AGX)] + no_debug!(RuntimePointers::ver<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct GlobalsSub { + pub(crate) unk_54: u16, + pub(crate) unk_56: u16, + pub(crate) unk_58: u16, + pub(crate) unk_5a: U32, + pub(crate) unk_5e: U32, + pub(crate) unk_62: U32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_66_0: Array<0xc, u8>, + + pub(crate) unk_66: U32, + pub(crate) unk_6a: Array<0x16, u8>, + } + #[versions(AGX)] + default_zeroed!(GlobalsSub::ver); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Globals { + pub(crate) ktrace_enable: u32, + pub(crate) unk_4: Array<0x20, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_24_0: u32, + + pub(crate) unk_24: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_28_0: u32, + + pub(crate) unk_28: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_2c_0: u32, + + pub(crate) unk_2c: u32, + pub(crate) unk_30: u32, + pub(crate) unk_34: u32, + pub(crate) unk_38: Array<0x1c, u8>, + + pub(crate) sub: GlobalsSub::ver, + + pub(crate) unk_80: Array<0xf80, u8>, + pub(crate) unk_1000: Array<0x7000, u8>, + pub(crate) unk_8000: Array<0x900, u8>, + + #[ver(V >= V13_0B4 && V < V13_2)] + pub(crate) unk_8900_0: u32, + + pub(crate) unk_8900: u32, + pub(crate) pending_submissions: AtomicU32, + pub(crate) max_power: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_8914: u32, + pub(crate) unk_8918: u32, + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_8920: u32, + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + pub(crate) unk_8978: Array<0x44, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89bc_0: Array<0x3c, u8>, + + pub(crate) unk_89bc: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_89c4: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_89d4: Array<0xc, u8>, + pub(crate) unk_89e0: u32, + pub(crate) max_power_2: u32, + pub(crate) ppm_kp: F32, + pub(crate) ppm_ki_dt: F32, + pub(crate) unk_89f0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_0: Array<0x8, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_c: Array<0x50, u8>, + + pub(crate) unk_89f4: u32, + pub(crate) hws1: HwDataShared1, + pub(crate) hws2: HwDataShared2, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_0: Array<0x28, u8>, + + pub(crate) hws3: HwDataShared3, + pub(crate) unk_9004: Array<8, u8>, + pub(crate) unk_900c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_4: Array<0x14, u8>, + + pub(crate) unk_9010: Array<0x2c, u8>, + pub(crate) unk_903c: u32, + pub(crate) unk_9040: Array<0xc0, u8>, + pub(crate) unk_9100: Array<0x6f00, u8>, + pub(crate) unk_10000: Array<0xe50, u8>, + pub(crate) unk_10e50: u32, + pub(crate) unk_10e54: Array<0x2c, u8>, + pub(crate) fault_control: u32, + pub(crate) do_init: u32, + pub(crate) unk_10e88: Array<0x188, u8>, + pub(crate) idle_ts: U64, + pub(crate) idle_unk: U64, + pub(crate) unk_11020: u32, + pub(crate) unk_11024: u32, + pub(crate) unk_11028: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_10: u32, + + pub(crate) unk_1102c: u32, + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + pub(crate) pending_stamps: Array<0x110, PendingStamp>, + pub(crate) unk_117bc: u32, + pub(crate) fault_info: FaultInfo, + pub(crate) counter: u32, + pub(crate) unk_118dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_118e0_0: Array<0x9c, u8>, + + pub(crate) unk_118e0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_118e4_0: u32, + + pub(crate) unk_118e4: u32, + pub(crate) unk_118e8: u32, + pub(crate) unk_118ec: Array<0x15, u8>, + pub(crate) unk_11901: Array<0x43f, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11d40: Array<0x19c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11edc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11ee0: Array<0x1c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11efc: u32, + } + #[versions(AGX)] + default_zeroed!(Globals::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitData<'a> { + #[ver(V >= V13_0B4)] + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointers::ver>, + pub(crate) globals: GpuPointer<'a, super::Globals::ver>, + pub(crate) fw_status: GpuPointer<'a, super::FwStatus>, + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + pub(crate) __pad0: Pad<0x14>, + pub(crate) host_mapped_fw_allocations: u32, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: u32, + pub(crate) unk_b4: u32, + pub(crate) unk_b8: u32, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); + +#[versions(AGX)] +#[derive(Debug, Default)] +pub(crate) struct GpuGlobalStatsVtx {} + +#[versions(AGX)] +impl GpuStruct for GpuGlobalStatsVtx::ver { + type Raw<'a> = raw::GpuGlobalStatsVtx::ver; +} + +#[versions(AGX)] +#[derive(Debug, Default)] +pub(crate) struct GpuGlobalStatsFrag {} + +#[versions(AGX)] +impl GpuStruct for GpuGlobalStatsFrag::ver { + type Raw<'a> = raw::GpuGlobalStatsFrag::ver; +} + +#[derive(Debug, Default)] +pub(crate) struct GpuStatsComp {} + +impl GpuStruct for GpuStatsComp { + type Raw<'a> = raw::GpuStatsComp; +} + +#[versions(AGX)] +#[derive(Debug, Default)] +pub(crate) struct HwDataA {} + +#[versions(AGX)] +impl GpuStruct for HwDataA::ver { + type Raw<'a> = raw::HwDataA::ver; +} + +#[versions(AGX)] +#[derive(Debug, Default)] +pub(crate) struct HwDataB {} + +#[versions(AGX)] +impl GpuStruct for HwDataB::ver { + type Raw<'a> = raw::HwDataB::ver; +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Stats { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RuntimePointers { + pub(crate) stats: Stats::ver, + + pub(crate) hwdata_a: GpuObject, + pub(crate) unkptr_190: GpuArray, + pub(crate) unkptr_198: GpuArray, + pub(crate) hwdata_b: GpuObject, + + pub(crate) unkptr_1b8: GpuArray, + pub(crate) unkptr_1c0: GpuArray, + pub(crate) unkptr_1c8: GpuArray, + + pub(crate) buffer_mgr_ctl: GpuArray, +} + +#[versions(AGX)] +impl GpuStruct for RuntimePointers::ver { + type Raw<'a> = raw::RuntimePointers::ver<'a>; +} + +#[versions(AGX)] +#[derive(Debug, Default)] +pub(crate) struct Globals {} + +#[versions(AGX)] +impl GpuStruct for Globals::ver { + type Raw<'a> = raw::Globals::ver; +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct InitData { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + pub(crate) fw_status: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for InitData::ver { + type Raw<'a> = raw::InitData::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs new file mode 100644 index 00000000000000..a0bbf67b1b1d3e --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common GPU job firmware structures + +use super::types::*; +use crate::{default_zeroed, trivial_gpustruct}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct JobMeta { + pub(crate) unk_4: u32, + pub(crate) stamp: GpuWeakPointer, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) stamp_slot: u32, + pub(crate) evctl_index: u32, + pub(crate) flush_stamps: u32, + pub(crate) uuid: u32, + pub(crate) cmd_seq: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct EncoderParams<'a> { + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) encoder_id: u32, + pub(crate) unk_18: u32, + pub(crate) iogpu_compute_unk44: u32, + pub(crate) seq_buffer: GpuPointer<'a, &'a [u64]>, + pub(crate) unk_28: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobTimestamps { + pub(crate) start: AtomicU64, + pub(crate) end: AtomicU64, + } + default_zeroed!(JobTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RenderTimestamps { + pub(crate) vtx: JobTimestamps, + pub(crate) frag: JobTimestamps, + } + default_zeroed!(RenderTimestamps); +} + +trivial_gpustruct!(JobTimestamps); +trivial_gpustruct!(RenderTimestamps); diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs new file mode 100644 index 00000000000000..8deea3fb991488 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU firmware microsequence operations + +use super::types::*; +use super::{buffer, compute, fragment, initdata, vertex, workqueue}; +use crate::default_zeroed; + +pub(crate) trait Operation {} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +enum OpCode { + WaitForIdle = 0x01, + RetireStamp = 0x18, + #[allow(dead_code)] + Timestamp = 0x19, + StartVertex = 0x22, + FinalizeVertex = 0x23, + StartFragment = 0x24, + FinalizeFragment = 0x25, + StartCompute = 0x29, + FinalizeCompute = 0x2a, +} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum Pipe { + Vertex = 1 << 0, + Fragment = 1 << 8, + Compute = 1 << 15, +} + +pub(crate) const MAX_ATTACHMENTS: usize = 16; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub(crate) struct Attachment { + pub(crate) address: U64, + pub(crate) size: u32, + pub(crate) unk_c: u16, + pub(crate) unk_e: u16, +} +default_zeroed!(Attachment); + +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub(crate) struct Attachments { + pub(crate) list: Array, + pub(crate) count: u32, +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct OpHeader(u32); + +impl OpHeader { + const fn new(opcode: OpCode) -> OpHeader { + OpHeader(opcode as u32) + } + const fn with_args(opcode: OpCode, args: u32) -> OpHeader { + OpHeader(opcode as u32 | args) + } +} + +macro_rules! simple_op { + ($name:ident) => { + #[derive(Debug, Copy, Clone)] + pub(crate) struct $name(OpHeader); + + impl $name { + pub(crate) const HEADER: $name = $name(OpHeader::new(OpCode::$name)); + } + }; +} + +pub(crate) mod op { + use super::*; + + simple_op!(StartVertex); + simple_op!(FinalizeVertex); + simple_op!(StartFragment); + simple_op!(FinalizeFragment); + simple_op!(StartCompute); + simple_op!(FinalizeCompute); + + #[derive(Debug, Copy, Clone)] + pub(crate) struct RetireStamp(OpHeader); + impl RetireStamp { + pub(crate) const HEADER: RetireStamp = + RetireStamp(OpHeader::with_args(OpCode::RetireStamp, 0x40000000)); + } + + #[derive(Debug, Copy, Clone)] + pub(crate) struct WaitForIdle(OpHeader); + impl WaitForIdle { + pub(crate) const fn new(pipe: Pipe) -> WaitForIdle { + WaitForIdle(OpHeader::with_args(OpCode::WaitForIdle, (pipe as u32) << 8)) + } + } + + #[derive(Debug, Copy, Clone)] + pub(crate) struct Timestamp(OpHeader); + impl Timestamp { + #[allow(dead_code)] + pub(crate) const fn new(flag: bool) -> Timestamp { + Timestamp(OpHeader::with_args(OpCode::Timestamp, (flag as u32) << 31)) + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle { + pub(crate) header: op::WaitForIdle, +} + +impl Operation for WaitForIdle {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct RetireStamp { + pub(crate) header: op::RetireStamp, +} + +impl Operation for RetireStamp {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct Timestamp<'a> { + pub(crate) header: op::Timestamp, + pub(crate) cur_ts: GpuWeakPointer, + pub(crate) start_ts: GpuWeakPointer>>, + pub(crate) update_ts: GpuWeakPointer>>, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) unk_24: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: GpuWeakPointer, + + pub(crate) uuid: u32, + pub(crate) unk_30_padding: u32, +} + +#[versions(AGX)] +impl<'a> Operation for Timestamp::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartVertex<'a> { + pub(crate) header: op::StartVertex, + pub(crate) tiling_params: GpuWeakPointer, + pub(crate) job_params1: GpuWeakPointer>, + pub(crate) buffer: GpuWeakPointer, + pub(crate) scene: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_38: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_44: u32, + pub(crate) cmd_seq: U64, + pub(crate) unk_50: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + pub(crate) uuid: u32, + pub(crate) unk_70: u32, + pub(crate) unk_74: Array<0x1d, U64>, + pub(crate) unk_15c: u32, + pub(crate) unk_160: U64, + pub(crate) unk_168: u32, + pub(crate) unk_16c: u32, + pub(crate) unk_170: U64, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, + + pub(crate) unk_178: u32, +} + +#[versions(AGX)] +impl<'a> Operation for StartVertex::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeVertex { + pub(crate) header: op::FinalizeVertex, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_34: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_48: U64, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: U64, + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + + #[ver(G >= G14 && V < V13_0B4)] + pub(crate) unk_68_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) unk_70: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_74: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeVertex::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartFragment<'a> { + pub(crate) header: op::StartFragment, + pub(crate) job_params2: GpuWeakPointer, + pub(crate) job_params1: GpuWeakPointer>, + pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, + pub(crate) stats: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) tvb_overflow_count: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_50: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_5c: u32, + pub(crate) cmd_seq: U64, + pub(crate) unk_68: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) unk_84: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) unk_190: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartFragment::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeFragment { + pub(crate) header: op::FinalizeFragment, + pub(crate) uuid: u32, + pub(crate) unk_8: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_18: u32, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) unk_2c: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_60: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + pub(crate) unk_6c: U64, + pub(crate) unk_74: U64, + pub(crate) unk_7c: U64, + pub(crate) unk_84: U64, + pub(crate) unk_8c: U64, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_8c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) unk_98: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9c: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeFragment::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartCompute<'a> { + pub(crate) header: op::StartCompute, + pub(crate) unk_pointer: GpuWeakPointer>, + pub(crate) job_params1: GpuWeakPointer>, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) event_generation: u32, + pub(crate) cmd_seq: U64, + pub(crate) unk_38: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_44: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartCompute::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeCompute<'a> { + pub(crate) header: op::FinalizeCompute, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_18: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_24: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) unk_4c: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_5c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) unk_60: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_64: Array<0xd, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) unk_79: Array<0x7, u8>, +} + +#[versions(AGX)] +impl<'a> Operation for FinalizeCompute::ver<'a> {} diff --git a/drivers/gpu/drm/asahi/fw/mod.rs b/drivers/gpu/drm/asahi/fw/mod.rs new file mode 100644 index 00000000000000..a5649aa20d3a8e --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware structures for Apple AGX GPUs + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod event; +pub(crate) mod fragment; +pub(crate) mod initdata; +pub(crate) mod job; +pub(crate) mod microseq; +pub(crate) mod types; +pub(crate) mod vertex; +pub(crate) mod workqueue; diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs new file mode 100644 index 00000000000000..c1a07be1e047e7 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common types for firmware structure definitions + +use crate::{alloc, object}; +use core::fmt; +use core::ops::{Deref, DerefMut, Index, IndexMut}; + +pub(crate) use crate::event::EventValue; +pub(crate) use crate::object::{GpuPointer, GpuStruct, GpuWeakPointer}; +pub(crate) use crate::{f32, float::F32}; + +pub(crate) use ::alloc::boxed::Box; +pub(crate) use core::fmt::Debug; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::sync::atomic::{AtomicI32, AtomicU32, AtomicU64}; +pub(crate) use kernel::macros::versions; + +// Make the trait visible +pub(crate) use crate::alloc::Allocator as _Allocator; + +/// General allocator type used for the driver +pub(crate) type Allocator = alloc::DefaultAllocator; + +/// General GpuObject type used for the driver +pub(crate) type GpuObject = + object::GpuObject>; + +/// General GpuArray type used for the driver +pub(crate) type GpuArray = object::GpuArray>; + +/// General GpuOnlyArray type used for the driver +pub(crate) type GpuOnlyArray = + object::GpuOnlyArray>; + +/// A stamp slot that is shared between firmware and the driver. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct Stamp(pub(crate) AtomicU32); + +/// A stamp slot that is for private firmware use. +/// +/// This is a separate type to guard against pointer type confusion. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct FwStamp(pub(crate) AtomicU32); + +/// An unaligned u64 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U64(pub(crate) u64); + +unsafe impl Zeroed for U64 {} + +impl fmt::Debug for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +unsafe impl Zeroed for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +unsafe impl Zeroed for u8 {} +unsafe impl Zeroed for u16 {} +unsafe impl Zeroed for u32 {} +unsafe impl Zeroed for u64 {} +unsafe impl Zeroed for i8 {} +unsafe impl Zeroed for i16 {} +unsafe impl Zeroed for i32 {} +unsafe impl Zeroed for i64 {} + +/// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by +/// hand or not very useful. +#[macro_export] +macro_rules! no_debug { + ($type:ty) => { + impl Debug for $type { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "...") + } + } + }; +} + +/// Types which can be safely initialized with an all-zero bit pattern. +/// +/// See: https://github.com/rust-lang/rfcs/issues/2626 +/// +/// # Safety +/// +/// This trait must only be implemented if a type only contains primitive types which can be +/// zero-initialized, FFI structs intended to be zero-initialized, or other types which impl Zeroed. +pub(crate) unsafe trait Zeroed: Default { + fn zeroed() -> Self { + // SAFETY: The user is responsible for ensuring this is safe. + unsafe { core::mem::zeroed() } + } +} + +/// Implement Zeroed for a given type (and Default along with it). +/// +/// # Safety +/// +/// This macro must only be used if a type only contains primitive types which can be +/// zero-initialized, FFI structs intended to be zero-initialized, or other types which impl Zeroed. +#[macro_export] +macro_rules! default_zeroed { + (<$($lt:lifetime),*>, $type:ty) => { + impl<$($lt),*> Default for $type { + fn default() -> $type { + Zeroed::zeroed() + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl<$($lt),*> Zeroed for $type {} + }; + ($type:ty) => { + impl Default for $type { + fn default() -> $type { + Zeroed::zeroed() + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl Zeroed for $type {} + }; +} + +/// A convenience type for a number of padding bytes. Hidden from Debug formatting. +#[derive(Copy, Clone)] +#[repr(C, packed)] +pub(crate) struct Pad([u8; N]); + +/// SAFETY: Primitive type, safe to zero-init. +unsafe impl Zeroed for Pad {} + +impl Default for Pad { + fn default() -> Self { + Zeroed::zeroed() + } +} + +impl fmt::Debug for Pad { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(format_args!("")) + } +} + +/// A convenience type for a fixed-sized array with Default/Zeroed impls. +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) struct Array([T; N]); + +impl Array { + pub(crate) fn new(data: [T; N]) -> Self { + Self(data) + } +} + +// SAFETY: Arrays of Zeroed values can be safely Zeroed. +unsafe impl Zeroed for Array {} + +impl Default for Array { + fn default() -> Self { + Zeroed::zeroed() + } +} + +impl Index for Array { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl IndexMut for Array { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } +} + +impl Deref for Array { + type Target = [T; N]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Array { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl fmt::Debug for Array { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// Convenience macro to define an identically-named trivial GpuStruct with no inner fields for a +/// given raw type name. +#[macro_export] +macro_rules! trivial_gpustruct { + ($type:ident) => { + #[derive(Debug, Default)] + pub(crate) struct $type {} + + impl GpuStruct for $type { + type Raw<'a> = raw::$type; + } + }; +} diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs new file mode 100644 index 00000000000000..781220826bf623 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU vertex job firmware structures + +use super::types::*; +use super::{event, job, workqueue}; +use crate::{buffer, fw, microseq, mmu}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct TilingParameters { + pub(crate) rgn_size: u32, + pub(crate) unk_4: u32, + pub(crate) ppp_ctrl: u32, + pub(crate) x_max: u16, + pub(crate) y_max: u16, + pub(crate) te_screen: u32, + pub(crate) te_mtile1: u32, + pub(crate) te_mtile2: u32, + pub(crate) tiles_per_mtile: u32, + pub(crate) tpc_stride: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) unk_0: U64, + pub(crate) unk_8: F32, + pub(crate) unk_c: F32, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_tilemaps: Option>, + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) iogpu_unk_54: u32, + pub(crate) iogpu_unk_55: u32, + pub(crate) iogpu_unk_56: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta1: Option>, + pub(crate) utile_config: u32, + pub(crate) unk_4c: u32, + pub(crate) ppp_multisamplectl: U64, + pub(crate) tvb_heapmeta_2: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) unk_60: U64, + #[ver(G < G14)] + pub(crate) core_mask: Array<2, u32>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_80: U64, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) encoder_addr: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta2: Option>, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta3: Option>, + #[ver(G < G14)] + pub(crate) tiling_control: u32, + #[ver(G < G14)] + pub(crate) unk_ac: u32, + pub(crate) unk_b0: Array<6, U64>, + pub(crate) pipeline_base: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta4: Option>, + #[ver(G < G14)] + pub(crate) unk_f0: U64, + pub(crate) unk_f8: U64, + pub(crate) unk_100: Array<3, U64>, + pub(crate) unk_118: u32, + #[ver(G >= G14)] + pub(crate) __pad: Pad<{ 8 * 9 }>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + pub(crate) unk_480: Array<4, u32>, + pub(crate) unk_498: U64, + pub(crate) unk_4a0: u32, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_4ac: u32, + pub(crate) unk_4b0: U64, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: U64, + pub(crate) unk_4c4_padding: Array<0x48, u8>, + pub(crate) unk_50c: u32, + pub(crate) unk_510: U64, + pub(crate) unk_518: U64, + pub(crate) unk_520: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunVertex<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer_slot: u32, + pub(crate) unk_1c: u32, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) unk_34: u32, + pub(crate) job_params1: JobParameters1::ver<'a>, + pub(crate) unk_154: Array<0x268, u8>, + pub(crate) tiling_params: TilingParameters, + pub(crate) unk_3e8: Array<0x74, u8>, + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tpc_size: U64, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) fragment_stamp_slot: u32, + pub(crate) fragment_stamp_value: EventValue, + pub(crate) unk_pointee: u32, + pub(crate) unk_pad: u32, + pub(crate) job_params2: JobParameters2<'a>, + pub(crate) encoder_params: job::raw::EncoderParams<'a>, + pub(crate) unk_55c: u32, + pub(crate) unk_560: u32, + pub(crate) memoryless_rts_used: u32, + pub(crate) unk_568: u32, + pub(crate) unk_56c: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) cur_ts: U64, + pub(crate) start_ts: Option>, + pub(crate) end_ts: Option>, + pub(crate) unk_5c4: u32, + pub(crate) unk_5c8: u32, + pub(crate) unk_5cc: u32, + pub(crate) unk_5d0: u32, + pub(crate) client_sequence: u8, + pub(crate) pad_5d5: Array<3, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_5d8_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_5d8_4: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_5d8_d: Array<0x13, u8>, + + pub(crate) pad_5d8: Pad<0x8>, + // Alignment - handled by allocator + //#[ver(V >= V13_0B4)] + //pad_5e0: Pad<0x18>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunVertex { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, +} + +#[versions(AGX)] +impl GpuStruct for RunVertex::ver { + type Raw<'a> = raw::RunVertex::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunVertex::ver {} diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs new file mode 100644 index 00000000000000..947742e8dc8843 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU work queue firmware structes + +use super::event; +use super::types::*; +use crate::event::EventValue; +use crate::{default_zeroed, trivial_gpustruct}; +use kernel::sync::Arc; + +#[derive(Debug)] +#[repr(u32)] +pub(crate) enum CommandType { + RunVertex = 0, + RunFragment = 1, + #[allow(dead_code)] + RunBlitter = 2, + RunCompute = 3, + Barrier = 4, + InitBuffer = 6, +} + +pub(crate) trait Command: GpuStruct + Send + Sync {} + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Barrier { + pub(crate) tag: CommandType, + pub(crate) wait_stamp: GpuWeakPointer, + pub(crate) wait_value: EventValue, + pub(crate) wait_slot: u32, + pub(crate) stamp_self: EventValue, + pub(crate) uuid: u32, + pub(crate) unk: u32, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct GpuContextData { + pub(crate) unk_0: u8, + pub(crate) unk_1: u8, + unk_2: Array<0x2, u8>, + pub(crate) unk_4: u8, + pub(crate) unk_5: u8, + unk_6: Array<0x18, u8>, + pub(crate) unk_1e: u8, + pub(crate) unk_1f: u8, + unk_20: Array<0x3, u8>, + pub(crate) unk_23: u8, + unk_24: Array<0x1c, u8>, + } + + impl Default for GpuContextData { + fn default() -> Self { + Self { + unk_0: 0xff, + unk_1: 0xff, + unk_2: Default::default(), + unk_4: 0, + unk_5: 1, + unk_6: Default::default(), + unk_1e: 0xff, + unk_1f: 0, + unk_20: Default::default(), + unk_23: 2, + unk_24: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RingState { + pub(crate) gpu_doneptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) unk_10: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) unk_20: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) gpu_rptr: AtomicU32, + __pad3: Pad<0xc>, + pub(crate) cpu_wptr: AtomicU32, + __pad4: Pad<0xc>, + pub(crate) rb_size: u32, + __pad5: Pad<0xc>, + // This isn't part of the structure, but it's here as a + // debugging hack so we can inspect what ring position + // the driver considered complete and freeable. + pub(crate) cpu_freeptr: AtomicU32, + __pad6: Pad<0xc>, + } + default_zeroed!(RingState); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct Priority(u32, u32, U64, u32, u32, u32); + + pub(crate) const PRIORITY: [Priority; 4] = [ + Priority(0, 0, U64(0xffff_ffff_ffff_0000), 1, 0, 1), + Priority(1, 1, U64(0xffff_ffff_0000_0000), 0, 0, 0), + Priority(2, 2, U64(0xffff_0000_0000_0000), 0, 0, 2), + Priority(3, 3, U64(0x0000_0000_0000_0000), 0, 0, 3), + ]; + + impl Default for Priority { + fn default() -> Priority { + PRIORITY[2] + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct QueueInfo<'a> { + pub(crate) state: GpuPointer<'a, super::RingState>, + pub(crate) ring: GpuPointer<'a, &'a [u64]>, + pub(crate) notifier_list: GpuPointer<'a, event::NotifierList>, + pub(crate) gpu_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) gpu_rptr1: AtomicU32, + pub(crate) gpu_rptr2: AtomicU32, + pub(crate) gpu_rptr3: AtomicU32, + pub(crate) event_id: AtomicI32, + pub(crate) priority: Priority, + pub(crate) unk_4c: i32, + pub(crate) uuid: u32, + pub(crate) unk_54: i32, + pub(crate) unk_58: U64, + pub(crate) busy: AtomicU32, + pub(crate) __pad: Pad<0x20>, + pub(crate) unk_84_state: AtomicU32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: u32, + pub(crate) unk_94: u32, + pub(crate) pending: AtomicU32, + pub(crate) unk_9c: u32, + #[ver(V >= V13_2)] + pub(crate) unk_a0_0: u32, + pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, + pub(crate) unk_a8: U64, + #[ver(V >= V13_2)] + pub(crate) unk_b0: u32, + } +} + +trivial_gpustruct!(Barrier); +trivial_gpustruct!(GpuContextData); +trivial_gpustruct!(RingState); + +impl Command for Barrier {} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct QueueInfo { + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, + pub(crate) gpu_buf: GpuArray, + pub(crate) notifier_list: Arc>, + pub(crate) gpu_context: Arc>, +} + +#[versions(AGX)] +impl GpuStruct for QueueInfo::ver { + type Raw<'a> = raw::QueueInfo::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs new file mode 100644 index 00000000000000..cf7271d51b236d --- /dev/null +++ b/drivers/gpu/drm/asahi/gem.rs @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi driver GEM object implementation +//! +//! Basic wrappers and adaptations between generic GEM shmem objects and this driver's +//! view of what a GPU buffer object is. It is in charge of keeping track of all mappings for +//! each GEM object so we can remove them when a client (File) or a Vm are destroyed, as well as +//! implementing RTKit buffers on top of GEM objects for firmware use. + +use kernel::{ + bindings, + drm::{gem, gem::shmem}, + error::Result, + prelude::*, + soc::apple::rtkit, + sync::smutex::Mutex, +}; + +use kernel::drm::gem::BaseObject; + +use core::sync::atomic::{AtomicU64, Ordering}; + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::file::DrmFile; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; + +/// Represents the inner data of a GEM object for this driver. +pub(crate) struct DriverObject { + /// Whether this is a kernel-created object. + kernel: bool, + /// Object creation flags. + flags: u32, + /// VM ID for VM-private objects. + vm_id: Option, + /// Locked list of mapping tuples: (file_id, vm_id, mapping) + mappings: Mutex>, + /// ID for debug + id: u64, +} + +/// Type alias for the shmem GEM object type for this driver. +pub(crate) type Object = shmem::Object; + +/// Type alias for the SGTable type for this driver. +pub(crate) type SGTable = shmem::SGTable; + +/// A shared reference to a GEM object for this driver. +pub(crate) struct ObjectRef { + /// The underlying GEM object reference + pub(crate) gem: gem::ObjectRef>, + /// The kernel-side VMap of this object, if needed + vmap: Option>, +} + +static GEM_ID: AtomicU64 = AtomicU64::new(0); + +impl DriverObject { + /// Drop all object mappings for a given file ID. + /// + /// Used on file close. + fn drop_file_mappings(&self, file_id: u64) { + let mut mappings = self.mappings.lock(); + for (index, (mapped_fid, _mapped_vmid, _mapping)) in mappings.iter().enumerate() { + if *mapped_fid == file_id { + mappings.swap_remove(index); + return; + } + } + } + + /// Drop all object mappings for a given VM ID. + /// + /// Used on VM destroy. + fn drop_vm_mappings(&self, vm_id: u64) { + let mut mappings = self.mappings.lock(); + for (index, (_mapped_fid, mapped_vmid, _mapping)) in mappings.iter().enumerate() { + if *mapped_vmid == vm_id { + mappings.swap_remove(index); + return; + } + } + } +} + +impl ObjectRef { + /// Create a new wrapper for a raw GEM object reference. + pub(crate) fn new(gem: gem::ObjectRef>) -> ObjectRef { + ObjectRef { gem, vmap: None } + } + + /// Return the `VMap` for this object, creating it if necessary. + pub(crate) fn vmap(&mut self) -> Result<&mut shmem::VMap> { + if self.vmap.is_none() { + self.vmap = Some(self.gem.vmap()?); + } + Ok(self.vmap.as_mut().unwrap()) + } + + /// Return the IOVA of this object at which it is mapped in a given `Vm` identified by its ID, + /// if it is mapped in that `Vm`. + pub(crate) fn iova(&self, vm_id: u64) -> Option { + let mappings = self.gem.mappings.lock(); + for (_mapped_fid, mapped_vmid, mapping) in mappings.iter() { + if *mapped_vmid == vm_id { + return Some(mapping.iova()); + } + } + + None + } + + /// Returns the size of an object in bytes + pub(crate) fn size(&self) -> usize { + self.gem.size() + } + + /// Maps an object into a given `Vm` at any free address. + /// + /// Returns Err(EBUSY) if there is already a mapping. + pub(crate) fn map_into(&mut self, vm: &crate::mmu::Vm) -> Result { + let vm_id = vm.id(); + + if self.gem.vm_id.is_some() && self.gem.vm_id != Some(vm_id) { + return Err(EINVAL); + } + + let mut mappings = self.gem.mappings.lock(); + for (_mapped_fid, mapped_vmid, _mapping) in mappings.iter() { + if *mapped_vmid == vm_id { + return Err(EBUSY); + } + } + + let sgt = self.gem.sg_table()?; + let new_mapping = vm.map(self.gem.size(), sgt)?; + + let iova = new_mapping.iova(); + mappings.try_push((vm.file_id(), vm_id, new_mapping))?; + Ok(iova) + } + + /// Maps an object into a given `Vm` at any free address within a given range. + /// + /// Returns Err(EBUSY) if there is already a mapping. + pub(crate) fn map_into_range( + &mut self, + vm: &crate::mmu::Vm, + start: u64, + end: u64, + alignment: u64, + prot: u32, + guard: bool, + ) -> Result { + let vm_id = vm.id(); + + if self.gem.vm_id.is_some() && self.gem.vm_id != Some(vm_id) { + return Err(EINVAL); + } + + let mut mappings = self.gem.mappings.lock(); + for (_mapped_fid, mapped_vmid, _mapping) in mappings.iter() { + if *mapped_vmid == vm_id { + return Err(EBUSY); + } + } + + let sgt = self.gem.sg_table()?; + let new_mapping = + vm.map_in_range(self.gem.size(), sgt, alignment, start, end, prot, guard)?; + + let iova = new_mapping.iova(); + mappings.try_push((vm.file_id(), vm_id, new_mapping))?; + Ok(iova) + } + + /// Maps an object into a given `Vm` at a specific address. + /// + /// Returns Err(EBUSY) if there is already a mapping. + /// Returns Err(ENOSPC) if the requested address is already busy. + pub(crate) fn map_at( + &mut self, + vm: &crate::mmu::Vm, + addr: u64, + prot: u32, + guard: bool, + ) -> Result { + let vm_id = vm.id(); + + if self.gem.vm_id.is_some() && self.gem.vm_id != Some(vm_id) { + return Err(EINVAL); + } + + let mut mappings = self.gem.mappings.lock(); + for (_mapped_fid, mapped_vmid, _mapping) in mappings.iter() { + if *mapped_vmid == vm_id { + return Err(EBUSY); + } + } + + let sgt = self.gem.sg_table()?; + let new_mapping = vm.map_at(addr, self.gem.size(), sgt, prot, guard)?; + + let iova = new_mapping.iova(); + assert!(iova == addr as usize); + mappings.try_push((vm.file_id(), vm_id, new_mapping))?; + Ok(()) + } + + /// Drop all mappings for this object owned by a given `Vm` identified by its ID. + pub(crate) fn drop_vm_mappings(&mut self, vm_id: u64) { + self.gem.drop_vm_mappings(vm_id); + } + + /// Drop all mappings for this object owned by a given `File` identified by its ID. + pub(crate) fn drop_file_mappings(&mut self, file_id: u64) { + self.gem.drop_file_mappings(file_id); + } +} + +/// Create a new kernel-owned GEM object. +pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { + let mut gem = shmem::Object::::new(dev, size)?; + gem.kernel = true; + gem.flags = 0; + + gem.set_exportable(false); + + mod_pr_debug!("DriverObject new kernel object id={}\n", gem.id); + Ok(ObjectRef::new(gem.into_ref())) +} + +/// Create a new user-owned GEM object with the given flags. +pub(crate) fn new_object( + dev: &AsahiDevice, + size: usize, + flags: u32, + vm_id: Option, +) -> Result { + let mut gem = shmem::Object::::new(dev, size)?; + gem.kernel = false; + gem.flags = flags; + gem.vm_id = vm_id; + + gem.set_exportable(vm_id.is_none()); + gem.set_wc(flags & bindings::ASAHI_GEM_WRITEBACK == 0); + + mod_pr_debug!( + "DriverObject new user object: vm_id={:?} id={}\n", + vm_id, + gem.id + ); + Ok(ObjectRef::new(gem.into_ref())) +} + +/// Look up a GEM object handle for a `File` and return an `ObjectRef` for it. +pub(crate) fn lookup_handle(file: &DrmFile, handle: u32) -> Result { + Ok(ObjectRef::new(shmem::Object::lookup_handle(file, handle)?)) +} + +impl gem::BaseDriverObject for DriverObject { + /// Callback to create the inner data of a GEM object + fn new(_dev: &AsahiDevice, _size: usize) -> Result { + let id = GEM_ID.fetch_add(1, Ordering::Relaxed); + mod_pr_debug!("DriverObject::new id={}\n", id); + Ok(DriverObject { + kernel: false, + flags: 0, + vm_id: None, + mappings: Mutex::new(Vec::new()), + id, + }) + } + + /// Callback to drop all mappings for a GEM object owned by a given `File` + fn close(obj: &Object, file: &DrmFile) { + mod_pr_debug!("DriverObject::close vm_id={:?} id={}\n", obj.vm_id, obj.id); + obj.drop_file_mappings(file.inner().file_id()); + } +} + +impl Drop for DriverObject { + fn drop(&mut self) { + mod_pr_debug!("DriverObject::drop vm_id={:?} id={}\n", self.vm_id, self.id); + } +} + +impl shmem::DriverObject for DriverObject { + type Driver = crate::driver::AsahiDriver; +} + +impl rtkit::Buffer for ObjectRef { + fn iova(&self) -> Result { + self.iova(0).ok_or(EIO) + } + fn buf(&mut self) -> Result<&mut [u8]> { + let vmap = self.vmap.as_mut().ok_or(ENOMEM)?; + Ok(vmap.as_mut_slice()) + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs new file mode 100644 index 00000000000000..3000872a2485e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -0,0 +1,1093 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU manager +//! +//! This module is the root of all GPU firmware management for a given driver instance. It is +//! responsible for initialization, owning the top-level managers (events, UAT, etc.), and +//! communicating with the raw RtKit endpoints to send and receive messages to/from the GPU +//! firmware. +//! +//! It is also the point where diverging driver firmware/GPU variants (using the versions macro) +//! are unified, so that the top level of the driver itself (in `driver`) does not have to concern +//! itself with version dependence. + +use core::any::Any; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use core::time::Duration; + +use kernel::{ + delay::coarse_sleep, + error::code::*, + macros::versions, + prelude::*, + soc::apple::rtkit, + sync::{smutex::Mutex, Arc, Guard, UniqueArc}, + time, + types::ForeignOwnable, +}; + +use crate::alloc::Allocator; +use crate::box_in_place; +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::fw::channels::PipeType; +use crate::fw::types::U64; +use crate::{ + alloc, buffer, channel, event, fw, gem, hw, initdata, mem, mmu, queue, regs, workqueue, +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gpu; + +/// Firmware endpoint for init & incoming notifications. +const EP_FIRMWARE: u8 = 0x20; + +/// Doorbell endpoint for work/message submissions. +const EP_DOORBELL: u8 = 0x21; + +/// Initialize the GPU firmware. +const MSG_INIT: u64 = 0x81 << 48; +const INIT_DATA_MASK: u64 = (1 << 44) - 1; + +/// TX channel doorbell. +const MSG_TX_DOORBELL: u64 = 0x83 << 48; +/// Firmware control channel doorbell. +const MSG_FWCTL: u64 = 0x84 << 48; +// /// Halt the firmware (?). +// const MSG_HALT: u64 = 0x85 << 48; + +/// Receive channel doorbell notification. +const MSG_RX_DOORBELL: u64 = 0x42 << 48; + +/// Doorbell number for firmware kicks/wakeups. +const DOORBELL_KICKFW: u64 = 0x10; +/// Doorbell number for device control channel kicks. +const DOORBELL_DEVCTRL: u64 = 0x11; + +// Upper kernel half VA address ranges. +/// Private (cached) firmware structure VA range base. +const IOVA_KERN_PRIV_BASE: u64 = 0xffffffa000000000; +/// Private (cached) firmware structure VA range top. +const IOVA_KERN_PRIV_TOP: u64 = 0xffffffa7ffffffff; +/// Shared (uncached) firmware structure VA range base. +const IOVA_KERN_SHARED_BASE: u64 = 0xffffffa800000000; +/// Shared (uncached) firmware structure VA range top. +const IOVA_KERN_SHARED_TOP: u64 = 0xffffffa9ffffffff; +/// Shared (uncached) read-only firmware structure VA range base. +const IOVA_KERN_SHARED_RO_BASE: u64 = 0xffffffaa00000000; +/// Shared (uncached) read-only firmware structure VA range top. +const IOVA_KERN_SHARED_RO_TOP: u64 = 0xffffffabffffffff; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_GPU_BASE: u64 = 0xffffffaf00000000; +/// GPU/FW shared structure VA range top. +const IOVA_KERN_GPU_TOP: u64 = 0xffffffafffffffff; + +/// Timeout for entering the halt state after a fault or request. +const HALT_ENTER_TIMEOUT_MS: u64 = 100; + +/// Maximum amount of firmware-private memory garbage allowed before collection. +/// Collection flushes the FW cache and is expensive, so this needs to be +/// reasonably high. +const MAX_FW_ALLOC_GARBAGE: usize = 16 * 1024 * 1024; + +/// Global allocators used for kernel-half structures. +pub(crate) struct KernelAllocators { + pub(crate) private: alloc::DefaultAllocator, + pub(crate) shared: alloc::DefaultAllocator, + pub(crate) shared_ro: alloc::DefaultAllocator, + pub(crate) gpu: alloc::DefaultAllocator, +} + +/// Receive (GPU->driver) ring buffer channels. +#[versions(AGX)] +struct RxChannels { + event: channel::EventChannel, + fw_log: channel::FwLogChannel, + ktrace: channel::KTraceChannel, + stats: channel::StatsChannel::ver, +} + +/// GPU work submission pipe channels (driver->GPU). +#[versions(AGX)] +struct PipeChannels { + pub(crate) vtx: Vec>, + pub(crate) frag: Vec>, + pub(crate) comp: Vec>, +} + +/// Misc command transmit (driver->GPU) channels. +#[versions(AGX)] +struct TxChannels { + pub(crate) device_control: channel::DeviceControlChannel::ver, +} + +/// Number of work submission pipes per type, one for each priority level. +const NUM_PIPES: usize = 4; + +/// A generic monotonically incrementing ID used to uniquely identify object instances within the +/// driver. +pub(crate) struct ID(AtomicU64); + +impl ID { + /// Create a new ID counter with a given value. + fn new(val: u64) -> ID { + ID(AtomicU64::new(val)) + } + + /// Fetch the next unique ID. + pub(crate) fn next(&self) -> u64 { + self.0.fetch_add(1, Ordering::Relaxed) + } +} + +impl Default for ID { + /// IDs default to starting at 2, as 0/1 are considered reserved for the system. + fn default() -> Self { + Self::new(2) + } +} + +/// A guard representing one active submission on the GPU. When dropped, decrements the active +/// submission count. +pub(crate) struct OpGuard(Arc); + +impl Drop for OpGuard { + fn drop(&mut self) { + self.0.end_op(); + } +} + +/// Set of global sequence IDs used in the driver. +#[derive(Default)] +pub(crate) struct SequenceIDs { + /// `File` instance ID. + pub(crate) file: ID, + /// `Vm` instance ID. + pub(crate) vm: ID, + /// Submission instance ID. + pub(crate) submission: ID, + /// `Queue` instance ID. + pub(crate) queue: ID, +} + +/// Top-level GPU manager that owns all the global state relevant to the driver instance. +#[versions(AGX)] +pub(crate) struct GpuManager { + dev: AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: Box, + pub(crate) initdata: Box>, + uat: Box, + crashed: AtomicBool, + alloc: Mutex, + io_mappings: Vec, + rtkit: Mutex>>>, + rx_channels: Mutex>, + tx_channels: Mutex>, + fwctl_channel: Mutex>, + pipes: PipeChannels::ver, + event_manager: Arc, + buffer_mgr: buffer::BufferManager, + ids: SequenceIDs, +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + /// Cast Arc as an Any type. + fn arc_as_any(self: Arc) -> Arc; + /// Initialize the GPU. + fn init(&self) -> Result; + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, Mutex>; + /// Create a new `Vm` given a unique `File` ID. + fn new_vm(&self, file_id: u64) -> Result; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; + /// Create a new user command queue. + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + caps: u32, + ) -> Result>; + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Kick the firmware (wake it up if asleep). + /// + /// This should be useful to reduce latency on work submission, so we can ask the firmware to + /// wake up while we do some preparatory work for the work submission. + fn kick_firmware(&self) -> Result; + /// Invalidate a GPU scheduler context. Must be called before the relevant structures are freed. + fn invalidate_context( + &self, + context: &fw::types::GpuObject, + ) -> Result; + /// Flush the entire firmware cache. + /// + /// TODO: Does this actually work? + fn flush_fw_cache(&self) -> Result; + /// Handle a GPU work timeout event. + fn handle_timeout(&self, counter: u32, event_slot: u32); + /// Handle a GPU fault event. + fn handle_fault(&self); + /// Wait for the GPU to become idle and power off. + fn wait_for_poweroff(&self, timeout: usize) -> Result; + /// Send a firmware control command (secure cache flush). + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result; + /// Get the static GPU configuration for this SoC. + fn get_cfg(&self) -> &'static hw::HwConfig; + /// Get the dynamic GPU configuration for this SoC. + fn get_dyncfg(&self) -> &hw::DynConfig; +} + +/// Private generic trait for functions that don't need to escape this module. +trait GpuManagerPriv { + /// Decrement the pending submission counter. + fn end_op(&self); +} + +#[versions(AGX)] +#[vtable] +impl rtkit::Operations for GpuManager::ver { + type Data = Arc; + type Buffer = gem::ObjectRef; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let dev = &data.dev; + //dev_info!(dev, "RtKit message: {:#x}:{:#x}\n", ep, msg); + + if ep != EP_FIRMWARE || msg != MSG_RX_DOORBELL { + dev_err!(dev, "Unknown message: {:#x}:{:#x}\n", ep, msg); + return; + } + + let mut ch = data.rx_channels.lock(); + + ch.fw_log.poll(); + ch.ktrace.poll(); + ch.stats.poll(); + ch.event.poll(); + } + + fn crashed(data: ::Borrowed<'_>) { + let dev = &data.dev; + dev_err!(dev, "GPU firmware crashed, failing all jobs\n"); + + data.crashed.store(true, Ordering::Relaxed); + data.event_manager.fail_all(workqueue::WorkError::NoDevice); + } + + fn shmem_alloc( + data: ::Borrowed<'_>, + size: usize, + ) -> Result { + let dev = &data.dev; + mod_dev_dbg!(dev, "shmem_alloc() {:#x} bytes\n", size); + + let mut obj = gem::new_kernel_object(dev, size)?; + obj.vmap()?; + let iova = obj.map_into(data.uat.kernel_vm())?; + mod_dev_dbg!(dev, "shmem_alloc() -> VA {:#x}\n", iova); + Ok(obj) + } +} + +#[versions(AGX)] +impl GpuManager::ver { + /// Create a new GpuManager of this version/GPU combination. + #[inline(never)] + pub(crate) fn new( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + ) -> Result> { + let uat = Self::make_uat(dev, cfg)?; + let dyncfg = Self::make_dyncfg(dev, res, cfg, &uat)?; + + let mut alloc = KernelAllocators { + private: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_PRIV_BASE, + IOVA_KERN_PRIV_TOP, + 0x80, + mmu::PROT_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel Private"), + true, + )?, + shared: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_BASE, + IOVA_KERN_SHARED_TOP, + 0x80, + mmu::PROT_FW_SHARED_RW, + 1024 * 1024, + true, + fmt!("Kernel Shared"), + false, + )?, + shared_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RO_BASE, + IOVA_KERN_SHARED_RO_TOP, + 0x80, + mmu::PROT_FW_SHARED_RO, + 64 * 1024, + true, + fmt!("Kernel RO Shared"), + false, + )?, + gpu: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_BASE, + IOVA_KERN_GPU_TOP, + 0x80, + mmu::PROT_GPU_FW_SHARED_RW, + 64 * 1024, + true, + fmt!("Kernel GPU Shared"), + false, + )?, + }; + + let event_manager = Self::make_event_manager(&mut alloc)?; + let initdata = Self::make_initdata(cfg, &dyncfg, &mut alloc)?; + let mut mgr = Self::make_mgr(dev, cfg, dyncfg, uat, alloc, event_manager, initdata)?; + + { + let fwctl = mgr.fwctl_channel.lock(); + let p_fwctl = fwctl.to_raw(); + core::mem::drop(fwctl); + + mgr.initdata.fw_status.with_mut(|raw, _inner| { + raw.fwctl_channel = p_fwctl; + }); + } + + { + let txc = mgr.tx_channels.lock(); + let p_device_control = txc.device_control.to_raw(); + core::mem::drop(txc); + + let rxc = mgr.rx_channels.lock(); + let p_event = rxc.event.to_raw(); + let p_fw_log = rxc.fw_log.to_raw(); + let p_ktrace = rxc.ktrace.to_raw(); + let p_stats = rxc.stats.to_raw(); + let p_fwlog_buf = rxc.fw_log.get_buf(); + core::mem::drop(rxc); + + mgr.initdata.runtime_pointers.with_mut(|raw, _inner| { + raw.device_control = p_device_control; + raw.event = p_event; + raw.fw_log = p_fw_log; + raw.ktrace = p_ktrace; + raw.stats = p_stats; + raw.fwlog_buf = Some(p_fwlog_buf); + }); + } + + let mut p_pipes: Vec = Vec::new(); + + for ((v, f), c) in mgr + .pipes + .vtx + .iter() + .zip(&mgr.pipes.frag) + .zip(&mgr.pipes.comp) + { + p_pipes.try_push(fw::initdata::raw::PipeChannels::ver { + vtx: v.lock().to_raw(), + frag: f.lock().to_raw(), + comp: c.lock().to_raw(), + })?; + } + + mgr.initdata.runtime_pointers.with_mut(|raw, _inner| { + for (i, p) in p_pipes.into_iter().enumerate() { + raw.pipes[i].vtx = p.vtx; + raw.pipes[i].frag = p.frag; + raw.pipes[i].comp = p.comp; + } + }); + + for (i, map) in cfg.io_mappings.iter().enumerate() { + if let Some(map) = map.as_ref() { + mgr.iomap(i, map)?; + } + } + + let mgr = Arc::from(mgr); + + let rtkit = Box::try_new(rtkit::RtKit::::new( + dev, + None, + 0, + mgr.clone(), + )?)?; + + *mgr.rtkit.lock() = Some(rtkit); + + { + let mut rxc = mgr.rx_channels.lock(); + rxc.event.set_manager(mgr.clone()); + } + + Ok(mgr) + } + + /// Build the entire GPU InitData structure tree and return it as a boxed GpuObject. + fn make_initdata( + cfg: &'static hw::HwConfig, + dyncfg: &hw::DynConfig, + alloc: &mut KernelAllocators, + ) -> Result>> { + let mut builder = initdata::InitDataBuilder::ver::new(alloc, cfg, dyncfg); + builder.build() + } + + /// Create a fresh boxed Uat instance. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_uat(dev: &AsahiDevice, cfg: &'static hw::HwConfig) -> Result> { + Ok(Box::try_new(mmu::Uat::new(dev, cfg)?)?) + } + + /// Actually create the final GpuManager instance, as a UniqueArc. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_mgr( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: Box, + uat: Box, + mut alloc: KernelAllocators, + event_manager: Arc, + initdata: Box>, + ) -> Result> { + let mut pipes = PipeChannels::ver { + vtx: Vec::new(), + frag: Vec::new(), + comp: Vec::new(), + }; + + for _i in 0..=NUM_PIPES - 1 { + pipes + .vtx + .try_push(Mutex::new(channel::PipeChannel::ver::new(dev, &mut alloc)?))?; + pipes + .frag + .try_push(Mutex::new(channel::PipeChannel::ver::new(dev, &mut alloc)?))?; + pipes + .comp + .try_push(Mutex::new(channel::PipeChannel::ver::new(dev, &mut alloc)?))?; + } + + UniqueArc::try_new(GpuManager::ver { + dev: dev.clone(), + cfg, + dyncfg, + initdata, + uat, + io_mappings: Vec::new(), + rtkit: Mutex::new(None), + crashed: AtomicBool::new(false), + rx_channels: Mutex::new(box_in_place!(RxChannels::ver { + event: channel::EventChannel::new(dev, &mut alloc, event_manager.clone())?, + fw_log: channel::FwLogChannel::new(dev, &mut alloc)?, + ktrace: channel::KTraceChannel::new(dev, &mut alloc)?, + stats: channel::StatsChannel::ver::new(dev, &mut alloc)?, + })?), + tx_channels: Mutex::new(Box::try_new(TxChannels::ver { + device_control: channel::DeviceControlChannel::ver::new(dev, &mut alloc)?, + })?), + fwctl_channel: Mutex::new(Box::try_new(channel::FwCtlChannel::new(dev, &mut alloc)?)?), + pipes, + event_manager, + buffer_mgr: buffer::BufferManager::new()?, + alloc: Mutex::new(alloc), + ids: Default::default(), + }) + } + + /// Fetch and validate the GPU dynamic configuration from the device tree and hardware. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_dyncfg( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + uat: &mmu::Uat, + ) -> Result> { + let gpu_id = res.get_gpu_id()?; + + dev_info!(dev, "GPU Information:\n"); + dev_info!( + dev, + " Type: {:?}{:?}\n", + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + dev_info!(dev, " Max dies: {}\n", gpu_id.max_dies); + dev_info!(dev, " Clusters: {}\n", gpu_id.num_clusters); + dev_info!( + dev, + " Cores: {} ({})\n", + gpu_id.num_cores, + gpu_id.num_cores * gpu_id.num_clusters + ); + dev_info!( + dev, + " Frags: {} ({})\n", + gpu_id.num_frags, + gpu_id.num_frags * gpu_id.num_clusters + ); + dev_info!( + dev, + " GPs: {} ({})\n", + gpu_id.num_gps, + gpu_id.num_gps * gpu_id.num_clusters + ); + dev_info!(dev, " Core masks: {:#x?}\n", gpu_id.core_masks); + dev_info!(dev, " Active cores: {}\n", gpu_id.total_active_cores); + + dev_info!(dev, "Getting configuration from device tree...\n"); + let pwr_cfg = hw::PwrConfig::load(dev, cfg)?; + dev_info!(dev, "Dynamic configuration fetched\n"); + + if gpu_id.gpu_gen != cfg.gpu_gen || gpu_id.gpu_variant != cfg.gpu_variant { + dev_err!( + dev, + "GPU type mismatch (expected {:?}{:?}, found {:?}{:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + return Err(EIO); + } + if gpu_id.num_clusters > cfg.max_num_clusters { + dev_err!( + dev, + "Too many clusters ({} > {})\n", + gpu_id.num_clusters, + cfg.max_num_clusters + ); + return Err(EIO); + } + if gpu_id.num_cores > cfg.max_num_cores { + dev_err!( + dev, + "Too many cores ({} > {})\n", + gpu_id.num_cores, + cfg.max_num_cores + ); + return Err(EIO); + } + if gpu_id.num_frags > cfg.max_num_frags { + dev_err!( + dev, + "Too many frags ({} > {})\n", + gpu_id.num_frags, + cfg.max_num_frags + ); + return Err(EIO); + } + if gpu_id.num_gps > cfg.max_num_gps { + dev_err!( + dev, + "Too many GPs ({} > {})\n", + gpu_id.num_gps, + cfg.max_num_gps + ); + return Err(EIO); + } + + Ok(Box::try_new(hw::DynConfig { + pwr: pwr_cfg, + uat_ttb_base: uat.ttb_base(), + id: gpu_id, + })?) + } + + /// Create the global GPU event manager, and return an `Arc<>` to it. + fn make_event_manager(alloc: &mut KernelAllocators) -> Result> { + Arc::try_new(event::EventManager::new(alloc)?) + } + + /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified + /// index. + fn iomap(&mut self, index: usize, map: &hw::IOMapping) -> Result { + let off = map.base & mmu::UAT_PGMSK; + let base = map.base - off; + let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; + let mapping = self + .uat + .kernel_vm() + .map_io(base, end - base, map.writable)?; + + self.initdata.runtime_pointers.hwdata_b.with_mut(|raw, _| { + raw.io_mappings[index] = fw::initdata::raw::IOMapping { + phys_addr: U64(map.base as u64), + virt_addr: U64((mapping.iova() + off) as u64), + size: map.size as u32, + range_size: map.range_size as u32, + readwrite: U64(map.writable as u64), + }; + }); + + self.io_mappings.try_push(mapping)?; + Ok(()) + } + + /// Mark work associated with currently in-progress event slots as failed, after a fault or + /// timeout. + fn mark_pending_events(&self, culprit_slot: Option, error: workqueue::WorkError) { + dev_err!(self.dev, " Pending events:\n"); + + self.initdata.globals.with(|raw, _inner| { + for i in raw.pending_stamps.iter() { + let info = i.info.load(Ordering::Relaxed); + let wait_value = i.wait_value.load(Ordering::Relaxed); + + if info & 1 != 0 { + let slot = info >> 3; + let flags = info & 0x7; + dev_err!( + self.dev, + " [{}] flags={} value={:#x}\n", + slot, + flags, + wait_value + ); + let error = if culprit_slot.is_some() && culprit_slot != Some(slot) { + workqueue::WorkError::Killed + } else { + error + }; + self.event_manager.mark_error(slot, wait_value, error); + i.info.store(0, Ordering::Relaxed); + i.wait_value.store(0, Ordering::Relaxed); + } + } + }); + } + + /// Fetch the GPU MMU fault information from the hardware registers. + fn get_fault_info(&self) -> Option { + let data = self.dev.data(); + + let res = match data.resources() { + Some(res) => res, + None => { + dev_err!(self.dev, " Failed to acquire resources\n"); + return None; + } + }; + + let info = res.get_fault_info(); + if info.is_some() { + dev_err!(self.dev, " Fault info: {:#x?}\n", info.as_ref().unwrap()); + } + info + } + + /// Resume the GPU firmware after it halts (due to a timeout, fault, or request). + fn recover(&self) { + self.initdata.fw_status.with(|raw, _inner| { + let halt_count = raw.flags.halt_count.load(Ordering::Relaxed); + let mut halted = raw.flags.halted.load(Ordering::Relaxed); + dev_err!(self.dev, " Halt count: {}\n", halt_count); + dev_err!(self.dev, " Halted: {}\n", halted); + + if halted == 0 { + let timeout = time::ktime_get() + Duration::from_millis(HALT_ENTER_TIMEOUT_MS); + while time::ktime_get() < timeout { + halted = raw.flags.halted.load(Ordering::Relaxed); + if halted != 0 { + break; + } + mem::sync(); + } + halted = raw.flags.halted.load(Ordering::Relaxed); + } + + if debug_enabled(DebugFlags::NoGpuRecovery) { + dev_crit!(self.dev, " GPU recovery is disabled, wedging forever!\n"); + } else if halted != 0 { + dev_err!(self.dev, " Attempting recovery...\n"); + raw.flags.halted.store(0, Ordering::SeqCst); + raw.flags.resume.store(1, Ordering::SeqCst); + } else { + dev_err!(self.dev, " Cannot recover.\n"); + } + }); + } + + /// Return the packed GPU enabled core masks. + // Only used for some versions + #[allow(dead_code)] + pub(crate) fn core_masks_packed(&self) -> &[u32] { + self.dyncfg.id.core_masks_packed.as_slice() + } + + /// Kick a submission pipe for a submitted job to tell the firmware to start processing it. + pub(crate) fn run_job(&self, job: workqueue::JobSubmission::ver<'_>) -> Result { + mod_dev_dbg!(self.dev, "GPU: run_job\n"); + + let pipe_type = job.pipe_type(); + mod_dev_dbg!(self.dev, "GPU: run_job: pipe_type={:?}\n", pipe_type); + + let pipes = match pipe_type { + PipeType::Vertex => &self.pipes.vtx, + PipeType::Fragment => &self.pipes.frag, + PipeType::Compute => &self.pipes.comp, + }; + + let index: usize = job.priority() as usize; + let mut pipe = pipes.get(index).ok_or(EIO)?.lock(); + + mod_dev_dbg!(self.dev, "GPU: run_job: run()\n"); + job.run(&mut pipe); + mod_dev_dbg!(self.dev, "GPU: run_job: ring doorbell\n"); + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + rtk.send_message( + EP_DOORBELL, + MSG_TX_DOORBELL | pipe_type as u64 | ((index as u64) << 2), + )?; + mod_dev_dbg!(self.dev, "GPU: run_job: done\n"); + + Ok(()) + } + + pub(crate) fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + pub(crate) fn start_op(self: &Arc) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); + + mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + self.kick_firmware()?; + Ok(OpGuard(self.clone())) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn init(&self) -> Result { + self.tx_channels.lock().device_control.send( + &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), + ); + + let initdata = self.initdata.gpu_va().get(); + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + + rtk.boot()?; + rtk.start_endpoint(EP_FIRMWARE)?; + rtk.start_endpoint(EP_DOORBELL)?; + rtk.send_message(EP_FIRMWARE, MSG_INIT | (initdata & INIT_DATA_MASK))?; + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + core::mem::drop(guard); + + self.kick_firmware()?; + Ok(()) + } + + fn update_globals(&self) { + let mut timeout: u32 = 2; + if debug_enabled(DebugFlags::WaitForPowerOff) { + timeout = 0; + } else if debug_enabled(DebugFlags::KeepGpuPowered) { + timeout = 5000; + } + + self.initdata.globals.with(|raw, _inner| { + raw.idle_off_delay_ms.store(timeout, Ordering::Relaxed); + }); + } + + fn alloc(&self) -> Guard<'_, Mutex> { + let mut guard = self.alloc.lock(); + let (garbage_count, garbage_bytes) = guard.private.garbage(); + if garbage_bytes > MAX_FW_ALLOC_GARBAGE { + mod_dev_dbg!( + self.dev, + "Collecting kalloc garbage ({} objects, {} bytes)\n", + garbage_count, + garbage_bytes + ); + if self.flush_fw_cache().is_err() { + dev_err!(self.dev, "Failed to flush FW cache\n"); + } else { + guard.private.collect_garbage(garbage_count); + } + } + + guard + } + + fn new_vm(&self, file_id: u64) -> Result { + self.uat.new_vm(self.ids.vm.next(), file_id) + } + + fn bind_vm(&self, vm: &mmu::Vm) -> Result { + self.uat.bind(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + caps: u32, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(Box::try_new(queue::Queue::ver::new( + &self.dev, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + caps, + )?)?) + } + + fn kick_firmware(&self) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_KICKFW)?; + + Ok(()) + } + + fn invalidate_context( + &self, + context: &fw::types::GpuObject, + ) -> Result { + mod_dev_dbg!( + self.dev, + "Invalidating GPU context @ {:?}\n", + context.weak_pointer() + ); + + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.alloc.lock(); + let (garbage_count, _) = guard.private.garbage(); + + let dc = context.with( + |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + ctx_23: raw.unk_23, + __pad0: Default::default(), + unk_c: 0, + unk_10: 0, + ctx_0: raw.unk_0, + ctx_1: raw.unk_1, + ctx_4: raw.unk_4, + __pad1: Default::default(), + unk_18: 0, + gpu_context: Some(context.weak_pointer()), + __pad2: Default::default(), + }, + ); + + mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + + mod_dev_dbg!( + self.dev, + "GPU context invalidated: {:?}\n", + context.weak_pointer() + ); + + // The invalidation does a cache flush, so it is okay to collect garbage + guard.private.collect_garbage(garbage_count); + + Ok(()) + } + + fn flush_fw_cache(&self) -> Result { + mod_dev_dbg!(self.dev, "Flushing coprocessor data cache\n"); + + if self.is_crashed() { + return Err(ENODEV); + } + + // ctx_0 == 0xff or ctx_1 == 0xff cause no effect on context, + // but this command does a full cache flush too, so abuse it + // for that. + + let dc = fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + ctx_23: 0, + __pad0: Default::default(), + unk_c: 0, + unk_10: 0, + ctx_0: 0xff, + ctx_1: 0xff, + ctx_4: 0, + __pad1: Default::default(), + unk_18: 0, + gpu_context: None, + __pad2: Default::default(), + }; + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + Ok(()) + } + + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn handle_timeout(&self, counter: u32, event_slot: u32) { + dev_err!(self.dev, " (\\________/) \n"); + dev_err!(self.dev, " | | \n"); + dev_err!(self.dev, "'.| \\ , / |.'\n"); + dev_err!(self.dev, "--| / (( \\ |--\n"); + dev_err!(self.dev, ".'| _-_- |'.\n"); + dev_err!(self.dev, " |________| \n"); + dev_err!(self.dev, "** GPU timeout nya~!!!!! **\n"); + dev_err!(self.dev, " Event slot: {}\n", event_slot); + dev_err!(self.dev, " Timeout count: {}\n", counter); + + // If we have fault info, consider it a fault. + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Timeout, + }; + self.mark_pending_events(Some(event_slot), error); + self.recover(); + } + + fn handle_fault(&self) { + dev_err!(self.dev, " (\\________/) \n"); + dev_err!(self.dev, " | | \n"); + dev_err!(self.dev, "'.| \\ , / |.'\n"); + dev_err!(self.dev, "--| / (( \\ |--\n"); + dev_err!(self.dev, ".'| _-_- |'.\n"); + dev_err!(self.dev, " |________| \n"); + dev_err!(self.dev, "GPU fault nya~!!!!!\n"); + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Unknown, + }; + self.mark_pending_events(None, error); + self.recover(); + } + + fn wait_for_poweroff(&self, timeout: usize) -> Result { + self.initdata.runtime_pointers.hwdata_a.with(|raw, _inner| { + for _i in 0..timeout { + if raw.pwr_status.load(Ordering::Relaxed) == 4 { + return Ok(()); + } + coarse_sleep(Duration::from_millis(1)); + } + Err(ETIMEDOUT) + }) + } + + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut fwctl = self.fwctl_channel.lock(); + let token = fwctl.send(&msg); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_FWCTL)?; + } + fwctl.wait_for(token)?; + Ok(()) + } + + fn get_cfg(&self) -> &'static hw::HwConfig { + self.cfg + } + + fn get_dyncfg(&self) -> &hw::DynConfig { + &self.dyncfg + } +} + +#[versions(AGX)] +impl GpuManagerPriv for GpuManager::ver { + fn end_op(&self) { + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_sub(1, Ordering::Release)); + + mod_dev_dbg!(self.dev, "OP end (pending: {})\n", val - 1); + } +} diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs new file mode 100644 index 00000000000000..a92bb70aeae808 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use alloc::vec::Vec; +use kernel::c_str; +use kernel::device::RawDevice; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod t600x; +pub(crate) mod t8103; +pub(crate) mod t8112; + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// GPU driver/hardware features, from the UABI. +pub(crate) mod feat { + /// Backwards-compatible features. + pub(crate) mod compat {} + + /// Backwards-incompatible features. + pub(crate) mod incompat { + use kernel::bindings; + + /// Hardware requires Z/S compression to be mandatorily enabled. + pub(crate) const MANDATORY_ZS_COMPRESSION: u64 = + bindings::drm_asahi_feat_incompat_DRM_ASAHI_FEAT_MANDATORY_ZS_COMPRESSION as u64; + } +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: Vec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Size of the mapping. + pub(crate) size: usize, + /// Range size of the mapping (for arrays?) + pub(crate) range_size: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + size: usize, + range_size: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + size, + range_size, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + pub(crate) gpu_core: GpuCore, + /// Compatible feature bitmask for this GPU. + pub(crate) gpu_feat_compat: u64, + /// Incompatible feature bitmask for this GPU. + pub(crate) gpu_feat_incompat: u64, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: usize, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_die0_sensor_mask: u64, + /// Temperature sensor list (alternate). + pub(crate) fast_die0_sensor_mask_alt: u64, + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Maximum number of dies supported. + pub(crate) max_dies: u32, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: Vec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: Vec, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: Vec, + /// GPU power zone list. + pub(crate) power_zones: Vec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: Vec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: Vec, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, +} + +impl PwrConfig { + /// Load the GPU power configuration from the device tree. + pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { + let mut perf_states = Vec::new(); + + let node = dev.of_node().ok_or(EIO)?; + let opps = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + + let mut max_power_mw: u32 = 0; + let mut max_freq_mhz: u32 = 0; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: Vec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = opp.get_property(c_str!("opp-microwatt"))?; + + if volt_uv.len() != cfg.max_num_clusters as usize { + dev_err!( + dev, + "Invalid opp-microvolt length (expected {}, got {})\n", + cfg.max_num_clusters, + volt_uv.len() + ); + return Err(EINVAL); + } + + volt_uv.iter_mut().for_each(|a| *a /= 1000); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + max_power_mw = max_power_mw.max(pwr_mw); + + let freq_mhz: u32 = (freq_hz / 1_000_000).try_into()?; + max_freq_mhz = max_freq_mhz.max(freq_mhz); + + perf_states.try_push(PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + })?; + } + + let pz_data = prop!("apple,power-zones", Vec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev, "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = Vec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.try_push(PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + })?; + } + + let core_leak_coef: Vec = prop!("apple,core-leak-coef"); + let sram_leak_coef: Vec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw, + max_freq_mhz, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: prop!("apple,min-sram-microvolt"), + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period: prop!("apple,power-sample-period"), + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + + perf_states, + power_zones, + }) + } + + pub(crate) fn min_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_base_pstate as usize].freq_hz / 1000 + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs new file mode 100644 index 00000000000000..2654f51cf40ad7 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(mcc_count: usize, has_die1: bool) -> [Option; 20] { + [ + Some(IOMapping::new(0x404d00000, 0x1c000, 0x1c000, true)), // Fender + Some(IOMapping::new(0x20e100000, 0x4000, 0x4000, false)), // AICTimer + Some(IOMapping::new(0x28e104000, 0x4000, 0x4000, true)), // AICSWInt + Some(IOMapping::new(0x404000000, 0x20000, 0x20000, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x28e494000, 0x1000, 0x1000, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404d80000, 0x8000, 0x8000, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, 0x1000, 0x1000, true)), // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + mcc_count * 0xd8000, + 0xd8000, + true, + )), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + Some(IOMapping::new(0x2643c4000, 0x1000, 0x1000, true)), // NIA Special agent idle register die 0 + if has_die1 { + // NIA Special agent idle register die 1 + Some(IOMapping::new(0x22643c4000, 0x1000, 0x1000, true)) + } else { + None + }, + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, 0x1000, 0x1000, true)), // ? + Some(IOMapping::new(0x28e3c0000, 0x1000, 0x1000, false)), // ? + ] +} + +pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { + chip_id: 0x6002, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G13C, + gpu_feat_compat: 0, + gpu_feat_incompat: feat::incompat::MANDATORY_ZS_COMPRESSION, + + base_clock_hz: 24_000_000, + uat_oas: 42, + max_num_clusters: 8, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + + render: HwRenderConfig { + tiling_control: 0xa540, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_4e0: 4, + unk_534: 1, + unk_ab8: 0x2084, + unk_abc: 0x80, + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0xffff, + shared2_tab: &[-1, -1, -1, -1, 0x2aa, 0xaaa, -1, -1, 0, 0], + shared2_unk_508: 0xcc00001, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([9.838]), + &f32!([9.819]), + &f32!([9.826]), + &f32!([9.799]), + &f32!([9.799]), + &f32!([9.826]), + &f32!([9.819]), + &f32!([9.838]), + ], + unk_coef_b: &[ + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + ], + global_tab: Some(&[ + 0, 1, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + fast_die0_sensor_mask: 0x8080808080808080, + fast_die0_sensor_mask_alt: 0x9090909090909090, + fast_die0_sensor_present: 0xff, + io_mappings: &iomaps(16, true), +}; + +pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { + chip_id: 0x6001, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G13C, + + max_num_clusters: 4, + fast_die0_sensor_mask: 0x80808080, + fast_die0_sensor_mask_alt: 0x90909090, + fast_die0_sensor_present: 0x0f, + io_mappings: &iomaps(8, false), + ..HWCONFIG_T6002 +}; + +pub(crate) const HWCONFIG_T6000: super::HwConfig = HwConfig { + chip_id: 0x6000, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G13S, + + max_num_clusters: 2, + fast_die0_sensor_mask: 0x8080, + fast_die0_sensor_mask_alt: 0x9090, + fast_die0_sensor_present: 0x03, + io_mappings: &iomaps(4, false), + ..HWCONFIG_T6001 +}; diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs new file mode 100644 index 00000000000000..3d38b088a0f576 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8103 platforms (M1). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8103, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G13G, + gpu_feat_compat: 0, + gpu_feat_incompat: 0, + + base_clock_hz: 24_000_000, + uat_oas: 40, + max_num_clusters: 1, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + + render: HwRenderConfig { + // bit 0: disable clustering (always) + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: -220, + unk_8cc: 9880, + unk_e24: 112, + }, + db: HwConfigB { + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0x48, + unk_abc: 0x8, + unk_b30: 0, + }, + shared1_tab: &[ + -1, 0x7282, 0x50ea, 0x370a, 0x25be, 0x1c1f, 0x16fb, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ], + shared1_a4: 0xffff, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0, 0], + shared2_unk_508: 0xc00007, + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + global_tab: None, + fast_die0_sensor_mask: 0x12, + fast_die0_sensor_mask_alt: 0x12, + fast_die0_sensor_present: 0x01, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, 0x1c000, 0x1c000, true)), // Fender + Some(IOMapping::new(0x20e100000, 0x4000, 0x4000, false)), // AICTimer + Some(IOMapping::new(0x23b104000, 0x4000, 0x4000, true)), // AICSWInt + Some(IOMapping::new(0x204000000, 0x20000, 0x20000, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2e8000, 0x1000, 0x1000, false)), // AnalogTempSensorControllerRegs + Some(IOMapping::new(0x23bc00000, 0x1000, 0x1000, true)), // PMPDoorbell + Some(IOMapping::new(0x204d80000, 0x5000, 0x5000, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, 0x1000, 0x1000, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, 0xd6400, 0xd6400, true)), // MCache registers + None, // AICBankedRegisters + Some(IOMapping::new(0x23b738000, 0x1000, 0x1000, true)), // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + None, // + None, // + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs new file mode 100644 index 00000000000000..5624dca130be45 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8112 platforms (M2). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8112, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G14G, + gpu_feat_compat: 0, + gpu_feat_incompat: 0, + + base_clock_hz: 24_000_000, + uat_oas: 40, + max_num_clusters: 1, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + + render: HwRenderConfig { + // TODO: this is unused here, may be present in newer FW + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0x2048, + unk_abc: 0x4000, + unk_b30: 1, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[-1, -1, -1, -1, -1, -1, -1, -1, 0xaa5aa, 0], + shared2_unk_508: 0xc00000, + sram_k: f32!(1.02), + // 13.2: last coef changed from 6.6 to 5.3, assuming that was a fix we can backport + unk_coef_a: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + unk_coef_b: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + global_tab: None, + fast_die0_sensor_mask: 0x6800, + fast_die0_sensor_mask_alt: 0x6800, + fast_die0_sensor_present: 0x02, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, 0x14000, 0x14000, true)), // Fender + Some(IOMapping::new(0x20e100000, 0x4000, 0x4000, false)), // AICTimer + Some(IOMapping::new(0x23b0c4000, 0x4000, 0x4000, true)), // AICSWInt + Some(IOMapping::new(0x204000000, 0x20000, 0x20000, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2c0000, 0x1000, 0x1000, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x204d80000, 0x8000, 0x8000, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, 0x1000, 0x1000, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, 0xd6400, 0xd6400, true)), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + Some(IOMapping::new(0x204e00000, 0x10000, 0x10000, true)), // CRE registers + Some(IOMapping::new(0x27d050000, 0x4000, 0x4000, true)), // Streaming codec registers + Some(IOMapping::new(0x23b3d0000, 0x1000, 0x1000, true)), // + Some(IOMapping::new(0x23b3c0000, 0x1000, 0x1000, true)), // + ], +}; diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs new file mode 100644 index 00000000000000..472c42169130b3 --- /dev/null +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -0,0 +1,777 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::{box_in_place, f32, place}; +use crate::{gpu, hw, mmu}; +use kernel::error::Result; +use kernel::macros::versions; + +/// Builder helper for the global GPU InitData. +#[versions(AGX)] +pub(crate) struct InitDataBuilder<'a> { + alloc: &'a mut gpu::KernelAllocators, + cfg: &'a hw::HwConfig, + dyncfg: &'a hw::DynConfig, +} + +#[versions(AGX)] +impl<'a> InitDataBuilder::ver<'a> { + /// Create a new InitData builder + pub(crate) fn new( + alloc: &'a mut gpu::KernelAllocators, + cfg: &'a hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilder::ver<'a> { + InitDataBuilder::ver { alloc, cfg, dyncfg } + } + + /// Create the HwDataShared1 structure, which is used in two places in InitData. + #[inline(never)] + fn hw_shared1(cfg: &hw::HwConfig) -> raw::HwDataShared1 { + let mut ret = raw::HwDataShared1 { + unk_a4: cfg.shared1_a4, + ..Default::default() + }; + for (i, val) in cfg.shared1_tab.iter().enumerate() { + ret.table[i] = *val; + } + ret + } + + fn init_curve( + curve: &mut raw::HwDataShared2Curve, + unk_0: u32, + unk_4: u32, + t1: &[i16], + t2: &[i16], + t3: &[&[i32]], + ) { + curve.unk_0 = unk_0; + curve.unk_4 = unk_4; + (*curve.t1)[..t1.len()].copy_from_slice(t1); + (*curve.t1)[t1.len()..].fill(t1[0]); + (*curve.t2)[..t2.len()].copy_from_slice(t2); + (*curve.t2)[t2.len()..].fill(t2[0]); + for (i, a) in curve.t3.iter_mut().enumerate() { + a.fill(0x3ffffff); + if i < t3.len() { + let b = t3[i]; + (**a)[..b.len()].copy_from_slice(b); + } + } + } + + /// Create the HwDataShared2 structure, which is used in two places in InitData. + #[inline(never)] + fn hw_shared2(cfg: &hw::HwConfig) -> Result> { + let mut ret = box_in_place!(raw::HwDataShared2 { + unk_28: Array::new([0xff; 16]), + t8112: Default::default(), + unk_508: cfg.shared2_unk_508, + ..Default::default() + })?; + + for (i, val) in cfg.shared2_tab.iter().enumerate() { + ret.table[i] = *val; + } + + if cfg.chip_id == 0x8112 { + ret.t8112.unk_14 = 0x6000000; + Self::init_curve(&mut ret.t8112.curve1, 0, 0x20000000, &[-1], &[0x0f07], &[]); + Self::init_curve( + &mut ret.t8112.curve2, + 7, + 0x80000000, + &[-1, 25740, 17429, 12550, 9597, 7910, 6657, 5881, 5421], + &[ + 0x0f07, 0x04c0, 0x06c0, 0x08c0, 0x0ac0, 0x0c40, 0x0dc0, 0x0ec0, 0x0f80, + ], + &[ + &[0x3ffffff, 107, 101, 94, 87, 82, 77, 73, 71], + &[ + 0x3ffffff, 38240, 36251, 33562, 31368, 29379, 27693, 26211, 25370, + ], + &[ + 0x3ffffff, 123933, 117485, 108771, 101661, 95217, 89751, 84948, 82222, + ], + ], + ); + } + + Ok(ret) + } + + /// Create the HwDataShared3 structure, which is used in two places in InitData. + #[inline(never)] + fn hw_shared3(cfg: &hw::HwConfig) -> Result> { + let mut ret = box_in_place!(raw::HwDataShared3 { + ..Default::default() + })?; + + if cfg.chip_id == 0x8112 { + ret.unk_0 = 1; + ret.unk_4 = 500; + ret.unk_8 = 5; + ret.table.copy_from_slice(&[ + 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, + 10700, 10700, 10700, 10700, + ]); + ret.unk_4c = 1; + } + + Ok(ret) + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data(dyncfg: &'a hw::DynConfig) -> raw::T81xxData { + raw::T81xxData { + unk_d8c: 0x80000000, + unk_d90: 4, + unk_d9c: f32!(0.6), + unk_da4: f32!(0.4), + unk_dac: f32!(0.38552), + unk_db8: f32!(65536.0), + unk_dbc: f32!(13.56), + max_pstate_scaled: 100 * dyncfg.pwr.perf_max_pstate, + ..Default::default() + } + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + #[inline(never)] + fn hwdata_a(&mut self) -> Result> { + self.alloc + .private + .new_inplace(Default::default(), |_inner, ptr| { + let pwr = &self.dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + #[ver(V >= V13_0B4)] + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + #[ver(V >= V13_0B4)] + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = max_ps - base_ps; + + let base_clock_khz = self.cfg.base_clock_hz / 1000; + let clocks_per_period = base_clock_khz * period_ms; + + let raw = place!( + ptr, + raw::HwDataA::ver { + clocks_per_period: clocks_per_period, + #[ver(V >= V13_0B4)] + clocks_per_period_2: clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 + - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + #[ver(V >= V13_0B4)] + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(self.cfg.fast_die0_sensor_mask), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: self.cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: self.cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms + * pwr.avg_power_target_filter_tc, + #[ver(V >= V13_0B4)] + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + #[ver(V >= V13_0B4)] + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data: match self.cfg.chip_id { + 0x8103 | 0x8112 => Self::t81xx_data(self.dyncfg), + _ => Default::default(), + }, + #[ver(V >= V13_0B4)] + unk_e10_0: raw::HwDataA130Extra { + unk_38: 4, + unk_3c: 8000, + unk_40: 2500, + unk_48: 0xffffffff, + unk_4c: 50, + unk_54: 50, + unk_58: 0x1, + unk_60: f32!(0.8888889), + unk_64: f32!(0.6666667), + unk_68: f32!(0.11111111), + unk_6c: f32!(0.33333333), + unk_70: f32!(-0.4), + unk_74: f32!(-0.8), + unk_7c: f32!(65536.0), + unk_80: f32!(-5.0), + unk_84: f32!(-10.0), + unk_8c: 40, + max_pstate_scaled_1: max_ps_scaled, + unk_9c: f32!(8000.0), + unk_a0: 1400, + unk_a8: 72, + unk_ac: 24, + unk_b0: 1728000, + unk_b8: 576000, + unk_c4: f32!(65536.0), + unk_114: f32!(65536.0), + unk_124: 40, + max_pstate_scaled_2: max_ps_scaled, + ..Default::default() + }, + fast_die0_sensor_mask_2: U64(self.cfg.fast_die0_sensor_mask), + unk_e24: self.cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(self.cfg.fast_die0_sensor_mask_alt), + #[ver(V < V13_0B4)] + fast_die0_sensor_present: U64(self.cfg.fast_die0_sensor_present as u64), + unk_163c: 1, + unk_3644: 0, + hws1: Self::hw_shared1(self.cfg), + hws2: *Self::hw_shared2(self.cfg)?, + hws3: *Self::hw_shared3(self.cfg)?, + unk_3ce8: 1, + ..Default::default() + } + ); + + for i in 0..self.dyncfg.pwr.perf_states.len() { + raw.sram_k[i] = self.cfg.sram_k; + } + + for (i, coef) in pwr.core_leak_coef.iter().enumerate() { + raw.core_leak_coef[i] = *coef; + } + + for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { + raw.sram_leak_coef[i] = *coef; + } + + for i in 0..self.dyncfg.id.num_clusters as usize { + if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + #[ver(V >= V13_0B4)] + raw.power_zones[i].unk_10 = 1320000000; + } + + Ok(raw) + }) + } + + /// Create the HwDataB structure. This mostly contains GPU-related configuration. + #[inline(never)] + fn hwdata_b(&mut self) -> Result> { + self.alloc + .private + .new_inplace(Default::default(), |_inner, ptr| { + let raw = place!( + ptr, + raw::HwDataB::ver { + // Userspace VA map related + #[ver(V < V13_0B4)] + unk_0: U64(0x13_00000000), + unk_8: U64(0x14_00000000), + #[ver(V < V13_0B4)] + unk_10: U64(0x1_00000000), + unk_18: U64(0xffc00000), + unk_20: U64(0x11_00000000), + unk_28: U64(0x11_00000000), + // userspace address? + unk_30: U64(0x6f_ffff8000), + // unmapped? + unkptr_38: U64(0xffffffa0_11800000), + // TODO: yuv matrices + chip_id: self.cfg.chip_id, + unk_454: 0x1, + unk_458: 0x1, + unk_460: 0x1, + unk_464: 0x1, + unk_468: 0x1, + unk_47c: 0x1, + unk_484: 0x1, + unk_48c: 0x1, + base_clock_khz: self.cfg.base_clock_hz / 1000, + power_sample_period: self.dyncfg.pwr.power_sample_period, + unk_49c: 0x1, + unk_4a0: 0x1, + unk_4a4: 0x1, + unk_4c0: 0x1f, + unk_4e0: U64(self.cfg.db.unk_4e0), + unk_4f0: 0x1, + unk_4f4: 0x1, + unk_504: 0x31, + unk_524: 0x1, // use_secure_cache_flush + unk_534: self.cfg.db.unk_534, + num_frags: self.dyncfg.id.num_frags * self.dyncfg.id.num_clusters, + unk_554: 0x1, + uat_ttb_base: U64(self.dyncfg.uat_ttb_base), + gpu_core_id: self.cfg.gpu_core as u32, + gpu_rev_id: self.dyncfg.id.gpu_rev_id as u32, + num_cores: self.dyncfg.id.num_cores * self.dyncfg.id.num_clusters, + max_pstate: self.dyncfg.pwr.perf_states.len() as u32 - 1, + #[ver(V < V13_0B4)] + num_pstates: self.dyncfg.pwr.perf_states.len() as u32, + #[ver(V < V13_0B4)] + min_sram_volt: self.dyncfg.pwr.min_sram_microvolt / 1000, + #[ver(V < V13_0B4)] + unk_ab8: self.cfg.db.unk_ab8, + #[ver(V < V13_0B4)] + unk_abc: self.cfg.db.unk_abc, + #[ver(V < V13_0B4)] + unk_ac0: 0x1020, + + #[ver(V >= V13_0B4)] + unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), + #[ver(V < V13_0B4)] + unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), + unk_b10: 0x1, + unk_b24: 0x1, + unk_b28: 0x1, + unk_b2c: 0x1, + unk_b30: self.cfg.db.unk_b30, + #[ver(V >= V13_0B4)] + unk_b38_0: 1, + #[ver(V >= V13_0B4)] + unk_b38_4: 1, + unk_b38: Array::new([0xffffffff; 12]), + #[ver(V >= V13_0B4)] + unk_c3c: 0x19, + ..Default::default() + } + ); + + let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; + let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; + let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; + let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; + + for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { + raw.frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); + raw.voltages[i][j] = *mv; + raw.voltages_sram[i][j] = sram_mv; + } + raw.sram_k[i] = self.cfg.sram_k; + raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; + raw.rel_boost_freqs[i] = if i > base_ps { + (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) + } else { + 0 + }; + } + + Ok(raw) + }) + } + + /// Create the Globals structure, which contains global firmware config including more power + /// configuration data and globals used to exchange state between the firmware and driver. + #[inline(never)] + fn globals(&mut self) -> Result> { + self.alloc + .shared + .new_inplace(Default::default(), |_inner, ptr| { + let pwr = &self.dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + + let raw = place!( + ptr, + raw::Globals::ver { + //ktrace_enable: 0xffffffff, + ktrace_enable: 0, + #[ver(V >= V13_2)] + unk_24_0: 3000, + unk_24: 0, + #[ver(V >= V13_0B4)] + unk_28_0: 0, // debug + unk_28: 1, + #[ver(V >= V13_0B4)] + unk_2c_0: 0, + unk_2c: 1, + unk_30: 0, + unk_34: 120, + sub: raw::GlobalsSub::ver { + unk_54: 0xffff, + unk_56: 40, + unk_58: 0xffff, + unk_5e: U32(1), + unk_66: U32(1), + ..Default::default() + }, + unk_8900: 1, + pending_submissions: AtomicU32::new(0), + max_power: pwr.max_power_mw, + max_pstate_scaled: max_ps_scaled, + max_pstate_scaled_2: max_ps_scaled, + max_pstate_scaled_3: max_ps_scaled, + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: avg_power_filter_tc_periods, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + unk_89bc: self.cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_89c4: self.cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_89e0: 1, + max_power_2: pwr.max_power_mw, + ppm_kp: pwr.ppm_kp, + ppm_ki_dt: pwr.ppm_ki * period_s, + #[ver(V >= V13_0B4)] + unk_89f4_8: 1, + unk_89f4: 0, + hws1: Self::hw_shared1(self.cfg), + hws2: *Self::hw_shared2(self.cfg)?, + hws3: *Self::hw_shared3(self.cfg)?, + unk_900c: 1, + #[ver(V >= V13_0B4)] + unk_9010_0: 1, + #[ver(V >= V13_0B4)] + unk_903c: 1, + #[ver(V < V13_0B4)] + unk_903c: 0, + fault_control: *crate::fault_control.read(), + do_init: 1, + unk_11020: 40, + unk_11024: 10, + unk_11028: 250, + #[ver(V >= V13_0B4)] + unk_1102c_0: 1, + #[ver(V >= V13_0B4)] + unk_1102c_4: 1, + #[ver(V >= V13_0B4)] + unk_1102c_8: 100, + #[ver(V >= V13_0B4)] + unk_1102c_c: 1, + idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), + fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + unk_118e0: 40, + #[ver(V >= V13_0B4)] + unk_118e4_0: 50, + #[ver(V >= V13_0B4)] + unk_11edc: 0, + #[ver(V >= V13_0B4)] + unk_11efc: 0, + ..Default::default() + } + ); + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + + if let Some(tab) = self.cfg.global_tab.as_ref() { + for (i, x) in tab.iter().enumerate() { + raw.unk_118ec[i] = *x; + } + raw.unk_118e8 = 1; + } + + Ok(raw) + }) + } + + /// Create the RuntimePointers structure, which contains pointers to most of the other + /// structures including the ring buffer channels, statistics structures, and HwDataA/HwDataB. + #[inline(never)] + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + + let pointers: Box = box_in_place!(RuntimePointers::ver { + stats: Stats::ver { + vtx: self.alloc.private.new_default::()?, + frag: self.alloc.private.new_inplace( + Default::default(), + |_inner, ptr: &mut MaybeUninit| { + Ok(place!( + ptr, + raw::GpuGlobalStatsFrag::ver { + stats: raw::GpuStatsFrag::ver { + cur_stamp_id: -1, + unk_118: -1, + ..Default::default() + }, + ..Default::default() + } + )) + }, + )?, + comp: self.alloc.private.new_default::()?, + }, + + hwdata_a: hwa, + unkptr_190: self.alloc.private.array_empty(0x80)?, + unkptr_198: self.alloc.private.array_empty(0xc0)?, + hwdata_b: hwb, + + unkptr_1b8: self.alloc.private.array_empty(0x1000)?, + unkptr_1c0: self.alloc.private.array_empty(0x300)?, + unkptr_1c8: self.alloc.private.array_empty(0x1000)?, + + buffer_mgr_ctl: self.alloc.gpu.array_empty(127)?, + })?; + + self.alloc.private.new_boxed(pointers, |inner, ptr| { + Ok(place!( + ptr, + raw::RuntimePointers::ver { + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + + hwdata_a: inner.hwdata_a.gpu_pointer(), + unkptr_190: inner.unkptr_190.gpu_pointer(), + unkptr_198: inner.unkptr_198.gpu_pointer(), + hwdata_b: inner.hwdata_b.gpu_pointer(), + hwdata_b_2: inner.hwdata_b.gpu_pointer(), + + fwlog_buf: None, + + unkptr_1b8: inner.unkptr_1b8.gpu_pointer(), + unkptr_1c0: inner.unkptr_1c0.gpu_pointer(), + unkptr_1c8: inner.unkptr_1c8.gpu_pointer(), + + buffer_mgr_ctl: inner.buffer_mgr_ctl.gpu_pointer(), + buffer_mgr_ctl_2: inner.buffer_mgr_ctl.gpu_pointer(), + + __pad0: Default::default(), + unk_160: U64(0), + unk_168: U64(0), + unk_1d0: 0, + unk_1d4: 0, + unk_1d8: Default::default(), + + __pad1: Default::default(), + gpu_scratch: raw::RuntimeScratch { + unk_6b38: 0xff, + ..Default::default() + }, + } + )) + }) + } + + /// Create the FwStatus structure, which is used to coordinate the firmware halt state between + /// the firmware and the driver. + #[inline(never)] + fn fw_status(&mut self) -> Result> { + self.alloc + .shared + .new_object(Default::default(), |_inner| Default::default()) + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + #[inline(never)] + fn uat_level_info( + cfg: &hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + let inner: Box = box_in_place!(InitData::ver { + unk_buf: self.alloc.shared_ro.array_empty(0x4000)?, + runtime_pointers: self.runtime_pointers()?, + globals: self.globals()?, + fw_status: self.fw_status()?, + })?; + + Ok(Box::try_new(self.alloc.shared_ro.new_boxed( + inner, + |inner, ptr| { + Ok(place!( + ptr, + raw::InitData::ver { + #[ver(V >= V13_0B4)] + ver_info: Array::new([1, 1, 16, 1]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + fw_status: inner.fw_status.gpu_pointer(), + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new([ + Self::uat_level_info(self.cfg, 36, 8), + Self::uat_level_info(self.cfg, 25, 2048), + Self::uat_level_info(self.cfg, 14, 2048), + ]), + __pad0: Default::default(), + host_mapped_fw_allocations: 1, + unk_ac: 0, + unk_b0: 0, + unk_b4: 0, + unk_b8: 0, + } + )) + }, + )?)?) + } +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs new file mode 100644 index 00000000000000..491d4f8a4016c9 --- /dev/null +++ b/drivers/gpu/drm/asahi/mem.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! ARM64 low level memory operations. +//! +//! This GPU uses CPU-side `tlbi` outer-shareable instructions to manage its TLBs. +//! Yes, really. Even though the VA address spaces are unrelated. +//! +//! Right now we pick our own ASIDs and don't coordinate with the CPU. This might result +//! in needless TLB shootdowns on the CPU side... TODO: fix this. + +use core::arch::asm; +use core::cmp::min; + +use crate::debug::*; +use crate::mmu; + +type Asid = u8; + +/// Invalidate the entire GPU TLB. +#[inline(always)] +pub(crate) fn tlbi_all() { + unsafe { + asm!(".arch armv8.4-a", "tlbi vmalle1os",); + } +} + +/// Invalidate all TLB entries for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_asid(asid: Asid) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi aside1os, {x}", + x = in(reg) ((asid as u64) << 48) + ); + } +} + +/// Invalidate a single page for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_page(asid: Asid, va: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi vae1os, {x}", + x = in(reg) val + ); + } +} + +/// Invalidate a range of pages for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + let start_pg = va >> mmu::UAT_PGBIT; + let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); + let pages = end_pg - start_pg; + + if pages == 1 { + tlbi_page(asid, va); + return; + } + + // Page count is always in units of 2 + let num = ((pages + 1) >> 1) as u64; + // base: 5 bits + // exp: 2 bits + // pages = (base + 1) << (5 * exp + 1) + // 0:00000 -> 2 pages = 2 << 0 + // 0:11111 -> 32 * 2 pages = 2 << 5 + // 1:00000 -> 1 * 32 * 2 pages = 2 << 5 + // 1:11111 -> 32 * 32 * 2 pages = 2 << 10 + // 2:00000 -> 1 * 32 * 32 * 2 pages = 2 << 10 + // 2:11111 -> 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:00000 -> 1 * 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:11111 -> 32 * 32 * 32 * 32 * 2 pages = 2 << 20 + let exp = min(3, (64 - num.leading_zeros()) / 5); + let bits = 5 * exp; + let mut base = (num + (1 << bits) - 1) >> bits; + + val |= (exp as u64) << 44; + + while base > 32 { + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | (31 << 39) + ); + } + base -= 32; + } + + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | ((base - 1) << 39) + ); + } +} + +/// Issue a memory barrier (`dsb sy`). +#[inline(always)] +pub(crate) fn sync() { + unsafe { + asm!("dsb sy"); + } +} diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs new file mode 100644 index 00000000000000..dca94ebc53a1b7 --- /dev/null +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU Micro operation sequence builder +//! +//! As part of a single job submisssion to the GPU, the GPU firmware interprets a sequence of +//! commands that we call a "microsequence". These are responsible for setting up the job execution, +//! timestamping the process, waiting for completion, tearing up any resources, and signaling +//! completion to the driver via the event stamp mechanism. +//! +//! Although the microsequences used by the macOS driver are usually quite uniform and simple, the +//! firmware actually implements enough operations to make this interpreter Turing-complete (!). +//! Most of those aren't implemented yet, since we don't need them, but they could come in handy in +//! the future to do strange things or work around firmware bugs... +//! +//! This module simply implements a collection of microsequence operations that can be appended to +//! and later concatenated into one buffer, ready for firmware execution. + +use crate::fw::microseq; +pub(crate) use crate::fw::microseq::*; +use crate::fw::types::*; +use kernel::prelude::*; + +/// MicroSequence object type, which is just an opaque byte array. +pub(crate) type MicroSequence = GpuArray; + +/// MicroSequence builder. +pub(crate) struct Builder { + ops: Vec, +} + +impl Builder { + /// Create a new Builder object + pub(crate) fn new() -> Builder { + Builder { ops: Vec::new() } + } + + /// Get the relative offset from the current pointer to a given target offset. + /// + /// Used for relative jumps. + pub(crate) fn offset_to(&self, target: i32) -> i32 { + target - self.ops.len() as i32 + } + + /// Add an operation to the end of the sequence. + pub(crate) fn add(&mut self, op: T) -> Result { + let off = self.ops.len(); + let p: *const T = &op; + let p: *const u8 = p as *const u8; + let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; + self.ops.try_extend_from_slice(s)?; + Ok(off as i32) + } + + /// Collect all submitted operations into a finalized GPU object. + pub(crate) fn build(self, alloc: &mut Allocator) -> Result { + let mut array = alloc.array_empty::(self.ops.len())?; + + array.as_mut_slice().clone_from_slice(self.ops.as_slice()); + Ok(array) + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs new file mode 100644 index 00000000000000..226ca0b7c1d74e --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -0,0 +1,1249 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU UAT (MMU) management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the global MMU structures, including a shared handoff structure +//! that is used to coordinate VM management operations with the firmware, the TTBAT which points +//! to currently active GPU VM contexts, as well as the individual `Vm` operations to map and +//! unmap buffer objects into a single user or kernel address space. +//! +//! The actual page table management is delegated to the common kernel `io_pgtable` code. + +use core::fmt::Debug; +use core::mem::size_of; +use core::ptr::{addr_of_mut, NonNull}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}; +use core::time::Duration; + +use kernel::{ + bindings, c_str, delay, device, + drm::mm, + error::{to_result, Result}, + io_pgtable, + io_pgtable::{prot, AppleUAT, IoPageTable}, + prelude::*, + sync::{smutex::Mutex, Guard}, + sync::{Arc, LockClassKey, UniqueArc}, + time, + types::ForeignOwnable, +}; + +use crate::debug::*; +use crate::no_debug; +use crate::{driver, fw, gem, hw, mem, slotalloc}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; + +/// PPL magic number for the handoff region +const PPL_MAGIC: u64 = 0x4b1d000000000002; + +/// Number of supported context entries in the TTBAT +const UAT_NUM_CTX: usize = 64; +/// First context available for users +const UAT_USER_CTX_START: usize = 1; +/// Number of available user contexts +const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; + +/// Number of bits in a page offset. +pub(crate) const UAT_PGBIT: usize = 14; +/// UAT page size. +pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; +/// UAT page offset mask. +pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; + +type Pte = AtomicU64; + +/// Number of PTEs per page. +const UAT_NPTE: usize = UAT_PGSZ / size_of::(); + +/// UAT input address space (user) +pub(crate) const UAT_IAS: usize = 39; +/// "Fake" kernel UAT input address space (one page level lower) +pub(crate) const UAT_IAS_KERN: usize = 36; + +/// Lower/user base VA +const IOVA_USER_BASE: usize = UAT_PGSZ; +/// Lower/user top VA +const IOVA_USER_TOP: usize = (1 << UAT_IAS) - 1; +/// Upper/kernel base VA +// const IOVA_TTBR1_BASE: usize = 0xffffff8000000000; +/// Driver-managed kernel base VA +const IOVA_KERN_BASE: usize = 0xffffffa000000000; +/// Driver-managed kernel top VA +const IOVA_KERN_TOP: usize = 0xffffffafffffffff; + +const TTBR_VALID: u64 = 0x1; // BIT(0) +const TTBR_ASID_SHIFT: usize = 48; + +const PTE_TABLE: u64 = 0x3; // BIT(0) | BIT(1) + +// Mapping protection types + +// Note: prot::CACHE means "cache coherency", which for UAT means *uncached*, +// since uncached mappings from the GFX ASC side are cache coherent with the AP cache. +// Not having that flag means *cached noncoherent*. + +/// Firmware MMIO R/W +pub(crate) const PROT_FW_MMIO_RW: u32 = + prot::PRIV | prot::READ | prot::WRITE | prot::CACHE | prot::MMIO; +/// Firmware MMIO R/O +pub(crate) const PROT_FW_MMIO_RO: u32 = prot::PRIV | prot::READ | prot::CACHE | prot::MMIO; +/// Firmware shared (uncached) RW +pub(crate) const PROT_FW_SHARED_RW: u32 = prot::PRIV | prot::READ | prot::WRITE | prot::CACHE; +/// Firmware shared (uncached) RO +pub(crate) const PROT_FW_SHARED_RO: u32 = prot::PRIV | prot::READ | prot::CACHE; +/// Firmware private (cached) RW +pub(crate) const PROT_FW_PRIV_RW: u32 = prot::PRIV | prot::READ | prot::WRITE; +/* +/// Firmware private (cached) RO +pub(crate) const PROT_FW_PRIV_RO: u32 = prot::PRIV | prot::READ; +*/ +/// Firmware/GPU shared (uncached) RW +pub(crate) const PROT_GPU_FW_SHARED_RW: u32 = prot::READ | prot::WRITE | prot::CACHE; +/// Firmware/GPU shared (private) RW +pub(crate) const PROT_GPU_FW_PRIV_RW: u32 = prot::READ | prot::WRITE; +/// GPU shared/coherent RW +pub(crate) const PROT_GPU_SHARED_RW: u32 = prot::READ | prot::WRITE | prot::CACHE | prot::NOEXEC; +/// GPU shared/coherent RO +pub(crate) const PROT_GPU_SHARED_RO: u32 = prot::READ | prot::CACHE | prot::NOEXEC; +/// GPU shared/coherent WO +pub(crate) const PROT_GPU_SHARED_WO: u32 = prot::WRITE | prot::CACHE | prot::NOEXEC; +/* +/// GPU private/noncoherent RW +pub(crate) const PROT_GPU_PRIV_RW: u32 = prot::READ | prot::WRITE | prot::NOEXEC; +/// GPU private/noncoherent RO +pub(crate) const PROT_GPU_PRIV_RO: u32 = prot::READ | prot::NOEXEC; +*/ + +type PhysAddr = bindings::phys_addr_t; + +/// A pre-allocated memory region for UAT management +struct UatRegion { + base: PhysAddr, + map: NonNull, +} + +/// It's safe to share UAT region records across threads. +unsafe impl Send for UatRegion {} +unsafe impl Sync for UatRegion {} + +/// Handoff region flush info structure +#[repr(C)] +struct FlushInfo { + state: AtomicU64, + addr: AtomicU64, + size: AtomicU64, +} + +/// UAT Handoff region layout +#[repr(C)] +struct Handoff { + magic_ap: AtomicU64, + magic_fw: AtomicU64, + + lock_ap: AtomicU8, + lock_fw: AtomicU8, + // Implicit padding: 2 bytes + turn: AtomicU32, + cur_slot: AtomicU32, + // Implicit padding: 4 bytes + flush: [FlushInfo; UAT_NUM_CTX + 1], + + unk2: AtomicU8, + // Implicit padding: 7 bytes + unk3: AtomicU64, +} + +const HANDOFF_SIZE: usize = size_of::(); + +/// One VM slot in the TTBAT +#[repr(C)] +struct SlotTTBS { + ttb0: AtomicU64, + ttb1: AtomicU64, +} + +const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); + +// We need at least page 0 (ttb0) +const PAGETABLES_SIZE: usize = UAT_PGSZ; + +/// Inner data for a Vm instance. This is reference-counted by the outer Vm object. +struct VmInner { + dev: driver::AsahiDevice, + is_kernel: bool, + min_va: usize, + max_va: usize, + page_table: AppleUAT, + mm: mm::Allocator<(), MappingInner>, + uat_inner: Arc, + active_users: usize, + binding: Option>, + bind_token: Option, + id: u64, +} + +impl VmInner { + /// Returns the slot index, if this VM is bound. + fn slot(&self) -> Option { + if self.is_kernel { + // The GFX ASC does not care about the ASID. Pick an arbitrary one. + // TODO: This needs to be a persistently reserved ASID once we integrate + // with the ARM64 kernel ASID machinery to avoid overlap. + Some(0) + } else { + // We don't check whether we lost the slot, which could cause unnecessary + // invalidations against another Vm. However, this situation should be very + // rare (e.g. a Vm lost its slot, which means 63 other Vms bound in the + // interim, and then it gets killed / drops its mappings without doing any + // final rendering). Anything doing active maps/unmaps is probably also + // rendering and therefore likely bound. + self.bind_token + .as_ref() + .map(|token| (token.last_slot() + UAT_USER_CTX_START as u32)) + } + } + + /// Returns the translation table base for this Vm + fn ttb(&self) -> u64 { + self.page_table.cfg().ttbr + } + + /// Map an IOVA to the shifted address the underlying io_pgtable uses. + fn map_iova(&self, iova: usize, size: usize) -> Result { + if iova < self.min_va || (iova + size - 1) > self.max_va { + Err(EINVAL) + } else if self.is_kernel { + Ok(iova - self.min_va) + } else { + Ok(iova) + } + } + + /// Map a contiguous range of virtual->physical pages. + fn map_pages( + &mut self, + mut iova: usize, + mut paddr: usize, + pgsize: usize, + pgcount: usize, + prot: u32, + ) -> Result { + let mut left = pgcount; + while left > 0 { + let mapped_iova = self.map_iova(iova, pgsize * left)?; + let mapped = self + .page_table + .map_pages(mapped_iova, paddr, pgsize, left, prot)?; + assert!(mapped <= left * pgsize); + + left -= mapped / pgsize; + paddr += mapped; + iova += mapped; + } + Ok(pgcount * pgsize) + } + + /// Unmap a contiguous range of pages. + fn unmap_pages(&mut self, mut iova: usize, pgsize: usize, pgcount: usize) -> Result { + let mut left = pgcount; + while left > 0 { + let mapped_iova = self.map_iova(iova, pgsize * left)?; + let unmapped = self.page_table.unmap_pages(mapped_iova, pgsize, left); + assert!(unmapped <= left * pgsize); + + left -= unmapped / pgsize; + iova += unmapped; + } + + Ok(pgcount * pgsize) + } + + /// Map an `mm::Node` representing an mapping in VA space. + fn map_node(&mut self, node: &mm::Node<(), MappingInner>, prot: u32) -> Result { + let mut iova = node.start() as usize; + let sgt = node.sgt.as_ref().ok_or(EINVAL)?; + + for range in sgt.iter() { + let addr = range.dma_address(); + let len = range.dma_len(); + + if (addr | len | iova) & UAT_PGMSK != 0 { + dev_err!( + self.dev, + "MMU: Mapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + addr, + len, + iova + ); + return Err(EINVAL); + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x}\n", + addr, + len, + iova + ); + + self.map_pages(iova, addr, UAT_PGSZ, len >> UAT_PGBIT, prot)?; + + iova += len; + } + Ok(()) + } +} + +/// Shared reference to a virtual memory address space ([`Vm`]). +#[derive(Clone)] +pub(crate) struct Vm { + id: u64, + file_id: u64, + inner: Arc>, +} +no_debug!(Vm); + +/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). +pub(crate) struct SlotInner(); + +impl slotalloc::SlotItem for SlotInner { + type Data = (); +} + +/// Represents a single user of a binding of a [`Vm`] to a slot. +/// +/// The number of users is counted, and the slot will be freed when it drops to 0. +#[derive(Debug)] +pub(crate) struct VmBind(Vm, u32); + +impl VmBind { + /// Returns the slot that this `Vm` is bound to. + pub(crate) fn slot(&self) -> u32 { + self.1 + } +} + +impl Drop for VmBind { + fn drop(&mut self) { + let mut inner = self.0.inner.lock(); + + assert_ne!(inner.active_users, 0); + inner.active_users -= 1; + mod_pr_debug!("MMU: slot {} active users {}\n", self.1, inner.active_users); + if inner.active_users == 0 { + inner.binding = None; + } + } +} + +impl Clone for VmBind { + fn clone(&self) -> VmBind { + let mut inner = self.0.inner.lock(); + + inner.active_users += 1; + mod_pr_debug!("MMU: slot {} active users {}\n", self.1, inner.active_users); + VmBind(self.0.clone(), self.1) + } +} + +/// Inner data required for an object mapping into a [`Vm`]. +pub(crate) struct MappingInner { + owner: Arc>, + uat_inner: Arc, + prot: u32, + mapped_size: usize, + sgt: Option, +} + +/// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. +pub(crate) struct Mapping(mm::Node<(), MappingInner>); + +impl Mapping { + /// Returns the IOVA base of this mapping + pub(crate) fn iova(&self) -> usize { + self.0.start() as usize + } + + /// Returns the size of this mapping in bytes + pub(crate) fn size(&self) -> usize { + self.0.mapped_size + } + + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the + /// coprocessor cache. This is required to safely unmap cached/private mappings. + fn remap_uncached_and_flush(&mut self) { + let mut owner = self.0.owner.lock(); + mod_dev_dbg!( + owner.dev, + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + // The IOMMU API does not allow us to remap things in-place... + // just do an unmap and map again for now. + // Do not try to unmap guard page (-1) + if owner + .unmap_pages(self.iova(), UAT_PGSZ, self.size() >> UAT_PGBIT) + .is_err() + { + dev_err!( + owner.dev, + "MMU: unmap for remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + let prot = self.0.prot | prot::CACHE; + if owner.map_node(&self.0, prot).is_err() { + dev_err!( + owner.dev, + "MMU: remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + // If we don't have (and have never had) a VM slot, just return + let slot = match owner.slot() { + None => return, + Some(slot) => slot, + }; + + let flush_slot = if owner.is_kernel { + // If this is a kernel mapping, always flush on index 64 + UAT_NUM_CTX as u32 + } else { + // Otherwise, check if this slot is the active one, otherwise return + // Also check that we actually own this slot + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.0.uat_inner.lock(); + uat_inner.handoff().lock(); + let cur_slot = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); + uat_inner.handoff().unlock(); + if cur_slot == Some(slot) && ttb_cur == ttb { + slot + } else { + return; + } + }; + + // FIXME: There is a race here, though it'll probably never happen in practice. + // In theory, it's possible for the ASC to finish using our slot, whatever command + // it was processing to complete, the slot to be lost to another context, and the ASC + // to begin using it again with a different page table, thus faulting when it gets a + // flush request here. In practice, the chance of this happening is probably vanishingly + // small, as all 62 other slots would have to be recycled or in use before that slot can + // be reused, and the ASC using user contexts at all is very rare. + + // Still, the locking around UAT/Handoff/TTBs should probably be redesigned to better + // model the interactions with the firmware and avoid these races. + // Possibly TTB changes should be tied to slot locks: + + // Flush: + // - Can early check handoff here (no need to lock). + // If user slot and it doesn't match the active ASC slot, + // we can elide the flush as the ASC guarantees it flushes + // TLBs/caches when it switches context. We just need a + // barrier to ensure ordering. + // - Lock TTB slot + // - If user ctx: + // - Lock handoff AP-side + // - Lock handoff dekker + // - Check TTB & handoff cur ctx + // - Perform flush if necessary + // - This implies taking the fwring lock + // + // TTB change: + // - lock TTB slot + // - lock handoff AP-side + // - lock handoff dekker + // change TTB + + // Lock this flush slot, and write the range to it + let flush = self.0.uat_inner.lock_flush(flush_slot); + let pages = self.size() >> UAT_PGBIT; + flush.begin_flush(self.iova() as u64, self.size() as u64); + if pages >= 0x10000 { + dev_err!(owner.dev, "MMU: Flush too big ({:#x} pages))\n", pages); + } + + let cmd = fw::channels::FwCtlMsg { + addr: fw::types::U64(self.iova() as u64), + unk_8: 0, + slot: flush_slot, + page_count: pages as u16, + unk_12: 2, // ? + }; + + // Tell the firmware to do a cache flush + if let Err(e) = owner.dev.data().gpu.fwctl(cmd) { + dev_err!( + owner.dev, + "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", + self.iova(), + self.size(), + e + ); + } + + // Finish the flush + flush.end_flush(); + + // Slot is unlocked here + } +} + +impl Drop for Mapping { + fn drop(&mut self) { + // This is the main unmap function for UAT mappings. + // The sequence of operations here is finicky, due to the interaction + // between cached GFX ASC mappings and the page tables. These mappings + // always have to be flushed from the cache before being unmapped. + + // For uncached mappings, just unmapping and flushing the TLB is sufficient. + + // For cached mappings, this is the required sequence: + // 1. Remap it as uncached + // 2. Flush the TLB range + // 3. If kernel VA mapping OR user VA mapping and handoff.current_slot() == slot: + // a. Take a lock for this slot + // b. Write the flush range to the right context slot in handoff area + // c. Issue a cache invalidation request via FwCtl queue + // d. Poll for completion via queue + // e. Check for completion flag in the handoff area + // f. Drop the lock + // 4. Unmap + // 5. Flush the TLB range again + + // prot::CACHE means "cache coherent" which means *uncached* here. + if self.0.prot & prot::CACHE == 0 { + self.remap_uncached_and_flush(); + } + + let mut owner = self.0.owner.lock(); + mod_dev_dbg!( + owner.dev, + "MMU: unmap {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + if owner + .unmap_pages(self.iova(), UAT_PGSZ, self.size() >> UAT_PGBIT) + .is_err() + { + dev_err!( + owner.dev, + "MMU: unmap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + if let Some(asid) = owner.slot() { + mem::tlbi_range(asid as u8, self.iova(), self.size()); + mod_dev_dbg!( + owner.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + self.iova(), + self.size() + ); + mem::sync(); + } + } +} + +/// Shared UAT global data structures +struct UatShared { + handoff_rgn: UatRegion, + ttbs_rgn: UatRegion, +} + +impl UatShared { + /// Returns the handoff region area + fn handoff(&self) -> &Handoff { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.handoff_rgn.map.as_ptr() as *mut Handoff).as_ref() }.unwrap() + } + + /// Returns the TTBAT area + fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.ttbs_rgn.map.as_ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + } +} + +// SAFETY: Nothing here is unsafe to send across threads. +unsafe impl Send for UatShared {} + +/// Inner data for the top-level UAT instance. +struct UatInner { + shared: Mutex, + handoff_flush: [Mutex; UAT_NUM_CTX + 1], +} + +impl UatInner { + /// Take the lock on the shared data and return the guard. + fn lock(&self) -> Guard<'_, Mutex> { + self.shared.lock() + } + + /// Take a lock on a handoff flush slot and return the guard. + fn lock_flush(&self, slot: u32) -> Guard<'_, Mutex> { + self.handoff_flush[slot as usize].lock() + } +} + +/// Top-level UAT manager object +pub(crate) struct Uat { + dev: driver::AsahiDevice, + cfg: &'static hw::HwConfig, + pagetables_rgn: UatRegion, + + inner: Arc, + slots: slotalloc::SlotAllocator, + + kernel_vm: Vm, + _kernel_lower_vm: Vm, +} + +impl Drop for UatRegion { + fn drop(&mut self) { + // SAFETY: the pointer is valid by the type invariant + unsafe { bindings::memunmap(self.map.as_ptr()) }; + } +} + +impl Handoff { + /// Lock the handoff region from firmware access + fn lock(&self) { + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + + while self.lock_fw.load(Ordering::Relaxed) != 0 { + if self.turn.load(Ordering::Relaxed) != 0 { + self.lock_ap.store(0, Ordering::Relaxed); + while self.turn.load(Ordering::Relaxed) != 0 {} + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + } + } + fence(Ordering::Acquire); + } + + /// Unlock the handoff region, allowing firmware access + fn unlock(&self) { + self.turn.store(1, Ordering::Relaxed); + self.lock_ap.store(0, Ordering::Release); + } + + /// Returns the current Vm slot mapped by the firmware for lower/unprivileged access, if any. + fn current_slot(&self) -> Option { + let slot = self.cur_slot.load(Ordering::Relaxed); + if slot == 0 || slot == u32::MAX { + None + } else { + Some(slot) + } + } + + /// Initialize the handoff region + fn init(&self) -> Result { + self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); + self.unk3.store(0, Ordering::Relaxed); + fence(Ordering::SeqCst); + + let timeout = time::ktime_get() + Duration::from_millis(1000); + + self.lock(); + while time::ktime_get() < timeout { + if self.magic_fw.load(Ordering::Relaxed) == PPL_MAGIC { + break; + } else { + self.unlock(); + delay::coarse_sleep(Duration::from_millis(10)); + self.lock(); + } + } + + if self.magic_fw.load(Ordering::Relaxed) != PPL_MAGIC { + self.unlock(); + pr_err!("Handoff: Failed to initialize (firmware not running?)\n"); + return Err(EIO); + } + + self.unlock(); + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Represents a single flush info slot in the handoff region. +/// +/// # Invariants +/// The pointer is valid and there is no aliasing HandoffFlush instance. +struct HandoffFlush(*const FlushInfo); + +// SAFETY: These pointers are safe to send across threads. +unsafe impl Send for HandoffFlush {} + +impl HandoffFlush { + /// Set up a flush operation for the coprocessor + fn begin_flush(&self, start: u64, size: u64) { + let flush = unsafe { self.0.as_ref().unwrap() }; + + let state = flush.state.load(Ordering::Relaxed); + if state != 0 { + pr_err!("Handoff: expected flush state 0, got {}\n", state); + } + flush.addr.store(start, Ordering::Relaxed); + flush.size.store(size, Ordering::Relaxed); + flush.state.store(1, Ordering::Relaxed); + } + + /// Complete a flush operation for the coprocessor + fn end_flush(&self) { + let flush = unsafe { self.0.as_ref().unwrap() }; + let state = flush.state.load(Ordering::Relaxed); + if state != 2 { + pr_err!("Handoff: expected flush state 2, got {}\n", state); + } + flush.state.store(0, Ordering::Relaxed); + } +} + +// We do not implement FlushOps, since we flush manually in this module after +// page table operations. Just provide dummy implementations. +impl io_pgtable::FlushOps for Uat { + type Data = (); + + fn tlb_flush_all(_data: ::Borrowed<'_>) {} + fn tlb_flush_walk( + _data: ::Borrowed<'_>, + _iova: usize, + _size: usize, + _granule: usize, + ) { + } + fn tlb_add_page( + _data: ::Borrowed<'_>, + _iova: usize, + _granule: usize, + ) { + } +} + +static LOCK_KEY: LockClassKey = LockClassKey::new(); + +impl Vm { + /// Create a new virtual memory address space + fn new( + dev: driver::AsahiDevice, + uat_inner: Arc, + cfg: &'static hw::HwConfig, + is_kernel: bool, + id: u64, + file_id: u64, + ) -> Result { + let page_table = AppleUAT::new( + &dev, + io_pgtable::Config { + pgsize_bitmap: UAT_PGSZ, + ias: if is_kernel { UAT_IAS_KERN } else { UAT_IAS }, + oas: cfg.uat_oas, + coherent_walk: true, + quirks: 0, + }, + (), + )?; + let min_va = if is_kernel { + IOVA_KERN_BASE + } else { + IOVA_USER_BASE + }; + let max_va = if is_kernel { + IOVA_KERN_TOP + } else { + IOVA_USER_TOP + }; + + let mm = mm::Allocator::new( + min_va as u64, + (max_va - min_va + 1) as u64, + (), + c_str!("asahi Vm"), + &LOCK_KEY, + )?; + + Ok(Vm { + id, + file_id, + inner: Arc::try_new(Mutex::new(VmInner { + dev, + min_va, + max_va, + is_kernel, + page_table, + mm, + uat_inner, + binding: None, + bind_token: None, + active_users: 0, + id, + }))?, + }) + } + + /// Get the translation table base for this Vm + fn ttb(&self) -> u64 { + self.inner.lock().ttb() + } + + /// Map a GEM object (using its `SGTable`) into this Vm at a free address. + pub(crate) fn map(&self, size: usize, sgt: gem::SGTable) -> Result { + let mut inner = self.inner.lock(); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.insert_node( + MappingInner { + owner: self.inner.clone(), + uat_inner, + prot: PROT_FW_SHARED_RW, + sgt: Some(sgt), + mapped_size: size, + }, + (size + UAT_PGSZ) as u64, // Add guard page + )?; + + inner.map_node(&node, PROT_FW_SHARED_RW)?; + Ok(Mapping(node)) + } + + /// Map a GEM object (using its `SGTable`) into this Vm at a free address in a given range. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range( + &self, + size: usize, + sgt: gem::SGTable, + alignment: u64, + start: u64, + end: u64, + prot: u32, + guard: bool, + ) -> Result { + let mut inner = self.inner.lock(); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.insert_node_in_range( + MappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + sgt: Some(sgt), + mapped_size: size, + }, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + alignment, + 0, + start, + end, + mm::InsertMode::Best, + )?; + + inner.map_node(&node, prot)?; + Ok(Mapping(node)) + } + + /// Map a GEM object (using its `SGTable`) into this Vm at a specific address. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_at( + &self, + addr: u64, + size: usize, + sgt: gem::SGTable, + prot: u32, + guard: bool, + ) -> Result { + let mut inner = self.inner.lock(); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + MappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + sgt: Some(sgt), + mapped_size: size, + }, + addr, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + 0, + )?; + + inner.map_node(&node, prot)?; + Ok(Mapping(node)) + } + + /// Add a direct MMIO mapping to this Vm at a free address. + pub(crate) fn map_io(&self, phys: usize, size: usize, rw: bool) -> Result { + let prot = if rw { PROT_FW_MMIO_RW } else { PROT_FW_MMIO_RO }; + let mut inner = self.inner.lock(); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.insert_node( + MappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + sgt: None, + mapped_size: size, + }, + (size + UAT_PGSZ) as u64, // Add guard page + )?; + + let iova = node.start() as usize; + + if (phys | size | iova) & UAT_PGMSK != 0 { + dev_err!( + inner.dev, + "MMU: Mapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + + dev_info!( + inner.dev, + "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", + phys, + size, + iova + ); + + inner.map_pages(iova, phys, UAT_PGSZ, size >> UAT_PGBIT, prot)?; + + Ok(Mapping(node)) + } + + /// Returns the unique ID of this Vm + pub(crate) fn id(&self) -> u64 { + self.id + } + + /// Returns the unique File ID of the owner of this Vm + pub(crate) fn file_id(&self) -> u64 { + self.file_id + } +} + +impl Drop for VmInner { + fn drop(&mut self) { + assert_eq!(self.active_users, 0); + + mod_pr_debug!( + "VmInner::Drop [{}]: bind_token={:?}\n", + self.id, + self.bind_token + ); + + // Make sure this VM is not mapped to a TTB if it was + if let Some(token) = self.bind_token.take() { + let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; + let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); + let handoff_cur = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); + let inval = ttb_cur == ttb; + if inval { + if handoff_cur == Some(idx as u32) { + pr_err!( + "VmInner::drop owning slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); + } + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // In principle we dropped all the Mappings already, but we might as + // well play it safe and invalidate the whole ASID. + if inval { + mod_pr_debug!( + "VmInner::Drop [{}]: need inval for ASID {:#x}\n", + self.id, + idx + ); + mem::tlbi_asid(idx as u8); + mem::sync(); + } + } + } +} + +impl Uat { + /// Map a bootloader-preallocated memory region + fn map_region( + dev: &dyn device::RawDevice, + name: &CStr, + size: usize, + cached: bool, + ) -> Result { + let rdev = dev.raw_device(); + + let mut res = core::mem::MaybeUninit::::uninit(); + + let res = unsafe { + let idx = bindings::of_property_match_string( + (*rdev).of_node, + c_str!("memory-region-names").as_char_ptr(), + name.as_char_ptr(), + ); + to_result(idx)?; + + let np = bindings::of_parse_phandle( + (*rdev).of_node, + c_str!("memory-region").as_char_ptr(), + idx, + ); + if np.is_null() { + dev_err!(dev, "Missing {} region\n", name); + return Err(EINVAL); + } + let ret = bindings::of_address_to_resource(np, 0, res.as_mut_ptr()); + bindings::of_node_put(np); + + if ret < 0 { + dev_err!(dev, "Failed to get {} region\n", name); + to_result(ret)? + } + + res.assume_init() + }; + + let rgn_size: usize = unsafe { bindings::resource_size(&res) } as usize; + + if size > rgn_size { + dev_err!( + dev, + "Region {} is too small (expected {}, got {})\n", + name, + size, + rgn_size + ); + return Err(ENOMEM); + } + + let flags = if cached { + bindings::MEMREMAP_WB + } else { + bindings::MEMREMAP_WC + }; + let map = unsafe { bindings::memremap(res.start, rgn_size, flags.into()) }; + let map = NonNull::new(map); + + match map { + None => { + dev_err!(dev, "Failed to remap {} region\n", name); + Err(ENOMEM) + } + Some(map) => Ok(UatRegion { + base: res.start, + map, + }), + } + } + + /// Returns a view into the root kernel (upper half) page table + fn kpt0(&self) -> &[Pte; UAT_NPTE] { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.pagetables_rgn.map.as_ptr() as *mut [Pte; UAT_NPTE]).as_ref() }.unwrap() + } + + /// Returns a reference to the global kernel (upper half) `Vm` + pub(crate) fn kernel_vm(&self) -> &Vm { + &self.kernel_vm + } + + /// Returns the base physical address of the TTBAT region. + pub(crate) fn ttb_base(&self) -> u64 { + let inner = self.inner.lock(); + + inner.ttbs_rgn.base + } + + /// Binds a `Vm` to a slot, preferring the last used one. + pub(crate) fn bind(&self, vm: &Vm) -> Result { + let mut inner = vm.inner.lock(); + + if inner.binding.is_none() { + assert_eq!(inner.active_users, 0); + + let slot = self.slots.get(inner.bind_token)?; + if slot.changed() { + mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); + let idx = (slot.slot() as usize) + UAT_USER_CTX_START; + let ttb = inner.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.inner.lock(); + let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); + if uat_inner.handoff().current_slot() == Some(idx as u32) { + pr_err!( + "Vm::bind to slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + ttbs[idx].ttb0.store(ttb, Ordering::Relaxed); + ttbs[idx].ttb1.store(0, Ordering::Relaxed); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // Make sure all TLB entries from the previous owner of this ASID are gone + mem::tlbi_asid(idx as u8); + mem::sync(); + } + + inner.bind_token = Some(slot.token()); + inner.binding = Some(slot); + } + + inner.active_users += 1; + + let slot = inner.binding.as_ref().unwrap().slot() + UAT_USER_CTX_START as u32; + mod_pr_debug!("MMU: slot {} active users {}\n", slot, inner.active_users); + Ok(VmBind(vm.clone(), slot)) + } + + /// Creates a new `Vm` linked to this UAT. + pub(crate) fn new_vm(&self, id: u64, file_id: u64) -> Result { + Vm::new( + self.dev.clone(), + self.inner.clone(), + self.cfg, + false, + id, + file_id, + ) + } + + /// Creates the reference-counted inner data for a new `Uat` instance. + #[inline(never)] + fn make_inner(dev: &driver::AsahiDevice) -> Result> { + let handoff_rgn = Self::map_region(dev, c_str!("handoff"), HANDOFF_SIZE, false)?; + let ttbs_rgn = Self::map_region(dev, c_str!("ttbs"), SLOTS_SIZE, false)?; + + dev_info!(dev, "MMU: Initializing kernel page table\n"); + + let mut inner = UniqueArc::::try_new_uninit()?; + let ptr = inner.as_mut_ptr(); + + Ok(unsafe { + let handoff = &(handoff_rgn.map.as_ptr() as *mut Handoff).as_ref().unwrap(); + + for i in 0..UAT_NUM_CTX + 1 { + addr_of_mut!((*ptr).handoff_flush[i]) + .write(Mutex::new(HandoffFlush(&handoff.flush[i]))); + } + + addr_of_mut!((*ptr).shared).write(Mutex::new(UatShared { + handoff_rgn, + ttbs_rgn, + })); + + inner.assume_init() + } + .into()) + } + + /// Creates a new `Uat` instance given the relevant hardware config. + #[inline(never)] + pub(crate) fn new(dev: &driver::AsahiDevice, cfg: &'static hw::HwConfig) -> Result { + dev_info!(dev, "MMU: Initializing...\n"); + + let inner = Self::make_inner(dev)?; + + let pagetables_rgn = Self::map_region(dev, c_str!("pagetables"), PAGETABLES_SIZE, true)?; + + dev_info!(dev, "MMU: Creating kernel page tables\n"); + let kernel_lower_vm = Vm::new(dev.clone(), inner.clone(), cfg, false, 1, 0)?; + let kernel_vm = Vm::new(dev.clone(), inner.clone(), cfg, true, 0, 0)?; + + dev_info!(dev, "MMU: Kernel page tables created\n"); + + let ttb0 = kernel_lower_vm.ttb(); + let ttb1 = kernel_vm.ttb(); + + let uat = Self { + dev: dev.clone(), + cfg, + pagetables_rgn, + kernel_vm, + _kernel_lower_vm: kernel_lower_vm, + inner, + slots: slotalloc::SlotAllocator::new(UAT_USER_CTX as u32, (), |_inner, _slot| { + SlotInner() + })?, + }; + + let inner = uat.inner.lock(); + + inner.handoff().init()?; + + dev_info!(dev, "MMU: Initializing TTBs\n"); + + inner.handoff().lock(); + + let ttbs = inner.ttbs(); + + ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::Relaxed); + ttbs[0] + .ttb1 + .store(uat.pagetables_rgn.base | TTBR_VALID, Ordering::Relaxed); + + for ctx in &ttbs[1..] { + ctx.ttb0.store(0, Ordering::Relaxed); + ctx.ttb1.store(0, Ordering::Relaxed); + } + + inner.handoff().unlock(); + + core::mem::drop(inner); + + uat.kpt0()[2].store(ttb1 | PTE_TABLE, Ordering::Relaxed); + + dev_info!(dev, "MMU: initialized\n"); + + Ok(uat) + } +} + +impl Drop for Uat { + fn drop(&mut self) { + // Unmap what we mapped + self.kpt0()[2].store(0, Ordering::Relaxed); + + // Make sure we flush the TLBs + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + } +} diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs new file mode 100644 index 00000000000000..449899b881813a --- /dev/null +++ b/drivers/gpu/drm/asahi/object.rs @@ -0,0 +1,704 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi GPU object model +//! +//! The AGX GPU includes a coprocessor that uses a large number of shared memory structures to +//! communicate with the driver. These structures contain GPU VA pointers to each other, which are +//! directly dereferenced by the firmware and are expected to always be valid for the usage +//! lifetime of the containing struct (which is an implicit contract, not explicitly managed). +//! Any faults cause an unrecoverable firmware crash, requiring a full system reboot. +//! +//! In order to manage this complexity safely, we implement a GPU object model using Rust's type +//! system to enforce GPU object lifetime relationships. GPU objects represent an allocated piece +//! of memory of a given type, mapped to the GPU (and usually also the CPU). On the CPU side, +//! these objects are associated with a pure Rust structure that contains the objects it depends +//! on (or references to them). This allows us to map Rust lifetimes into the GPU object model +//! system. Then, GPU VA pointers also inherit those lifetimes, which means the Rust borrow checker +//! can ensure that all pointers are assigned an address that is guaranteed to outlive the GPU +//! object it points to. +//! +//! Since the firmware object model does have self-referencing pointers (and there is of course no +//! underlying revocability mechanism to make it safe), we must have an escape hatch. GPU pointers +//! can be weak pointers, which do not enforce lifetimes. In those cases, it is the user's +//! responsibility to ensure that lifetime requirements are met. +//! +//! In other words, the model is necessarily leaky and there is no way to fully map Rust safety to +//! GPU firmware object safety. The goal of the model is to make it easy to model the lifetimes of +//! GPU objects and have the compiler help in avoiding mistakes, rather than to guarantee safety +//! 100% of the time as would be the case for CPU-side Rust code. + +// TODO: There is a fundamental soundness issue with sharing memory with the GPU (that even affects +// C code too). Since the GPU is free to mutate that memory at any time, normal reference invariants +// cannot be enforced on the CPU side. For example, the compiler could perform an optimization that +// assumes that a given memory location does not change between two reads, and causes UB otherwise, +// and then the GPU could mutate that memory out from under the CPU. +// +// For cases where we *expect* this to happen, we use atomic types, which avoid this issue. However, +// doing so for every single field of every type is a non-starter. Right now, there seems to be no +// good solution for this that does not come with significant performance or ergonomics downsides. +// +// In *practice* we are almost always only writing GPU memory, and only reading from atomics, so the +// chances of this actually triggering UB (e.g. a security issue that can be triggered from the GPU +// side) due to a compiler optimization are very slim. +// +// Further discussion: https://github.com/rust-lang/unsafe-code-guidelines/issues/152 + +use kernel::{error::code::*, prelude::*}; + +use alloc::boxed::Box; +use core::fmt; +use core::fmt::Debug; +use core::fmt::Formatter; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::num::NonZeroU64; +use core::ops::{Deref, DerefMut, Index, IndexMut}; +use core::{mem, ptr, slice}; + +use crate::alloc::Allocation; +use crate::debug::*; +use crate::fw::types::Zeroed; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Object; + +/// A GPU-side strong pointer, which is a 64-bit non-zero VA with an associated lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuPointer<'a, T: ?Sized>(NonZeroU64, PhantomData<&'a T>); + +impl<'a, T: ?Sized> GpuPointer<'a, T> { + /// Logical OR the pointer with an arbitrary `u64`. This is used when GPU struct fields contain + /// misc flag fields in the upper bits. The lifetime is retained. This is GPU-unsafe in + /// principle, but we assert that only non-implemented address bits are touched, which is safe + /// for pointers used by the GPU (not by firmware). + pub(crate) fn or(&self, other: u64) -> GpuPointer<'a, T> { + // This will fail for kernel-half pointers, which should not be ORed. + assert_eq!(self.0.get() & other, 0); + // Assert that we only touch the high bits. + assert_eq!(other & 0xffffffffff, 0); + GpuPointer(self.0 | other, PhantomData) + } + + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_ptr` macro to get pointers to inner fields, hence we mark + /// it `unsafe` to discourage direct use. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuPointer<'a, U> { + GpuPointer::<'a, U>( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } +} + +impl<'a, T: ?Sized> Debug for GpuPointer<'a, T> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuPointer, keeping the +/// lifetime. +#[macro_export] +macro_rules! inner_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // This mirrors kernel::offset_of(), except we use type inference to avoid having to know + // the type of the pointer explicitly. + fn uninit_from<'a, T: GpuStruct>(_: GpuPointer<'a, T>) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// A GPU-side weak pointer, which is a 64-bit non-zero VA with no lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuWeakPointer(NonZeroU64, PhantomData<*const T>); + +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Send for GpuWeakPointer {} +unsafe impl Sync for GpuWeakPointer {} + +// Weak pointers can be copied/cloned regardless of their target type. +impl Copy for GpuWeakPointer {} + +impl Clone for GpuWeakPointer { + fn clone(&self) -> Self { + *self + } +} + +impl GpuWeakPointer { + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_ptr` macro to get pointers to inner fields, hence we mark + /// it `unsafe` to discourage direct use. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuWeakPointer { + GpuWeakPointer::( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } + + /// Upgrade a weak pointer into a strong pointer. This is not considered safe from the GPU + /// perspective. + pub(crate) unsafe fn upgrade<'a>(&self) -> GpuPointer<'a, T> { + GpuPointer(self.0, PhantomData) + } +} + +impl Debug for GpuWeakPointer { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuWeakPointer. +#[macro_export] +macro_rules! inner_weak_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // See inner_ptr() + fn uninit_from(_: GpuWeakPointer) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// Types that implement this trait represent a GPU structure from the CPU side. +/// +/// The `Raw` type represents the actual raw structure definition on the GPU side. +/// +/// Types implementing [`GpuStruct`] must have fields owning any objects (or strong references +/// to them) that GPU pointers in the `Raw` structure point to. This mechanism is used to enforce +/// lifetimes. +pub(crate) trait GpuStruct: 'static { + /// The type of the GPU-side structure definition representing the firmware struct layout. + type Raw<'a>; +} + +/// An instance of a GPU object in memory. +/// +/// # Invariants +/// `raw` must point to a valid mapping of the `T::Raw` type associated with the `alloc` allocation. +/// `gpu_ptr` must be the GPU address of the same object. +pub(crate) struct GpuObject> { + raw: *mut T::Raw<'static>, + alloc: U, + gpu_ptr: GpuWeakPointer, + inner: Box, +} + +impl> GpuObject { + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that constructs the `T::Raw` type given a reference to the + /// `GpuStruct`. This is the mechanism used to enforce lifetimes. + pub(crate) fn new( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result { + let size = mem::size_of::>(); + if size > 0x1000 { + dev_crit!( + alloc.device(), + "Allocating {} of size {:#x}, with new, please use new_boxed!\n", + core::any::type_name::(), + size + ); + } + if alloc.size() < size { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let mut raw = callback(&inner); + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is + // identical to the type of `raw` other than the lifetime. + unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + mem::forget(raw); + Ok(Self { + raw: p, + gpu_ptr, + alloc, + inner: Box::try_new(inner)?, + }) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_boxed( + alloc: U, + inner: Box, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + let raw = callback(&inner, unsafe { &mut *p })?; + if p as *mut T::Raw<'_> != raw as *mut _ { + dev_err!( + alloc.device(), + "Allocation callback returned a mismatched reference ({})\n", + core::any::type_name::(), + ); + return Err(EINVAL); + } + Ok(Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner, + }) + } + + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_inplace( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + GpuObject::::new_boxed(alloc, Box::try_new(inner)?, callback) + } + + /// Create a new GpuObject given an allocator, with callback-based initialization. + /// + /// This is used when the construction of the `T` type requires knowing the GPU VA address of + /// the structure that is being constructed ahead of time. The first callback constructs a + /// `Box` given the pointer to the about-to-be-initialized GPU structure, and the second + /// callback initializes that structure as in `new_boxed`. + pub(crate) fn new_prealloc( + alloc: U, + inner_cb: impl FnOnce(GpuWeakPointer) -> Result>, + raw_cb: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let inner = inner_cb(gpu_ptr)?; + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + let raw = raw_cb(&*inner, unsafe { &mut *p })?; + if p as *mut T::Raw<'_> != raw as *mut _ { + dev_err!( + alloc.device(), + "Allocation callback returned a mismatched reference ({})\n", + core::any::type_name::(), + ); + return Err(EINVAL); + } + Ok(Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner, + }) + } + + /// Returns the GPU VA of this object (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr.0 + } + + /// Returns a strong GPU pointer to this object, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, T> { + GpuPointer(self.gpu_ptr.0, PhantomData) + } + + /// Returns a weak GPU pointer to this object, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + GpuWeakPointer(self.gpu_ptr.0, PhantomData) + } + + /// Perform a mutation to the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a mutable reference to the `GpuStruct` type. + pub(crate) fn with_mut( + &mut self, + callback: impl for<'a> FnOnce(&'a mut ::Raw<'a>, &'a mut T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + } + + /// Access the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a reference to the `GpuStruct` type. + pub(crate) fn with( + &self, + callback: impl for<'a> FnOnce(&'a ::Raw<'a>, &'a T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + } +} + +impl> Deref for GpuObject { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl> DerefMut for GpuObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl> Debug for GpuObject +where + ::Raw<'static>: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + // SAFETY: `self.raw` is valid per the type invariant. + .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) + .field("inner", &format_args!("{:#X?}", &self.inner)) + .field("alloc", &format_args!("{:?}", &self.alloc)) + .finish() + } +} + +impl> GpuObject +where + for<'a> ::Raw<'a>: Default + Zeroed, +{ + /// Create a new GpuObject with default data. `T` must implement `Default` and `T::Raw` must + /// implement `Zeroed`, since the GPU-side memory is initialized by zeroing. + pub(crate) fn new_default(alloc: U) -> Result { + GpuObject::::new_inplace(alloc, Default::default(), |_inner, raw| { + // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroed`. + Ok(unsafe { + ptr::write_bytes(raw, 0, 1); + (*raw).assume_init_mut() + }) + }) + } +} + +impl> Drop for GpuObject { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Send for GpuObject {} +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Sync for GpuObject {} + +/// Trait used to erase the type of a GpuObject, used when we need to keep a list of heterogenous +/// objects around. +pub(crate) trait OpaqueGpuObject: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; +} + +impl> OpaqueGpuObject for GpuObject { + fn gpu_va(&self) -> NonZeroU64 { + Self::gpu_va(self) + } +} + +/// An array of raw GPU objects that is only accessible to the GPU (no CPU-side mapping required). +/// +/// This must necessarily be uninitialized as far as the GPU is concerned, so it cannot be used +/// when initialization is required. +/// +/// # Invariants +/// +/// `alloc` is valid and at least as large as `len` times the size of one `T`. +/// `gpu_ptr` is valid and points to the allocation start. +pub(crate) struct GpuOnlyArray> { + len: usize, + alloc: U, + gpu_ptr: NonZeroU64, + _p: PhantomData, +} + +impl> GpuOnlyArray { + /// Allocate a new GPU-only array with the given length. + pub(crate) fn new(alloc: U, count: usize) -> Result> { + let bytes = count * mem::size_of::(); + let gpu_ptr = NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?; + if alloc.size() < bytes { + return Err(ENOMEM); + } + Ok(Self { + len: count, + alloc, + gpu_ptr, + _p: PhantomData, + }) + } + + /// Returns the GPU VA of this arraw (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr + } + + /// Returns a strong GPU pointer to this array, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, &'_ [T]> { + GpuPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a weak GPU pointer to this array, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer<[T]> { + GpuWeakPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a pointer to an offset within the array (as a subslice). + pub(crate) fn gpu_offset_pointer(&self, offset: usize) -> GpuPointer<'_, &'_ [T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /* Not used yet + /// Returns a weak pointer to an offset within the array (as a subslice). + pub(crate) fn weak_offset_pointer(&self, offset: usize) -> GpuWeakPointer<[T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns a pointer to an element within the array. + pub(crate) fn gpu_item_pointer(&self, index: usize) -> GpuPointer<'_, &'_ T> { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + */ + + /// Returns a weak pointer to an element within the array. + pub(crate) fn weak_item_pointer(&self, index: usize) -> GpuWeakPointer { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns the length of the array. + pub(crate) fn len(&self) -> usize { + self.len + } +} + +impl> Debug for GpuOnlyArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("len", &format_args!("{:#X?}", self.len())) + .finish() + } +} + +impl> Drop for GpuOnlyArray { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +/// An array of raw GPU objects that is also CPU-accessible. +/// +/// # Invariants +/// +/// `raw` is valid and points to the CPU-side view of the array (which must have one). +pub(crate) struct GpuArray> { + raw: *mut T, + array: GpuOnlyArray, +} + +/* Not used yet +impl> GpuArray { + /// Allocate a new GPU array, copying the contents from a slice. + pub(crate) fn new(alloc: U, data: &[T]) -> Result> { + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr(); + let inner = GpuOnlyArray::new(alloc, data.len())?; + // SAFETY: `p` is valid per the Allocation type invariant, and GpuOnlyArray guarantees + // that its size is at least as large as `data.len()`. + unsafe { ptr::copy(data.as_ptr(), p, data.len()) }; + Ok(Self { + raw: p, + array: inner, + }) + } +} +*/ + +impl> GpuArray { + /// Allocate a new GPU array, initializing each element to its default. + pub(crate) fn empty(alloc: U, count: usize) -> Result> { + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T; + let inner = GpuOnlyArray::new(alloc, count)?; + let mut pi = p; + for _i in 0..count { + // SAFETY: `pi` is valid per the Allocation type invariant, and GpuOnlyArray guarantees + // that it can never iterate beyond the buffer length. + unsafe { + pi.write(Default::default()); + pi = pi.add(1); + } + } + Ok(Self { + raw: p, + array: inner, + }) + } +} + +impl> GpuArray { + /// Get a slice view of the array contents. + pub(crate) fn as_slice(&self) -> &[T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts(self.raw, self.len) } + } + + /// Get a mutable slice view of the array contents. + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts_mut(self.raw, self.len) } + } +} + +impl> Deref for GpuArray { + type Target = GpuOnlyArray; + + fn deref(&self) -> &GpuOnlyArray { + &self.array + } +} + +impl> Index for GpuArray { + type Output = T; + + fn index(&self, index: usize) -> &T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &*(self.raw.add(index)) } + } +} + +impl> IndexMut for GpuArray { + fn index_mut(&mut self, index: usize) -> &mut T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &mut *(self.raw.add(index)) } + } +} + +// SAFETY: GpuArray are Send as long as the contained type itself is Send +unsafe impl> Send for GpuArray {} +// SAFETY: GpuArray are Sync as long as the contained type itself is Sync +unsafe impl> Sync for GpuArray {} + +impl> Debug for GpuArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("array", &format_args!("{:#X?}", self.as_slice())) + .finish() + } +} diff --git a/drivers/gpu/drm/asahi/place.rs b/drivers/gpu/drm/asahi/place.rs new file mode 100644 index 00000000000000..40c51f4fab8db2 --- /dev/null +++ b/drivers/gpu/drm/asahi/place.rs @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! "Placement new" macro +//! +//! This cursed abomination of a declarative macro is used to emulate a "placement new" feature, +//! which allows initializing objects directly in a user-provided memory region without first +//! going through the stack. +//! +//! This driver needs to manage several large GPU objects of a fixed layout. Linux kernel stacks are +//! very small, so it is impossible to create these objects on the stack. While the compiler can +//! sometimes optimize away the stack copy and directly instantiate in target memory, this is not +//! guaranteed and not reliable. Therefore, we need some mechanism to ergonomically initialize +//! complex structures directly in a pre-allocated piece of memory. +//! +//! This issue also affects some driver-internal structs which are large/complex enough to overflow +//! the stack. While this can be solved by breaking them up into pieces and using `Box` more +//! liberally, this has performance implications and still isn't very nice. This macro can also be +//! used to solve this issue. +//! +//! # Further reading +//! https://github.com/rust-lang/rust/issues/27779#issuecomment-378416911 +//! https://internals.rust-lang.org/t/removal-of-all-unstable-placement-features/7223 + +/// Initialize a `MaybeUninit` in-place, without constructing the value on the stack first. +/// +/// This macro is analogous to `MaybeUninit::write()`. In other words, +/// `place!(foo, bar)` is equivalent to `MaybeUninit::write(foo, bar)`, except that `bar` is not +/// constructed first, but rather its fields (if it is a structure constructor) are copied one by +/// one into the correct location in the `MaybeUninit`. +/// +/// The macro supports most Rust initialization syntax including type paths, generic arguments, +/// and nested structures. Nested structures are themselves initialized in-place field by field. +/// `..Default::default()` is supported, but this macro converts it to `..Zeroed::zeroed()`, as it +/// initializes those structs by zero-initializing the underlying memory. Usage of +/// `..Default::default()` with a type not implementing `Zeroed` will result in a compile error. +/// +/// Usage: +/// ``` +/// let mut buf = MaybeUninit::uninit(); +/// let mut_ref = place!(&mut buf, MyStruct { +/// b: true, +/// s: String::from("works"), +/// i: str::parse::("123").unwrap(), +/// v: vec![String::from("works")], +/// x: foo::MyOtherCoolStruct { +/// a: false, +/// b: String::from("Hello, world!"), +/// }, +/// y: foo::MyOtherCoolStruct { +/// a: false, +/// b: String::from("Hello, world!"), +/// }, +/// z: foo::MyCoolGenericStruct:: { +/// a: false, +/// b: String::from("Hello, world!"), +/// }, +/// }; +/// // `mut_ref` is now a mutable reference to the `buf`, which is now safely initialized. +/// ``` +/// +/// Based on https://crates.io/crates/place by DianaNites, with contributions by Joshua Barretto. +#[macro_export] +macro_rules! place { + // Top-level struct + (@STRUCT $ptr:ident, _TOP, $typ:path, {$($typ_init:tt)*} { $($fields:tt)* }) => {{ + place!(@STRUCT_ZERO $ptr, {$($typ_init)*} { $($fields)* }); + place!(@STRUCT_CHECK $ptr, {$($typ_init)*} { $($fields)* } { + place!(@FIELDS $ptr, $($fields)*); + }); + }}; + // Nested structure + (@STRUCT $ptr:ident, $f_struct:ident, $typ:path, {$($typ_init:tt)*} { $($fields:tt)* }) => {{ + use core::ptr::addr_of_mut; + let buf = unsafe { addr_of_mut!((*$ptr).$f_struct) }; + place!(@STRUCT_ZERO buf, {$($typ_init)*} { $($fields)* }); + place!(@STRUCT_CHECK $ptr, {$($typ_init)*} { $($fields)* } { + place!(@FIELDS buf, $($fields)*); + }); + }}; + + // Zero-initialize structure if the initializer ends in ..default::Default() + (@STRUCT_ZERO $ptr:ident, {$($typ_init:tt)*} { $($f:ident $(: $v:expr)?),* $(,)? }) => {}; + (@STRUCT_ZERO $ptr:ident, {$($typ_init:tt)*} { $($($f:ident $(: $v:expr)?),*,)? ..Default::default() }) => {{ + // Check that the structure actually implements Zeroed + const _: () = { + fn _check_default() { + let _ = $($typ_init)* { + ..Zeroed::zeroed() + }; + } + }; + use core::ptr; + unsafe { ptr::write_bytes($ptr, 0, 1) }; + + }}; + + // Check that all fields are specified + (@STRUCT_CHECK $ptr:ident, {$($typ_init:tt)*} { $($($f:ident $(: $v:expr)?),*,)? ..Default::default() } {$($body:tt)*}) => { + if false { + #[allow(clippy::redundant_field_names)] + let _x = $($typ_init)* { + $($( + $f $(: $v)? + ),* + ,)? + ..Zeroed::zeroed() + }; + } else { + {$($body)*} + } + }; + (@STRUCT_CHECK $ptr:ident, {$($typ_init:tt)*} { $($f:ident $(: $v:expr)?),* $(,)? } {$($body:tt)*}) => { + if false { + #[allow(clippy::redundant_field_names)] + let _x = $($typ_init)* { + $( + $f $(: $v)? + ),* + }; + } else { + {$($body)*} + } + }; + // Top-level scalar + (@SCALAR $ptr:ident, _TOP, $val:expr) => { + let tmp = $val; + unsafe { $ptr.write(tmp); } + }; + // Regular field + (@SCALAR $ptr:ident, $f:ident, $val:expr) => {{ + use core::ptr::addr_of_mut; + let tmp = $val; + unsafe { addr_of_mut!((*$ptr).$f).write(tmp); } + }}; + // Type-like name followed by braces is a nested structure + (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {{ $($fields:tt)* } $($tail:tt)*}) => { + place!(@STRUCT $ptr, $f, $($head)*, {$($head)*} { $($fields)* }); + place!(@FIELDS $ptr $($tail)*) + }; + // Type-like name followed by ::ident, append to head + (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {::$id:ident $($tail:tt)*}) => { + place!(@PARTIAL $ptr, $f, {$($head)* :: $id}, {$($tail)*}); + }; + // Type-like name followed by ::, append to head + (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {::<$($gen:ty),*> $($tail:tt)*}) => { + place!(@PARTIAL $ptr, $f, {$($head)* :: <$($gen),*>}, {$($tail)*}); + }; + // Type-like name followed by ::<'lifetime>, append to head + (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {::<$li:lifetime> $($tail:tt)*}) => { + place!(@PARTIAL $ptr, $f, {$($head)* :: <$li>}, {$($tail)*}); + }; + // Anything else, parse it as an expression + (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {$($tail:tt)*}) => { + place!(@EXPR $ptr, $f, $($head)* $($tail)*) + }; + // Expression followed by more fields + (@EXPR $ptr:ident, $f:ident, $val:expr, $($tail:tt)*) => { + place!(@SCALAR $ptr, $f, $val); + place!(@FIELDS $ptr, $($tail)*) + }; + // Last field expression, without a trailing comma + (@EXPR $ptr:ident, $f:ident, $val:expr) => { + place!(@SCALAR $ptr, $f, $val); + }; + // Field with a value starting with an ident, start incremental type parsing + (@FIELDS $ptr:ident, $f:ident : $id:ident $($tail:tt)*) => { + place!(@PARTIAL $ptr, $f, {$id}, {$($tail)*}); + }; + // Same, but starting with ::ident + (@FIELDS $ptr:ident, $f:ident : ::$id:ident $($tail:tt)*) => { + place!(@PARTIAL $ptr, $f, {::$id}, {$($tail)*}); + }; + // Otherwise, parse it as an expression + (@FIELDS $ptr:ident, $f:ident : $($tail:tt)*) => { + place!(@EXPR $ptr, $f, $($tail)*) + }; + // Default terminating case + (@FIELDS $ptr:ident, ..Default::default() ) => {}; + // Terminating case + (@FIELDS $ptr:ident $(,)? ) => {}; + ( + $buf:expr, + $($val:tt)* + ) => {{ + use core::mem::MaybeUninit; + // Ensures types are correct + let obj: &mut MaybeUninit<_> = $buf; + let top_ptr = obj.as_mut_ptr(); + place!(@FIELDS top_ptr, _TOP: $($val)*); + // SAFETY: All fields have been initialized above + // The compiler ensures that all fields were used, all types were correct, + // and that size and alignment are correct. + unsafe { obj.assume_init_mut() } + }}; +} + +/// Helper macro to get the struct type part of a struct initialization expression. +#[macro_export] +#[doc(hidden)] +macro_rules! get_type { + ($t:ty { $($val:tt)* }) => { + $t + }; +} + +/// Like `Box::try_new(...)`, but with in-place initialization. +#[macro_export] +macro_rules! box_in_place { + ($($val:tt)*) => {{ + use $crate::place; + let b = Box::<$crate::get_type!($($val)*)>::try_new_uninit(); + match b { + Ok(mut p) => { + place!((&mut *p), $($val)*); + Ok(unsafe { p.assume_init() }) + } + Err(e) => Err(e) + } + }}; +} + +// TODO: figure out how to make this run +#[cfg(test)] +mod tests { + use super::*; + use core::mem::MaybeUninit; + + #[derive(Debug, PartialEq)] + struct MyCoolStruct { + b: bool, + s: String, + i: i32, + v: Vec, + x: MyOtherCoolStruct, + y: MyOtherCoolStruct, + z: foo::MyCoolGenericStruct, + } + + #[derive(Debug, PartialEq)] + struct MyDefaultStruct { + b: bool, + i: i32, + j: i16, + } + default_zeroed!(MyDefaultStruct); + + mod foo { + #[derive(Debug, PartialEq)] + pub struct MyOtherCoolStruct { + pub a: bool, + pub b: String, + } + #[derive(Debug, PartialEq)] + pub struct MyCoolGenericStruct { + pub a: T, + pub b: U, + } + } + + use foo::MyOtherCoolStruct; + + #[test] + fn test_initialized() { + let mut buf: MaybeUninit = MaybeUninit::uninit(); + + let x: &mut MyCoolStruct = place!( + &mut buf, + MyCoolStruct { + b: true, + s: String::from("works"), + i: str::parse::("123").unwrap(), + v: vec![String::from("works")], + x: MyOtherCoolStruct { + a: false, + b: String::from("Hello, world!"), + }, + y: foo::MyOtherCoolStruct { + a: false, + b: String::from("Hello, world!"), + }, + z: foo::MyCoolGenericStruct:: { + a: false, + b: String::from("Hello, world!"), + } + } + ); + //dbg!(x); + + assert_eq!( + x, + &MyCoolStruct { + b: true, + s: String::from("works"), + i: str::parse::("123").unwrap(), + v: vec![String::from("works")], + x: foo::MyOtherCoolStruct { + a: false, + b: String::from("Hello, world!"), + }, + y: foo::MyOtherCoolStruct { + a: false, + b: String::from("Hello, world!"), + }, + z: foo::MyCoolGenericStruct:: { + a: false, + b: String::from("Hello, world!"), + }, + }, + ); + } + + #[test] + fn test_default() { + let mut buf: MaybeUninit = MaybeUninit::uninit(); + + let x: &mut MyDefaultStruct = place!( + &mut buf, + MyDefaultStruct { + b: true, + i: 1, + ..Default::default() + } + ); + + assert_eq!( + x, + &MyDefaultStruct { + b: true, + i: 1, + j: 0, + }, + ); + } + + #[test] + fn test_scalar() { + let mut buf: MaybeUninit = MaybeUninit::uninit(); + + let x: &mut u32 = place!(&mut buf, 1234); + + assert_eq!(x, &mut 1234u32); + } +} diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs new file mode 100644 index 00000000000000..127b4ccc6eca05 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common queue functionality. +//! +//! Shared helpers used by the submission logic for multiple command types. + +use crate::fw::microseq; +use crate::fw::types::*; + +use kernel::bindings; +use kernel::io_buffer::IoBufferReader; +use kernel::prelude::*; +use kernel::user_ptr::UserSlicePtr; + +use core::mem::MaybeUninit; + +pub(super) fn build_attachments(pointer: u64, count: u32) -> Result { + if count as usize > microseq::MAX_ATTACHMENTS { + return Err(EINVAL); + } + + const STRIDE: usize = core::mem::size_of::(); + let size = STRIDE * count as usize; + + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = unsafe { UserSlicePtr::new(pointer as usize as *mut _, size).reader() }; + + let mut attachments: microseq::Attachments = Default::default(); + + for i in 0..count { + let mut att: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: The size of `att` is STRIDE + unsafe { reader.read_raw(att.as_mut_ptr() as *mut u8, STRIDE)? }; + + // SAFETY: All bit patterns in the struct are valid + let att = unsafe { att.assume_init() }; + + let cache_lines = (att.size + 127) >> 7; + let order = 1; + attachments.list[i as usize] = microseq::Attachment { + address: U64(att.pointer), + size: cache_lines, + unk_c: 0x17, + unk_e: order, + }; + + attachments.count += 1; + } + + Ok(attachments) +} diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs new file mode 100644 index 00000000000000..1d50ae16fa4c05 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] +#![allow(unused_imports)] + +//! Compute work queue. +//! +//! A compute queue consists of one underlying WorkQueue. +//! This module is in charge of creating all of the firmware structures required to submit compute +//! work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::{alloc, channel, event, file, fw, gpu, microseq, mmu, workqueue}; +use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; +use core::mem::MaybeUninit; +use core::sync::atomic::Ordering; +use kernel::bindings; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::io_buffer::IoBufferReader; +use kernel::prelude::*; +use kernel::sync::{smutex::Mutex, Arc}; +use kernel::user_ptr::UserSlicePtr; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; + +#[versions(AGX)] +impl super::Queue::ver { + /// Submit work to a compute queue. + pub(super) fn submit_compute( + &self, + job: &mut Job, + cmd: &bindings::drm_asahi_command, + result_writer: Option, + id: u64, + flush_stamps: bool, + ) -> Result { + if cmd.cmd_type != bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE { + return Err(EINVAL); + } + + let dev = self.dev.data(); + let gpu = match dev.gpu.as_any().downcast_ref::() { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev, "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + mod_dev_dbg!(self.dev, "[Submission {}] Compute!\n", id); + + let mut cmdbuf_reader = unsafe { + UserSlicePtr::new( + cmd.cmd_buffer as usize as *mut _, + core::mem::size_of::(), + ) + .reader() + }; + + let mut cmdbuf: MaybeUninit = MaybeUninit::uninit(); + unsafe { + cmdbuf_reader.read_raw( + cmdbuf.as_mut_ptr() as *mut u8, + core::mem::size_of::(), + )?; + } + let cmdbuf = unsafe { cmdbuf.assume_init() }; + + if cmdbuf.flags != 0 { + return Err(EINVAL); + } + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let notifier = self.notifier.clone(); + + let fence = job.fence.clone(); + let comp_job = job.get_comp()?; + let ev_comp = comp_job.event_info(); + + // TODO: Is this the same on all GPUs? Is this really for preemption? + let preempt_size = 0x7fa0; + let preempt2_off = 0x7f80; + let preempt3_off = 0x7f88; + let preempt4_off = 0x7f90; + let preempt5_off = 0x7f98; + + let preempt_buf = self.ualloc.lock().array_empty(preempt_size)?; + + let mut seq_buf = self.ualloc.lock().array_empty(0x800)?; + for i in 1..0x400 { + seq_buf[i] = (i + 1) as u64; + } + + mod_dev_dbg!( + self.dev, + "[Submission {}] Event #{} {:#x?} -> {:#x?}\n", + id, + ev_comp.slot, + ev_comp.value, + ev_comp.value.next(), + ); + + let timestamps = Arc::try_new(kalloc.shared.new_default::()?)?; + + let uuid = cmdbuf.cmd_id; + + let unk3 = debug_enabled(debug::DebugFlags::Debug3); + + mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); + + // TODO: check + #[ver(V >= V13_0B4)] + let count = self.counter.fetch_add(1, Ordering::Relaxed); + + let comp = GpuObject::new_prealloc( + kalloc.private.alloc_object()?, + |ptr: GpuWeakPointer| { + let mut builder = microseq::Builder::new(); + + let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); + + let start_comp = builder.add(microseq::StartCompute::ver { + header: microseq::op::StartCompute::HEADER, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + job_params1: inner_weak_ptr!(ptr, job_params1), + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x1, + event_generation: self.id as u32, + cmd_seq: U64(ev_comp.cmd_seq), + unk_38: 0x0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_44: 0x0, + uuid, + attachments: common::build_attachments( + cmdbuf.attachments, + cmdbuf.attachment_count, + )?, + padding: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + counter: U64(count), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if result_writer.is_some() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, start_ts), + work_queue: ev_comp.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid, + unk_30_padding: 0, + })?; + } + + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), + })?; + + if result_writer.is_some() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, end_ts), + work_queue: ev_comp.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_comp); + builder.add(microseq::FinalizeCompute::ver { + header: microseq::op::FinalizeCompute::HEADER, + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + #[ver(V < V13_0B4)] + unk_18: 0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_24: 0, + uuid, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + unk_38: 0, + unk_3c: 0, + unk_40: 0, + unk_44: 0, + unk_48: 0, + unk_4c: 0, + unk_50: 0, + unk_54: 0, + unk_58: 0, + #[ver(G == G14 && V < V13_0B4)] + unk_5c_g14: U64(0), + restart_branch_offset: off, + unk_60: unk3.into(), + #[ver(V >= V13_0B4)] + unk_64: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + unk_79: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + Ok(box_in_place!(fw::compute::RunCompute::ver { + notifier: notifier.clone(), + preempt_buf: preempt_buf, + seq_buf: seq_buf, + micro_seq: builder.build(&mut kalloc.private)?, + vm_bind: vm_bind.clone(), + timestamps: timestamps.clone(), + })?) + }, + |inner, ptr| { + Ok(place!( + ptr, + fw::compute::raw::RunCompute::ver { + tag: fw::workqueue::CommandType::RunCompute, + #[ver(V >= V13_0B4)] + counter: U64(count), + unk_4: 0, + vm_slot: vm_bind.slot(), + notifier: inner.notifier.gpu_pointer(), + unk_pointee: Default::default(), + job_params1: fw::compute::raw::JobParameters1 { + preempt_buf1: inner.preempt_buf.gpu_pointer(), + encoder: U64(cmdbuf.encoder_ptr), + // buf2-5 Only if internal program is used + preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), + preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), + preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), + preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), + pipeline_base: U64(0x11_00000000), + unk_38: U64(0x8c60), + unk_40: cmdbuf.ctx_switch_prog, // Internal program addr | 1 + unk_44: 0, + compute_layout_addr: U64(cmdbuf.buffer_descriptor), // Only if internal program used + unk_50: cmdbuf.buffer_descriptor_size, // 0x40 if internal program used + unk_54: 0, + unk_58: 1, + unk_5c: 0, + iogpu_unk_40: cmdbuf.iogpu_unk_40, // 0x1c if internal program used + }, + unk_b8: Default::default(), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + job_params2: fw::compute::raw::JobParameters2::ver { + #[ver(V >= V13_0B4)] + unk_0_0: 0, + unk_0: Default::default(), + preempt_buf1: inner.preempt_buf.gpu_pointer(), + encoder_end: U64(cmdbuf.encoder_end), + unk_34: Default::default(), + #[ver(V < V13_0B4)] + unk_5c: 0, + }, + encoder_params: fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + unk_c: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: cmdbuf.encoder_id, + unk_18: 0x0, // fixed + iogpu_compute_unk44: cmdbuf.iogpu_unk_44, + seq_buffer: inner.seq_buf.gpu_pointer(), + unk_28: U64(0x0), // fixed + }, + meta: fw::job::raw::JobMeta { + unk_4: 0, + stamp: ev_comp.stamp_pointer, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + stamp_slot: ev_comp.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid, + cmd_seq: ev_comp.cmd_seq as u32, + }, + cur_ts: U64(0), + start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), + end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), + unk_2c0: 0, + unk_2c4: 0, + unk_2c8: 0, + unk_2cc: 0, + client_sequence: slot_client_seq, + pad_2d1: Default::default(), + unk_2d4: 0, + unk_2d8: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_2e1: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: U32(0), + #[ver(V >= V13_0B4)] + unk_pad: Default::default(), + } + )) + }, + )?; + + core::mem::drop(alloc); + + fence.add_command(); + comp_job.add_cb(comp, vm_bind.slot(), move |cmd, error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + if let Some(mut rw) = result_writer { + let mut result: bindings::drm_asahi_result_compute = Default::default(); + + cmd.timestamps.with(|raw, _inner| { + result.ts_start = raw.start.load(Ordering::Relaxed); + result.ts_end = raw.end.load(Ordering::Relaxed); + }); + + if let Some(err) = error { + result.info = err.into(); + } else { + result.info.status = bindings::drm_asahi_status_DRM_ASAHI_STATUS_COMPLETE; + } + + rw.write(result); + } + + fence.command_complete(); + })?; + + notifier.threshold.with(|raw, _inner| { + raw.increment(); + }); + + comp_job.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs new file mode 100644 index 00000000000000..ede31b841b1866 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -0,0 +1,717 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(unused_imports)] + +//! Submission queue management +//! +//! This module implements the userspace view of submission queues and the logic to map userspace +//! submissions to firmware queues. + +use kernel::dma_fence::*; +use kernel::io_buffer::IoBufferReader; +use kernel::prelude::*; +use kernel::user_ptr::UserSlicePtr; +use kernel::{ + bindings, c_str, dma_fence, + drm::gem::shmem::VMap, + drm::sched, + macros::versions, + prelude::*, + sync::{smutex::Mutex, Arc, Guard, UniqueArc}, +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::util::*; +use crate::{alloc, buffer, channel, event, file, fw, gem, gpu, microseq, mmu, object, workqueue}; +use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; + +use core::any::Any; +use core::mem::MaybeUninit; +use core::sync::atomic::{AtomicU64, Ordering}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Queue; + +const WQ_SIZE: u32 = 0x500; + +mod common; +mod compute; +mod render; + +/// Trait implemented by all versioned queues. +pub(crate) trait Queue: Send + Sync { + fn submit( + &mut self, + id: u64, + in_syncs: Vec, + out_syncs: Vec, + result_buf: Option, + commands: Vec, + ) -> Result; +} + +#[versions(AGX)] +struct SubQueue { + wq: Arc, +} + +#[versions(AGX)] +impl SubQueue::ver { + fn new_job(&mut self) -> SubQueueJob::ver { + SubQueueJob::ver { + wq: self.wq.clone(), + job: None, + } + } +} + +#[versions(AGX)] +struct SubQueueJob { + wq: Arc, + job: Option, +} + +#[versions(AGX)] +impl SubQueueJob::ver { + fn get(&mut self) -> Result<&mut workqueue::Job::ver> { + if self.job.is_none() { + mod_pr_debug!("SubQueueJob: Creating {:?} job\n", self.wq.pipe_type()); + self.job.replace(self.wq.new_job()?); + } + Ok(self.job.as_mut().expect("expected a Job")) + } + + fn commit(&mut self) -> Result { + match self.job.as_mut() { + Some(job) => job.commit(), + None => Ok(()), + } + } + + fn can_submit(&self) -> bool { + match self.job.as_ref() { + None => true, + Some(job) => job.can_submit(), + } + } +} + +#[versions(AGX)] +pub(crate) struct Queue { + dev: AsahiDevice, + sched: sched::Scheduler, + entity: sched::Entity, + vm: mmu::Vm, + ualloc: Arc>, + q_vtx: Option, + q_frag: Option, + q_comp: Option, + buffer: Option>, + gpu_context: Arc>, + notifier_list: Arc>, + notifier: Arc>, + id: u64, + fence_ctx: FenceContexts, + #[ver(V >= V13_0B4)] + counter: AtomicU64, +} + +#[versions(AGX)] +#[derive(Default)] +pub(crate) struct JobFence { + id: u64, + pending: AtomicU64, +} + +#[versions(AGX)] +impl JobFence::ver { + fn add_command(self: &FenceObject) { + self.pending.fetch_add(1, Ordering::Relaxed); + } + + fn command_complete(self: &FenceObject) { + let remain = self.pending.fetch_sub(1, Ordering::Relaxed) - 1; + mod_pr_debug!( + "JobFence[{}]: Command complete (remain: {})\n", + self.id, + remain + ); + if remain == 0 { + mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); + if self.signal().is_err() { + pr_err!("JobFence[{}]: Fence signal failed\n", self.id); + } + } + } +} + +#[versions(AGX)] +#[vtable] +impl dma_fence::FenceOps for JobFence::ver { + const USE_64BIT_SEQNO: bool = true; + + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("queue") + } +} + +#[versions(AGX)] +pub(crate) struct QueueJob { + dev: AsahiDevice, + vm_bind: mmu::VmBind, + op_guard: Option, + sj_vtx: Option, + sj_frag: Option, + sj_comp: Option, + fence: UserFence, +} + +#[versions(AGX)] +impl QueueJob::ver { + fn get_vtx(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_vtx.as_mut().ok_or(EINVAL)?.get() + } + fn get_frag(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_frag.as_mut().ok_or(EINVAL)?.get() + } + fn get_comp(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_comp.as_mut().ok_or(EINVAL)?.get() + } + + fn commit(&mut self) -> Result { + mod_dev_dbg!(self.dev, "QueueJob: Committing\n"); + + self.sj_vtx.as_mut().map(|a| a.commit()).unwrap_or(Ok(()))?; + self.sj_frag + .as_mut() + .map(|a| a.commit()) + .unwrap_or(Ok(()))?; + self.sj_comp.as_mut().map(|a| a.commit()).unwrap_or(Ok(())) + } +} + +#[versions(AGX)] +impl sched::JobImpl for QueueJob::ver { + fn can_run(job: &mut sched::Job) -> bool { + mod_dev_dbg!(job.dev, "QueueJob: Checking runnability\n"); + + if let Some(sj) = job.sj_vtx.as_ref() { + if !sj.can_submit() { + mod_dev_dbg!(job.dev, "QueueJob: Blocking due to vertex queue full\n"); + return false; + } + } + if let Some(sj) = job.sj_frag.as_ref() { + if !sj.can_submit() { + mod_dev_dbg!(job.dev, "QueueJob: Blocking due to fragment queue full\n"); + return false; + } + } + if let Some(sj) = job.sj_comp.as_ref() { + if !sj.can_submit() { + mod_dev_dbg!(job.dev, "QueueJob: Blocking due to compute queue full\n"); + return false; + } + } + true + } + + #[allow(unused_assignments)] + fn run(job: &mut sched::Job) -> Result> { + mod_dev_dbg!(job.dev, "QueueJob: Running Job\n"); + + let dev = job.dev.data(); + let gpu = match dev + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(job.dev, "GpuManager mismatched with QueueJob!\n"); + return Err(EIO); + } + }; + + if job.op_guard.is_none() { + job.op_guard = Some(gpu.start_op()?); + } + + // First submit all the commands for each queue. This can fail. + + let mut frag_job = None; + let mut frag_sub = None; + if let Some(sj) = job.sj_frag.as_mut() { + frag_job = sj.job.take(); + if let Some(wqjob) = frag_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob: Submit fragment\n"); + frag_sub = Some(wqjob.submit()?); + } + } + + let mut vtx_job = None; + let mut vtx_sub = None; + if let Some(sj) = job.sj_vtx.as_mut() { + vtx_job = sj.job.take(); + if let Some(wqjob) = vtx_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob: Submit vertex\n"); + vtx_sub = Some(wqjob.submit()?); + } + } + + let mut comp_job = None; + let mut comp_sub = None; + if let Some(sj) = job.sj_comp.as_mut() { + comp_job = sj.job.take(); + if let Some(wqjob) = comp_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob: Submit compute\n"); + comp_sub = Some(wqjob.submit()?); + } + } + + // Now we fully commit to running the job + mod_dev_dbg!(job.dev, "QueueJob: Run fragment\n"); + frag_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob: Run vertex\n"); + vtx_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob: Run compute\n"); + comp_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob: Drop compute job\n"); + core::mem::drop(comp_job); + mod_dev_dbg!(job.dev, "QueueJob: Drop vertex job\n"); + core::mem::drop(vtx_job); + mod_dev_dbg!(job.dev, "QueueJob: Drop fragment job\n"); + core::mem::drop(frag_job); + + Ok(Some(Fence::from_fence(&job.fence))) + } + + fn timed_out(job: &mut sched::Job) -> sched::Status { + // FIXME: Handle timeouts properly + dev_err!( + job.dev, + "Job timed out on the DRM scheduler, things will probably break\n" + ); + sched::Status::NoDevice + } +} + +#[versions(AGX)] +impl Drop for QueueJob::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "QueueJob {}: Dropping\n", self.id); + } +} + +struct ResultWriter { + vmap: VMap, + offset: usize, + len: usize, +} + +impl ResultWriter { + fn write(&mut self, mut value: T) { + let p: *mut u8 = &mut value as *mut _ as *mut u8; + // SAFETY: We know `p` points to a type T of that size, and UAPI types must have + // no padding and all bit patterns valid. + let slice = unsafe { core::slice::from_raw_parts_mut(p, core::mem::size_of::()) }; + let len = slice.len().min(self.len); + self.vmap.as_mut_slice()[self.offset..self.offset + len].copy_from_slice(&slice[..len]); + } +} + +static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_CLASS_KEY: kernel::sync::LockClassKey = kernel::sync::LockClassKey::new(); + +#[versions(AGX)] +impl Queue::ver { + /// Create a new user queue. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: mmu::Vm, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + event_manager: Arc, + mgr: &buffer::BufferManager, + id: u64, + priority: u32, + caps: u32, + ) -> Result { + mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); + + let data = dev.data(); + + let gpu_context: Arc> = Arc::try_new( + alloc + .shared + .new_object(Default::default(), |_inner| Default::default())?, + )?; + + let mut notifier_list = alloc.private.new_default::()?; + + let self_ptr = notifier_list.weak_pointer(); + notifier_list.with_mut(|raw, _inner| { + raw.list_head.next = Some(inner_weak_ptr!(self_ptr, list_head)); + }); + + let threshold = alloc.shared.new_default::()?; + + let notifier: Arc> = + Arc::try_new(alloc.private.new_inplace( + fw::event::Notifier::ver { threshold }, + |inner, ptr: &mut MaybeUninit>| { + Ok(place!( + ptr, + fw::event::raw::Notifier::ver { + threshold: inner.threshold.gpu_pointer(), + generation: AtomicU32::new(id as u32), + cur_count: AtomicU32::new(0), + unk_10: AtomicU32::new(0x50), + state: Default::default() + } + )) + }, + )?)?; + + let sched = sched::Scheduler::new(dev, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; + // Priorities are handled by the AGX scheduler, there is no meaning within a + // per-queue scheduler. + let entity = sched::Entity::new(&sched, sched::Priority::Normal)?; + + let mut ret = Queue::ver { + dev: dev.clone(), + sched, + entity, + vm, + ualloc, + q_vtx: None, + q_frag: None, + q_comp: None, + buffer: None, + gpu_context, + notifier_list: Arc::try_new(notifier_list)?, + notifier, + id, + fence_ctx: FenceContexts::new(1, QUEUE_NAME, &QUEUE_CLASS_KEY)?, + #[ver(V >= V13_0B4)] + counter: AtomicU64::new(0), + }; + + // Rendering structures + if caps & bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER != 0 { + let buffer = + buffer::Buffer::ver::new(&*data.gpu, alloc, ret.ualloc.clone(), ualloc_priv, mgr)?; + let tvb_blocks = { + let lock = crate::THIS_MODULE.kernel_param_lock(); + *crate::initial_tvb_size.read(&lock) + }; + + buffer.ensure_blocks(tvb_blocks)?; + + ret.buffer = Some(Mutex::new(buffer)); + ret.q_vtx = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + alloc, + event_manager.clone(), + ret.gpu_context.clone(), + ret.notifier_list.clone(), + channel::PipeType::Vertex, + id, + priority, + WQ_SIZE, + )?, + }); + } + + // Rendering & blit structures + if caps + & (bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER + | bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_BLIT) + != 0 + { + ret.q_frag = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + alloc, + event_manager.clone(), + ret.gpu_context.clone(), + ret.notifier_list.clone(), + channel::PipeType::Fragment, + id, + priority, + WQ_SIZE, + )?, + }); + } + + // Compute structures + if caps & bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE != 0 { + ret.q_comp = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + alloc, + event_manager, + ret.gpu_context.clone(), + ret.notifier_list.clone(), + channel::PipeType::Compute, + id, + priority, + WQ_SIZE, + )?, + }); + } + + mod_dev_dbg!(dev, "[Queue {}] Queue created\n", id); + Ok(ret) + } +} + +const SQ_RENDER: usize = bindings::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_RENDER as usize; +const SQ_COMPUTE: usize = bindings::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_COMPUTE as usize; +const SQ_COUNT: usize = bindings::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_COUNT as usize; + +#[versions(AGX)] +impl Queue for Queue::ver { + fn submit( + &mut self, + id: u64, + in_syncs: Vec, + out_syncs: Vec, + result_buf: Option, + commands: Vec, + ) -> Result { + let dev = self.dev.data(); + let gpu = match dev + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(self.dev, "GpuManager mismatched with JobImpl!\n"); + return Err(EIO); + } + }; + + mod_dev_dbg!(self.dev, "[Submission {}] Submit job\n", id); + + if gpu.is_crashed() { + dev_err!( + self.dev, + "[Submission {}] GPU is crashed, cannot submit\n", + id + ); + return Err(ENODEV); + } + + // Empty submissions are not legal + if commands.is_empty() { + return Err(EINVAL); + } + + let op_guard = if !in_syncs.is_empty() { + Some(gpu.start_op()?) + } else { + None + }; + + let mut events: [Vec>; SQ_COUNT] = + Default::default(); + + events[SQ_RENDER].try_push(self.q_frag.as_ref().and_then(|a| a.wq.event_info()))?; + events[SQ_COMPUTE].try_push(self.q_comp.as_ref().and_then(|a| a.wq.event_info()))?; + + let vm_bind = gpu.bind_vm(&self.vm)?; + let vm_slot = vm_bind.slot(); + + mod_dev_dbg!(self.dev, "[Submission {}] Creating job\n", id); + let mut job = self.entity.new_job(QueueJob::ver { + dev: self.dev.clone(), + vm_bind, + op_guard, + sj_vtx: self.q_vtx.as_mut().map(|a| a.new_job()), + sj_frag: self.q_frag.as_mut().map(|a| a.new_job()), + sj_comp: self.q_comp.as_mut().map(|a| a.new_job()), + fence: self + .fence_ctx + .new_fence::( + 0, + JobFence::ver { + id, + pending: Default::default(), + }, + )? + .into(), + })?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Adding {} in_syncs\n", + id, + in_syncs.len() + ); + for sync in in_syncs { + job.add_dependency(sync.fence.expect("in_sync missing fence"))?; + } + + let mut last_render = None; + let mut last_compute = None; + + for (i, cmd) in commands.iter().enumerate() { + match cmd.cmd_type { + bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => last_render = Some(i), + bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => last_compute = Some(i), + _ => return Err(EINVAL), + } + } + + mod_dev_dbg!( + self.dev, + "[Submission {}] Submitting {} commands\n", + id, + commands.len() + ); + for (i, cmd) in commands.into_iter().enumerate() { + for (queue_idx, index) in cmd.barriers.iter().enumerate() { + if *index == bindings::DRM_ASAHI_BARRIER_NONE as u32 { + continue; + } + if let Some(event) = events[queue_idx].get(*index as usize).ok_or(EINVAL)? { + let mut alloc = gpu.alloc(); + let queue_job = match cmd.cmd_type { + bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, + bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, + _ => return Err(EINVAL), + }; + mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); + let barrier: GpuObject = alloc.private.new_inplace( + Default::default(), + |_inner, ptr: &mut MaybeUninit| { + Ok(place!( + ptr, + fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: event.fw_stamp_pointer, + wait_value: event.value, + wait_slot: event.slot, + stamp_self: queue_job.event_info().value.next(), + uuid: 0xffffbbbb, + unk: 0, + } + )) + }, + )?; + mod_dev_dbg!(self.dev, "[Submission {}] Add Explicit Barrier\n", id); + queue_job.add(barrier, vm_slot)?; + } else { + assert!(*index == 0); + } + } + + let result_writer = match result_buf.as_ref() { + None => { + if cmd.result_offset != 0 || cmd.result_size != 0 { + return Err(EINVAL); + } + None + } + Some(buf) => { + if cmd.result_size != 0 { + if cmd + .result_offset + .checked_add(cmd.result_size) + .ok_or(EINVAL)? + > buf.size() as u64 + { + return Err(EINVAL); + } + Some(ResultWriter { + vmap: buf.gem.vmap()?, + offset: cmd.result_offset.try_into()?, + len: cmd.result_size.try_into()?, + }) + } else { + None + } + } + }; + + match cmd.cmd_type { + bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + self.submit_render( + &mut job, + &cmd, + result_writer, + id, + last_render.unwrap() == i, + )?; + events[SQ_RENDER].try_push(Some( + job.sj_frag + .as_ref() + .expect("No frag queue?") + .job + .as_ref() + .expect("No frag job?") + .event_info(), + ))?; + } + bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + self.submit_compute( + &mut job, + &cmd, + result_writer, + id, + last_compute.unwrap() == i, + )?; + events[SQ_COMPUTE].try_push(Some( + job.sj_comp + .as_ref() + .expect("No comp queue?") + .job + .as_ref() + .expect("No comp job?") + .event_info(), + ))?; + } + _ => return Err(EINVAL), + } + } + + mod_dev_dbg!(self.dev, "Queue: Committing job\n"); + job.commit()?; + + mod_dev_dbg!(self.dev, "Queue: Arming job\n"); + let job = job.arm(); + let out_fence = job.fences().finished(); + mod_dev_dbg!(self.dev, "Queue: Pushing job\n"); + job.push(); + + mod_dev_dbg!(self.dev, "Queue: Adding {} out_syncs\n", out_syncs.len()); + for sync in out_syncs { + sync.syncobj.replace_fence(Some(&out_fence)); + } + + Ok(()) + } +} + +#[versions(AGX)] +impl Drop for Queue::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.id); + let dev = self.dev.data(); + if dev.gpu.invalidate_context(&self.gpu_context).is_err() { + dev_err!(self.dev, "Queue::drop: Failed to invalidate GPU context!\n"); + } + } +} diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs new file mode 100644 index 00000000000000..6a0a2a84a040ec --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -0,0 +1,1181 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] +#![allow(unused_imports)] + +//! Render work queue. +//! +//! A render queue consists of two underlying WorkQueues, one for vertex and one for fragment work. +//! This module is in charge of creating all of the firmware structures required to submit 3D +//! rendering work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::util::*; +use crate::workqueue::WorkError; +use crate::{alloc, buffer, channel, event, file, fw, gpu, microseq, mmu, workqueue}; +use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; +use core::mem::MaybeUninit; +use core::sync::atomic::Ordering; +use kernel::bindings; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::gem::shmem::VMap; +use kernel::drm::sched::Job; +use kernel::io_buffer::IoBufferReader; +use kernel::prelude::*; +use kernel::sync::{smutex::Mutex, Arc}; +use kernel::user_ptr::UserSlicePtr; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Render; + +/// Tiling/Vertex control bit to disable using more than one GPU cluster. This results in decreased +/// throughput but also less latency, which is probably desirable for light vertex loads where the +/// overhead of clustering/merging would exceed the time it takes to just run the job on one +/// cluster. +const TILECTL_DISABLE_CLUSTERING: u32 = 1u32 << 0; + +struct RenderResult { + result: bindings::drm_asahi_result_render, + vtx_complete: bool, + frag_complete: bool, + vtx_error: Option, + frag_error: Option, + writer: super::ResultWriter, +} + +impl RenderResult { + fn commit(&mut self) { + if !self.vtx_complete || !self.frag_complete { + return; + } + + let mut error = self.vtx_error.take(); + if let Some(frag_error) = self.frag_error.take() { + if error.is_none() || error == Some(WorkError::Killed) { + error = Some(frag_error); + } + } + + if let Some(err) = error { + self.result.info = err.into(); + } else { + self.result.info.status = bindings::drm_asahi_status_DRM_ASAHI_STATUS_COMPLETE; + } + + self.writer.write(self.result); + } +} + +#[versions(AGX)] +impl super::Queue::ver { + /// Get the appropriate tiling parameters for a given userspace command buffer. + fn get_tiling_params( + cmdbuf: &bindings::drm_asahi_cmd_render, + num_clusters: u32, + ) -> Result { + let width: u32 = cmdbuf.fb_width; + let height: u32 = cmdbuf.fb_height; + let layers: u32 = cmdbuf.layers; + + if width > 65536 || height > 65536 { + return Err(EINVAL); + } + + if layers == 0 || layers > 2048 { + return Err(EINVAL); + } + + let tile_width = 32u32; + let tile_height = 32u32; + + let utile_width = cmdbuf.utile_width; + let utile_height = cmdbuf.utile_height; + + match (utile_width, utile_height) { + (32, 32) | (32, 16) | (16, 16) => (), + _ => return Err(EINVAL), + }; + + let utiles_per_tile_x = tile_width / utile_width; + let utiles_per_tile_y = tile_height / utile_height; + + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + + let tiles_x = (width + tile_width - 1) / tile_width; + let tiles_y = (height + tile_height - 1) / tile_height; + let tiles = tiles_x * tiles_y; + + let mtiles_x = 4u32; + let mtiles_y = 4u32; + let mtiles = mtiles_x * mtiles_y; + + // TODO: *samples + let tiles_per_mtile_x = align(div_ceil(tiles_x, mtiles_x), 4); + let tiles_per_mtile_y = align(div_ceil(tiles_y, mtiles_y), 4); + let tiles_per_mtile = tiles_per_mtile_x * tiles_per_mtile_y; + + let mtile_x1 = tiles_per_mtile_x; + let mtile_x2 = 2 * tiles_per_mtile_x; + let mtile_x3 = 3 * tiles_per_mtile_x; + + let mtile_y1 = tiles_per_mtile_y; + let mtile_y2 = 2 * tiles_per_mtile_y; + let mtile_y3 = 3 * tiles_per_mtile_y; + + let rgn_entry_size = 5; + // Macrotile stride in 32-bit words + let rgn_size = align(rgn_entry_size * tiles_per_mtile * utiles_per_tile, 4) / 4; + let tilemap_size = (4 * rgn_size * mtiles * layers) as usize; + + let tpc_entry_size = 8; + // TPC stride in 32-bit words + let tpc_mtile_stride = tpc_entry_size * utiles_per_tile * tiles_per_mtile / 4; + let tpc_size = (num_clusters * (4 * tpc_mtile_stride * mtiles) * layers) as usize; + + // No idea where this comes from, but it fits what macOS does... + // TODO: layers? + let meta1_blocks = if num_clusters > 1 { + div_ceil(align(tiles_x, 2) * align(tiles_y, 4), 0x1980) + } else { + 0 + }; + + let min_tvb_blocks = + div_ceil(tiles_x * tiles_y, 128).max(if num_clusters > 1 { 9 } else { 8 }) as usize; + + // Sometimes clustering seems to use twice the cluster tilemap count + // and twice the meta4 size. TODO: Is this random or can we calculate + // it somehow??? Does it go higher??? + let cluster_factor = 2; + + Ok(buffer::TileInfo { + tiles_x, + tiles_y, + tiles, + utile_width, + utile_height, + //mtiles_x, + //mtiles_y, + tiles_per_mtile_x, + tiles_per_mtile_y, + //tiles_per_mtile, + utiles_per_mtile_x: tiles_per_mtile_x * utiles_per_tile_x, + utiles_per_mtile_y: tiles_per_mtile_y * utiles_per_tile_y, + //utiles_per_mtile: tiles_per_mtile * utiles_per_tile, + tilemap_size, + tpc_size, + meta1_blocks, + min_tvb_blocks, + cluster_factor, + params: fw::vertex::raw::TilingParameters { + rgn_size, + unk_4: 0x88, + ppp_ctrl: cmdbuf.ppp_ctrl, + x_max: (width - 1) as u16, + y_max: (height - 1) as u16, + te_screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + te_mtile1: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + te_mtile2: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + tiles_per_mtile, + tpc_stride: tpc_mtile_stride, + unk_24: 0x100, + unk_28: if layers > 1 { + 0xe000 | (layers - 1) + } else { + 0x8000 + }, + }, + }) + } + + /// Submit work to a render queue. + pub(super) fn submit_render( + &self, + job: &mut Job, + cmd: &bindings::drm_asahi_command, + result_writer: Option, + id: u64, + flush_stamps: bool, + ) -> Result { + if cmd.cmd_type != bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { + return Err(EINVAL); + } + + mod_dev_dbg!(self.dev, "[Submission {}] Render!\n", id); + + let mut cmdbuf_reader = unsafe { + UserSlicePtr::new( + cmd.cmd_buffer as usize as *mut _, + core::mem::size_of::(), + ) + .reader() + }; + + let mut cmdbuf: MaybeUninit = MaybeUninit::uninit(); + unsafe { + cmdbuf_reader.read_raw( + cmdbuf.as_mut_ptr() as *mut u8, + core::mem::size_of::(), + )?; + } + let cmdbuf = unsafe { cmdbuf.assume_init() }; + + if cmdbuf.flags + & !(bindings::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES + | bindings::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S + | bindings::ASAHI_RENDER_MEMORYLESS_RTS_USED + | bindings::ASAHI_RENDER_PROCESS_EMPTY_TILES + | bindings::ASAHI_RENDER_NO_VERTEX_CLUSTERING) as u64 + != 0 + { + return Err(EINVAL); + } + + if cmdbuf.flags & bindings::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 != 0 { + // Not supported yet + return Err(EINVAL); + } + + if cmdbuf.fb_width == 0 + || cmdbuf.fb_height == 0 + || cmdbuf.fb_width > 16384 + || cmdbuf.fb_height > 16384 + { + mod_dev_dbg!( + self.dev, + "[Submission {}] Invalid dimensions {}x{}\n", + id, + cmdbuf.fb_width, + cmdbuf.fb_height + ); + return Err(EINVAL); + } + + let dev = self.dev.data(); + let gpu = match dev.gpu.as_any().downcast_ref::() { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev, "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let nclusters = gpu.get_dyncfg().id.num_clusters; + + // Can be set to false to disable clustering (for simpler jobs), but then the + // core masks below should be adjusted to cover a single rolling cluster. + let mut clustering = nclusters > 1; + + if debug_enabled(debug::DebugFlags::DisableClustering) + || cmdbuf.flags & bindings::ASAHI_RENDER_NO_VERTEX_CLUSTERING as u64 != 0 + { + clustering = false; + } + + #[ver(G < G14)] + let tiling_control = { + let render_cfg = gpu.get_cfg().render; + let mut tiling_control = render_cfg.tiling_control; + + if !clustering { + tiling_control |= TILECTL_DISABLE_CLUSTERING; + } + tiling_control + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + + let buffer = self.buffer.as_ref().ok_or(EINVAL)?.lock(); + + let scene = Arc::try_new(buffer.new_scene(kalloc, &tile_info)?)?; + + let notifier = self.notifier.clone(); + + let tvb_autogrown = buffer.auto_grow()?; + if tvb_autogrown { + let new_size = buffer.block_count() as usize; + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to overflows\n", + id, + new_size * buffer::BLOCK_SIZE, + new_size, + ); + } + + let tvb_grown = buffer.ensure_blocks(tile_info.min_tvb_blocks)?; + if tvb_grown { + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to dimensions ({}x{})\n", + id, + tile_info.min_tvb_blocks * buffer::BLOCK_SIZE, + tile_info.min_tvb_blocks, + cmdbuf.fb_width, + cmdbuf.fb_height + ); + } + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let ev_vtx = job.get_vtx()?.event_info(); + let ev_frag = job.get_frag()?.event_info(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert event #{} -> {:#x?}\n", + id, + ev_vtx.slot, + ev_vtx.value.next(), + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag event #{} -> {:#x?}\n", + id, + ev_frag.slot, + ev_frag.value.next(), + ); + + let uuid_3d = cmdbuf.cmd_3d_id; + let uuid_ta = cmdbuf.cmd_ta_id; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert UUID = {:#x?}\n", + id, + uuid_ta + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag UUID = {:#x?}\n", + id, + uuid_3d + ); + + let fence = job.fence.clone(); + let frag_job = job.get_frag()?; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Barrier\n", id); + let barrier: GpuObject = kalloc.private.new_inplace( + Default::default(), + |_inner, ptr: &mut MaybeUninit| { + Ok(place!( + ptr, + fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: ev_vtx.fw_stamp_pointer, + wait_value: ev_vtx.value.next(), + wait_slot: ev_vtx.slot, + stamp_self: ev_frag.value.next(), + uuid: uuid_3d, + unk: 0, + } + )) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Barrier\n", id); + frag_job.add(barrier, vm_bind.slot())?; + + let timestamps = Arc::try_new(kalloc.shared.new_default::()?)?; + + let unk1 = debug_enabled(debug::DebugFlags::Debug1); + let unk2 = debug_enabled(debug::DebugFlags::Debug2); + let unk3 = debug_enabled(debug::DebugFlags::Debug3); + + let mut tile_config: u64 = 0; + if !unk1 { + tile_config |= 0x280; + } + if cmdbuf.layers > 1 { + tile_config |= 1; + } + if cmdbuf.flags & bindings::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 != 0 { + tile_config |= 0x10000; + } + + let mut utile_config = + ((tile_info.utile_width / 16) << 12) | ((tile_info.utile_height / 16) << 14); + utile_config |= match cmdbuf.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(EINVAL), + }; + + let frag_result = result_writer + .map(|writer| { + let mut result = RenderResult { + result: Default::default(), + vtx_complete: false, + frag_complete: false, + vtx_error: None, + frag_error: None, + writer, + }; + + if tvb_autogrown { + result.result.flags |= bindings::DRM_ASAHI_RESULT_RENDER_TVB_GROW_OVF as u64; + } + if tvb_grown { + result.result.flags |= bindings::DRM_ASAHI_RESULT_RENDER_TVB_GROW_MIN as u64; + } + result.result.tvb_size_bytes = buffer.size() as u64; + + Arc::try_new(Mutex::new(result)) + }) + .transpose()?; + + let vtx_result = frag_result.clone(); + + // TODO: check + #[ver(V >= V13_0B4)] + let count_frag = self.counter.fetch_add(2, Ordering::Relaxed); + #[ver(V >= V13_0B4)] + let count_vtx = count_frag + 1; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); + let frag = GpuObject::new_prealloc( + kalloc.private.alloc_object()?, + |ptr: GpuWeakPointer| { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), + stats + ); + + let start_frag = builder.add(microseq::StartFragment::ver { + header: microseq::op::StartFragment::HEADER, + job_params2: inner_weak_ptr!(ptr, job_params2), + job_params1: inner_weak_ptr!(ptr, job_params1), + scene: scene.gpu_pointer(), + stats, + busy_flag: inner_weak_ptr!(ptr, busy_flag), + tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_50: 0x1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + unk_5c: 0, + cmd_seq: U64(ev_frag.cmd_seq), + unk_68: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + unk_7c: 0, + unk_80: 0, + unk_84: 0, + uuid: uuid_3d, + attachments: common::build_attachments( + cmdbuf.attachments, + cmdbuf.attachment_count, + )?, + unk_190: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if frag_result.is_some() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, start_ts), + work_queue: ev_frag.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), + })?; + + if frag_result.is_some() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, end_ts), + work_queue: ev_frag.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_frag); + builder.add(microseq::FinalizeFragment::ver { + header: microseq::op::FinalizeFragment::HEADER, + uuid: uuid_3d, + unk_8: 0, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + unk_18: 0, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + unk_2c: U64(1), + stats, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + busy_flag: inner_weak_ptr!(ptr, busy_flag), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_60: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + unk_6c: U64(0), + unk_74: U64(0), + unk_7c: U64(0), + unk_84: U64(0), + unk_8c: U64(0), + #[ver(G == G14 && V < V13_0B4)] + unk_8c_g14: U64(0), + restart_branch_offset: off, + unk_98: unk3.into(), + #[ver(V >= V13_0B4)] + unk_9c: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + Ok(box_in_place!(fw::fragment::RunFragment::ver { + notifier: notifier.clone(), + scene: scene.clone(), + micro_seq: builder.build(&mut kalloc.private)?, + vm_bind: vm_bind.clone(), + aux_fb: self.ualloc.lock().array_empty(0x8000)?, + timestamps: timestamps.clone(), + })?) + }, + |inner, ptr| { + let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { + iogpu_unk_214: cmdbuf.iogpu_unk_214, + unk2: 0, + width: cmdbuf.fb_width, + height: cmdbuf.fb_height, + #[ver(V >= V13_0B4)] + unk3: U64(0x100000), + }; + + Ok(place!( + ptr, + fw::fragment::raw::RunFragment::ver { + tag: fw::workqueue::CommandType::RunFragment, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + vm_slot: vm_bind.slot(), + unk_8: 0, + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + notifier: inner.notifier.gpu_pointer(), + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + samples: cmdbuf.samples, + tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, + tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, + unk_50: U64(0), + unk_58: U64(0), + merge_upper_x: F32::from_bits(cmdbuf.merge_upper_x), + merge_upper_y: F32::from_bits(cmdbuf.merge_upper_y), + unk_68: U64(0), + tile_count: U64(tile_info.tiles as u64), + job_params1: fw::fragment::raw::JobParameters1::ver { + utile_config: utile_config, + unk_4: 0, + clear_pipeline: fw::fragment::raw::ClearPipelineBinding { + pipeline_bind: U64(cmdbuf.load_pipeline_bind as u64), + address: U64(cmdbuf.load_pipeline as u64), + }, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + scissor_array: U64(cmdbuf.scissor_array), + depth_bias_array: U64(cmdbuf.depth_bias_array), + aux_fb_info: aux_fb_info, + depth_dimensions: U64(cmdbuf.depth_dimensions as u64), + visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), + zls_ctrl: U64(cmdbuf.zls_ctrl), + #[ver(G >= G14)] + unk_58_g14_0: U64(0x4040404), + #[ver(G >= G14)] + unk_58_g14_8: U64(0), + depth_buffer_ptr1: U64(cmdbuf.depth_buffer_1), + depth_buffer_ptr2: U64(cmdbuf.depth_buffer_2), + stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_1), + stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_2), + #[ver(G >= G14)] + unk_68_g14_0: Default::default(), + unk_78: Default::default(), + depth_meta_buffer_ptr1: U64(cmdbuf.depth_meta_buffer_1), + unk_a0: Default::default(), + depth_meta_buffer_ptr2: U64(cmdbuf.depth_meta_buffer_2), + unk_b0: Default::default(), + stencil_meta_buffer_ptr1: U64(cmdbuf.stencil_meta_buffer_1), + unk_c0: Default::default(), + stencil_meta_buffer_ptr2: U64(cmdbuf.stencil_meta_buffer_2), + unk_d0: Default::default(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), + mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + tvb_heapmeta_2: inner.scene.tvb_heapmeta_pointer(), + tile_config: U64(tile_config), + aux_fb: inner.aux_fb.gpu_pointer(), + unk_108: Default::default(), + pipeline_base: U64(0x11_00000000), + unk_140: U64(0x8c60), + unk_148: U64(0x0), + unk_150: U64(0x0), + unk_158: U64(0x1c), + unk_160: U64(0), + unk_168_padding: Default::default(), + #[ver(V < V13_0B4)] + __pad0: Default::default(), + }, + job_params2: fw::fragment::raw::JobParameters2 { + store_pipeline_bind: cmdbuf.store_pipeline_bind, + store_pipeline_addr: cmdbuf.store_pipeline, + unk_8: 0x0, + unk_c: 0x0, + merge_upper_x: F32::from_bits(cmdbuf.merge_upper_x), + merge_upper_y: F32::from_bits(cmdbuf.merge_upper_y), + unk_18: U64(0x0), + utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, + utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, + unk_24: 0x0, + tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), + iogpu_unk_212: cmdbuf.iogpu_unk_212, + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + // TODO: does this flag need to be exposed to userspace? + isp_bgobjvals: cmdbuf.isp_bgobjvals | 0x400, + unk_38: 0x0, + unk_3c: 0x1, + unk_40: 0, + }, + job_params3: fw::fragment::raw::JobParameters3::ver { + unk_44_padding: Default::default(), + depth_bias_array: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.depth_bias_array), + unk_padding: U64(0), + }, + scissor_array: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.scissor_array), + unk_padding: U64(0), + }, + visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), + unk_118: U64(0x0), + unk_120: Default::default(), + unk_reload_pipeline: fw::fragment::raw::ClearPipelineBinding { + pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), + address: U64(cmdbuf.partial_reload_pipeline as u64), + }, + unk_258: U64(0), + unk_260: U64(0), + unk_268: U64(0), + unk_270: U64(0), + reload_pipeline: fw::fragment::raw::ClearPipelineBinding { + pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), + address: U64(cmdbuf.partial_reload_pipeline as u64), + }, + zls_ctrl: U64(cmdbuf.zls_ctrl), + unk_290: U64(0x0), + depth_buffer_ptr1: U64(cmdbuf.depth_buffer_1), + unk_2a0: U64(0x0), + unk_2a8: U64(0x0), + depth_buffer_ptr2: U64(cmdbuf.depth_buffer_2), + depth_buffer_ptr3: U64(cmdbuf.depth_buffer_3), + depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_3), + stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_1), + unk_2d0: U64(0x0), + unk_2d8: U64(0x0), + stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_2), + stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_3), + stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_3), + unk_2f8: Default::default(), + iogpu_unk_212: cmdbuf.iogpu_unk_212, + unk_30c: 0x0, + aux_fb_info: aux_fb_info, + unk_320_padding: Default::default(), + unk_partial_store_pipeline: + fw::fragment::raw::StorePipelineBinding::new( + cmdbuf.partial_store_pipeline_bind, + cmdbuf.partial_store_pipeline + ), + partial_store_pipeline: fw::fragment::raw::StorePipelineBinding::new( + cmdbuf.partial_store_pipeline_bind, + cmdbuf.partial_store_pipeline + ), + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + isp_bgobjvals: cmdbuf.isp_bgobjvals, + iogpu_unk_49: cmdbuf.iogpu_unk_49, + unk_37c: 0x0, + unk_380: U64(0x0), + unk_388: U64(0x0), + #[ver(V >= V13_0B4)] + unk_390_0: U64(0x0), + depth_dimensions: U64(cmdbuf.depth_dimensions as u64), + }, + unk_758_flag: 0, + unk_75c_flag: 0, + unk_buf: Default::default(), + busy_flag: 0, + tvb_overflow_count: 0, + unk_878: 0, + encoder_params: fw::job::raw::EncoderParams { + unk_8: (cmdbuf.flags + & bindings::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 + != 0) as u32, + unk_c: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: cmdbuf.encoder_id, + unk_18: 0x0, // fixed + iogpu_compute_unk44: 0xffffffff, + seq_buffer: inner.scene.seq_buf_pointer(), + unk_28: U64(0x0), // fixed + }, + process_empty_tiles: (cmdbuf.flags + & bindings::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 + != 0) as u32, + no_clear_pipeline_textures: (cmdbuf.flags + & bindings::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES as u64 + != 0) as u32, + unk_param: unk2.into(), // 1 for boot stuff? + unk_pointee: 0, + meta: fw::job::raw::JobMeta { + unk_4: 0, + stamp: ev_frag.stamp_pointer, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + stamp_slot: ev_frag.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_3d, + cmd_seq: ev_frag.cmd_seq as u32, + }, + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + unk_buf_10: U64(1), + cur_ts: U64(0), + start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), + end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), + unk_914: 0, + unk_918: U64(0), + unk_920: 0, + client_sequence: slot_client_seq, + pad_925: Default::default(), + #[ver(V >= V13_0B4)] + unk_928_0: 0, + #[ver(V >= V13_0B4)] + unk_928_4: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_928_d: Default::default(), + } + )) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); + fence.add_command(); + + frag_job.add_cb(frag, vm_bind.slot(), move |cmd, error| { + if let Some(err) = error { + fence.set_error(err.into()); + } + if let Some(mut res) = frag_result.as_ref().map(|a| a.lock()) { + cmd.timestamps.with(|raw, _inner| { + res.result.fragment_ts_start = raw.frag.start.load(Ordering::Relaxed); + res.result.fragment_ts_end = raw.frag.end.load(Ordering::Relaxed); + }); + cmd.with(|raw, _inner| { + res.result.num_tvb_overflows = raw.tvb_overflow_count; + }); + res.frag_error = error; + res.frag_complete = true; + res.commit(); + } + fence.command_complete(); + })?; + + let fence = job.fence.clone(); + let vtx_job = job.get_vtx()?; + + if scene.rebind() || tvb_grown || tvb_autogrown { + mod_dev_dbg!(self.dev, "[Submission {}] Create Bind Buffer\n", id); + let bind_buffer = kalloc.private.new_inplace( + fw::buffer::InitBuffer::ver { + scene: scene.clone(), + }, + |inner, ptr: &mut MaybeUninit>| { + Ok(place!( + ptr, + fw::buffer::raw::InitBuffer::ver { + tag: fw::workqueue::CommandType::InitBuffer, + vm_slot: vm_bind.slot(), + buffer_slot: inner.scene.slot(), + unk_c: 0, + block_count: buffer.block_count(), + buffer: inner.scene.buffer_pointer(), + stamp_value: ev_vtx.value.next(), + } + )) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Bind Buffer\n", id); + vtx_job.add(bind_buffer, vm_bind.slot())?; + } + + mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); + let vtx = GpuObject::new_prealloc( + kalloc.private.alloc_object()?, + |ptr: GpuWeakPointer| { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), + stats + ); + + let start_vtx = builder.add(microseq::StartVertex::ver { + header: microseq::op::StartVertex::HEADER, + tiling_params: inner_weak_ptr!(ptr, tiling_params), + job_params1: inner_weak_ptr!(ptr, job_params1), + buffer: scene.weak_buffer_pointer(), + scene: scene.weak_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_38: 1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + unk_44: 0, + cmd_seq: U64(ev_vtx.cmd_seq), + unk_50: 0, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + unk_64: 0x0, // fixed + unk_68: unk1.into(), + uuid: uuid_ta, + unk_70: 0x0, // fixed + unk_74: Default::default(), // fixed + unk_15c: 0x0, // fixed + unk_160: U64(0x0), // fixed + unk_168: 0x0, // fixed + unk_16c: 0x0, // fixed + unk_170: U64(0x0), // fixed + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + unk_178: 0x0, // padding? + })?; + + if vtx_result.is_some() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, start_ts), + work_queue: ev_vtx.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), + })?; + + if vtx_result.is_some() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, end_ts), + work_queue: ev_vtx.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_vtx); + builder.add(microseq::FinalizeVertex::ver { + header: microseq::op::FinalizeVertex::HEADER, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x0, // fixed + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_34: 0x0, // fixed + uuid: uuid_ta, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + unk_48: U64(0x0), // fixed + unk_50: 0x0, // fixed + unk_54: 0x0, // fixed + unk_58: U64(0x0), // fixed + unk_60: 0x0, // fixed + unk_64: 0x0, // fixed + unk_68: 0x0, // fixed + #[ver(G >= G14 && V < V13_0B4)] + unk_68_g14: U64(0), + restart_branch_offset: off, + unk_70: 0x0, // fixed + #[ver(V >= V13_0B4)] + unk_74: Default::default(), // Ventura + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + Ok(box_in_place!(fw::vertex::RunVertex::ver { + notifier: notifier, + scene: scene.clone(), + micro_seq: builder.build(&mut kalloc.private)?, + vm_bind: vm_bind.clone(), + timestamps: timestamps, + })?) + }, + |inner, ptr| { + #[ver(G < G14)] + let core_masks = gpu.core_masks_packed(); + Ok(place!( + ptr, + fw::vertex::raw::RunVertex::ver { + tag: fw::workqueue::CommandType::RunVertex, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + vm_slot: vm_bind.slot(), + unk_8: 0, + notifier: inner.notifier.gpu_pointer(), + buffer_slot: inner.scene.slot(), + unk_1c: 0, + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + unk_34: 0, + job_params1: fw::vertex::raw::JobParameters1::ver { + unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 + unk_8: f32!(1e-20), // fixed + unk_c: f32!(1e-20), // fixed + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + #[ver(G < G14)] + tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), + tpc: inner.scene.tpc_pointer(), + tvb_heapmeta: inner + .scene + .tvb_heapmeta_pointer() + .or(0x8000_0000_0000_0000), + iogpu_unk_54: 0x6b0003, // fixed + iogpu_unk_55: 0x3a0012, // fixed + iogpu_unk_56: U64(0x1), // fixed + #[ver(G < G14)] + tvb_cluster_meta1: inner + .scene + .meta_1_pointer() + .map(|x| x.or((tile_info.meta1_blocks as u64) << 50)), + utile_config: utile_config, + unk_4c: 0, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed + tvb_heapmeta_2: inner.scene.tvb_heapmeta_pointer(), + #[ver(G < G14)] + unk_60: U64(0x0), // fixed + #[ver(G < G14)] + core_mask: Array::new([ + *core_masks.first().unwrap_or(&0), + *core_masks.get(1).unwrap_or(&0), + ]), + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + preempt_buf2: inner.scene.preempt_buf_2_pointer(), + unk_80: U64(0x1), // fixed + preempt_buf3: inner + .scene + .preempt_buf_3_pointer() + .or(0x4_0000_0000_0000), // check + encoder_addr: U64(cmdbuf.encoder_ptr), + #[ver(G < G14)] + tvb_cluster_meta2: inner.scene.meta_2_pointer(), + #[ver(G < G14)] + tvb_cluster_meta3: inner.scene.meta_3_pointer(), + #[ver(G < G14)] + tiling_control: tiling_control, + #[ver(G < G14)] + unk_ac: Default::default(), // fixed + unk_b0: Default::default(), // fixed + pipeline_base: U64(0x11_00000000), + #[ver(G < G14)] + tvb_cluster_meta4: inner + .scene + .meta_4_pointer() + .map(|x| x.or(0x3000_0000_0000_0000)), + #[ver(G < G14)] + unk_f0: U64(0x1c + align(tile_info.meta1_blocks, 4) as u64), + unk_f8: U64(0x8c60), // fixed + unk_100: Default::default(), // fixed + unk_118: 0x1c, // fixed + #[ver(G >= G14)] + __pad: Default::default(), + }, + unk_154: Default::default(), + tiling_params: tile_info.params, + unk_3e8: Default::default(), + tpc: inner.scene.tpc_pointer(), + tpc_size: U64(tile_info.tpc_size as u64), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + fragment_stamp_slot: ev_frag.slot, + fragment_stamp_value: ev_frag.value.next(), + unk_pointee: 0, + unk_pad: 0, + job_params2: fw::vertex::raw::JobParameters2 { + unk_480: Default::default(), // fixed + unk_498: U64(0x0), // fixed + unk_4a0: 0x0, // fixed + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + unk_4ac: 0x0, // fixed + unk_4b0: U64(0x0), // fixed + unk_4b8: 0x0, // fixed + unk_4bc: U64(0x0), // fixed + unk_4c4_padding: Default::default(), + unk_50c: 0x0, // fixed + unk_510: U64(0x0), // fixed + unk_518: U64(0x0), // fixed + unk_520: U64(0x0), // fixed + }, + encoder_params: fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + unk_c: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: cmdbuf.encoder_id, + unk_18: 0x0, // fixed + iogpu_compute_unk44: 0xffffffff, + seq_buffer: inner.scene.seq_buf_pointer(), + unk_28: U64(0x0), // fixed + }, + unk_55c: 0, + unk_560: 0, + memoryless_rts_used: (cmdbuf.flags + & bindings::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 + != 0) as u32, + unk_568: 0, + unk_56c: 0, + meta: fw::job::raw::JobMeta { + unk_4: 0, + stamp: ev_vtx.stamp_pointer, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + stamp_slot: ev_vtx.slot, + evctl_index: 0, // fixed + flush_stamps: 0, + uuid: uuid_ta, + cmd_seq: ev_vtx.cmd_seq as u32, + }, + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + unk_buf_10: U64(0), + cur_ts: U64(0), + start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), + end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), + unk_5c4: 0, + unk_5c8: 0, + unk_5cc: 0, + unk_5d0: 0, + client_sequence: slot_client_seq, + pad_5d5: Default::default(), + #[ver(V >= V13_0B4)] + unk_5d8_0: 0, + #[ver(V >= V13_0B4)] + unk_5d8_4: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_5d8_d: Default::default(), + pad_5d8: Default::default(), + } + )) + }, + )?; + + core::mem::drop(alloc); + + mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); + fence.add_command(); + vtx_job.add_cb(vtx, vm_bind.slot(), move |cmd, error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + if let Some(mut res) = vtx_result.as_ref().map(|a| a.lock()) { + cmd.timestamps.with(|raw, _inner| { + res.result.vertex_ts_start = raw.vtx.start.load(Ordering::Relaxed); + res.result.vertex_ts_end = raw.vtx.end.load(Ordering::Relaxed); + }); + res.result.tvb_usage_bytes = cmd.scene.used_bytes() as u64; + if cmd.scene.overflowed() { + res.result.flags |= bindings::DRM_ASAHI_RESULT_RENDER_TVB_OVERFLOWED as u64; + } + res.vtx_error = error; + res.vtx_complete = true; + res.commit(); + } + fence.command_complete(); + })?; + + mod_dev_dbg!(self.dev, "[Submission {}] Increment counters\n", id); + self.notifier.threshold.with(|raw, _inner| { + raw.increment(); + raw.increment(); + }); + + // TODO: handle rollbacks, move to job submit? + buffer.increment(); + + job.get_vtx()?.next_seq(); + job.get_frag()?.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs new file mode 100644 index 00000000000000..019d7214793d3e --- /dev/null +++ b/drivers/gpu/drm/asahi/regs.rs @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU MMIO register abstraction +//! +//! Since the vast majority of the interactions with the GPU are brokered through the firmware, +//! there is very little need to interact directly with GPU MMIO register. This module abstracts +//! the few operations that require that, mainly reading the MMU fault status, reading GPU ID +//! information, and starting the GPU firmware coprocessor. + +use crate::hw; +use kernel::{device, io_mem::IoMem, platform, prelude::*}; + +/// Size of the ASC control MMIO region. +pub(crate) const ASC_CTL_SIZE: usize = 0x4000; + +/// Size of the SGX MMIO region. +pub(crate) const SGX_SIZE: usize = 0x1000000; + +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; // BIT(4) + +const FAULT_INFO: usize = 0x17030; + +const ID_VERSION: usize = 0xd04000; +const ID_UNK08: usize = 0xd04008; +const ID_COUNTS_1: usize = 0xd04010; +const ID_COUNTS_2: usize = 0xd04014; +const ID_UNK18: usize = 0xd04018; +const ID_CLUSTERS: usize = 0xd0401c; + +const CORE_MASK_0: usize = 0xd01500; +const CORE_MASK_1: usize = 0xd01514; + +/// Enum representing the unit that caused an MMU fault. +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultUnit { + /// Decompress / pixel fetch + DCMP(u8), + /// USC L1 Cache (device loads/stores) + UL1C(u8), + /// Compress / pixel store + CMP(u8), + GSL1(u8), + IAP(u8), + VCE(u8), + /// Tiling Engine + TE(u8), + RAS(u8), + /// Vertex Data Master + VDM(u8), + PPP(u8), + /// ISP Parameter Fetch + IPF(u8), + IPF_CPF(u8), + VF(u8), + VF_CPF(u8), + /// Depth/Stencil load/store + ZLS(u8), + + /// Parameter Management + dPM, + /// Compute Data Master + dCDM_KS(u8), + dIPP, + dIPP_CS, + // Vertex Data Master + dVDM_CSD, + dVDM_SSD, + dVDM_ILF, + dVDM_ILD, + dRDE(u8), + FC, + GSL2, + + /// Graphics L2 Cache Control? + GL2CC_META(u8), + GL2CC_MB, + + /// Parameter Management + gPM_SP(u8), + /// Vertex Data Master - CSD + gVDM_CSD_SP(u8), + gVDM_SSD_SP(u8), + gVDM_ILF_SP(u8), + gVDM_TFP_SP(u8), + gVDM_MMB_SP(u8), + /// Compute Data Master + gCDM_CS_KS0_SP(u8), + gCDM_CS_KS1_SP(u8), + gCDM_CS_KS2_SP(u8), + gCDM_KS0_SP(u8), + gCDM_KS1_SP(u8), + gCDM_KS2_SP(u8), + gIPP_SP(u8), + gIPP_CS_SP(u8), + gRDE0_SP(u8), + gRDE1_SP(u8), + + Unknown(u8), +} + +/// Reason for an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultReason { + Unmapped, + AfFault, + WriteOnly, + ReadOnly, + NoAccess, + Unknown(u8), +} + +/// Collection of information about an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) struct FaultInfo { + pub(crate) address: u64, + pub(crate) sideband: u8, + pub(crate) vm_slot: u32, + pub(crate) unit_code: u8, + pub(crate) unit: FaultUnit, + pub(crate) level: u8, + pub(crate) unk_5: u8, + pub(crate) read: bool, + pub(crate) reason: FaultReason, +} + +/// Device resources for this GPU instance. +pub(crate) struct Resources { + dev: device::Device, + asc: IoMem, + sgx: IoMem, +} + +impl Resources { + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &mut platform::Device) -> Result { + // TODO: add device abstraction to ioremap by name + let asc_res = unsafe { pdev.ioremap_resource(0)? }; + let sgx_res = unsafe { pdev.ioremap_resource(1)? }; + + Ok(Resources { + // SAFETY: This device does DMA via the UAT IOMMU. + dev: device::Device::from_dev(pdev), + asc: asc_res, + sgx: sgx_res, + }) + } + + fn sgx_read32(&self, off: usize) -> u32 { + self.sgx.readl_relaxed(off) + } + + /* Not yet used + fn sgx_write32(&self, off: usize, val: u32) { + self.sgx.writel_relaxed(val, off) + } + */ + + fn sgx_read64(&self, off: usize) -> u64 { + self.sgx.readq_relaxed(off) + } + + /* Not yet used + fn sgx_write64(&self, off: usize, val: u64) { + self.sgx.writeq_relaxed(val, off) + } + */ + + /// Initialize the MMIO registers for the GPU. + pub(crate) fn init_mmio(&self) -> Result { + // Nothing to do for now... + + Ok(()) + } + + /// Start the ASC coprocessor CPU. + pub(crate) fn start_cpu(&self) -> Result { + let val = self.asc.readl_relaxed(CPU_CONTROL); + + self.asc.writel_relaxed(val | CPU_RUN, CPU_CONTROL); + + Ok(()) + } + + /// Get the GPU identification info from registers. + /// + /// See [`hw::GpuIdConfig`] for the result. + pub(crate) fn get_gpu_id(&self) -> Result { + let id_version = self.sgx_read32(ID_VERSION); + let id_unk08 = self.sgx_read32(ID_UNK08); + let id_counts_1 = self.sgx_read32(ID_COUNTS_1); + let id_counts_2 = self.sgx_read32(ID_COUNTS_2); + let id_unk18 = self.sgx_read32(ID_UNK18); + let id_clusters = self.sgx_read32(ID_CLUSTERS); + + dev_info!( + self.dev, + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + let core_mask_0 = self.sgx_read32(CORE_MASK_0); + let core_mask_1 = self.sgx_read32(CORE_MASK_1); + let mut core_mask = (core_mask_0 as u64) | ((core_mask_1 as u64) << 32); + + dev_info!(self.dev, "Core mask: {:#x}\n", core_mask); + + let num_clusters = (id_clusters >> 12) & 0xff; + let num_cores = id_counts_1 & 0xff; + + if num_cores * num_clusters > 64 { + dev_err!( + self.dev, + "Too many total cores ({} x {} > 64)\n", + num_clusters, + num_cores + ); + return Err(ENODEV); + } + + let mut core_masks = Vec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = (1u64 << num_cores) - 1; + for _i in 0..num_clusters { + let mask = core_mask & max_core_mask; + core_masks.try_push(mask as u32)?; + core_mask >>= num_cores; + total_active_cores += mask.count_ones(); + } + let mut core_masks_packed = Vec::new(); + core_masks_packed.try_push(core_mask_0)?; + if core_mask_1 != 0 { + core_masks_packed.try_push(core_mask_1)?; + } + + if core_mask != 0 { + dev_err!(self.dev, "Leftover core mask: {:#x}\n", core_mask); + return Err(EIO); + } + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev, "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }; + + Ok(hw::GpuIdConfig { + gpu_gen: match (id_version >> 24) & 0xff { + 4 => hw::GpuGen::G13, + 5 => hw::GpuGen::G14, + a => { + dev_err!(self.dev, "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }, + gpu_variant: match (id_version >> 16) & 0xff { + 1 => hw::GpuVariant::P, // Guess + 2 => hw::GpuVariant::G, + 3 => hw::GpuVariant::S, + 4 => { + if num_clusters > 4 { + hw::GpuVariant::D + } else { + hw::GpuVariant::C + } + } + a => { + dev_err!(self.dev, "Unknown GPU variant {}\n", a); + return Err(ENODEV); + } + }, + gpu_rev, + gpu_rev_id, + max_dies: (id_clusters >> 20) & 0xf, + num_clusters, + num_cores, + num_frags: (id_counts_1 >> 8) & 0xff, + num_gps: (id_counts_2 >> 16) & 0xff, + total_active_cores, + core_masks, + core_masks_packed, + }) + } + + /// Get the fault information from the MMU status register, if one occurred. + pub(crate) fn get_fault_info(&self) -> Option { + let fault_info = self.sgx_read64(FAULT_INFO); + + if fault_info & 1 == 0 { + return None; + } + + let unit_code = ((fault_info >> 9) & 0xff) as u8; + let unit = match unit_code { + 0x00..=0x9f => match unit_code & 0xf { + 0x0 => FaultUnit::DCMP(unit_code >> 4), + 0x1 => FaultUnit::UL1C(unit_code >> 4), + 0x2 => FaultUnit::CMP(unit_code >> 4), + 0x3 => FaultUnit::GSL1(unit_code >> 4), + 0x4 => FaultUnit::IAP(unit_code >> 4), + 0x5 => FaultUnit::VCE(unit_code >> 4), + 0x6 => FaultUnit::TE(unit_code >> 4), + 0x7 => FaultUnit::RAS(unit_code >> 4), + 0x8 => FaultUnit::VDM(unit_code >> 4), + 0x9 => FaultUnit::PPP(unit_code >> 4), + 0xa => FaultUnit::IPF(unit_code >> 4), + 0xb => FaultUnit::IPF_CPF(unit_code >> 4), + 0xc => FaultUnit::VF(unit_code >> 4), + 0xd => FaultUnit::VF_CPF(unit_code >> 4), + 0xe => FaultUnit::ZLS(unit_code >> 4), + _ => FaultUnit::Unknown(unit_code), + }, + 0xa1 => FaultUnit::dPM, + 0xa2 => FaultUnit::dCDM_KS(0), + 0xa3 => FaultUnit::dCDM_KS(1), + 0xa4 => FaultUnit::dCDM_KS(2), + 0xa5 => FaultUnit::dIPP, + 0xa6 => FaultUnit::dIPP_CS, + 0xa7 => FaultUnit::dVDM_CSD, + 0xa8 => FaultUnit::dVDM_SSD, + 0xa9 => FaultUnit::dVDM_ILF, + 0xaa => FaultUnit::dVDM_ILD, + 0xab => FaultUnit::dRDE(0), + 0xac => FaultUnit::dRDE(1), + 0xad => FaultUnit::FC, + 0xae => FaultUnit::GSL2, + 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), + 0xb8 => FaultUnit::GL2CC_MB, + 0xe0..=0xff => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), + 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), + 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), + 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), + 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), + 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), + 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), + 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), + 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + _ => FaultUnit::Unknown(unit_code), + }; + + let reason = match (fault_info >> 1) & 0x7 { + 0 => FaultReason::Unmapped, + 1 => FaultReason::AfFault, + 2 => FaultReason::WriteOnly, + 3 => FaultReason::ReadOnly, + 4 => FaultReason::NoAccess, + a => FaultReason::Unknown(a as u8), + }; + + Some(FaultInfo { + address: (fault_info >> 30) << 6, + sideband: ((fault_info >> 23) & 0x7f) as u8, + vm_slot: ((fault_info >> 17) & 0x3f) as u32, + unit_code, + unit, + level: ((fault_info >> 7) & 3) as u8, + unk_5: ((fault_info >> 5) & 3) as u8, + read: (fault_info & (1 << 4)) != 0, + reason, + }) + } +} diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs new file mode 100644 index 00000000000000..6493111643fe75 --- /dev/null +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Generic slot allocator +//! +//! This is a simple allocator to manage fixed-size pools of GPU resources that are transiently +//! required during command execution. Each item resides in a "slot" at a given index. Users borrow +//! and return free items from the available pool. +//! +//! Allocations are "sticky", and return a token that callers can use to request the same slot +//! again later. This allows slots to be lazily invalidated, so that multiple uses by the same user +//! avoid any actual cleanup work. +//! +//! The allocation policy is currently a simple LRU mechanism, doing a full linear scan over the +//! slots when no token was previously provided. This is probably good enough, since in the absence +//! of serious system contention most allocation requests will be immediately fulfilled from the +//! previous slot without doing an LRU scan. + +use core::ops::{Deref, DerefMut}; +use kernel::{ + error::{code::*, Result}, + prelude::*, + sync::{Arc, CondVar, Mutex, UniqueArc}, +}; + +/// Trait representing a single item within a slot. +pub(crate) trait SlotItem { + /// Arbitrary user data associated with the SlotAllocator. + type Data; + + /// Called eagerly when this item is released back into the available pool. + fn release(&mut self, _data: &mut Self::Data, _slot: u32) {} +} + +/// Trivial implementation for users which do not require any slot data nor any allocator data. +impl SlotItem for () { + type Data = (); +} + +/// Represents a current or previous allocation of an item from a slot. Users keep `SlotToken`s +/// around across allocations to request that, if possible, the same slot be reused. +#[derive(Copy, Clone, Debug)] +pub(crate) struct SlotToken { + time: u64, + slot: u32, +} + +impl SlotToken { + /// Returns the slot index that this token represents a past assignment to. + pub(crate) fn last_slot(&self) -> u32 { + self.slot + } +} + +/// A guard representing active ownership of a slot. +pub(crate) struct Guard { + item: Option, + changed: bool, + token: SlotToken, + alloc: Arc>, +} + +impl Guard { + /// Returns the active slot owned by this `Guard`. + pub(crate) fn slot(&self) -> u32 { + self.token.slot + } + + /// Returns `true` if the slot changed since the last allocation (or no `SlotToken` was + /// provided), or `false` if the previously allocated slot was successfully re-acquired with + /// no other users in the interim. + pub(crate) fn changed(&self) -> bool { + self.changed + } + + /// Returns a `SlotToken` that can be used to re-request the same slot at a later time, after + /// this `Guard` is dropped. + pub(crate) fn token(&self) -> SlotToken { + self.token + } +} + +impl Deref for Guard { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.item.as_ref().expect("SlotItem Guard lost our item!") + } +} + +impl DerefMut for Guard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.item.as_mut().expect("SlotItem Guard lost our item!") + } +} + +/// A slot item that is currently free. +struct Entry { + item: T, + get_time: u64, + drop_time: u64, +} + +/// Inner data for the `SlotAllocator`, protected by a `Mutex`. +struct SlotAllocatorInner { + data: T::Data, + slots: Vec>>, + get_count: u64, + drop_count: u64, +} + +/// A single slot allocator instance. +struct SlotAllocatorOuter { + inner: Mutex>, + cond: CondVar, +} + +/// A shared reference to a slot allocator instance. +pub(crate) struct SlotAllocator(Arc>); + +impl SlotAllocator { + /// Creates a new `SlotAllocator`, with a fixed number of slots and arbitrary associated data. + /// + /// The caller provides a constructor callback which takes a reference to the `T::Data` and + /// creates a single slot. This is called during construction to create all the initial + /// items, which then live the lifetime of the `SlotAllocator`. + pub(crate) fn new( + num_slots: u32, + mut data: T::Data, + mut constructor: impl FnMut(&mut T::Data, u32) -> T, + ) -> Result> { + let mut slots = Vec::try_with_capacity(num_slots as usize)?; + + for i in 0..num_slots { + slots + .try_push(Some(Entry { + item: constructor(&mut data, i), + get_time: 0, + drop_time: 0, + })) + .expect("try_push() failed after reservation"); + } + + let inner = SlotAllocatorInner { + data, + slots, + get_count: 0, + drop_count: 0, + }; + + let mut alloc = Pin::from(UniqueArc::try_new(SlotAllocatorOuter { + // SAFETY: `condvar_init!` is called below. + cond: unsafe { CondVar::new() }, + // SAFETY: `mutex_init!` is called below. + inner: unsafe { Mutex::new(inner) }, + })?); + + // SAFETY: `cond` is pinned when `alloc` is. + let pinned = unsafe { alloc.as_mut().map_unchecked_mut(|s| &mut s.cond) }; + kernel::condvar_init!(pinned, "SlotAllocator::cond"); + + // SAFETY: `inner` is pinned when `alloc` is. + let pinned = unsafe { alloc.as_mut().map_unchecked_mut(|s| &mut s.inner) }; + kernel::mutex_init!(pinned, "SlotAllocator::inner"); + + Ok(SlotAllocator(alloc.into())) + } + + /// Calls a callback on the inner data associated with this allocator, taking the lock. + pub(crate) fn with_inner(&self, cb: impl FnOnce(&mut T::Data) -> RetVal) -> RetVal { + let mut inner = self.0.inner.lock(); + cb(&mut inner.data) + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + pub(crate) fn get(&self, token: Option) -> Result> { + self.get_inner(token, |_a, _b| Ok(())) + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + /// + /// This version allows the caller to pass in a callback that gets a mutable reference to the + /// user data for the allocator and the freshly acquired slot, which is called before the + /// allocator lock is released. This can be used to perform bookkeeping associated with + /// specific slots (such as tracking their current owner). + pub(crate) fn get_inner( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + let mut inner = self.0.inner.lock(); + + if let Some(token) = token { + let slot = &mut inner.slots[token.slot as usize]; + if slot.is_some() { + let count = slot.as_ref().unwrap().get_time; + if count == token.time { + let mut guard = Guard { + item: Some(slot.take().unwrap().item), + token, + changed: false, + alloc: self.0.clone(), + }; + cb(&mut inner.data, &mut guard)?; + return Ok(guard); + } + } + } + + let mut first = true; + let slot = loop { + let mut oldest_time = u64::MAX; + let mut oldest_slot = 0u32; + + for (i, slot) in inner.slots.iter().enumerate() { + if let Some(slot) = slot.as_ref() { + if slot.drop_time < oldest_time { + oldest_slot = i as u32; + oldest_time = slot.drop_time; + } + } + } + + if oldest_time == u64::MAX { + if first { + pr_warn!( + "{}: out of slots, blocking\n", + core::any::type_name::() + ); + } + first = false; + if self.0.cond.wait(&mut inner) { + return Err(ERESTARTSYS); + } + } else { + break oldest_slot; + } + }; + + inner.get_count += 1; + + let item = inner.slots[slot as usize] + .take() + .expect("Someone stole our slot?") + .item; + + let mut guard = Guard { + item: Some(item), + changed: true, + token: SlotToken { + time: inner.get_count, + slot, + }, + alloc: self.0.clone(), + }; + + cb(&mut inner.data, &mut guard)?; + Ok(guard) + } +} + +impl Clone for SlotAllocator { + fn clone(&self) -> Self { + SlotAllocator(self.0.clone()) + } +} + +impl Drop for Guard { + fn drop(&mut self) { + let mut inner = self.alloc.inner.lock(); + if inner.slots[self.token.slot as usize].is_some() { + pr_crit!( + "{}: tried to return an item into a full slot ({})\n", + core::any::type_name::(), + self.token.slot + ); + } else { + inner.drop_count += 1; + let mut item = self.item.take().expect("Guard lost its item"); + item.release(&mut inner.data, self.token.slot); + inner.slots[self.token.slot as usize] = Some(Entry { + item, + get_time: self.token.time, + drop_time: inner.drop_count, + }); + self.alloc.cond.notify_one(); + } + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs new file mode 100644 index 00000000000000..8d1a37f17cd887 --- /dev/null +++ b/drivers/gpu/drm/asahi/util.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Miscellaneous utility functions + +use core::ops::{Add, BitAnd, Div, Not, Sub}; + +/// Aligns an integer type to a power of two. +pub(crate) fn align(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Add + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + (a + b - one) & !(b - one) +} + +/// Integer division rounding up. +pub(crate) fn div_ceil(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Add + + Sub + + Div, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + (a + b - one) / b +} diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs new file mode 100644 index 00000000000000..837d82369ebfa8 --- /dev/null +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -0,0 +1,853 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU command execution queues +//! +//! The AGX GPU firmware schedules GPU work commands out of work queues, which are ring buffers of +//! pointers to work commands. There can be an arbitrary number of work queues. Work queues have an +//! associated type (vertex, fragment, or compute) and may only contain generic commands or commands +//! specific to that type. +//! +//! This module manages queueing work commands into a work queue and submitting them for execution +//! by the firmware. An active work queue needs an event to signal completion of its work, which is +//! owned by what we call a batch. This event then notifies the work queue when work is completed, +//! and that triggers freeing of all resources associated with that work. An idle work queue gives +//! up its associated event. + +use crate::debug::*; +use crate::fw::channels::PipeType; +use crate::fw::types::*; +use crate::fw::workqueue::*; +use crate::object::OpaqueGpuObject; +use crate::regs::FaultReason; +use crate::{box_in_place, place}; +use crate::{channel, event, fw, gpu, object, regs}; +use core::num::NonZeroU64; +use core::sync::atomic::Ordering; +use kernel::{ + bindings, + error::code::*, + prelude::*, + sync::{Arc, Guard, Mutex, UniqueArc}, + types::Opaque, +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; + +const MAX_JOB_SLOTS: u32 = 127; + +/// An enum of possible errors that might cause a piece of work to fail execution. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum WorkError { + /// GPU timeout (command execution took too long). + Timeout, + /// GPU MMU fault (invalid access). + Fault(regs::FaultInfo), + /// Work failed due to an error caused by other concurrent GPU work. + Killed, + /// The GPU crashed. + NoDevice, + /// Unknown reason. + Unknown, +} + +impl From for bindings::drm_asahi_result_info { + fn from(err: WorkError) -> Self { + match err { + WorkError::Fault(info) => Self { + status: bindings::drm_asahi_status_DRM_ASAHI_STATUS_FAULT, + fault_type: match info.reason { + FaultReason::Unmapped => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_UNMAPPED, + FaultReason::AfFault => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_AF_FAULT, + FaultReason::WriteOnly => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_WRITE_ONLY, + FaultReason::ReadOnly => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_READ_ONLY, + FaultReason::NoAccess => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_NO_ACCESS, + FaultReason::Unknown(_) => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_UNKNOWN, + }, + unit: info.unit_code.into(), + sideband: info.sideband.into(), + level: info.level, + extra: info.unk_5.into(), + is_read: info.read as u8, + pad: 0, + address: info.address, + }, + a => Self { + status: match a { + WorkError::Timeout => bindings::drm_asahi_status_DRM_ASAHI_STATUS_TIMEOUT, + WorkError::Killed => bindings::drm_asahi_status_DRM_ASAHI_STATUS_KILLED, + WorkError::NoDevice => bindings::drm_asahi_status_DRM_ASAHI_STATUS_NO_DEVICE, + _ => bindings::drm_asahi_status_DRM_ASAHI_STATUS_UNKNOWN_ERROR, + }, + ..Default::default() + }, + } + } +} + +impl From for kernel::error::Error { + fn from(err: WorkError) -> Self { + match err { + WorkError::Timeout => ETIMEDOUT, + // Not EFAULT because that's for userspace faults + WorkError::Fault(_) => EIO, + WorkError::Unknown => ENODATA, + WorkError::Killed => ECANCELED, + WorkError::NoDevice => ENODEV, + } + } +} + +struct SubmittedWork +where + O: OpaqueGpuObject, + C: FnOnce(O, Option) + Send + Sync + 'static, +{ + object: O, + value: EventValue, + error: Option, + wptr: u32, + vm_slot: u32, + callback: C, +} + +trait GenSubmittedWork: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; + fn value(&self) -> event::EventValue; + fn wptr(&self) -> u32; + fn set_wptr(&mut self, wptr: u32); + fn mark_error(&mut self, error: WorkError); + fn complete(self: Box); +} + +impl) + Send + Sync> GenSubmittedWork + for SubmittedWork +{ + fn gpu_va(&self) -> NonZeroU64 { + self.object.gpu_va() + } + + fn value(&self) -> event::EventValue { + self.value + } + + fn wptr(&self) -> u32 { + self.wptr + } + + fn set_wptr(&mut self, wptr: u32) { + self.wptr = wptr; + } + + fn complete(self: Box) { + let SubmittedWork { + object, + value, + error, + wptr, + vm_slot, + callback, + } = *self; + + callback(object, error); + } + + fn mark_error(&mut self, error: WorkError) { + mod_pr_debug!("WorkQueue: Command at value {:#x?} failed\n", self.value); + self.error = Some(match error { + WorkError::Fault(info) if info.vm_slot != self.vm_slot => WorkError::Killed, + err => err, + }); + } +} + +/// Inner data for managing a single work queue. +#[versions(AGX)] +struct WorkQueueInner { + event_manager: Arc, + info: GpuObject, + new: bool, + pipe_type: PipeType, + size: u32, + wptr: u32, + pending: Vec>, + last_token: Option, + pending_jobs: usize, + last_submitted: Option, + last_completed: Option, + event: Option<(event::Event, event::EventValue)>, + priority: u32, + commit_seq: u64, + submit_seq: u64, +} + +/// An instance of a work queue. +#[versions(AGX)] +pub(crate) struct WorkQueue { + info_pointer: GpuWeakPointer, + inner: Mutex, +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the GPU done pointer, representing how many work items have been completed by the + /// GPU. + fn doneptr(&self) -> u32 { + self.info + .state + .with(|raw, _inner| raw.gpu_doneptr.load(Ordering::Acquire)) + } +} + +#[versions(AGX)] +#[derive(Copy, Clone)] +pub(crate) struct QueueEventInfo { + pub(crate) stamp_pointer: GpuWeakPointer, + pub(crate) fw_stamp_pointer: GpuWeakPointer, + pub(crate) slot: u32, + pub(crate) value: event::EventValue, + pub(crate) cmd_seq: u64, + pub(crate) info_ptr: GpuWeakPointer, +} + +#[versions(AGX)] +pub(crate) struct Job { + wq: Arc, + wq_size: u32, + event_info: QueueEventInfo::ver, + start_value: EventValue, + pending: Vec>, + committed: bool, + submitted: bool, + event_count: usize, +} + +#[versions(AGX)] +pub(crate) struct JobSubmission<'a> { + inner: Option>>, + wptr: u32, + event_count: usize, + command_count: usize, +} + +#[versions(AGX)] +impl Job::ver { + pub(crate) fn event_info(&self) -> QueueEventInfo::ver { + let mut info = self.event_info; + info.cmd_seq += self.event_count as u64; + + info + } + + pub(crate) fn next_seq(&mut self) { + self.event_count += 1; + self.event_info.value.increment(); + } + + pub(crate) fn add( + &mut self, + command: O, + vm_slot: u32, + ) -> Result { + self.add_cb(command, vm_slot, |_, _| {}) + } + + pub(crate) fn add_cb( + &mut self, + command: O, + vm_slot: u32, + callback: impl FnOnce(O, Option) + Sync + Send + 'static, + ) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to mutate committed Job\n"); + return Err(EINVAL); + } + + self.pending.try_push(Box::try_new(SubmittedWork::<_, _> { + object: command, + value: self.event_info.value.next(), + error: None, + callback, + wptr: 0, + vm_slot, + })?)?; + + Ok(()) + } + + pub(crate) fn commit(&mut self) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to commit committed Job\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::commit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + let ev = inner.event.as_mut().expect("WorkQueue: Job lost its event"); + + if ev.1 != self.start_value { + pr_err!( + "WorkQueue: Job::commit() out of order (event slot {} {:?} != {:?}\n", + ev.0.slot(), + ev.1, + self.start_value + ); + return Err(EINVAL); + } + + ev.1 = self.event_info.value; + inner.commit_seq += self.pending.len() as u64; + self.committed = true; + + Ok(()) + } + + pub(crate) fn can_submit(&self) -> bool { + self.wq.free_slots() > self.event_count && self.wq.free_space() > self.pending.len() + } + + pub(crate) fn submit(&mut self) -> Result> { + if !self.committed { + pr_err!("WorkQueue: Tried to submit uncommitted Job\n"); + return Err(EINVAL); + } + + if self.submitted { + pr_err!("WorkQueue: Tried to submit Job twice\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::submit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + if inner.submit_seq != self.event_info.cmd_seq { + pr_err!( + "WorkQueue: Job::submit() out of order (submit_seq {} != {})\n", + inner.submit_seq, + self.event_info.cmd_seq + ); + return Err(EINVAL); + } + + if inner.commit_seq < (self.event_info.cmd_seq + self.pending.len() as u64) { + pr_err!( + "WorkQueue: Job::submit() out of order (commit_seq {} != {})\n", + inner.commit_seq, + (self.event_info.cmd_seq + self.pending.len() as u64) + ); + return Err(EINVAL); + } + + let mut wptr = inner.wptr; + let command_count = self.pending.len(); + + if inner.free_space() <= command_count { + pr_err!("WorkQueue: Job does not fit in ring buffer\n"); + return Err(EBUSY); + } + + inner.pending.try_reserve(command_count)?; + + inner.last_submitted = inner.event.as_ref().map(|e| e.1); + + for mut command in self.pending.drain(..) { + command.set_wptr(wptr); + + let next_wptr = (wptr + 1) % inner.size; + assert!(inner.doneptr() != next_wptr); + inner.info.ring[wptr as usize] = command.gpu_va().get(); + wptr = next_wptr; + + // Cannot fail, since we did a try_reserve(1) above + inner + .pending + .try_push(command) + .expect("try_push() failed after try_reserve()"); + } + + self.submitted = true; + + Ok(JobSubmission::ver { + inner: Some(inner), + wptr, + command_count, + event_count: self.event_count, + }) + } +} + +#[versions(AGX)] +impl<'a> JobSubmission::ver<'a> { + pub(crate) fn run(mut self, channel: &mut channel::PipeChannel::ver) { + let command_count = self.command_count; + let mut inner = self.inner.take().expect("No inner?"); + let wptr = self.wptr; + core::mem::forget(self); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_wptr.store(wptr, Ordering::Release)); + + inner.wptr = wptr; + + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + + let event_slot = event.0.slot(); + + let msg = fw::channels::RunWorkQueueMsg::ver { + pipe_type: inner.pipe_type, + work_queue: Some(inner.info.weak_pointer()), + wptr: inner.wptr, + event_slot, + is_new: inner.new, + __pad: Default::default(), + }; + channel.send(&msg); + inner.new = false; + + inner.submit_seq += command_count as u64; + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.as_ref().expect("No inner?").pipe_type + } + + pub(crate) fn priority(&self) -> u32 { + self.inner.as_ref().expect("No inner?").priority + } +} + +#[versions(AGX)] +impl Drop for Job::ver { + fn drop(&mut self) { + mod_pr_debug!("WorkQueue: Dropping Job\n"); + let mut inner = self.wq.inner.lock(); + + if self.committed && !self.submitted { + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("Job lost its event"); + mod_pr_debug!( + "WorkQueue({:?}): Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.pending.len() + ); + event.1.sub(self.event_count as u32); + inner.commit_seq -= self.pending.len() as u64; + } + + inner.pending_jobs -= 1; + + if inner.pending.is_empty() && inner.pending_jobs == 0 { + mod_pr_debug!("WorkQueue({:?}): Dropping event\n", inner.pipe_type); + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + mod_pr_debug!("WorkQueue({:?}): Dropped Job\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl<'a> Drop for JobSubmission::ver<'a> { + fn drop(&mut self) { + let inner = self.inner.as_mut().expect("No inner?"); + mod_pr_debug!("WorkQueue({:?}): Dropping JobSubmission\n", inner.pipe_type); + + let new_len = inner.pending.len() - self.command_count; + inner.pending.truncate(new_len); + + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + mod_pr_debug!( + "WorkQueue({:?}): Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.command_count + ); + event.1.sub(self.event_count as u32); + inner.commit_seq -= self.command_count as u64; + mod_pr_debug!("WorkQueue({:?}): Dropped JobSubmission\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the number of free entries in the workqueue + pub(crate) fn free_space(&self) -> usize { + self.size as usize - self.pending.len() - 1 + } + + pub(crate) fn free_slots(&self) -> usize { + let busy_slots = if let Some(ls) = self.last_submitted { + let lc = self + .last_completed + .expect("last_submitted but not completed?"); + ls.delta(&lc) + } else { + 0 + }; + + ((MAX_JOB_SLOTS as i32) - busy_slots).max(0) as usize + } +} + +#[versions(AGX)] +impl WorkQueue::ver { + /// Create a new WorkQueue of a given type and priority. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + alloc: &mut gpu::KernelAllocators, + event_manager: Arc, + gpu_context: Arc>, + notifier_list: Arc>, + pipe_type: PipeType, + id: u64, + priority: u32, + size: u32, + ) -> Result> { + let mut info = box_in_place!(QueueInfo::ver { + state: alloc.shared.new_default::()?, + ring: alloc.shared.array_empty(size as usize)?, + gpu_buf: alloc.private.array_empty(0x2c18)?, + notifier_list: notifier_list, + gpu_context: gpu_context, + })?; + + info.state.with_mut(|raw, _inner| { + raw.rb_size = size; + }); + + let inner = WorkQueueInner::ver { + event_manager, + info: alloc.private.new_boxed(info, |inner, ptr| { + Ok(place!( + ptr, + raw::QueueInfo::ver { + state: inner.state.gpu_pointer(), + ring: inner.ring.gpu_pointer(), + notifier_list: inner.notifier_list.gpu_pointer(), + gpu_buf: inner.gpu_buf.gpu_pointer(), + gpu_rptr1: Default::default(), + gpu_rptr2: Default::default(), + gpu_rptr3: Default::default(), + event_id: AtomicI32::new(-1), + priority: *raw::PRIORITY.get(priority as usize).ok_or(EINVAL)?, + unk_4c: -1, + uuid: id as u32, + unk_54: -1, + unk_58: Default::default(), + busy: Default::default(), + __pad: Default::default(), + unk_84_state: Default::default(), + unk_88: 0, + unk_8c: 0, + unk_90: 0, + unk_94: 0, + pending: Default::default(), + unk_9c: 0, + #[ver(V >= V13_2)] + unk_a0_0: 0, + gpu_context: inner.gpu_context.gpu_pointer(), + unk_a8: Default::default(), + #[ver(V >= V13_2)] + unk_b0: 0, + } + )) + })?, + new: true, + pipe_type, + size, + wptr: 0, + pending: Vec::new(), + last_token: None, + event: None, + priority, + pending_jobs: 0, + commit_seq: 0, + submit_seq: 0, + last_completed: None, + last_submitted: None, + }; + + let mut queue = Pin::from(UniqueArc::try_new(Self { + info_pointer: inner.info.weak_pointer(), + // SAFETY: `mutex_init!` is called below. + inner: unsafe { Mutex::new(inner) }, + })?); + + // SAFETY: `inner` is pinned when `queue` is. + let pinned = unsafe { queue.as_mut().map_unchecked_mut(|s| &mut s.inner) }; + match pipe_type { + PipeType::Vertex => kernel::mutex_init!(pinned, "WorkQueue::inner (Vertex)"), + PipeType::Fragment => kernel::mutex_init!(pinned, "WorkQueue::inner (Fragment)"), + PipeType::Compute => kernel::mutex_init!(pinned, "WorkQueue::inner (Compute)"), + } + + Ok(queue.into()) + } + + /// Returns the QueueInfo pointer for this workqueue, as a weak pointer. + pub(crate) fn info_pointer(&self) -> GpuWeakPointer { + self.info_pointer + } + + pub(crate) fn event_info(&self) -> Option { + let inner = self.inner.lock(); + + inner.event.as_ref().map(|ev| QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + info_ptr: self.info_pointer, + }) + } + + pub(crate) fn new_job(self: &Arc) -> Result { + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue({:?}): Grabbing event\n", inner.pipe_type); + let event = inner.event_manager.get(inner.last_token, self.clone())?; + let cur = event.current(); + inner.last_token = Some(event.token()); + mod_pr_debug!( + "WorkQueue({:?}): Grabbed event slot {}: {:#x?}\n", + inner.pipe_type, + event.slot(), + cur + ); + inner.event = Some((event, cur)); + inner.last_submitted = Some(cur); + inner.last_completed = Some(cur); + } + + inner.pending_jobs += 1; + + let ev = &inner.event.as_ref().unwrap(); + + mod_pr_debug!("WorkQueue({:?}): New job\n", inner.pipe_type); + Ok(Job::ver { + wq: self.clone(), + wq_size: inner.size, + event_info: QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + info_ptr: self.info_pointer, + }, + start_value: ev.1, + pending: Vec::new(), + event_count: 0, + committed: false, + submitted: false, + }) + } + + /// Return the number of free entries in the workqueue + pub(crate) fn free_space(&self) -> usize { + self.inner.lock().free_space() + } + + /// Return the number of free job slots in the workqueue + pub(crate) fn free_slots(&self) -> usize { + self.inner.lock().free_slots() + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.lock().pipe_type + } +} + +/// Trait used to erase the version-specific type of WorkQueues, to avoid leaking +/// version-specificity into the event module. +pub(crate) trait WorkQueue { + fn signal(&self) -> bool; + fn mark_error(&self, value: event::EventValue, error: WorkError); + fn fail_all(&self, error: WorkError); +} + +#[versions(AGX)] +impl WorkQueue for WorkQueue::ver { + /// Signal a workqueue that some work was completed. + /// + /// This will check the event stamp value to find out exactly how many commands were processed. + fn signal(&self) -> bool { + let mut inner = self.inner.lock(); + let event = inner.event.as_ref(); + let value = match event { + None => { + pr_err!("WorkQueue: signal() called but no event?\n"); + return true; + } + Some(event) => event.0.current(), + }; + + inner.last_completed = Some(value); + + mod_pr_debug!( + "WorkQueue({:?}): Signaling event {:?} value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + let mut completed_commands: usize = 0; + + for cmd in inner.pending.iter() { + if cmd.value() <= value { + mod_pr_debug!( + "WorkQueue({:?}): Command at value {:#x?} complete\n", + inner.pipe_type, + cmd.value() + ); + completed_commands += 1; + } else { + break; + } + } + + if completed_commands == 0 { + return inner.pending.is_empty(); + } + + let mut completed = Vec::new(); + + if completed.try_reserve(completed_commands).is_err() { + pr_crit!( + "WorkQueue({:?}): Failed to allocated space for {} completed commands\n", + inner.pipe_type, + completed_commands + ); + } + + let last_wptr = 0; + + let pipe_type = inner.pipe_type; + + for cmd in inner.pending.drain(..completed_commands) { + if completed.try_push(cmd).is_err() { + pr_crit!( + "WorkQueue({:?}): Failed to signal a completed command\n", + pipe_type, + ); + } + } + + mod_pr_debug!( + "WorkQueue({:?}): Completed {} commands\n", + inner.pipe_type, + completed_commands + ); + + if let Some(i) = completed.last() { + inner + .info + .state + .with(|raw, _inner| raw.cpu_freeptr.store(i.wptr(), Ordering::Release)); + } + + let empty = inner.pending.is_empty(); + if empty && inner.pending_jobs == 0 { + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + + core::mem::drop(inner); + + for cmd in completed { + cmd.complete(); + } + + empty + } + + /// Mark this queue's work up to a certain stamp value as having failed. + fn mark_error(&self, value: event::EventValue, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + pr_err!("WorkQueue: signal_fault() called but no event?\n"); + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Signaling fault for event {:?} at value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + for cmd in inner.pending.iter_mut() { + if cmd.value() <= value { + cmd.mark_error(error); + } else { + break; + } + } + } + + /// Mark all of this queue's work as having failed, and complete it. + fn fail_all(&self, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + pr_err!("WorkQueue: fail_all() called but no event?\n"); + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Failing all jobs {:?}\n", + inner.pipe_type, + error + ); + + let mut cmds = Vec::new(); + + core::mem::swap(&mut inner.pending, &mut cmds); + + if inner.pending_jobs == 0 { + inner.event = None; + } + + core::mem::drop(inner); + + for mut cmd in cmds { + cmd.mark_error(error); + cmd.complete(); + } + } +} + +#[versions(AGX)] +impl Drop for WorkQueue::ver { + fn drop(&mut self) { + mod_pr_debug!("WorkQueue({:?}): Dropping\n", self.inner.lock().pipe_type); + } +} From cec69d207acd3acc562df17d341fd4932509d506 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 1 Mar 2023 16:40:14 +0900 Subject: [PATCH 485/635] drm/asahi: queue: Add more debugging IDs Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/mod.rs | 49 +++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index ede31b841b1866..11833905dbdc80 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -169,6 +169,8 @@ pub(crate) struct QueueJob { sj_frag: Option, sj_comp: Option, fence: UserFence, + did_run: bool, + id: u64, } #[versions(AGX)] @@ -198,23 +200,35 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { fn can_run(job: &mut sched::Job) -> bool { - mod_dev_dbg!(job.dev, "QueueJob: Checking runnability\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); if let Some(sj) = job.sj_vtx.as_ref() { if !sj.can_submit() { - mod_dev_dbg!(job.dev, "QueueJob: Blocking due to vertex queue full\n"); + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to vertex queue full\n", + job.id + ); return false; } } if let Some(sj) = job.sj_frag.as_ref() { if !sj.can_submit() { - mod_dev_dbg!(job.dev, "QueueJob: Blocking due to fragment queue full\n"); + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to fragment queue full\n", + job.id + ); return false; } } if let Some(sj) = job.sj_comp.as_ref() { if !sj.can_submit() { - mod_dev_dbg!(job.dev, "QueueJob: Blocking due to compute queue full\n"); + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to compute queue full\n", + job.id + ); return false; } } @@ -223,7 +237,7 @@ impl sched::JobImpl for QueueJob::ver { #[allow(unused_assignments)] fn run(job: &mut sched::Job) -> Result> { - mod_dev_dbg!(job.dev, "QueueJob: Running Job\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); let dev = job.dev.data(); let gpu = match dev @@ -250,7 +264,7 @@ impl sched::JobImpl for QueueJob::ver { if let Some(sj) = job.sj_frag.as_mut() { frag_job = sj.job.take(); if let Some(wqjob) = frag_job.as_mut() { - mod_dev_dbg!(job.dev, "QueueJob: Submit fragment\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Submit fragment\n", job.id); frag_sub = Some(wqjob.submit()?); } } @@ -260,7 +274,7 @@ impl sched::JobImpl for QueueJob::ver { if let Some(sj) = job.sj_vtx.as_mut() { vtx_job = sj.job.take(); if let Some(wqjob) = vtx_job.as_mut() { - mod_dev_dbg!(job.dev, "QueueJob: Submit vertex\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Submit vertex\n", job.id); vtx_sub = Some(wqjob.submit()?); } } @@ -270,28 +284,30 @@ impl sched::JobImpl for QueueJob::ver { if let Some(sj) = job.sj_comp.as_mut() { comp_job = sj.job.take(); if let Some(wqjob) = comp_job.as_mut() { - mod_dev_dbg!(job.dev, "QueueJob: Submit compute\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Submit compute\n", job.id); comp_sub = Some(wqjob.submit()?); } } // Now we fully commit to running the job - mod_dev_dbg!(job.dev, "QueueJob: Run fragment\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); frag_sub.map(|a| gpu.run_job(a)).transpose()?; - mod_dev_dbg!(job.dev, "QueueJob: Run vertex\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Run vertex\n", job.id); vtx_sub.map(|a| gpu.run_job(a)).transpose()?; - mod_dev_dbg!(job.dev, "QueueJob: Run compute\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Run compute\n", job.id); comp_sub.map(|a| gpu.run_job(a)).transpose()?; - mod_dev_dbg!(job.dev, "QueueJob: Drop compute job\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop compute job\n", job.id); core::mem::drop(comp_job); - mod_dev_dbg!(job.dev, "QueueJob: Drop vertex job\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop vertex job\n", job.id); core::mem::drop(vtx_job); - mod_dev_dbg!(job.dev, "QueueJob: Drop fragment job\n"); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); core::mem::drop(frag_job); + job.did_run = true; + Ok(Some(Fence::from_fence(&job.fence))) } @@ -299,7 +315,8 @@ impl sched::JobImpl for QueueJob::ver { // FIXME: Handle timeouts properly dev_err!( job.dev, - "Job timed out on the DRM scheduler, things will probably break\n" + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", + job.id, job.did_run ); sched::Status::NoDevice } @@ -552,6 +569,8 @@ impl Queue for Queue::ver { }, )? .into(), + did_run: false, + id, })?; mod_dev_dbg!( From b4c26736ac259241f4e13187e2b5aa9144940cf1 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 1 Mar 2023 16:40:26 +0900 Subject: [PATCH 486/635] drm/asahi: render: Always flush stamps on both vertex & frag They could complete in either order, and not forcing flushes on both can leave us with stuck stamps... Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/render.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 6a0a2a84a040ec..da3e3e656bae0f 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -1109,7 +1109,7 @@ impl super::Queue::ver { stamp_value: ev_vtx.value.next(), stamp_slot: ev_vtx.slot, evctl_index: 0, // fixed - flush_stamps: 0, + flush_stamps: flush_stamps as u32, uuid: uuid_ta, cmd_seq: ev_vtx.cmd_seq as u32, }, From 7d13c98dc45f967f4d7d39525969286830f29a1e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 5 Mar 2023 23:29:52 +0900 Subject: [PATCH 487/635] rust: Fix bindgen on arm64 Signed-off-by: Hector Martin --- rust/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/Makefile b/rust/Makefile index 2328977cd5791f..be34d7653a554c 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -262,6 +262,7 @@ bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ # Derived from `scripts/Makefile.clang`. BINDGEN_TARGET_x86 := x86_64-linux-gnu +BINDGEN_TARGET_arm64 := aarch64-linux-gnu BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) # All warnings are inhibited since GCC builds are very experimental, From 506ea8cea576bfc3a8e5325e8f0a1918f48b8b6e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 5 Mar 2023 23:40:22 +0900 Subject: [PATCH 488/635] rust: helpers: Fix spinlock helper for various spinlock modes Signed-off-by: Hector Martin --- rust/helpers.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/helpers.c b/rust/helpers.c index 0983d09cc8abc7..3b128be210829c 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -55,7 +55,12 @@ void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, struct lock_class_key *key) { #ifdef CONFIG_DEBUG_SPINLOCK +# ifndef CONFIG_PREEMPT_RT __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); +# else + rt_mutex_base_init(&lock->lock); + __rt_spin_lock_init(lock, name, key, false); +# endif #else spin_lock_init(lock); #endif From 95b14b2242483bdafc3972a20bacd46738520554 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 6 Mar 2023 19:57:22 +0900 Subject: [PATCH 489/635] drm/scheduler: Clean up jobs when the scheduler is torn down. drm_sched_fini() currently leaves any pending jobs dangling, which causes segfaults and other badness when job completion fences are signaled after the scheduler is torn down. Explicitly detach all jobs from their completion callbacks and free them. This makes it possible to write a sensible safe abstraction for drm_sched, without having to externally duplicate the tracking of in-flight jobs. This shouldn't regress any existing drivers, since calling drm_sched_fini() with any pending jobs is broken and this change should be a no-op if there are no pending jobs. Signed-off-by: Asahi Lina --- drivers/gpu/drm/scheduler/sched_main.c | 27 ++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 3cf591946c53dd..8760a84abf8408 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -1114,10 +1114,33 @@ EXPORT_SYMBOL(drm_sched_init); void drm_sched_fini(struct drm_gpu_scheduler *sched) { struct drm_sched_entity *s_entity; + struct drm_sched_job *s_job, *tmp; int i; - if (sched->thread) - kthread_stop(sched->thread); + if (!sched->thread) + return; + + /* + * Stop the scheduler, detaching all jobs from their hardware callbacks + * and cleaning up complete jobs. + */ + drm_sched_stop(sched, NULL); + + /* + * Iterate through the pending job list and free all jobs. + * This assumes the driver has either guaranteed jobs are already stopped, or that + * otherwise it is responsible for keeping any necessary data structures for + * in-progress jobs alive even when the free_job() callback is called early (e.g. by + * putting them in its own queue or doing its own refcounting). + */ + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { + spin_lock(&sched->job_list_lock); + list_del_init(&s_job->list); + spin_unlock(&sched->job_list_lock); + sched->ops->free_job(s_job); + } + + kthread_stop(sched->thread); for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { struct drm_sched_rq *rq = &sched->sched_rq[i]; From 2f682b3f537159067e88d2d3372d6dd60de85ddd Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 6 Mar 2023 20:29:50 +0900 Subject: [PATCH 490/635] drm/asahi: Move GPU context drop inside refcount Instead of invalidating the GPU context when the Queue gets dropped (at which point jobs might still be running), wrap the object and put the drop logic in the wrapper, which is inside the Arc<>. Since the WorkQueues stay alive until all commands are complete (via the EventManager reference), that means the GPU context will only be invalidated when there is no work in progress or queued for it and the Arc<> loses its last reference. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/workqueue.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 14 ++------- drivers/gpu/drm/asahi/workqueue.rs | 43 +++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs index 947742e8dc8843..e81025b6c014a1 100644 --- a/drivers/gpu/drm/asahi/fw/workqueue.rs +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -159,7 +159,7 @@ pub(crate) struct QueueInfo { pub(crate) ring: GpuArray, pub(crate) gpu_buf: GpuArray, pub(crate) notifier_list: Arc>, - pub(crate) gpu_context: Arc>, + pub(crate) gpu_context: Arc, } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 11833905dbdc80..563ff882d9071c 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -109,7 +109,7 @@ pub(crate) struct Queue { q_frag: Option, q_comp: Option, buffer: Option>, - gpu_context: Arc>, + gpu_context: Arc, notifier_list: Arc>, notifier: Arc>, id: u64, @@ -369,12 +369,6 @@ impl Queue::ver { let data = dev.data(); - let gpu_context: Arc> = Arc::try_new( - alloc - .shared - .new_object(Default::default(), |_inner| Default::default())?, - )?; - let mut notifier_list = alloc.private.new_default::()?; let self_ptr = notifier_list.weak_pointer(); @@ -416,7 +410,7 @@ impl Queue::ver { q_frag: None, q_comp: None, buffer: None, - gpu_context, + gpu_context: Arc::try_new(workqueue::GpuContext::new(dev, alloc)?)?, notifier_list: Arc::try_new(notifier_list)?, notifier, id, @@ -728,9 +722,5 @@ impl Queue for Queue::ver { impl Drop for Queue::ver { fn drop(&mut self) { mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.id); - let dev = self.dev.data(); - if dev.gpu.invalidate_context(&self.gpu_context).is_err() { - dev_err!(self.dev, "Queue::drop: Failed to invalidate GPU context!\n"); - } } } diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 837d82369ebfa8..449552d3ade0b9 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -19,8 +19,8 @@ use crate::fw::types::*; use crate::fw::workqueue::*; use crate::object::OpaqueGpuObject; use crate::regs::FaultReason; -use crate::{box_in_place, place}; -use crate::{channel, event, fw, gpu, object, regs}; +use crate::{box_in_place, no_debug, place}; +use crate::{channel, driver, event, fw, gpu, object, regs}; use core::num::NonZeroU64; use core::sync::atomic::Ordering; use kernel::{ @@ -97,6 +97,43 @@ impl From for kernel::error::Error { } } +/// A GPU context tracking structure, which must be explicitly invalidated when dropped. +pub(crate) struct GpuContext { + dev: driver::AsahiDevice, + data: GpuObject, +} +no_debug!(GpuContext); + +impl GpuContext { + /// Allocate a new GPU context. + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(GpuContext { + dev: dev.clone(), + data: alloc + .shared + .new_object(Default::default(), |_inner| Default::default())?, + }) + } + + /// Returns the GPU pointer to the inner GPU context data structure. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { + self.data.gpu_pointer() + } +} + +impl Drop for GpuContext { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "GpuContext: Invalidating GPU context\n"); + let dev = self.dev.data(); + if dev.gpu.invalidate_context(&self.data).is_err() { + dev_err!(self.dev, "GpuContext: Failed to invalidate GPU context!\n"); + } + } +} + struct SubmittedWork where O: OpaqueGpuObject, @@ -512,7 +549,7 @@ impl WorkQueue::ver { pub(crate) fn new( alloc: &mut gpu::KernelAllocators, event_manager: Arc, - gpu_context: Arc>, + gpu_context: Arc, notifier_list: Arc>, pipe_type: PipeType, id: u64, From 9ed6040dc405a0c0caf7adb715efb2f3340ea267 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 6 Mar 2023 21:04:22 +0900 Subject: [PATCH 491/635] drm/asahi: Promote two fields in vert/frag structs to all versions Seen on 12.3: asahi 206400000.gpu: Allocator: Corruption after object of type asahi::fw::fragment::RunFragmentG13V12_3 at 0xffffffa00009be00:0x928 + 0x0..0x5 5 bytes sounds like exactly the two fields we had as >= 13_0B4 at the end of the structs, so let's just enable them for all versions. Plus we already had them for compute! Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/fragment.rs | 10 +++------- drivers/gpu/drm/asahi/fw/vertex.rs | 12 ++++-------- drivers/gpu/drm/asahi/queue/render.rs | 18 +++++++----------- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs index 20594233efe831..2dda766b0b335c 100644 --- a/drivers/gpu/drm/asahi/fw/fragment.rs +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -245,18 +245,14 @@ pub(crate) mod raw { pub(crate) unk_920: u32, pub(crate) client_sequence: u8, pub(crate) pad_925: Array<3, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_928_0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_928_4: u8, + pub(crate) unk_928: u32, + pub(crate) unk_92c: u8, #[ver(V >= V13_0B4)] pub(crate) unk_ts: U64, #[ver(V >= V13_0B4)] - pub(crate) unk_928_d: Array<0x1b, u8>, + pub(crate) unk_92d_8: Array<0x1b, u8>, // Alignment - handled by allocator //pad_928: [u8; 0x18], } diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs index 781220826bf623..c46f1a9deb3bf1 100644 --- a/drivers/gpu/drm/asahi/fw/vertex.rs +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -147,20 +147,16 @@ pub(crate) mod raw { pub(crate) unk_5d0: u32, pub(crate) client_sequence: u8, pub(crate) pad_5d5: Array<3, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_5d8_0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_5d8_4: u8, + pub(crate) unk_5d8: u32, + pub(crate) unk_5dc: u8, #[ver(V >= V13_0B4)] pub(crate) unk_ts: U64, #[ver(V >= V13_0B4)] - pub(crate) unk_5d8_d: Array<0x13, u8>, + pub(crate) unk_5dd_8: Array<0x13, u8>, - pub(crate) pad_5d8: Pad<0x8>, + pub(crate) pad_5dd: Pad<0x8>, // Alignment - handled by allocator //#[ver(V >= V13_0B4)] //pad_5e0: Pad<0x18>, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index da3e3e656bae0f..8515239afc8543 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -795,14 +795,12 @@ impl super::Queue::ver { unk_920: 0, client_sequence: slot_client_seq, pad_925: Default::default(), - #[ver(V >= V13_0B4)] - unk_928_0: 0, - #[ver(V >= V13_0B4)] - unk_928_4: 0, + unk_928: 0, + unk_92c: 0, #[ver(V >= V13_0B4)] unk_ts: U64(0), #[ver(V >= V13_0B4)] - unk_928_d: Default::default(), + unk_92d_8: Default::default(), } )) }, @@ -1126,15 +1124,13 @@ impl super::Queue::ver { unk_5d0: 0, client_sequence: slot_client_seq, pad_5d5: Default::default(), - #[ver(V >= V13_0B4)] - unk_5d8_0: 0, - #[ver(V >= V13_0B4)] - unk_5d8_4: 0, + unk_5d8: 0, + unk_5dc: 0, #[ver(V >= V13_0B4)] unk_ts: U64(0), #[ver(V >= V13_0B4)] - unk_5d8_d: Default::default(), - pad_5d8: Default::default(), + unk_5dd_8: Default::default(), + pad_5dd: Default::default(), } )) }, From 8c78020ed0e1f6bfd377bee3991947b12d47c37a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 6 Mar 2023 21:11:39 +0900 Subject: [PATCH 492/635] drm/asahi: Remove padding from Vertex commands, move to 13_0B4+ This was probably papering over the issue in the previous commit. Doing this makes both command types have the same structure at the end, which makes sense... let's see how it goes. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/fragment.rs | 2 -- drivers/gpu/drm/asahi/fw/vertex.rs | 7 +------ drivers/gpu/drm/asahi/queue/render.rs | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs index 2dda766b0b335c..eca275efb967a4 100644 --- a/drivers/gpu/drm/asahi/fw/fragment.rs +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -253,8 +253,6 @@ pub(crate) mod raw { #[ver(V >= V13_0B4)] pub(crate) unk_92d_8: Array<0x1b, u8>, - // Alignment - handled by allocator - //pad_928: [u8; 0x18], } } diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs index c46f1a9deb3bf1..959a0913e693fa 100644 --- a/drivers/gpu/drm/asahi/fw/vertex.rs +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -154,12 +154,7 @@ pub(crate) mod raw { pub(crate) unk_ts: U64, #[ver(V >= V13_0B4)] - pub(crate) unk_5dd_8: Array<0x13, u8>, - - pub(crate) pad_5dd: Pad<0x8>, - // Alignment - handled by allocator - //#[ver(V >= V13_0B4)] - //pad_5e0: Pad<0x18>, + pub(crate) unk_5dd_8: Array<0x1b, u8>, } } diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 8515239afc8543..4b6542472be562 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -1130,7 +1130,6 @@ impl super::Queue::ver { unk_ts: U64(0), #[ver(V >= V13_0B4)] unk_5dd_8: Default::default(), - pad_5dd: Default::default(), } )) }, From 114292499ef4204878b906f8a42255108dc246df Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Mon, 6 Mar 2023 21:13:31 +0900 Subject: [PATCH 493/635] drm/asahi: alloc: Make corruption ranges end-inclusive And use Rust syntax to make it clear that it is. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index 5397da5f64e183..d918b19e97214c 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -159,12 +159,12 @@ impl Drop for GenericAlloc { .unwrap_or(0); dev_warn!( self.device(), - "Allocator: Corruption after object of type {} at {:#x}:{:#x} + {:#x}..{:#x}\n", + "Allocator: Corruption after object of type {} at {:#x}:{:#x} + {:#x}..={:#x}\n", core::any::type_name::(), self.gpu_ptr(), self.size(), first_err, - self.padding - last_err + self.padding - last_err - 1 ); } } From bd924e820e345213a2faec20e7fbf4cd4e2aeb45 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 21 Oct 2022 16:42:26 +0900 Subject: [PATCH 494/635] rust: module_param: Tolerate a trailing newline when parsing This is the same behavior as kstrtol/kstrtoul, and allows simple `echo 0 > /sys/module/foo/parameters/bar` commands to work. Signed-off-by: Asahi Lina --- rust/kernel/module_param.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs index b597abf0abba4e..746c73108fc805 100644 --- a/rust/kernel/module_param.rs +++ b/rust/kernel/module_param.rs @@ -121,8 +121,8 @@ pub trait ModuleParam: core::fmt::Display + core::marker::Sized { /// Strings beginning with `0x`, `0o`, or `0b` are parsed as hex, octal, or /// binary respectively. Strings beginning with `0` otherwise are parsed as /// octal. Anything else is parsed as decimal. A leading `+` or `-` is also -/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be -/// successfully parsed. +/// permitted. The string may contain a trailing newline. Any string parsed +/// by [`kstrtol()`] or [`kstrtoul()`] will be successfully parsed. /// /// [`kstrtol()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtol /// [`kstrtoul()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtoul @@ -131,6 +131,7 @@ trait ParseInt: Sized { fn checked_neg(self) -> Option; fn from_str_unsigned(src: &str) -> Result { + let src = src.strip_suffix('\n').unwrap_or(src); let (radix, digits) = if let Some(n) = src.strip_prefix("0x") { (16, n) } else if let Some(n) = src.strip_prefix("0X") { From 7b6c9fd8e431f9c1054fa952f9a3bd145c6b6758 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Tue, 7 Mar 2023 15:42:18 +0900 Subject: [PATCH 495/635] drm/asahi: channel: Increase timeouts, add sleeping mode Increase timeouts to 1s, but switch to sleeping between polls at the 10ms boundary so we don't busy-loop when the firmware is too busy to service rings (which can happen in rare cases). Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/channel.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 03aeecc75da102..0b3c3b65c2796a 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -155,16 +155,28 @@ where /// Wait for a previously submitted message to be popped off of the ring by the GPU firmware. /// /// This busy-loops, and is intended to be used for rare cases when we need to block for - /// completion of a cache management or invalidation operation synchronously. + /// completion of a cache management or invalidation operation synchronously (which + /// the firmware normally completes fast enough not to be worth sleeping for). + /// If the poll takes longer than 10ms, this switches to sleeping between polls. pub(crate) fn wait_for(&mut self, wptr: u32, timeout_ms: u64) -> Result { - let timeout = time::ktime_get() + Duration::from_millis(timeout_ms); + const MAX_FAST_POLL: u64 = 10; + let start = time::ktime_get(); + let timeout_fast = start + Duration::from_millis(timeout_ms.min(MAX_FAST_POLL)); + let timeout_slow = start + Duration::from_millis(timeout_ms); self.ring.state.with(|raw, _inner| { - while time::ktime_get() < timeout { + while time::ktime_get() < timeout_fast { if T::rptr(raw) == wptr { return Ok(()); } mem::sync(); } + while time::ktime_get() < timeout_slow { + if T::rptr(raw) == wptr { + return Ok(()); + } + coarse_sleep(Duration::from_millis(5)); + mem::sync(); + } Err(ETIMEDOUT) }) } @@ -179,7 +191,7 @@ pub(crate) struct DeviceControlChannel { #[versions(AGX)] impl DeviceControlChannel::ver { - const COMMAND_TIMEOUT_MS: u64 = 100; + const COMMAND_TIMEOUT_MS: u64 = 1000; /// Allocate a new Device Control channel. pub(crate) fn new( @@ -248,7 +260,7 @@ pub(crate) struct FwCtlChannel { } impl FwCtlChannel { - const COMMAND_TIMEOUT_MS: u64 = 100; + const COMMAND_TIMEOUT_MS: u64 = 1000; /// Allocate a new Firmware Control channel. pub(crate) fn new( From 5dff1431bb0e41fa9c1910a44aca4be3acd0c952 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Tue, 7 Mar 2023 22:20:44 +0900 Subject: [PATCH 496/635] drm/asahi: Clippy cleanups Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/asahi.rs | 3 --- drivers/gpu/drm/asahi/event.rs | 7 ++----- drivers/gpu/drm/asahi/file.rs | 8 ++++---- drivers/gpu/drm/asahi/queue/compute.rs | 6 ++---- drivers/gpu/drm/asahi/queue/mod.rs | 16 +++++----------- drivers/gpu/drm/asahi/queue/render.rs | 5 +---- drivers/gpu/drm/asahi/workqueue.rs | 16 +++------------- 7 files changed, 17 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 425e8d6ada70e3..e511d83f4cd150 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -1,8 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #![recursion_limit = "1024"] -#![allow(dead_code)] -#![allow(unused_imports)] -#![allow(unused_variables)] //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. diff --git a/drivers/gpu/drm/asahi/event.rs b/drivers/gpu/drm/asahi/event.rs index 3ae18d507ddc7e..ccf00e4104bef7 100644 --- a/drivers/gpu/drm/asahi/event.rs +++ b/drivers/gpu/drm/asahi/event.rs @@ -46,11 +46,6 @@ pub(crate) type Event = slotalloc::Guard; pub(crate) struct EventValue(u32); impl EventValue { - /// Returns the counter portion of the value. - pub(crate) fn counter(&self) -> u32 { - self.0 >> 8 - } - /// Returns the `EventValue` that succeeds this one. pub(crate) fn next(&self) -> EventValue { EventValue(self.0.wrapping_add(0x100)) @@ -61,12 +56,14 @@ impl EventValue { self.0 = self.0.wrapping_add(0x100); } + /* Not used /// Increments this `EventValue` in place by a certain count. pub(crate) fn add(&mut self, val: u32) { self.0 = self .0 .wrapping_add(val.checked_mul(0x100).expect("Adding too many events")); } + */ /// Increments this `EventValue` in place by a certain count. pub(crate) fn sub(&mut self, val: u32) { diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 2e3d90585c0978..fa554e1a12c9af 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -109,7 +109,7 @@ impl SyncItem { // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = unsafe { UserSlicePtr::new(ptr as usize as *mut _, size).reader() }; - for i in 0..count { + for _i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE @@ -455,7 +455,7 @@ impl File { } pub(crate) fn do_gem_bind( - device: &AsahiDevice, + _device: &AsahiDevice, data: &mut bindings::drm_asahi_gem_bind, file: &DrmFile, ) -> Result { @@ -525,7 +525,7 @@ impl File { } pub(crate) fn do_gem_unbind_all( - device: &AsahiDevice, + _device: &AsahiDevice, data: &mut bindings::drm_asahi_gem_bind, file: &DrmFile, ) -> Result { @@ -714,7 +714,7 @@ impl File { let mut reader = unsafe { UserSlicePtr::new(data.commands as usize as *mut _, size).reader() }; - for i in 0..data.command_count { + for _i in 0..data.command_count { let mut cmd: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 1d50ae16fa4c05..6590382c75afa9 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #![allow(clippy::unusual_byte_groupings)] -#![allow(unused_imports)] //! Compute work queue. //! @@ -11,11 +10,10 @@ use super::common; use crate::alloc::Allocator; use crate::debug::*; -use crate::driver::AsahiDevice; use crate::fw::types::*; use crate::gpu::GpuManager; -use crate::{alloc, channel, event, file, fw, gpu, microseq, mmu, workqueue}; use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; +use crate::{fw, gpu, microseq}; use core::mem::MaybeUninit; use core::sync::atomic::Ordering; use kernel::bindings; @@ -23,7 +21,7 @@ use kernel::dma_fence::RawDmaFence; use kernel::drm::sched::Job; use kernel::io_buffer::IoBufferReader; use kernel::prelude::*; -use kernel::sync::{smutex::Mutex, Arc}; +use kernel::sync::Arc; use kernel::user_ptr::UserSlicePtr; const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 563ff882d9071c..107df04140e2ba 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -#![allow(unused_imports)] //! Submission queue management //! @@ -7,16 +6,13 @@ //! submissions to firmware queues. use kernel::dma_fence::*; -use kernel::io_buffer::IoBufferReader; use kernel::prelude::*; -use kernel::user_ptr::UserSlicePtr; use kernel::{ bindings, c_str, dma_fence, drm::gem::shmem::VMap, drm::sched, macros::versions, - prelude::*, - sync::{smutex::Mutex, Arc, Guard, UniqueArc}, + sync::{smutex::Mutex, Arc}, }; use crate::alloc::Allocator; @@ -24,11 +20,9 @@ use crate::debug::*; use crate::driver::AsahiDevice; use crate::fw::types::*; use crate::gpu::GpuManager; -use crate::util::*; -use crate::{alloc, buffer, channel, event, file, fw, gem, gpu, microseq, mmu, object, workqueue}; -use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; +use crate::{alloc, buffer, channel, event, file, fw, gem, gpu, mmu, workqueue}; +use crate::{inner_weak_ptr, place}; -use core::any::Any; use core::mem::MaybeUninit; use core::sync::atomic::{AtomicU64, Ordering}; @@ -101,7 +95,7 @@ impl SubQueueJob::ver { #[versions(AGX)] pub(crate) struct Queue { dev: AsahiDevice, - sched: sched::Scheduler, + _sched: sched::Scheduler, entity: sched::Entity, vm: mmu::Vm, ualloc: Arc>, @@ -402,7 +396,7 @@ impl Queue::ver { let mut ret = Queue::ver { dev: dev.clone(), - sched, + _sched: sched, entity, vm, ualloc, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 4b6542472be562..318c952df020b9 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT #![allow(clippy::unusual_byte_groupings)] -#![allow(unused_imports)] //! Render work queue. //! @@ -11,18 +10,16 @@ use super::common; use crate::alloc::Allocator; use crate::debug::*; -use crate::driver::AsahiDevice; use crate::fw::types::*; use crate::gpu::GpuManager; use crate::util::*; use crate::workqueue::WorkError; -use crate::{alloc, buffer, channel, event, file, fw, gpu, microseq, mmu, workqueue}; use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; +use crate::{buffer, fw, gpu, microseq, workqueue}; use core::mem::MaybeUninit; use core::sync::atomic::Ordering; use kernel::bindings; use kernel::dma_fence::RawDmaFence; -use kernel::drm::gem::shmem::VMap; use kernel::drm::sched::Job; use kernel::io_buffer::IoBufferReader; use kernel::prelude::*; diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 449552d3ade0b9..ce1d1f89e48e0d 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -28,7 +28,6 @@ use kernel::{ error::code::*, prelude::*, sync::{Arc, Guard, Mutex, UniqueArc}, - types::Opaque, }; const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; @@ -178,10 +177,10 @@ impl) + Send + Sync> GenSubmi fn complete(self: Box) { let SubmittedWork { object, - value, + value: _, error, - wptr, - vm_slot, + wptr: _, + vm_slot: _, callback, } = *self; @@ -249,7 +248,6 @@ pub(crate) struct QueueEventInfo { #[versions(AGX)] pub(crate) struct Job { wq: Arc, - wq_size: u32, event_info: QueueEventInfo::ver, start_value: EventValue, pending: Vec>, @@ -637,11 +635,6 @@ impl WorkQueue::ver { Ok(queue.into()) } - /// Returns the QueueInfo pointer for this workqueue, as a weak pointer. - pub(crate) fn info_pointer(&self) -> GpuWeakPointer { - self.info_pointer - } - pub(crate) fn event_info(&self) -> Option { let inner = self.inner.lock(); @@ -681,7 +674,6 @@ impl WorkQueue::ver { mod_pr_debug!("WorkQueue({:?}): New job\n", inner.pipe_type); Ok(Job::ver { wq: self.clone(), - wq_size: inner.size, event_info: QueueEventInfo::ver { stamp_pointer: ev.0.stamp_pointer(), fw_stamp_pointer: ev.0.fw_stamp_pointer(), @@ -775,8 +767,6 @@ impl WorkQueue for WorkQueue::ver { ); } - let last_wptr = 0; - let pipe_type = inner.pipe_type; for cmd in inner.pending.drain(..completed_commands) { From 8101a9b450f908c46b4fd0576b952c5dbf831de4 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Tue, 7 Mar 2023 22:20:58 +0900 Subject: [PATCH 497/635] drm/asahi: Add missing timeline syncobj signaling support Still untested, but it might even work now? Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 107df04140e2ba..15988af33cf35d 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -704,8 +704,13 @@ impl Queue for Queue::ver { job.push(); mod_dev_dbg!(self.dev, "Queue: Adding {} out_syncs\n", out_syncs.len()); - for sync in out_syncs { - sync.syncobj.replace_fence(Some(&out_fence)); + for mut sync in out_syncs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj + .add_point(chain, &out_fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(&out_fence)); + } } Ok(()) From cfb91548fd7ffbfba9892dc4d27a8e7314cc69e3 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Mar 2023 20:00:00 +0900 Subject: [PATCH 498/635] drm/asahi: Identify vertex attachment list This needs to go into the UAPI... Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/microseq.rs | 17 ++++++----------- drivers/gpu/drm/asahi/queue/compute.rs | 4 +--- drivers/gpu/drm/asahi/queue/render.rs | 16 +++++----------- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index 8deea3fb991488..a51e58279ec196 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -169,13 +169,8 @@ pub(crate) struct StartVertex<'a> { pub(crate) unk_64: u32, pub(crate) unk_68: u32, pub(crate) uuid: u32, - pub(crate) unk_70: u32, - pub(crate) unk_74: Array<0x1d, U64>, - pub(crate) unk_15c: u32, - pub(crate) unk_160: U64, - pub(crate) unk_168: u32, - pub(crate) unk_16c: u32, - pub(crate) unk_170: U64, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, #[ver(V >= V13_0B4)] pub(crate) counter: U64, @@ -217,7 +212,7 @@ pub(crate) struct FinalizeVertex { pub(crate) unk_68_g14: U64, pub(crate) restart_branch_offset: i32, - pub(crate) unk_70: u32, + pub(crate) has_attachments: u32, #[ver(V >= V13_0B4)] pub(crate) unk_74: Array<0x10, u8>, @@ -254,7 +249,7 @@ pub(crate) struct StartFragment<'a> { pub(crate) unk_84: u32, pub(crate) uuid: u32, pub(crate) attachments: Attachments, - pub(crate) unk_190: u32, + pub(crate) padding: u32, #[ver(V >= V13_0B4)] pub(crate) counter: U64, @@ -297,7 +292,7 @@ pub(crate) struct FinalizeFragment { pub(crate) unk_8c_g14: U64, pub(crate) restart_branch_offset: i32, - pub(crate) unk_98: u32, + pub(crate) has_attachments: u32, #[ver(V >= V13_0B4)] pub(crate) unk_9c: Array<0x10, u8>, @@ -368,7 +363,7 @@ pub(crate) struct FinalizeCompute<'a> { pub(crate) unk_5c_g14: U64, pub(crate) restart_branch_offset: i32, - pub(crate) unk_60: u32, + pub(crate) has_attachments: u32, #[ver(V >= V13_0B4)] pub(crate) unk_64: Array<0xd, u8>, diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 6590382c75afa9..822beae38f9367 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -122,8 +122,6 @@ impl super::Queue::ver { let uuid = cmdbuf.cmd_id; - let unk3 = debug_enabled(debug::DebugFlags::Debug3); - mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); // TODO: check @@ -223,7 +221,7 @@ impl super::Queue::ver { #[ver(G == G14 && V < V13_0B4)] unk_5c_g14: U64(0), restart_branch_offset: off, - unk_60: unk3.into(), + has_attachments: (cmdbuf.attachment_count > 0) as u32, #[ver(V >= V13_0B4)] unk_64: Default::default(), #[ver(V >= V13_0B4)] diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 318c952df020b9..a05ce3a8048b52 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -397,7 +397,6 @@ impl super::Queue::ver { let unk1 = debug_enabled(debug::DebugFlags::Debug1); let unk2 = debug_enabled(debug::DebugFlags::Debug2); - let unk3 = debug_enabled(debug::DebugFlags::Debug3); let mut tile_config: u64 = 0; if !unk1 { @@ -489,7 +488,7 @@ impl super::Queue::ver { cmdbuf.attachments, cmdbuf.attachment_count, )?, - unk_190: 0, + padding: 0, #[ver(V >= V13_0B4)] counter: U64(count_frag), #[ver(V >= V13_0B4)] @@ -557,7 +556,7 @@ impl super::Queue::ver { #[ver(G == G14 && V < V13_0B4)] unk_8c_g14: U64(0), restart_branch_offset: off, - unk_98: unk3.into(), + has_attachments: (cmdbuf.attachment_count > 0) as u32, #[ver(V >= V13_0B4)] unk_9c: Default::default(), })?; @@ -885,13 +884,8 @@ impl super::Queue::ver { unk_64: 0x0, // fixed unk_68: unk1.into(), uuid: uuid_ta, - unk_70: 0x0, // fixed - unk_74: Default::default(), // fixed - unk_15c: 0x0, // fixed - unk_160: U64(0x0), // fixed - unk_168: 0x0, // fixed - unk_16c: 0x0, // fixed - unk_170: U64(0x0), // fixed + attachments: Default::default(), // TODO: Vertex attachments + padding: 0, #[ver(V >= V13_0B4)] counter: U64(count_vtx), #[ver(V >= V13_0B4)] @@ -957,7 +951,7 @@ impl super::Queue::ver { #[ver(G >= G14 && V < V13_0B4)] unk_68_g14: U64(0), restart_branch_offset: off, - unk_70: 0x0, // fixed + has_attachments: 0, // TODO: Vertex attachments #[ver(V >= V13_0B4)] unk_74: Default::default(), // Ventura })?; From f6e2933770f8b6eaa7f76479b1d772f045663036 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Mar 2023 20:34:15 +0900 Subject: [PATCH 499/635] drm/asahi: Track event sequences properly Looks like these fields of the commands count events (actual GPU commands), not work commands. The previous implementation was mixing these things up... Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/job.rs | 2 +- drivers/gpu/drm/asahi/fw/microseq.rs | 6 +++--- drivers/gpu/drm/asahi/queue/compute.rs | 4 ++-- drivers/gpu/drm/asahi/queue/render.rs | 8 ++++---- drivers/gpu/drm/asahi/workqueue.rs | 11 ++++++++++- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index a0bbf67b1b1d3e..f8dcc1d8bdf44c 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -19,7 +19,7 @@ pub(crate) mod raw { pub(crate) evctl_index: u32, pub(crate) flush_stamps: u32, pub(crate) uuid: u32, - pub(crate) cmd_seq: u32, + pub(crate) event_seq: u32, } #[derive(Debug)] diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index a51e58279ec196..1adf57901a98d8 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -162,7 +162,7 @@ pub(crate) struct StartVertex<'a> { pub(crate) event_generation: u32, pub(crate) buffer_slot: u32, pub(crate) unk_44: u32, - pub(crate) cmd_seq: U64, + pub(crate) event_seq: U64, pub(crate) unk_50: u32, pub(crate) unk_pointer: GpuWeakPointer, pub(crate) unk_job_buf: GpuWeakPointer, @@ -240,7 +240,7 @@ pub(crate) struct StartFragment<'a> { pub(crate) event_generation: u32, pub(crate) buffer_slot: u32, pub(crate) unk_5c: u32, - pub(crate) cmd_seq: U64, + pub(crate) event_seq: U64, pub(crate) unk_68: u32, pub(crate) unk_758_flag: GpuWeakPointer, pub(crate) unk_job_buf: GpuWeakPointer, @@ -313,7 +313,7 @@ pub(crate) struct StartCompute<'a> { pub(crate) vm_slot: u32, pub(crate) unk_28: u32, pub(crate) event_generation: u32, - pub(crate) cmd_seq: U64, + pub(crate) event_seq: U64, pub(crate) unk_38: u32, pub(crate) job_params2: GpuWeakPointer>, pub(crate) unk_44: u32, diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 822beae38f9367..198e3bfd35ad6c 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -144,7 +144,7 @@ impl super::Queue::ver { vm_slot: vm_bind.slot(), unk_28: 0x1, event_generation: self.id as u32, - cmd_seq: U64(ev_comp.cmd_seq), + event_seq: U64(ev_comp.event_seq), unk_38: 0x0, job_params2: inner_weak_ptr!(ptr, job_params2), unk_44: 0x0, @@ -305,7 +305,7 @@ impl super::Queue::ver { evctl_index: 0, // fixed flush_stamps: flush_stamps as u32, uuid: uuid, - cmd_seq: ev_comp.cmd_seq as u32, + event_seq: ev_comp.event_seq as u32, }, cur_ts: U64(0), start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index a05ce3a8048b52..978d88dcf40ad2 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -476,7 +476,7 @@ impl super::Queue::ver { event_generation: self.id as u32, buffer_slot: scene.slot(), unk_5c: 0, - cmd_seq: U64(ev_frag.cmd_seq), + event_seq: U64(ev_frag.event_seq), unk_68: 0, unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), @@ -777,7 +777,7 @@ impl super::Queue::ver { evctl_index: 0, // fixed flush_stamps: flush_stamps as u32, uuid: uuid_3d, - cmd_seq: ev_frag.cmd_seq as u32, + event_seq: ev_frag.event_seq as u32, }, unk_after_meta: unk1.into(), unk_buf_0: U64(0), @@ -877,7 +877,7 @@ impl super::Queue::ver { event_generation: self.id as u32, buffer_slot: scene.slot(), unk_44: 0, - cmd_seq: U64(ev_vtx.cmd_seq), + event_seq: U64(ev_vtx.event_seq), unk_50: 0, unk_pointer: inner_weak_ptr!(ptr, unk_pointee), unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), @@ -1100,7 +1100,7 @@ impl super::Queue::ver { evctl_index: 0, // fixed flush_stamps: flush_stamps as u32, uuid: uuid_ta, - cmd_seq: ev_vtx.cmd_seq as u32, + event_seq: ev_vtx.event_seq as u32, }, unk_after_meta: unk1.into(), unk_buf_0: U64(0), diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index ce1d1f89e48e0d..ab0178c37a2736 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -214,6 +214,7 @@ struct WorkQueueInner { priority: u32, commit_seq: u64, submit_seq: u64, + event_seq: u64, } /// An instance of a work queue. @@ -242,6 +243,7 @@ pub(crate) struct QueueEventInfo { pub(crate) slot: u32, pub(crate) value: event::EventValue, pub(crate) cmd_seq: u64, + pub(crate) event_seq: u64, pub(crate) info_ptr: GpuWeakPointer, } @@ -268,7 +270,8 @@ pub(crate) struct JobSubmission<'a> { impl Job::ver { pub(crate) fn event_info(&self) -> QueueEventInfo::ver { let mut info = self.event_info; - info.cmd_seq += self.event_count as u64; + info.cmd_seq += self.pending.len() as u64; + info.event_seq += self.event_count as u64; info } @@ -336,6 +339,7 @@ impl Job::ver { ev.1 = self.event_info.value; inner.commit_seq += self.pending.len() as u64; + inner.event_seq += self.event_count as u64; self.committed = true; Ok(()) @@ -480,6 +484,7 @@ impl Drop for Job::ver { ); event.1.sub(self.event_count as u32); inner.commit_seq -= self.pending.len() as u64; + inner.event_seq -= self.event_count as u64; } inner.pending_jobs -= 1; @@ -515,6 +520,7 @@ impl<'a> Drop for JobSubmission::ver<'a> { ); event.1.sub(self.event_count as u32); inner.commit_seq -= self.command_count as u64; + inner.event_seq -= self.event_count as u64; mod_pr_debug!("WorkQueue({:?}): Dropped JobSubmission\n", inner.pipe_type); } } @@ -614,6 +620,7 @@ impl WorkQueue::ver { pending_jobs: 0, commit_seq: 0, submit_seq: 0, + event_seq: 0, last_completed: None, last_submitted: None, }; @@ -644,6 +651,7 @@ impl WorkQueue::ver { slot: ev.0.slot(), value: ev.1, cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, info_ptr: self.info_pointer, }) } @@ -680,6 +688,7 @@ impl WorkQueue::ver { slot: ev.0.slot(), value: ev.1, cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, info_ptr: self.info_pointer, }, start_value: ev.1, From 5db3307ce2325c6885a900cdad7b5785770c1b9b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Mar 2023 22:21:29 +0900 Subject: [PATCH 500/635] drm/asahi: Fix compute stats size (maybe) Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/initdata.rs | 18 ++++++++++++------ drivers/gpu/drm/asahi/fw/microseq.rs | 4 ++-- drivers/gpu/drm/asahi/initdata.rs | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs index 44de0c1cccf35a..473adf3326bdb1 100644 --- a/drivers/gpu/drm/asahi/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -807,12 +807,16 @@ pub(crate) mod raw { #[versions(AGX)] default_zeroed!(GpuGlobalStatsFrag::ver); + #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct GpuStatsComp { - pub(crate) unk: Array<0x140, u8>, + pub(crate) unk: Array<0x180, u8>, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x580, u8>, } - default_zeroed!(GpuStatsComp); + #[versions(AGX)] + default_zeroed!(GpuStatsComp::ver); #[derive(Debug)] #[repr(C)] @@ -871,7 +875,7 @@ pub(crate) mod raw { pub(crate) unk_168: U64, pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx::ver>, pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, - pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp::ver>, pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, @@ -1185,11 +1189,13 @@ impl GpuStruct for GpuGlobalStatsFrag::ver { type Raw<'a> = raw::GpuGlobalStatsFrag::ver; } +#[versions(AGX)] #[derive(Debug, Default)] pub(crate) struct GpuStatsComp {} -impl GpuStruct for GpuStatsComp { - type Raw<'a> = raw::GpuStatsComp; +#[versions(AGX)] +impl GpuStruct for GpuStatsComp::ver { + type Raw<'a> = raw::GpuStatsComp::ver; } #[versions(AGX)] @@ -1215,7 +1221,7 @@ impl GpuStruct for HwDataB::ver { pub(crate) struct Stats { pub(crate) vtx: GpuObject, pub(crate) frag: GpuObject, - pub(crate) comp: GpuObject, + pub(crate) comp: GpuObject, } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index 1adf57901a98d8..274a2c7f741537 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -308,7 +308,7 @@ pub(crate) struct StartCompute<'a> { pub(crate) header: op::StartCompute, pub(crate) unk_pointer: GpuWeakPointer>, pub(crate) job_params1: GpuWeakPointer>, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, pub(crate) vm_slot: u32, pub(crate) unk_28: u32, @@ -339,7 +339,7 @@ impl<'a> Operation for StartCompute::ver<'a> {} #[repr(C)] pub(crate) struct FinalizeCompute<'a> { pub(crate) header: op::FinalizeCompute, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, pub(crate) vm_slot: u32, #[ver(V < V13_0B4)] diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 472c42169130b3..7b36df4ab151c0 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -640,7 +640,7 @@ impl<'a> InitDataBuilder::ver<'a> { )) }, )?, - comp: self.alloc.private.new_default::()?, + comp: self.alloc.private.new_default::()?, }, hwdata_a: hwa, From 19b3dc27055c3213ff4231b19cb84cc062e6761f Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Mar 2023 22:21:40 +0900 Subject: [PATCH 501/635] drm/asahi: Identify no_preemption flag Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/job.rs | 4 +++- drivers/gpu/drm/asahi/queue/compute.rs | 5 ++++- drivers/gpu/drm/asahi/queue/render.rs | 8 ++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index f8dcc1d8bdf44c..7027a763dffb11 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -11,7 +11,9 @@ pub(crate) mod raw { #[derive(Debug, Clone, Copy)] #[repr(C)] pub(crate) struct JobMeta { - pub(crate) unk_4: u32, + pub(crate) unk_0: u16, + pub(crate) unk_2: u8, + pub(crate) no_preemption: u8, pub(crate) stamp: GpuWeakPointer, pub(crate) fw_stamp: GpuWeakPointer, pub(crate) stamp_value: EventValue, diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 198e3bfd35ad6c..2df05ddf09b018 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -297,7 +297,10 @@ impl super::Queue::ver { unk_28: U64(0x0), // fixed }, meta: fw::job::raw::JobMeta { - unk_4: 0, + unk_0: 0, + unk_2: 0, + // TODO: make separate flag + no_preemption: ((cmdbuf.ctx_switch_prog & 1) == 0) as u8, stamp: ev_comp.stamp_pointer, fw_stamp: ev_comp.fw_stamp_pointer, stamp_value: ev_comp.value.next(), diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 978d88dcf40ad2..75673661bc9608 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -769,7 +769,9 @@ impl super::Queue::ver { unk_param: unk2.into(), // 1 for boot stuff? unk_pointee: 0, meta: fw::job::raw::JobMeta { - unk_4: 0, + unk_0: 0, + unk_2: 0, + no_preemption: 0, stamp: ev_frag.stamp_pointer, fw_stamp: ev_frag.fw_stamp_pointer, stamp_value: ev_frag.value.next(), @@ -1092,7 +1094,9 @@ impl super::Queue::ver { unk_568: 0, unk_56c: 0, meta: fw::job::raw::JobMeta { - unk_4: 0, + unk_0: 0, + unk_2: 0, + no_preemption: 0, stamp: ev_vtx.stamp_pointer, fw_stamp: ev_vtx.fw_stamp_pointer, stamp_value: ev_vtx.value.next(), From 2d490b48a02fdd67a812450f0c5f606a774e0b1d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 22 Mar 2023 16:17:59 +0900 Subject: [PATCH 502/635] drm/asahi: Add a flag to panic (oops) on GPU crashes This is useful so the driver won't try to abort everything, which leaves memory mostly untouched so it can be inspected from the hypervisor. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/debug.rs | 2 ++ drivers/gpu/drm/asahi/gpu.rs | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index 2f3a70e04cfd6b..f03a3f991d74a4 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -64,6 +64,8 @@ pub(crate) enum DebugFlags { Debug5 = 53, Debug6 = 54, Debug7 = 55, + + OopsOnGpuCrash = 63, } /// Update the cached global debug flags from the module parameter diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 3000872a2485e3..c7e3a5bb989756 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -278,10 +278,15 @@ impl rtkit::Operations for GpuManager::ver { fn crashed(data: ::Borrowed<'_>) { let dev = &data.dev; - dev_err!(dev, "GPU firmware crashed, failing all jobs\n"); data.crashed.store(true, Ordering::Relaxed); - data.event_manager.fail_all(workqueue::WorkError::NoDevice); + + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware crashed"); + } else { + dev_err!(dev, "GPU firmware crashed, failing all jobs\n"); + data.event_manager.fail_all(workqueue::WorkError::NoDevice); + } } fn shmem_alloc( From 8b5abf161a71cec718a596eb69bfe70fd77a797c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 22 Mar 2023 15:52:50 +0900 Subject: [PATCH 503/635] drm/asahi: Align GEM object sizes to the UAT page size Makes sure object sizes are 16K aligned on 4K kernels. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gem.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index cf7271d51b236d..f87131de1b4b43 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -20,9 +20,7 @@ use kernel::drm::gem::BaseObject; use core::sync::atomic::{AtomicU64, Ordering}; -use crate::debug::*; -use crate::driver::AsahiDevice; -use crate::file::DrmFile; +use crate::{debug::*, driver::AsahiDevice, file::DrmFile, mmu, util::*}; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -221,7 +219,7 @@ impl ObjectRef { /// Create a new kernel-owned GEM object. pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { - let mut gem = shmem::Object::::new(dev, size)?; + let mut gem = shmem::Object::::new(dev, align(size, mmu::UAT_PGSZ))?; gem.kernel = true; gem.flags = 0; @@ -238,7 +236,7 @@ pub(crate) fn new_object( flags: u32, vm_id: Option, ) -> Result { - let mut gem = shmem::Object::::new(dev, size)?; + let mut gem = shmem::Object::::new(dev, align(size, mmu::UAT_PGSZ))?; gem.kernel = false; gem.flags = flags; gem.vm_id = vm_id; From 629a3a824d11f100b5cfe8bea138cd4bc4e6406c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 15 Mar 2023 21:27:58 +0900 Subject: [PATCH 504/635] drm/asahi: Identify and set barrier_type field Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/workqueue.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 2 +- drivers/gpu/drm/asahi/queue/render.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs index e81025b6c014a1..86472bd6f7ae53 100644 --- a/drivers/gpu/drm/asahi/fw/workqueue.rs +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -34,7 +34,7 @@ pub(crate) mod raw { pub(crate) wait_slot: u32, pub(crate) stamp_self: EventValue, pub(crate) uuid: u32, - pub(crate) unk: u32, + pub(crate) barrier_type: u32, } #[derive(Debug, Clone, Copy)] diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 15988af33cf35d..a7d7af8612bdd9 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -613,7 +613,7 @@ impl Queue for Queue::ver { wait_slot: event.slot, stamp_self: queue_job.event_info().value.next(), uuid: 0xffffbbbb, - unk: 0, + barrier_type: 0, } )) }, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 75673661bc9608..79693469774c23 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -384,7 +384,7 @@ impl super::Queue::ver { wait_slot: ev_vtx.slot, stamp_self: ev_frag.value.next(), uuid: uuid_3d, - unk: 0, + barrier_type: 0, } )) }, From 7e7954c951b232ad0a0f3c45a8bb81eb072c02fd Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 21:28:29 +0900 Subject: [PATCH 505/635] arm64: dts: apple: Add T602x GPU node Signed-off-by: Asahi Lina --- arch/arm64/boot/dts/apple/t6021.dtsi | 3 + arch/arm64/boot/dts/apple/t602x-common.dtsi | 78 +++++++++++++++++++++ arch/arm64/boot/dts/apple/t602x-die0.dtsi | 49 ++++++++++++- 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 5a46ca207a1d5c..e304014456ed13 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -19,6 +19,9 @@ #ifndef GPU_REPEAT # define GPU_REPEAT(x) #endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif #include "t602x-common.dtsi" diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index 1224e4f12a5347..80ba81c47a3424 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -9,6 +9,10 @@ #address-cells = <2>; #size-cells = <2>; + aliases { + gpu = &gpu; + }; + cpus { #address-cells = <2>; #size-cells = <0>; @@ -434,6 +438,80 @@ }; }; + gpu_cs_opp: opp-table-gpu-cs { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <1024000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1140000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + + gpu_afr_opp: opp-table-gpu-afr { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <552000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <760000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <980000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1098000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + pmu-e { compatible = "apple,blizzard-pmu"; interrupt-parent = <&aic>; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 6ca16f05b3ede8..9b42022efffcf0 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -593,8 +593,55 @@ apple,firmware-compat = <0 0 0>; operating-points-v2 = <&gpu_opp>; - /* TODO perf stuff */ + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + + apple,min-sram-microvolt = <790000>; + apple,csafr-min-sram-microvolt = <812000>; apple,perf-base-pstate = <1>; + + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <1.5125>; + apple,avg-power-kp = <0.38>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-integral-gain = <700.0>; + apple,fast-die0-proportional-gain = <34.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <1.62>; + apple,perf-integral-gain2 = <1.62>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <5.4>; + apple,perf-proportional-gain = <5.4>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; + apple,ppm-kp = <0.1>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + apple,pwr-sample-period-aic-clks = <200000>; + apple,se-engagement-criteria = <700>; + apple,se-filter-time-constant = <9>; + apple,se-filter-time-constant-1 = <3>; + apple,se-inactive-threshold = <2500>; + apple,se-ki = <-50.0>; + apple,se-ki-1 = <-100.0>; + apple,se-kp = <-5.0>; + apple,se-kp-1 = <-10.0>; + apple,se-reset-criteria = <50>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); + apple,cs-leak-coef = GPU_DIE_REPEAT(400.0); + apple,afr-leak-coef = GPU_DIE_REPEAT(200.0); }; agx_mbox: mbox@406408000 { From 3a2c32a653d5fb144ef0104bf92888eccfe196ec Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 22 Mar 2023 16:14:56 +0900 Subject: [PATCH 506/635] drm/asahi: Make multi-page TLB invals use the CPU page size. This is just a guess, but worst case we over-invalidate which is better than under-invalidating. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/mem.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 491d4f8a4016c9..05f9a796862450 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -80,6 +80,11 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); let pages = end_pg - start_pg; + // Guess? It's possible that the page count is in terms of 4K pages + // when the CPU is in 4K mode... + #[cfg(CONFIG_ARM64_4K_PAGES)] + let pages = 4 * pages; + if pages == 1 { tlbi_page(asid, va); return; From 1f22a4904827d8e87e01a9f488c981bbf4953f5d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 29 Mar 2023 20:22:08 +0900 Subject: [PATCH 507/635] drm/asahi: Move to uapi crate Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/file.rs | 77 +++++++++++++------------- drivers/gpu/drm/asahi/gem.rs | 4 +- drivers/gpu/drm/asahi/hw/mod.rs | 4 +- drivers/gpu/drm/asahi/queue/common.rs | 6 +- drivers/gpu/drm/asahi/queue/compute.rs | 16 +++--- drivers/gpu/drm/asahi/queue/mod.rs | 35 ++++++------ drivers/gpu/drm/asahi/queue/render.rs | 48 ++++++++-------- drivers/gpu/drm/asahi/workqueue.rs | 26 ++++----- 8 files changed, 108 insertions(+), 108 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index fa554e1a12c9af..7f575816b84d02 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -17,7 +17,7 @@ use kernel::io_buffer::{IoBufferReader, IoBufferWriter}; use kernel::prelude::*; use kernel::sync::{smutex::Mutex, Arc}; use kernel::user_ptr::UserSlicePtr; -use kernel::{bindings, dma_fence, drm, xarray}; +use kernel::{dma_fence, drm, uapi, xarray}; const DEBUG_CLASS: DebugFlags = DebugFlags::File; @@ -49,13 +49,13 @@ pub(crate) struct SyncItem { } impl SyncItem { - fn parse_one(file: &DrmFile, data: bindings::drm_asahi_sync, out: bool) -> Result { + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { if data.extensions != 0 { return Err(EINVAL); } match data.sync_type { - bindings::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { return Err(EINVAL); } @@ -72,7 +72,7 @@ impl SyncItem { timeline_value: data.timeline_value, }) } - bindings::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; let fence = if out { None @@ -103,14 +103,14 @@ impl SyncItem { fn parse_array(file: &DrmFile, ptr: u64, count: u32, out: bool) -> Result> { let mut vec = Vec::try_with_capacity(count as usize)?; - const STRIDE: usize = core::mem::size_of::(); + const STRIDE: usize = core::mem::size_of::(); let size = STRIDE * count as usize; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = unsafe { UserSlicePtr::new(ptr as usize as *mut _, size).reader() }; for _i in 0..count { - let mut sync: MaybeUninit = MaybeUninit::uninit(); + let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE unsafe { reader.read_raw(sync.as_mut_ptr() as *mut u8, STRIDE)? }; @@ -191,7 +191,7 @@ impl File { /// IOCTL: get_param: Get a driver parameter value. pub(crate) fn get_params( device: &AsahiDevice, - data: &mut bindings::drm_asahi_get_params, + data: &mut uapi::drm_asahi_get_params, file: &DrmFile, ) -> Result { mod_dev_dbg!(device, "[File {}]: IOCTL: get_params\n", file.inner().id); @@ -202,8 +202,8 @@ impl File { return Err(EINVAL); } - let mut params = bindings::drm_asahi_params_global { - unstable_uabi_version: bindings::DRM_ASAHI_UNSTABLE_UABI_VERSION, + let mut params = uapi::drm_asahi_params_global { + unstable_uabi_version: uapi::DRM_ASAHI_UNSTABLE_UABI_VERSION, pad0: 0, feat_compat: gpu.get_cfg().gpu_feat_compat, @@ -220,7 +220,7 @@ impl File { num_frags_per_cluster: gpu.get_dyncfg().id.num_frags, num_gps_per_cluster: gpu.get_dyncfg().id.num_gps, num_cores_total_active: gpu.get_dyncfg().id.total_active_cores, - core_masks: [0; bindings::DRM_ASAHI_MAX_CLUSTERS as usize], + core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], vm_page_size: mmu::UAT_PGSZ as u32, pad1: 0, @@ -239,16 +239,15 @@ impl File { max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), max_power_mw: gpu.get_dyncfg().pwr.max_power_mw, - result_render_size: core::mem::size_of::() as u32, - result_compute_size: core::mem::size_of::() as u32, + result_render_size: core::mem::size_of::() as u32, + result_compute_size: core::mem::size_of::() as u32, }; for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).try_into()?; } - let size = - core::mem::size_of::().min(data.size.try_into()?); + let size = core::mem::size_of::().min(data.size.try_into()?); // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = @@ -263,7 +262,7 @@ impl File { /// IOCTL: vm_create: Create a new `Vm`. pub(crate) fn vm_create( device: &AsahiDevice, - data: &mut bindings::drm_asahi_vm_create, + data: &mut uapi::drm_asahi_vm_create, file: &DrmFile, ) -> Result { if data.extensions != 0 { @@ -335,7 +334,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDevice, - data: &mut bindings::drm_asahi_vm_destroy, + data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { if data.extensions != 0 { @@ -352,7 +351,7 @@ impl File { /// IOCTL: gem_create: Create a new GEM object. pub(crate) fn gem_create( device: &AsahiDevice, - data: &mut bindings::drm_asahi_gem_create, + data: &mut uapi::drm_asahi_gem_create, file: &DrmFile, ) -> Result { mod_dev_dbg!( @@ -363,13 +362,13 @@ impl File { ); if data.extensions != 0 - || (data.flags & !(bindings::ASAHI_GEM_WRITEBACK | bindings::ASAHI_GEM_VM_PRIVATE)) != 0 - || (data.flags & bindings::ASAHI_GEM_VM_PRIVATE == 0 && data.vm_id != 0) + || (data.flags & !(uapi::ASAHI_GEM_WRITEBACK | uapi::ASAHI_GEM_VM_PRIVATE)) != 0 + || (data.flags & uapi::ASAHI_GEM_VM_PRIVATE == 0 && data.vm_id != 0) { return Err(EINVAL); } - let vm_id = if data.flags & bindings::ASAHI_GEM_VM_PRIVATE != 0 { + let vm_id = if data.flags & uapi::ASAHI_GEM_VM_PRIVATE != 0 { Some( file.inner() .vms() @@ -402,7 +401,7 @@ impl File { /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. pub(crate) fn gem_mmap_offset( device: &AsahiDevice, - data: &mut bindings::drm_asahi_gem_mmap_offset, + data: &mut uapi::drm_asahi_gem_mmap_offset, file: &DrmFile, ) -> Result { mod_dev_dbg!( @@ -424,7 +423,7 @@ impl File { /// IOCTL: gem_bind: Map or unmap a GEM object into a Vm. pub(crate) fn gem_bind( device: &AsahiDevice, - data: &mut bindings::drm_asahi_gem_bind, + data: &mut uapi::drm_asahi_gem_bind, file: &DrmFile, ) -> Result { mod_dev_dbg!( @@ -445,9 +444,9 @@ impl File { } match data.op { - bindings::drm_asahi_bind_op_ASAHI_BIND_OP_BIND => Self::do_gem_bind(device, data, file), - bindings::drm_asahi_bind_op_ASAHI_BIND_OP_UNBIND => Err(ENOTSUPP), - bindings::drm_asahi_bind_op_ASAHI_BIND_OP_UNBIND_ALL => { + uapi::drm_asahi_bind_op_ASAHI_BIND_OP_BIND => Self::do_gem_bind(device, data, file), + uapi::drm_asahi_bind_op_ASAHI_BIND_OP_UNBIND => Err(ENOTSUPP), + uapi::drm_asahi_bind_op_ASAHI_BIND_OP_UNBIND_ALL => { Self::do_gem_unbind_all(device, data, file) } _ => Err(EINVAL), @@ -456,7 +455,7 @@ impl File { pub(crate) fn do_gem_bind( _device: &AsahiDevice, - data: &mut bindings::drm_asahi_gem_bind, + data: &mut uapi::drm_asahi_gem_bind, file: &DrmFile, ) -> Result { if data.offset != 0 { @@ -467,7 +466,7 @@ impl File { return Err(EINVAL); // Must be page aligned } - if (data.flags & !(bindings::ASAHI_BIND_READ | bindings::ASAHI_BIND_WRITE)) != 0 { + if (data.flags & !(uapi::ASAHI_BIND_READ | uapi::ASAHI_BIND_WRITE)) != 0 { return Err(EINVAL); } @@ -497,13 +496,13 @@ impl File { return Err(EINVAL); } - let prot = if data.flags & bindings::ASAHI_BIND_READ != 0 { - if data.flags & bindings::ASAHI_BIND_WRITE != 0 { + let prot = if data.flags & uapi::ASAHI_BIND_READ != 0 { + if data.flags & uapi::ASAHI_BIND_WRITE != 0 { mmu::PROT_GPU_SHARED_RW } else { mmu::PROT_GPU_SHARED_RO } - } else if data.flags & bindings::ASAHI_BIND_WRITE != 0 { + } else if data.flags & uapi::ASAHI_BIND_WRITE != 0 { mmu::PROT_GPU_SHARED_WO } else { return Err(EINVAL); // Must specify one of ASAHI_BIND_{READ,WRITE} @@ -526,7 +525,7 @@ impl File { pub(crate) fn do_gem_unbind_all( _device: &AsahiDevice, - data: &mut bindings::drm_asahi_gem_bind, + data: &mut uapi::drm_asahi_gem_bind, file: &DrmFile, ) -> Result { if data.flags != 0 || data.offset != 0 || data.range != 0 || data.addr != 0 { @@ -555,7 +554,7 @@ impl File { /// IOCTL: queue_create: Create a new command submission queue of a given type. pub(crate) fn queue_create( device: &AsahiDevice, - data: &mut bindings::drm_asahi_queue_create, + data: &mut uapi::drm_asahi_queue_create, file: &DrmFile, ) -> Result { let file_id = file.inner().id; @@ -575,9 +574,9 @@ impl File { || data.priority > 3 || data.queue_caps == 0 || (data.queue_caps - & !(bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER - | bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_BLIT - | bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE)) + & !(uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER + | uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_BLIT + | uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE)) != 0 { return Err(EINVAL); @@ -610,7 +609,7 @@ impl File { /// IOCTL: queue_destroy: Destroy a command submission queue. pub(crate) fn queue_destroy( _device: &AsahiDevice, - data: &mut bindings::drm_asahi_queue_destroy, + data: &mut uapi::drm_asahi_queue_destroy, file: &DrmFile, ) -> Result { if data.extensions != 0 { @@ -632,7 +631,7 @@ impl File { /// IOCTL: submit: Submit GPU work to a command submission queue. pub(crate) fn submit( device: &AsahiDevice, - data: &mut bindings::drm_asahi_submit, + data: &mut uapi::drm_asahi_submit, file: &DrmFile, ) -> Result { if data.extensions != 0 @@ -707,7 +706,7 @@ impl File { ); let mut commands = Vec::try_with_capacity(data.command_count as usize)?; - const STRIDE: usize = core::mem::size_of::(); + const STRIDE: usize = core::mem::size_of::(); let size = STRIDE * data.command_count as usize; // SAFETY: We only read this once, so there are no TOCTOU issues. @@ -715,7 +714,7 @@ impl File { unsafe { UserSlicePtr::new(data.commands as usize as *mut _, size).reader() }; for _i in 0..data.command_count { - let mut cmd: MaybeUninit = MaybeUninit::uninit(); + let mut cmd: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE unsafe { reader.read_raw(cmd.as_mut_ptr() as *mut u8, STRIDE)? }; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index f87131de1b4b43..2d6eb213268de5 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -8,12 +8,12 @@ //! implementing RTKit buffers on top of GEM objects for firmware use. use kernel::{ - bindings, drm::{gem, gem::shmem}, error::Result, prelude::*, soc::apple::rtkit, sync::smutex::Mutex, + uapi, }; use kernel::drm::gem::BaseObject; @@ -242,7 +242,7 @@ pub(crate) fn new_object( gem.vm_id = vm_id; gem.set_exportable(vm_id.is_none()); - gem.set_wc(flags & bindings::ASAHI_GEM_WRITEBACK == 0); + gem.set_wc(flags & uapi::ASAHI_GEM_WRITEBACK == 0); mod_pr_debug!( "DriverObject new user object: vm_id={:?} id={}\n", diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index a92bb70aeae808..5d3e89aef0d31b 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -90,11 +90,11 @@ pub(crate) mod feat { /// Backwards-incompatible features. pub(crate) mod incompat { - use kernel::bindings; + use kernel::uapi; /// Hardware requires Z/S compression to be mandatorily enabled. pub(crate) const MANDATORY_ZS_COMPRESSION: u64 = - bindings::drm_asahi_feat_incompat_DRM_ASAHI_FEAT_MANDATORY_ZS_COMPRESSION as u64; + uapi::drm_asahi_feat_incompat_DRM_ASAHI_FEAT_MANDATORY_ZS_COMPRESSION as u64; } } diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index 127b4ccc6eca05..9dbc345d1dfef9 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -7,9 +7,9 @@ use crate::fw::microseq; use crate::fw::types::*; -use kernel::bindings; use kernel::io_buffer::IoBufferReader; use kernel::prelude::*; +use kernel::uapi; use kernel::user_ptr::UserSlicePtr; use core::mem::MaybeUninit; @@ -19,7 +19,7 @@ pub(super) fn build_attachments(pointer: u64, count: u32) -> Result(); + const STRIDE: usize = core::mem::size_of::(); let size = STRIDE * count as usize; // SAFETY: We only read this once, so there are no TOCTOU issues. @@ -28,7 +28,7 @@ pub(super) fn build_attachments(pointer: u64, count: u32) -> Result = MaybeUninit::uninit(); + let mut att: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `att` is STRIDE unsafe { reader.read_raw(att.as_mut_ptr() as *mut u8, STRIDE)? }; diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 2df05ddf09b018..995371470a2584 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -16,12 +16,12 @@ use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; use crate::{fw, gpu, microseq}; use core::mem::MaybeUninit; use core::sync::atomic::Ordering; -use kernel::bindings; use kernel::dma_fence::RawDmaFence; use kernel::drm::sched::Job; use kernel::io_buffer::IoBufferReader; use kernel::prelude::*; use kernel::sync::Arc; +use kernel::uapi; use kernel::user_ptr::UserSlicePtr; const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; @@ -32,12 +32,12 @@ impl super::Queue::ver { pub(super) fn submit_compute( &self, job: &mut Job, - cmd: &bindings::drm_asahi_command, + cmd: &uapi::drm_asahi_command, result_writer: Option, id: u64, flush_stamps: bool, ) -> Result { - if cmd.cmd_type != bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE { + if cmd.cmd_type != uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE { return Err(EINVAL); } @@ -58,16 +58,16 @@ impl super::Queue::ver { let mut cmdbuf_reader = unsafe { UserSlicePtr::new( cmd.cmd_buffer as usize as *mut _, - core::mem::size_of::(), + core::mem::size_of::(), ) .reader() }; - let mut cmdbuf: MaybeUninit = MaybeUninit::uninit(); + let mut cmdbuf: MaybeUninit = MaybeUninit::uninit(); unsafe { cmdbuf_reader.read_raw( cmdbuf.as_mut_ptr() as *mut u8, - core::mem::size_of::(), + core::mem::size_of::(), )?; } let cmdbuf = unsafe { cmdbuf.assume_init() }; @@ -342,7 +342,7 @@ impl super::Queue::ver { fence.set_error(err.into()) } if let Some(mut rw) = result_writer { - let mut result: bindings::drm_asahi_result_compute = Default::default(); + let mut result: uapi::drm_asahi_result_compute = Default::default(); cmd.timestamps.with(|raw, _inner| { result.ts_start = raw.start.load(Ordering::Relaxed); @@ -352,7 +352,7 @@ impl super::Queue::ver { if let Some(err) = error { result.info = err.into(); } else { - result.info.status = bindings::drm_asahi_status_DRM_ASAHI_STATUS_COMPLETE; + result.info.status = uapi::drm_asahi_status_DRM_ASAHI_STATUS_COMPLETE; } rw.write(result); diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index a7d7af8612bdd9..47f88b4bb21f04 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -8,11 +8,12 @@ use kernel::dma_fence::*; use kernel::prelude::*; use kernel::{ - bindings, c_str, dma_fence, + c_str, dma_fence, drm::gem::shmem::VMap, drm::sched, macros::versions, sync::{smutex::Mutex, Arc}, + uapi, }; use crate::alloc::Allocator; @@ -42,7 +43,7 @@ pub(crate) trait Queue: Send + Sync { in_syncs: Vec, out_syncs: Vec, result_buf: Option, - commands: Vec, + commands: Vec, ) -> Result; } @@ -414,7 +415,7 @@ impl Queue::ver { }; // Rendering structures - if caps & bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER != 0 { + if caps & uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER != 0 { let buffer = buffer::Buffer::ver::new(&*data.gpu, alloc, ret.ualloc.clone(), ualloc_priv, mgr)?; let tvb_blocks = { @@ -441,8 +442,8 @@ impl Queue::ver { // Rendering & blit structures if caps - & (bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER - | bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_BLIT) + & (uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER + | uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_BLIT) != 0 { ret.q_frag = Some(SubQueue::ver { @@ -460,7 +461,7 @@ impl Queue::ver { } // Compute structures - if caps & bindings::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE != 0 { + if caps & uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE != 0 { ret.q_comp = Some(SubQueue::ver { wq: workqueue::WorkQueue::ver::new( alloc, @@ -480,9 +481,9 @@ impl Queue::ver { } } -const SQ_RENDER: usize = bindings::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_RENDER as usize; -const SQ_COMPUTE: usize = bindings::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_COMPUTE as usize; -const SQ_COUNT: usize = bindings::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_COUNT as usize; +const SQ_RENDER: usize = uapi::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_RENDER as usize; +const SQ_COMPUTE: usize = uapi::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_COMPUTE as usize; +const SQ_COUNT: usize = uapi::drm_asahi_subqueue_DRM_ASAHI_SUBQUEUE_COUNT as usize; #[versions(AGX)] impl Queue for Queue::ver { @@ -492,7 +493,7 @@ impl Queue for Queue::ver { in_syncs: Vec, out_syncs: Vec, result_buf: Option, - commands: Vec, + commands: Vec, ) -> Result { let dev = self.dev.data(); let gpu = match dev @@ -576,8 +577,8 @@ impl Queue for Queue::ver { for (i, cmd) in commands.iter().enumerate() { match cmd.cmd_type { - bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => last_render = Some(i), - bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => last_compute = Some(i), + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => last_render = Some(i), + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => last_compute = Some(i), _ => return Err(EINVAL), } } @@ -590,14 +591,14 @@ impl Queue for Queue::ver { ); for (i, cmd) in commands.into_iter().enumerate() { for (queue_idx, index) in cmd.barriers.iter().enumerate() { - if *index == bindings::DRM_ASAHI_BARRIER_NONE as u32 { + if *index == uapi::DRM_ASAHI_BARRIER_NONE as u32 { continue; } if let Some(event) = events[queue_idx].get(*index as usize).ok_or(EINVAL)? { let mut alloc = gpu.alloc(); let queue_job = match cmd.cmd_type { - bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, - bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), }; mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); @@ -654,7 +655,7 @@ impl Queue for Queue::ver { }; match cmd.cmd_type { - bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { self.submit_render( &mut job, &cmd, @@ -672,7 +673,7 @@ impl Queue for Queue::ver { .event_info(), ))?; } - bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { self.submit_compute( &mut job, &cmd, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 79693469774c23..f93b8db6e96405 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -18,12 +18,12 @@ use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; use crate::{buffer, fw, gpu, microseq, workqueue}; use core::mem::MaybeUninit; use core::sync::atomic::Ordering; -use kernel::bindings; use kernel::dma_fence::RawDmaFence; use kernel::drm::sched::Job; use kernel::io_buffer::IoBufferReader; use kernel::prelude::*; use kernel::sync::{smutex::Mutex, Arc}; +use kernel::uapi; use kernel::user_ptr::UserSlicePtr; const DEBUG_CLASS: DebugFlags = DebugFlags::Render; @@ -35,7 +35,7 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Render; const TILECTL_DISABLE_CLUSTERING: u32 = 1u32 << 0; struct RenderResult { - result: bindings::drm_asahi_result_render, + result: uapi::drm_asahi_result_render, vtx_complete: bool, frag_complete: bool, vtx_error: Option, @@ -59,7 +59,7 @@ impl RenderResult { if let Some(err) = error { self.result.info = err.into(); } else { - self.result.info.status = bindings::drm_asahi_status_DRM_ASAHI_STATUS_COMPLETE; + self.result.info.status = uapi::drm_asahi_status_DRM_ASAHI_STATUS_COMPLETE; } self.writer.write(self.result); @@ -70,7 +70,7 @@ impl RenderResult { impl super::Queue::ver { /// Get the appropriate tiling parameters for a given userspace command buffer. fn get_tiling_params( - cmdbuf: &bindings::drm_asahi_cmd_render, + cmdbuf: &uapi::drm_asahi_cmd_render, num_clusters: u32, ) -> Result { let width: u32 = cmdbuf.fb_width; @@ -192,12 +192,12 @@ impl super::Queue::ver { pub(super) fn submit_render( &self, job: &mut Job, - cmd: &bindings::drm_asahi_command, + cmd: &uapi::drm_asahi_command, result_writer: Option, id: u64, flush_stamps: bool, ) -> Result { - if cmd.cmd_type != bindings::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { + if cmd.cmd_type != uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { return Err(EINVAL); } @@ -206,32 +206,32 @@ impl super::Queue::ver { let mut cmdbuf_reader = unsafe { UserSlicePtr::new( cmd.cmd_buffer as usize as *mut _, - core::mem::size_of::(), + core::mem::size_of::(), ) .reader() }; - let mut cmdbuf: MaybeUninit = MaybeUninit::uninit(); + let mut cmdbuf: MaybeUninit = MaybeUninit::uninit(); unsafe { cmdbuf_reader.read_raw( cmdbuf.as_mut_ptr() as *mut u8, - core::mem::size_of::(), + core::mem::size_of::(), )?; } let cmdbuf = unsafe { cmdbuf.assume_init() }; if cmdbuf.flags - & !(bindings::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES - | bindings::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S - | bindings::ASAHI_RENDER_MEMORYLESS_RTS_USED - | bindings::ASAHI_RENDER_PROCESS_EMPTY_TILES - | bindings::ASAHI_RENDER_NO_VERTEX_CLUSTERING) as u64 + & !(uapi::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES + | uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S + | uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED + | uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES + | uapi::ASAHI_RENDER_NO_VERTEX_CLUSTERING) as u64 != 0 { return Err(EINVAL); } - if cmdbuf.flags & bindings::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 != 0 { + if cmdbuf.flags & uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 != 0 { // Not supported yet return Err(EINVAL); } @@ -267,7 +267,7 @@ impl super::Queue::ver { let mut clustering = nclusters > 1; if debug_enabled(debug::DebugFlags::DisableClustering) - || cmdbuf.flags & bindings::ASAHI_RENDER_NO_VERTEX_CLUSTERING as u64 != 0 + || cmdbuf.flags & uapi::ASAHI_RENDER_NO_VERTEX_CLUSTERING as u64 != 0 { clustering = false; } @@ -405,7 +405,7 @@ impl super::Queue::ver { if cmdbuf.layers > 1 { tile_config |= 1; } - if cmdbuf.flags & bindings::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 != 0 { + if cmdbuf.flags & uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 != 0 { tile_config |= 0x10000; } @@ -430,10 +430,10 @@ impl super::Queue::ver { }; if tvb_autogrown { - result.result.flags |= bindings::DRM_ASAHI_RESULT_RENDER_TVB_GROW_OVF as u64; + result.result.flags |= uapi::DRM_ASAHI_RESULT_RENDER_TVB_GROW_OVF as u64; } if tvb_grown { - result.result.flags |= bindings::DRM_ASAHI_RESULT_RENDER_TVB_GROW_MIN as u64; + result.result.flags |= uapi::DRM_ASAHI_RESULT_RENDER_TVB_GROW_MIN as u64; } result.result.tvb_size_bytes = buffer.size() as u64; @@ -750,7 +750,7 @@ impl super::Queue::ver { unk_878: 0, encoder_params: fw::job::raw::EncoderParams { unk_8: (cmdbuf.flags - & bindings::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 + & uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 != 0) as u32, unk_c: 0x0, // fixed unk_10: 0x0, // fixed @@ -761,10 +761,10 @@ impl super::Queue::ver { unk_28: U64(0x0), // fixed }, process_empty_tiles: (cmdbuf.flags - & bindings::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 + & uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 != 0) as u32, no_clear_pipeline_textures: (cmdbuf.flags - & bindings::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES as u64 + & uapi::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES as u64 != 0) as u32, unk_param: unk2.into(), // 1 for boot stuff? unk_pointee: 0, @@ -1089,7 +1089,7 @@ impl super::Queue::ver { unk_55c: 0, unk_560: 0, memoryless_rts_used: (cmdbuf.flags - & bindings::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 + & uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 != 0) as u32, unk_568: 0, unk_56c: 0, @@ -1145,7 +1145,7 @@ impl super::Queue::ver { }); res.result.tvb_usage_bytes = cmd.scene.used_bytes() as u64; if cmd.scene.overflowed() { - res.result.flags |= bindings::DRM_ASAHI_RESULT_RENDER_TVB_OVERFLOWED as u64; + res.result.flags |= uapi::DRM_ASAHI_RESULT_RENDER_TVB_OVERFLOWED as u64; } res.vtx_error = error; res.vtx_complete = true; diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index ab0178c37a2736..205e15715e9615 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -24,10 +24,10 @@ use crate::{channel, driver, event, fw, gpu, object, regs}; use core::num::NonZeroU64; use core::sync::atomic::Ordering; use kernel::{ - bindings, error::code::*, prelude::*, sync::{Arc, Guard, Mutex, UniqueArc}, + uapi, }; const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; @@ -49,18 +49,18 @@ pub(crate) enum WorkError { Unknown, } -impl From for bindings::drm_asahi_result_info { +impl From for uapi::drm_asahi_result_info { fn from(err: WorkError) -> Self { match err { WorkError::Fault(info) => Self { - status: bindings::drm_asahi_status_DRM_ASAHI_STATUS_FAULT, + status: uapi::drm_asahi_status_DRM_ASAHI_STATUS_FAULT, fault_type: match info.reason { - FaultReason::Unmapped => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_UNMAPPED, - FaultReason::AfFault => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_AF_FAULT, - FaultReason::WriteOnly => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_WRITE_ONLY, - FaultReason::ReadOnly => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_READ_ONLY, - FaultReason::NoAccess => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_NO_ACCESS, - FaultReason::Unknown(_) => bindings::drm_asahi_fault_DRM_ASAHI_FAULT_UNKNOWN, + FaultReason::Unmapped => uapi::drm_asahi_fault_DRM_ASAHI_FAULT_UNMAPPED, + FaultReason::AfFault => uapi::drm_asahi_fault_DRM_ASAHI_FAULT_AF_FAULT, + FaultReason::WriteOnly => uapi::drm_asahi_fault_DRM_ASAHI_FAULT_WRITE_ONLY, + FaultReason::ReadOnly => uapi::drm_asahi_fault_DRM_ASAHI_FAULT_READ_ONLY, + FaultReason::NoAccess => uapi::drm_asahi_fault_DRM_ASAHI_FAULT_NO_ACCESS, + FaultReason::Unknown(_) => uapi::drm_asahi_fault_DRM_ASAHI_FAULT_UNKNOWN, }, unit: info.unit_code.into(), sideband: info.sideband.into(), @@ -72,10 +72,10 @@ impl From for bindings::drm_asahi_result_info { }, a => Self { status: match a { - WorkError::Timeout => bindings::drm_asahi_status_DRM_ASAHI_STATUS_TIMEOUT, - WorkError::Killed => bindings::drm_asahi_status_DRM_ASAHI_STATUS_KILLED, - WorkError::NoDevice => bindings::drm_asahi_status_DRM_ASAHI_STATUS_NO_DEVICE, - _ => bindings::drm_asahi_status_DRM_ASAHI_STATUS_UNKNOWN_ERROR, + WorkError::Timeout => uapi::drm_asahi_status_DRM_ASAHI_STATUS_TIMEOUT, + WorkError::Killed => uapi::drm_asahi_status_DRM_ASAHI_STATUS_KILLED, + WorkError::NoDevice => uapi::drm_asahi_status_DRM_ASAHI_STATUS_NO_DEVICE, + _ => uapi::drm_asahi_status_DRM_ASAHI_STATUS_UNKNOWN_ERROR, }, ..Default::default() }, From e5e4efa75e15861e302abde30a60a8c8329047a8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 5 Apr 2023 15:18:41 +0900 Subject: [PATCH 508/635] rust: Fix container_of!() Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 84f89e3845dafb..3caca288a2664a 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -226,8 +226,15 @@ macro_rules! offset_of { #[macro_export] macro_rules! container_of { ($ptr:expr, $type:ty, $($f:tt)*) => {{ - let ptr = $ptr as *const _ as *const u8; + let ptr = $ptr as *const u8; let offset = $crate::offset_of!($type, $($f)*); - ptr.wrapping_offset(-offset) as *const $type + let outer = ptr.wrapping_offset(-offset) as *const $type; + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + // SAFETY: The two pointers are within the same allocation block. + let inner = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + build_assert!(inner == $ptr); + outer }} } From c30435814139e48dc3ea756cbad9703fd5a3ef4f Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 5 Apr 2023 17:44:13 +0900 Subject: [PATCH 509/635] drm/scheduler: Fix UAF in drm_sched_fence_get_timeline_name A signaled scheduler fence can outlive its scheduler, since fences are independencly reference counted. Therefore, we can't reference the scheduler in the get_timeline_name() implementation. Fixes oopses on `cat /sys/kernel/debug/dma_buf/bufinfo` when shared dma-bufs reference fences from GPU schedulers that no longer exist. Signed-off-by: Asahi Lina --- drivers/gpu/drm/scheduler/sched_entity.c | 7 ++++++- drivers/gpu/drm/scheduler/sched_fence.c | 4 +++- include/drm/gpu_scheduler.h | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index e0a8890a62e23a..924cba8e71d427 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -368,7 +368,12 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) /* * Fence is from the same scheduler, only need to wait for - * it to be scheduled + * it to be scheduled. + * + * Note: s_fence->sched could have been freed and reallocated + * as another scheduler. This false positive case is okay, as if + * the old scheduler was freed all of its jobs must have + * signaled their completion fences. */ fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c index 7fd869520ef2cd..33b145dfa38c08 100644 --- a/drivers/gpu/drm/scheduler/sched_fence.c +++ b/drivers/gpu/drm/scheduler/sched_fence.c @@ -66,7 +66,7 @@ static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence) static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f) { struct drm_sched_fence *fence = to_drm_sched_fence(f); - return (const char *)fence->sched->name; + return (const char *)fence->sched_name; } static void drm_sched_fence_free_rcu(struct rcu_head *rcu) @@ -168,6 +168,8 @@ void drm_sched_fence_init(struct drm_sched_fence *fence, unsigned seq; fence->sched = entity->rq->sched; + strlcpy(fence->sched_name, entity->rq->sched->name, + sizeof(fence->sched_name)); seq = atomic_inc_return(&entity->fence_seq); dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled, &fence->lock, entity->fence_context, seq); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 6d558538fc626b..fe85a16aeb323f 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -288,6 +288,11 @@ struct drm_sched_fence { * @lock: the lock used by the scheduled and the finished fences. */ spinlock_t lock; + /** + * @sched_name: the name of the scheduler that owns this fence. We + * keep a copy here since fences can outlive their scheduler. + */ + char sched_name[16]; /** * @owner: job owner for debugging */ From 947d55dc45334e7c8fd2789d63457766d8d584e5 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 5 Apr 2023 23:09:27 +0900 Subject: [PATCH 510/635] drm/asahi: workqueue: Panic on context inval timeouts if requested Treat this like a GPU firmware crash, since it's at the very least a deadlock and a bad place to be in... Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/workqueue.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 205e15715e9615..056899215205bd 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -129,6 +129,9 @@ impl Drop for GpuContext { let dev = self.dev.data(); if dev.gpu.invalidate_context(&self.data).is_err() { dev_err!(self.dev, "GpuContext: Failed to invalidate GPU context!\n"); + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware timed out"); + } } } } From 935ec202ce0086997a05ad38f5582c5903710471 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 5 Apr 2023 23:10:12 +0900 Subject: [PATCH 511/635] drm/asahi: workqueue: Keep a Device reference here too We need this to offload freeing completed work to the global GPU manager, since doing it in the event handler context is a recipe for deadlocks if the GPU faults. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/mod.rs | 3 +++ drivers/gpu/drm/asahi/workqueue.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 47f88b4bb21f04..6159299debf538 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -428,6 +428,7 @@ impl Queue::ver { ret.buffer = Some(Mutex::new(buffer)); ret.q_vtx = Some(SubQueue::ver { wq: workqueue::WorkQueue::ver::new( + dev, alloc, event_manager.clone(), ret.gpu_context.clone(), @@ -448,6 +449,7 @@ impl Queue::ver { { ret.q_frag = Some(SubQueue::ver { wq: workqueue::WorkQueue::ver::new( + dev, alloc, event_manager.clone(), ret.gpu_context.clone(), @@ -464,6 +466,7 @@ impl Queue::ver { if caps & uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_COMPUTE != 0 { ret.q_comp = Some(SubQueue::ver { wq: workqueue::WorkQueue::ver::new( + dev, alloc, event_manager, ret.gpu_context.clone(), diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 056899215205bd..9caad6cc69fd24 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -202,6 +202,7 @@ impl) + Send + Sync> GenSubmi /// Inner data for managing a single work queue. #[versions(AGX)] struct WorkQueueInner { + dev: driver::AsahiDevice, event_manager: Arc, info: GpuObject, new: bool, @@ -554,6 +555,7 @@ impl WorkQueue::ver { /// Create a new WorkQueue of a given type and priority. #[allow(clippy::too_many_arguments)] pub(crate) fn new( + dev: &driver::AsahiDevice, alloc: &mut gpu::KernelAllocators, event_manager: Arc, gpu_context: Arc, @@ -576,6 +578,7 @@ impl WorkQueue::ver { }); let inner = WorkQueueInner::ver { + dev: dev.clone(), event_manager, info: alloc.private.new_boxed(info, |inner, ptr| { Ok(place!( From 04b52c1c5f10f777cd7dfd616009a98d723b485a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 5 Apr 2023 23:11:18 +0900 Subject: [PATCH 512/635] drm/asahi: Drop completed work asynchronously If we drop work in the main event completion thread, we can deadlock when we try to invalidate GPU contexts if the firmware is stuck. Instead, stash completed work in the global GPU manager and free it whenever we allocate anything. In the future this should move to a dedicated cleanup thread or something like that... Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 28 ++++++++++++++++++++++ drivers/gpu/drm/asahi/workqueue.rs | 37 ++++++++++++++---------------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index c7e3a5bb989756..0d2742eb82e34a 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -188,6 +188,7 @@ pub(crate) struct GpuManager { event_manager: Arc, buffer_mgr: buffer::BufferManager, ids: SequenceIDs, + garbage_work: Mutex>>, } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -245,6 +246,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; + /// Register completed work as garbage + fn add_completed_work(&self, work: Vec>); } /// Private generic trait for functions that don't need to escape this module. @@ -526,6 +529,7 @@ impl GpuManager::ver { buffer_mgr: buffer::BufferManager::new()?, alloc: Mutex::new(alloc), ids: Default::default(), + garbage_work: Mutex::new(Vec::new()), }) } @@ -846,6 +850,12 @@ impl GpuManager for GpuManager::ver { } fn alloc(&self) -> Guard<'_, Mutex> { + /* + * TODO: This should be done in a workqueue or something. + * Clean up completed jobs + */ + self.garbage_work.lock().clear(); + let mut guard = self.alloc.lock(); let (garbage_count, garbage_bytes) = guard.private.garbage(); if garbage_bytes > MAX_FW_ALLOC_GARBAGE { @@ -1083,6 +1093,24 @@ impl GpuManager for GpuManager::ver { fn get_dyncfg(&self) -> &hw::DynConfig { &self.dyncfg } + + fn add_completed_work(&self, work: Vec>) { + let mut garbage = self.garbage_work.lock(); + + if garbage.try_reserve(work.len()).is_err() { + dev_err!( + self.dev, + "Failed to reserve space for completed work, deadlock possible.\n" + ); + return; + } + + for i in work { + garbage + .try_push(i) + .expect("try_push() failed after try_reserve()"); + } + } } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 9caad6cc69fd24..fb278550fd7b49 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -139,26 +139,26 @@ impl Drop for GpuContext { struct SubmittedWork where O: OpaqueGpuObject, - C: FnOnce(O, Option) + Send + Sync + 'static, + C: FnOnce(&mut O, Option) + Send + Sync + 'static, { object: O, value: EventValue, error: Option, wptr: u32, vm_slot: u32, - callback: C, + callback: Option, } -trait GenSubmittedWork: Send + Sync { +pub(crate) trait GenSubmittedWork: Send + Sync { fn gpu_va(&self) -> NonZeroU64; fn value(&self) -> event::EventValue; fn wptr(&self) -> u32; fn set_wptr(&mut self, wptr: u32); fn mark_error(&mut self, error: WorkError); - fn complete(self: Box); + fn complete(&mut self); } -impl) + Send + Sync> GenSubmittedWork +impl) + Send + Sync> GenSubmittedWork for SubmittedWork { fn gpu_va(&self) -> NonZeroU64 { @@ -177,17 +177,10 @@ impl) + Send + Sync> GenSubmi self.wptr = wptr; } - fn complete(self: Box) { - let SubmittedWork { - object, - value: _, - error, - wptr: _, - vm_slot: _, - callback, - } = *self; - - callback(object, error); + fn complete(&mut self) { + if let Some(cb) = self.callback.take() { + cb(&mut self.object, self.error); + } } fn mark_error(&mut self, error: WorkError) { @@ -297,7 +290,7 @@ impl Job::ver { &mut self, command: O, vm_slot: u32, - callback: impl FnOnce(O, Option) + Sync + Send + 'static, + callback: impl FnOnce(&mut O, Option) + Sync + Send + 'static, ) -> Result { if self.committed { pr_err!("WorkQueue: Tried to mutate committed Job\n"); @@ -308,7 +301,7 @@ impl Job::ver { object: command, value: self.event_info.value.next(), error: None, - callback, + callback: Some(callback), wptr: 0, vm_slot, })?)?; @@ -776,7 +769,7 @@ impl WorkQueue for WorkQueue::ver { if completed.try_reserve(completed_commands).is_err() { pr_crit!( - "WorkQueue({:?}): Failed to allocated space for {} completed commands\n", + "WorkQueue({:?}): Failed to allocate space for {} completed commands\n", inner.pipe_type, completed_commands ); @@ -813,12 +806,16 @@ impl WorkQueue for WorkQueue::ver { inner.last_completed = None; } + let dev = inner.dev.clone(); core::mem::drop(inner); - for cmd in completed { + for cmd in completed.iter_mut() { cmd.complete(); } + let gpu = &dev.data().gpu; + gpu.add_completed_work(completed); + empty } From d34d42bbe2ce0e2fc4a794ffcc66d88ac240ee77 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 5 Apr 2023 23:41:17 +0900 Subject: [PATCH 513/635] drm/asahi: Also free unused GPU contexts asynchronously Since workqueues get freed in the event thread, that can drop the GpuContext there and deadlock when the firmware halts. Delegate it to the GpuManager too. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 154 +++++++++++++++++------------ drivers/gpu/drm/asahi/workqueue.rs | 22 ++--- 2 files changed, 99 insertions(+), 77 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 0d2742eb82e34a..221e742c6ee08d 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -189,6 +189,8 @@ pub(crate) struct GpuManager { buffer_mgr: buffer::BufferManager, ids: SequenceIDs, garbage_work: Mutex>>, + #[allow(clippy::vec_box)] + garbage_contexts: Mutex>>>, } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -225,11 +227,6 @@ pub(crate) trait GpuManager: Send + Sync { /// This should be useful to reduce latency on work submission, so we can ask the firmware to /// wake up while we do some preparatory work for the work submission. fn kick_firmware(&self) -> Result; - /// Invalidate a GPU scheduler context. Must be called before the relevant structures are freed. - fn invalidate_context( - &self, - context: &fw::types::GpuObject, - ) -> Result; /// Flush the entire firmware cache. /// /// TODO: Does this actually work? @@ -248,6 +245,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_dyncfg(&self) -> &hw::DynConfig; /// Register completed work as garbage fn add_completed_work(&self, work: Vec>); + /// Register an unused context as garbage + fn free_context(&self, data: Box>); } /// Private generic trait for functions that don't need to escape this module. @@ -530,6 +529,7 @@ impl GpuManager::ver { alloc: Mutex::new(alloc), ids: Default::default(), garbage_work: Mutex::new(Vec::new()), + garbage_contexts: Mutex::new(Vec::new()), }) } @@ -804,6 +804,66 @@ impl GpuManager::ver { self.kick_firmware()?; Ok(OpGuard(self.clone())) } + + fn invalidate_context( + &self, + context: &fw::types::GpuObject, + ) -> Result { + mod_dev_dbg!( + self.dev, + "Invalidating GPU context @ {:?}\n", + context.weak_pointer() + ); + + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.alloc.lock(); + let (garbage_count, _) = guard.private.garbage(); + + let dc = context.with( + |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + ctx_23: raw.unk_23, + __pad0: Default::default(), + unk_c: 0, + unk_10: 0, + ctx_0: raw.unk_0, + ctx_1: raw.unk_1, + ctx_4: raw.unk_4, + __pad1: Default::default(), + unk_18: 0, + gpu_context: Some(context.weak_pointer()), + __pad2: Default::default(), + }, + ); + + mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + + mod_dev_dbg!( + self.dev, + "GPU context invalidated: {:?}\n", + context.weak_pointer() + ); + + // The invalidation does a cache flush, so it is okay to collect garbage + guard.private.collect_garbage(garbage_count); + + Ok(()) + } } #[versions(AGX)] @@ -856,6 +916,19 @@ impl GpuManager for GpuManager::ver { */ self.garbage_work.lock().clear(); + /* Clean up idle contexts */ + let mut garbage_ctx = Vec::new(); + core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); + + for ctx in garbage_ctx { + if self.invalidate_context(&ctx).is_err() { + dev_err!(self.dev, "GpuContext: Failed to invalidate GPU context!\n"); + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware timed out"); + } + } + } + let mut guard = self.alloc.lock(); let (garbage_count, garbage_bytes) = guard.private.garbage(); if garbage_bytes > MAX_FW_ALLOC_GARBAGE { @@ -919,66 +992,6 @@ impl GpuManager for GpuManager::ver { Ok(()) } - fn invalidate_context( - &self, - context: &fw::types::GpuObject, - ) -> Result { - mod_dev_dbg!( - self.dev, - "Invalidating GPU context @ {:?}\n", - context.weak_pointer() - ); - - if self.is_crashed() { - return Err(ENODEV); - } - - let mut guard = self.alloc.lock(); - let (garbage_count, _) = guard.private.garbage(); - - let dc = context.with( - |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { - unk_4: 0, - ctx_23: raw.unk_23, - __pad0: Default::default(), - unk_c: 0, - unk_10: 0, - ctx_0: raw.unk_0, - ctx_1: raw.unk_1, - ctx_4: raw.unk_4, - __pad1: Default::default(), - unk_18: 0, - gpu_context: Some(context.weak_pointer()), - __pad2: Default::default(), - }, - ); - - mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); - - let mut txch = self.tx_channels.lock(); - - let token = txch.device_control.send(&dc); - - { - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().unwrap(); - rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; - } - - txch.device_control.wait_for(token)?; - - mod_dev_dbg!( - self.dev, - "GPU context invalidated: {:?}\n", - context.weak_pointer() - ); - - // The invalidation does a cache flush, so it is okay to collect garbage - guard.private.collect_garbage(garbage_count); - - Ok(()) - } - fn flush_fw_cache(&self) -> Result { mod_dev_dbg!(self.dev, "Flushing coprocessor data cache\n"); @@ -1111,6 +1124,17 @@ impl GpuManager for GpuManager::ver { .expect("try_push() failed after try_reserve()"); } } + + fn free_context(&self, ctx: Box>) { + let mut garbage = self.garbage_contexts.lock(); + + if garbage.try_push(ctx).is_err() { + dev_err!( + self.dev, + "Failed to reserve space for freed context, deadlock possible.\n" + ); + } + } } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index fb278550fd7b49..1713e122e8cbaf 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -99,7 +99,7 @@ impl From for kernel::error::Error { /// A GPU context tracking structure, which must be explicitly invalidated when dropped. pub(crate) struct GpuContext { dev: driver::AsahiDevice, - data: GpuObject, + data: Option>>, } no_debug!(GpuContext); @@ -111,28 +111,26 @@ impl GpuContext { ) -> Result { Ok(GpuContext { dev: dev.clone(), - data: alloc - .shared - .new_object(Default::default(), |_inner| Default::default())?, + data: Some(Box::try_new( + alloc + .shared + .new_object(Default::default(), |_inner| Default::default())?, + )?), }) } /// Returns the GPU pointer to the inner GPU context data structure. pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { - self.data.gpu_pointer() + self.data.as_ref().unwrap().gpu_pointer() } } impl Drop for GpuContext { fn drop(&mut self) { - mod_dev_dbg!(self.dev, "GpuContext: Invalidating GPU context\n"); + mod_dev_dbg!(self.dev, "GpuContext: Freeing GPU context\n"); let dev = self.dev.data(); - if dev.gpu.invalidate_context(&self.data).is_err() { - dev_err!(self.dev, "GpuContext: Failed to invalidate GPU context!\n"); - if debug_enabled(DebugFlags::OopsOnGpuCrash) { - panic!("GPU firmware timed out"); - } - } + let data = self.data.take().unwrap(); + dev.gpu.free_context(data); } } From 4a30805f67992a9f95f97c3945e6b99f5565755d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 6 Apr 2023 00:29:11 +0900 Subject: [PATCH 514/635] drm/asahi: queue: Drop redundant mutex around Buffer Buffer is already internally locked. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/mod.rs | 4 ++-- drivers/gpu/drm/asahi/queue/render.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 6159299debf538..200781e237a63c 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -103,7 +103,7 @@ pub(crate) struct Queue { q_vtx: Option, q_frag: Option, q_comp: Option, - buffer: Option>, + buffer: Option, gpu_context: Arc, notifier_list: Arc>, notifier: Arc>, @@ -425,7 +425,7 @@ impl Queue::ver { buffer.ensure_blocks(tvb_blocks)?; - ret.buffer = Some(Mutex::new(buffer)); + ret.buffer = Some(buffer); ret.q_vtx = Some(SubQueue::ver { wq: workqueue::WorkQueue::ver::new( dev, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index f93b8db6e96405..1a3f6e3438aeb9 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -292,7 +292,7 @@ impl super::Queue::ver { let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; - let buffer = self.buffer.as_ref().ok_or(EINVAL)?.lock(); + let buffer = self.buffer.as_ref().ok_or(EINVAL)?; let scene = Arc::try_new(buffer.new_scene(kalloc, &tile_info)?)?; From 658131747473fd431db2b8da980dfc62870f6562 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 6 Apr 2023 00:29:46 +0900 Subject: [PATCH 515/635] drm/asahi: workqueue: Hold a Buffer reference in GpuContextData It looks like the firmware can clean up currently mapped buffers at any arbitrary time. Let's assume this is cleaned up by context inval (which is the only explicit inval macOS does). Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 4 ++++ drivers/gpu/drm/asahi/fw/workqueue.rs | 8 +++++++- drivers/gpu/drm/asahi/queue/mod.rs | 25 +++++++++++++++++++------ drivers/gpu/drm/asahi/workqueue.rs | 10 +++++----- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index 767ea161176f2d..5e629b884c4c51 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -663,6 +663,10 @@ impl Buffer::ver { raw.count.store(v + 1, Ordering::Relaxed); }); } + + pub(crate) fn any_ref(&self) -> Arc { + self.inner.clone() + } } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs index 86472bd6f7ae53..f213a4d923817e 100644 --- a/drivers/gpu/drm/asahi/fw/workqueue.rs +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -147,11 +147,17 @@ pub(crate) mod raw { } trivial_gpustruct!(Barrier); -trivial_gpustruct!(GpuContextData); trivial_gpustruct!(RingState); impl Command for Barrier {} +pub(crate) struct GpuContextData { + pub(crate) _buffer: Option>, +} +impl GpuStruct for GpuContextData { + type Raw<'a> = raw::GpuContextData; +} + #[versions(AGX)] #[derive(Debug)] pub(crate) struct QueueInfo { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 200781e237a63c..84e62e803b8271 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -395,6 +395,18 @@ impl Queue::ver { // per-queue scheduler. let entity = sched::Entity::new(&sched, sched::Priority::Normal)?; + let buffer = if caps & uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER != 0 { + Some(buffer::Buffer::ver::new( + &*data.gpu, + alloc, + ualloc.clone(), + ualloc_priv, + mgr, + )?) + } else { + None + }; + let mut ret = Queue::ver { dev: dev.clone(), _sched: sched, @@ -404,8 +416,12 @@ impl Queue::ver { q_vtx: None, q_frag: None, q_comp: None, - buffer: None, - gpu_context: Arc::try_new(workqueue::GpuContext::new(dev, alloc)?)?, + gpu_context: Arc::try_new(workqueue::GpuContext::new( + dev, + alloc, + buffer.as_ref().map(|b| b.any_ref()), + )?)?, + buffer, notifier_list: Arc::try_new(notifier_list)?, notifier, id, @@ -416,16 +432,13 @@ impl Queue::ver { // Rendering structures if caps & uapi::drm_asahi_queue_cap_DRM_ASAHI_QUEUE_CAP_RENDER != 0 { - let buffer = - buffer::Buffer::ver::new(&*data.gpu, alloc, ret.ualloc.clone(), ualloc_priv, mgr)?; let tvb_blocks = { let lock = crate::THIS_MODULE.kernel_param_lock(); *crate::initial_tvb_size.read(&lock) }; - buffer.ensure_blocks(tvb_blocks)?; + ret.buffer.as_ref().unwrap().ensure_blocks(tvb_blocks)?; - ret.buffer = Some(buffer); ret.q_vtx = Some(SubQueue::ver { wq: workqueue::WorkQueue::ver::new( dev, diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 1713e122e8cbaf..a50c5758b1acad 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -108,14 +108,14 @@ impl GpuContext { pub(crate) fn new( dev: &driver::AsahiDevice, alloc: &mut gpu::KernelAllocators, + buffer: Option>, ) -> Result { Ok(GpuContext { dev: dev.clone(), - data: Some(Box::try_new( - alloc - .shared - .new_object(Default::default(), |_inner| Default::default())?, - )?), + data: Some(Box::try_new(alloc.shared.new_object( + fw::workqueue::GpuContextData { _buffer: buffer }, + |_inner| Default::default(), + )?)?), }) } From 52312640eab0bbcd8d086eb6ddb2968be85b2d11 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 6 Apr 2023 17:03:27 +0900 Subject: [PATCH 516/635] drm/asahi: Fail params requests if the GPU is crashed This should make Mesa fail the driver load and fall back to software rendering early. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/file.rs | 4 ++++ drivers/gpu/drm/asahi/gpu.rs | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 7f575816b84d02..53d7e7ef00e48f 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -202,6 +202,10 @@ impl File { return Err(EINVAL); } + if gpu.is_crashed() { + return Err(ENODEV); + } + let mut params = uapi::drm_asahi_params_global { unstable_uabi_version: uapi::DRM_ASAHI_UNSTABLE_UABI_VERSION, pad0: 0, diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 221e742c6ee08d..52eb103628aad2 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -247,6 +247,8 @@ pub(crate) trait GpuManager: Send + Sync { fn add_completed_work(&self, work: Vec>); /// Register an unused context as garbage fn free_context(&self, data: Box>); + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; } /// Private generic trait for functions that don't need to escape this module. @@ -786,10 +788,6 @@ impl GpuManager::ver { Ok(()) } - pub(crate) fn is_crashed(&self) -> bool { - self.crashed.load(Ordering::Relaxed) - } - pub(crate) fn start_op(self: &Arc) -> Result { if self.is_crashed() { return Err(ENODEV); @@ -1135,6 +1133,10 @@ impl GpuManager for GpuManager::ver { ); } } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } } #[versions(AGX)] From 4bd09df0d3462018d9997c3be2dafbed4bda8b8b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 7 Apr 2023 17:45:16 +0900 Subject: [PATCH 517/635] I don't know how to fix drm_sched. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/mod.rs | 5 ++++- rust/kernel/drm/sched.rs | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 84e62e803b8271..fc03ca8972b681 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -390,7 +390,10 @@ impl Queue::ver { }, )?)?; - let sched = sched::Scheduler::new(dev, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; + // SAFETY: I give up. This is unsafe and there is no reasonable way to make it safe. + // Known broken. Will crash and burn and oops under corner cases like apps getting kill -9'd. + let sched = + unsafe { sched::Scheduler::new(dev, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))? }; // Priorities are handled by the AGX scheduler, there is no meaning within a // per-queue scheduler. let entity = sched::Entity::new(&sched, sched::Priority::Normal)?; diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index a5275cc1617921..7d132a8dab8b84 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -327,8 +327,16 @@ impl Scheduler { free_job: Some(free_job_cb::), }; /// Creates a new DRM Scheduler object + /// + /// # Safety + /// The user must magically ensure all of its Job objects outlive this scheduler, + /// but isn't allowed to hold a direct reference to the scheduler because dropping + /// the scheduler from the drop impl of a Job is not allowed, as it will deadlock. + /// + /// Yes, this is horribly broken, but it's how drm_sched is designed. + /// Please send help. // TODO: Shared timeout workqueues & scores - pub fn new( + pub unsafe fn new( device: &impl device::RawDevice, hw_submission: u32, hang_limit: u32, From 89b497260a0322ba3e6a77b91b5b82d912ebcf5e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 7 Apr 2023 20:38:42 +0900 Subject: [PATCH 518/635] drm/asahi: Identify more Render fields & update to UAPI 10007 Also set the large_tib flag properly (implied from tib size for now, not sure if this should be a separate flag?) Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/fragment.rs | 8 +++---- drivers/gpu/drm/asahi/fw/job.rs | 2 +- drivers/gpu/drm/asahi/fw/microseq.rs | 2 +- drivers/gpu/drm/asahi/queue/compute.rs | 6 +++--- drivers/gpu/drm/asahi/queue/render.rs | 29 ++++++++++++++------------ 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs index eca275efb967a4..a01b0d0d7807f8 100644 --- a/drivers/gpu/drm/asahi/fw/fragment.rs +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -132,7 +132,7 @@ pub(crate) mod raw { pub(crate) utiles_per_mtile_x: u16, pub(crate) unk_24: u32, pub(crate) tile_counts: u32, - pub(crate) iogpu_unk_212: u32, + pub(crate) tib_blocks: u32, pub(crate) isp_bgobjdepth: u32, pub(crate) isp_bgobjvals: u32, pub(crate) unk_38: u32, @@ -171,7 +171,7 @@ pub(crate) mod raw { pub(crate) stencil_buffer_ptr3: U64, pub(crate) stencil_meta_buffer_ptr3: U64, pub(crate) unk_2f8: Array<2, U64>, - pub(crate) iogpu_unk_212: u32, + pub(crate) tib_blocks: u32, pub(crate) unk_30c: u32, pub(crate) aux_fb_info: AuxFBInfo::ver, pub(crate) unk_320_padding: Array<0x10, u8>, @@ -179,7 +179,7 @@ pub(crate) mod raw { pub(crate) partial_store_pipeline: StorePipelineBinding, pub(crate) isp_bgobjdepth: u32, pub(crate) isp_bgobjvals: u32, - pub(crate) iogpu_unk_49: u32, + pub(crate) sample_size: u32, pub(crate) unk_37c: u32, pub(crate) unk_380: U64, pub(crate) unk_388: U64, @@ -230,7 +230,7 @@ pub(crate) mod raw { pub(crate) encoder_params: job::raw::EncoderParams<'a>, pub(crate) process_empty_tiles: u32, pub(crate) no_clear_pipeline_textures: u32, - pub(crate) unk_param: u32, + pub(crate) msaa_zs: u32, pub(crate) unk_pointee: u32, pub(crate) meta: job::raw::JobMeta, pub(crate) unk_after_meta: u32, diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index 7027a763dffb11..2384b5c1119665 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -28,7 +28,7 @@ pub(crate) mod raw { #[repr(C)] pub(crate) struct EncoderParams<'a> { pub(crate) unk_8: u32, - pub(crate) unk_c: u32, + pub(crate) large_tib: u32, pub(crate) unk_10: u32, pub(crate) encoder_id: u32, pub(crate) unk_18: u32, diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index 274a2c7f741537..f47740a00fb5d6 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -239,7 +239,7 @@ pub(crate) struct StartFragment<'a> { pub(crate) unk_50: u32, pub(crate) event_generation: u32, pub(crate) buffer_slot: u32, - pub(crate) unk_5c: u32, + pub(crate) large_tib: u32, pub(crate) event_seq: U64, pub(crate) unk_68: u32, pub(crate) unk_758_flag: GpuWeakPointer, diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 995371470a2584..69039edd3fbf8d 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -287,9 +287,9 @@ impl super::Queue::ver { unk_5c: 0, }, encoder_params: fw::job::raw::EncoderParams { - unk_8: 0x0, // fixed - unk_c: 0x0, // fixed - unk_10: 0x0, // fixed + unk_8: 0x0, // fixed + large_tib: 0x0, // check! + unk_10: 0x0, // fixed encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed iogpu_compute_unk44: cmdbuf.iogpu_unk_44, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 1a3f6e3438aeb9..0a12473c4ca89f 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -225,7 +225,9 @@ impl super::Queue::ver { | uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S | uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED | uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES - | uapi::ASAHI_RENDER_NO_VERTEX_CLUSTERING) as u64 + | uapi::ASAHI_RENDER_NO_VERTEX_CLUSTERING + | uapi::ASAHI_RENDER_MSAA_ZS + | uapi::ASAHI_RENDER_UNK_FLAG1) as u64 != 0 { return Err(EINVAL); @@ -395,8 +397,7 @@ impl super::Queue::ver { let timestamps = Arc::try_new(kalloc.shared.new_default::()?)?; - let unk1 = debug_enabled(debug::DebugFlags::Debug1); - let unk2 = debug_enabled(debug::DebugFlags::Debug2); + let unk1 = cmdbuf.flags & uapi::ASAHI_RENDER_UNK_FLAG1 as u64 != 0; let mut tile_config: u64 = 0; if !unk1 { @@ -449,6 +450,8 @@ impl super::Queue::ver { #[ver(V >= V13_0B4)] let count_vtx = count_frag + 1; + let large_tib = cmdbuf.tib_blocks > 8; + mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); let frag = GpuObject::new_prealloc( kalloc.private.alloc_object()?, @@ -475,7 +478,7 @@ impl super::Queue::ver { unk_50: 0x1, // fixed event_generation: self.id as u32, buffer_slot: scene.slot(), - unk_5c: 0, + large_tib: large_tib as u32, event_seq: U64(ev_frag.event_seq), unk_68: 0, unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), @@ -671,7 +674,7 @@ impl super::Queue::ver { utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, unk_24: 0x0, tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), - iogpu_unk_212: cmdbuf.iogpu_unk_212, + tib_blocks: cmdbuf.tib_blocks, isp_bgobjdepth: cmdbuf.isp_bgobjdepth, // TODO: does this flag need to be exposed to userspace? isp_bgobjvals: cmdbuf.isp_bgobjvals | 0x400, @@ -719,7 +722,7 @@ impl super::Queue::ver { stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_3), stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_3), unk_2f8: Default::default(), - iogpu_unk_212: cmdbuf.iogpu_unk_212, + tib_blocks: cmdbuf.tib_blocks, unk_30c: 0x0, aux_fb_info: aux_fb_info, unk_320_padding: Default::default(), @@ -734,7 +737,7 @@ impl super::Queue::ver { ), isp_bgobjdepth: cmdbuf.isp_bgobjdepth, isp_bgobjvals: cmdbuf.isp_bgobjvals, - iogpu_unk_49: cmdbuf.iogpu_unk_49, + sample_size: cmdbuf.sample_size, unk_37c: 0x0, unk_380: U64(0x0), unk_388: U64(0x0), @@ -752,8 +755,8 @@ impl super::Queue::ver { unk_8: (cmdbuf.flags & uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 != 0) as u32, - unk_c: 0x0, // fixed - unk_10: 0x0, // fixed + large_tib: large_tib as u32, // fixed + unk_10: 0x0, // fixed encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed iogpu_compute_unk44: 0xffffffff, @@ -766,7 +769,7 @@ impl super::Queue::ver { no_clear_pipeline_textures: (cmdbuf.flags & uapi::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES as u64 != 0) as u32, - unk_param: unk2.into(), // 1 for boot stuff? + msaa_zs: (cmdbuf.flags & uapi::ASAHI_RENDER_MSAA_ZS as u64 != 0) as u32, unk_pointee: 0, meta: fw::job::raw::JobMeta { unk_0: 0, @@ -1077,9 +1080,9 @@ impl super::Queue::ver { unk_520: U64(0x0), // fixed }, encoder_params: fw::job::raw::EncoderParams { - unk_8: 0x0, // fixed - unk_c: 0x0, // fixed - unk_10: 0x0, // fixed + unk_8: 0x0, // fixed + large_tib: 0x0, // fixed + unk_10: 0x0, // fixed encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed iogpu_compute_unk44: 0xffffffff, From aead264a64deebfb698ee2737c0007dd3bcac397 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 15:44:22 +0900 Subject: [PATCH 519/635] drm/asahi: Make stats structs opaque These change all the time and we don't use them, so let's just allocate them as big opaque blobs and stop worrying about them. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/initdata.rs | 124 +++++---------------------- drivers/gpu/drm/asahi/fw/microseq.rs | 12 +-- drivers/gpu/drm/asahi/initdata.rs | 21 +---- 3 files changed, 32 insertions(+), 125 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs index 473adf3326bdb1..923b560367194f 100644 --- a/drivers/gpu/drm/asahi/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -723,100 +723,46 @@ pub(crate) mod raw { #[versions(AGX)] default_zeroed!(HwDataB::ver); - #[derive(Debug, Clone, Copy)] - #[repr(C, packed)] - pub(crate) struct GpuQueueStatsVtx { - pub(crate) busy: u32, - pub(crate) unk_4: u32, - pub(crate) cur_cmdqueue: U64, - pub(crate) cur_count: u32, - pub(crate) unk_14: u32, - } - default_zeroed!(GpuQueueStatsVtx); - - #[versions(AGX)] - #[derive(Debug, Default, Clone, Copy)] + #[derive(Debug)] #[repr(C, packed)] pub(crate) struct GpuStatsVtx { - pub(crate) unk_4: u32, - pub(crate) queues: Array<0x4, GpuQueueStatsVtx>, - pub(crate) unk_68: Array<0x8, u8>, - pub(crate) unk_70: u32, - pub(crate) unk_74: u32, - pub(crate) unk_timestamp: U64, - pub(crate) unk_80: Array<0x40, u8>, + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, } + default_zeroed!(GpuStatsVtx); - #[derive(Debug, Default, Clone, Copy)] - #[repr(C, packed)] - pub(crate) struct GpuQueueStatsFrag { - pub(crate) busy: u32, - pub(crate) cur_cmdqueue: U64, - pub(crate) unk_c: u32, - pub(crate) unk_10: u32, - pub(crate) unk_14: Array<0x14, u8>, - } - - #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct GpuStatsFrag { - pub(crate) unk_0: Array<0x18, u8>, - pub(crate) queues: Array<0x4, GpuQueueStatsFrag>, - pub(crate) unk_d0: Array<0x38, u8>, - pub(crate) tvb_overflows_1: u32, - pub(crate) tvb_overflows_2: u32, - pub(crate) unk_f8: u32, - pub(crate) unk_fc: u32, - pub(crate) cur_stamp_id: i32, - pub(crate) unk_104: Array<0x14, u8>, - pub(crate) unk_118: i32, - pub(crate) unk_11c: u32, - pub(crate) unk_120: u32, - pub(crate) unk_124: u32, - pub(crate) unk_128: u32, - pub(crate) unk_12c: u32, - pub(crate) unk_timestamp: U64, - pub(crate) unk_134: Array<0x8c, u8>, + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, } - #[versions(AGX)] - default_zeroed!(GpuStatsFrag::ver); + default_zeroed!(GpuStatsFrag); - #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct GpuGlobalStatsVtx { pub(crate) total_cmds: u32, - pub(crate) stats: GpuStatsVtx::ver, - #[ver(V >= V13_0B4)] - pub(crate) unk_pad: Array<0x5c4, u8>, + pub(crate) stats: GpuStatsVtx, } - #[versions(AGX)] - default_zeroed!(GpuGlobalStatsVtx::ver); + default_zeroed!(GpuGlobalStatsVtx); - #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct GpuGlobalStatsFrag { pub(crate) total_cmds: u32, pub(crate) unk_4: u32, - pub(crate) stats: GpuStatsFrag::ver, - #[ver(V >= V13_0B4)] - pub(crate) unk_pad: Array<0x580, u8>, + pub(crate) stats: GpuStatsFrag, } - #[versions(AGX)] - default_zeroed!(GpuGlobalStatsFrag::ver); + default_zeroed!(GpuGlobalStatsFrag); - #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct GpuStatsComp { - pub(crate) unk: Array<0x180, u8>, - #[ver(V >= V13_0B4)] - pub(crate) unk_pad: Array<0x580, u8>, + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, } - #[versions(AGX)] - default_zeroed!(GpuStatsComp::ver); + default_zeroed!(GpuStatsComp); #[derive(Debug)] #[repr(C)] @@ -873,9 +819,9 @@ pub(crate) mod raw { pub(crate) __pad0: Pad<0x50>, pub(crate) unk_160: U64, pub(crate) unk_168: U64, - pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx::ver>, - pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, - pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp::ver>, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, @@ -1170,33 +1116,9 @@ where } trivial_gpustruct!(FwStatus); - -#[versions(AGX)] -#[derive(Debug, Default)] -pub(crate) struct GpuGlobalStatsVtx {} - -#[versions(AGX)] -impl GpuStruct for GpuGlobalStatsVtx::ver { - type Raw<'a> = raw::GpuGlobalStatsVtx::ver; -} - -#[versions(AGX)] -#[derive(Debug, Default)] -pub(crate) struct GpuGlobalStatsFrag {} - -#[versions(AGX)] -impl GpuStruct for GpuGlobalStatsFrag::ver { - type Raw<'a> = raw::GpuGlobalStatsFrag::ver; -} - -#[versions(AGX)] -#[derive(Debug, Default)] -pub(crate) struct GpuStatsComp {} - -#[versions(AGX)] -impl GpuStruct for GpuStatsComp::ver { - type Raw<'a> = raw::GpuStatsComp::ver; -} +trivial_gpustruct!(GpuGlobalStatsVtx); +trivial_gpustruct!(GpuGlobalStatsFrag); +trivial_gpustruct!(GpuStatsComp); #[versions(AGX)] #[derive(Debug, Default)] @@ -1219,9 +1141,9 @@ impl GpuStruct for HwDataB::ver { #[versions(AGX)] #[derive(Debug)] pub(crate) struct Stats { - pub(crate) vtx: GpuObject, - pub(crate) frag: GpuObject, - pub(crate) comp: GpuObject, + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, } #[versions(AGX)] diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index f47740a00fb5d6..7daedd50bb09d5 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -155,7 +155,7 @@ pub(crate) struct StartVertex<'a> { pub(crate) job_params1: GpuWeakPointer>, pub(crate) buffer: GpuWeakPointer, pub(crate) scene: GpuWeakPointer, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, pub(crate) vm_slot: u32, pub(crate) unk_38: u32, @@ -191,7 +191,7 @@ pub(crate) struct FinalizeVertex { pub(crate) header: op::FinalizeVertex, pub(crate) scene: GpuWeakPointer, pub(crate) buffer: GpuWeakPointer, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, pub(crate) vm_slot: u32, pub(crate) unk_28: u32, @@ -229,7 +229,7 @@ pub(crate) struct StartFragment<'a> { pub(crate) job_params2: GpuWeakPointer, pub(crate) job_params1: GpuWeakPointer>, pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) busy_flag: GpuWeakPointer, pub(crate) tvb_overflow_count: GpuWeakPointer, pub(crate) unk_pointer: GpuWeakPointer, @@ -274,7 +274,7 @@ pub(crate) struct FinalizeFragment { pub(crate) scene: GpuWeakPointer, pub(crate) buffer: GpuWeakPointer, pub(crate) unk_2c: U64, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) unk_pointer: GpuWeakPointer, pub(crate) busy_flag: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, @@ -308,7 +308,7 @@ pub(crate) struct StartCompute<'a> { pub(crate) header: op::StartCompute, pub(crate) unk_pointer: GpuWeakPointer>, pub(crate) job_params1: GpuWeakPointer>, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, pub(crate) vm_slot: u32, pub(crate) unk_28: u32, @@ -339,7 +339,7 @@ impl<'a> Operation for StartCompute::ver<'a> {} #[repr(C)] pub(crate) struct FinalizeCompute<'a> { pub(crate) header: op::FinalizeCompute, - pub(crate) stats: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, pub(crate) work_queue: GpuWeakPointer, pub(crate) vm_slot: u32, #[ver(V < V13_0B4)] diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 7b36df4ab151c0..29ee4bbd3a8a49 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -623,24 +623,9 @@ impl<'a> InitDataBuilder::ver<'a> { let pointers: Box = box_in_place!(RuntimePointers::ver { stats: Stats::ver { - vtx: self.alloc.private.new_default::()?, - frag: self.alloc.private.new_inplace( - Default::default(), - |_inner, ptr: &mut MaybeUninit| { - Ok(place!( - ptr, - raw::GpuGlobalStatsFrag::ver { - stats: raw::GpuStatsFrag::ver { - cur_stamp_id: -1, - unk_118: -1, - ..Default::default() - }, - ..Default::default() - } - )) - }, - )?, - comp: self.alloc.private.new_default::()?, + vtx: self.alloc.private.new_default::()?, + frag: self.alloc.private.new_default::()?, + comp: self.alloc.private.new_default::()?, }, hwdata_a: hwa, From 49c0be226246b3757d35ecab54955b44fc5ac73d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 15:45:54 +0900 Subject: [PATCH 520/635] drm/asahi: Drop 13.2 top-level support We will not be supporting this version in the end. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/driver.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index d49d8b1934a4f5..a1422c53b3761d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -124,15 +124,9 @@ impl platform::Driver for AsahiDriver { (hw::GpuGen::G13, &[12, 3, 0]) => { gpu::GpuManagerG13V12_3::new(reg.device(), &res, cfg)? as Arc } - (hw::GpuGen::G13, &[13, 2, 0]) => { - gpu::GpuManagerG13V13_2::new(reg.device(), &res, cfg)? as Arc - } (hw::GpuGen::G14, &[12, 4, 0]) => { gpu::GpuManagerG14V12_4::new(reg.device(), &res, cfg)? as Arc } - (hw::GpuGen::G14, &[13, 2, 0]) => { - gpu::GpuManagerG14V13_2::new(reg.device(), &res, cfg)? as Arc - } _ => { dev_info!( dev, From 6740562f9f0e8860b561dcda48920c8c6b1ef5c3 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 15:44:46 +0900 Subject: [PATCH 521/635] rust: macros: versions: Add V13_3 and G14X Also remove V13_2 from the build list, since we will not support it. Signed-off-by: Asahi Lina --- rust/macros/versions.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs index 6dfdfd7d25c55d..994567fd871856 100644 --- a/rust/macros/versions.rs +++ b/rust/macros/versions.rs @@ -47,14 +47,16 @@ struct VersionConfig { static AGX_VERSIONS: VersionConfig = VersionConfig { fields: &["G", "V"], - enums: &[&["G13", "G14"], &["V12_3", "V12_4", "V13_0B4", "V13_2"]], + enums: &[ + &["G13", "G14", "G14X"], + &["V12_3", "V12_4", "V13_0B4", "V13_2", "V13_3"], + ], versions: &[ &["G13", "V12_3"], &["G14", "V12_4"], - // &["G13", "V13_0B4"], - // &["G14", "V13_0B4"], - &["G13", "V13_2"], - &["G14", "V13_2"], + &["G13", "V13_3"], + &["G14", "V13_3"], + &["G14X", "V13_3"], ], }; From c34dd2257aff35b83ef3d7d76daab3033d51b325 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 16:25:10 +0900 Subject: [PATCH 522/635] drm/asahi: Initdata structure updates for G14X/V13_3 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/initdata.rs | 221 ++++++++++++++++++++++----- drivers/gpu/drm/asahi/initdata.rs | 56 ++++--- 2 files changed, 214 insertions(+), 63 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs index 923b560367194f..33c3d659fa30bf 100644 --- a/drivers/gpu/drm/asahi/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -68,14 +68,14 @@ pub(crate) mod raw { pub(crate) struct HwDataShared2Curve { pub(crate) unk_0: u32, pub(crate) unk_4: u32, - pub(crate) t1: Array<16, i16>, + pub(crate) t1: Array<16, u16>, pub(crate) t2: Array<16, i16>, pub(crate) t3: Array<8, Array<16, i32>>, } #[derive(Debug, Default)] #[repr(C)] - pub(crate) struct HwDataShared2T8112 { + pub(crate) struct HwDataShared2G14 { pub(crate) unk_0: Array<5, u32>, pub(crate) unk_14: u32, pub(crate) unk_18: Array<8, u32>, @@ -88,12 +88,11 @@ pub(crate) mod raw { pub(crate) struct HwDataShared2 { pub(crate) table: Array<10, i32>, pub(crate) unk_28: Array<0x10, u8>, - pub(crate) t8112: HwDataShared2T8112, + pub(crate) g14: HwDataShared2G14, pub(crate) unk_500: u32, pub(crate) unk_504: u32, pub(crate) unk_508: u32, pub(crate) unk_50c: u32, - pub(crate) unk_510: u32, } default_zeroed!(HwDataShared2); @@ -114,24 +113,24 @@ pub(crate) mod raw { pub(crate) unk_0: Array<0x38, u8>, pub(crate) unk_38: u32, pub(crate) unk_3c: u32, - pub(crate) unk_40: u32, + pub(crate) gpu_se_inactive_threshold: u32, pub(crate) unk_44: u32, - pub(crate) unk_48: u32, - pub(crate) unk_4c: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, pub(crate) unk_50: u32, pub(crate) unk_54: u32, pub(crate) unk_58: u32, pub(crate) unk_5c: u32, - pub(crate) unk_60: F32, - pub(crate) unk_64: F32, - pub(crate) unk_68: F32, - pub(crate) unk_6c: F32, - pub(crate) unk_70: F32, - pub(crate) unk_74: F32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, pub(crate) unk_78: F32, pub(crate) unk_7c: F32, - pub(crate) unk_80: F32, - pub(crate) unk_84: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, pub(crate) unk_88: u32, pub(crate) unk_8c: u32, pub(crate) max_pstate_scaled_1: u32, @@ -140,12 +139,10 @@ pub(crate) mod raw { pub(crate) unk_9c: F32, pub(crate) unk_a0: u32, pub(crate) unk_a4: u32, - pub(crate) unk_a8: u32, - pub(crate) unk_ac: u32, - pub(crate) unk_b0: u32, - pub(crate) unk_b4: u32, - pub(crate) unk_b8: u32, - pub(crate) unk_bc: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, pub(crate) unk_c0: u32, pub(crate) unk_c4: F32, pub(crate) unk_c8: Array<0x4c, u8>, @@ -199,6 +196,27 @@ pub(crate) mod raw { pub(crate) pad: u32, } + #[versions(AGX)] + const MAX_CORES_PER_CLUSTER: usize = { + #[ver(G >= G14X)] + { + 16 + } + #[ver(G < G14X)] + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + #[versions(AGX)] #[repr(C)] pub(crate) struct HwDataA { @@ -373,6 +391,8 @@ pub(crate) mod raw { pub(crate) pad_840: Pad<0x2c>, pub(crate) unk_86c: u32, pub(crate) fast_die0_sensor_mask: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask: U64, pub(crate) fast_die0_release_temp_cc: u32, pub(crate) unk_87c: i32, pub(crate) unk_880: u32, @@ -408,8 +428,10 @@ pub(crate) mod raw { pub(crate) pad_8f8: u32, pub(crate) pad_8fc: u32, pub(crate) unk_900: Array<0x24, u8>, - pub(crate) unk_coef_a1: Array<8, Array<8, F32>>, - pub(crate) unk_coef_a2: Array<8, Array<8, F32>>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + pub(crate) pad_b24: Pad<0x70>, pub(crate) max_pstate_scaled_11: u32, pub(crate) freq_with_off: u32, @@ -418,6 +440,10 @@ pub(crate) mod raw { pub(crate) unk_ba8: U64, pub(crate) unk_bb0: u32, pub(crate) unk_bb4: u32, + + #[ver(V >= V13_3)] + pub(crate) pad_bb8_0: Pad<0x200>, + pub(crate) pad_bb8: Pad<0x74>, pub(crate) unk_c2c: u32, pub(crate) power_zone_count: u32, @@ -483,28 +509,44 @@ pub(crate) mod raw { pub(crate) unk_e10_0: HwDataA130Extra, pub(crate) unk_e10: Array<0xc, u8>, + pub(crate) fast_die0_sensor_mask_2: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_2: U64, + pub(crate) unk_e24: u32, pub(crate) unk_e28: u32, pub(crate) unk_e2c: Pad<0x1c>, - pub(crate) unk_coef_b1: Array<8, Array<8, F32>>, - pub(crate) unk_coef_b2: Array<8, Array<8, F32>>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + #[ver(G >= G14X)] + pub(crate) pad_1048_0: Pad<0x600>, + pub(crate) pad_1048: Pad<0x5e4>, + pub(crate) fast_die0_sensor_mask_alt: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_alt: U64, #[ver(V < V13_0B4)] pub(crate) fast_die0_sensor_present: U64, pub(crate) unk_163c: u32, pub(crate) unk_1640: Array<0x2000, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_3640_0: Array<0x2000, u8>, + pub(crate) unk_3640: u32, pub(crate) unk_3644: u32, pub(crate) hws1: HwDataShared1, #[ver(V >= V13_0B4)] - pub(crate) unk_pad1: Pad<0x20>, + pub(crate) unk_hws2: Array<16, u16>, pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, pub(crate) unk_3c04: u32, pub(crate) hws3: HwDataShared3, pub(crate) unk_3c58: Array<0x3c, u8>, @@ -529,10 +571,13 @@ pub(crate) mod raw { pub(crate) unk_3cf0: u32, pub(crate) core_leak_coef: Array<8, F32>, pub(crate) sram_leak_coef: Array<8, F32>, - pub(crate) unk_3d34: Array<0x38, u8>, #[ver(V >= V13_0B4)] - pub(crate) unk_3d6c: Array<0x38, u8>, + pub(crate) aux_leak_coef: AuxLeakCoef, + #[ver(V >= V13_0B4)] + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, } #[versions(AGX)] default_zeroed!(HwDataA::ver); @@ -549,6 +594,37 @@ pub(crate) mod raw { pub(crate) readwrite: U64, } + #[versions(AGX)] + const IO_MAPPING_COUNT: usize = { + #[ver(V < V13_0B4)] + { + 0x14 + } + #[ver(V >= V13_0B4 && V < V13_3)] + { + 0x17 + } + #[ver(V >= V13_3)] + { + 0x18 + } + }; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + #[versions(AGX)] #[derive(Debug)] #[repr(C)] @@ -575,10 +651,10 @@ pub(crate) mod raw { pub(crate) yuv_matrices: Array<0x3f, Array<3, Array<4, i16>>>, pub(crate) pad_1c8: Pad<0x8>, - pub(crate) io_mappings: Array<0x14, IOMapping>, + pub(crate) io_mappings: Array, #[ver(V >= V13_0B4)] - pub(crate) unk_450_0: Array<0x68, u8>, + pub(crate) sgx_sram_ptr: U64, pub(crate) chip_id: u32, pub(crate) unk_454: u32, @@ -664,11 +740,18 @@ pub(crate) mod raw { pub(crate) frequencies: Array<0x10, u32>, pub(crate) voltages: Array<0x10, [u32; 0x8]>, pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + #[ver(V >= V13_3)] + pub(crate) unk_9f4_0: Pad<64>, + pub(crate) sram_k: Array<0x10, F32>, pub(crate) unk_9f4: Array<0x10, u32>, pub(crate) rel_max_powers: Array<0x10, u32>, pub(crate) rel_boost_freqs: Array<0x10, u32>, + #[ver(V >= V13_3)] + pub(crate) unk_arr_0: Array<32, u32>, + #[ver(V < V13_0B4)] pub(crate) min_sram_volt: u32, @@ -682,7 +765,10 @@ pub(crate) mod raw { pub(crate) unk_ac0: u32, #[ver(V >= V13_0B4)] - pub(crate) unk_ac4_0: Array<0x1f0, u8>, + pub(crate) aux_ps: HwDataBAuxPStates, + + #[ver(V >= V13_3)] + pub(crate) pad_ac4_0: Array<0x44c, u8>, pub(crate) pad_ac4: Pad<0x8>, pub(crate) unk_acc: u32, @@ -695,6 +781,10 @@ pub(crate) mod raw { pub(crate) unk_b04: u32, pub(crate) unk_b08: u32, pub(crate) unk_b0c: u32, + + #[ver(G >= G14X)] + pub(crate) pad_b10_0: Array<0x8, u8>, + pub(crate) unk_b10: u32, pub(crate) pad_b14: Pad<0x8>, pub(crate) unk_b1c: u32, @@ -711,12 +801,18 @@ pub(crate) mod raw { #[ver(V >= V13_0B4)] pub(crate) unk_b38_4: u32, + #[ver(V >= V13_3)] + pub(crate) unk_b38_8: u32, + pub(crate) unk_b38: Array<0xc, u32>, pub(crate) unk_b68: u32, #[ver(V >= V13_0B4)] pub(crate) unk_b6c: Array<0xd0, u8>, + #[ver(G >= G14X)] + pub(crate) unk_c3c_0: Array<0x8, u8>, + #[ver(V >= V13_0B4)] pub(crate) unk_c3c: u32, } @@ -764,6 +860,7 @@ pub(crate) mod raw { } default_zeroed!(GpuStatsComp); + #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct RuntimeScratch { @@ -797,10 +894,15 @@ pub(crate) mod raw { pub(crate) unk_ctr5: u32, pub(crate) unk_6afc: u32, pub(crate) pad_6b00: Pad<0x38>, + + #[ver(G >= G14X)] + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + pub(crate) unk_6b38: u32, pub(crate) pad_6b3c: Pad<0x84>, } - default_zeroed!(RuntimeScratch); + #[versions(AGX)] + default_zeroed!(RuntimeScratch::ver); pub(crate) type BufferMgrCtl = Array<4, u32>; @@ -829,15 +931,19 @@ pub(crate) mod raw { pub(crate) hwdata_b_2: GpuPointer<'a, super::HwDataB::ver>, pub(crate) fwlog_buf: Option>, pub(crate) unkptr_1b8: GpuPointer<'a, &'a [u8]>, + + #[ver(G < G14X)] pub(crate) unkptr_1c0: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14X)] pub(crate) unkptr_1c8: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_1d0: u32, pub(crate) unk_1d4: u32, pub(crate) unk_1d8: Array<0x3c, u8>, pub(crate) buffer_mgr_ctl: GpuPointer<'a, &'a [BufferMgrCtl]>, pub(crate) buffer_mgr_ctl_2: GpuPointer<'a, &'a [BufferMgrCtl]>, pub(crate) __pad1: Pad<0x5c>, - pub(crate) gpu_scratch: RuntimeScratch, + pub(crate) gpu_scratch: RuntimeScratch::ver, } #[versions(AGX)] no_debug!(RuntimePointers::ver<'_>); @@ -904,7 +1010,10 @@ pub(crate) mod raw { pub(crate) unk_24: u32, #[ver(V >= V13_0B4)] - pub(crate) unk_28_0: u32, + pub(crate) debug: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_28_4: u32, pub(crate) unk_28: u32, @@ -922,8 +1031,11 @@ pub(crate) mod raw { pub(crate) unk_1000: Array<0x7000, u8>, pub(crate) unk_8000: Array<0x900, u8>, - #[ver(V >= V13_0B4 && V < V13_2)] - pub(crate) unk_8900_0: u32, + #[ver(G >= G14X)] + pub(crate) unk_8900_pad: Array<0x484c, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_8900_pad2: Array<0x54, u8>, pub(crate) unk_8900: u32, pub(crate) pending_submissions: AtomicU32, @@ -968,12 +1080,23 @@ pub(crate) mod raw { #[ver(V >= V13_0B4)] pub(crate) unk_89f4_c: Array<0x50, u8>, + #[ver(V >= V13_3)] + pub(crate) unk_89f4_5c: Array<0xc, u8>, + pub(crate) unk_89f4: u32, pub(crate) hws1: HwDataShared1, pub(crate) hws2: HwDataShared2, #[ver(V >= V13_0B4)] - pub(crate) unk_hws2_0: Array<0x28, u8>, + pub(crate) unk_hws2_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_4: Array<0x8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_24: u32, + + pub(crate) unk_hws2_28: u32, pub(crate) hws3: HwDataShared3, pub(crate) unk_9004: Array<8, u8>, @@ -992,6 +1115,10 @@ pub(crate) mod raw { pub(crate) unk_10000: Array<0xe50, u8>, pub(crate) unk_10e50: u32, pub(crate) unk_10e54: Array<0x2c, u8>, + + #[ver(G >= G14X && V < V13_3 || G <= G14 && V >= V13_3)] + pub(crate) unk_x_pad: Array<0x4, u8>, + pub(crate) fault_control: u32, pub(crate) do_init: u32, pub(crate) unk_10e88: Array<0x188, u8>, @@ -1020,7 +1147,12 @@ pub(crate) mod raw { pub(crate) idle_off_delay_ms: AtomicU32, pub(crate) fender_idle_off_delay_ms: u32, pub(crate) fw_early_wake_timeout_ms: u32, - pub(crate) pending_stamps: Array<0x110, PendingStamp>, + #[ver(V >= V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) pending_stamps: Array<0x100, PendingStamp>, + #[ver(V < V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) unkpad_ps: Pad<0x78>, pub(crate) unk_117bc: u32, pub(crate) fault_info: FaultInfo, pub(crate) counter: u32, @@ -1029,6 +1161,12 @@ pub(crate) mod raw { #[ver(V >= V13_0B4)] pub(crate) unk_118e0_0: Array<0x9c, u8>, + #[ver(G >= G14X)] + pub(crate) unk_118e0_9c: Array<0x580, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_118e0_9c_x: Array<0x8, u8>, + pub(crate) unk_118e0: u32, #[ver(V >= V13_0B4)] @@ -1036,8 +1174,8 @@ pub(crate) mod raw { pub(crate) unk_118e4: u32, pub(crate) unk_118e8: u32, - pub(crate) unk_118ec: Array<0x15, u8>, - pub(crate) unk_11901: Array<0x43f, u8>, + pub(crate) unk_118ec: Array<0x400, u8>, + pub(crate) unk_11cec: Array<0x54, u8>, #[ver(V >= V13_0B4)] pub(crate) unk_11d40: Array<0x19c, u8>, @@ -1050,6 +1188,9 @@ pub(crate) mod raw { #[ver(V >= V13_0B4)] pub(crate) unk_11efc: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_11f00: Array<0x280, u8>, } #[versions(AGX)] default_zeroed!(Globals::ver); diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 29ee4bbd3a8a49..9e1a01dfaaad53 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -54,7 +54,7 @@ impl<'a> InitDataBuilder::ver<'a> { curve: &mut raw::HwDataShared2Curve, unk_0: u32, unk_4: u32, - t1: &[i16], + t1: &[u16], t2: &[i16], t3: &[&[i32]], ) { @@ -78,7 +78,7 @@ impl<'a> InitDataBuilder::ver<'a> { fn hw_shared2(cfg: &hw::HwConfig) -> Result> { let mut ret = box_in_place!(raw::HwDataShared2 { unk_28: Array::new([0xff; 16]), - t8112: Default::default(), + g14: Default::default(), unk_508: cfg.shared2_unk_508, ..Default::default() })?; @@ -88,13 +88,20 @@ impl<'a> InitDataBuilder::ver<'a> { } if cfg.chip_id == 0x8112 { - ret.t8112.unk_14 = 0x6000000; - Self::init_curve(&mut ret.t8112.curve1, 0, 0x20000000, &[-1], &[0x0f07], &[]); + ret.g14.unk_14 = 0x6000000; Self::init_curve( - &mut ret.t8112.curve2, + &mut ret.g14.curve1, + 0, + 0x20000000, + &[0xffff], + &[0x0f07], + &[], + ); + Self::init_curve( + &mut ret.g14.curve2, 7, 0x80000000, - &[-1, 25740, 17429, 12550, 9597, 7910, 6657, 5881, 5421], + &[0xffff, 25740, 17429, 12550, 9597, 7910, 6657, 5881, 5421], &[ 0x0f07, 0x04c0, 0x06c0, 0x08c0, 0x0ac0, 0x0c40, 0x0dc0, 0x0ec0, 0x0f80, ], @@ -306,28 +313,28 @@ impl<'a> InitDataBuilder::ver<'a> { unk_e10_0: raw::HwDataA130Extra { unk_38: 4, unk_3c: 8000, - unk_40: 2500, - unk_48: 0xffffffff, - unk_4c: 50, + gpu_se_inactive_threshold: 2500, + gpu_se_engagement_criteria: -1, + gpu_se_reset_criteria: 50, unk_54: 50, unk_58: 0x1, - unk_60: f32!(0.8888889), - unk_64: f32!(0.6666667), - unk_68: f32!(0.11111111), - unk_6c: f32!(0.33333333), - unk_70: f32!(-0.4), - unk_74: f32!(-0.8), + gpu_se_filter_a_neg: f32!(0.8888889), + gpu_se_filter_1_a_neg: f32!(0.6666667), + gpu_se_filter_a: f32!(0.11111111), + gpu_se_filter_1_a: f32!(0.33333333), + gpu_se_ki_dt: f32!(-0.4), + gpu_se_ki_1_dt: f32!(-0.8), unk_7c: f32!(65536.0), - unk_80: f32!(-5.0), - unk_84: f32!(-10.0), + gpu_se_kp: f32!(-5.0), + gpu_se_kp_1: f32!(-10.0), unk_8c: 40, max_pstate_scaled_1: max_ps_scaled, unk_9c: f32!(8000.0), unk_a0: 1400, - unk_a8: 72, - unk_ac: 24, - unk_b0: 1728000, - unk_b8: 576000, + gpu_se_filter_time_constant_ms: 72, + gpu_se_filter_time_constant_1_ms: 24, + gpu_se_filter_time_constant_clks: U64(1728000), + gpu_se_filter_time_constant_1_clks: U64(576000), unk_c4: f32!(65536.0), unk_114: f32!(65536.0), unk_124: 40, @@ -520,7 +527,7 @@ impl<'a> InitDataBuilder::ver<'a> { unk_24_0: 3000, unk_24: 0, #[ver(V >= V13_0B4)] - unk_28_0: 0, // debug + debug: 0, unk_28: 1, #[ver(V >= V13_0B4)] unk_2c_0: 0, @@ -664,7 +671,10 @@ impl<'a> InitDataBuilder::ver<'a> { fwlog_buf: None, unkptr_1b8: inner.unkptr_1b8.gpu_pointer(), + + #[ver(G < G14X)] unkptr_1c0: inner.unkptr_1c0.gpu_pointer(), + #[ver(G < G14X)] unkptr_1c8: inner.unkptr_1c8.gpu_pointer(), buffer_mgr_ctl: inner.buffer_mgr_ctl.gpu_pointer(), @@ -678,7 +688,7 @@ impl<'a> InitDataBuilder::ver<'a> { unk_1d8: Default::default(), __pad1: Default::default(), - gpu_scratch: raw::RuntimeScratch { + gpu_scratch: raw::RuntimeScratch::ver { unk_6b38: 0xff, ..Default::default() }, From 5e1331e3bd68d4f7d7c88ec078cb2040e3f056af Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 17:41:20 +0900 Subject: [PATCH 523/635] drm/asahi: hw: Add new config fields for t8112/t602x This includes new coefficients for t8112 that are needed for the correct table calculation code. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/hw/mod.rs | 43 +++++++++++++++++++++++++++++-- drivers/gpu/drm/asahi/hw/t600x.rs | 24 ++++++++++++----- drivers/gpu/drm/asahi/hw/t8103.rs | 15 +++++++++-- drivers/gpu/drm/asahi/hw/t8112.rs | 26 +++++++++++++++++-- drivers/gpu/drm/asahi/initdata.rs | 6 ++--- 5 files changed, 99 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 5d3e89aef0d31b..29471f55877591 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -109,6 +109,12 @@ pub(crate) struct PState { pub(crate) pwr_mw: u32, } +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + /// A power zone definition (we have no idea what this is but Apple puts them in the DT). #[allow(missing_docs)] #[derive(Debug, Copy, Clone)] @@ -175,6 +181,14 @@ pub(crate) struct HwRenderConfig { pub(crate) tiling_control: u32, } +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + /// Static hardware configuration for a given SoC model. #[derive(Debug)] pub(crate) struct HwConfig { @@ -195,6 +209,8 @@ pub(crate) struct HwConfig { pub(crate) base_clock_hz: u32, /// Output address space for the UAT on this SoC. pub(crate) uat_oas: usize, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, /// Maximum number of clusters on this SoC. pub(crate) max_num_clusters: u32, /// Maximum number of cores per cluster for this GPU. @@ -226,6 +242,23 @@ pub(crate) struct HwConfig { pub(crate) shared2_tab: &'static [i32], /// HwDataShared2.unk_508. pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.unk_hws2_0. + pub(crate) unk_hws2_0: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + /// Constant related to SRAM voltages. pub(crate) sram_k: F32, /// Unknown per-cluster coefficients 1. @@ -234,15 +267,21 @@ pub(crate) struct HwConfig { pub(crate) unk_coef_b: &'static [&'static [F32]], /// Unknown table in Global struct. pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, /// Temperature sensor list (8 bits per sensor). - pub(crate) fast_die0_sensor_mask: u64, + pub(crate) fast_sensor_mask: [u64; 2], /// Temperature sensor list (alternate). - pub(crate) fast_die0_sensor_mask_alt: u64, + pub(crate) fast_sensor_mask_alt: [u64; 2], /// Temperature sensor present bitmask. pub(crate) fast_die0_sensor_present: u32, /// Required MMIO mappings for this GPU/firmware. pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, } /// Dynamic (fetched from hardware/DT) configuration. diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs index 2654f51cf40ad7..af809547a643d4 100644 --- a/drivers/gpu/drm/asahi/hw/t600x.rs +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -51,6 +51,7 @@ pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { base_clock_hz: 24_000_000, uat_oas: 42, + num_dies: 2, max_num_clusters: 8, max_num_cores: 8, max_num_frags: 8, @@ -83,6 +84,13 @@ pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { shared1_a4: 0xffff, shared2_tab: &[-1, -1, -1, -1, 0x2aa, 0xaaa, -1, -1, 0, 0], shared2_unk_508: 0xcc00001, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + unk_hws2_0: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, sram_k: f32!(1.02), unk_coef_a: &[ &f32!([9.838]), @@ -107,10 +115,13 @@ pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { global_tab: Some(&[ 0, 1, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, ]), - fast_die0_sensor_mask: 0x8080808080808080, - fast_die0_sensor_mask_alt: 0x9090909090909090, + has_csafr: false, + fast_sensor_mask: [0x8080808080808080, 0], + fast_sensor_mask_alt: [0x9090909090909090, 0], fast_die0_sensor_present: 0xff, io_mappings: &iomaps(16, true), + sram_base: None, + sram_size: None, }; pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { @@ -118,9 +129,10 @@ pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { gpu_variant: GpuVariant::C, gpu_core: GpuCore::G13C, + num_dies: 1, max_num_clusters: 4, - fast_die0_sensor_mask: 0x80808080, - fast_die0_sensor_mask_alt: 0x90909090, + fast_sensor_mask: [0x80808080, 0], + fast_sensor_mask_alt: [0x90909090, 0], fast_die0_sensor_present: 0x0f, io_mappings: &iomaps(8, false), ..HWCONFIG_T6002 @@ -132,8 +144,8 @@ pub(crate) const HWCONFIG_T6000: super::HwConfig = HwConfig { gpu_core: GpuCore::G13S, max_num_clusters: 2, - fast_die0_sensor_mask: 0x8080, - fast_die0_sensor_mask_alt: 0x9090, + fast_sensor_mask: [0x8080, 0], + fast_sensor_mask_alt: [0x9090, 0], fast_die0_sensor_present: 0x03, io_mappings: &iomaps(4, false), ..HWCONFIG_T6001 diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs index 3d38b088a0f576..8cfdc54cfe03d9 100644 --- a/drivers/gpu/drm/asahi/hw/t8103.rs +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -16,6 +16,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { base_clock_hz: 24_000_000, uat_oas: 40, + num_dies: 1, max_num_clusters: 1, max_num_cores: 8, max_num_frags: 8, @@ -48,12 +49,20 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { shared1_a4: 0xffff, shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0, 0], shared2_unk_508: 0xc00007, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + unk_hws2_0: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, sram_k: f32!(1.02), unk_coef_a: &[], unk_coef_b: &[], global_tab: None, - fast_die0_sensor_mask: 0x12, - fast_die0_sensor_mask_alt: 0x12, + has_csafr: false, + fast_sensor_mask: [0x12, 0], + fast_sensor_mask_alt: [0x12, 0], fast_die0_sensor_present: 0x01, io_mappings: &[ Some(IOMapping::new(0x204d00000, 0x1c000, 0x1c000, true)), // Fender @@ -77,4 +86,6 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { None, // None, // ], + sram_base: None, + sram_size: None, }; diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs index 5624dca130be45..b4d58019e3bf78 100644 --- a/drivers/gpu/drm/asahi/hw/t8112.rs +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -16,6 +16,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { base_clock_hz: 24_000_000, uat_oas: 40, + num_dies: 1, max_num_clusters: 1, max_num_cores: 10, max_num_frags: 10, @@ -49,13 +50,32 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { shared1_a4: 0, shared2_tab: &[-1, -1, -1, -1, -1, -1, -1, -1, 0xaa5aa, 0], shared2_unk_508: 0xc00000, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 7200, + t2: &[ + 0xf07, 0x4c0, 0x6c0, 0x8c0, 0xac0, 0xc40, 0xdc0, 0xec0, 0xf80, + ], + t3_coefs: &[0, 20, 28, 36, 44, 50, 56, 60, 63], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 5, + shared3_tab: &[ + 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, 10700, + 10700, 10700, 10700, + ], + unk_hws2_0: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), // 13.2: last coef changed from 6.6 to 5.3, assuming that was a fix we can backport unk_coef_a: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], unk_coef_b: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], global_tab: None, - fast_die0_sensor_mask: 0x6800, - fast_die0_sensor_mask_alt: 0x6800, + has_csafr: false, + fast_sensor_mask: [0x6800, 0], + fast_sensor_mask_alt: [0x6800, 0], fast_die0_sensor_present: 0x02, io_mappings: &[ Some(IOMapping::new(0x204d00000, 0x14000, 0x14000, true)), // Fender @@ -79,4 +99,6 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { Some(IOMapping::new(0x23b3d0000, 0x1000, 0x1000, true)), // Some(IOMapping::new(0x23b3c0000, 0x1000, 0x1000, true)), // ], + sram_base: None, + sram_size: None, }; diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 9e1a01dfaaad53..1930b3f24007c6 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -260,7 +260,7 @@ impl<'a> InitDataBuilder::ver<'a> { max_pstate_scaled_7: max_ps_scaled, unk_alpha_neg: f32!(0.8), unk_alpha: f32!(0.2), - fast_die0_sensor_mask: U64(self.cfg.fast_die0_sensor_mask), + fast_die0_sensor_mask: U64(self.cfg.fast_sensor_mask[0]), fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, unk_87c: self.cfg.da.unk_87c, unk_880: 0x4, @@ -341,10 +341,10 @@ impl<'a> InitDataBuilder::ver<'a> { max_pstate_scaled_2: max_ps_scaled, ..Default::default() }, - fast_die0_sensor_mask_2: U64(self.cfg.fast_die0_sensor_mask), + fast_die0_sensor_mask_2: U64(self.cfg.fast_sensor_mask[0]), unk_e24: self.cfg.da.unk_e24, unk_e28: 1, - fast_die0_sensor_mask_alt: U64(self.cfg.fast_die0_sensor_mask_alt), + fast_die0_sensor_mask_alt: U64(self.cfg.fast_sensor_mask_alt[0]), #[ver(V < V13_0B4)] fast_die0_sensor_present: U64(self.cfg.fast_die0_sensor_present as u64), unk_163c: 1, From 1736a816f3ddff292ca6448f999bb558b3a52e8a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 17:42:13 +0900 Subject: [PATCH 524/635] drm/asahi: Implement correct shared2 table calculation This was already missing in t8112, since the tables aren't static and vary device to device. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/initdata.rs | 96 ++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 1930b3f24007c6..2f0eeb0e35d851 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -15,6 +15,7 @@ use crate::fw::initdata::*; use crate::fw::types::*; use crate::{box_in_place, f32, place}; use crate::{gpu, hw, mmu}; +use alloc::vec::Vec; use kernel::error::Result; use kernel::macros::versions; @@ -56,7 +57,7 @@ impl<'a> InitDataBuilder::ver<'a> { unk_4: u32, t1: &[u16], t2: &[i16], - t3: &[&[i32]], + t3: &[Vec], ) { curve.unk_0 = unk_0; curve.unk_4 = unk_4; @@ -67,7 +68,7 @@ impl<'a> InitDataBuilder::ver<'a> { for (i, a) in curve.t3.iter_mut().enumerate() { a.fill(0x3ffffff); if i < t3.len() { - let b = t3[i]; + let b = &t3[i]; (**a)[..b.len()].copy_from_slice(b); } } @@ -75,7 +76,10 @@ impl<'a> InitDataBuilder::ver<'a> { /// Create the HwDataShared2 structure, which is used in two places in InitData. #[inline(never)] - fn hw_shared2(cfg: &hw::HwConfig) -> Result> { + fn hw_shared2( + cfg: &hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> Result> { let mut ret = box_in_place!(raw::HwDataShared2 { unk_28: Array::new([0xff; 16]), g14: Default::default(), @@ -87,36 +91,57 @@ impl<'a> InitDataBuilder::ver<'a> { ret.table[i] = *val; } - if cfg.chip_id == 0x8112 { - ret.g14.unk_14 = 0x6000000; - Self::init_curve( - &mut ret.g14.curve1, - 0, - 0x20000000, - &[0xffff], - &[0x0f07], - &[], - ); - Self::init_curve( - &mut ret.g14.curve2, - 7, - 0x80000000, - &[0xffff, 25740, 17429, 12550, 9597, 7910, 6657, 5881, 5421], - &[ - 0x0f07, 0x04c0, 0x06c0, 0x08c0, 0x0ac0, 0x0c40, 0x0dc0, 0x0ec0, 0x0f80, - ], - &[ - &[0x3ffffff, 107, 101, 94, 87, 82, 77, 73, 71], - &[ - 0x3ffffff, 38240, 36251, 33562, 31368, 29379, 27693, 26211, 25370, - ], - &[ - 0x3ffffff, 123933, 117485, 108771, 101661, 95217, 89751, 84948, 82222, - ], - ], - ); + let curve_cfg = match cfg.shared2_curves.as_ref() { + None => return Ok(ret), + Some(a) => a, + }; + + let mut t1 = Vec::new(); + let mut t3 = Vec::new(); + + for _ in 0..curve_cfg.t3_scales.len() { + t3.try_push(Vec::new())?; + } + + for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { + let t3_coef = curve_cfg.t3_coefs[i]; + if t3_coef == 0 { + t1.try_push(0xffff)?; + for j in t3.iter_mut() { + j.try_push(0x3ffffff)?; + } + continue; + } + + let f_mhz = (ps.freq_hz / 1000) as u64; + let v_max = ps.max_volt_mv() as u64; + + t1.try_push( + (1000000000 * (curve_cfg.t1_coef as u64) / (f_mhz * v_max)) + .try_into() + .unwrap(), + )?; + + for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { + t3[j].try_push( + (t3_coef as u64 * 1000000000 * *scale as u64 / (f_mhz * v_max * 6)) + .try_into() + .unwrap(), + )?; + } } + ret.g14.unk_14 = 0x6000000; + Self::init_curve( + &mut ret.g14.curve1, + 0, + 0x20000000, + &[0xffff], + &[0x0f07], + &[], + ); + Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); + Ok(ret) } @@ -350,7 +375,7 @@ impl<'a> InitDataBuilder::ver<'a> { unk_163c: 1, unk_3644: 0, hws1: Self::hw_shared1(self.cfg), - hws2: *Self::hw_shared2(self.cfg)?, + hws2: *Self::hw_shared2(self.cfg, self.dyncfg)?, hws3: *Self::hw_shared3(self.cfg)?, unk_3ce8: 1, ..Default::default() @@ -392,6 +417,11 @@ impl<'a> InitDataBuilder::ver<'a> { raw.power_zones[i].unk_10 = 1320000000; } + #[ver(V >= V13_0B4)] + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + Ok(raw) }) } @@ -568,7 +598,7 @@ impl<'a> InitDataBuilder::ver<'a> { unk_89f4_8: 1, unk_89f4: 0, hws1: Self::hw_shared1(self.cfg), - hws2: *Self::hw_shared2(self.cfg)?, + hws2: *Self::hw_shared2(self.cfg, self.dyncfg)?, hws3: *Self::hw_shared3(self.cfg)?, unk_900c: 1, #[ver(V >= V13_0B4)] From d54993caee32e55d7510e59081ffd723e8c61aeb Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 17:44:39 +0900 Subject: [PATCH 525/635] drm/asahi: initdata: Pull shared3 table from the HwConfig Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/initdata.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 2f0eeb0e35d851..a775bc3d640fb1 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -152,14 +152,11 @@ impl<'a> InitDataBuilder::ver<'a> { ..Default::default() })?; - if cfg.chip_id == 0x8112 { + if !cfg.shared3_tab.is_empty() { ret.unk_0 = 1; ret.unk_4 = 500; - ret.unk_8 = 5; - ret.table.copy_from_slice(&[ - 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, - 10700, 10700, 10700, 10700, - ]); + ret.unk_8 = cfg.shared3_unk; + ret.table.copy_from_slice(cfg.shared3_tab); ret.unk_4c = 1; } From a7d01068f712eb81a3b7121495b0a2a43401e4b3 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 18:19:48 +0900 Subject: [PATCH 526/635] drm/asahi: initdata: Implement new cfg/dyncfg dependent fields --- drivers/gpu/drm/asahi/hw/mod.rs | 29 +++++++++- drivers/gpu/drm/asahi/initdata.rs | 92 ++++++++++++++++++++----------- 2 files changed, 88 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 29471f55877591..97e5f48b1272f1 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -409,6 +409,18 @@ pub(crate) struct PwrConfig { pub(crate) pwr_integral_min_clamp: u32, pub(crate) pwr_min_duty_cycle: u32, pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, } impl PwrConfig { @@ -501,6 +513,8 @@ impl PwrConfig { return Err(EINVAL); } + let power_sample_period: u32 = prop!("apple,power-sample-period"); + Ok(PwrConfig { core_leak_coef, sram_leak_coef, @@ -536,7 +550,7 @@ impl PwrConfig { perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), perf_reset_iters: prop!("apple,perf-reset-iters", 6), perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), - power_sample_period: prop!("apple,power-sample-period"), + power_sample_period, ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), ppm_ki: prop!("apple,ppm-ki"), ppm_kp: prop!("apple,ppm-kp"), @@ -545,6 +559,19 @@ impl PwrConfig { pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), perf_states, power_zones, diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index a775bc3d640fb1..11ba86e50da73a 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -206,8 +206,13 @@ impl<'a> InitDataBuilder::ver<'a> { let max_ps_scaled = 100 * max_ps; let boost_ps_count = max_ps - base_ps; + #[allow(unused_variables)] let base_clock_khz = self.cfg.base_clock_hz / 1000; - let clocks_per_period = base_clock_khz * period_ms; + let clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = + self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; let raw = place!( ptr, @@ -332,36 +337,53 @@ impl<'a> InitDataBuilder::ver<'a> { _ => Default::default(), }, #[ver(V >= V13_0B4)] - unk_e10_0: raw::HwDataA130Extra { - unk_38: 4, - unk_3c: 8000, - gpu_se_inactive_threshold: 2500, - gpu_se_engagement_criteria: -1, - gpu_se_reset_criteria: 50, - unk_54: 50, - unk_58: 0x1, - gpu_se_filter_a_neg: f32!(0.8888889), - gpu_se_filter_1_a_neg: f32!(0.6666667), - gpu_se_filter_a: f32!(0.11111111), - gpu_se_filter_1_a: f32!(0.33333333), - gpu_se_ki_dt: f32!(-0.4), - gpu_se_ki_1_dt: f32!(-0.8), - unk_7c: f32!(65536.0), - gpu_se_kp: f32!(-5.0), - gpu_se_kp_1: f32!(-10.0), - unk_8c: 40, - max_pstate_scaled_1: max_ps_scaled, - unk_9c: f32!(8000.0), - unk_a0: 1400, - gpu_se_filter_time_constant_ms: 72, - gpu_se_filter_time_constant_1_ms: 24, - gpu_se_filter_time_constant_clks: U64(1728000), - gpu_se_filter_time_constant_1_clks: U64(576000), - unk_c4: f32!(65536.0), - unk_114: f32!(65536.0), - unk_124: 40, - max_pstate_scaled_2: max_ps_scaled, - ..Default::default() + unk_e10_0: { + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + raw::HwDataA130Extra { + unk_38: 4, + unk_3c: 8000, + gpu_se_inactive_threshold: pwr.se_inactive_threshold, + gpu_se_engagement_criteria: pwr.se_engagement_criteria, + gpu_se_reset_criteria: pwr.se_reset_criteria, + unk_54: 50, + unk_58: 0x1, + gpu_se_filter_a_neg: f32!(1.0) - filter_a, + gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, + gpu_se_filter_a: filter_a, + gpu_se_filter_1_a: filter_1_a, + gpu_se_ki_dt: pwr.se_ki * period_s, + gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, + unk_7c: f32!(65536.0), + gpu_se_kp: pwr.se_kp, + gpu_se_kp_1: pwr.se_kp_1, + + #[ver(V >= V13_3)] + unk_8c: 100, + #[ver(V < V13_3)] + unk_8c: 40, + + max_pstate_scaled_1: max_ps_scaled, + unk_9c: f32!(8000.0), + unk_a0: 1400, + gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant + * period_ms, + gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 + * period_ms, + gpu_se_filter_time_constant_clks: U64((pwr + .se_filter_time_constant + * clocks_per_period_coarse) + .into()), + gpu_se_filter_time_constant_1_clks: U64((pwr + .se_filter_time_constant_1 + * clocks_per_period_coarse) + .into()), + unk_c4: f32!(65536.0), + unk_114: f32!(65536.0), + unk_124: 40, + max_pstate_scaled_2: max_ps_scaled, + ..Default::default() + } }, fast_die0_sensor_mask_2: U64(self.cfg.fast_sensor_mask[0]), unk_e24: self.cfg.da.unk_e24, @@ -562,7 +584,7 @@ impl<'a> InitDataBuilder::ver<'a> { unk_30: 0, unk_34: 120, sub: raw::GlobalsSub::ver { - unk_54: 0xffff, + unk_54: self.cfg.global_unk_54, unk_56: 40, unk_58: 0xffff, unk_5e: U32(1), @@ -597,6 +619,12 @@ impl<'a> InitDataBuilder::ver<'a> { hws1: Self::hw_shared1(self.cfg), hws2: *Self::hw_shared2(self.cfg, self.dyncfg)?, hws3: *Self::hw_shared3(self.cfg)?, + #[ver(V >= V13_0B4)] + unk_hws2_0: self.cfg.unk_hws2_0, + #[ver(V >= V13_0B4)] + unk_hws2_4: self.cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), + #[ver(V >= V13_0B4)] + unk_hws2_24: self.cfg.unk_hws2_24, unk_900c: 1, #[ver(V >= V13_0B4)] unk_9010_0: 1, From 44a4d3852903e60599c00af75535d7fa3d05caf2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 19:09:51 +0900 Subject: [PATCH 527/635] rust: macros: versions: Support conditionals on block statements This makes stuff like: #[ver(...)] if foo { // version and runtime condition specific code } work. Signed-off-by: Asahi Lina --- rust/macros/versions.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs index 994567fd871856..4caa88483f7787 100644 --- a/rust/macros/versions.rs +++ b/rust/macros/versions.rs @@ -1,4 +1,4 @@ -use proc_macro::{Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; //use crate::helpers::expect_punct; @@ -39,6 +39,29 @@ fn drop_until_punct(it: &mut impl Iterator, delimiter: &str) { } } +fn drop_until_braces(it: &mut impl Iterator) { + let mut depth: isize = 0; + for token in it.by_ref() { + match token { + TokenTree::Punct(punct) => match punct.as_char() { + '<' => { + depth += 1; + } + '>' => { + depth -= 1; + } + _ => (), + }, + TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => { + if depth == 0 { + break; + } + } + _ => (), + } + } +} + struct VersionConfig { fields: &'static [&'static str], enums: &'static [&'static [&'static str]], @@ -150,6 +173,12 @@ fn filter_versions( } else { let first = it.next().unwrap(); match &first { + TokenTree::Ident(ident) + if ["while", "for", "loop", "if", "match", "unsafe", "fn"] + .contains(&ident.to_string().as_str()) => + { + drop_until_braces(&mut it); + } TokenTree::Group(_) => (), _ => { drop_until_punct(&mut it, ",;"); From 7ef215d8591b08d0d56ce16c7ddafc7de97c00c8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 19:10:43 +0900 Subject: [PATCH 528/635] drm/asahi: fw.types: Impl Zeroed for F32 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/types.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs index c1a07be1e047e7..5397a5be057442 100644 --- a/drivers/gpu/drm/asahi/fw/types.rs +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -87,6 +87,7 @@ unsafe impl Zeroed for i8 {} unsafe impl Zeroed for i16 {} unsafe impl Zeroed for i32 {} unsafe impl Zeroed for i64 {} +unsafe impl Zeroed for F32 {} /// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by /// hand or not very useful. From e327674960eaf499c380d678475d03b3cc296c8f Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 19:12:39 +0900 Subject: [PATCH 529/635] drm/asahi: Load and set CS/AFR performance and leakage data Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/hw/mod.rs | 119 +++++++++++++++++++++--------- drivers/gpu/drm/asahi/initdata.rs | 38 ++++++++++ 2 files changed, 122 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 97e5f48b1272f1..c6a0ea9cbb6984 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -324,6 +324,23 @@ pub(crate) struct GpuIdConfig { pub(crate) core_masks_packed: Vec, } +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: Vec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: Vec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: Vec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: Vec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + /// Configurable GPU power settings from the device tree. #[derive(Debug)] pub(crate) struct PwrConfig { @@ -337,6 +354,8 @@ pub(crate) struct PwrConfig { /// SRAM leakage coefficient per cluster. pub(crate) sram_leak_coef: Vec, + pub(crate) csafr: Option, + /// Maximum total power of the GPU in milliwatts. pub(crate) max_power_mw: u32, /// Maximum frequency of the GPU in megahertz. @@ -424,45 +443,37 @@ pub(crate) struct PwrConfig { } impl PwrConfig { - /// Load the GPU power configuration from the device tree. - pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { + fn load_opp( + dev: &AsahiDevice, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { let mut perf_states = Vec::new(); let node = dev.of_node().ok_or(EIO)?; - let opps = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; - - let mut max_power_mw: u32 = 0; - let mut max_freq_mhz: u32 = 0; - - macro_rules! prop { - ($prop:expr, $default:expr) => {{ - node.get_opt_property(c_str!($prop)) - .map_err(|e| { - dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); - e - })? - .unwrap_or($default) - }}; - ($prop:expr) => {{ - node.get_property(c_str!($prop)).map_err(|e| { - dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); - e - })? - }}; - } + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; for opp in opps.children() { let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; let mut volt_uv: Vec = opp.get_property(c_str!("opp-microvolt"))?; - let pwr_uw: u32 = opp.get_property(c_str!("opp-microwatt"))?; - - if volt_uv.len() != cfg.max_num_clusters as usize { + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { dev_err!( dev, "Invalid opp-microvolt length (expected {}, got {})\n", - cfg.max_num_clusters, + voltage_count, volt_uv.len() ); return Err(EINVAL); @@ -472,10 +483,6 @@ impl PwrConfig { let volt_mv = volt_uv; let pwr_mw = pwr_uw / 1000; - max_power_mw = max_power_mw.max(pwr_mw); - - let freq_mhz: u32 = (freq_hz / 1_000_000).try_into()?; - max_freq_mhz = max_freq_mhz.max(freq_mhz); perf_states.try_push(PState { freq_hz: freq_hz.try_into()?, @@ -484,6 +491,35 @@ impl PwrConfig { })?; } + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree. + pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + let pz_data = prop!("apple,power-zones", Vec::new()); if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { @@ -513,14 +549,26 @@ impl PwrConfig { return Err(EINVAL); } + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + let power_sample_period: u32 = prop!("apple,power-sample-period"); Ok(PwrConfig { core_leak_coef, sram_leak_coef, - max_power_mw, - max_freq_mhz, + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, perf_base_pstate: prop!("apple,perf-base-pstate", 1), perf_max_pstate: perf_states.len() as u32 - 1, @@ -575,6 +623,7 @@ impl PwrConfig { perf_states, power_zones, + csafr, }) } diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 11ba86e50da73a..aaaa4f9c70fd0e 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -413,6 +413,19 @@ impl<'a> InitDataBuilder::ver<'a> { raw.sram_leak_coef[i] = *coef; } + #[ver(V >= V13_0B4)] + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + for i in 0..self.dyncfg.id.num_clusters as usize { if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); @@ -548,6 +561,31 @@ impl<'a> InitDataBuilder::ver<'a> { }; } + #[ver(V >= V13_0B4)] + if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + Ok(raw) }) } From 296c2be9740d1ea6bb36bcd9f173719f27053c26 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 19:16:03 +0900 Subject: [PATCH 530/635] drm/asahi: Instantiate driver for 13.3 versions Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/driver.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index a1422c53b3761d..0052f0aa61e0a5 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -120,13 +120,22 @@ impl platform::Driver for AsahiDriver { let compat: Vec = node.get_property(c_str!("apple,firmware-compat"))?; let reg = drm::drv::Registration::::new(&dev)?; - let gpu = match (cfg.gpu_gen, compat.as_slice()) { - (hw::GpuGen::G13, &[12, 3, 0]) => { + let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { + (hw::GpuGen::G13, _, &[12, 3, 0]) => { gpu::GpuManagerG13V12_3::new(reg.device(), &res, cfg)? as Arc } - (hw::GpuGen::G14, &[12, 4, 0]) => { + (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { gpu::GpuManagerG14V12_4::new(reg.device(), &res, cfg)? as Arc } + (hw::GpuGen::G13, _, &[13, 3, 0]) => { + gpu::GpuManagerG13V13_3::new(reg.device(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 3, 0]) => { + gpu::GpuManagerG14V13_3::new(reg.device(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, _, &[13, 3, 0]) => { + gpu::GpuManagerG14XV13_3::new(reg.device(), &res, cfg)? as Arc + } _ => { dev_info!( dev, From 458faea2c99b95d981ae59485f24ba33d8d11c8d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 20:52:42 +0900 Subject: [PATCH 531/635] drm/asahi: regs: Implement ID and fault register changes for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 2 +- drivers/gpu/drm/asahi/hw/mod.rs | 5 ++ drivers/gpu/drm/asahi/regs.rs | 131 ++++++++++++++++++++++++++------ 3 files changed, 112 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 52eb103628aad2..6a6fb8a0ae9aa4 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -711,7 +711,7 @@ impl GpuManager::ver { } }; - let info = res.get_fault_info(); + let info = res.get_fault_info(self.cfg); if info.is_some() { dev_err!(self.dev, " Fault info: {:#x?}\n", info.as_ref().unwrap()); } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index c6a0ea9cbb6984..8041b214e5ce79 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -68,6 +68,11 @@ pub(crate) enum GpuCore { G13C = 13, // G14P = 14, G14G = 15, + G14S = 16, + G14C = 17, + // G15M = 18, + // G15P_AGX2 = 19, + // G15P = 20, } /// GPU revision ID. Note: Part of the firmware ABI. diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 019d7214793d3e..ca25c171fda2ad 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -31,6 +31,10 @@ const ID_CLUSTERS: usize = 0xd0401c; const CORE_MASK_0: usize = 0xd01500; const CORE_MASK_1: usize = 0xd01514; +const CORE_MASKS_G14X: usize = 0xe01500; +const FAULT_INFO_G14X: usize = 0xd8c0; +const FAULT_ADDR_G14X: usize = 0xd8c8; + /// Enum representing the unit that caused an MMU fault. #[allow(non_camel_case_types)] #[allow(clippy::upper_case_acronyms)] @@ -98,6 +102,16 @@ pub(crate) enum FaultUnit { gRDE0_SP(u8), gRDE1_SP(u8), + gCDM_CS, + gCDM_ID, + gCDM_CSR, + gCDM_CSW, + gCDM_CTXR, + gCDM_CTXW, + gIPP, + gIPP_CS, + gKSM_RCE, + Unknown(u8), } @@ -206,43 +220,75 @@ impl Resources { id_clusters ); - let core_mask_0 = self.sgx_read32(CORE_MASK_0); - let core_mask_1 = self.sgx_read32(CORE_MASK_1); - let mut core_mask = (core_mask_0 as u64) | ((core_mask_1 as u64) << 32); + let gpu_gen = (id_version >> 24) & 0xff; + + let mut core_mask_regs = Vec::new(); - dev_info!(self.dev, "Core mask: {:#x}\n", core_mask); + let num_clusters = match gpu_gen { + 4 | 5 => { + // G13 | G14G + core_mask_regs.try_push(self.sgx_read32(CORE_MASK_0))?; + core_mask_regs.try_push(self.sgx_read32(CORE_MASK_1))?; + (id_clusters >> 12) & 0xff + } + 6 => { + // G14X + core_mask_regs.try_push(self.sgx_read32(CORE_MASKS_G14X))?; + core_mask_regs.try_push(self.sgx_read32(CORE_MASKS_G14X + 4))?; + core_mask_regs.try_push(self.sgx_read32(CORE_MASKS_G14X + 8))?; + (id_clusters >> 28) & 0xf // TODO: guess + } + a => { + dev_err!(self.dev, "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }; + + let mut core_masks_packed = Vec::new(); + core_masks_packed.try_extend_from_slice(&core_mask_regs)?; + + dev_info!(self.dev, "Core masks: {:#x?}\n", core_masks_packed); - let num_clusters = (id_clusters >> 12) & 0xff; let num_cores = id_counts_1 & 0xff; - if num_cores * num_clusters > 64 { + if num_cores > 32 { dev_err!( self.dev, - "Too many total cores ({} x {} > 64)\n", - num_clusters, + "Too many cores per cluster ({} > 32)\n", num_cores ); return Err(ENODEV); } + if num_cores * num_clusters > (core_mask_regs.len() * 32) as u32 { + dev_err!( + self.dev, + "Too many total cores ({} x {} > {})\n", + num_clusters, + num_cores, + core_mask_regs.len() * 32 + ); + return Err(ENODEV); + } + let mut core_masks = Vec::new(); let mut total_active_cores: u32 = 0; - let max_core_mask = (1u64 << num_cores) - 1; - for _i in 0..num_clusters { - let mask = core_mask & max_core_mask; - core_masks.try_push(mask as u32)?; - core_mask >>= num_cores; + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.try_push(mask)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] >>= num_cores; + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); + } + } total_active_cores += mask.count_ones(); } - let mut core_masks_packed = Vec::new(); - core_masks_packed.try_push(core_mask_0)?; - if core_mask_1 != 0 { - core_masks_packed.try_push(core_mask_1)?; - } - if core_mask != 0 { - dev_err!(self.dev, "Leftover core mask: {:#x}\n", core_mask); + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!(self.dev, "Leftover core mask: {:#x?}\n", core_mask_regs); return Err(EIO); } @@ -263,6 +309,7 @@ impl Resources { gpu_gen: match (id_version >> 24) & 0xff { 4 => hw::GpuGen::G13, 5 => hw::GpuGen::G14, + 6 => hw::GpuGen::G14, // G14X has a separate ID a => { dev_err!(self.dev, "Unknown GPU generation {}\n", a); return Err(ENODEV); @@ -289,7 +336,7 @@ impl Resources { max_dies: (id_clusters >> 20) & 0xf, num_clusters, num_cores, - num_frags: (id_counts_1 >> 8) & 0xff, + num_frags: num_cores, // Used to be id_counts_1[15:8] but does not work for G14X num_gps: (id_counts_2 >> 16) & 0xff, total_active_cores, core_masks, @@ -298,13 +345,25 @@ impl Resources { } /// Get the fault information from the MMU status register, if one occurred. - pub(crate) fn get_fault_info(&self) -> Option { - let fault_info = self.sgx_read64(FAULT_INFO); + pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { + let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; + + let fault_info = if g14x { + self.sgx_read64(FAULT_INFO_G14X) + } else { + self.sgx_read64(FAULT_INFO) + }; if fault_info & 1 == 0 { return None; } + let fault_addr = if g14x { + self.sgx_read64(FAULT_ADDR_G14X) + } else { + fault_info >> 30 + }; + let unit_code = ((fault_info >> 9) & 0xff) as u8; let unit = match unit_code { 0x00..=0x9f => match unit_code & 0xf { @@ -341,7 +400,29 @@ impl Resources { 0xae => FaultUnit::GSL2, 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), 0xb8 => FaultUnit::GL2CC_MB, - 0xe0..=0xff => match unit_code & 0xf { + 0xd0..=0xdf if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gCDM_CS, + 0x1 => FaultUnit::gCDM_ID, + 0x2 => FaultUnit::gCDM_CSR, + 0x3 => FaultUnit::gCDM_CSW, + 0x4 => FaultUnit::gCDM_CTXR, + 0x5 => FaultUnit::gCDM_CTXW, + 0x6 => FaultUnit::gIPP, + 0x7 => FaultUnit::gIPP_CS, + 0x8 => FaultUnit::gKSM_RCE, + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if !g14x => match unit_code & 0xf { 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), @@ -373,7 +454,7 @@ impl Resources { }; Some(FaultInfo { - address: (fault_info >> 30) << 6, + address: fault_addr << 6, sideband: ((fault_info >> 23) & 0x7f) as u8, vm_slot: ((fault_info >> 17) & 0x3f) as u32, unit_code, From 60289e603acb55daac6783b8896b1f334dd7debf Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 21:06:24 +0900 Subject: [PATCH 532/635] drm/asahi: hw: Add t602x device data Includes preliminary T6022 and T6020 data (untested). Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/hw/mod.rs | 1 + drivers/gpu/drm/asahi/hw/t602x.rs | 159 ++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 drivers/gpu/drm/asahi/hw/t602x.rs diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 8041b214e5ce79..b8c52be7104509 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -14,6 +14,7 @@ use kernel::prelude::*; const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; +pub(crate) mod t602x; pub(crate) mod t8103; pub(crate) mod t8112; diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs new file mode 100644 index 00000000000000..a953c1b14168db --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(mcc_count: usize) -> [Option; 24] { + [ + Some(IOMapping::new(0x404d00000, 0x144000, 0x144000, true)), // Fender + Some(IOMapping::new(0x20e100000, 0x4000, 0x4000, false)), // AICTimer + Some(IOMapping::new(0x28e106000, 0x4000, 0x4000, true)), // AICSWInt + Some(IOMapping::new(0x404000000, 0x20000, 0x20000, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x28e478000, 0x4000, 0x4000, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404e08000, 0x8000, 0x8000, true)), // MetrologySensorRegs + None, // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + mcc_count * 0xd8000, + 0xd8000, + true, + )), // MCache registers + Some(IOMapping::new(0x28e118000, 0x4000, 0x4000, false)), // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, 0x4000, 0x4000, true)), // ? + Some(IOMapping::new(0x28e3c0000, 0x4000, 0x4000, false)), // ? + Some(IOMapping::new(0x28e3d8000, 0x4000, 0x4000, true)), // ? + Some(IOMapping::new(0x404eac000, 0x4000, 0x4000, true)), // ? + None, + None, + ] +} + +// TODO: Tentative +pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { + chip_id: 0x6022, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G14C, + gpu_feat_compat: 0, + gpu_feat_incompat: feat::incompat::MANDATORY_ZS_COMPRESSION, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x40, + + render: HwRenderConfig { + tiling_control: 0x180340, + }, + + da: HwConfigA { + unk_87c: 500, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0, // Unused + unk_abc: 0, // Unused + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0xaaaaa, 0], + shared2_unk_508: 0xc00007, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 11000, + t2: &[ + 0xf07, 0x4c0, 0x680, 0x8c0, 0xa80, 0xc40, 0xd80, 0xec0, 0xf40, + ], + t3_coefs: &[0, 20, 27, 36, 43, 50, 55, 60, 62], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 8, + shared3_tab: &[ + 125, 125, 125, 125, 125, 125, 125, 125, 7500, 125, 125, 125, 125, 125, 125, 125, + ], + unk_hws2_0: 700, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + unk_hws2_24: 6, + global_unk_54: 4000, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + ], + unk_coef_b: &[ + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + ], + global_tab: Some(&[ + 0, 2, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: true, + fast_sensor_mask: [0x40005000c000d00, 0x40005000c000d00], + fast_sensor_mask_alt: [0x140015001d001d00, 0x140015001d001d00], + fast_die0_sensor_present: 0, // Unused + io_mappings: &iomaps(16), + sram_base: Some(0x404d60000), + sram_size: Some(0x20000), +}; + +pub(crate) const HWCONFIG_T6021: super::HwConfig = HwConfig { + chip_id: 0x6021, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G14C, + + num_dies: 1, + max_num_clusters: 4, + fast_sensor_mask: [0x40005000c000d00, 0], + fast_sensor_mask_alt: [0x140015001d001d00, 0], + io_mappings: &iomaps(8), + ..HWCONFIG_T6022 +}; + +pub(crate) const HWCONFIG_T6020: super::HwConfig = HwConfig { + chip_id: 0x6020, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G14S, + + max_num_clusters: 2, + fast_sensor_mask: [0xc000d00, 0], + fast_sensor_mask_alt: [0x1d001d00, 0], + io_mappings: &iomaps(4), + ..HWCONFIG_T6021 +}; From 60e79cd00e7f3ed5e301415a4d28b2791265c467 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 21:06:56 +0900 Subject: [PATCH 533/635] drm/asahi: driver: Bind to t6020 and t6021 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/driver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 0052f0aa61e0a5..26e79b36592f67 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -81,6 +81,8 @@ kernel::define_of_id_table! {ASAHI_ID_TABLE, &'static hw::HwConfig, [ (of::DeviceId::Compatible(b"apple,agx-t6000"), Some(&hw::t600x::HWCONFIG_T6000)), (of::DeviceId::Compatible(b"apple,agx-t6001"), Some(&hw::t600x::HWCONFIG_T6001)), (of::DeviceId::Compatible(b"apple,agx-t6002"), Some(&hw::t600x::HWCONFIG_T6002)), + (of::DeviceId::Compatible(b"apple,agx-t6020"), Some(&hw::t602x::HWCONFIG_T6020)), + (of::DeviceId::Compatible(b"apple,agx-t6021"), Some(&hw::t602x::HWCONFIG_T6021)), ]} /// Platform Driver implementation for `AsahiDriver`. From 5e44637a4d85f6439045030ccb1dfd1ea98a2daf Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 22:11:50 +0900 Subject: [PATCH 534/635] drm/asahi: initdata: Add version IDs for G14/G14X 13.3 TODO: G13(X) Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/initdata.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index aaaa4f9c70fd0e..c3b66a911bf414 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -835,8 +835,12 @@ impl<'a> InitDataBuilder::ver<'a> { Ok(place!( ptr, raw::InitData::ver { - #[ver(V >= V13_0B4)] - ver_info: Array::new([1, 1, 16, 1]), + #[ver(V == V13_3 && G == G13)] + ver_info: Array::new([1, 1, 16, 1]), // TODO + #[ver(V == V13_3 && G == G14)] + ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), + #[ver(V == V13_3 && G == G14X)] + ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), unk_buf: inner.unk_buf.gpu_pointer(), unk_8: 0, unk_c: 0, From c19d84a0787a2a8cce18d5115399c0025385595f Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 22:30:01 +0900 Subject: [PATCH 535/635] drm/asahi: initdata: Value updates for 13.3 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/initdata.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index c3b66a911bf414..78dda4101cfcc8 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -534,12 +534,19 @@ impl<'a> InitDataBuilder::ver<'a> { #[ver(V >= V13_0B4)] unk_b38_4: 1, unk_b38: Array::new([0xffffffff; 12]), - #[ver(V >= V13_0B4)] + #[ver(V >= V13_0B4 && V < V13_3)] unk_c3c: 0x19, + #[ver(V >= V13_3)] + unk_c3c: 0x1a, ..Default::default() } ); + #[ver(V >= V13_3)] + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; @@ -616,7 +623,9 @@ impl<'a> InitDataBuilder::ver<'a> { #[ver(V >= V13_0B4)] debug: 0, unk_28: 1, - #[ver(V >= V13_0B4)] + #[ver(V >= V13_3)] + unk_2c_0: 1, + #[ver(V >= V13_0B4 && V < V13_3)] unk_2c_0: 0, unk_2c: 1, unk_30: 0, From 65136b61a8e5290ece01177f93f5d3b4f316ab02 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 22:30:22 +0900 Subject: [PATCH 536/635] drm/asahi: initdata: Pad voltages out like macOS does This removes some diff noise. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/initdata.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 78dda4101cfcc8..7c010004e5b525 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -559,6 +559,10 @@ impl<'a> InitDataBuilder::ver<'a> { raw.voltages[i][j] = *mv; raw.voltages_sram[i][j] = sram_mv; } + for j in ps.volt_mv.len()..raw.voltages[i].len() { + raw.voltages[i][j] = raw.voltages[i][0]; + raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; + } raw.sram_k[i] = self.cfg.sram_k; raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; raw.rel_boost_freqs[i] = if i > base_ps { From 8b49deea205487465fcd3c54f53c13584bd71b5e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 23:12:27 +0900 Subject: [PATCH 537/635] drm/asahi: gpu: Implement SGX SRAM mapping Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 29 +++++++++++++++++++++++++---- drivers/gpu/drm/asahi/mmu.rs | 3 +-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 6a6fb8a0ae9aa4..ba74008e562180 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -438,6 +438,22 @@ impl GpuManager::ver { } } + #[ver(V >= V13_0B4)] + if let Some(base) = cfg.sram_base { + let size = cfg.sram_size.unwrap() as usize; + + let mapping = + mgr.uat + .kernel_vm() + .map_io(base as usize, size, mmu::PROT_FW_SHARED_RW)?; + + mgr.initdata.runtime_pointers.hwdata_b.with_mut(|raw, _| { + raw.sgx_sram_ptr = U64(mapping.iova() as u64); + }); + + mgr.io_mappings.try_push(mapping)?; + } + let mgr = Arc::from(mgr); let rtkit = Box::try_new(rtkit::RtKit::::new( @@ -647,10 +663,15 @@ impl GpuManager::ver { let off = map.base & mmu::UAT_PGMSK; let base = map.base - off; let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; - let mapping = self - .uat - .kernel_vm() - .map_io(base, end - base, map.writable)?; + let mapping = self.uat.kernel_vm().map_io( + base, + end - base, + if map.writable { + mmu::PROT_FW_MMIO_RW + } else { + mmu::PROT_FW_MMIO_RO + }, + )?; self.initdata.runtime_pointers.hwdata_b.with_mut(|raw, _| { raw.io_mappings[index] = fw::initdata::raw::IOMapping { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 226ca0b7c1d74e..624db1a5fa3a65 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -899,8 +899,7 @@ impl Vm { } /// Add a direct MMIO mapping to this Vm at a free address. - pub(crate) fn map_io(&self, phys: usize, size: usize, rw: bool) -> Result { - let prot = if rw { PROT_FW_MMIO_RW } else { PROT_FW_MMIO_RO }; + pub(crate) fn map_io(&self, phys: usize, size: usize, prot: u32) -> Result { let mut inner = self.inner.lock(); let uat_inner = inner.uat_inner.clone(); From 83b6c58c47b807a63fff605696f688fbd1b8b0a6 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 23:16:23 +0900 Subject: [PATCH 538/635] drm/asahi: gpu: Move the buffer manager control to the kernel lower VM This is a mitigation introduced in macOS 13.3. We were already OK without mapping the kernel address space to user contexts, but it seems we need to do that for a new G14X thing. Doing this keeps the buffer manager stuff safe. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 19 +++++++++++++++++++ drivers/gpu/drm/asahi/initdata.rs | 3 +++ drivers/gpu/drm/asahi/mmu.rs | 9 +++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index ba74008e562180..9e4097eb5974ce 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -81,6 +81,11 @@ const IOVA_KERN_GPU_BASE: u64 = 0xffffffaf00000000; /// GPU/FW shared structure VA range top. const IOVA_KERN_GPU_TOP: u64 = 0xffffffafffffffff; +/// GPU/FW low shared structure VA range base. +const IOVA_KERN_GPU_LOW_BASE: u64 = 0x20_0000_0000; +/// GPU/FW low shared structure VA range top. +const IOVA_KERN_GPU_LOW_TOP: u64 = 0x20_ffff_ffff; + /// Timeout for entering the halt state after a fault or request. const HALT_ENTER_TIMEOUT_MS: u64 = 100; @@ -94,7 +99,9 @@ pub(crate) struct KernelAllocators { pub(crate) private: alloc::DefaultAllocator, pub(crate) shared: alloc::DefaultAllocator, pub(crate) shared_ro: alloc::DefaultAllocator, + #[allow(dead_code)] pub(crate) gpu: alloc::DefaultAllocator, + pub(crate) gpu_low: alloc::DefaultAllocator, } /// Receive (GPU->driver) ring buffer channels. @@ -369,6 +376,18 @@ impl GpuManager::ver { fmt!("Kernel GPU Shared"), false, )?, + gpu_low: alloc::DefaultAllocator::new( + dev, + uat.kernel_lower_vm(), + IOVA_KERN_GPU_LOW_BASE, + IOVA_KERN_GPU_LOW_TOP, + 0x80, + mmu::PROT_GPU_FW_SHARED_RW, + 64 * 1024, + true, + fmt!("Kernel GPU Lower"), + false, + )?, }; let event_manager = Self::make_event_manager(&mut alloc)?; diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 7c010004e5b525..3c3bd31b553130 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -750,7 +750,10 @@ impl<'a> InitDataBuilder::ver<'a> { unkptr_1c0: self.alloc.private.array_empty(0x300)?, unkptr_1c8: self.alloc.private.array_empty(0x1000)?, + #[ver(V < V13_3)] buffer_mgr_ctl: self.alloc.gpu.array_empty(127)?, + #[ver(V >= V13_3)] + buffer_mgr_ctl: self.alloc.gpu_low.array_empty(127)?, })?; self.alloc.private.new_boxed(pointers, |inner, ptr| { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 624db1a5fa3a65..da085c38af9aed 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -611,7 +611,7 @@ pub(crate) struct Uat { slots: slotalloc::SlotAllocator, kernel_vm: Vm, - _kernel_lower_vm: Vm, + kernel_lower_vm: Vm, } impl Drop for UatRegion { @@ -1082,6 +1082,11 @@ impl Uat { &self.kernel_vm } + /// Returns a reference to the local kernel (lower half) `Vm` + pub(crate) fn kernel_lower_vm(&self) -> &Vm { + &self.kernel_lower_vm + } + /// Returns the base physical address of the TTBAT region. pub(crate) fn ttb_base(&self) -> u64 { let inner = self.inner.lock(); @@ -1196,7 +1201,7 @@ impl Uat { cfg, pagetables_rgn, kernel_vm, - _kernel_lower_vm: kernel_lower_vm, + kernel_lower_vm, inner, slots: slotalloc::SlotAllocator::new(UAT_USER_CTX as u32, (), |_inner, _slot| { SlotInner() From 13f0dbdc014fb5476e94a9974d655eb7757920ea Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 23:33:18 +0900 Subject: [PATCH 539/635] drm/asahi: gpu,mmu: Map kernel AS to user contexts on G14X+ We were avoiding this to avoid security implications, but it seems G14X needs to read command buffers directly from the hardware as well as write to a new scene object... Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 9 ++++++++- drivers/gpu/drm/asahi/mmu.rs | 25 ++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 9e4097eb5974ce..fc5929167c52eb 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -507,7 +507,14 @@ impl GpuManager::ver { /// Force disable inlining to avoid blowing up the stack. #[inline(never)] fn make_uat(dev: &AsahiDevice, cfg: &'static hw::HwConfig) -> Result> { - Ok(Box::try_new(mmu::Uat::new(dev, cfg)?)?) + // G14X has a new thing in the Scene structure that unfortunately requires + // write access from user contexts. Hopefully it's not security-sensitive. + #[ver(G >= G14X)] + let map_kernel_to_user = true; + #[ver(G < G14X)] + let map_kernel_to_user = false; + + Ok(Box::try_new(mmu::Uat::new(dev, cfg, map_kernel_to_user)?)?) } /// Actually create the final GpuManager instance, as a UniqueArc. diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index da085c38af9aed..f75402b54c613f 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -562,6 +562,8 @@ impl Drop for Mapping { /// Shared UAT global data structures struct UatShared { + kernel_ttb1: u64, + map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, } @@ -979,6 +981,7 @@ impl Drop for VmInner { ); } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); + uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); } uat_inner.handoff().unlock(); core::mem::drop(uat_inner); @@ -1108,6 +1111,13 @@ impl Uat { let ttb = inner.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + } else { + 0 + }; + let ttbs = uat_inner.ttbs(); uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { @@ -1117,7 +1127,7 @@ impl Uat { ); } ttbs[idx].ttb0.store(ttb, Ordering::Relaxed); - ttbs[idx].ttb1.store(0, Ordering::Relaxed); + ttbs[idx].ttb1.store(ttb1, Ordering::Relaxed); uat_inner.handoff().unlock(); core::mem::drop(uat_inner); @@ -1169,6 +1179,8 @@ impl Uat { } addr_of_mut!((*ptr).shared).write(Mutex::new(UatShared { + kernel_ttb1: 0, + map_kernel_to_user: false, handoff_rgn, ttbs_rgn, })); @@ -1180,7 +1192,11 @@ impl Uat { /// Creates a new `Uat` instance given the relevant hardware config. #[inline(never)] - pub(crate) fn new(dev: &driver::AsahiDevice, cfg: &'static hw::HwConfig) -> Result { + pub(crate) fn new( + dev: &driver::AsahiDevice, + cfg: &'static hw::HwConfig, + map_kernel_to_user: bool, + ) -> Result { dev_info!(dev, "MMU: Initializing...\n"); let inner = Self::make_inner(dev)?; @@ -1208,7 +1224,10 @@ impl Uat { })?, }; - let inner = uat.inner.lock(); + let mut inner = uat.inner.lock(); + + inner.map_kernel_to_user = map_kernel_to_user; + inner.kernel_ttb1 = uat.pagetables_rgn.base; inner.handoff().init()?; From a190f6b458391e71a375f608a32403639084086f Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 23:43:40 +0900 Subject: [PATCH 540/635] drm/asahi: buffer: Scene update for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 12 +++++++++++- drivers/gpu/drm/asahi/fw/buffer.rs | 13 ++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index 5e629b884c4c51..35447e8a5b2be2 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -593,6 +593,8 @@ impl Buffer::ver { clustering: clustering, preempt_buf: preempt_buf, seq_buf: seq_buf, + #[ver(G >= G14X)] + control_word: alloc.gpu.array_empty(1)?, })?; // Could be made strong, but we wind up with a deadlock if we try to grab the @@ -608,16 +610,24 @@ impl Buffer::ver { let scene = alloc.shared.new_boxed(scene_inner, |inner, ptr| { Ok(place!( ptr, - buffer::raw::Scene { + buffer::raw::Scene::ver { + #[ver(G >= G14X)] + control_word: inner.control_word.gpu_pointer(), + #[ver(G >= G14X)] + control_word2: inner.control_word.gpu_pointer(), pass_page_count: AtomicU32::new(0), unk_4: 0, unk_8: U64(0), unk_10: U64(0), user_buffer: inner.user_buffer.gpu_pointer(), unk_20: 0, + #[ver(V >= V13_3)] + unk_28: U64(0), stats: stats_pointer, total_page_count: AtomicU32::new(0), + #[ver(G < G14X)] unk_30: U64(0), + #[ver(G < G14X)] unk_38: U64(0), } )) diff --git a/drivers/gpu/drm/asahi/fw/buffer.rs b/drivers/gpu/drm/asahi/fw/buffer.rs index a8a46787951895..89d0927f2902bb 100644 --- a/drivers/gpu/drm/asahi/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/fw/buffer.rs @@ -81,18 +81,27 @@ pub(crate) mod raw { pub(crate) unk_90: Array<0x30, u8>, } + #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct Scene<'a> { + #[ver(G >= G14X)] + pub(crate) control_word: GpuPointer<'a, &'a [u32]>, + #[ver(G >= G14X)] + pub(crate) control_word2: GpuPointer<'a, &'a [u32]>, pub(crate) pass_page_count: AtomicU32, pub(crate) unk_4: u32, pub(crate) unk_8: U64, pub(crate) unk_10: U64, pub(crate) user_buffer: GpuPointer<'a, &'a [u8]>, pub(crate) unk_20: u32, + #[ver(V >= V13_3)] + pub(crate) unk_28: U64, pub(crate) stats: GpuWeakPointer, pub(crate) total_page_count: AtomicU32, + #[ver(G < G14X)] pub(crate) unk_30: U64, // pad + #[ver(G < G14X)] pub(crate) unk_38: U64, // pad } @@ -143,6 +152,8 @@ pub(crate) struct Scene { pub(crate) clustering: Option, pub(crate) preempt_buf: GpuArray, pub(crate) seq_buf: GpuArray, + #[ver(G >= G14X)] + pub(crate) control_word: GpuArray, } #[versions(AGX)] @@ -150,7 +161,7 @@ no_debug!(Scene::ver); #[versions(AGX)] impl GpuStruct for Scene::ver { - type Raw<'a> = raw::Scene<'a>; + type Raw<'a> = raw::Scene::ver<'a>; } #[versions(AGX)] From 4bd998c3b129efc7160d550945c0681b6905226c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 23:44:00 +0900 Subject: [PATCH 541/635] drm/asahi: fw.channels: Message length & ID updates for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/channels.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs index db5ac9a3ded5cd..0c52c206dd26ff 100644 --- a/drivers/gpu/drm/asahi/fw/channels.rs +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -130,9 +130,9 @@ pub(crate) struct RunWorkQueueMsg { pub(crate) wptr: u32, pub(crate) event_slot: u32, pub(crate) is_new: bool, - #[ver(V >= V13_2 && G >= G14)] + #[ver(V >= V13_2 && G == G14)] pub(crate) __pad: Pad<0x2b>, - #[ver(V < V13_2 || G < G14)] + #[ver(V < V13_2 || G != G14)] pub(crate) __pad: Pad<0x1b>, } @@ -141,11 +141,11 @@ pub(crate) type PipeMsg = RunWorkQueueMsg::ver; #[versions(AGX)] pub(crate) const DEVICECONTROL_SZ: usize = { - #[ver(V < V13_2 || G < G14)] + #[ver(V < V13_2 || G != G14)] { 0x2c } - #[ver(V >= V13_2 && G >= G14)] + #[ver(V >= V13_2 && G == G14)] { 0x3c } @@ -182,6 +182,8 @@ pub(crate) enum DeviceControlMsg { Unk14(Array), Unk15(Array), Unk16(Array), + #[ver(V >= V13_3)] + Unk17(Array), DestroyContext { unk_4: u32, ctx_23: u8, From cad97b1b1f062ac370ece6f6bc193f2c7b547140 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 19 Apr 2023 23:48:03 +0900 Subject: [PATCH 542/635] drm/asahi: fw.workqueue: Barrier update for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/workqueue.rs | 2 ++ drivers/gpu/drm/asahi/queue/mod.rs | 1 + drivers/gpu/drm/asahi/queue/render.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs index f213a4d923817e..6260ccfde64395 100644 --- a/drivers/gpu/drm/asahi/fw/workqueue.rs +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -35,6 +35,8 @@ pub(crate) mod raw { pub(crate) stamp_self: EventValue, pub(crate) uuid: u32, pub(crate) barrier_type: u32, + // G14X addition + pub(crate) padding: Pad<0x20>, } #[derive(Debug, Clone, Copy)] diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index fc03ca8972b681..b6a3fdda5c004e 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -634,6 +634,7 @@ impl Queue for Queue::ver { stamp_self: queue_job.event_info().value.next(), uuid: 0xffffbbbb, barrier_type: 0, + padding: Default::default(), } )) }, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 0a12473c4ca89f..0ea13d2118e576 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -387,6 +387,7 @@ impl super::Queue::ver { stamp_self: ev_frag.value.next(), uuid: uuid_3d, barrier_type: 0, + padding: Default::default(), } )) }, From c6e76977324ec42879e9f913248b3f1b804e68b6 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 00:04:02 +0900 Subject: [PATCH 543/635] iommu/io-pgtable: Hack in FW-RW/GPU-RO mode into UAT io_pgtable This is ugly, we need a better way of expressing this. Signed-off-by: Asahi Lina --- drivers/iommu/io-pgtable-arm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index bbcbf30a064bbe..9d3a8c604d0287 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -424,6 +424,9 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, if (prot & IOMMU_WRITE) { /* Firmware RW */ pte |= APPLE_UAT_UXN; + if (!(prot & IOMMU_READ)) + /* Hack: Firmware RW, GPU RO */ + pte |= APPLE_UAT_PXN; } else if (!(prot & IOMMU_READ)) { /* No access */ pte |= APPLE_UAT_PXN; From 9ac011b459943e0379c8c1c987d171ca440a7b92 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 00:10:07 +0900 Subject: [PATCH 544/635] gpu: Add a FW-RW/GPU-RO allocator We will use this for command buffers, since G14X needs that. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 38 ++++++++++++++++++++++++++++++++++-- drivers/gpu/drm/asahi/mmu.rs | 2 ++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index fc5929167c52eb..c4e0181be01209 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -67,7 +67,11 @@ const DOORBELL_DEVCTRL: u64 = 0x11; /// Private (cached) firmware structure VA range base. const IOVA_KERN_PRIV_BASE: u64 = 0xffffffa000000000; /// Private (cached) firmware structure VA range top. -const IOVA_KERN_PRIV_TOP: u64 = 0xffffffa7ffffffff; +const IOVA_KERN_PRIV_TOP: u64 = 0xffffffa5ffffffff; +/// Private (cached) GPU-RO firmware structure VA range base. +const IOVA_KERN_GPU_RO_BASE: u64 = 0xffffffa600000000; +/// Private (cached) GPU-RO firmware structure VA range top. +const IOVA_KERN_GPU_RO_TOP: u64 = 0xffffffa7ffffffff; /// Shared (uncached) firmware structure VA range base. const IOVA_KERN_SHARED_BASE: u64 = 0xffffffa800000000; /// Shared (uncached) firmware structure VA range top. @@ -101,6 +105,7 @@ pub(crate) struct KernelAllocators { pub(crate) shared_ro: alloc::DefaultAllocator, #[allow(dead_code)] pub(crate) gpu: alloc::DefaultAllocator, + pub(crate) gpu_ro: alloc::DefaultAllocator, pub(crate) gpu_low: alloc::DefaultAllocator, } @@ -376,6 +381,18 @@ impl GpuManager::ver { fmt!("Kernel GPU Shared"), false, )?, + gpu_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RO_BASE, + IOVA_KERN_GPU_RO_TOP, + 0x80, + mmu::PROT_GPU_RO_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel GPU RO Shared"), + true, + )?, gpu_low: alloc::DefaultAllocator::new( dev, uat.kernel_lower_vm(), @@ -866,6 +883,7 @@ impl GpuManager::ver { let mut guard = self.alloc.lock(); let (garbage_count, _) = guard.private.garbage(); + let (garbage_count_gpuro, _) = guard.gpu_ro.garbage(); let dc = context.with( |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { @@ -906,6 +924,7 @@ impl GpuManager::ver { // The invalidation does a cache flush, so it is okay to collect garbage guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(garbage_count_gpuro); Ok(()) } @@ -979,7 +998,7 @@ impl GpuManager for GpuManager::ver { if garbage_bytes > MAX_FW_ALLOC_GARBAGE { mod_dev_dbg!( self.dev, - "Collecting kalloc garbage ({} objects, {} bytes)\n", + "Collecting kalloc/private garbage ({} objects, {} bytes)\n", garbage_count, garbage_bytes ); @@ -990,6 +1009,21 @@ impl GpuManager for GpuManager::ver { } } + let (garbage_count, garbage_bytes) = guard.gpu_ro.garbage(); + if garbage_bytes > MAX_FW_ALLOC_GARBAGE { + mod_dev_dbg!( + self.dev, + "Collecting kalloc/gpuro garbage ({} objects, {} bytes)\n", + garbage_count, + garbage_bytes + ); + if self.flush_fw_cache().is_err() { + dev_err!(self.dev, "Failed to flush FW cache\n"); + } else { + guard.gpu_ro.collect_garbage(garbage_count); + } + } + guard } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f75402b54c613f..59bf050b5ec19d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -103,6 +103,8 @@ pub(crate) const PROT_FW_PRIV_RO: u32 = prot::PRIV | prot::READ; pub(crate) const PROT_GPU_FW_SHARED_RW: u32 = prot::READ | prot::WRITE | prot::CACHE; /// Firmware/GPU shared (private) RW pub(crate) const PROT_GPU_FW_PRIV_RW: u32 = prot::READ | prot::WRITE; +/// Firmware-RW/GPU-RO shared (private) RW +pub(crate) const PROT_GPU_RO_FW_PRIV_RW: u32 = prot::PRIV | prot::WRITE; /// GPU shared/coherent RW pub(crate) const PROT_GPU_SHARED_RW: u32 = prot::READ | prot::WRITE | prot::CACHE | prot::NOEXEC; /// GPU shared/coherent RO From 8bd00c0dd659fff6128433d0a8f886e72a0bf641 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 00:13:47 +0900 Subject: [PATCH 545/635] drm/asahi: render,compute: Allocate command buffers from gpuro Needed by G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/queue/render.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 69039edd3fbf8d..9bec92783454a6 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -129,7 +129,7 @@ impl super::Queue::ver { let count = self.counter.fetch_add(1, Ordering::Relaxed); let comp = GpuObject::new_prealloc( - kalloc.private.alloc_object()?, + kalloc.gpu_ro.alloc_object()?, |ptr: GpuWeakPointer| { let mut builder = microseq::Builder::new(); diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 0ea13d2118e576..d7a4859951749b 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -455,7 +455,7 @@ impl super::Queue::ver { mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); let frag = GpuObject::new_prealloc( - kalloc.private.alloc_object()?, + kalloc.gpu_ro.alloc_object()?, |ptr: GpuWeakPointer| { let mut builder = microseq::Builder::new(); @@ -861,7 +861,7 @@ impl super::Queue::ver { mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); let vtx = GpuObject::new_prealloc( - kalloc.private.alloc_object()?, + kalloc.gpu_ro.alloc_object()?, |ptr: GpuWeakPointer| { let mut builder = microseq::Builder::new(); From fce8d6303c4d2ee96f49ef45f0f8bb355cb8fd8e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 00:17:30 +0900 Subject: [PATCH 546/635] drm/asahi: render: Add missing unk1 field to StartFragment Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/render.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index d7a4859951749b..5869fd49fe91c6 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -486,7 +486,7 @@ impl super::Queue::ver { unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), unk_7c: 0, unk_80: 0, - unk_84: 0, + unk_84: unk1.into(), uuid: uuid_3d, attachments: common::build_attachments( cmdbuf.attachments, From d0807b5d5a2e99ae54be05d3702456b9ad45f268 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 00:21:35 +0900 Subject: [PATCH 547/635] drm/asahi: buffer,workqueue: Update some conditionals for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 2 +- drivers/gpu/drm/asahi/fw/buffer.rs | 2 +- drivers/gpu/drm/asahi/fw/workqueue.rs | 4 ++-- drivers/gpu/drm/asahi/workqueue.rs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index 35447e8a5b2be2..56c61c9c8604cb 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -353,7 +353,7 @@ impl Buffer::ver { unk_10: 0x0, gpu_counter2: 0x0, unk_18: 0x0, - #[ver(V < V13_0B4)] + #[ver(V < V13_0B4 || G >= G14X)] unk_1c: 0x0, page_list: inner.page_list.gpu_pointer(), page_list_size: (4 * max_pages).try_into()?, diff --git a/drivers/gpu/drm/asahi/fw/buffer.rs b/drivers/gpu/drm/asahi/fw/buffer.rs index 89d0927f2902bb..9cab2765791b4d 100644 --- a/drivers/gpu/drm/asahi/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/fw/buffer.rs @@ -52,7 +52,7 @@ pub(crate) mod raw { pub(crate) gpu_counter2: u32, pub(crate) unk_18: u32, - #[ver(V < V13_0B4)] + #[ver(V < V13_0B4 || G >= G14X)] pub(crate) unk_1c: u32, pub(crate) page_list: GpuPointer<'a, &'a [u32]>, diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs index 6260ccfde64395..8ad8bb1b0eee22 100644 --- a/drivers/gpu/drm/asahi/fw/workqueue.rs +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -139,11 +139,11 @@ pub(crate) mod raw { pub(crate) unk_94: u32, pub(crate) pending: AtomicU32, pub(crate) unk_9c: u32, - #[ver(V >= V13_2)] + #[ver(V >= V13_2 && G < G14X)] pub(crate) unk_a0_0: u32, pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, pub(crate) unk_a8: U64, - #[ver(V >= V13_2)] + #[ver(V >= V13_2 && G < G14X)] pub(crate) unk_b0: u32, } } diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index a50c5758b1acad..022cd2e39e6498 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -597,11 +597,11 @@ impl WorkQueue::ver { unk_94: 0, pending: Default::default(), unk_9c: 0, - #[ver(V >= V13_2)] + #[ver(V >= V13_2 && G < G14X)] unk_a0_0: 0, gpu_context: inner.gpu_context.gpu_pointer(), unk_a8: Default::default(), - #[ver(V >= V13_2)] + #[ver(V >= V13_2 && G < G14X)] unk_b0: 0, } )) From ddb61fdfa97d9d967d24e62be87b89081d53a14d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 01:01:47 +0900 Subject: [PATCH 548/635] drm/asahi: fw,render: Render command structure updates for G14X The first two job params structs got replaced with a register write array. Also shuffle around paddings a bit to make it make more sense. This does not yet populate the registers. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/fragment.rs | 19 +++++++--- drivers/gpu/drm/asahi/fw/job.rs | 29 +++++++++++++++ drivers/gpu/drm/asahi/fw/microseq.rs | 18 +++++++--- drivers/gpu/drm/asahi/fw/vertex.rs | 13 +++++-- drivers/gpu/drm/asahi/queue/render.rs | 51 +++++++++++++++++++++------ 5 files changed, 106 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs index a01b0d0d7807f8..8830a268eb0475 100644 --- a/drivers/gpu/drm/asahi/fw/fragment.rs +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -111,11 +111,11 @@ pub(crate) mod raw { pub(crate) unk_160: U64, #[ver(G < G14)] - pub(crate) unk_168_padding: Array<0x1d8, u8>, + pub(crate) __pad: Pad<0x1d8>, #[ver(G >= G14)] - pub(crate) unk_168_padding: Array<0x1a8, u8>, + pub(crate) __pad: Pad<0x1a8>, #[ver(V < V13_0B4)] - pub(crate) __pad0: Pad<0x8>, + pub(crate) __pad1: Pad<0x8>, } #[derive(Debug)] @@ -138,13 +138,13 @@ pub(crate) mod raw { pub(crate) unk_38: u32, pub(crate) unk_3c: u32, pub(crate) unk_40: u32, + pub(crate) __pad: Pad<0xac>, } #[versions(AGX)] #[derive(Debug)] #[repr(C)] pub(crate) struct JobParameters3 { - pub(crate) unk_44_padding: Array<0xac, u8>, pub(crate) depth_bias_array: ArrayAddr, pub(crate) scissor_array: ArrayAddr, pub(crate) visibility_result_buffer: U64, @@ -174,7 +174,8 @@ pub(crate) mod raw { pub(crate) tib_blocks: u32, pub(crate) unk_30c: u32, pub(crate) aux_fb_info: AuxFBInfo::ver, - pub(crate) unk_320_padding: Array<0x10, u8>, + pub(crate) tile_config: U64, + pub(crate) unk_328_padding: Array<0x8, u8>, pub(crate) unk_partial_store_pipeline: StorePipelineBinding, pub(crate) partial_store_pipeline: StorePipelineBinding, pub(crate) isp_bgobjdepth: u32, @@ -218,8 +219,14 @@ pub(crate) mod raw { pub(crate) merge_upper_y: F32, pub(crate) unk_68: U64, pub(crate) tile_count: U64, + + #[ver(G < G14X)] pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] pub(crate) job_params2: JobParameters2, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + pub(crate) job_params3: JobParameters3::ver, pub(crate) unk_758_flag: u32, pub(crate) unk_75c_flag: u32, @@ -232,6 +239,8 @@ pub(crate) mod raw { pub(crate) no_clear_pipeline_textures: u32, pub(crate) msaa_zs: u32, pub(crate) unk_pointee: u32, + #[ver(V >= V13_3)] + pub(crate) unk_v13_3: u32, pub(crate) meta: job::raw::JobMeta, pub(crate) unk_after_meta: u32, pub(crate) unk_buf_0: U64, diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index 2384b5c1119665..55f8230cd1b118 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -52,6 +52,35 @@ pub(crate) mod raw { pub(crate) frag: JobTimestamps, } default_zeroed!(RenderTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Register { + pub(crate) number: u32, + pub(crate) value: U64, + } + + impl Register { + fn new(number: u32, value: u64) -> Register { + Register { + number, + value: U64(value), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RegisterArray { + pub(crate) registers: Array<128, Register>, + pub(crate) pad: Array<0x100, u8>, + + pub(crate) addr: GpuWeakPointer<[Register]>, + pub(crate) count: u16, + pub(crate) length: u16, + pub(crate) unk_pad: u32, + } + default_zeroed!(RegisterArray); } trivial_gpustruct!(JobTimestamps); diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index 7daedd50bb09d5..eaa5aeb34c2866 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -3,7 +3,7 @@ //! GPU firmware microsequence operations use super::types::*; -use super::{buffer, compute, fragment, initdata, vertex, workqueue}; +use super::{buffer, compute, fragment, initdata, job, vertex, workqueue}; use crate::default_zeroed; pub(crate) trait Operation {} @@ -151,8 +151,10 @@ impl<'a> Operation for Timestamp::ver<'a> {} #[repr(C)] pub(crate) struct StartVertex<'a> { pub(crate) header: op::StartVertex, - pub(crate) tiling_params: GpuWeakPointer, - pub(crate) job_params1: GpuWeakPointer>, + pub(crate) tiling_params: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, pub(crate) buffer: GpuWeakPointer, pub(crate) scene: GpuWeakPointer, pub(crate) stats: GpuWeakPointer, @@ -226,8 +228,10 @@ impl Operation for FinalizeVertex::ver {} #[repr(C)] pub(crate) struct StartFragment<'a> { pub(crate) header: op::StartFragment, - pub(crate) job_params2: GpuWeakPointer, - pub(crate) job_params1: GpuWeakPointer>, + pub(crate) job_params2: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, pub(crate) stats: GpuWeakPointer, pub(crate) busy_flag: GpuWeakPointer, @@ -244,6 +248,8 @@ pub(crate) struct StartFragment<'a> { pub(crate) unk_68: u32, pub(crate) unk_758_flag: GpuWeakPointer, pub(crate) unk_job_buf: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_7c_0: U64, pub(crate) unk_7c: u32, pub(crate) unk_80: u32, pub(crate) unk_84: u32, @@ -282,6 +288,8 @@ pub(crate) struct FinalizeFragment { pub(crate) vm_slot: u32, pub(crate) unk_60: u32, pub(crate) unk_758_flag: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_6c_0: U64, pub(crate) unk_6c: U64, pub(crate) unk_74: U64, pub(crate) unk_7c: U64, diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs index 959a0913e693fa..898c1e24c9f8d8 100644 --- a/drivers/gpu/drm/asahi/fw/vertex.rs +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -25,6 +25,7 @@ pub(crate) mod raw { pub(crate) tpc_stride: u32, pub(crate) unk_24: u32, pub(crate) unk_28: u32, + pub(crate) __pad: Pad<0x74>, } #[versions(AGX)] @@ -75,7 +76,9 @@ pub(crate) mod raw { pub(crate) unk_100: Array<3, U64>, pub(crate) unk_118: u32, #[ver(G >= G14)] - pub(crate) __pad: Pad<{ 8 * 9 }>, + pub(crate) __pad: Pad<{ 8 * 9 + 0x268 }>, + #[ver(G < G14)] + pub(crate) __pad: Pad<0x268>, } #[derive(Debug)] @@ -114,10 +117,14 @@ pub(crate) mod raw { pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, pub(crate) unk_34: u32, + + #[ver(G < G14X)] pub(crate) job_params1: JobParameters1::ver<'a>, - pub(crate) unk_154: Array<0x268, u8>, + #[ver(G < G14X)] pub(crate) tiling_params: TilingParameters, - pub(crate) unk_3e8: Array<0x74, u8>, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, pub(crate) tpc_size: U64, pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 5869fd49fe91c6..4e10caf435a124 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -184,6 +184,7 @@ impl super::Queue::ver { } else { 0x8000 }, + __pad: Default::default(), }, }) } @@ -466,8 +467,16 @@ impl super::Queue::ver { let start_frag = builder.add(microseq::StartFragment::ver { header: microseq::op::StartFragment::HEADER, - job_params2: inner_weak_ptr!(ptr, job_params2), - job_params1: inner_weak_ptr!(ptr, job_params1), + #[ver(G < G14X)] + job_params2: Some(inner_weak_ptr!(ptr, job_params2)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + job_params2: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), scene: scene.gpu_pointer(), stats, busy_flag: inner_weak_ptr!(ptr, busy_flag), @@ -484,6 +493,8 @@ impl super::Queue::ver { unk_68: 0, unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + #[ver(V >= V13_3)] + unk_7c_0: U64(0), unk_7c: 0, unk_80: 0, unk_84: unk1.into(), @@ -552,6 +563,8 @@ impl super::Queue::ver { vm_slot: vm_bind.slot(), unk_60: 0, unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + #[ver(V >= V13_3)] + unk_6c_0: U64(0), unk_6c: U64(0), unk_74: U64(0), unk_7c: U64(0), @@ -613,6 +626,7 @@ impl super::Queue::ver { merge_upper_y: F32::from_bits(cmdbuf.merge_upper_y), unk_68: U64(0), tile_count: U64(tile_info.tiles as u64), + #[ver(G < G14X)] job_params1: fw::fragment::raw::JobParameters1::ver { utile_config: utile_config, unk_4: 0, @@ -659,10 +673,11 @@ impl super::Queue::ver { unk_150: U64(0x0), unk_158: U64(0x1c), unk_160: U64(0), - unk_168_padding: Default::default(), + __pad: Default::default(), #[ver(V < V13_0B4)] - __pad0: Default::default(), + __pad1: Default::default(), }, + #[ver(G < G14X)] job_params2: fw::fragment::raw::JobParameters2 { store_pipeline_bind: cmdbuf.store_pipeline_bind, store_pipeline_addr: cmdbuf.store_pipeline, @@ -682,9 +697,11 @@ impl super::Queue::ver { unk_38: 0x0, unk_3c: 0x1, unk_40: 0, + __pad: Default::default(), }, + #[ver(G >= G14X)] + registers: Default::default(), job_params3: fw::fragment::raw::JobParameters3::ver { - unk_44_padding: Default::default(), depth_bias_array: fw::fragment::raw::ArrayAddr { ptr: U64(cmdbuf.depth_bias_array), unk_padding: U64(0), @@ -726,7 +743,8 @@ impl super::Queue::ver { tib_blocks: cmdbuf.tib_blocks, unk_30c: 0x0, aux_fb_info: aux_fb_info, - unk_320_padding: Default::default(), + tile_config: U64(tile_config), + unk_328_padding: Default::default(), unk_partial_store_pipeline: fw::fragment::raw::StorePipelineBinding::new( cmdbuf.partial_store_pipeline_bind, @@ -772,6 +790,8 @@ impl super::Queue::ver { != 0) as u32, msaa_zs: (cmdbuf.flags & uapi::ASAHI_RENDER_MSAA_ZS as u64 != 0) as u32, unk_pointee: 0, + #[ver(V >= V13_3)] + unk_v13_3: 0, meta: fw::job::raw::JobMeta { unk_0: 0, unk_2: 0, @@ -872,8 +892,16 @@ impl super::Queue::ver { let start_vtx = builder.add(microseq::StartVertex::ver { header: microseq::op::StartVertex::HEADER, - tiling_params: inner_weak_ptr!(ptr, tiling_params), - job_params1: inner_weak_ptr!(ptr, job_params1), + #[ver(G < G14X)] + tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + tiling_params: None, + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), buffer: scene.weak_buffer_pointer(), scene: scene.weak_pointer(), stats, @@ -992,6 +1020,7 @@ impl super::Queue::ver { scene: inner.scene.gpu_pointer(), unk_buffer_buf: inner.scene.kernel_buffer_pointer(), unk_34: 0, + #[ver(G < G14X)] job_params1: fw::vertex::raw::JobParameters1::ver { unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 unk_8: f32!(1e-20), // fixed @@ -1051,12 +1080,12 @@ impl super::Queue::ver { unk_f8: U64(0x8c60), // fixed unk_100: Default::default(), // fixed unk_118: 0x1c, // fixed - #[ver(G >= G14)] __pad: Default::default(), }, - unk_154: Default::default(), + #[ver(G < G14X)] tiling_params: tile_info.params, - unk_3e8: Default::default(), + #[ver(G >= G14X)] + registers: Default::default(), tpc: inner.scene.tpc_pointer(), tpc_size: U64(tile_info.tpc_size as u64), microsequence: inner.micro_seq.gpu_pointer(), From a770a35da4376b8385dd883d52d1600ad06c8f0c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 01:07:02 +0900 Subject: [PATCH 549/635] drm/asahi: fw.microseq: Implement and use WaitForIdle2 command Needed for G14X. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/microseq.rs | 10 ++++++++++ drivers/gpu/drm/asahi/queue/compute.rs | 5 +++++ drivers/gpu/drm/asahi/queue/render.rs | 10 ++++++++++ 3 files changed, 25 insertions(+) diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs index eaa5aeb34c2866..d07a172907d5b7 100644 --- a/drivers/gpu/drm/asahi/fw/microseq.rs +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -12,6 +12,7 @@ pub(crate) trait Operation {} #[repr(u32)] enum OpCode { WaitForIdle = 0x01, + WaitForIdle2 = 0x02, RetireStamp = 0x18, #[allow(dead_code)] Timestamp = 0x19, @@ -83,6 +84,7 @@ pub(crate) mod op { simple_op!(FinalizeFragment); simple_op!(StartCompute); simple_op!(FinalizeCompute); + simple_op!(WaitForIdle2); #[derive(Debug, Copy, Clone)] pub(crate) struct RetireStamp(OpHeader); @@ -117,6 +119,14 @@ pub(crate) struct WaitForIdle { impl Operation for WaitForIdle {} +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle2 { + pub(crate) header: op::WaitForIdle2, +} + +impl Operation for WaitForIdle2 {} + #[derive(Debug)] #[repr(C)] pub(crate) struct RetireStamp { diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 9bec92783454a6..1a8f7e1186b1a5 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -177,9 +177,14 @@ impl super::Queue::ver { })?; } + #[ver(G < G14X)] builder.add(microseq::WaitForIdle { header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; if result_writer.is_some() { builder.add(microseq::Timestamp::ver { diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 4e10caf435a124..162f20443e1145 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -525,9 +525,14 @@ impl super::Queue::ver { })?; } + #[ver(G < G14X)] builder.add(microseq::WaitForIdle { header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; if frag_result.is_some() { builder.add(microseq::Timestamp::ver { @@ -942,9 +947,14 @@ impl super::Queue::ver { })?; } + #[ver(G < G14X)] builder.add(microseq::WaitForIdle { header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; if vtx_result.is_some() { builder.add(microseq::Timestamp::ver { From 1b915a430f1304439c6306d281e8aaf461779732 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 01:20:42 +0900 Subject: [PATCH 550/635] drm/asahi: object: Add a gpu_ptr argument new_prealloc raw fn This allows self-referential pointers. We already had this in the inner cb for back-references, so this allows direct self-references. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 1 + drivers/gpu/drm/asahi/object.rs | 3 ++- drivers/gpu/drm/asahi/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/queue/render.rs | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index d918b19e97214c..659a86b02688d2 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -257,6 +257,7 @@ pub(crate) trait Allocator { raw_cb: impl for<'a> FnOnce( &'a T, &'a mut MaybeUninit>, + GpuWeakPointer, ) -> Result<&'a mut T::Raw<'a>>, ) -> Result>> { GpuObject::>::new_prealloc( diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 449899b881813a..2fdc000e297a81 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -334,6 +334,7 @@ impl> GpuObject { raw_cb: impl for<'a> FnOnce( &'a T, &'a mut MaybeUninit>, + GpuWeakPointer, ) -> Result<&'a mut T::Raw<'a>>, ) -> Result { if alloc.size() < mem::size_of::>() { @@ -350,7 +351,7 @@ impl> GpuObject { let inner = inner_cb(gpu_ptr)?; let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - let raw = raw_cb(&*inner, unsafe { &mut *p })?; + let raw = raw_cb(&*inner, unsafe { &mut *p }, gpu_ptr)?; if p as *mut T::Raw<'_> != raw as *mut _ { dev_err!( alloc.device(), diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 1a8f7e1186b1a5..4f7ae9785a6b9b 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -248,7 +248,7 @@ impl super::Queue::ver { timestamps: timestamps.clone(), })?) }, - |inner, ptr| { + |inner, ptr, _gpu_ptr| { Ok(place!( ptr, fw::compute::raw::RunCompute::ver { diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 162f20443e1145..b3b0791f18004f 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -596,7 +596,7 @@ impl super::Queue::ver { timestamps: timestamps.clone(), })?) }, - |inner, ptr| { + |inner, ptr, _gpu_ptr| { let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { iogpu_unk_214: cmdbuf.iogpu_unk_214, unk2: 0, @@ -1012,7 +1012,7 @@ impl super::Queue::ver { timestamps: timestamps, })?) }, - |inner, ptr| { + |inner, ptr, _gpu_ptr| { #[ver(G < G14)] let core_masks = gpu.core_masks_packed(); Ok(place!( From 0bf8efb1eeb9d1a25dc7995d2f5221f5bfac864b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 01:35:09 +0900 Subject: [PATCH 551/635] drm/asahi: fw.job: Add a helper constructor to make register lists Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/job.rs | 30 +++++++++++++++++++++++++-- drivers/gpu/drm/asahi/queue/render.rs | 10 +++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index 55f8230cd1b118..f7b679e61c574b 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -59,6 +59,7 @@ pub(crate) mod raw { pub(crate) number: u32, pub(crate) value: U64, } + default_zeroed!(Register); impl Register { fn new(number: u32, value: u64) -> Register { @@ -75,12 +76,37 @@ pub(crate) mod raw { pub(crate) registers: Array<128, Register>, pub(crate) pad: Array<0x100, u8>, - pub(crate) addr: GpuWeakPointer<[Register]>, + pub(crate) addr: GpuWeakPointer>, pub(crate) count: u16, pub(crate) length: u16, pub(crate) unk_pad: u32, } - default_zeroed!(RegisterArray); + + impl RegisterArray { + pub(crate) fn new( + self_ptr: GpuWeakPointer>, + cb: impl FnOnce(&mut RegisterArray), + ) -> RegisterArray { + let mut array = RegisterArray { + registers: Default::default(), + pad: Default::default(), + addr: self_ptr, + count: 0, + length: 0, + unk_pad: 0, + }; + + cb(&mut array); + + array + } + + pub(crate) fn add(&mut self, number: u32, value: u64) { + self.registers[self.count as usize] = Register::new(number, value); + self.count += 1; + self.length += core::mem::size_of::() as u16; + } + } } trivial_gpustruct!(JobTimestamps); diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index b3b0791f18004f..ce0ecdf92a234f 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -705,7 +705,10 @@ impl super::Queue::ver { __pad: Default::default(), }, #[ver(G >= G14X)] - registers: Default::default(), + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_gpu_ptr, registers.registers), + |r| {} + ), job_params3: fw::fragment::raw::JobParameters3::ver { depth_bias_array: fw::fragment::raw::ArrayAddr { ptr: U64(cmdbuf.depth_bias_array), @@ -1095,7 +1098,10 @@ impl super::Queue::ver { #[ver(G < G14X)] tiling_params: tile_info.params, #[ver(G >= G14X)] - registers: Default::default(), + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_gpu_ptr, registers.registers), + |r| {} + ), tpc: inner.scene.tpc_pointer(), tpc_size: U64(tile_info.tpc_size as u64), microsequence: inner.micro_seq.gpu_pointer(), From 642330374ba1c7c0a1a895dc713f09c3c235fe8c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 02:01:08 +0900 Subject: [PATCH 552/635] drm/asahi: object: Implement From for u64 Useful when converting GPU pointers to raw register values. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/object.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 2fdc000e297a81..b38c2f9d80e756 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -99,6 +99,12 @@ impl<'a, T: ?Sized> Debug for GpuPointer<'a, T> { } } +impl<'a, T: ?Sized> From> for u64 { + fn from(value: GpuPointer<'a, T>) -> Self { + value.0.get() + } +} + /// Take a pointer to a sub-field within a structure pointed to by a GpuPointer, keeping the /// lifetime. #[macro_export] @@ -162,6 +168,12 @@ impl GpuWeakPointer { } } +impl From> for u64 { + fn from(value: GpuWeakPointer) -> Self { + value.0.get() + } +} + impl Debug for GpuWeakPointer { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { let val = self.0; From 43513e48819a538b9df8b2b574e85b78cc0a0335 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 02:02:34 +0900 Subject: [PATCH 553/635] drm/asahi: render: Add fragment register list for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/render.rs | 89 ++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index ce0ecdf92a234f..3aa8c5a52b0b79 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -140,6 +140,7 @@ impl super::Queue::ver { 0 }; + // TODO: large_tib needs larger min_tvb_blocks apparently let min_tvb_blocks = div_ceil(tiles_x * tiles_y, 128).max(if num_clusters > 1 { 9 } else { 8 }) as usize; @@ -707,7 +708,93 @@ impl super::Queue::ver { #[ver(G >= G14X)] registers: fw::job::raw::RegisterArray::new( inner_weak_ptr!(_gpu_ptr, registers.registers), - |r| {} + |r| { + r.add(0x1739, 1); + r.add(0x10009, utile_config.into()); + r.add(0x15379, cmdbuf.store_pipeline_bind.into()); + r.add(0x15381, cmdbuf.store_pipeline.into()); + r.add(0x15369, cmdbuf.load_pipeline_bind.into()); + r.add(0x15371, cmdbuf.load_pipeline.into()); + r.add(0x15131, cmdbuf.merge_upper_x.into()); + r.add(0x15139, cmdbuf.merge_upper_y.into()); + r.add(0x100a1, 0); + r.add(0x15069, 0); + r.add(0x15071, 0); // pointer + r.add(0x16058, 0); + r.add(0x10019, cmdbuf.ppp_multisamplectl); + + let isp_mtile_size = (tile_info.utiles_per_mtile_y + | (tile_info.utiles_per_mtile_x << 16)) + .into(); + r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE + r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE + r.add( + 0x100d9, + (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)) + .into(), + ); // TE_SCREEN + r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x15109, cmdbuf.scissor_array); // ISP_SCISSOR_BASE + r.add(0x15101, cmdbuf.depth_bias_array); // ISP_DBIAS_BASE + r.add(0x15021, cmdbuf.iogpu_unk_214.into()); // aux_fb_info.unk_1 + r.add( + 0x15211, + ((cmdbuf.fb_height as u64) << 32) | cmdbuf.fb_width as u64, + ); // aux_fb_info.{width, heigh + r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 + r.add(0x10051, cmdbuf.tib_blocks.into()); // s1.unk_2c + r.add(0x15321, cmdbuf.depth_dimensions.into()); // ISP_ZLS_PIXELS + r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH + r.add(0x15309, cmdbuf.isp_bgobjvals.into()); // ISP_BGOBJVALS + r.add(0x15311, cmdbuf.visibility_result_buffer); // ISP_OCLQRY_BASE + r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL + r.add(0x15349, 0x4040404); // s2.unk_58_g14_0 + r.add(0x15351, 0); // s2.unk_58_g14_8 + r.add(0x15329, cmdbuf.depth_buffer_1); // ISP_ZLOAD_BASE + r.add(0x15331, cmdbuf.depth_buffer_2); // ISP_ZSTORE_BASE + r.add(0x15339, cmdbuf.stencil_buffer_1); // ISP_STENCIL_LOAD_BASE + r.add(0x15341, cmdbuf.stencil_buffer_2); // ISP_STENCIL_STORE_BASE + r.add(0x15231, 0); + r.add(0x15221, 0); + r.add(0x15239, 0); + r.add(0x15229, 0); + r.add(0x15401, 0); + r.add(0x15421, 0); + r.add(0x15409, 0); + r.add(0x15429, 0); + r.add(0x153c1, cmdbuf.depth_meta_buffer_1); + r.add(0x15411, 0); + r.add(0x153c9, cmdbuf.depth_meta_buffer_2); + r.add(0x15431, 0); + r.add(0x153d1, cmdbuf.stencil_meta_buffer_1); + r.add(0x15419, 0); + r.add(0x153d9, cmdbuf.stencil_meta_buffer_2); + r.add(0x15439, 0); + r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x16060, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x10039, tile_config); // tile_config ISP_CTL? + r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN + r.add(0x11821, 0x0); // some shader? + r.add(0x11829, 0); + r.add(0x11f79, 0); + r.add(0x15359, 0); + r.add(0x10069, 0x11_00000000); // USC_EXEC_BASE_ISP + r.add(0x16020, 0); + r.add(0x16461, inner.aux_fb.gpu_pointer().into()); + r.add(0x16090, inner.aux_fb.gpu_pointer().into()); + r.add(0x120a1, 0x1c); // s2.unk_158 + r.add(0x160a8, 0); + r.add( + 0x16068, + 0x0000000_00036011 + | (((tile_info.tiles_x - 1) as u64) << 44) + | (((tile_info.tiles_y - 1) as u64) << 53) + | (if unk1 { 0 } else { 0x20_00000000 }) + | ((utile_config as u64 & 0xf000) << 28), + ); + r.add(0x160b8, 0x0); + } ), job_params3: fw::fragment::raw::JobParameters3::ver { depth_bias_array: fw::fragment::raw::ArrayAddr { From ad6d5a3966c90c89823e5837c99faccd798fd4fb Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Thu, 20 Apr 2023 02:24:09 +0900 Subject: [PATCH 554/635] drm/asahi: render: Add vertex register list for G14X Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/render.rs | 95 ++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 3aa8c5a52b0b79..8321dd5e40ec0f 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -276,7 +276,7 @@ impl super::Queue::ver { clustering = false; } - #[ver(G < G14)] + #[ver(G != G14)] let tiling_control = { let render_cfg = gpu.get_cfg().render; let mut tiling_control = render_cfg.tiling_control; @@ -1187,7 +1187,98 @@ impl super::Queue::ver { #[ver(G >= G14X)] registers: fw::job::raw::RegisterArray::new( inner_weak_ptr!(_gpu_ptr, registers.registers), - |r| {} + |r| { + r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 + r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); + + let cl_tilemaps_ptr = inner + .scene + .cluster_tilemaps_pointer() + .map_or(0, |a| a.into()); + r.add(0x1c041, cl_tilemaps_ptr); + r.add(0x1c9d0, cl_tilemaps_ptr); + r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR + + let tvb_heapmeta_ptr = inner + .scene + .tvb_heapmeta_pointer() + .or(0x8000_0000_0000_0000) + .into(); + r.add(0x1c031, tvb_heapmeta_ptr); + r.add(0x1c9c0, tvb_heapmeta_ptr); + r.add(0x1c051, 0x3a0012006b0003); // iogpu_unk_54/55 + r.add(0x1c061, 1); // iogpu_unk_56 + r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config + r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL + r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c958, 1); // s2.unk_80 + r.add( + 0x1c950, + inner + .scene + .preempt_buf_3_pointer() + .or(0x4_0000_0000_0000) + .into(), + ); + r.add(0x1c930, 0); // VCE related addr, lsb to enable + r.add(0x1c880, cmdbuf.encoder_ptr); // VDM_CTRL_STREAM_BASE + r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. + r.add( + 0x1c948, + inner.scene.meta_2_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta2 + r.add( + 0x1c888, + inner.scene.meta_3_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta3 + r.add(0x1c890, tiling_control.into()); // tvb_tiling_control + r.add(0x1c918, 0x4); + r.add(0x1c079, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x1c9d8, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x1c089, 0); + r.add(0x1c9e0, 0); + let cl_meta_4_pointer = + inner.scene.meta_4_pointer().map_or(0, |a| a.into()); + r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1c9a8, 0x1c); // + meta1_blocks? min_free_tvb_pages? + r.add( + 0x1c920, + inner.scene.meta_1_pointer().map_or(0, |a| a.into()), + ); // ??? | meta1_blocks? + r.add(0x10151, 0); + r.add(0x1c199, 0); + r.add(0x1c1a1, 0); + r.add(0x1c1a9, 0); // 0x10151 bit 1 enables + r.add(0x1c1b1, 0); + r.add(0x1c1b9, 0); + r.add(0x10061, 0x11_00000000); // USC_EXEC_BASE_TA + r.add(0x11801, 0); // some shader? + r.add(0x11809, 0); // maybe arg? + r.add(0x11f71, 0); + r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG + r.add(0x1c850, tile_info.params.rgn_size.into()); + r.add(0x10131, tile_info.params.unk_4.into()); + r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL + r.add( + 0x10129, + tile_info.params.x_max as u64 + | ((tile_info.params.y_max as u64) << 16), + ); // PPP_SCREEN + r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN + r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 + r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 + r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE + r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC + r.add(0x10171, tile_info.params.unk_24.into()); + r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX + r.add(0x12099, 0x1c); + r.add(0x1c9e8, 0); + } ), tpc: inner.scene.tpc_pointer(), tpc_size: U64(tile_info.tpc_size as u64), From c9e7b68df288b4e65c930a8e26f6df523b2858b8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 17:11:49 +0900 Subject: [PATCH 555/635] drm/asahi: fw.channels: Fix DestroyContext layout for V13_3 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/channels.rs | 10 ++++++++-- drivers/gpu/drm/asahi/gpu.rs | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs index 0c52c206dd26ff..dae3095df0a72e 100644 --- a/drivers/gpu/drm/asahi/fw/channels.rs +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -187,16 +187,22 @@ pub(crate) enum DeviceControlMsg { DestroyContext { unk_4: u32, ctx_23: u8, + #[ver(V < V13_3)] __pad0: Pad<3>, - unk_c: u32, - unk_10: u32, + unk_c: U32, + unk_10: U32, ctx_0: u8, ctx_1: u8, ctx_4: u8, + #[ver(V < V13_3)] __pad1: Pad<1>, + #[ver(V < V13_3)] unk_18: u32, gpu_context: Option>, + #[ver(V < V13_3)] __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x20 }>, + #[ver(V >= V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x18 }>, }, Unk18(Array), Initialize(Pad), diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index c4e0181be01209..60b98b81fd621f 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -31,7 +31,7 @@ use crate::box_in_place; use crate::debug::*; use crate::driver::AsahiDevice; use crate::fw::channels::PipeType; -use crate::fw::types::U64; +use crate::fw::types::{U32, U64}; use crate::{ alloc, buffer, channel, event, fw, gem, hw, initdata, mem, mmu, queue, regs, workqueue, }; @@ -889,13 +889,16 @@ impl GpuManager::ver { |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { unk_4: 0, ctx_23: raw.unk_23, + #[ver(V < V13_3)] __pad0: Default::default(), - unk_c: 0, - unk_10: 0, + unk_c: U32(0), + unk_10: U32(0), ctx_0: raw.unk_0, ctx_1: raw.unk_1, ctx_4: raw.unk_4, + #[ver(V < V13_3)] __pad1: Default::default(), + #[ver(V < V13_3)] unk_18: 0, gpu_context: Some(context.weak_pointer()), __pad2: Default::default(), @@ -1084,14 +1087,18 @@ impl GpuManager for GpuManager::ver { let dc = fw::channels::DeviceControlMsg::ver::DestroyContext { unk_4: 0, + ctx_23: 0, + #[ver(V < V13_3)] __pad0: Default::default(), - unk_c: 0, - unk_10: 0, + unk_c: U32(0), + unk_10: U32(0), ctx_0: 0xff, ctx_1: 0xff, ctx_4: 0, + #[ver(V < V13_3)] __pad1: Default::default(), + #[ver(V < V13_3)] unk_18: 0, gpu_context: None, __pad2: Default::default(), From c3252f472bd3fb503b87784621cf68e8525e2c54 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 17:14:52 +0900 Subject: [PATCH 556/635] drm/asahi: gpu,mmu: Fix MMIO maps overlapping allocators Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 4 ++-- drivers/gpu/drm/asahi/mmu.rs | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 60b98b81fd621f..7172c5ff6b93b9 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -81,9 +81,9 @@ const IOVA_KERN_SHARED_RO_BASE: u64 = 0xffffffaa00000000; /// Shared (uncached) read-only firmware structure VA range top. const IOVA_KERN_SHARED_RO_TOP: u64 = 0xffffffabffffffff; /// GPU/FW shared structure VA range base. -const IOVA_KERN_GPU_BASE: u64 = 0xffffffaf00000000; +const IOVA_KERN_GPU_BASE: u64 = 0xffffffac00000000; /// GPU/FW shared structure VA range top. -const IOVA_KERN_GPU_TOP: u64 = 0xffffffafffffffff; +const IOVA_KERN_GPU_TOP: u64 = 0xffffffadffffffff; /// GPU/FW low shared structure VA range base. const IOVA_KERN_GPU_LOW_BASE: u64 = 0x20_0000_0000; diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 59bf050b5ec19d..954b4438f0d233 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -73,6 +73,10 @@ const IOVA_KERN_BASE: usize = 0xffffffa000000000; /// Driver-managed kernel top VA const IOVA_KERN_TOP: usize = 0xffffffafffffffff; +/// Range reserved for MMIO maps +const IOVA_KERN_MMIO_BASE: usize = 0xffffffaf00000000; +const IOVA_KERN_MMIO_TOP: usize = 0xffffffafffffffff; + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; @@ -907,7 +911,7 @@ impl Vm { let mut inner = self.inner.lock(); let uat_inner = inner.uat_inner.clone(); - let node = inner.mm.insert_node( + let node = inner.mm.insert_node_in_range( MappingInner { owner: self.inner.clone(), uat_inner, @@ -916,6 +920,11 @@ impl Vm { mapped_size: size, }, (size + UAT_PGSZ) as u64, // Add guard page + UAT_PGSZ as u64, + 0, + IOVA_KERN_MMIO_BASE as u64, + IOVA_KERN_MMIO_TOP as u64, + mm::InsertMode::Best, )?; let iova = node.start() as usize; From 1a80362cf70476360c91384e6c43f52d47001970 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 17:15:29 +0900 Subject: [PATCH 557/635] drm/asahi: hw: Put clustering sizes into the HwConfig struct Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/hw/mod.rs | 12 ++++++++++++ drivers/gpu/drm/asahi/hw/t600x.rs | 7 +++++++ drivers/gpu/drm/asahi/hw/t602x.rs | 7 +++++++ drivers/gpu/drm/asahi/hw/t8103.rs | 1 + drivers/gpu/drm/asahi/hw/t8112.rs | 1 + 5 files changed, 28 insertions(+) diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index b8c52be7104509..d641a103a09166 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -195,6 +195,16 @@ pub(crate) struct HwConfigShared2Curves { pub(crate) t3_scales: &'static [u32], } +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + /// Static hardware configuration for a given SoC model. #[derive(Debug)] pub(crate) struct HwConfig { @@ -233,6 +243,8 @@ pub(crate) struct HwConfig { /// Required size of the third preemption buffer. pub(crate) preempt3_size: usize, + pub(crate) clustering: Option, + /// Rendering-relevant configuration. pub(crate) render: HwRenderConfig, diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs index af809547a643d4..842f4cd1820a8b 100644 --- a/drivers/gpu/drm/asahi/hw/t600x.rs +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -60,6 +60,13 @@ pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { preempt1_size: 0x540, preempt2_size: 0x280, preempt3_size: 0x20, + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0x190 * 8, // CHECK + meta3_size: 0x280 * 8, + meta4_size: 0x30 * 16, + max_splits: 16, + }), render: HwRenderConfig { tiling_control: 0xa540, diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs index a953c1b14168db..ef22a225e8f8fa 100644 --- a/drivers/gpu/drm/asahi/hw/t602x.rs +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -60,6 +60,13 @@ pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { preempt1_size: 0x540, preempt2_size: 0x280, preempt3_size: 0x40, + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 8, + meta3_size: 0x280 * 8, + meta4_size: 0x10 * 64, + max_splits: 64, + }), render: HwRenderConfig { tiling_control: 0x180340, diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs index 8cfdc54cfe03d9..4647f0afd51794 100644 --- a/drivers/gpu/drm/asahi/hw/t8103.rs +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -25,6 +25,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { preempt1_size: 0x540, preempt2_size: 0x280, preempt3_size: 0x20, + clustering: None, render: HwRenderConfig { // bit 0: disable clustering (always) diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs index b4d58019e3bf78..c7ee5b8583e7ec 100644 --- a/drivers/gpu/drm/asahi/hw/t8112.rs +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -25,6 +25,7 @@ pub(crate) const HWCONFIG: super::HwConfig = HwConfig { preempt1_size: 0x540, preempt2_size: 0x280, preempt3_size: 0x20, + clustering: None, render: HwRenderConfig { // TODO: this is unused here, may be present in newer FW From d0c06deab16a1c7b54e0b8d5cdee4814fce55338 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 17:16:02 +0900 Subject: [PATCH 558/635] drm/asahi: buffer: Calculate cluster sizes using config data Also be more conservative about things, using the worst-case sizes instead of the real cluster count. It's unclear what goes down with lower-cluster-count machines anyway, and the buffers (except the tilemaps) aren't large enough to be worth optimizing more. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 35 +++++++++++++++------------ drivers/gpu/drm/asahi/queue/render.rs | 6 ----- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index 56c61c9c8604cb..b6715877eb7402 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -36,7 +36,7 @@ use crate::debug::*; use crate::fw::buffer; use crate::fw::types::*; use crate::util::*; -use crate::{alloc, fw, gpu, mmu, slotalloc}; +use crate::{alloc, fw, gpu, hw, mmu, slotalloc}; use crate::{box_in_place, place}; use core::sync::atomic::Ordering; use kernel::prelude::*; @@ -93,8 +93,6 @@ pub(crate) struct TileInfo { pub(crate) meta1_blocks: u32, /// Minimum number of TVB blocks for this render. pub(crate) min_tvb_blocks: usize, - /// XXX: Allocation factor for cluster tilemaps and meta4. Always 2? - pub(crate) cluster_factor: usize, /// Tiling parameter structure passed to firmware. pub(crate) params: fw::vertex::raw::TilingParameters, } @@ -292,6 +290,7 @@ struct BufferInner { tpc: Option>>, kernel_buffer: GpuArray, stats: GpuObject, + cfg: &'static hw::HwConfig, preempt1_size: usize, preempt2_size: usize, preempt3_size: usize, @@ -407,6 +406,7 @@ impl Buffer::ver { tpc: None, kernel_buffer, stats, + cfg: gpu.get_cfg(), preempt1_size, preempt2_size, preempt3_size, @@ -521,8 +521,8 @@ impl Buffer::ver { // This seems to be a list, with 4x2 bytes of headers and 8 bytes per entry. // On single-cluster devices, the used length always seems to be 1. - // On M1 Ultra, it can grow and usually doesn't exceed 8 * cluster_factor - // entries. macOS allocates a whole 64K * 0x80 for this, so let's go with + // On M1 Ultra, it can grow and usually doesn't exceed 64 entries. + // macOS allocates a whole 64K * 0x80 for this, so let's go with // that to be safe... let user_buffer = inner.ualloc.lock().array_empty(if inner.num_clusters > 1 { 0x10080 @@ -562,22 +562,27 @@ impl Buffer::ver { } }; - // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) - let meta1_size = align(tile_info.meta1_blocks as usize * 0x44, 0x80); - // check - let meta2_size = align(0x190 * inner.num_clusters, 0x80); - let meta3_size = align(0x280 * inner.num_clusters, 0x80); - // Like user_buffer for single-cluster modes, 0x30 per cluster * the cluster - // factor. - let meta4_size = align(0x30 * inner.num_clusters * tile_info.cluster_factor, 0x80); - let meta_size = meta1_size + meta2_size + meta3_size + meta4_size; + let mut meta1_size = 0; + let mut meta2_size = 0; + let mut meta3_size = 0; let clustering = if inner.num_clusters > 1 { + let cfg = inner.cfg.clustering.as_ref().unwrap(); + + // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) + // Unused on t602x? + meta1_size = align(tile_info.meta1_blocks as usize * cfg.meta1_blocksize, 0x80); + meta2_size = align(cfg.meta2_size, 0x80); + meta3_size = align(cfg.meta3_size, 0x80); + let meta4_size = cfg.meta4_size; + + let meta_size = meta1_size + meta2_size + meta3_size + meta4_size; + mod_pr_debug!("Buffer: Allocating clustering buffers\n"); let tilemaps = inner .ualloc .lock() - .array_empty(inner.num_clusters * tilemap_size * tile_info.cluster_factor)?; + .array_empty(cfg.max_splits * tilemap_size)?; let meta = inner.ualloc.lock().array_empty(meta_size)?; Some(buffer::ClusterBuffers { tilemaps, meta }) } else { diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 8321dd5e40ec0f..7c9dc5128b7dd4 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -144,11 +144,6 @@ impl super::Queue::ver { let min_tvb_blocks = div_ceil(tiles_x * tiles_y, 128).max(if num_clusters > 1 { 9 } else { 8 }) as usize; - // Sometimes clustering seems to use twice the cluster tilemap count - // and twice the meta4 size. TODO: Is this random or can we calculate - // it somehow??? Does it go higher??? - let cluster_factor = 2; - Ok(buffer::TileInfo { tiles_x, tiles_y, @@ -167,7 +162,6 @@ impl super::Queue::ver { tpc_size, meta1_blocks, min_tvb_blocks, - cluster_factor, params: fw::vertex::raw::TilingParameters { rgn_size, unk_4: 0x88, From 4d9aafa57d7027654a9c1e641397badf44416219 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 17:28:07 +0900 Subject: [PATCH 559/635] drm/asahi: render: Misc value updates for G14X/V13_3 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/render.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 7c9dc5128b7dd4..b105bec46739a5 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -815,6 +815,9 @@ impl super::Queue::ver { address: U64(cmdbuf.partial_reload_pipeline as u64), }, zls_ctrl: U64(cmdbuf.zls_ctrl), + #[ver(G >= G14X)] + unk_290: U64(0x4040404), + #[ver(G < G14X)] unk_290: U64(0x0), depth_buffer_ptr1: U64(cmdbuf.depth_buffer_1), unk_2a0: U64(0x0), @@ -897,7 +900,10 @@ impl super::Queue::ver { unk_after_meta: unk1.into(), unk_buf_0: U64(0), unk_buf_8: U64(0), + #[ver(G < G14X)] unk_buf_10: U64(1), + #[ver(G >= G14X)] + unk_buf_10: U64(0), cur_ts: U64(0), start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), From 980056740831b371dc854e625b5010271e05988d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 20:04:46 +0900 Subject: [PATCH 560/635] drm/asahi: render: Update for new UAPI Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/common.rs | 12 +++-- drivers/gpu/drm/asahi/queue/render.rs | 70 ++++++++++++++------------- 2 files changed, 45 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index 9dbc345d1dfef9..d46f0b0fe93da8 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -36,13 +36,19 @@ pub(super) fn build_attachments(pointer: u64, count: u32) -> Result 6 { + return Err(EINVAL); + } + let cache_lines = (att.size + 127) >> 7; - let order = 1; attachments.list[i as usize] = microseq::Attachment { address: U64(att.pointer), - size: cache_lines, + size: cache_lines.try_into()?, unk_c: 0x17, - unk_e: order, + unk_e: att.order as u16, }; attachments.count += 1; diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index b105bec46739a5..a908fc732efdce 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -222,8 +222,7 @@ impl super::Queue::ver { | uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED | uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES | uapi::ASAHI_RENDER_NO_VERTEX_CLUSTERING - | uapi::ASAHI_RENDER_MSAA_ZS - | uapi::ASAHI_RENDER_UNK_FLAG1) as u64 + | uapi::ASAHI_RENDER_MSAA_ZS) as u64 != 0 { return Err(EINVAL); @@ -394,7 +393,7 @@ impl super::Queue::ver { let timestamps = Arc::try_new(kalloc.shared.new_default::()?)?; - let unk1 = cmdbuf.flags & uapi::ASAHI_RENDER_UNK_FLAG1 as u64 != 0; + let unk1 = false; let mut tile_config: u64 = 0; if !unk1 { @@ -495,8 +494,8 @@ impl super::Queue::ver { unk_84: unk1.into(), uuid: uuid_3d, attachments: common::build_attachments( - cmdbuf.attachments, - cmdbuf.attachment_count, + cmdbuf.fragment_attachments, + cmdbuf.fragment_attachment_count, )?, padding: 0, #[ver(V >= V13_0B4)] @@ -573,7 +572,7 @@ impl super::Queue::ver { #[ver(G == G14 && V < V13_0B4)] unk_8c_g14: U64(0), restart_branch_offset: off, - has_attachments: (cmdbuf.attachment_count > 0) as u32, + has_attachments: (cmdbuf.fragment_attachment_count > 0) as u32, #[ver(V >= V13_0B4)] unk_9c: Default::default(), })?; @@ -645,20 +644,20 @@ impl super::Queue::ver { unk_58_g14_0: U64(0x4040404), #[ver(G >= G14)] unk_58_g14_8: U64(0), - depth_buffer_ptr1: U64(cmdbuf.depth_buffer_1), - depth_buffer_ptr2: U64(cmdbuf.depth_buffer_2), - stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_1), - stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_2), + depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), + depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), + stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), + stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), #[ver(G >= G14)] unk_68_g14_0: Default::default(), unk_78: Default::default(), - depth_meta_buffer_ptr1: U64(cmdbuf.depth_meta_buffer_1), + depth_meta_buffer_ptr1: U64(cmdbuf.depth_meta_buffer_load), unk_a0: Default::default(), - depth_meta_buffer_ptr2: U64(cmdbuf.depth_meta_buffer_2), + depth_meta_buffer_ptr2: U64(cmdbuf.depth_meta_buffer_store), unk_b0: Default::default(), - stencil_meta_buffer_ptr1: U64(cmdbuf.stencil_meta_buffer_1), + stencil_meta_buffer_ptr1: U64(cmdbuf.stencil_meta_buffer_load), unk_c0: Default::default(), - stencil_meta_buffer_ptr2: U64(cmdbuf.stencil_meta_buffer_2), + stencil_meta_buffer_ptr2: U64(cmdbuf.stencil_meta_buffer_store), unk_d0: Default::default(), tvb_tilemap: inner.scene.tvb_tilemap_pointer(), tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), @@ -744,10 +743,10 @@ impl super::Queue::ver { r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL r.add(0x15349, 0x4040404); // s2.unk_58_g14_0 r.add(0x15351, 0); // s2.unk_58_g14_8 - r.add(0x15329, cmdbuf.depth_buffer_1); // ISP_ZLOAD_BASE - r.add(0x15331, cmdbuf.depth_buffer_2); // ISP_ZSTORE_BASE - r.add(0x15339, cmdbuf.stencil_buffer_1); // ISP_STENCIL_LOAD_BASE - r.add(0x15341, cmdbuf.stencil_buffer_2); // ISP_STENCIL_STORE_BASE + r.add(0x15329, cmdbuf.depth_buffer_load); // ISP_ZLOAD_BASE + r.add(0x15331, cmdbuf.depth_buffer_store); // ISP_ZSTORE_BASE + r.add(0x15339, cmdbuf.stencil_buffer_load); // ISP_STENCIL_LOAD_BASE + r.add(0x15341, cmdbuf.stencil_buffer_store); // ISP_STENCIL_STORE_BASE r.add(0x15231, 0); r.add(0x15221, 0); r.add(0x15239, 0); @@ -756,13 +755,13 @@ impl super::Queue::ver { r.add(0x15421, 0); r.add(0x15409, 0); r.add(0x15429, 0); - r.add(0x153c1, cmdbuf.depth_meta_buffer_1); + r.add(0x153c1, cmdbuf.depth_meta_buffer_load); r.add(0x15411, 0); - r.add(0x153c9, cmdbuf.depth_meta_buffer_2); + r.add(0x153c9, cmdbuf.depth_meta_buffer_store); r.add(0x15431, 0); - r.add(0x153d1, cmdbuf.stencil_meta_buffer_1); + r.add(0x153d1, cmdbuf.stencil_meta_buffer_load); r.add(0x15419, 0); - r.add(0x153d9, cmdbuf.stencil_meta_buffer_2); + r.add(0x153d9, cmdbuf.stencil_meta_buffer_store); r.add(0x15439, 0); r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); r.add(0x16060, inner.scene.tvb_heapmeta_pointer().into()); @@ -819,18 +818,18 @@ impl super::Queue::ver { unk_290: U64(0x4040404), #[ver(G < G14X)] unk_290: U64(0x0), - depth_buffer_ptr1: U64(cmdbuf.depth_buffer_1), + depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), unk_2a0: U64(0x0), unk_2a8: U64(0x0), - depth_buffer_ptr2: U64(cmdbuf.depth_buffer_2), - depth_buffer_ptr3: U64(cmdbuf.depth_buffer_3), - depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_3), - stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_1), + depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), + depth_buffer_ptr3: U64(cmdbuf.depth_buffer_reload), + depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_reload), + stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), unk_2d0: U64(0x0), unk_2d8: U64(0x0), - stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_2), - stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_3), - stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_3), + stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), + stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_reload), + stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_reload), unk_2f8: Default::default(), tib_blocks: cmdbuf.tib_blocks, unk_30c: 0x0, @@ -866,8 +865,8 @@ impl super::Queue::ver { unk_8: (cmdbuf.flags & uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 != 0) as u32, - large_tib: large_tib as u32, // fixed - unk_10: 0x0, // fixed + large_tib: large_tib as u32, + unk_10: 0x0, // fixed encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed iogpu_compute_unk44: 0xffffffff, @@ -1013,7 +1012,10 @@ impl super::Queue::ver { unk_64: 0x0, // fixed unk_68: unk1.into(), uuid: uuid_ta, - attachments: Default::default(), // TODO: Vertex attachments + attachments: common::build_attachments( + cmdbuf.vertex_attachments, + cmdbuf.vertex_attachment_count, + )?, padding: 0, #[ver(V >= V13_0B4)] counter: U64(count_vtx), @@ -1085,7 +1087,7 @@ impl super::Queue::ver { #[ver(G >= G14 && V < V13_0B4)] unk_68_g14: U64(0), restart_branch_offset: off, - has_attachments: 0, // TODO: Vertex attachments + has_attachments: (cmdbuf.vertex_attachment_count > 0) as u32, #[ver(V >= V13_0B4)] unk_74: Default::default(), // Ventura })?; From 42d199691b30bca60628fbdbcaf6662fd34fd3b8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 20:05:17 +0900 Subject: [PATCH 561/635] DO NOT MERGE: drm/asahi: Add an experimental UAPI extension This lets us play around with unknown stuff from userspace, without having to commit to adding it to the UAPI (or knowing whether it's safe to expose at all). Signed-off-by: Asahi Lina --- include/uapi/drm/asahi_drm.h | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 01f57a20d97401..d805dffa2138e2 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -403,7 +403,75 @@ struct drm_asahi_cmd_render { __u32 depth_dimensions; __u32 isp_bgobjdepth; __u32 isp_bgobjvals; +}; + +#define ASAHI_RENDER_UNK_UNK1 (1UL << 0) +#define ASAHI_RENDER_UNK_SET_TILE_CONFIG (1UL << 1) +#define ASAHI_RENDER_UNK_SET_UTILE_CONFIG (1UL << 2) +#define ASAHI_RENDER_UNK_LARGE_TIB (1UL << 3) +#define ASAHI_RENDER_UNK_SMALL_TIB (1UL << 4) +#define ASAHI_RENDER_UNK_SET_AUX_FB_UNK (1UL << 5) +#define ASAHI_RENDER_UNK_SET_G14_UNK (1UL << 6) +#define ASAHI_RENDER_UNK_SET_FRG_UNK_140 (1UL << 7) +#define ASAHI_RENDER_UNK_SET_FRG_UNK_158 (1UL << 8) +#define ASAHI_RENDER_UNK_SET_FRG_TILECFG (1UL << 9) +#define ASAHI_RENDER_UNK_SET_LOAD_BGOBJVALS (1UL << 10) +#define ASAHI_RENDER_UNK_SET_FRG_UNK_38 (1UL << 11) +#define ASAHI_RENDER_UNK_SET_FRG_UNK_3C (1UL << 12) +#define ASAHI_RENDER_UNK_SET_FRG_UNK_40 (1UL << 13) +#define ASAHI_RENDER_UNK_SET_RELOAD_ZLSCTRL (1UL << 14) +#define ASAHI_RENDER_UNK_SET_UNK_BUF_10 (1UL << 15) + +#define ASAHI_RENDER_UNK_SET_FRG_IGCMPUNK44 (1UL << 20) +#define ASAHI_RENDER_UNK_SET_FRG_SEQ_BUFFER (1UL << 21) +#define ASAHI_RENDER_UNK_SET_IOGPU_UNK54 (1UL << 22) +#define ASAHI_RENDER_UNK_SET_IOGPU_UNK56 (1UL << 23) +#define ASAHI_RENDER_UNK_SET_TILING_CONTROL (1UL << 24) +#define ASAHI_RENDER_UNK_SET_TILING_CONTROL_2 (1UL << 25) +#define ASAHI_RENDER_UNK_SET_VTX_UNK_F0 (1UL << 26) +#define ASAHI_RENDER_UNK_SET_VTX_UNK_F8 (1UL << 27) +#define ASAHI_RENDER_UNK_SET_VTX_UNK_118 (1UL << 28) +#define ASAHI_RENDER_UNK_SET_VTX_IGCMPUNK44 (1UL << 29) +#define ASAHI_RENDER_UNK_SET_VTX_SEQ_BUFFER (1UL << 30) + +#define ASAHI_RENDER_EXT_UNKNOWNS 0xff00 + +/* XXX: Do not upstream this struct */ +struct drm_asahi_cmd_render_unknowns { + /** @type: Type ID of this extension */ + __u32 type; + __u32 pad; + /** @next: Pointer to the next extension struct, if any */ + __u64 next; + + __u64 flags; + __u64 tile_config; + __u64 utile_config; + + __u64 aux_fb_unk; + __u64 g14_unk; + __u64 frg_unk_140; + __u64 frg_unk_158; + __u64 frg_tilecfg; + __u64 load_bgobjvals; + __u64 frg_unk_38; + __u64 frg_unk_3c; + __u64 frg_unk_40; + __u64 reload_zlsctrl; + __u64 unk_buf_10; + __u64 frg_iogpucmp_unk44; + __u64 frg_seq_buffer; + + __u64 iogpu_unk54; + __u64 iogpu_unk56; + __u64 tiling_control; + __u64 tiling_control_2; + __u64 vtx_unk_f0; + __u64 vtx_unk_f8; + __u64 vtx_unk_118; + __u64 vtx_iogpucmp_unk44; + __u64 vtx_seq_buffer; }; struct drm_asahi_cmd_compute { From 8fee0447fe2270dca26ddb2fd16ccef000f483be Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Wed, 26 Apr 2023 20:19:19 +0900 Subject: [PATCH 562/635] drm/asahi: render: Implement unknown value UAPI extension --- drivers/gpu/drm/asahi/debug.rs | 1 + drivers/gpu/drm/asahi/fw/compute.rs | 2 +- drivers/gpu/drm/asahi/fw/fragment.rs | 2 +- drivers/gpu/drm/asahi/fw/job.rs | 4 +- drivers/gpu/drm/asahi/fw/vertex.rs | 5 +- drivers/gpu/drm/asahi/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/queue/render.rs | 221 ++++++++++++++++++++----- 7 files changed, 183 insertions(+), 54 deletions(-) diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index f03a3f991d74a4..4f20d55db0337f 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -65,6 +65,7 @@ pub(crate) enum DebugFlags { Debug6 = 54, Debug7 = 55, + AllowUnknownOverrides = 62, OopsOnGpuCrash = 63, } diff --git a/drivers/gpu/drm/asahi/fw/compute.rs b/drivers/gpu/drm/asahi/fw/compute.rs index 0dbcd77c5e3e39..32f7f7a5a87d6f 100644 --- a/drivers/gpu/drm/asahi/fw/compute.rs +++ b/drivers/gpu/drm/asahi/fw/compute.rs @@ -63,7 +63,7 @@ pub(crate) mod raw { pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, pub(crate) microsequence_size: u32, pub(crate) job_params2: JobParameters2::ver<'a>, - pub(crate) encoder_params: job::raw::EncoderParams<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, pub(crate) meta: job::raw::JobMeta, pub(crate) cur_ts: U64, pub(crate) start_ts: Option>, diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs index 8830a268eb0475..078c7cfed9c0f0 100644 --- a/drivers/gpu/drm/asahi/fw/fragment.rs +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -234,7 +234,7 @@ pub(crate) mod raw { pub(crate) busy_flag: u32, pub(crate) tvb_overflow_count: u32, pub(crate) unk_878: u32, - pub(crate) encoder_params: job::raw::EncoderParams<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, pub(crate) process_empty_tiles: u32, pub(crate) no_clear_pipeline_textures: u32, pub(crate) msaa_zs: u32, diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs index f7b679e61c574b..bcae56edbb0218 100644 --- a/drivers/gpu/drm/asahi/fw/job.rs +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -26,14 +26,14 @@ pub(crate) mod raw { #[derive(Debug)] #[repr(C)] - pub(crate) struct EncoderParams<'a> { + pub(crate) struct EncoderParams { pub(crate) unk_8: u32, pub(crate) large_tib: u32, pub(crate) unk_10: u32, pub(crate) encoder_id: u32, pub(crate) unk_18: u32, pub(crate) iogpu_compute_unk44: u32, - pub(crate) seq_buffer: GpuPointer<'a, &'a [u64]>, + pub(crate) seq_buffer: U64, pub(crate) unk_28: U64, } diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs index 898c1e24c9f8d8..ee76150af1e220 100644 --- a/drivers/gpu/drm/asahi/fw/vertex.rs +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -40,8 +40,7 @@ pub(crate) mod raw { pub(crate) tvb_cluster_tilemaps: Option>, pub(crate) tpc: GpuPointer<'a, &'a [u8]>, pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, - pub(crate) iogpu_unk_54: u32, - pub(crate) iogpu_unk_55: u32, + pub(crate) iogpu_unk_54: U64, pub(crate) iogpu_unk_56: U64, #[ver(G < G14)] pub(crate) tvb_cluster_meta1: Option>, @@ -134,7 +133,7 @@ pub(crate) mod raw { pub(crate) unk_pointee: u32, pub(crate) unk_pad: u32, pub(crate) job_params2: JobParameters2<'a>, - pub(crate) encoder_params: job::raw::EncoderParams<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, pub(crate) unk_55c: u32, pub(crate) unk_560: u32, pub(crate) memoryless_rts_used: u32, diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 4f7ae9785a6b9b..7692c29e0f5a66 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -298,7 +298,7 @@ impl super::Queue::ver { encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed iogpu_compute_unk44: cmdbuf.iogpu_unk_44, - seq_buffer: inner.seq_buf.gpu_pointer(), + seq_buffer: U64(inner.seq_buf.gpu_pointer().into()), unk_28: U64(0x0), // fixed }, meta: fw::job::raw::JobMeta { diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index a908fc732efdce..bb8db1c1dcfb3f 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -248,6 +248,46 @@ impl super::Queue::ver { return Err(EINVAL); } + let mut unks: uapi::drm_asahi_cmd_render_unknowns = Default::default(); + + let mut ext_ptr = cmdbuf.extensions; + while ext_ptr != 0 { + let ext_type = u32::from_ne_bytes( + unsafe { UserSlicePtr::new(ext_ptr as usize as *mut _, 4) } + .read_all()? + .try_into() + .or(Err(EINVAL))?, + ); + + match ext_type { + uapi::ASAHI_RENDER_EXT_UNKNOWNS => { + if !debug_enabled(debug::DebugFlags::AllowUnknownOverrides) { + return Err(EINVAL); + } + let mut ext_reader = unsafe { + UserSlicePtr::new( + ext_ptr as usize as *mut _, + core::mem::size_of::(), + ) + .reader() + }; + unsafe { + ext_reader.read_raw( + &mut unks as *mut _ as *mut u8, + core::mem::size_of::(), + )?; + } + + ext_ptr = unks.next; + } + _ => return Err(EINVAL), + } + } + + if unks.pad != 0 { + return Err(EINVAL); + } + let dev = self.dev.data(); let gpu = match dev.gpu.as_any().downcast_ref::() { Some(gpu) => gpu, @@ -270,7 +310,7 @@ impl super::Queue::ver { } #[ver(G != G14)] - let tiling_control = { + let mut tiling_control = { let render_cfg = gpu.get_cfg().render; let mut tiling_control = render_cfg.tiling_control; @@ -393,7 +433,7 @@ impl super::Queue::ver { let timestamps = Arc::try_new(kalloc.shared.new_default::()?)?; - let unk1 = false; + let unk1 = unks.flags & uapi::ASAHI_RENDER_UNK_UNK1 as u64 != 0; let mut tile_config: u64 = 0; if !unk1 { @@ -446,7 +486,104 @@ impl super::Queue::ver { #[ver(V >= V13_0B4)] let count_vtx = count_frag + 1; - let large_tib = cmdbuf.tib_blocks > 8; + let mut large_tib = cmdbuf.tib_blocks > 8; + + // Unknowns handling + + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_TILE_CONFIG as u64 != 0 { + tile_config = unks.tile_config; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_UTILE_CONFIG as u64 != 0 { + utile_config = unks.utile_config as u32; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_LARGE_TIB as u64 != 0 { + large_tib = true; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SMALL_TIB as u64 != 0 { + large_tib = false; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_AUX_FB_UNK as u64 == 0 { + unks.aux_fb_unk = 0x100000; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_G14_UNK as u64 == 0 { + unks.g14_unk = 0x4040404; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_UNK_140 as u64 == 0 { + unks.frg_unk_140 = 0x8c60; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_UNK_158 as u64 == 0 { + unks.frg_unk_158 = 0x1c; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_TILECFG as u64 == 0 { + unks.frg_tilecfg = 0x0000000_00036011 + | (((tile_info.tiles_x - 1) as u64) << 44) + | (((tile_info.tiles_y - 1) as u64) << 53) + | (if unk1 { 0 } else { 0x20_00000000 }) + | ((utile_config as u64 & 0xf000) << 28); + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_LOAD_BGOBJVALS as u64 == 0 { + unks.load_bgobjvals = cmdbuf.isp_bgobjvals.into(); + #[ver(G < G14X)] + unks.load_bgobjvals |= 0x400; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_UNK_38 as u64 == 0 { + unks.frg_unk_38 = 0; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_UNK_3C as u64 == 0 { + unks.frg_unk_3c = 1; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_UNK_40 as u64 == 0 { + unks.frg_unk_40 = 0; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_RELOAD_ZLSCTRL as u64 == 0 { + unks.reload_zlsctrl = cmdbuf.zls_ctrl; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_UNK_BUF_10 as u64 == 0 { + #[ver(G < G14X)] + unks.unk_buf_10 = 1; + #[ver(G >= G14X)] + unks.unk_buf_10 = 0; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_IGCMPUNK44 as u64 == 0 { + unks.frg_iogpucmp_unk44 = 0xffffffff; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_FRG_SEQ_BUFFER as u64 == 0 { + unks.frg_seq_buffer = scene.seq_buf_pointer().into(); + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_IOGPU_UNK54 as u64 == 0 { + unks.iogpu_unk54 = 0x3a0012006b0003; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_IOGPU_UNK56 as u64 == 0 { + unks.iogpu_unk56 = 1; + } + #[ver(G != G14)] + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_TILING_CONTROL as u64 != 0 { + tiling_control = unks.tiling_control as u32; + } + #[ver(G != G14)] + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_TILING_CONTROL_2 as u64 == 0 { + #[ver(G < G14X)] + unks.tiling_control_2 = 0; + #[ver(G >= G14X)] + unks.tiling_control_2 = 4; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_VTX_UNK_F0 as u64 == 0 { + unks.vtx_unk_f0 = 0x1c; + #[ver(G < G14X)] + unks.vtx_unk_f0 += align(tile_info.meta1_blocks, 4) as u64; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_VTX_UNK_F8 as u64 == 0 { + unks.vtx_unk_f8 = 0x8c60; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_VTX_UNK_118 as u64 == 0 { + unks.vtx_unk_118 = 0x1c; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_VTX_IGCMPUNK44 as u64 == 0 { + unks.vtx_iogpucmp_unk44 = 0xffffffff; + } + if unks.flags & uapi::ASAHI_RENDER_UNK_SET_VTX_SEQ_BUFFER as u64 == 0 { + unks.vtx_seq_buffer = scene.seq_buf_pointer().into(); + } mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); let frag = GpuObject::new_prealloc( @@ -597,7 +734,7 @@ impl super::Queue::ver { width: cmdbuf.fb_width, height: cmdbuf.fb_height, #[ver(V >= V13_0B4)] - unk3: U64(0x100000), + unk3: U64(unks.aux_fb_unk), }; Ok(place!( @@ -641,7 +778,7 @@ impl super::Queue::ver { visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), zls_ctrl: U64(cmdbuf.zls_ctrl), #[ver(G >= G14)] - unk_58_g14_0: U64(0x4040404), + unk_58_g14_0: U64(unks.g14_unk), #[ver(G >= G14)] unk_58_g14_8: U64(0), depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), @@ -667,10 +804,10 @@ impl super::Queue::ver { aux_fb: inner.aux_fb.gpu_pointer(), unk_108: Default::default(), pipeline_base: U64(0x11_00000000), - unk_140: U64(0x8c60), + unk_140: U64(unks.frg_unk_140), unk_148: U64(0x0), unk_150: U64(0x0), - unk_158: U64(0x1c), + unk_158: U64(unks.frg_unk_158), unk_160: U64(0), __pad: Default::default(), #[ver(V < V13_0B4)] @@ -692,10 +829,10 @@ impl super::Queue::ver { tib_blocks: cmdbuf.tib_blocks, isp_bgobjdepth: cmdbuf.isp_bgobjdepth, // TODO: does this flag need to be exposed to userspace? - isp_bgobjvals: cmdbuf.isp_bgobjvals | 0x400, - unk_38: 0x0, - unk_3c: 0x1, - unk_40: 0, + isp_bgobjvals: unks.load_bgobjvals as u32, + unk_38: unks.frg_unk_38 as u32, + unk_3c: unks.frg_unk_3c as u32, + unk_40: unks.frg_unk_40 as u32, __pad: Default::default(), }, #[ver(G >= G14X)] @@ -734,14 +871,14 @@ impl super::Queue::ver { 0x15211, ((cmdbuf.fb_height as u64) << 32) | cmdbuf.fb_width as u64, ); // aux_fb_info.{width, heigh - r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 + r.add(0x15049, unks.aux_fb_unk); // s2.aux_fb_info.unk3 r.add(0x10051, cmdbuf.tib_blocks.into()); // s1.unk_2c r.add(0x15321, cmdbuf.depth_dimensions.into()); // ISP_ZLS_PIXELS r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH - r.add(0x15309, cmdbuf.isp_bgobjvals.into()); // ISP_BGOBJVALS + r.add(0x15309, unks.load_bgobjvals); // ISP_BGOBJVALS r.add(0x15311, cmdbuf.visibility_result_buffer); // ISP_OCLQRY_BASE r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL - r.add(0x15349, 0x4040404); // s2.unk_58_g14_0 + r.add(0x15349, unks.g14_unk); // s2.unk_58_g14_0 r.add(0x15351, 0); // s2.unk_58_g14_8 r.add(0x15329, cmdbuf.depth_buffer_load); // ISP_ZLOAD_BASE r.add(0x15331, cmdbuf.depth_buffer_store); // ISP_ZSTORE_BASE @@ -776,16 +913,9 @@ impl super::Queue::ver { r.add(0x16020, 0); r.add(0x16461, inner.aux_fb.gpu_pointer().into()); r.add(0x16090, inner.aux_fb.gpu_pointer().into()); - r.add(0x120a1, 0x1c); // s2.unk_158 + r.add(0x120a1, unks.frg_unk_158); r.add(0x160a8, 0); - r.add( - 0x16068, - 0x0000000_00036011 - | (((tile_info.tiles_x - 1) as u64) << 44) - | (((tile_info.tiles_y - 1) as u64) << 53) - | (if unk1 { 0 } else { 0x20_00000000 }) - | ((utile_config as u64 & 0xf000) << 28), - ); + r.add(0x16068, unks.frg_tilecfg); r.add(0x160b8, 0x0); } ), @@ -813,23 +943,23 @@ impl super::Queue::ver { pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), address: U64(cmdbuf.partial_reload_pipeline as u64), }, - zls_ctrl: U64(cmdbuf.zls_ctrl), + zls_ctrl: U64(unks.reload_zlsctrl), #[ver(G >= G14X)] - unk_290: U64(0x4040404), + unk_290: U64(unks.g14_unk), #[ver(G < G14X)] unk_290: U64(0x0), depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), unk_2a0: U64(0x0), unk_2a8: U64(0x0), depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), - depth_buffer_ptr3: U64(cmdbuf.depth_buffer_reload), - depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_reload), + depth_buffer_ptr3: U64(cmdbuf.depth_buffer_partial), + depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_partial), stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), unk_2d0: U64(0x0), unk_2d8: U64(0x0), stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), - stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_reload), - stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_reload), + stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_partial), + stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_partial), unk_2f8: Default::default(), tib_blocks: cmdbuf.tib_blocks, unk_30c: 0x0, @@ -869,8 +999,8 @@ impl super::Queue::ver { unk_10: 0x0, // fixed encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed - iogpu_compute_unk44: 0xffffffff, - seq_buffer: inner.scene.seq_buf_pointer(), + iogpu_compute_unk44: unks.frg_iogpucmp_unk44 as u32, + seq_buffer: U64(unks.frg_seq_buffer), unk_28: U64(0x0), // fixed }, process_empty_tiles: (cmdbuf.flags @@ -1135,9 +1265,8 @@ impl super::Queue::ver { .scene .tvb_heapmeta_pointer() .or(0x8000_0000_0000_0000), - iogpu_unk_54: 0x6b0003, // fixed - iogpu_unk_55: 0x3a0012, // fixed - iogpu_unk_56: U64(0x1), // fixed + iogpu_unk_54: U64(unks.iogpu_unk54), // fixed + iogpu_unk_56: U64(unks.iogpu_unk56), // fixed #[ver(G < G14)] tvb_cluster_meta1: inner .scene @@ -1169,7 +1298,7 @@ impl super::Queue::ver { #[ver(G < G14)] tiling_control: tiling_control, #[ver(G < G14)] - unk_ac: Default::default(), // fixed + unk_ac: unks.tiling_control_2 as u32, // fixed unk_b0: Default::default(), // fixed pipeline_base: U64(0x11_00000000), #[ver(G < G14)] @@ -1178,10 +1307,10 @@ impl super::Queue::ver { .meta_4_pointer() .map(|x| x.or(0x3000_0000_0000_0000)), #[ver(G < G14)] - unk_f0: U64(0x1c + align(tile_info.meta1_blocks, 4) as u64), - unk_f8: U64(0x8c60), // fixed - unk_100: Default::default(), // fixed - unk_118: 0x1c, // fixed + unk_f0: U64(unks.vtx_unk_f0), + unk_f8: U64(unks.vtx_unk_f8), // fixed + unk_100: Default::default(), // fixed + unk_118: unks.vtx_unk_118 as u32, // fixed __pad: Default::default(), }, #[ver(G < G14X)] @@ -1209,8 +1338,8 @@ impl super::Queue::ver { .into(); r.add(0x1c031, tvb_heapmeta_ptr); r.add(0x1c9c0, tvb_heapmeta_ptr); - r.add(0x1c051, 0x3a0012006b0003); // iogpu_unk_54/55 - r.add(0x1c061, 1); // iogpu_unk_56 + r.add(0x1c051, unks.iogpu_unk54); // iogpu_unk_54/55 + r.add(0x1c061, unks.iogpu_unk56); // iogpu_unk_56 r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); @@ -1238,7 +1367,7 @@ impl super::Queue::ver { inner.scene.meta_3_pointer().map_or(0, |a| a.into()), ); // tvb_cluster_meta3 r.add(0x1c890, tiling_control.into()); // tvb_tiling_control - r.add(0x1c918, 0x4); + r.add(0x1c918, unks.tiling_control_2); r.add(0x1c079, inner.scene.tvb_heapmeta_pointer().into()); r.add(0x1c9d8, inner.scene.tvb_heapmeta_pointer().into()); r.add(0x1c089, 0); @@ -1247,7 +1376,7 @@ impl super::Queue::ver { inner.scene.meta_4_pointer().map_or(0, |a| a.into()); r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 - r.add(0x1c9a8, 0x1c); // + meta1_blocks? min_free_tvb_pages? + r.add(0x1c9a8, unks.vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? r.add( 0x1c920, inner.scene.meta_1_pointer().map_or(0, |a| a.into()), @@ -1278,7 +1407,7 @@ impl super::Queue::ver { r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC r.add(0x10171, tile_info.params.unk_24.into()); r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX - r.add(0x12099, 0x1c); + r.add(0x12099, unks.vtx_unk_118); r.add(0x1c9e8, 0); } ), @@ -1311,8 +1440,8 @@ impl super::Queue::ver { unk_10: 0x0, // fixed encoder_id: cmdbuf.encoder_id, unk_18: 0x0, // fixed - iogpu_compute_unk44: 0xffffffff, - seq_buffer: inner.scene.seq_buf_pointer(), + iogpu_compute_unk44: unks.vtx_iogpucmp_unk44 as u32, + seq_buffer: U64(unks.vtx_seq_buffer), unk_28: U64(0x0), // fixed }, unk_55c: 0, From 5fcef8887dc7a85f27cc96eb531db718a47834f8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 28 Apr 2023 20:11:26 +0900 Subject: [PATCH 563/635] rust: Enable type_alias_impl_trait This is required to make PinInit work as a return value from a trait function. We indirect via an associated type to avoid return_position_impl_trait_in_trait, which is an incomplete feature. Signed-off-by: Asahi Lina --- rust/kernel/lib.rs | 1 + scripts/Makefile.build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 3caca288a2664a..923d11dc5518c2 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -23,6 +23,7 @@ #![feature(new_uninit)] #![feature(pin_macro)] #![feature(receiver_trait)] +#![feature(type_alias_impl_trait)] #![feature(unsize)] // Ensure conditional compilation based on the kernel configuration works; diff --git a/scripts/Makefile.build b/scripts/Makefile.build index c3e2e2f257afdf..ae0038a4f9b246 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -277,7 +277,7 @@ $(obj)/%.lst: $(src)/%.c FORCE # Compile Rust sources (.rs) # --------------------------------------------------------------------------- -rust_allowed_features := allocator_api,const_refs_to_cell,new_uninit,pin_macro +rust_allowed_features := allocator_api,const_refs_to_cell,new_uninit,pin_macro,type_alias_impl_trait rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ From 92826b4159a41732c60e51a737c566a1ba45afe5 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 28 Apr 2023 20:12:35 +0900 Subject: [PATCH 564/635] rust: kernel: lock: Add Lock::pin_init() This allows initializing a lock using pin_init!(), instead of requiring the inner data to be passed through the stack. Signed-off-by: Asahi Lina --- rust/kernel/sync/lock.rs | 30 +++++++++++++++++++++++++++++- rust/kernel/sync/lock/mutex.rs | 13 +++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index a2216325632dff..d493c5d191048b 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -6,7 +6,9 @@ //! spinlocks, raw spinlocks) to be provided with minimal effort. use super::LockClassKey; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; +use crate::{ + bindings, init::PinInit, pin_init, str::CStr, try_pin_init, types::Opaque, types::ScopeGuard, +}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; @@ -87,6 +89,7 @@ pub struct Lock { _pin: PhantomPinned, /// The data protected by the lock. + #[pin] pub(crate) data: UnsafeCell, } @@ -111,6 +114,31 @@ impl Lock { }), }) } + + /// Constructs a new lock initialiser taking an initialiser. + pub fn pin_init( + t: impl PinInit, + name: &'static CStr, + key: &'static LockClassKey, + ) -> impl PinInit + where + E: core::convert::From, + { + try_pin_init!(Self { + // SAFETY: We are just forwarding the initialization across a + // cast away from UnsafeCell, so the pin_init_from_closure and + // __pinned_init() requirements are in sync. + data <- unsafe { crate::init::pin_init_from_closure(move |slot: *mut UnsafeCell| { + t.__pinned_init(slot as *mut T) + })}, + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }? E) + } } impl Lock { diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 923472f04af4bd..06fe685501b4e3 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -18,6 +18,19 @@ macro_rules! new_mutex { }; } +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class, +/// given an initialiser for the inner type. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex_pinned { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::pin_init( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + /// A mutual exclusion primitive. /// /// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, From 61e91c1ea7f5409483fab53ffd60d5a69c2263a1 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 28 Apr 2023 20:14:32 +0900 Subject: [PATCH 565/635] rust: drm: gem: Allow pinning GEM object driver data This requires type_alias_impl_trait. Signed-off-by: Asahi Lina --- rust/kernel/drm/gem/mod.rs | 42 ++++++++++++++++++++++++--------- rust/kernel/drm/gem/shmem.rs | 45 +++++++++++++++++++++--------------- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 68018289556984..658416b95818a5 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -15,12 +15,16 @@ use crate::{ error::{to_result, Result}, prelude::*, }; -use core::{mem, mem::ManuallyDrop, ops::Deref, ops::DerefMut}; +use core::{marker::PhantomPinned, mem, mem::ManuallyDrop, ops::Deref, ops::DerefMut}; /// GEM object functions, which must be implemented by drivers. pub trait BaseDriverObject: Sync + Send + Sized { + /// The return type of the new() function. Should be `impl PinInit`. + /// TODO: Remove this when return_position_impl_trait_in_trait is stable. + type Initializer: PinInit; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &device::Device, size: usize) -> Result; + fn new(dev: &device::Device, size: usize) -> Self::Initializer; /// Open a new handle to an existing object, associated with a File. fn open( @@ -204,14 +208,21 @@ impl BaseObject for T {} /// A base GEM object. #[repr(C)] +#[pin_data] pub struct Object { obj: bindings::drm_gem_object, // The DRM core ensures the Device exists as long as its objects exist, so we don't need to // manage the reference count here. dev: ManuallyDrop>, + #[pin] inner: T, + #[pin] + _p: PhantomPinned, } +// SAFETY: This struct is safe to zero-initialize +unsafe impl init::Zeroable for bindings::drm_gem_object {} + impl Object { /// The size of this object's structure. pub const SIZE: usize = mem::size_of::(); @@ -232,23 +243,31 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &device::Device, size: usize) -> Result> { - let mut obj: Box = Box::try_new(Self { + pub fn new(dev: &device::Device, size: usize) -> Result>> { + let obj: Pin> = Box::pin_init(try_pin_init!(Self { // SAFETY: This struct is expected to be zero-initialized - obj: unsafe { mem::zeroed() }, + obj: bindings::drm_gem_object { + funcs: &Self::OBJECT_FUNCS, + ..Default::default() + }, + inner <- T::new(dev, size), // SAFETY: The drm subsystem guarantees that the drm_device will live as long as // the GEM object lives, so we can conjure a reference out of thin air. dev: ManuallyDrop::new(unsafe { device::Device::from_raw(dev.ptr) }), - inner: T::new(dev, size)?, - })?; + _p: PhantomPinned + }))?; - obj.obj.funcs = &Self::OBJECT_FUNCS; to_result(unsafe { - bindings::drm_gem_object_init(dev.raw() as *mut _, &mut obj.obj, size) + bindings::drm_gem_object_init(dev.raw() as *mut _, &obj.obj as *const _ as *mut _, size) })?; - let obj_ref = UniqueObjectRef { - ptr: Box::leak(obj), + // SAFETY: We never move out of self + let obj_ref = unsafe { + Pin::new_unchecked(UniqueObjectRef { + // SAFETY: We never move out of the Box + ptr: Box::leak(Pin::into_inner_unchecked(obj)), + _p: PhantomPinned, + }) }; Ok(obj_ref) @@ -330,6 +349,7 @@ pub struct UniqueObjectRef { // Invariant: the pointer is valid and initialized, and this ObjectRef owns the only reference // to it. ptr: *mut T, + _p: PhantomPinned, } impl UniqueObjectRef { diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fe5f976b484a5f..17cfb51cf36449 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -10,11 +10,10 @@ use crate::{ prelude::*, }; use core::{ - marker::PhantomData, + marker::{PhantomData, PhantomPinned}, mem, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, - ptr::addr_of_mut, slice, }; @@ -66,14 +65,20 @@ const SHMEM_VM_OPS: bindings::vm_operations_struct = vm_numa_fields! { /// A shmem-backed GEM object. #[repr(C)] +#[pin_data] pub struct Object { + #[pin] obj: bindings::drm_gem_shmem_object, // The DRM core ensures the Device exists as long as its objects exist, so we don't need to // manage the reference count here. dev: ManuallyDrop>, + #[pin] inner: T, } +// SAFETY: drm_gem_shmem_object is safe to zero-initialize +unsafe impl init::Zeroable for bindings::drm_gem_shmem_object {} + unsafe extern "C" fn gem_create_object( raw_dev: *mut bindings::drm_device, size: usize, @@ -82,31 +87,30 @@ unsafe extern "C" fn gem_create_object( // so we can conjure up a reference from thin air and never drop it. let dev = ManuallyDrop::new(unsafe { device::Device::from_raw(raw_dev) }); - let inner = match T::new(&*dev, size) { - Ok(v) => v, - Err(e) => return e.to_ptr(), - }; - let p = unsafe { - bindings::krealloc( - core::ptr::null(), - Object::::SIZE, - bindings::GFP_KERNEL | bindings::__GFP_ZERO, - ) as *mut Object + bindings::krealloc(core::ptr::null(), Object::::SIZE, bindings::GFP_KERNEL) + as *mut Object }; if p.is_null() { return ENOMEM.to_ptr(); } - // SAFETY: p is valid as long as the alloc succeeded - unsafe { - addr_of_mut!((*p).dev).write(dev); - addr_of_mut!((*p).inner).write(inner); + let init = try_pin_init!(Object { + obj <- init::zeroed(), + inner <- T::new(&*dev, size), + dev, + }); + + // SAFETY: p is a valid pointer to an uninitialized Object. + if let Err(e) = unsafe { init.__pinned_init(p) } { + // SAFETY: p is a valid pointer from `krealloc` and __pinned_init guarantees we can dealloc it. + unsafe { bindings::kfree(p as *mut _) }; + + return e.to_ptr(); } - // SAFETY: drm_gem_shmem_object is safe to zero-init, and - // the rest of Object has been initialized + // SAFETY: __pinned_init() guarantees the object has been initialized let new: &mut Object = unsafe { &mut *(p as *mut _) }; new.obj.base.funcs = &Object::::VTABLE; @@ -164,7 +168,10 @@ impl Object { // SAFETY: The gem_create_object callback ensures this is a valid Object, // so we can take a unique reference to it. - let obj_ref = gem::UniqueObjectRef { ptr: p }; + let obj_ref = gem::UniqueObjectRef { + ptr: p, + _p: PhantomPinned, + }; Ok(obj_ref) } From 844d720b8604515836819a77a307daafe8d3265e Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 28 Apr 2023 20:15:23 +0900 Subject: [PATCH 566/635] drm/asahi: Update for new upstream APIs Replace all usage of smutex::Mutex with real mutexes using pin_init!(). This could still use some optimization (there's redundant re-boxing going on), but for now it works and runs. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 8 +- drivers/gpu/drm/asahi/driver.rs | 5 +- drivers/gpu/drm/asahi/event.rs | 4 + drivers/gpu/drm/asahi/file.rs | 10 +-- drivers/gpu/drm/asahi/gem.rs | 19 ++--- drivers/gpu/drm/asahi/gpu.rs | 117 +++++++++++++++----------- drivers/gpu/drm/asahi/mmu.rs | 108 ++++++++++++++---------- drivers/gpu/drm/asahi/queue/mod.rs | 2 +- drivers/gpu/drm/asahi/queue/render.rs | 5 +- drivers/gpu/drm/asahi/slotalloc.rs | 29 +++---- drivers/gpu/drm/asahi/workqueue.rs | 41 ++++----- 11 files changed, 197 insertions(+), 151 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index b6715877eb7402..c3b33f950fe5f4 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -40,7 +40,8 @@ use crate::{alloc, fw, gpu, hw, mmu, slotalloc}; use crate::{box_in_place, place}; use core::sync::atomic::Ordering; use kernel::prelude::*; -use kernel::sync::{smutex::Mutex, Arc}; +use kernel::sync::{Arc, Mutex}; +use kernel::{c_str, new_mutex, static_lock_class}; const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; @@ -392,7 +393,7 @@ impl Buffer::ver { })?; Ok(Buffer::ver { - inner: Arc::try_new(Mutex::new(BufferInner::ver { + inner: Arc::pin_init(new_mutex!(BufferInner::ver { info, ualloc, ualloc_priv, @@ -702,6 +703,9 @@ impl BufferManager { NUM_BUFFERS, (), |_inner, _slot| (), + c_str!("BufferManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), )?)) } } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 26e79b36592f67..dcf4e9ce3c0971 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -152,12 +152,13 @@ impl platform::Driver for AsahiDriver { let data = kernel::new_device_data!(reg, res, AsahiData { dev, gpu }, "Asahi::Registrations")?; - let data = Arc::::from(data); + let data: Arc = data.into(); data.gpu.init()?; kernel::drm_device_register!( - data.registrations().ok_or(ENXIO)?.as_pinned_mut(), + // TODO: Expose an API to get a pinned reference here + unsafe { Pin::new_unchecked(&mut *data.registrations().ok_or(ENXIO)?) }, data.clone(), 0 )?; diff --git a/drivers/gpu/drm/asahi/event.rs b/drivers/gpu/drm/asahi/event.rs index ccf00e4104bef7..a793dc508da880 100644 --- a/drivers/gpu/drm/asahi/event.rs +++ b/drivers/gpu/drm/asahi/event.rs @@ -16,6 +16,7 @@ use core::cmp; use core::sync::atomic::Ordering; use kernel::prelude::*; use kernel::sync::Arc; +use kernel::{c_str, static_lock_class}; const DEBUG_CLASS: DebugFlags = DebugFlags::Event; @@ -158,6 +159,9 @@ impl EventManager { gpu_stamp: inner.stamps.weak_item_pointer(slot as usize), gpu_fw_stamp: inner.fw_stamps.weak_item_pointer(slot as usize), }, + c_str!("EventManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), )?, }) } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 53d7e7ef00e48f..6ea4d45c9d0176 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -15,9 +15,9 @@ use kernel::dma_fence::RawDmaFence; use kernel::drm::gem::BaseObject; use kernel::io_buffer::{IoBufferReader, IoBufferWriter}; use kernel::prelude::*; -use kernel::sync::{smutex::Mutex, Arc}; +use kernel::sync::{Arc, Mutex}; use kernel::user_ptr::UserSlicePtr; -use kernel::{dma_fence, drm, uapi, xarray}; +use kernel::{dma_fence, drm, new_mutex, uapi, xarray}; const DEBUG_CLASS: DebugFlags = DebugFlags::File; @@ -287,7 +287,7 @@ impl File { file_id, id ); - let ualloc = Arc::try_new(Mutex::new(alloc::DefaultAllocator::new( + let ualloc = Arc::pin_init(new_mutex!(alloc::DefaultAllocator::new( device, &vm, VM_DRV_GPU_START, @@ -299,7 +299,7 @@ impl File { fmt!("File {} VM {} GPU Shared", file_id, id), false, )?))?; - let ualloc_priv = Arc::try_new(Mutex::new(alloc::DefaultAllocator::new( + let ualloc_priv = Arc::pin_init(new_mutex!(alloc::DefaultAllocator::new( device, &vm, VM_DRV_GPUFW_START, @@ -605,7 +605,7 @@ impl File { .new_queue(vm, ualloc, ualloc_priv, data.priority, data.queue_caps)?; data.queue_id = resv.index().try_into()?; - resv.store(Arc::try_new(Mutex::new(queue))?)?; + resv.store(Arc::pin_init(new_mutex!(queue))?)?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 2d6eb213268de5..649368ba6c41e6 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -10,9 +10,10 @@ use kernel::{ drm::{gem, gem::shmem}, error::Result, + new_mutex, prelude::*, soc::apple::rtkit, - sync::smutex::Mutex, + sync::Mutex, uapi, }; @@ -25,6 +26,7 @@ use crate::{debug::*, driver::AsahiDevice, file::DrmFile, mmu, util::*}; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. +#[pin_data] pub(crate) struct DriverObject { /// Whether this is a kernel-created object. kernel: bool, @@ -33,6 +35,7 @@ pub(crate) struct DriverObject { /// VM ID for VM-private objects. vm_id: Option, /// Locked list of mapping tuples: (file_id, vm_id, mapping) + #[pin] mappings: Mutex>, /// ID for debug id: u64, @@ -258,15 +261,17 @@ pub(crate) fn lookup_handle(file: &DrmFile, handle: u32) -> Result { } impl gem::BaseDriverObject for DriverObject { + type Initializer = impl PinInit; + /// Callback to create the inner data of a GEM object - fn new(_dev: &AsahiDevice, _size: usize) -> Result { + fn new(_dev: &AsahiDevice, _size: usize) -> Self::Initializer { let id = GEM_ID.fetch_add(1, Ordering::Relaxed); mod_pr_debug!("DriverObject::new id={}\n", id); - Ok(DriverObject { + try_pin_init!(DriverObject { kernel: false, flags: 0, vm_id: None, - mappings: Mutex::new(Vec::new()), + mappings <- new_mutex!(Vec::new()), id, }) } @@ -278,12 +283,6 @@ impl gem::BaseDriverObject for DriverObject { } } -impl Drop for DriverObject { - fn drop(&mut self) { - mod_pr_debug!("DriverObject::drop vm_id={:?} id={}\n", self.vm_id, self.id); - } -} - impl shmem::DriverObject for DriverObject { type Driver = crate::driver::AsahiDriver; } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 7172c5ff6b93b9..7226efede039fa 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -19,15 +19,18 @@ use kernel::{ delay::coarse_sleep, error::code::*, macros::versions, + new_mutex, prelude::*, soc::apple::rtkit, - sync::{smutex::Mutex, Arc, Guard, UniqueArc}, + sync::{ + lock::{mutex::MutexBackend, Guard}, + Arc, Mutex, UniqueArc, + }, time, types::ForeignOwnable, }; use crate::alloc::Allocator; -use crate::box_in_place; use crate::debug::*; use crate::driver::AsahiDevice; use crate::fw::channels::PipeType; @@ -111,6 +114,7 @@ pub(crate) struct KernelAllocators { /// Receive (GPU->driver) ring buffer channels. #[versions(AGX)] +#[pin_data] struct RxChannels { event: channel::EventChannel, fw_log: channel::FwLogChannel, @@ -121,13 +125,14 @@ struct RxChannels { /// GPU work submission pipe channels (driver->GPU). #[versions(AGX)] struct PipeChannels { - pub(crate) vtx: Vec>, - pub(crate) frag: Vec>, - pub(crate) comp: Vec>, + pub(crate) vtx: Vec>>>, + pub(crate) frag: Vec>>>, + pub(crate) comp: Vec>>>, } /// Misc command transmit (driver->GPU) channels. #[versions(AGX)] +#[pin_data] struct TxChannels { pub(crate) device_control: channel::DeviceControlChannel::ver, } @@ -190,19 +195,22 @@ pub(crate) struct GpuManager { pub(crate) initdata: Box>, uat: Box, crashed: AtomicBool, - alloc: Mutex, + alloc: Pin>>, io_mappings: Vec, - rtkit: Mutex>>>, - rx_channels: Mutex>, - tx_channels: Mutex>, - fwctl_channel: Mutex>, + rtkit: Pin>>>>, + rx_channels: Pin>>, + tx_channels: Pin>>, + fwctl_channel: Pin>>, pipes: PipeChannels::ver, event_manager: Arc, buffer_mgr: buffer::BufferManager, ids: SequenceIDs, - garbage_work: Mutex>>, + #[allow(clippy::type_complexity)] + garbage_work: Pin>>>>, #[allow(clippy::vec_box)] - garbage_contexts: Mutex>>>, + #[allow(clippy::type_complexity)] + garbage_contexts: + Pin>>>>>, } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -218,7 +226,7 @@ pub(crate) trait GpuManager: Send + Sync { /// TODO: Unclear what can and cannot be updated like this. fn update_globals(&self); /// Get a reference to the KernelAllocators. - fn alloc(&self) -> Guard<'_, Mutex>; + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; /// Create a new `Vm` given a unique `File` ID. fn new_vm(&self, file_id: u64) -> Result; /// Bind a `Vm` to an available slot and return the `VmBind`. @@ -492,12 +500,7 @@ impl GpuManager::ver { let mgr = Arc::from(mgr); - let rtkit = Box::try_new(rtkit::RtKit::::new( - dev, - None, - 0, - mgr.clone(), - )?)?; + let rtkit = rtkit::RtKit::::new(dev, None, 0, mgr.clone())?; *mgr.rtkit.lock() = Some(rtkit); @@ -554,44 +557,64 @@ impl GpuManager::ver { }; for _i in 0..=NUM_PIPES - 1 { - pipes - .vtx - .try_push(Mutex::new(channel::PipeChannel::ver::new(dev, &mut alloc)?))?; - pipes - .frag - .try_push(Mutex::new(channel::PipeChannel::ver::new(dev, &mut alloc)?))?; - pipes - .comp - .try_push(Mutex::new(channel::PipeChannel::ver::new(dev, &mut alloc)?))?; + pipes.vtx.try_push(Box::pin_init(new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_vtx", + ))?)?; + pipes.frag.try_push(Box::pin_init(new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_frag", + ))?)?; + pipes.comp.try_push(Box::pin_init(new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_comp", + ))?)?; } - UniqueArc::try_new(GpuManager::ver { + let fwctl_channel = Box::pin_init(new_mutex!( + channel::FwCtlChannel::new(dev, &mut alloc)?, + "fwctl_channel" + ))?; + let rx_channels = Box::pin_init(new_mutex!( + RxChannels::ver { + event: channel::EventChannel::new(dev, &mut alloc, event_manager.clone())?, + fw_log: channel::FwLogChannel::new(dev, &mut alloc)?, + ktrace: channel::KTraceChannel::new(dev, &mut alloc)?, + stats: channel::StatsChannel::ver::new(dev, &mut alloc)?, + }, + "rx_channels" + ))?; + let tx_channels = Box::pin_init(new_mutex!( + TxChannels::ver { + device_control: channel::DeviceControlChannel::ver::new(dev, &mut alloc)?, + }, + "tx_channels" + ))?; + + let alloc = Box::pin_init(new_mutex!(alloc, "alloc"))?; + + let x = UniqueArc::try_new(GpuManager::ver { dev: dev.clone(), cfg, dyncfg, initdata, uat, io_mappings: Vec::new(), - rtkit: Mutex::new(None), + rtkit: Box::pin_init(new_mutex!(None, "rtkit"))?, crashed: AtomicBool::new(false), - rx_channels: Mutex::new(box_in_place!(RxChannels::ver { - event: channel::EventChannel::new(dev, &mut alloc, event_manager.clone())?, - fw_log: channel::FwLogChannel::new(dev, &mut alloc)?, - ktrace: channel::KTraceChannel::new(dev, &mut alloc)?, - stats: channel::StatsChannel::ver::new(dev, &mut alloc)?, - })?), - tx_channels: Mutex::new(Box::try_new(TxChannels::ver { - device_control: channel::DeviceControlChannel::ver::new(dev, &mut alloc)?, - })?), - fwctl_channel: Mutex::new(Box::try_new(channel::FwCtlChannel::new(dev, &mut alloc)?)?), - pipes, event_manager, + alloc, + fwctl_channel, + rx_channels, + tx_channels, + pipes, buffer_mgr: buffer::BufferManager::new()?, - alloc: Mutex::new(alloc), ids: Default::default(), - garbage_work: Mutex::new(Vec::new()), - garbage_contexts: Mutex::new(Vec::new()), - }) + garbage_work: Box::pin_init(new_mutex!(Vec::new(), "garbage_work"))?, + garbage_contexts: Box::pin_init(new_mutex!(Vec::new(), "garbage_contexts"))?, + })?; + + Ok(x) } /// Fetch and validate the GPU dynamic configuration from the device tree and hardware. @@ -697,7 +720,7 @@ impl GpuManager::ver { /// Create the global GPU event manager, and return an `Arc<>` to it. fn make_event_manager(alloc: &mut KernelAllocators) -> Result> { - Arc::try_new(event::EventManager::new(alloc)?) + Ok(Arc::try_new(event::EventManager::new(alloc)?)?) } /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified @@ -976,7 +999,7 @@ impl GpuManager for GpuManager::ver { }); } - fn alloc(&self) -> Guard<'_, Mutex> { + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend> { /* * TODO: This should be done in a workqueue or something. * Clean up completed jobs diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 954b4438f0d233..dc05f6cf36fe3d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -10,6 +10,7 @@ //! //! The actual page table management is delegated to the common kernel `io_pgtable` code. +use core::convert::Infallible; use core::fmt::Debug; use core::mem::size_of; use core::ptr::{addr_of_mut, NonNull}; @@ -22,9 +23,13 @@ use kernel::{ error::{to_result, Result}, io_pgtable, io_pgtable::{prot, AppleUAT, IoPageTable}, + new_mutex, prelude::*, - sync::{smutex::Mutex, Guard}, - sync::{Arc, LockClassKey, UniqueArc}, + static_lock_class, + sync::{ + lock::{mutex::MutexBackend, Guard}, + Arc, Mutex, + }, time, types::ForeignOwnable, }; @@ -592,19 +597,22 @@ impl UatShared { unsafe impl Send for UatShared {} /// Inner data for the top-level UAT instance. +#[pin_data] struct UatInner { + #[pin] shared: Mutex, + #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } impl UatInner { /// Take the lock on the shared data and return the guard. - fn lock(&self) -> Guard<'_, Mutex> { + fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } /// Take a lock on a handoff flush slot and return the guard. - fn lock_flush(&self, slot: u32) -> Guard<'_, Mutex> { + fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() } } @@ -755,8 +763,6 @@ impl io_pgtable::FlushOps for Uat { } } -static LOCK_KEY: LockClassKey = LockClassKey::new(); - impl Vm { /// Create a new virtual memory address space fn new( @@ -794,25 +800,28 @@ impl Vm { (max_va - min_va + 1) as u64, (), c_str!("asahi Vm"), - &LOCK_KEY, + static_lock_class!(), )?; Ok(Vm { id, file_id, - inner: Arc::try_new(Mutex::new(VmInner { - dev, - min_va, - max_va, - is_kernel, - page_table, - mm, - uat_inner, - binding: None, - bind_token: None, - active_users: 0, - id, - }))?, + inner: Arc::pin_init(new_mutex!( + VmInner { + dev, + min_va, + max_va, + is_kernel, + page_table, + mm, + uat_inner, + binding: None, + bind_token: None, + active_users: 0, + id, + }, + "VmInner" + ))?, }) } @@ -1178,27 +1187,33 @@ impl Uat { dev_info!(dev, "MMU: Initializing kernel page table\n"); - let mut inner = UniqueArc::::try_new_uninit()?; - let ptr = inner.as_mut_ptr(); - - Ok(unsafe { - let handoff = &(handoff_rgn.map.as_ptr() as *mut Handoff).as_ref().unwrap(); - - for i in 0..UAT_NUM_CTX + 1 { - addr_of_mut!((*ptr).handoff_flush[i]) - .write(Mutex::new(HandoffFlush(&handoff.flush[i]))); - } - - addr_of_mut!((*ptr).shared).write(Mutex::new(UatShared { - kernel_ttb1: 0, - map_kernel_to_user: false, - handoff_rgn, - ttbs_rgn, - })); - - inner.assume_init() - } - .into()) + Arc::pin_init(unsafe { + init::pin_init_from_closure( + move |slot: *mut UatInner| -> core::result::Result<(), Infallible> { + let handoff = &(handoff_rgn.map.as_ptr() as *mut Handoff).as_ref().unwrap(); + + for i in 0..UAT_NUM_CTX + 1 { + new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + .__pinned_init(addr_of_mut!((*slot).handoff_flush[i])) + .expect("infallible"); + } + + new_mutex!( + UatShared { + kernel_ttb1: 0, + map_kernel_to_user: false, + handoff_rgn, + ttbs_rgn, + }, + "uat_shared" + ) + .__pinned_init(addr_of_mut!((*slot).shared)) + .expect("infallible"); + + Ok(()) + }, + ) + }) } /// Creates a new `Uat` instance given the relevant hardware config. @@ -1230,9 +1245,14 @@ impl Uat { kernel_vm, kernel_lower_vm, inner, - slots: slotalloc::SlotAllocator::new(UAT_USER_CTX as u32, (), |_inner, _slot| { - SlotInner() - })?, + slots: slotalloc::SlotAllocator::new( + UAT_USER_CTX as u32, + (), + |_inner, _slot| SlotInner(), + c_str!("Uat::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, }; let mut inner = uat.inner.lock(); diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index b6a3fdda5c004e..804d4b7c0e8ccb 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -12,7 +12,7 @@ use kernel::{ drm::gem::shmem::VMap, drm::sched, macros::versions, - sync::{smutex::Mutex, Arc}, + sync::{Arc, Mutex}, uapi, }; diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index bb8db1c1dcfb3f..2d007af1450e84 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -21,8 +21,9 @@ use core::sync::atomic::Ordering; use kernel::dma_fence::RawDmaFence; use kernel::drm::sched::Job; use kernel::io_buffer::IoBufferReader; +use kernel::new_mutex; use kernel::prelude::*; -use kernel::sync::{smutex::Mutex, Arc}; +use kernel::sync::Arc; use kernel::uapi; use kernel::user_ptr::UserSlicePtr; @@ -474,7 +475,7 @@ impl super::Queue::ver { } result.result.tvb_size_bytes = buffer.size() as u64; - Arc::try_new(Mutex::new(result)) + Arc::pin_init(new_mutex!(result, "render result")) }) .transpose()?; diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index 6493111643fe75..bd2da42245cbc4 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -19,7 +19,8 @@ use core::ops::{Deref, DerefMut}; use kernel::{ error::{code::*, Result}, prelude::*, - sync::{Arc, CondVar, Mutex, UniqueArc}, + str::CStr, + sync::{Arc, CondVar, LockClassKey, Mutex}, }; /// Trait representing a single item within a slot. @@ -109,8 +110,11 @@ struct SlotAllocatorInner { } /// A single slot allocator instance. +#[pin_data] struct SlotAllocatorOuter { + #[pin] inner: Mutex>, + #[pin] cond: CondVar, } @@ -127,6 +131,9 @@ impl SlotAllocator { num_slots: u32, mut data: T::Data, mut constructor: impl FnMut(&mut T::Data, u32) -> T, + name: &'static CStr, + lock_key1: &'static LockClassKey, + lock_key2: &'static LockClassKey, ) -> Result> { let mut slots = Vec::try_with_capacity(num_slots as usize)?; @@ -147,22 +154,14 @@ impl SlotAllocator { drop_count: 0, }; - let mut alloc = Pin::from(UniqueArc::try_new(SlotAllocatorOuter { - // SAFETY: `condvar_init!` is called below. - cond: unsafe { CondVar::new() }, + let alloc = Arc::pin_init(pin_init!(SlotAllocatorOuter { // SAFETY: `mutex_init!` is called below. - inner: unsafe { Mutex::new(inner) }, - })?); - - // SAFETY: `cond` is pinned when `alloc` is. - let pinned = unsafe { alloc.as_mut().map_unchecked_mut(|s| &mut s.cond) }; - kernel::condvar_init!(pinned, "SlotAllocator::cond"); - - // SAFETY: `inner` is pinned when `alloc` is. - let pinned = unsafe { alloc.as_mut().map_unchecked_mut(|s| &mut s.inner) }; - kernel::mutex_init!(pinned, "SlotAllocator::inner"); + inner <- Mutex::new(inner, name, lock_key1), + // SAFETY: `condvar_init!` is called below. + cond <- CondVar::new(name, lock_key2), + }))?; - Ok(SlotAllocator(alloc.into())) + Ok(SlotAllocator(alloc)) } /// Calls a callback on the inner data associated with this allocator, taking the lock. diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 022cd2e39e6498..b121a25bc9b559 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -26,7 +26,10 @@ use core::sync::atomic::Ordering; use kernel::{ error::code::*, prelude::*, - sync::{Arc, Guard, Mutex, UniqueArc}, + sync::{ + lock::{mutex::MutexBackend, Guard}, + Arc, Mutex, + }, uapi, }; @@ -214,8 +217,10 @@ struct WorkQueueInner { /// An instance of a work queue. #[versions(AGX)] +#[pin_data] pub(crate) struct WorkQueue { info_pointer: GpuWeakPointer, + #[pin] inner: Mutex, } @@ -255,7 +260,7 @@ pub(crate) struct Job { #[versions(AGX)] pub(crate) struct JobSubmission<'a> { - inner: Option>>, + inner: Option>, wptr: u32, event_count: usize, command_count: usize, @@ -622,21 +627,18 @@ impl WorkQueue::ver { last_submitted: None, }; - let mut queue = Pin::from(UniqueArc::try_new(Self { - info_pointer: inner.info.weak_pointer(), - // SAFETY: `mutex_init!` is called below. - inner: unsafe { Mutex::new(inner) }, - })?); - - // SAFETY: `inner` is pinned when `queue` is. - let pinned = unsafe { queue.as_mut().map_unchecked_mut(|s| &mut s.inner) }; - match pipe_type { - PipeType::Vertex => kernel::mutex_init!(pinned, "WorkQueue::inner (Vertex)"), - PipeType::Fragment => kernel::mutex_init!(pinned, "WorkQueue::inner (Fragment)"), - PipeType::Compute => kernel::mutex_init!(pinned, "WorkQueue::inner (Compute)"), - } + let info_pointer = inner.info.weak_pointer(); + + let mutex_init = match pipe_type { + PipeType::Vertex => kernel::new_mutex!(inner, "WorkQueue::inner (Vertex)"), + PipeType::Fragment => kernel::new_mutex!(inner, "WorkQueue::inner (Fragment)"), + PipeType::Compute => kernel::new_mutex!(inner, "WorkQueue::inner (Compute)"), + }; - Ok(queue.into()) + Arc::pin_init(pin_init!(Self { + info_pointer, + inner <- mutex_init, + })) } pub(crate) fn event_info(&self) -> Option { @@ -881,10 +883,3 @@ impl WorkQueue for WorkQueue::ver { } } } - -#[versions(AGX)] -impl Drop for WorkQueue::ver { - fn drop(&mut self) { - mod_pr_debug!("WorkQueue({:?}): Dropping\n", self.inner.lock().pipe_type); - } -} From d6504f53eae529f9095a1deb47f8385e98f78ab2 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 28 Apr 2023 21:00:25 +0900 Subject: [PATCH 567/635] drm/asahi: Make GpuManager pinned This lets us get rid of a bunch of Pin> and Box<> indirection, though we still need temporary boxes due to borrow checker issues during initialization. Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/gpu.rs | 196 ++++++++++++++++++++--------------- 1 file changed, 115 insertions(+), 81 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 7226efede039fa..a66330d45ab21e 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -188,29 +188,34 @@ pub(crate) struct SequenceIDs { /// Top-level GPU manager that owns all the global state relevant to the driver instance. #[versions(AGX)] +#[pin_data] pub(crate) struct GpuManager { dev: AsahiDevice, cfg: &'static hw::HwConfig, - dyncfg: Box, - pub(crate) initdata: Box>, - uat: Box, + dyncfg: hw::DynConfig, + pub(crate) initdata: fw::types::GpuObject, + uat: mmu::Uat, crashed: AtomicBool, - alloc: Pin>>, + #[pin] + alloc: Mutex, io_mappings: Vec, - rtkit: Pin>>>>, - rx_channels: Pin>>, - tx_channels: Pin>>, - fwctl_channel: Pin>>, + #[pin] + rtkit: Mutex>>, + #[pin] + rx_channels: Mutex, + #[pin] + tx_channels: Mutex, + #[pin] + fwctl_channel: Mutex, pipes: PipeChannels::ver, event_manager: Arc, buffer_mgr: buffer::BufferManager, ids: SequenceIDs, - #[allow(clippy::type_complexity)] - garbage_work: Pin>>>>, + #[pin] + garbage_work: Mutex>>, #[allow(clippy::vec_box)] - #[allow(clippy::type_complexity)] - garbage_contexts: - Pin>>>>>, + #[pin] + garbage_contexts: Mutex>>>, } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -424,9 +429,12 @@ impl GpuManager::ver { let p_fwctl = fwctl.to_raw(); core::mem::drop(fwctl); - mgr.initdata.fw_status.with_mut(|raw, _inner| { - raw.fwctl_channel = p_fwctl; - }); + mgr.as_mut() + .initdata_mut() + .fw_status + .with_mut(|raw, _inner| { + raw.fwctl_channel = p_fwctl; + }); } { @@ -442,14 +450,17 @@ impl GpuManager::ver { let p_fwlog_buf = rxc.fw_log.get_buf(); core::mem::drop(rxc); - mgr.initdata.runtime_pointers.with_mut(|raw, _inner| { - raw.device_control = p_device_control; - raw.event = p_event; - raw.fw_log = p_fw_log; - raw.ktrace = p_ktrace; - raw.stats = p_stats; - raw.fwlog_buf = Some(p_fwlog_buf); - }); + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + raw.device_control = p_device_control; + raw.event = p_event; + raw.fw_log = p_fw_log; + raw.ktrace = p_ktrace; + raw.stats = p_stats; + raw.fwlog_buf = Some(p_fwlog_buf); + }); } let mut p_pipes: Vec = Vec::new(); @@ -468,17 +479,20 @@ impl GpuManager::ver { })?; } - mgr.initdata.runtime_pointers.with_mut(|raw, _inner| { - for (i, p) in p_pipes.into_iter().enumerate() { - raw.pipes[i].vtx = p.vtx; - raw.pipes[i].frag = p.frag; - raw.pipes[i].comp = p.comp; - } - }); + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + for (i, p) in p_pipes.into_iter().enumerate() { + raw.pipes[i].vtx = p.vtx; + raw.pipes[i].frag = p.frag; + raw.pipes[i].comp = p.comp; + } + }); for (i, map) in cfg.io_mappings.iter().enumerate() { if let Some(map) = map.as_ref() { - mgr.iomap(i, map)?; + Self::iomap(&mut mgr, i, map)?; } } @@ -491,11 +505,15 @@ impl GpuManager::ver { .kernel_vm() .map_io(base as usize, size, mmu::PROT_FW_SHARED_RW)?; - mgr.initdata.runtime_pointers.hwdata_b.with_mut(|raw, _| { - raw.sgx_sram_ptr = U64(mapping.iova() as u64); - }); + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.sgx_sram_ptr = U64(mapping.iova() as u64); + }); - mgr.io_mappings.try_push(mapping)?; + mgr.as_mut().io_mappings_mut().try_push(mapping)?; } let mgr = Arc::from(mgr); @@ -512,6 +530,20 @@ impl GpuManager::ver { Ok(mgr) } + /// Return a mutable reference to the initdata member + fn initdata_mut( + self: Pin<&mut Self>, + ) -> &mut fw::types::GpuObject { + // SAFETY: initdata does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().initdata } + } + + /// Return a mutable reference to the io_mappings member + fn io_mappings_mut(self: Pin<&mut Self>) -> &mut Vec { + // SAFETY: io_mappings does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().io_mappings } + } + /// Build the entire GPU InitData structure tree and return it as a boxed GpuObject. fn make_initdata( cfg: &'static hw::HwConfig, @@ -549,7 +581,7 @@ impl GpuManager::ver { mut alloc: KernelAllocators, event_manager: Arc, initdata: Box>, - ) -> Result> { + ) -> Result>> { let mut pipes = PipeChannels::ver { vtx: Vec::new(), frag: Vec::new(), @@ -571,48 +603,42 @@ impl GpuManager::ver { ))?)?; } - let fwctl_channel = Box::pin_init(new_mutex!( - channel::FwCtlChannel::new(dev, &mut alloc)?, - "fwctl_channel" - ))?; - let rx_channels = Box::pin_init(new_mutex!( - RxChannels::ver { - event: channel::EventChannel::new(dev, &mut alloc, event_manager.clone())?, - fw_log: channel::FwLogChannel::new(dev, &mut alloc)?, - ktrace: channel::KTraceChannel::new(dev, &mut alloc)?, - stats: channel::StatsChannel::ver::new(dev, &mut alloc)?, - }, - "rx_channels" - ))?; - let tx_channels = Box::pin_init(new_mutex!( - TxChannels::ver { - device_control: channel::DeviceControlChannel::ver::new(dev, &mut alloc)?, - }, - "tx_channels" - ))?; + let fwctl_channel = channel::FwCtlChannel::new(dev, &mut alloc)?; - let alloc = Box::pin_init(new_mutex!(alloc, "alloc"))?; + let event_manager_clone = event_manager.clone(); + let alloc_ref = &mut alloc; + let rx_channels = Box::init(try_init!(RxChannels::ver { + event: channel::EventChannel::new(dev, alloc_ref, event_manager_clone)?, + fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, + ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, + stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, + }))?; - let x = UniqueArc::try_new(GpuManager::ver { + let alloc_ref = &mut alloc; + let tx_channels = Box::init(try_init!(TxChannels::ver { + device_control: channel::DeviceControlChannel::ver::new(dev, alloc_ref)?, + }))?; + + let x = UniqueArc::pin_init(try_pin_init!(GpuManager::ver { dev: dev.clone(), cfg, - dyncfg, - initdata, - uat, + dyncfg: *dyncfg, + initdata: *initdata, + uat: *uat, io_mappings: Vec::new(), - rtkit: Box::pin_init(new_mutex!(None, "rtkit"))?, + rtkit <- new_mutex!(None, "rtkit"), crashed: AtomicBool::new(false), event_manager, - alloc, - fwctl_channel, - rx_channels, - tx_channels, + alloc <- new_mutex!(alloc, "alloc"), + fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), + rx_channels <- new_mutex!(*rx_channels, "rx_channels"), + tx_channels <- new_mutex!(*tx_channels, "tx_channels"), pipes, buffer_mgr: buffer::BufferManager::new()?, ids: Default::default(), - garbage_work: Box::pin_init(new_mutex!(Vec::new(), "garbage_work"))?, - garbage_contexts: Box::pin_init(new_mutex!(Vec::new(), "garbage_contexts"))?, - })?; + garbage_work <- new_mutex!(Vec::new(), "garbage_work"), + garbage_contexts <- new_mutex!(Vec::new(), "garbage_contexts"), + }))?; Ok(x) } @@ -725,11 +751,15 @@ impl GpuManager::ver { /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified /// index. - fn iomap(&mut self, index: usize, map: &hw::IOMapping) -> Result { + fn iomap( + this: &mut Pin>, + index: usize, + map: &hw::IOMapping, + ) -> Result { let off = map.base & mmu::UAT_PGMSK; let base = map.base - off; let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; - let mapping = self.uat.kernel_vm().map_io( + let mapping = this.uat.kernel_vm().map_io( base, end - base, if map.writable { @@ -739,17 +769,21 @@ impl GpuManager::ver { }, )?; - self.initdata.runtime_pointers.hwdata_b.with_mut(|raw, _| { - raw.io_mappings[index] = fw::initdata::raw::IOMapping { - phys_addr: U64(map.base as u64), - virt_addr: U64((mapping.iova() + off) as u64), - size: map.size as u32, - range_size: map.range_size as u32, - readwrite: U64(map.writable as u64), - }; - }); + this.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.io_mappings[index] = fw::initdata::raw::IOMapping { + phys_addr: U64(map.base as u64), + virt_addr: U64((mapping.iova() + off) as u64), + size: map.size as u32, + range_size: map.range_size as u32, + readwrite: U64(map.writable as u64), + }; + }); - self.io_mappings.try_push(mapping)?; + this.as_mut().io_mappings_mut().try_push(mapping)?; Ok(()) } From 6a960c6fc5bc55390564b40a7b377a9ecfe29f67 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Thu, 13 Apr 2023 10:37:50 +0200 Subject: [PATCH 568/635] rust: macros: fix usage of `#[allow]` in `quote!` When using `quote!` as part of an expression that was not the last one in a function, the `#[allow(clippy::vec_init_then_push)]` attribute would be present on an expression, which is not allowed. This patch refactors that part of the macro to use a statement instead. Signed-off-by: Benno Lossin --- rust/macros/quote.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs index c8e08b3c1e4cfb..dddbb4e6f4cb88 100644 --- a/rust/macros/quote.rs +++ b/rust/macros/quote.rs @@ -39,12 +39,14 @@ impl ToTokens for TokenStream { /// [`quote_spanned!`](https://docs.rs/quote/latest/quote/macro.quote_spanned.html) macro from the /// `quote` crate but provides only just enough functionality needed by the current `macros` crate. macro_rules! quote_spanned { - ($span:expr => $($tt:tt)*) => { - #[allow(clippy::vec_init_then_push)] - { - let mut tokens = ::std::vec::Vec::new(); - let span = $span; - quote_spanned!(@proc tokens span $($tt)*); + ($span:expr => $($tt:tt)*) => {{ + let mut tokens; + #[allow(clippy::vec_init_then_push)] + { + tokens = ::std::vec::Vec::new(); + let span = $span; + quote_spanned!(@proc tokens span $($tt)*); + } ::proc_macro::TokenStream::from_iter(tokens) }}; (@proc $v:ident $span:ident) => {}; From 1b88918cdff287cb1032ee9df2570a4e88cc5de3 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sun, 23 Apr 2023 19:29:01 +0200 Subject: [PATCH 569/635] rust: macros: refactor generics parsing of `#[pin_data]` into its own function Other macros might also want to parse generics. Additionally this makes the code easier to read, as the next commit will introduce more code in `#[pin_data]`. Also add more comments to explain how parsing generics work. Signed-off-by: Benno Lossin --- rust/macros/helpers.rs | 86 ++++++++++++++++++++++++++++++++++++++++- rust/macros/pin_data.rs | 70 +++++---------------------------- 2 files changed, 94 insertions(+), 62 deletions(-) diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs index 99f9de209c8451..e95a19b2f52da2 100644 --- a/rust/macros/helpers.rs +++ b/rust/macros/helpers.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use proc_macro::{token_stream, Group, TokenTree}; +use proc_macro::{token_stream, Group, Punct, Spacing, TokenStream, TokenTree}; pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option { if let Some(TokenTree::Ident(ident)) = it.next() { @@ -89,3 +89,87 @@ pub(crate) fn get_string(it: &mut token_stream::IntoIter, expected_name: &str) - assert_eq!(expect_punct(it), ','); string } + +pub(crate) struct Generics { + pub(crate) impl_generics: Vec, + pub(crate) ty_generics: Vec, +} + +/// Parses the given `TokenStream` into `Generics` and the rest. +/// +/// The generics are not present in the rest, but a where clause might remain. +pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec) { + // `impl_generics`, the declared generics with their bounds. + let mut impl_generics = vec![]; + // Only the names of the generics, without any bounds. + let mut ty_generics = vec![]; + // Tokens not related to the generics e.g. the `where` token and definition. + let mut rest = vec![]; + // The current level of `<`. + let mut nesting = 0; + let mut toks = input.into_iter(); + // If we are at the beginning of a generic parameter. + let mut at_start = true; + for tt in &mut toks { + match tt.clone() { + TokenTree::Punct(p) if p.as_char() == '<' => { + if nesting >= 1 { + // This is inside of the generics and part of some bound. + impl_generics.push(tt); + } + nesting += 1; + } + TokenTree::Punct(p) if p.as_char() == '>' => { + // This is a parsing error, so we just end it here. + if nesting == 0 { + break; + } else { + nesting -= 1; + if nesting >= 1 { + // We are still inside of the generics and part of some bound. + impl_generics.push(tt); + } + if nesting == 0 { + break; + } + } + } + tt => { + if nesting == 1 { + // Here depending on the token, it might be a generic variable name. + match &tt { + // Ignore const. + TokenTree::Ident(i) if i.to_string() == "const" => {} + TokenTree::Ident(_) if at_start => { + ty_generics.push(tt.clone()); + // We also already push the `,` token, this makes it easier to append + // generics. + ty_generics.push(TokenTree::Punct(Punct::new(',', Spacing::Alone))); + at_start = false; + } + TokenTree::Punct(p) if p.as_char() == ',' => at_start = true, + // Lifetimes begin with `'`. + TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { + ty_generics.push(tt.clone()); + } + _ => {} + } + } + if nesting >= 1 { + impl_generics.push(tt); + } else if nesting == 0 { + // If we haven't entered the generics yet, we still want to keep these tokens. + rest.push(tt); + } + } + } + } + rest.extend(toks); + ( + Generics { + impl_generics, + ty_generics, + }, + rest, + ) +} diff --git a/rust/macros/pin_data.rs b/rust/macros/pin_data.rs index 954149d771813c..c593b05d9e8c18 100644 --- a/rust/macros/pin_data.rs +++ b/rust/macros/pin_data.rs @@ -1,71 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -use proc_macro::{Punct, Spacing, TokenStream, TokenTree}; +use crate::helpers::{parse_generics, Generics}; +use proc_macro::TokenStream; pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { // This proc-macro only does some pre-parsing and then delegates the actual parsing to // `kernel::__pin_data!`. - // - // In here we only collect the generics, since parsing them in declarative macros is very - // elaborate. We also do not need to analyse their structure, we only need to collect them. - // `impl_generics`, the declared generics with their bounds. - let mut impl_generics = vec![]; - // Only the names of the generics, without any bounds. - let mut ty_generics = vec![]; - // Tokens not related to the generics e.g. the `impl` token. - let mut rest = vec![]; - // The current level of `<`. - let mut nesting = 0; - let mut toks = input.into_iter(); - // If we are at the beginning of a generic parameter. - let mut at_start = true; - for tt in &mut toks { - match tt.clone() { - TokenTree::Punct(p) if p.as_char() == '<' => { - if nesting >= 1 { - impl_generics.push(tt); - } - nesting += 1; - } - TokenTree::Punct(p) if p.as_char() == '>' => { - if nesting == 0 { - break; - } else { - nesting -= 1; - if nesting >= 1 { - impl_generics.push(tt); - } - if nesting == 0 { - break; - } - } - } - tt => { - if nesting == 1 { - match &tt { - TokenTree::Ident(i) if i.to_string() == "const" => {} - TokenTree::Ident(_) if at_start => { - ty_generics.push(tt.clone()); - ty_generics.push(TokenTree::Punct(Punct::new(',', Spacing::Alone))); - at_start = false; - } - TokenTree::Punct(p) if p.as_char() == ',' => at_start = true, - TokenTree::Punct(p) if p.as_char() == '\'' && at_start => { - ty_generics.push(tt.clone()); - } - _ => {} - } - } - if nesting >= 1 { - impl_generics.push(tt); - } else if nesting == 0 { - rest.push(tt); - } - } - } - } - rest.extend(toks); + let ( + Generics { + impl_generics, + ty_generics, + }, + mut rest, + ) = parse_generics(input); // This should be the body of the struct `{...}`. let last = rest.pop(); quote!(::kernel::__pin_data! { From 0e20ebe84fdc1147cec5fa0c71112494dbf0b693 Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Sun, 23 Apr 2023 19:31:23 +0200 Subject: [PATCH 570/635] rust: macros: replace Self with the concrete type in #[pin_data] When using `#[pin_data]` on a struct that used `Self` in the field types, a type error would be emitted when trying to use `pin_init!`. Since an internal type would be referenced by `Self` instead of the defined struct. This patch fixes this issue by replacing all occurrences of `Self` in the `#[pin_data]` macro with the concrete type circumventing the issue. Since rust allows type definitions inside of blocks, which are expressions, the macro also checks for these and emits a compile error when it finds `trait`, `enum`, `union`, `struct` or `impl`. These keywords allow creating new `Self` contexts, which conflicts with the current implementation of replacing every `Self` ident. If these were allowed, some `Self` idents would be replaced incorrectly. Signed-off-by: Benno Lossin Reported-by: Alice Ryhl --- rust/macros/pin_data.rs | 108 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/rust/macros/pin_data.rs b/rust/macros/pin_data.rs index c593b05d9e8c18..6d58cfda9872e6 100644 --- a/rust/macros/pin_data.rs +++ b/rust/macros/pin_data.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use crate::helpers::{parse_generics, Generics}; -use proc_macro::TokenStream; +use proc_macro::{Group, Punct, Spacing, TokenStream, TokenTree}; pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { // This proc-macro only does some pre-parsing and then delegates the actual parsing to @@ -12,16 +12,116 @@ pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream { impl_generics, ty_generics, }, - mut rest, + rest, ) = parse_generics(input); + // The struct definition might contain the `Self` type. Since `__pin_data!` will define a new + // type with the same generics and bounds, this poses a problem, since `Self` will refer to the + // new type as opposed to this struct definition. Therefore we have to replace `Self` with the + // concrete name. + + // Errors that occur when replacing `Self` with `struct_name`. + let mut errs = TokenStream::new(); + // The name of the struct with ty_generics. + let struct_name = rest + .iter() + .skip_while(|tt| !matches!(tt, TokenTree::Ident(i) if i.to_string() == "struct")) + .nth(1) + .and_then(|tt| match tt { + TokenTree::Ident(_) => { + let tt = tt.clone(); + let mut res = vec![tt]; + if !ty_generics.is_empty() { + // We add this, so it is maximally compatible with e.g. `Self::CONST` which + // will be replaced by `StructName::<$generics>::CONST`. + res.push(TokenTree::Punct(Punct::new(':', Spacing::Joint))); + res.push(TokenTree::Punct(Punct::new(':', Spacing::Alone))); + res.push(TokenTree::Punct(Punct::new('<', Spacing::Alone))); + res.extend(ty_generics.iter().cloned()); + res.push(TokenTree::Punct(Punct::new('>', Spacing::Alone))); + } + Some(res) + } + _ => None, + }) + .unwrap_or_else(|| { + // If we did not find the name of the struct then we will use `Self` as the replacement + // and add a compile error to ensure it does not compile. + errs.extend( + "::core::compile_error!(\"Could not locate type name.\");" + .parse::() + .unwrap(), + ); + "Self".parse::().unwrap().into_iter().collect() + }); + let impl_generics = impl_generics + .into_iter() + .flat_map(|tt| replace_self_and_deny_type_defs(&struct_name, tt, &mut errs)) + .collect::>(); + let mut rest = rest + .into_iter() + .flat_map(|tt| { + // We ignore top level `struct` tokens, since they would emit a compile error. + if matches!(&tt, TokenTree::Ident(i) if i.to_string() == "struct") { + vec![tt] + } else { + replace_self_and_deny_type_defs(&struct_name, tt, &mut errs) + } + }) + .collect::>(); // This should be the body of the struct `{...}`. let last = rest.pop(); - quote!(::kernel::__pin_data! { + let mut quoted = quote!(::kernel::__pin_data! { parse_input: @args(#args), @sig(#(#rest)*), @impl_generics(#(#impl_generics)*), @ty_generics(#(#ty_generics)*), @body(#last), - }) + }); + quoted.extend(errs); + quoted +} + +/// Replaces `Self` with `struct_name` and errors on `enum`, `trait`, `struct` `union` and `impl` +/// keywords. +/// +/// The error is appended to `errs` to allow normal parsing to continue. +fn replace_self_and_deny_type_defs( + struct_name: &Vec, + tt: TokenTree, + errs: &mut TokenStream, +) -> Vec { + match tt { + TokenTree::Ident(ref i) + if i.to_string() == "enum" + || i.to_string() == "trait" + || i.to_string() == "struct" + || i.to_string() == "union" + || i.to_string() == "impl" => + { + errs.extend( + format!( + "::core::compile_error!(\"Cannot use `{i}` inside of struct definition with \ + `#[pin_data]`.\");" + ) + .parse::() + .unwrap() + .into_iter() + .map(|mut tok| { + tok.set_span(tt.span()); + tok + }), + ); + vec![tt] + } + TokenTree::Ident(i) if i.to_string() == "Self" => struct_name.clone(), + TokenTree::Literal(_) | TokenTree::Punct(_) | TokenTree::Ident(_) => vec![tt], + TokenTree::Group(g) => vec![TokenTree::Group(Group::new( + g.delimiter(), + g.stream() + .into_iter() + .flat_map(|tt| replace_self_and_deny_type_defs(struct_name, tt, errs)) + .collect(), + ))], + } } From 1939919aa0895d7935e7c7be41417813a0bb73ba Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Mon, 24 Apr 2023 10:01:46 +0200 Subject: [PATCH 571/635] rust: init: update macro expansion example in docs Also improve the explaining comments. Signed-off-by: Benno Lossin --- rust/kernel/init/macros.rs | 85 +++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 37 deletions(-) diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index 541cfad1d8beed..00aa4e956c0ae9 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -16,8 +16,9 @@ //! //! We will look at the following example: //! -//! ```rust +//! ```rust,ignore //! # use kernel::init::*; +//! # use core::pin::Pin; //! #[pin_data] //! #[repr(C)] //! struct Bar { @@ -71,11 +72,12 @@ //! //! Here is the definition of `Bar` from our example: //! -//! ```rust +//! ```rust,ignore //! # use kernel::init::*; //! #[pin_data] //! #[repr(C)] //! struct Bar { +//! #[pin] //! t: T, //! pub x: usize, //! } @@ -83,7 +85,7 @@ //! //! This expands to the following code: //! -//! ```rust +//! ```rust,ignore //! // Firstly the normal definition of the struct, attributes are preserved: //! #[repr(C)] //! struct Bar { @@ -116,20 +118,22 @@ //! unsafe fn t( //! self, //! slot: *mut T, -//! init: impl ::kernel::init::Init, +//! // Since `t` is `#[pin]`, this is `PinInit`. +//! init: impl ::kernel::init::PinInit, //! ) -> ::core::result::Result<(), E> { -//! unsafe { ::kernel::init::Init::__init(init, slot) } +//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } //! } //! pub unsafe fn x( //! self, //! slot: *mut usize, +//! // Since `x` is not `#[pin]`, this is `Init`. //! init: impl ::kernel::init::Init, //! ) -> ::core::result::Result<(), E> { //! unsafe { ::kernel::init::Init::__init(init, slot) } //! } //! } //! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct -//! // that we constructed beforehand. +//! // that we constructed above. //! unsafe impl ::kernel::init::__internal::HasPinData for Bar { //! type PinData = __ThePinData; //! unsafe fn __pin_data() -> Self::PinData { @@ -160,6 +164,8 @@ //! struct __Unpin<'__pin, T> { //! __phantom_pin: ::core::marker::PhantomData &'__pin ()>, //! __phantom: ::core::marker::PhantomData) -> Bar>, +//! // Our only `#[pin]` field is `t`. +//! t: T, //! } //! #[doc(hidden)] //! impl<'__pin, T> @@ -193,7 +199,7 @@ //! //! Here is the impl on `Bar` defining the new function: //! -//! ```rust +//! ```rust,ignore //! impl Bar { //! fn new(t: T) -> impl PinInit { //! pin_init!(Self { t, x: 0 }) @@ -203,7 +209,7 @@ //! //! This expands to the following code: //! -//! ```rust +//! ```rust,ignore //! impl Bar { //! fn new(t: T) -> impl PinInit { //! { @@ -232,25 +238,31 @@ //! // that will refer to this struct instead of the one defined above. //! struct __InitOk; //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. -//! unsafe { ::core::ptr::write(&raw mut (*slot).t, t) }; +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; //! // Since initialization could fail later (not in this case, since the error -//! // type is `Infallible`) we will need to drop this field if it fails. This -//! // `DropGuard` will drop the field when it gets dropped and has not yet -//! // been forgotten. We make a reference to it, so users cannot `mem::forget` -//! // it from the initializer, since the name is the same as the field. +//! // type is `Infallible`) we will need to drop this field if there is an +//! // error later. This `DropGuard` will drop the field when it gets dropped +//! // and has not yet been forgotten. We make a reference to it, so users +//! // cannot `mem::forget` it from the initializer, since the name is the same +//! // as the field (including hygiene). //! let t = &unsafe { -//! ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).t) +//! ::kernel::init::__internal::DropGuard::new( +//! ::core::addr_of_mut!((*slot).t), +//! ) //! }; //! // Expansion of `x: 0,`: //! // Since this can be an arbitrary expression we cannot place it inside of //! // the `unsafe` block, so we bind it here. //! let x = 0; -//! unsafe { ::core::ptr::write(&raw mut (*slot).x, x) }; +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; +//! // We again create a `DropGuard`. //! let x = &unsafe { -//! ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).x) +//! ::kernel::init::__internal::DropGuard::new( +//! ::core::addr_of_mut!((*slot).x), +//! ) //! }; //! -//! // Here we use the type checker to ensuer that every field has been +//! // Here we use the type checker to ensure that every field has been //! // initialized exactly once, since this is `if false` it will never get //! // executed, but still type-checked. //! // Additionally we abuse `slot` to automatically infer the correct type for @@ -272,7 +284,7 @@ //! }; //! } //! // Since initialization has successfully completed, we can now forget the -//! // guards. +//! // guards. This is not `mem::forget`, since we only have `&DropGuard`. //! unsafe { ::kernel::init::__internal::DropGuard::forget(t) }; //! unsafe { ::kernel::init::__internal::DropGuard::forget(x) }; //! } @@ -280,7 +292,7 @@ //! // `__InitOk` that we need to return. //! Ok(__InitOk) //! }); -//! // Change the return type of the closure. +//! // Change the return type from `__InitOk` to `()`. //! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { //! init(slot).map(|__InitOk| ()) //! }; @@ -299,7 +311,7 @@ //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the //! differences/new things in the expansion of the `Foo` definition: //! -//! ```rust +//! ```rust,ignore //! #[pin_data(PinnedDrop)] //! struct Foo { //! a: usize, @@ -310,7 +322,7 @@ //! //! This expands to the following code: //! -//! ```rust +//! ```rust,ignore //! struct Foo { //! a: usize, //! b: Bar, @@ -330,8 +342,6 @@ //! unsafe fn b( //! self, //! slot: *mut Bar, -//! // Note that this is `PinInit` instead of `Init`, this is because `b` is -//! // structurally pinned, as marked by the `#[pin]` attribute. //! init: impl ::kernel::init::PinInit, E>, //! ) -> ::core::result::Result<(), E> { //! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } @@ -359,14 +369,13 @@ //! struct __Unpin<'__pin> { //! __phantom_pin: ::core::marker::PhantomData &'__pin ()>, //! __phantom: ::core::marker::PhantomData Foo>, -//! // Since this field is `#[pin]`, it is listed here. //! b: Bar, //! } //! #[doc(hidden)] //! impl<'__pin> ::core::marker::Unpin for Foo where __Unpin<'__pin>: ::core::marker::Unpin {} //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like -//! // before, instead we implement it here and delegate to `PinnedDrop`. +//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. //! impl ::core::ops::Drop for Foo { //! fn drop(&mut self) { //! // Since we are getting dropped, no one else has a reference to `self` and thus we @@ -388,7 +397,7 @@ //! //! Here is the `PinnedDrop` impl for `Foo`: //! -//! ```rust +//! ```rust,ignore //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { @@ -399,7 +408,7 @@ //! //! This expands to the following code: //! -//! ```rust +//! ```rust,ignore //! // `unsafe`, full path and the token parameter are added, everything else stays the same. //! unsafe impl ::kernel::init::PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { @@ -410,10 +419,10 @@ //! //! ## `pin_init!` on `Foo` //! -//! Since we already took a look at `pin_init!` on `Bar`, this section will only explain the -//! differences/new things in the expansion of `pin_init!` on `Foo`: +//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion +//! of `pin_init!` on `Foo`: //! -//! ```rust +//! ```rust,ignore //! let a = 42; //! let initializer = pin_init!(Foo { //! a, @@ -423,7 +432,7 @@ //! //! This expands to the following code: //! -//! ```rust +//! ```rust,ignore //! let a = 42; //! let initializer = { //! struct __InitOk; @@ -438,13 +447,15 @@ //! >(data, move |slot| { //! { //! struct __InitOk; -//! unsafe { ::core::ptr::write(&raw mut (*slot).a, a) }; -//! let a = &unsafe { ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).a) }; +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; +//! let a = &unsafe { +//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) +//! }; //! let b = Bar::new(36); -//! // Here we use `data` to access the correct field and require that `b` is of type -//! // `PinInit, Infallible>`. -//! unsafe { data.b(&raw mut (*slot).b, b)? }; -//! let b = &unsafe { ::kernel::init::__internal::DropGuard::new(&raw mut (*slot).b) }; +//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; +//! let b = &unsafe { +//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) +//! }; //! //! #[allow(unreachable_code, clippy::diverging_sub_expression)] //! if false { From f45fb0ecf4f6e7bee5a6dfb6d682c1f071400cfd Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Fri, 5 May 2023 08:12:33 +0200 Subject: [PATCH 572/635] rust: add derive macro for `Zeroable` Add a derive proc-macro for the `Zeroable` trait. The macro supports structs where every field implements the `Zeroable` trait. This way `unsafe` implementations can be avoided. Suggested-by: Asahi Lina Signed-off-by: Benno Lossin --- rust/kernel/init/macros.rs | 28 ++++++++++++++++++++++++++++ rust/kernel/prelude.rs | 2 +- rust/macros/lib.rs | 20 ++++++++++++++++++++ rust/macros/quote.rs | 6 ++++++ rust/macros/zeroable.rs | 26 ++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 rust/macros/zeroable.rs diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index 00aa4e956c0ae9..ef0755b6116216 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -980,3 +980,31 @@ macro_rules! __pin_data { } }; } + +#[doc(hidden)] +#[macro_export] +macro_rules! __derive_zeroable { + (parse_input: + @sig( + $(#[$($struct_attr:tt)*])* + $vis:vis struct $name:ident + $(where $($whr:tt)*)? + ), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @body({ + $( + $(#[$($field_attr:tt)*])* + $field:ident : $field_ty:ty + ),* $(,)? + }), + ) => { + // SAFETY: every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl<$($impl_generics)*> $crate::Zeroable for $name<$($ty_generics)*> + where + $($field_ty: $crate::Zeroable,)* + $($($whr)*)? + {} + }; +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index e62b4d0045a9c2..e1f16239b3caea 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -18,7 +18,7 @@ pub use core::pin::Pin; pub use alloc::{boxed::Box, vec::Vec}; #[doc(no_inline)] -pub use macros::{module, pin_data, pinned_drop, vtable}; +pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; pub use super::build_assert; diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index e22915b9c45454..0e539590047c67 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -11,6 +11,7 @@ mod pin_data; mod pinned_drop; mod versions; mod vtable; +mod zeroable; use proc_macro::TokenStream; @@ -253,3 +254,22 @@ pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream { pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream { pinned_drop::pinned_drop(args, input) } + +/// Derives the [`Zeroable`] trait for the given struct. +/// +/// This can only be used for structs where every field implements the [`Zeroable`] trait. +/// +/// # Examples +/// +/// ```rust +/// #[derive(Zeroable)] +/// pub struct DriverData { +/// id: i64, +/// buf_ptr: *mut u8, +/// len: usize, +/// } +/// ``` +#[proc_macro_derive(Zeroable)] +pub fn derive_zeroable(input: TokenStream) -> TokenStream { + zeroable::derive(input) +} diff --git a/rust/macros/quote.rs b/rust/macros/quote.rs index dddbb4e6f4cb88..b76c198a4ed503 100644 --- a/rust/macros/quote.rs +++ b/rust/macros/quote.rs @@ -124,6 +124,12 @@ macro_rules! quote_spanned { )); quote_spanned!(@proc $v $span $($tt)*); }; + (@proc $v:ident $span:ident ; $($tt:tt)*) => { + $v.push(::proc_macro::TokenTree::Punct( + ::proc_macro::Punct::new(';', ::proc_macro::Spacing::Alone) + )); + quote_spanned!(@proc $v $span $($tt)*); + }; (@proc $v:ident $span:ident $id:ident $($tt:tt)*) => { $v.push(::proc_macro::TokenTree::Ident(::proc_macro::Ident::new(stringify!($id), $span))); quote_spanned!(@proc $v $span $($tt)*); diff --git a/rust/macros/zeroable.rs b/rust/macros/zeroable.rs new file mode 100644 index 00000000000000..82968b5dcca11f --- /dev/null +++ b/rust/macros/zeroable.rs @@ -0,0 +1,26 @@ +use crate::{ + helpers::{parse_generics, Generics}, + quote, +}; +use proc_macro::TokenStream; + +pub(crate) fn derive(input: TokenStream) -> TokenStream { + let ( + Generics { + impl_generics, + ty_generics, + }, + mut rest, + ) = parse_generics(input); + // This should be the body of the struct `{...}`. + let last = rest.pop(); + quote! { + ::kernel::__derive_zeroable!( + parse_input: + @sig(#(#rest)*), + @impl_generics(#(#impl_generics)*), + @ty_generics(#(#ty_generics)*), + @body(#last), + ); + } +} From 139b3351b36018b08fa807654e39160984ec9dbc Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Fri, 5 May 2023 10:16:33 +0200 Subject: [PATCH 573/635] rust: init: make guards in the init macros hygienic Use hygienic identifiers for the guards instead of the field names. This makes the init macros feel more like normal struct initializers, since assigning identifiers with the name of a field does not create conflicts. Suggested-by: Asahi Lina Signed-off-by: Benno Lossin --- rust/kernel/init.rs | 119 ++++++++++++--------------------- rust/kernel/init/__internal.rs | 25 ++----- rust/macros/zeroable.rs | 5 +- 3 files changed, 47 insertions(+), 102 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 4ebfb08dab1154..26cf3b056d95d8 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -205,7 +205,6 @@ use crate::{ use alloc::boxed::Box; use core::{ alloc::AllocError, - cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, @@ -636,6 +635,7 @@ macro_rules! try_pin_init { $crate::try_pin_init!(init_slot: @data(data), @slot(slot), + @guards(), @munch_fields($($fields)*,), ); // We use unreachable code to ensure that all fields have been mentioned exactly @@ -650,10 +650,6 @@ macro_rules! try_pin_init { @acc(), ); } - // Forget all guards, since initialization was a success. - $crate::try_pin_init!(forget_guards: - @munch_fields($($fields)*,), - ); } Ok(__InitOk) } @@ -667,13 +663,17 @@ macro_rules! try_pin_init { (init_slot: @data($data:ident), @slot($slot:ident), + @guards($($guards:ident,)*), @munch_fields($(,)?), ) => { - // Endpoint of munching, no fields are left. + // Endpoint of munching, no fields are left. If execution reaches this point, all fields + // have been initialized. Therefore we can now dismiss the guards by forgetting them. + $(::core::mem::forget($guards);)* }; (init_slot: @data($data:ident), @slot($slot:ident), + @guards($($guards:ident,)*), // In-place initialization syntax. @munch_fields($field:ident <- $val:expr, $($rest:tt)*), ) => { @@ -686,22 +686,24 @@ macro_rules! try_pin_init { unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; // Create the drop guard. // - // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. + // Users cannot access this field due to macro hygiene. // // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { + let guard = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::try_pin_init!(init_slot: @data($data), @slot($slot), + @guards(guard, $($guards,)*), @munch_fields($($rest)*), ); }; (init_slot: @data($data:ident), @slot($slot:ident), + @guards($($guards:ident,)*), // Direct value init, this is safe for every field. @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), ) => { @@ -712,16 +714,17 @@ macro_rules! try_pin_init { unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; // Create the drop guard: // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. + // Users cannot access this field due to macro hygiene. // // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { + let guard = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::try_pin_init!(init_slot: @data($data), @slot($slot), + @guards(guard, $($guards,)*), @munch_fields($($rest)*), ); }; @@ -766,29 +769,6 @@ macro_rules! try_pin_init { @acc($($acc)* $field: ::core::panic!(),), ); }; - (forget_guards: - @munch_fields($(,)?), - ) => { - // Munching finished. - }; - (forget_guards: - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_pin_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; - (forget_guards: - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_pin_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; } /// Construct an in-place initializer for `struct`s. @@ -903,6 +883,7 @@ macro_rules! try_init { // Initialize every field. $crate::try_init!(init_slot: @slot(slot), + @guards(), @munch_fields($($fields)*,), ); // We use unreachable code to ensure that all fields have been mentioned exactly @@ -917,10 +898,6 @@ macro_rules! try_init { @acc(), ); } - // Forget all guards, since initialization was a success. - $crate::try_init!(forget_guards: - @munch_fields($($fields)*,), - ); } Ok(__InitOk) } @@ -933,57 +910,68 @@ macro_rules! try_init { }}; (init_slot: @slot($slot:ident), + @guards($($guards:ident,)*), @munch_fields( $(,)?), ) => { - // Endpoint of munching, no fields are left. + // Endpoint of munching, no fields are left. If execution reaches this point, all fields + // have been initialized. Therefore we can now dismiss the guards by forgetting them. + $(::core::mem::forget($guards);)* }; (init_slot: @slot($slot:ident), + @guards($($guards:ident,)*), @munch_fields($field:ident <- $val:expr, $($rest:tt)*), ) => { - let $field = $val; - // Call the initializer. - // - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - unsafe { - $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; + { + let $field = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + unsafe { + $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; + } } // Create the drop guard. // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. + // Users cannot access this field due to macro hygiene. // // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { + let guard = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::try_init!(init_slot: @slot($slot), + @guards(guard, $($guards,)*), @munch_fields($($rest)*), ); }; (init_slot: @slot($slot:ident), + @guards($($guards:ident,)*), // Direct value init. @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), ) => { - $(let $field = $val;)? - // Call the initializer. - // - // SAFETY: The memory at `slot` is uninitialized. - unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + { + $(let $field = $val;)? + // Call the initializer. + // + // SAFETY: The memory at `slot` is uninitialized. + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + } // Create the drop guard. // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. + // Users cannot access this field due to macro hygiene. // // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { + let guard = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::try_init!(init_slot: @slot($slot), + @guards(guard, $($guards,)*), @munch_fields($($rest)*), ); }; @@ -1028,29 +1016,6 @@ macro_rules! try_init { @acc($($acc)*$field: ::core::panic!(),), ); }; - (forget_guards: - @munch_fields($(,)?), - ) => { - // Munching finished. - }; - (forget_guards: - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; - (forget_guards: - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; } /// A pin-initializer for the type `T`. diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 44751fb62b51a2..7abd1fb65e412d 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -174,7 +174,6 @@ impl StackInit { /// Can be forgotten to prevent the drop. pub struct DropGuard { ptr: *mut T, - do_drop: Cell, } impl DropGuard { @@ -190,32 +189,16 @@ impl DropGuard { /// - will not be dropped by any other means. #[inline] pub unsafe fn new(ptr: *mut T) -> Self { - Self { - ptr, - do_drop: Cell::new(true), - } - } - - /// Prevents this guard from dropping the supplied pointer. - /// - /// # Safety - /// - /// This function is unsafe in order to prevent safe code from forgetting this guard. It should - /// only be called by the macros in this module. - #[inline] - pub unsafe fn forget(&self) { - self.do_drop.set(false); + Self { ptr } } } impl Drop for DropGuard { #[inline] fn drop(&mut self) { - if self.do_drop.get() { - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function - // ensuring that this operation is safe. - unsafe { ptr::drop_in_place(self.ptr) } - } + // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function + // ensuring that this operation is safe. + unsafe { ptr::drop_in_place(self.ptr) } } } diff --git a/rust/macros/zeroable.rs b/rust/macros/zeroable.rs index 82968b5dcca11f..487655768da3cb 100644 --- a/rust/macros/zeroable.rs +++ b/rust/macros/zeroable.rs @@ -1,7 +1,4 @@ -use crate::{ - helpers::{parse_generics, Generics}, - quote, -}; +use crate::helpers::{parse_generics, Generics}; use proc_macro::TokenStream; pub(crate) fn derive(input: TokenStream) -> TokenStream { From 473c7e411f0e439a2a797e14122bd5056881ddbc Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Fri, 5 May 2023 15:55:05 +0200 Subject: [PATCH 574/635] rust: init: wrap type checking struct initializers in a closure In the implementation of the init macros there is a `if false` statement that type checks the initializer to ensure every field is initialized. Since the next patch has a stack variable to store the struct, the function might allocate too much memory on debug builds. Putting the struct into a closure ensures that even in debug builds no stack overflow error is caused. In release builds this was not a problem since the code was optimized away due to the `if false`. Signed-off-by: Benno Lossin --- rust/kernel/init.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 26cf3b056d95d8..47227a444248f3 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -641,14 +641,16 @@ macro_rules! try_pin_init { // We use unreachable code to ensure that all fields have been mentioned exactly // once, this struct initializer will still be type-checked and complain with a // very natural error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] + #[allow(unreachable_code, clippy::diverging_sub_expression, clippy::redundant_closure_call)] if false { - $crate::try_pin_init!(make_initializer: - @slot(slot), - @type_name($t), - @munch_fields($($fields)*,), - @acc(), - ); + (|| { + $crate::try_pin_init!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + })(); } } Ok(__InitOk) @@ -889,14 +891,16 @@ macro_rules! try_init { // We use unreachable code to ensure that all fields have been mentioned exactly // once, this struct initializer will still be type-checked and complain with a // very natural error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] + #[allow(unreachable_code, clippy::diverging_sub_expression, clippy::redundant_closure_call)] if false { - $crate::try_init!(make_initializer: - @slot(slot), - @type_name($t), - @munch_fields($($fields)*,), - @acc(), - ); + (|| { + $crate::try_init!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + })(); } } Ok(__InitOk) From 7d1f99cbd98f1f782061a41c735e1c316574040d Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Fri, 5 May 2023 15:45:17 +0200 Subject: [PATCH 575/635] rust: init: add `..Zeroable::zeroed()` syntax for zeroing all missing fields Add the struct update syntax to the init macros, but only for `..Zeroable::zeroed()`. Adding this at the end of the struct initializer allows one to omit fields from the initializer, these fields will be initialized with 0x00 set to every byte. Only types that implement the `Zeroable` trait can utilize this. Suggested-by: Asahi Lina Signed-off-by: Benno Lossin --- rust/kernel/init.rs | 194 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 185 insertions(+), 9 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 47227a444248f3..3a45ef37383f55 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -505,14 +505,18 @@ macro_rules! stack_try_pin_init { /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. /// - In front of the initializer you can write `&this in` to have access to a [`NonNull`] /// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. /// /// For instance: /// /// ```rust /// # use kernel::pin_init; -/// # use macros::pin_data; +/// # use macros::{pin_data, Zeroable}; /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; /// #[pin_data] +/// #[derive(Zeroable)] /// struct Buf { /// // `ptr` points into `buf`. /// ptr: *mut u8, @@ -525,6 +529,10 @@ macro_rules! stack_try_pin_init { /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, /// pin: PhantomPinned, /// }); +/// pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::zeroed(), +/// }); /// ``` /// /// [`try_pin_init!`]: kernel::try_pin_init @@ -536,11 +544,12 @@ macro_rules! pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!( + $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), + @munch_fields($($fields)*), ) }; } @@ -589,28 +598,76 @@ macro_rules! try_pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!( + $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), @typ($t $(::<$($generics),*>)? ), @fields($($fields)*), @error($crate::error::Error), + @munch_fields($($fields)*), ) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - $crate::try_pin_init!( + $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), @typ($t $(::<$($generics),*>)? ), @fields($($fields)*), @error($err), + @munch_fields($($fields)*), + ) + }; + (parse_zeroable_end: + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + @munch_fields(), + ) => { + $crate::try_pin_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @zeroed(), // nothing means we do not zero unmentioned fields ) }; + (parse_zeroable_end: + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + @munch_fields(..Zeroable::zeroed()), + ) => { + $crate::try_pin_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @zeroed(()), // () means we zero unmentioned fields + ) + }; + (parse_zeroable_end: + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + @munch_fields($ignore:tt $($rest:tt)*), + ) => { + $crate::try_pin_init!(parse_zeroable_end: + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @munch_fields($($rest)*), + ) + }; ( @this($($this:ident)?), @typ($t:ident $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), + @zeroed($($init_zeroed:expr)?), ) => {{ // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return // type and shadow it later when we insert the arbitrary user code. That way there will be @@ -628,6 +685,19 @@ macro_rules! try_pin_init { { // Shadow the structure so it cannot be used to return early. struct __InitOk; + // If `$init_zeroed` is present, we should not error on unmentioned fields and + // set the whole struct to zero first. + // + // For type inference reasons we do not use `init::zeroed`, but instead + // write_bytes. + $({ + // We need to ensure the type actually implements `Zeroable`. + fn is_zeroable(ptr: *mut T) {} + is_zeroable(slot); + // SAFETY: the type implements `Zeroable`. + unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; + $init_zeroed + })? // Create the `this` so it can be referenced by the user inside of the // expressions creating the individual fields. $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? @@ -666,7 +736,7 @@ macro_rules! try_pin_init { @data($data:ident), @slot($slot:ident), @guards($($guards:ident,)*), - @munch_fields($(,)?), + @munch_fields($(..Zeroable::zeroed())? $(,)?), ) => { // Endpoint of munching, no fields are left. If execution reaches this point, all fields // have been initialized. Therefore we can now dismiss the guards by forgetting them. @@ -730,6 +800,28 @@ macro_rules! try_pin_init { @munch_fields($($rest)*), ); }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields(..Zeroable::zeroed() $(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. Without erroring on extra + // fields. We want to have the same semantics as a struct initializer with struct update + // syntax, so we create one first. + // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to + // get the correct type inference here: + unsafe { + // We have to force zeroed to have the correct type. + let mut zeroed = ::core::mem::zeroed(); + ::core::ptr::write($slot, zeroed); + zeroed = ::core::mem::zeroed(); + ::core::ptr::write($slot, $t { + $($acc)* + ..zeroed + }); + } + }; (make_initializer: @slot($slot:ident), @type_name($t:ident), @@ -793,11 +885,12 @@ macro_rules! init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!( + $crate::try_init!(parse_zeroable_end: @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), + @munch_fields($($fields)*), ) } } @@ -840,28 +933,76 @@ macro_rules! try_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!( + $crate::try_init!(parse_zeroable_end: @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error($crate::error::Error), + @munch_fields($($fields)*), ) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { + $crate::try_init!(parse_zeroable_end: + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @munch_fields($($fields)*), + ) + }; + (parse_zeroable_end: + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + @munch_fields(), + ) => { $crate::try_init!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error($err), + @zeroed(), // Nothing means we do not zero unmentioned fields. ) }; + (parse_zeroable_end: + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + @munch_fields(..Zeroable::zeroed()), + ) => { + $crate::try_init!( + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @zeroed(()), // () means we zero unmentioned fields. + ) + }; + (parse_zeroable_end: + @this($($this:ident)?), + @typ($t:ident $(::<$($generics:ty),*>)?), + @fields($($fields:tt)*), + @error($err:ty), + @munch_fields($ignore:tt $($rest:tt)*), + ) => { + $crate::try_init!(parse_zeroable_end: + @this($($this)?), + @typ($t $(::<$($generics),*>)?), + @fields($($fields)*), + @error($err), + @munch_fields($($rest)*), + ) + }; ( @this($($this:ident)?), @typ($t:ident $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), + @zeroed($($init_zeroed:expr)?), ) => {{ // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return // type and shadow it later when we insert the arbitrary user code. That way there will be @@ -879,6 +1020,19 @@ macro_rules! try_init { { // Shadow the structure so it cannot be used to return early. struct __InitOk; + // If `$init_zeroed` is present, we should not error on unmentioned fields and + // set the whole struct to zero first. + // + // For type inference reasons we do not use `init::zeroed`, but instead + // write_bytes. + $({ + // We need to ensure the type actually implements `Zeroable`. + fn is_zeroable(ptr: *mut T) {} + is_zeroable(slot); + // SAFETY: the type implements `Zeroable`. + unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; + $init_zeroed + })? // Create the `this` so it can be referenced by the user inside of the // expressions creating the individual fields. $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? @@ -915,7 +1069,7 @@ macro_rules! try_init { (init_slot: @slot($slot:ident), @guards($($guards:ident,)*), - @munch_fields( $(,)?), + @munch_fields($(..Zeroable::zeroed())? $(,)?), ) => { // Endpoint of munching, no fields are left. If execution reaches this point, all fields // have been initialized. Therefore we can now dismiss the guards by forgetting them. @@ -982,7 +1136,29 @@ macro_rules! try_init { (make_initializer: @slot($slot:ident), @type_name($t:ident), - @munch_fields( $(,)?), + @munch_fields(..Zeroable::zeroed() $(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. Without erroring on extra + // fields. We want to have the same semantics as a struct initializer with struct update + // syntax, so we create one first. + // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to + // get the correct type inference here: + unsafe { + // We have to force zeroed to have the correct type. + let mut zeroed = ::core::mem::zeroed(); + ::core::ptr::write($slot, zeroed); + zeroed = ::core::mem::zeroed(); + ::core::ptr::write($slot, $t { + $($acc)* + ..zeroed + }); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:ident), + @munch_fields($(,)?), @acc($($acc:tt)*), ) => { // Endpoint, nothing more to munch, create the initializer. From 7ba063f5c6e964f98ce16804437f9cb7c974baee Mon Sep 17 00:00:00 2001 From: Benno Lossin Date: Fri, 5 May 2023 16:08:41 +0200 Subject: [PATCH 576/635] rust: init: Add functions to create array initializers Add two functions `pin_init_array_from_fn` and `init_array_from_fn` that take a function that generates initializers for `T` from usize, the added functions then return an initializer for `[T; N]` where every element is initialized by an element returned from the generator function. Suggested-by: Asahi Lina Signed-off-by: Benno Lossin --- rust/kernel/init.rs | 90 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 3a45ef37383f55..143915440b01b8 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1334,6 +1334,96 @@ pub fn uninit() -> impl Init, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// let array: Box<[usize; 1000_000_000]>= Box::init(init_array_from_fn(|i| i)).unwrap(); +/// println!("{array:?}"); +/// ``` +pub fn init_array_from_fn( + mut make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::(); + for i in 0..N { + let init = make_init(i); + // SAFETY: since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + match unsafe { init.__init(ptr) } { + Ok(()) => {} + Err(e) => { + // We now free every element that has been initialized before: + for j in 0..i { + let ptr = unsafe { slot.add(j) }; + // SAFETY: The value was initialized in a previous iteration of the loop + // and since we return `Err` below, the caller will consider the memory at + // `slot` as uninitialized. + unsafe { ptr::drop_in_place(ptr) }; + } + return Err(e); + } + } + } + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { init_from_closure(init) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// let array: Arc<[Mutex; 1000_000_000]>= +/// Arc::pin_init(init_array_from_fn(|i| new_mutex!(i))).unwrap(); +/// println!("{array:?}"); +/// ``` +pub fn pin_init_array_from_fn( + mut make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::(); + for i in 0..N { + let init = make_init(i); + // SAFETY: since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__pinned_init` + // requirements. + match unsafe { init.__pinned_init(ptr) } { + Ok(()) => {} + Err(e) => { + // We now have to free every element that has been initialized before, since we + // have to abide by the drop guarantee. + for j in 0..i { + let ptr = unsafe { slot.add(j) }; + // SAFETY: The value was initialized in a previous iteration of the loop + // and since we return `Err` below, the caller will consider the memory at + // `slot` as uninitialized. + unsafe { ptr::drop_in_place(ptr) }; + } + return Err(e); + } + } + } + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { pin_init_from_closure(init) } +} + // SAFETY: Every type can be initialized by-value. unsafe impl Init for T { unsafe fn __init(self, slot: *mut T) -> Result<(), E> { From 58a114427266b45bd833aabd459e2995edb5badb Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 15:24:04 +0900 Subject: [PATCH 577/635] rust: init: Implement Zeroable::zeroed() By analogy to Default::default(), this just returns the zeroed representation of the type directly. init::zeroed() is the version that returns an initializer. Signed-off-by: Asahi Lina --- rust/kernel/init.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 143915440b01b8..d0b8a1ce5d176e 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1584,7 +1584,14 @@ pub unsafe trait PinnedDrop: __internal::HasPinData { /// ```rust,ignore /// let val: Self = unsafe { core::mem::zeroed() }; /// ``` -pub unsafe trait Zeroable {} +pub unsafe trait Zeroable: core::marker::Sized { + /// Create a new zeroed T. + /// + /// Directly returns a zeroed T, analogous to Default::default(). + fn zeroed() -> Self { + unsafe { core::mem::zeroed() } + } +} /// Create a new zeroed T. /// From ca44e0ab0886fc04bea667ac3403a8a84e7c02db Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 15:25:21 +0900 Subject: [PATCH 578/635] drm/asahi: Switch to the kernel crate Zeroable trait Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 4 +- drivers/gpu/drm/asahi/channel.rs | 8 ++-- drivers/gpu/drm/asahi/float.rs | 4 +- drivers/gpu/drm/asahi/fw/channels.rs | 2 +- drivers/gpu/drm/asahi/fw/types.rs | 60 +++++++++------------------- drivers/gpu/drm/asahi/object.rs | 8 ++-- drivers/gpu/drm/asahi/place.rs | 10 ++--- 7 files changed, 38 insertions(+), 58 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index 659a86b02688d2..571430995e4b3c 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -16,7 +16,7 @@ use kernel::{c_str, drm::mm, error::Result, prelude::*, str::CString, sync::Lock use crate::debug::*; use crate::driver::AsahiDevice; -use crate::fw::types::Zeroed; +use crate::fw::types::Zeroable; use crate::mmu; use crate::object::{GpuArray, GpuObject, GpuOnlyArray, GpuStruct, GpuWeakPointer}; @@ -244,7 +244,7 @@ pub(crate) trait Allocator { &mut self, ) -> Result>> where - for<'a> ::Raw<'a>: Default + Zeroed, + for<'a> ::Raw<'a>: Default + Zeroable, { GpuObject::>::new_default(self.alloc_object()?) } diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 0b3c3b65c2796a..b0069e5fb3a520 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -22,7 +22,7 @@ pub(crate) use crate::fw::channels::PipeType; /// A receive (FW->driver) channel. pub(crate) struct RxChannel where - for<'a> ::Raw<'a>: Debug + Default + Zeroed, + for<'a> ::Raw<'a>: Debug + Default + Zeroable, { ring: ChannelRing, // FIXME: needs feature(generic_const_exprs) @@ -33,7 +33,7 @@ where impl RxChannel where - for<'a> ::Raw<'a>: Debug + Default + Zeroed, + for<'a> ::Raw<'a>: Debug + Default + Zeroable, { /// Allocates a new receive channel with a given message count. pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { @@ -87,7 +87,7 @@ where /// A transmit (driver->FW) channel. pub(crate) struct TxChannel where - for<'a> ::Raw<'a>: Debug + Default + Zeroed, + for<'a> ::Raw<'a>: Debug + Default + Zeroable, { ring: ChannelRing, wptr: u32, @@ -96,7 +96,7 @@ where impl TxChannel where - for<'a> ::Raw<'a>: Debug + Default + Zeroed, + for<'a> ::Raw<'a>: Debug + Default + Zeroable, { /// Allocates a new cached transmit channel with a given message count. pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index e73b4b628cf9f5..f35134123a08d9 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -23,12 +23,14 @@ //! related to slightly non-compliant rounding. use core::ops; -use kernel::{of, prelude::*}; +use kernel::{init::Zeroable, of, prelude::*}; /// An IEEE754-compatible floating point number implemented in software. #[derive(Default, Debug, Copy, Clone)] pub(crate) struct F32(u32); +unsafe impl Zeroable for F32 {} + #[derive(Default, Debug, Copy, Clone)] struct F32U { sign: bool, diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs index dae3095df0a72e..1020c0157be713 100644 --- a/drivers/gpu/drm/asahi/fw/channels.rs +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -34,7 +34,7 @@ pub(crate) mod raw { pub(crate) trait RxChannelState: GpuStruct + Debug + Default where - for<'a> ::Raw<'a>: Default + Zeroed, + for<'a> ::Raw<'a>: Default + Zeroable, { const SUB_CHANNELS: usize; diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs index 5397a5be057442..956e693c9d90b0 100644 --- a/drivers/gpu/drm/asahi/fw/types.rs +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -14,6 +14,7 @@ pub(crate) use ::alloc::boxed::Box; pub(crate) use core::fmt::Debug; pub(crate) use core::marker::PhantomData; pub(crate) use core::sync::atomic::{AtomicI32, AtomicU32, AtomicU64}; +pub(crate) use kernel::init::Zeroable; pub(crate) use kernel::macros::versions; // Make the trait visible @@ -53,7 +54,7 @@ pub(crate) struct FwStamp(pub(crate) AtomicU32); #[repr(C, packed(1))] pub(crate) struct U64(pub(crate) u64); -unsafe impl Zeroed for U64 {} +unsafe impl Zeroable for U64 {} impl fmt::Debug for U64 { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -70,7 +71,7 @@ impl fmt::Debug for U64 { #[repr(C, packed(1))] pub(crate) struct U32(pub(crate) u32); -unsafe impl Zeroed for U32 {} +unsafe impl Zeroable for U32 {} impl fmt::Debug for U32 { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -79,16 +80,6 @@ impl fmt::Debug for U32 { } } -unsafe impl Zeroed for u8 {} -unsafe impl Zeroed for u16 {} -unsafe impl Zeroed for u32 {} -unsafe impl Zeroed for u64 {} -unsafe impl Zeroed for i8 {} -unsafe impl Zeroed for i16 {} -unsafe impl Zeroed for i32 {} -unsafe impl Zeroed for i64 {} -unsafe impl Zeroed for F32 {} - /// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by /// hand or not very useful. #[macro_export] @@ -102,46 +93,32 @@ macro_rules! no_debug { }; } -/// Types which can be safely initialized with an all-zero bit pattern. -/// -/// See: https://github.com/rust-lang/rfcs/issues/2626 -/// -/// # Safety -/// -/// This trait must only be implemented if a type only contains primitive types which can be -/// zero-initialized, FFI structs intended to be zero-initialized, or other types which impl Zeroed. -pub(crate) unsafe trait Zeroed: Default { - fn zeroed() -> Self { - // SAFETY: The user is responsible for ensuring this is safe. - unsafe { core::mem::zeroed() } - } -} - -/// Implement Zeroed for a given type (and Default along with it). +/// Implement Zeroable for a given type (and Default along with it). /// /// # Safety /// /// This macro must only be used if a type only contains primitive types which can be -/// zero-initialized, FFI structs intended to be zero-initialized, or other types which impl Zeroed. +/// zero-initialized, FFI structs intended to be zero-initialized, or other types which +/// impl Zeroable. #[macro_export] macro_rules! default_zeroed { (<$($lt:lifetime),*>, $type:ty) => { impl<$($lt),*> Default for $type { fn default() -> $type { - Zeroed::zeroed() + ::kernel::init::Zeroable::zeroed() } } // SAFETY: The user is responsible for ensuring this is safe. - unsafe impl<$($lt),*> Zeroed for $type {} + unsafe impl<$($lt),*> ::kernel::init::Zeroable for $type {} }; ($type:ty) => { impl Default for $type { fn default() -> $type { - Zeroed::zeroed() + ::kernel::init::Zeroable::zeroed() } } // SAFETY: The user is responsible for ensuring this is safe. - unsafe impl Zeroed for $type {} + unsafe impl ::kernel::init::Zeroable for $type {} }; } @@ -151,11 +128,11 @@ macro_rules! default_zeroed { pub(crate) struct Pad([u8; N]); /// SAFETY: Primitive type, safe to zero-init. -unsafe impl Zeroed for Pad {} +unsafe impl Zeroable for Pad {} impl Default for Pad { fn default() -> Self { - Zeroed::zeroed() + Zeroable::zeroed() } } @@ -165,7 +142,7 @@ impl fmt::Debug for Pad { } } -/// A convenience type for a fixed-sized array with Default/Zeroed impls. +/// A convenience type for a fixed-sized array with Default/Zeroable impls. #[derive(Copy, Clone)] #[repr(C)] pub(crate) struct Array([T; N]); @@ -176,12 +153,12 @@ impl Array { } } -// SAFETY: Arrays of Zeroed values can be safely Zeroed. -unsafe impl Zeroed for Array {} +// SAFETY: Arrays of Zeroable values can be safely Zeroable. +unsafe impl Zeroable for Array {} -impl Default for Array { +impl Default for Array { fn default() -> Self { - Zeroed::zeroed() + Zeroable::zeroed() } } @@ -224,11 +201,12 @@ impl fmt::Debug for Array { #[macro_export] macro_rules! trivial_gpustruct { ($type:ident) => { - #[derive(Debug, Default)] + #[derive(Debug)] pub(crate) struct $type {} impl GpuStruct for $type { type Raw<'a> = raw::$type; } + $crate::default_zeroed!($type); }; } diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index b38c2f9d80e756..11a26e839e3d84 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -57,7 +57,7 @@ use core::{mem, ptr, slice}; use crate::alloc::Allocation; use crate::debug::*; -use crate::fw::types::Zeroed; +use crate::fw::types::Zeroable; const DEBUG_CLASS: DebugFlags = DebugFlags::Object; @@ -450,13 +450,13 @@ where impl> GpuObject where - for<'a> ::Raw<'a>: Default + Zeroed, + for<'a> ::Raw<'a>: Default + Zeroable, { /// Create a new GpuObject with default data. `T` must implement `Default` and `T::Raw` must - /// implement `Zeroed`, since the GPU-side memory is initialized by zeroing. + /// implement `Zeroable`, since the GPU-side memory is initialized by zeroing. pub(crate) fn new_default(alloc: U) -> Result { GpuObject::::new_inplace(alloc, Default::default(), |_inner, raw| { - // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroed`. + // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroable`. Ok(unsafe { ptr::write_bytes(raw, 0, 1); (*raw).assume_init_mut() diff --git a/drivers/gpu/drm/asahi/place.rs b/drivers/gpu/drm/asahi/place.rs index 40c51f4fab8db2..33df7199b6caa1 100644 --- a/drivers/gpu/drm/asahi/place.rs +++ b/drivers/gpu/drm/asahi/place.rs @@ -30,9 +30,9 @@ /// /// The macro supports most Rust initialization syntax including type paths, generic arguments, /// and nested structures. Nested structures are themselves initialized in-place field by field. -/// `..Default::default()` is supported, but this macro converts it to `..Zeroed::zeroed()`, as it +/// `..Default::default()` is supported, but this macro converts it to `..Zeroable::zeroed()`, as it /// initializes those structs by zero-initializing the underlying memory. Usage of -/// `..Default::default()` with a type not implementing `Zeroed` will result in a compile error. +/// `..Default::default()` with a type not implementing `Zeroable` will result in a compile error. /// /// Usage: /// ``` @@ -81,11 +81,11 @@ macro_rules! place { // Zero-initialize structure if the initializer ends in ..default::Default() (@STRUCT_ZERO $ptr:ident, {$($typ_init:tt)*} { $($f:ident $(: $v:expr)?),* $(,)? }) => {}; (@STRUCT_ZERO $ptr:ident, {$($typ_init:tt)*} { $($($f:ident $(: $v:expr)?),*,)? ..Default::default() }) => {{ - // Check that the structure actually implements Zeroed + // Check that the structure actually implements Zeroable const _: () = { fn _check_default() { let _ = $($typ_init)* { - ..Zeroed::zeroed() + ..Zeroable::zeroed() }; } }; @@ -103,7 +103,7 @@ macro_rules! place { $f $(: $v)? ),* ,)? - ..Zeroed::zeroed() + ..Zeroable::zeroed() }; } else { {$($body)*} From 37d6f17fcca3c50407c4766e91b2ef129ef4b706 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 15:35:54 +0900 Subject: [PATCH 579/635] drm/asahi: mmu: Switch to pin_init_array_from_fn Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/mmu.rs | 44 +++++++++++++----------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index dc05f6cf36fe3d..a3e9cb633221c0 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -10,10 +10,9 @@ //! //! The actual page table management is delegated to the common kernel `io_pgtable` code. -use core::convert::Infallible; use core::fmt::Debug; use core::mem::size_of; -use core::ptr::{addr_of_mut, NonNull}; +use core::ptr::NonNull; use core::sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}; use core::time::Duration; @@ -1185,35 +1184,24 @@ impl Uat { let handoff_rgn = Self::map_region(dev, c_str!("handoff"), HANDOFF_SIZE, false)?; let ttbs_rgn = Self::map_region(dev, c_str!("ttbs"), SLOTS_SIZE, false)?; + let handoff = unsafe { &(handoff_rgn.map.as_ptr() as *mut Handoff).as_ref().unwrap() }; + dev_info!(dev, "MMU: Initializing kernel page table\n"); - Arc::pin_init(unsafe { - init::pin_init_from_closure( - move |slot: *mut UatInner| -> core::result::Result<(), Infallible> { - let handoff = &(handoff_rgn.map.as_ptr() as *mut Handoff).as_ref().unwrap(); - - for i in 0..UAT_NUM_CTX + 1 { - new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") - .__pinned_init(addr_of_mut!((*slot).handoff_flush[i])) - .expect("infallible"); - } - - new_mutex!( - UatShared { - kernel_ttb1: 0, - map_kernel_to_user: false, - handoff_rgn, - ttbs_rgn, - }, - "uat_shared" - ) - .__pinned_init(addr_of_mut!((*slot).shared)) - .expect("infallible"); - - Ok(()) + Arc::pin_init(try_pin_init!(UatInner { + handoff_flush <- init::pin_init_array_from_fn(|i| { + new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + }), + shared <- new_mutex!( + UatShared { + kernel_ttb1: 0, + map_kernel_to_user: false, + handoff_rgn, + ttbs_rgn, }, - ) - }) + "uat_shared" + ), + })) } /// Creates a new `Uat` instance given the relevant hardware config. From 9a101890470a71bc6da1044f302363d917d67a1f Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 16:22:09 +0900 Subject: [PATCH 580/635] rust: init: Support type paths in pin_init!() and friends This makes directly initializing structures with a type name that isn't in the current scope work properly. Signed-off-by: Asahi Lina --- rust/kernel/init.rs | 92 ++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index d0b8a1ce5d176e..18bca4797a2a0b 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -541,12 +541,12 @@ macro_rules! stack_try_pin_init { // module `__internal` inside of `init/__internal.rs`. #[macro_export] macro_rules! pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), @munch_fields($($fields)*), @@ -595,23 +595,23 @@ macro_rules! pin_init { // module `__internal` inside of `init/__internal.rs`. #[macro_export] macro_rules! try_pin_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)? ), + @typ($t $(::$p)* $(::<$($generics),*>)? ), @fields($($fields)*), @error($crate::error::Error), @munch_fields($($fields)*), ) }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)? ), + @typ($t $(::$p)* $(::<$($generics),*>)? ), @fields($($fields)*), @error($err), @munch_fields($($fields)*), @@ -619,14 +619,14 @@ macro_rules! try_pin_init { }; (parse_zeroable_end: @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @munch_fields(), ) => { $crate::try_pin_init!( @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @zeroed(), // nothing means we do not zero unmentioned fields @@ -634,14 +634,14 @@ macro_rules! try_pin_init { }; (parse_zeroable_end: @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @munch_fields(..Zeroable::zeroed()), ) => { $crate::try_pin_init!( @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @zeroed(()), // () means we zero unmentioned fields @@ -649,14 +649,14 @@ macro_rules! try_pin_init { }; (parse_zeroable_end: @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @munch_fields($ignore:tt $($rest:tt)*), ) => { $crate::try_pin_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @munch_fields($($rest)*), @@ -664,7 +664,7 @@ macro_rules! try_pin_init { }; ( @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @zeroed($($init_zeroed:expr)?), @@ -676,7 +676,7 @@ macro_rules! try_pin_init { // Get the pin data from the supplied type. let data = unsafe { use $crate::init::__internal::HasPinData; - $t$(::<$($generics),*>)?::__pin_data() + $t$(::$p)*$(::<$($generics),*>)?::__pin_data() }; // Ensure that `data` really is of type `PinData` and help with type inference: let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( @@ -716,7 +716,7 @@ macro_rules! try_pin_init { (|| { $crate::try_pin_init!(make_initializer: @slot(slot), - @type_name($t), + @type_name($t$(::$p)*), @munch_fields($($fields)*,), @acc(), ); @@ -802,7 +802,7 @@ macro_rules! try_pin_init { }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields(..Zeroable::zeroed() $(,)?), @acc($($acc:tt)*), ) => { @@ -816,7 +816,7 @@ macro_rules! try_pin_init { let mut zeroed = ::core::mem::zeroed(); ::core::ptr::write($slot, zeroed); zeroed = ::core::mem::zeroed(); - ::core::ptr::write($slot, $t { + ::core::ptr::write($slot, $t$(::$p)* { $($acc)* ..zeroed }); @@ -824,7 +824,7 @@ macro_rules! try_pin_init { }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields($(,)?), @acc($($acc:tt)*), ) => { @@ -832,33 +832,33 @@ macro_rules! try_pin_init { // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to // get the correct type inference here: unsafe { - ::core::ptr::write($slot, $t { + ::core::ptr::write($slot, $t$(::$p)* { $($acc)* }); } }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields($field:ident <- $val:expr, $($rest:tt)*), @acc($($acc:tt)*), ) => { $crate::try_pin_init!(make_initializer: @slot($slot), - @type_name($t), + @type_name($t$(::$p)*), @munch_fields($($rest)*), @acc($($acc)* $field: ::core::panic!(),), ); }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), @acc($($acc:tt)*), ) => { $crate::try_pin_init!(make_initializer: @slot($slot), - @type_name($t), + @type_name($t$(::$p)*), @munch_fields($($rest)*), @acc($($acc)* $field: ::core::panic!(),), ); @@ -882,12 +882,12 @@ macro_rules! try_pin_init { // module `__internal` inside of `init/__internal.rs`. #[macro_export] macro_rules! init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { $crate::try_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), @munch_fields($($fields)*), @@ -930,23 +930,23 @@ macro_rules! init { // module `__internal` inside of `init/__internal.rs`. #[macro_export] macro_rules! try_init { - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { $crate::try_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($crate::error::Error), @munch_fields($($fields)*), ) }; - ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { + ($(&$this:ident in)? $t:ident $(::$p:ident)* $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { $crate::try_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @munch_fields($($fields)*), @@ -954,14 +954,14 @@ macro_rules! try_init { }; (parse_zeroable_end: @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @munch_fields(), ) => { $crate::try_init!( @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @zeroed(), // Nothing means we do not zero unmentioned fields. @@ -969,14 +969,14 @@ macro_rules! try_init { }; (parse_zeroable_end: @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @munch_fields(..Zeroable::zeroed()), ) => { $crate::try_init!( @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @zeroed(()), // () means we zero unmentioned fields. @@ -984,14 +984,14 @@ macro_rules! try_init { }; (parse_zeroable_end: @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @munch_fields($ignore:tt $($rest:tt)*), ) => { $crate::try_init!(parse_zeroable_end: @this($($this)?), - @typ($t $(::<$($generics),*>)?), + @typ($t $(::$p)* $(::<$($generics),*>)?), @fields($($fields)*), @error($err), @munch_fields($($rest)*), @@ -999,7 +999,7 @@ macro_rules! try_init { }; ( @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), + @typ($t:ident $(::$p:ident)* $(::<$($generics:ty),*>)?), @fields($($fields:tt)*), @error($err:ty), @zeroed($($init_zeroed:expr)?), @@ -1011,7 +1011,7 @@ macro_rules! try_init { // Get the init data from the supplied type. let data = unsafe { use $crate::init::__internal::HasInitData; - $t$(::<$($generics),*>)?::__init_data() + $t$(::$p)*$(::<$($generics),*>)?::__init_data() }; // Ensure that `data` really is of type `InitData` and help with type inference: let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( @@ -1050,7 +1050,7 @@ macro_rules! try_init { (|| { $crate::try_init!(make_initializer: @slot(slot), - @type_name($t), + @type_name($t$(::$p)*), @munch_fields($($fields)*,), @acc(), ); @@ -1135,7 +1135,7 @@ macro_rules! try_init { }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields(..Zeroable::zeroed() $(,)?), @acc($($acc:tt)*), ) => { @@ -1149,7 +1149,7 @@ macro_rules! try_init { let mut zeroed = ::core::mem::zeroed(); ::core::ptr::write($slot, zeroed); zeroed = ::core::mem::zeroed(); - ::core::ptr::write($slot, $t { + ::core::ptr::write($slot, $t$(::$p)* { $($acc)* ..zeroed }); @@ -1157,7 +1157,7 @@ macro_rules! try_init { }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields($(,)?), @acc($($acc:tt)*), ) => { @@ -1165,33 +1165,33 @@ macro_rules! try_init { // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to // get the correct type inference here: unsafe { - ::core::ptr::write($slot, $t { + ::core::ptr::write($slot, $t$(::$p)* { $($acc)* }); } }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields($field:ident <- $val:expr, $($rest:tt)*), @acc($($acc:tt)*), ) => { $crate::try_init!(make_initializer: @slot($slot), - @type_name($t), + @type_name($t$(::$p)*), @munch_fields($($rest)*), @acc($($acc)*$field: ::core::panic!(),), ); }; (make_initializer: @slot($slot:ident), - @type_name($t:ident), + @type_name($t:ident $(::$p:ident)*), @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), @acc($($acc:tt)*), ) => { $crate::try_init!(make_initializer: @slot($slot), - @type_name($t), + @type_name($t$(::$p)*), @munch_fields($($rest)*), @acc($($acc)*$field: ::core::panic!(),), ); From d660b9c2a46187c814ff5222d137144db12606d7 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 17:08:56 +0900 Subject: [PATCH 581/635] drm/asahi: Add support for initializing GpuObjects with init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 37 ++++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/object.rs | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index 571430995e4b3c..21fa83b2183ed9 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -267,6 +267,43 @@ pub(crate) trait Allocator { ) } + /// Allocate a new GpuStruct object. See [`GpuObject::new_init_prealloc`]. + #[inline(never)] + fn new_init_prealloc< + 'a, + T: GpuStruct, + I: Init, + R: PinInit, kernel::error::Error>, + >( + &mut self, + inner_init: impl FnOnce(GpuWeakPointer) -> I, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result>> { + GpuObject::>::new_init_prealloc( + self.alloc_object()?, + inner_init, + raw_init, + ) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_init`]. + #[inline(never)] + fn new_init<'a, T: GpuStruct, R: PinInit, F>, E, F>( + &mut self, + inner_init: impl Init, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result>> + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + GpuObject::>::new_init_prealloc( + self.alloc_object()?, + |_p| inner_init, + raw_init, + ) + } + /// Allocate a generic buffer of the given size and alignment, applying the debug features if /// enabled to tag it and detect overflows. fn alloc_generic( diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 11a26e839e3d84..fb5dcc628d4fe9 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -380,6 +380,46 @@ impl> GpuObject { }) } + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_init_prealloc<'a, I: Init, R: PinInit, F>, E, F>( + alloc: U, + inner_init: impl FnOnce(GpuWeakPointer) -> I, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let inner = inner_init(gpu_ptr); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let ret = Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner: Box::init(inner)?, + }; + let q = &*ret.inner as *const T; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + Ok(ret) + } + /// Returns the GPU VA of this object (as a raw [`NonZeroU64`]) pub(crate) fn gpu_va(&self) -> NonZeroU64 { self.gpu_ptr.0 From cebd02628c8a4e0faa43f19884a51853cbfac1f7 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 17:09:20 +0900 Subject: [PATCH 582/635] drm/asahi: buffer: Replace place!() with init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 71 +++++++++++++++++---------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index c3b33f950fe5f4..cb15ea40141a45 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -37,7 +37,6 @@ use crate::fw::buffer; use crate::fw::types::*; use crate::util::*; use crate::{alloc, fw, gpu, hw, mmu, slotalloc}; -use crate::{box_in_place, place}; use core::sync::atomic::Ordering; use kernel::prelude::*; use kernel::sync::{Arc, Mutex}; @@ -335,17 +334,19 @@ impl Buffer::ver { let preempt2_size = num_clusters_adj * gpu.get_cfg().preempt2_size; let preempt3_size = num_clusters_adj * gpu.get_cfg().preempt3_size; - let inner = box_in_place!(buffer::Info::ver { - block_ctl: alloc.shared.new_default::()?, - counter: alloc.shared.new_default::()?, - page_list: ualloc_priv.lock().array_empty(max_pages)?, - block_list: ualloc_priv.lock().array_empty(max_blocks * 2)?, - })?; - - let info = alloc.private.new_boxed(inner, |inner, ptr| { - Ok(place!( - ptr, - buffer::raw::Info::ver { + let shared = &mut alloc.shared; + let info = alloc.private.new_init( + { + let ualloc_priv = &ualloc_priv; + try_init!(buffer::Info::ver { + block_ctl: shared.new_default::()?, + counter: shared.new_default::()?, + page_list: ualloc_priv.lock().array_empty(max_pages)?, + block_list: ualloc_priv.lock().array_empty(max_blocks * 2)?, + }) + }, + |inner, _p| { + try_init!(buffer::raw::Info::ver { gpu_counter: 0x0, unk_4: 0, last_id: 0x0, @@ -379,9 +380,9 @@ impl Buffer::ver { max_pages_nomemless: max_pages_nomemless.try_into()?, unk_8c: 0x0, unk_90: Default::default(), - } - )) - })?; + }) + }, + )?; // Technically similar to Scene below, let's play it safe. let kernel_buffer = alloc.shared.array_empty(0x40)?; @@ -590,33 +591,33 @@ impl Buffer::ver { None }; - let scene_inner = box_in_place!(buffer::Scene::ver { - user_buffer: user_buffer, - buffer: self.clone(), - tvb_heapmeta: tvb_heapmeta, - tvb_tilemap: tvb_tilemap, - tpc: tpc, - clustering: clustering, - preempt_buf: preempt_buf, - seq_buf: seq_buf, - #[ver(G >= G14X)] - control_word: alloc.gpu.array_empty(1)?, - })?; - // Could be made strong, but we wind up with a deadlock if we try to grab the // pointer through the inner.buffer path inside the closure. let stats_pointer = inner.stats.weak_pointer(); + let _gpu = &mut alloc.gpu; + // macOS allocates this as private. However, the firmware does not // DC CIVAC this before reading it (like it does most other things), // which causes odd cache incoherency bugs when combined with // speculation on the firmware side (maybe). This doesn't happen // on macOS because these structs are a circular pool that is mapped // already initialized. Just mark this shared for now. - let scene = alloc.shared.new_boxed(scene_inner, |inner, ptr| { - Ok(place!( - ptr, - buffer::raw::Scene::ver { + let scene = alloc.shared.new_init( + try_init!(buffer::Scene::ver { + user_buffer: user_buffer, + buffer: self.clone(), + tvb_heapmeta: tvb_heapmeta, + tvb_tilemap: tvb_tilemap, + tpc: tpc, + clustering: clustering, + preempt_buf: preempt_buf, + seq_buf: seq_buf, + #[ver(G >= G14X)] + control_word: _gpu.array_empty(1)?, + }), + |inner, _p| { + try_init!(buffer::raw::Scene::ver { #[ver(G >= G14X)] control_word: inner.control_word.gpu_pointer(), #[ver(G >= G14X)] @@ -635,9 +636,9 @@ impl Buffer::ver { unk_30: U64(0), #[ver(G < G14X)] unk_38: U64(0), - } - )) - })?; + }) + }, + )?; let mut rebind = false; From d9ce6bc1acca698f8abbee833fcef8b759a01935 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 17:19:33 +0900 Subject: [PATCH 583/635] drm/asahi: workqueue: Port to init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/workqueue.rs | 42 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index b121a25bc9b559..38ebe9deffd11d 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -17,9 +17,9 @@ use crate::debug::*; use crate::fw::channels::PipeType; use crate::fw::types::*; use crate::fw::workqueue::*; +use crate::no_debug; use crate::object::OpaqueGpuObject; use crate::regs::FaultReason; -use crate::{box_in_place, no_debug, place}; use crate::{channel, driver, event, fw, gpu, object, regs}; use core::num::NonZeroU64; use core::sync::atomic::Ordering; @@ -561,25 +561,27 @@ impl WorkQueue::ver { priority: u32, size: u32, ) -> Result> { - let mut info = box_in_place!(QueueInfo::ver { - state: alloc.shared.new_default::()?, - ring: alloc.shared.array_empty(size as usize)?, - gpu_buf: alloc.private.array_empty(0x2c18)?, - notifier_list: notifier_list, - gpu_context: gpu_context, - })?; - - info.state.with_mut(|raw, _inner| { - raw.rb_size = size; - }); - + let gpu_buf = alloc.private.array_empty(0x2c18)?; + let shared = &mut alloc.shared; let inner = WorkQueueInner::ver { dev: dev.clone(), event_manager, - info: alloc.private.new_boxed(info, |inner, ptr| { - Ok(place!( - ptr, - raw::QueueInfo::ver { + info: alloc.private.new_init( + try_init!(QueueInfo::ver { + state: { + let mut s = shared.new_default::()?; + s.with_mut(|raw, _inner| { + raw.rb_size = size; + }); + s + }, + ring: shared.array_empty(size as usize)?, + gpu_buf, + notifier_list: notifier_list, + gpu_context: gpu_context, + }), + |inner, _p| { + try_init!(raw::QueueInfo::ver { state: inner.state.gpu_pointer(), ring: inner.ring.gpu_pointer(), notifier_list: inner.notifier_list.gpu_pointer(), @@ -608,9 +610,9 @@ impl WorkQueue::ver { unk_a8: Default::default(), #[ver(V >= V13_2 && G < G14X)] unk_b0: 0, - } - )) - })?, + }) + }, + )?, new: true, pipe_type, size, From 7bed6290fac281f8251694abaf094b2bbb60565b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 17:50:02 +0900 Subject: [PATCH 584/635] drm/asahi: queue::mod: Port to init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/mod.rs | 56 +++++++++++++----------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 804d4b7c0e8ccb..d5a1a197281c06 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -21,10 +21,9 @@ use crate::debug::*; use crate::driver::AsahiDevice; use crate::fw::types::*; use crate::gpu::GpuManager; +use crate::inner_weak_ptr; use crate::{alloc, buffer, channel, event, file, fw, gem, gpu, mmu, workqueue}; -use crate::{inner_weak_ptr, place}; -use core::mem::MaybeUninit; use core::sync::atomic::{AtomicU64, Ordering}; const DEBUG_CLASS: DebugFlags = DebugFlags::Queue; @@ -374,19 +373,16 @@ impl Queue::ver { let threshold = alloc.shared.new_default::()?; let notifier: Arc> = - Arc::try_new(alloc.private.new_inplace( - fw::event::Notifier::ver { threshold }, - |inner, ptr: &mut MaybeUninit>| { - Ok(place!( - ptr, - fw::event::raw::Notifier::ver { - threshold: inner.threshold.gpu_pointer(), - generation: AtomicU32::new(id as u32), - cur_count: AtomicU32::new(0), - unk_10: AtomicU32::new(0x50), - state: Default::default() - } - )) + Arc::try_new(alloc.private.new_init( + try_init!(fw::event::Notifier::ver { threshold }), + |inner, _p| { + try_init!(fw::event::raw::Notifier::ver { + threshold: inner.threshold.gpu_pointer(), + generation: AtomicU32::new(id as u32), + cur_count: AtomicU32::new(0), + unk_10: AtomicU32::new(0x50), + state: Default::default() + }) }, )?)?; @@ -621,22 +617,20 @@ impl Queue for Queue::ver { _ => return Err(EINVAL), }; mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); - let barrier: GpuObject = alloc.private.new_inplace( - Default::default(), - |_inner, ptr: &mut MaybeUninit| { - Ok(place!( - ptr, - fw::workqueue::raw::Barrier { - tag: fw::workqueue::CommandType::Barrier, - wait_stamp: event.fw_stamp_pointer, - wait_value: event.value, - wait_slot: event.slot, - stamp_self: queue_job.event_info().value.next(), - uuid: 0xffffbbbb, - barrier_type: 0, - padding: Default::default(), - } - )) + let barrier = alloc.private.new_init( + kernel::init::zeroed::(), + |_inner, _p| { + let queue_job = &queue_job; + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: event.fw_stamp_pointer, + wait_value: event.value, + wait_slot: event.slot, + stamp_self: queue_job.event_info().value.next(), + uuid: 0xffffbbbb, + barrier_type: 0, + padding: Default::default(), + }) }, )?; mod_dev_dbg!(self.dev, "[Submission {}] Add Explicit Barrier\n", id); From bfcb3661cb453dc23ac860ea87a1149b420a0d40 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 18:00:57 +0900 Subject: [PATCH 585/635] drm/asahi: queue::compute: Port to init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/compute.rs | 402 +++++++++++++------------ 1 file changed, 202 insertions(+), 200 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs index 7692c29e0f5a66..3d8a95b3be5752 100644 --- a/drivers/gpu/drm/asahi/queue/compute.rs +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -12,8 +12,8 @@ use crate::alloc::Allocator; use crate::debug::*; use crate::fw::types::*; use crate::gpu::GpuManager; -use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; use crate::{fw, gpu, microseq}; +use crate::{inner_ptr, inner_weak_ptr}; use core::mem::MaybeUninit; use core::sync::atomic::Ordering; use kernel::dma_fence::RawDmaFence; @@ -128,214 +128,216 @@ impl super::Queue::ver { #[ver(V >= V13_0B4)] let count = self.counter.fetch_add(1, Ordering::Relaxed); - let comp = GpuObject::new_prealloc( + let comp = GpuObject::new_init_prealloc( kalloc.gpu_ro.alloc_object()?, |ptr: GpuWeakPointer| { - let mut builder = microseq::Builder::new(); - - let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); - - let start_comp = builder.add(microseq::StartCompute::ver { - header: microseq::op::StartCompute::HEADER, - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - job_params1: inner_weak_ptr!(ptr, job_params1), - stats, - work_queue: ev_comp.info_ptr, - vm_slot: vm_bind.slot(), - unk_28: 0x1, - event_generation: self.id as u32, - event_seq: U64(ev_comp.event_seq), - unk_38: 0x0, - job_params2: inner_weak_ptr!(ptr, job_params2), - unk_44: 0x0, - uuid, - attachments: common::build_attachments( - cmdbuf.attachments, - cmdbuf.attachment_count, - )?, - padding: Default::default(), - #[ver(V >= V13_0B4)] - unk_flag: inner_weak_ptr!(ptr, unk_flag), - #[ver(V >= V13_0B4)] - counter: U64(count), - #[ver(V >= V13_0B4)] - notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), - })?; - - if result_writer.is_some() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - cur_ts: inner_weak_ptr!(ptr, cur_ts), - start_ts: inner_weak_ptr!(ptr, start_ts), - update_ts: inner_weak_ptr!(ptr, start_ts), - work_queue: ev_comp.info_ptr, - unk_24: U64(0), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid, - unk_30_padding: 0, - })?; - } - - #[ver(G < G14X)] - builder.add(microseq::WaitForIdle { - header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), - })?; - #[ver(G >= G14X)] - builder.add(microseq::WaitForIdle2 { - header: microseq::op::WaitForIdle2::HEADER, - })?; - - if result_writer.is_some() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - cur_ts: inner_weak_ptr!(ptr, cur_ts), - start_ts: inner_weak_ptr!(ptr, start_ts), - update_ts: inner_weak_ptr!(ptr, end_ts), - work_queue: ev_comp.info_ptr, - unk_24: U64(0), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid, - unk_30_padding: 0, - })?; - } - - let off = builder.offset_to(start_comp); - builder.add(microseq::FinalizeCompute::ver { - header: microseq::op::FinalizeCompute::HEADER, - stats, - work_queue: ev_comp.info_ptr, - vm_slot: vm_bind.slot(), - #[ver(V < V13_0B4)] - unk_18: 0, - job_params2: inner_weak_ptr!(ptr, job_params2), - unk_24: 0, - uuid, - fw_stamp: ev_comp.fw_stamp_pointer, - stamp_value: ev_comp.value.next(), - unk_38: 0, - unk_3c: 0, - unk_40: 0, - unk_44: 0, - unk_48: 0, - unk_4c: 0, - unk_50: 0, - unk_54: 0, - unk_58: 0, - #[ver(G == G14 && V < V13_0B4)] - unk_5c_g14: U64(0), - restart_branch_offset: off, - has_attachments: (cmdbuf.attachment_count > 0) as u32, - #[ver(V >= V13_0B4)] - unk_64: Default::default(), - #[ver(V >= V13_0B4)] - unk_flag: inner_weak_ptr!(ptr, unk_flag), - #[ver(V >= V13_0B4)] - unk_79: Default::default(), - })?; - - builder.add(microseq::RetireStamp { - header: microseq::op::RetireStamp::HEADER, - })?; - - Ok(box_in_place!(fw::compute::RunCompute::ver { - notifier: notifier.clone(), + let has_result = result_writer.is_some(); + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + try_init!(fw::compute::RunCompute::ver { preempt_buf: preempt_buf, seq_buf: seq_buf, - micro_seq: builder.build(&mut kalloc.private)?, - vm_bind: vm_bind.clone(), - timestamps: timestamps.clone(), - })?) - }, - |inner, ptr, _gpu_ptr| { - Ok(place!( - ptr, - fw::compute::raw::RunCompute::ver { - tag: fw::workqueue::CommandType::RunCompute, - #[ver(V >= V13_0B4)] - counter: U64(count), - unk_4: 0, - vm_slot: vm_bind.slot(), - notifier: inner.notifier.gpu_pointer(), - unk_pointee: Default::default(), - job_params1: fw::compute::raw::JobParameters1 { - preempt_buf1: inner.preempt_buf.gpu_pointer(), - encoder: U64(cmdbuf.encoder_ptr), - // buf2-5 Only if internal program is used - preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), - preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), - preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), - preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), - pipeline_base: U64(0x11_00000000), - unk_38: U64(0x8c60), - unk_40: cmdbuf.ctx_switch_prog, // Internal program addr | 1 - unk_44: 0, - compute_layout_addr: U64(cmdbuf.buffer_descriptor), // Only if internal program used - unk_50: cmdbuf.buffer_descriptor_size, // 0x40 if internal program used - unk_54: 0, - unk_58: 1, - unk_5c: 0, - iogpu_unk_40: cmdbuf.iogpu_unk_40, // 0x1c if internal program used - }, - unk_b8: Default::default(), - microsequence: inner.micro_seq.gpu_pointer(), - microsequence_size: inner.micro_seq.len() as u32, - job_params2: fw::compute::raw::JobParameters2::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); + + let start_comp = builder.add(microseq::StartCompute::ver { + header: microseq::op::StartCompute::HEADER, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + job_params1: inner_weak_ptr!(ptr, job_params1), + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x1, + event_generation: self.id as u32, + event_seq: U64(ev_comp.event_seq), + unk_38: 0x0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_44: 0x0, + uuid, + attachments: common::build_attachments( + cmdbuf.attachments, + cmdbuf.attachment_count, + )?, + padding: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + counter: U64(count), #[ver(V >= V13_0B4)] - unk_0_0: 0, - unk_0: Default::default(), - preempt_buf1: inner.preempt_buf.gpu_pointer(), - encoder_end: U64(cmdbuf.encoder_end), - unk_34: Default::default(), + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if has_result { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, start_ts), + work_queue: ev_comp.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if has_result { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, end_ts), + work_queue: ev_comp.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_comp); + builder.add(microseq::FinalizeCompute::ver { + header: microseq::op::FinalizeCompute::HEADER, + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), #[ver(V < V13_0B4)] - unk_5c: 0, - }, - encoder_params: fw::job::raw::EncoderParams { - unk_8: 0x0, // fixed - large_tib: 0x0, // check! - unk_10: 0x0, // fixed - encoder_id: cmdbuf.encoder_id, - unk_18: 0x0, // fixed - iogpu_compute_unk44: cmdbuf.iogpu_unk_44, - seq_buffer: U64(inner.seq_buf.gpu_pointer().into()), - unk_28: U64(0x0), // fixed - }, - meta: fw::job::raw::JobMeta { - unk_0: 0, - unk_2: 0, - // TODO: make separate flag - no_preemption: ((cmdbuf.ctx_switch_prog & 1) == 0) as u8, - stamp: ev_comp.stamp_pointer, + unk_18: 0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_24: 0, + uuid, fw_stamp: ev_comp.fw_stamp_pointer, stamp_value: ev_comp.value.next(), - stamp_slot: ev_comp.slot, - evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, - uuid: uuid, - event_seq: ev_comp.event_seq as u32, - }, - cur_ts: U64(0), - start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), - end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), - unk_2c0: 0, - unk_2c4: 0, - unk_2c8: 0, - unk_2cc: 0, - client_sequence: slot_client_seq, - pad_2d1: Default::default(), - unk_2d4: 0, - unk_2d8: 0, - #[ver(V >= V13_0B4)] - unk_ts: U64(0), - #[ver(V >= V13_0B4)] - unk_2e1: Default::default(), - #[ver(V >= V13_0B4)] - unk_flag: U32(0), + unk_38: 0, + unk_3c: 0, + unk_40: 0, + unk_44: 0, + unk_48: 0, + unk_4c: 0, + unk_50: 0, + unk_54: 0, + unk_58: 0, + #[ver(G == G14 && V < V13_0B4)] + unk_5c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (cmdbuf.attachment_count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_64: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + unk_79: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(&mut kalloc.private)? + }, + notifier, + vm_bind, + timestamps, + }) + }, + |inner, _p| { + let vm_slot = vm_bind.slot(); + try_init!(fw::compute::raw::RunCompute::ver { + tag: fw::workqueue::CommandType::RunCompute, + #[ver(V >= V13_0B4)] + counter: U64(count), + unk_4: 0, + vm_slot, + notifier: inner.notifier.gpu_pointer(), + unk_pointee: Default::default(), + job_params1 <- try_init!(fw::compute::raw::JobParameters1 { + preempt_buf1: inner.preempt_buf.gpu_pointer(), + encoder: U64(cmdbuf.encoder_ptr), + // buf2-5 Only if internal program is used + preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), + preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), + preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), + preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), + pipeline_base: U64(0x11_00000000), + unk_38: U64(0x8c60), + unk_40: cmdbuf.ctx_switch_prog, // Internal program addr | 1 + unk_44: 0, + compute_layout_addr: U64(cmdbuf.buffer_descriptor), // Only if internal program used + unk_50: cmdbuf.buffer_descriptor_size, // 0x40 if internal program used + unk_54: 0, + unk_58: 1, + unk_5c: 0, + iogpu_unk_40: cmdbuf.iogpu_unk_40, // 0x1c if internal program used + }), + unk_b8: Default::default(), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + job_params2 <- try_init!(fw::compute::raw::JobParameters2::ver { #[ver(V >= V13_0B4)] - unk_pad: Default::default(), - } - )) + unk_0_0: 0, + unk_0: Default::default(), + preempt_buf1: inner.preempt_buf.gpu_pointer(), + encoder_end: U64(cmdbuf.encoder_end), + unk_34: Default::default(), + #[ver(V < V13_0B4)] + unk_5c: 0, + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + large_tib: 0x0, // check! + unk_10: 0x0, // fixed + encoder_id: cmdbuf.encoder_id, + unk_18: 0x0, // fixed + iogpu_compute_unk44: cmdbuf.iogpu_unk_44, + seq_buffer: U64(inner.seq_buf.gpu_pointer().into()), + unk_28: U64(0x0), // fixed + }), + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + // TODO: make separate flag + no_preemption: ((cmdbuf.ctx_switch_prog & 1) == 0) as u8, + stamp: ev_comp.stamp_pointer, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + stamp_slot: ev_comp.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid, + event_seq: ev_comp.event_seq as u32, + }), + cur_ts: U64(0), + start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), + end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), + unk_2c0: 0, + unk_2c4: 0, + unk_2c8: 0, + unk_2cc: 0, + client_sequence: slot_client_seq, + pad_2d1: Default::default(), + unk_2d4: 0, + unk_2d8: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_2e1: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: U32(0), + #[ver(V >= V13_0B4)] + unk_pad: Default::default(), + }) }, )?; From 21bef13c83a87457ab4e515a3062d62cbf46b5b9 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 18:32:36 +0900 Subject: [PATCH 586/635] rust: macros: versions: Make parsing stricter This was previously breaking the init!() macros which use <- as part of their syntax (and would've also broken operators like <). Restrict the angle bracket nesting parsing to only start after ::, unless we're in a struct definition. Signed-off-by: Asahi Lina --- rust/macros/versions.rs | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs index 4caa88483f7787..89d43096d35808 100644 --- a/rust/macros/versions.rs +++ b/rust/macros/versions.rs @@ -18,18 +18,29 @@ fn expect_punct(it: &mut impl Iterator) -> String { } } -fn drop_until_punct(it: &mut impl Iterator, delimiter: &str) { +fn drop_until_punct(it: &mut impl Iterator, delimiter: &str, is_struct: bool) { let mut depth: isize = 0; + let mut colons: isize = 0; for token in it.by_ref() { if let TokenTree::Punct(punct) = token { match punct.as_char() { + ':' => { + colons += 1; + } '<' => { - depth += 1; + if depth > 0 || colons == 2 || is_struct { + depth += 1; + } + colons = 0; } '>' => { - depth -= 1; + if depth > 0 { + depth -= 1; + } + colons = 0; } _ => { + colons = 0; if depth == 0 && delimiter.contains(&punct.to_string()) { break; } @@ -41,16 +52,26 @@ fn drop_until_punct(it: &mut impl Iterator, delimiter: &str) { fn drop_until_braces(it: &mut impl Iterator) { let mut depth: isize = 0; + let mut colons: isize = 0; for token in it.by_ref() { match token { TokenTree::Punct(punct) => match punct.as_char() { + ':' => { + colons += 1; + } '<' => { - depth += 1; + if depth > 0 || colons == 2 { + depth += 1; + } + colons = 0; } '>' => { - depth -= 1; + if depth > 0 { + depth -= 1; + } + colons = 0; } - _ => (), + _ => colons = 0, }, TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => { if depth == 0 { @@ -169,7 +190,7 @@ fn filter_versions( TokenTree::Ident(ident) if ident.to_string() == "ver" => { if check_version(config, ver, &mut grp_it) { } else if is_struct { - drop_until_punct(&mut it, ","); + drop_until_punct(&mut it, ",", true); } else { let first = it.next().unwrap(); match &first { @@ -181,7 +202,7 @@ fn filter_versions( } TokenTree::Group(_) => (), _ => { - drop_until_punct(&mut it, ",;"); + drop_until_punct(&mut it, ",;", false); } } } From 8e1f2143d815ca815d26695a9b3f5b3d54344ef1 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 18:42:55 +0900 Subject: [PATCH 587/635] drm/asahi: queue::render: Port to init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/queue/render.rs | 1682 ++++++++++++------------- 1 file changed, 840 insertions(+), 842 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs index 2d007af1450e84..292a3b2f06d125 100644 --- a/drivers/gpu/drm/asahi/queue/render.rs +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -14,8 +14,8 @@ use crate::fw::types::*; use crate::gpu::GpuManager; use crate::util::*; use crate::workqueue::WorkError; -use crate::{box_in_place, inner_ptr, inner_weak_ptr, place}; use crate::{buffer, fw, gpu, microseq, workqueue}; +use crate::{inner_ptr, inner_weak_ptr}; use core::mem::MaybeUninit; use core::sync::atomic::Ordering; use kernel::dma_fence::RawDmaFence; @@ -410,22 +410,19 @@ impl super::Queue::ver { let frag_job = job.get_frag()?; mod_dev_dbg!(self.dev, "[Submission {}] Create Barrier\n", id); - let barrier: GpuObject = kalloc.private.new_inplace( - Default::default(), - |_inner, ptr: &mut MaybeUninit| { - Ok(place!( - ptr, - fw::workqueue::raw::Barrier { - tag: fw::workqueue::CommandType::Barrier, - wait_stamp: ev_vtx.fw_stamp_pointer, - wait_value: ev_vtx.value.next(), - wait_slot: ev_vtx.slot, - stamp_self: ev_frag.value.next(), - uuid: uuid_3d, - barrier_type: 0, - padding: Default::default(), - } - )) + let barrier = kalloc.private.new_init( + kernel::init::zeroed::(), + |_inner, _p| { + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: ev_vtx.fw_stamp_pointer, + wait_value: ev_vtx.value.next(), + wait_slot: ev_vtx.slot, + stamp_self: ev_frag.value.next(), + uuid: uuid_3d, + barrier_type: 0, + padding: Default::default(), + }) }, )?; @@ -587,148 +584,157 @@ impl super::Queue::ver { } mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); - let frag = GpuObject::new_prealloc( + let frag = GpuObject::new_init_prealloc( kalloc.gpu_ro.alloc_object()?, |ptr: GpuWeakPointer| { - let mut builder = microseq::Builder::new(); - - let stats = inner_weak_ptr!( - gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), - stats - ); - - let start_frag = builder.add(microseq::StartFragment::ver { - header: microseq::op::StartFragment::HEADER, - #[ver(G < G14X)] - job_params2: Some(inner_weak_ptr!(ptr, job_params2)), - #[ver(G < G14X)] - job_params1: Some(inner_weak_ptr!(ptr, job_params1)), - #[ver(G >= G14X)] - job_params1: None, - #[ver(G >= G14X)] - job_params2: None, - #[ver(G >= G14X)] - registers: inner_weak_ptr!(ptr, registers), - scene: scene.gpu_pointer(), - stats, - busy_flag: inner_weak_ptr!(ptr, busy_flag), - tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - work_queue: ev_frag.info_ptr, - work_item: ptr, - vm_slot: vm_bind.slot(), - unk_50: 0x1, // fixed - event_generation: self.id as u32, - buffer_slot: scene.slot(), - large_tib: large_tib as u32, - event_seq: U64(ev_frag.event_seq), - unk_68: 0, - unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), - unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), - #[ver(V >= V13_3)] - unk_7c_0: U64(0), - unk_7c: 0, - unk_80: 0, - unk_84: unk1.into(), - uuid: uuid_3d, - attachments: common::build_attachments( - cmdbuf.fragment_attachments, - cmdbuf.fragment_attachment_count, - )?, - padding: 0, - #[ver(V >= V13_0B4)] - counter: U64(count_frag), - #[ver(V >= V13_0B4)] - notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), - })?; - - if frag_result.is_some() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - cur_ts: inner_weak_ptr!(ptr, cur_ts), - start_ts: inner_weak_ptr!(ptr, start_ts), - update_ts: inner_weak_ptr!(ptr, start_ts), - work_queue: ev_frag.info_ptr, - unk_24: U64(0), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + let has_result = frag_result.is_some(); + let scene = scene.clone(); + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::fragment::RunFragment::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), + stats + ); + + let start_frag = builder.add(microseq::StartFragment::ver { + header: microseq::op::StartFragment::HEADER, + #[ver(G < G14X)] + job_params2: Some(inner_weak_ptr!(ptr, job_params2)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + job_params2: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + scene: scene.gpu_pointer(), + stats, + busy_flag: inner_weak_ptr!(ptr, busy_flag), + tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_50: 0x1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + large_tib: large_tib as u32, + event_seq: U64(ev_frag.event_seq), + unk_68: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + #[ver(V >= V13_3)] + unk_7c_0: U64(0), + unk_7c: 0, + unk_80: 0, + unk_84: unk1.into(), + uuid: uuid_3d, + attachments: common::build_attachments( + cmdbuf.fragment_attachments, + cmdbuf.fragment_attachment_count, + )?, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if has_result { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, start_ts), + work_queue: ev_frag.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } - #[ver(G < G14X)] - builder.add(microseq::WaitForIdle { - header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), - })?; - #[ver(G >= G14X)] - builder.add(microseq::WaitForIdle2 { - header: microseq::op::WaitForIdle2::HEADER, - })?; - - if frag_result.is_some() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - cur_ts: inner_weak_ptr!(ptr, cur_ts), - start_ts: inner_weak_ptr!(ptr, start_ts), - update_ts: inner_weak_ptr!(ptr, end_ts), - work_queue: ev_frag.info_ptr, - unk_24: U64(0), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if has_result { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, end_ts), + work_queue: ev_frag.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } - let off = builder.offset_to(start_frag); - builder.add(microseq::FinalizeFragment::ver { - header: microseq::op::FinalizeFragment::HEADER, - uuid: uuid_3d, - unk_8: 0, - fw_stamp: ev_frag.fw_stamp_pointer, - stamp_value: ev_frag.value.next(), - unk_18: 0, - scene: scene.weak_pointer(), - buffer: scene.weak_buffer_pointer(), - unk_2c: U64(1), - stats, - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - busy_flag: inner_weak_ptr!(ptr, busy_flag), - work_queue: ev_frag.info_ptr, - work_item: ptr, - vm_slot: vm_bind.slot(), - unk_60: 0, - unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), - #[ver(V >= V13_3)] - unk_6c_0: U64(0), - unk_6c: U64(0), - unk_74: U64(0), - unk_7c: U64(0), - unk_84: U64(0), - unk_8c: U64(0), - #[ver(G == G14 && V < V13_0B4)] - unk_8c_g14: U64(0), - restart_branch_offset: off, - has_attachments: (cmdbuf.fragment_attachment_count > 0) as u32, - #[ver(V >= V13_0B4)] - unk_9c: Default::default(), - })?; - - builder.add(microseq::RetireStamp { - header: microseq::op::RetireStamp::HEADER, - })?; - - Ok(box_in_place!(fw::fragment::RunFragment::ver { - notifier: notifier.clone(), - scene: scene.clone(), - micro_seq: builder.build(&mut kalloc.private)?, - vm_bind: vm_bind.clone(), + let off = builder.offset_to(start_frag); + builder.add(microseq::FinalizeFragment::ver { + header: microseq::op::FinalizeFragment::HEADER, + uuid: uuid_3d, + unk_8: 0, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + unk_18: 0, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + unk_2c: U64(1), + stats, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + busy_flag: inner_weak_ptr!(ptr, busy_flag), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_60: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + #[ver(V >= V13_3)] + unk_6c_0: U64(0), + unk_6c: U64(0), + unk_74: U64(0), + unk_7c: U64(0), + unk_84: U64(0), + unk_8c: U64(0), + #[ver(G == G14 && V < V13_0B4)] + unk_8c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (cmdbuf.fragment_attachment_count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_9c: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + builder.build(private)? + }, + notifier, + scene, + vm_bind, aux_fb: self.ualloc.lock().array_empty(0x8000)?, - timestamps: timestamps.clone(), - })?) + timestamps, + }) }, - |inner, ptr, _gpu_ptr| { + |inner, _ptr| { + let vm_slot = vm_bind.slot(); let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { iogpu_unk_214: cmdbuf.iogpu_unk_214, unk2: 0, @@ -738,318 +744,312 @@ impl super::Queue::ver { unk3: U64(unks.aux_fb_unk), }; - Ok(place!( - ptr, - fw::fragment::raw::RunFragment::ver { - tag: fw::workqueue::CommandType::RunFragment, - #[ver(V >= V13_0B4)] - counter: U64(count_frag), - vm_slot: vm_bind.slot(), - unk_8: 0, - microsequence: inner.micro_seq.gpu_pointer(), - microsequence_size: inner.micro_seq.len() as u32, - notifier: inner.notifier.gpu_pointer(), - buffer: inner.scene.buffer_pointer(), - scene: inner.scene.gpu_pointer(), - unk_buffer_buf: inner.scene.kernel_buffer_pointer(), - tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + try_init!(fw::fragment::raw::RunFragment::ver { + tag: fw::workqueue::CommandType::RunFragment, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + vm_slot, + unk_8: 0, + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + notifier: inner.notifier.gpu_pointer(), + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + samples: cmdbuf.samples, + tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, + tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, + unk_50: U64(0), + unk_58: U64(0), + merge_upper_x: F32::from_bits(cmdbuf.merge_upper_x), + merge_upper_y: F32::from_bits(cmdbuf.merge_upper_y), + unk_68: U64(0), + tile_count: U64(tile_info.tiles as u64), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { + utile_config, + unk_4: 0, + clear_pipeline: fw::fragment::raw::ClearPipelineBinding { + pipeline_bind: U64(cmdbuf.load_pipeline_bind as u64), + address: U64(cmdbuf.load_pipeline as u64), + }, ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - samples: cmdbuf.samples, - tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, - tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, - unk_50: U64(0), - unk_58: U64(0), + scissor_array: U64(cmdbuf.scissor_array), + depth_bias_array: U64(cmdbuf.depth_bias_array), + aux_fb_info, + depth_dimensions: U64(cmdbuf.depth_dimensions as u64), + visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), + zls_ctrl: U64(cmdbuf.zls_ctrl), + #[ver(G >= G14)] + unk_58_g14_0: U64(unks.g14_unk), + #[ver(G >= G14)] + unk_58_g14_8: U64(0), + depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), + depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), + stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), + stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), + #[ver(G >= G14)] + unk_68_g14_0: Default::default(), + unk_78: Default::default(), + depth_meta_buffer_ptr1: U64(cmdbuf.depth_meta_buffer_load), + unk_a0: Default::default(), + depth_meta_buffer_ptr2: U64(cmdbuf.depth_meta_buffer_store), + unk_b0: Default::default(), + stencil_meta_buffer_ptr1: U64(cmdbuf.stencil_meta_buffer_load), + unk_c0: Default::default(), + stencil_meta_buffer_ptr2: U64(cmdbuf.stencil_meta_buffer_store), + unk_d0: Default::default(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), + mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + tvb_heapmeta_2: inner.scene.tvb_heapmeta_pointer(), + tile_config: U64(tile_config), + aux_fb: inner.aux_fb.gpu_pointer(), + unk_108: Default::default(), + pipeline_base: U64(0x11_00000000), + unk_140: U64(unks.frg_unk_140), + unk_148: U64(0x0), + unk_150: U64(0x0), + unk_158: U64(unks.frg_unk_158), + unk_160: U64(0), + __pad: Default::default(), + #[ver(V < V13_0B4)] + __pad1: Default::default(), + }), + #[ver(G < G14X)] + job_params2 <- try_init!(fw::fragment::raw::JobParameters2 { + store_pipeline_bind: cmdbuf.store_pipeline_bind, + store_pipeline_addr: cmdbuf.store_pipeline, + unk_8: 0x0, + unk_c: 0x0, merge_upper_x: F32::from_bits(cmdbuf.merge_upper_x), merge_upper_y: F32::from_bits(cmdbuf.merge_upper_y), - unk_68: U64(0), - tile_count: U64(tile_info.tiles as u64), - #[ver(G < G14X)] - job_params1: fw::fragment::raw::JobParameters1::ver { - utile_config: utile_config, - unk_4: 0, - clear_pipeline: fw::fragment::raw::ClearPipelineBinding { - pipeline_bind: U64(cmdbuf.load_pipeline_bind as u64), - address: U64(cmdbuf.load_pipeline as u64), - }, - ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - scissor_array: U64(cmdbuf.scissor_array), - depth_bias_array: U64(cmdbuf.depth_bias_array), - aux_fb_info: aux_fb_info, - depth_dimensions: U64(cmdbuf.depth_dimensions as u64), - visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), - zls_ctrl: U64(cmdbuf.zls_ctrl), - #[ver(G >= G14)] - unk_58_g14_0: U64(unks.g14_unk), - #[ver(G >= G14)] - unk_58_g14_8: U64(0), - depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), - depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), - stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), - stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), - #[ver(G >= G14)] - unk_68_g14_0: Default::default(), - unk_78: Default::default(), - depth_meta_buffer_ptr1: U64(cmdbuf.depth_meta_buffer_load), - unk_a0: Default::default(), - depth_meta_buffer_ptr2: U64(cmdbuf.depth_meta_buffer_store), - unk_b0: Default::default(), - stencil_meta_buffer_ptr1: U64(cmdbuf.stencil_meta_buffer_load), - unk_c0: Default::default(), - stencil_meta_buffer_ptr2: U64(cmdbuf.stencil_meta_buffer_store), - unk_d0: Default::default(), - tvb_tilemap: inner.scene.tvb_tilemap_pointer(), - tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), - mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), - tvb_heapmeta_2: inner.scene.tvb_heapmeta_pointer(), - tile_config: U64(tile_config), - aux_fb: inner.aux_fb.gpu_pointer(), - unk_108: Default::default(), - pipeline_base: U64(0x11_00000000), - unk_140: U64(unks.frg_unk_140), - unk_148: U64(0x0), - unk_150: U64(0x0), - unk_158: U64(unks.frg_unk_158), - unk_160: U64(0), - __pad: Default::default(), - #[ver(V < V13_0B4)] - __pad1: Default::default(), - }, - #[ver(G < G14X)] - job_params2: fw::fragment::raw::JobParameters2 { - store_pipeline_bind: cmdbuf.store_pipeline_bind, - store_pipeline_addr: cmdbuf.store_pipeline, - unk_8: 0x0, - unk_c: 0x0, - merge_upper_x: F32::from_bits(cmdbuf.merge_upper_x), - merge_upper_y: F32::from_bits(cmdbuf.merge_upper_y), - unk_18: U64(0x0), - utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, - utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, - unk_24: 0x0, - tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), - tib_blocks: cmdbuf.tib_blocks, - isp_bgobjdepth: cmdbuf.isp_bgobjdepth, - // TODO: does this flag need to be exposed to userspace? - isp_bgobjvals: unks.load_bgobjvals as u32, - unk_38: unks.frg_unk_38 as u32, - unk_3c: unks.frg_unk_3c as u32, - unk_40: unks.frg_unk_40 as u32, - __pad: Default::default(), + unk_18: U64(0x0), + utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, + utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, + unk_24: 0x0, + tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), + tib_blocks: cmdbuf.tib_blocks, + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + // TODO: does this flag need to be exposed to userspace? + isp_bgobjvals: unks.load_bgobjvals as u32, + unk_38: unks.frg_unk_38 as u32, + unk_3c: unks.frg_unk_3c as u32, + unk_40: unks.frg_unk_40 as u32, + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1739, 1); + r.add(0x10009, utile_config.into()); + r.add(0x15379, cmdbuf.store_pipeline_bind.into()); + r.add(0x15381, cmdbuf.store_pipeline.into()); + r.add(0x15369, cmdbuf.load_pipeline_bind.into()); + r.add(0x15371, cmdbuf.load_pipeline.into()); + r.add(0x15131, cmdbuf.merge_upper_x.into()); + r.add(0x15139, cmdbuf.merge_upper_y.into()); + r.add(0x100a1, 0); + r.add(0x15069, 0); + r.add(0x15071, 0); // pointer + r.add(0x16058, 0); + r.add(0x10019, cmdbuf.ppp_multisamplectl); + + let isp_mtile_size = (tile_info.utiles_per_mtile_y + | (tile_info.utiles_per_mtile_x << 16)) + .into(); + r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE + r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE + r.add( + 0x100d9, + (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)).into(), + ); // TE_SCREEN + r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x15109, cmdbuf.scissor_array); // ISP_SCISSOR_BASE + r.add(0x15101, cmdbuf.depth_bias_array); // ISP_DBIAS_BASE + r.add(0x15021, cmdbuf.iogpu_unk_214.into()); // aux_fb_info.unk_1 + r.add( + 0x15211, + ((cmdbuf.fb_height as u64) << 32) | cmdbuf.fb_width as u64, + ); // aux_fb_info.{width, heigh + r.add(0x15049, unks.aux_fb_unk); // s2.aux_fb_info.unk3 + r.add(0x10051, cmdbuf.tib_blocks.into()); // s1.unk_2c + r.add(0x15321, cmdbuf.depth_dimensions.into()); // ISP_ZLS_PIXELS + r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH + r.add(0x15309, unks.load_bgobjvals); // ISP_BGOBJVALS + r.add(0x15311, cmdbuf.visibility_result_buffer); // ISP_OCLQRY_BASE + r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL + r.add(0x15349, unks.g14_unk); // s2.unk_58_g14_0 + r.add(0x15351, 0); // s2.unk_58_g14_8 + r.add(0x15329, cmdbuf.depth_buffer_load); // ISP_ZLOAD_BASE + r.add(0x15331, cmdbuf.depth_buffer_store); // ISP_ZSTORE_BASE + r.add(0x15339, cmdbuf.stencil_buffer_load); // ISP_STENCIL_LOAD_BASE + r.add(0x15341, cmdbuf.stencil_buffer_store); // ISP_STENCIL_STORE_BASE + r.add(0x15231, 0); + r.add(0x15221, 0); + r.add(0x15239, 0); + r.add(0x15229, 0); + r.add(0x15401, 0); + r.add(0x15421, 0); + r.add(0x15409, 0); + r.add(0x15429, 0); + r.add(0x153c1, cmdbuf.depth_meta_buffer_load); + r.add(0x15411, 0); + r.add(0x153c9, cmdbuf.depth_meta_buffer_store); + r.add(0x15431, 0); + r.add(0x153d1, cmdbuf.stencil_meta_buffer_load); + r.add(0x15419, 0); + r.add(0x153d9, cmdbuf.stencil_meta_buffer_store); + r.add(0x15439, 0); + r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x16060, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x10039, tile_config); // tile_config ISP_CTL? + r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN + r.add(0x11821, 0x0); // some shader? + r.add(0x11829, 0); + r.add(0x11f79, 0); + r.add(0x15359, 0); + r.add(0x10069, 0x11_00000000); // USC_EXEC_BASE_ISP + r.add(0x16020, 0); + r.add(0x16461, inner.aux_fb.gpu_pointer().into()); + r.add(0x16090, inner.aux_fb.gpu_pointer().into()); + r.add(0x120a1, unks.frg_unk_158); + r.add(0x160a8, 0); + r.add(0x16068, unks.frg_tilecfg); + r.add(0x160b8, 0x0); + } + ), + job_params3 <- try_init!(fw::fragment::raw::JobParameters3::ver { + depth_bias_array: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.depth_bias_array), + unk_padding: U64(0), }, - #[ver(G >= G14X)] - registers: fw::job::raw::RegisterArray::new( - inner_weak_ptr!(_gpu_ptr, registers.registers), - |r| { - r.add(0x1739, 1); - r.add(0x10009, utile_config.into()); - r.add(0x15379, cmdbuf.store_pipeline_bind.into()); - r.add(0x15381, cmdbuf.store_pipeline.into()); - r.add(0x15369, cmdbuf.load_pipeline_bind.into()); - r.add(0x15371, cmdbuf.load_pipeline.into()); - r.add(0x15131, cmdbuf.merge_upper_x.into()); - r.add(0x15139, cmdbuf.merge_upper_y.into()); - r.add(0x100a1, 0); - r.add(0x15069, 0); - r.add(0x15071, 0); // pointer - r.add(0x16058, 0); - r.add(0x10019, cmdbuf.ppp_multisamplectl); - - let isp_mtile_size = (tile_info.utiles_per_mtile_y - | (tile_info.utiles_per_mtile_x << 16)) - .into(); - r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE - r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE - r.add( - 0x100d9, - (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)) - .into(), - ); // TE_SCREEN - r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); - r.add(0x15109, cmdbuf.scissor_array); // ISP_SCISSOR_BASE - r.add(0x15101, cmdbuf.depth_bias_array); // ISP_DBIAS_BASE - r.add(0x15021, cmdbuf.iogpu_unk_214.into()); // aux_fb_info.unk_1 - r.add( - 0x15211, - ((cmdbuf.fb_height as u64) << 32) | cmdbuf.fb_width as u64, - ); // aux_fb_info.{width, heigh - r.add(0x15049, unks.aux_fb_unk); // s2.aux_fb_info.unk3 - r.add(0x10051, cmdbuf.tib_blocks.into()); // s1.unk_2c - r.add(0x15321, cmdbuf.depth_dimensions.into()); // ISP_ZLS_PIXELS - r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH - r.add(0x15309, unks.load_bgobjvals); // ISP_BGOBJVALS - r.add(0x15311, cmdbuf.visibility_result_buffer); // ISP_OCLQRY_BASE - r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL - r.add(0x15349, unks.g14_unk); // s2.unk_58_g14_0 - r.add(0x15351, 0); // s2.unk_58_g14_8 - r.add(0x15329, cmdbuf.depth_buffer_load); // ISP_ZLOAD_BASE - r.add(0x15331, cmdbuf.depth_buffer_store); // ISP_ZSTORE_BASE - r.add(0x15339, cmdbuf.stencil_buffer_load); // ISP_STENCIL_LOAD_BASE - r.add(0x15341, cmdbuf.stencil_buffer_store); // ISP_STENCIL_STORE_BASE - r.add(0x15231, 0); - r.add(0x15221, 0); - r.add(0x15239, 0); - r.add(0x15229, 0); - r.add(0x15401, 0); - r.add(0x15421, 0); - r.add(0x15409, 0); - r.add(0x15429, 0); - r.add(0x153c1, cmdbuf.depth_meta_buffer_load); - r.add(0x15411, 0); - r.add(0x153c9, cmdbuf.depth_meta_buffer_store); - r.add(0x15431, 0); - r.add(0x153d1, cmdbuf.stencil_meta_buffer_load); - r.add(0x15419, 0); - r.add(0x153d9, cmdbuf.stencil_meta_buffer_store); - r.add(0x15439, 0); - r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); - r.add(0x16060, inner.scene.tvb_heapmeta_pointer().into()); - r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? - r.add(0x10039, tile_config); // tile_config ISP_CTL? - r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN - r.add(0x11821, 0x0); // some shader? - r.add(0x11829, 0); - r.add(0x11f79, 0); - r.add(0x15359, 0); - r.add(0x10069, 0x11_00000000); // USC_EXEC_BASE_ISP - r.add(0x16020, 0); - r.add(0x16461, inner.aux_fb.gpu_pointer().into()); - r.add(0x16090, inner.aux_fb.gpu_pointer().into()); - r.add(0x120a1, unks.frg_unk_158); - r.add(0x160a8, 0); - r.add(0x16068, unks.frg_tilecfg); - r.add(0x160b8, 0x0); - } - ), - job_params3: fw::fragment::raw::JobParameters3::ver { - depth_bias_array: fw::fragment::raw::ArrayAddr { - ptr: U64(cmdbuf.depth_bias_array), - unk_padding: U64(0), - }, - scissor_array: fw::fragment::raw::ArrayAddr { - ptr: U64(cmdbuf.scissor_array), - unk_padding: U64(0), - }, - visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), - unk_118: U64(0x0), - unk_120: Default::default(), - unk_reload_pipeline: fw::fragment::raw::ClearPipelineBinding { - pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), - address: U64(cmdbuf.partial_reload_pipeline as u64), - }, - unk_258: U64(0), - unk_260: U64(0), - unk_268: U64(0), - unk_270: U64(0), - reload_pipeline: fw::fragment::raw::ClearPipelineBinding { - pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), - address: U64(cmdbuf.partial_reload_pipeline as u64), - }, - zls_ctrl: U64(unks.reload_zlsctrl), - #[ver(G >= G14X)] - unk_290: U64(unks.g14_unk), - #[ver(G < G14X)] - unk_290: U64(0x0), - depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), - unk_2a0: U64(0x0), - unk_2a8: U64(0x0), - depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), - depth_buffer_ptr3: U64(cmdbuf.depth_buffer_partial), - depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_partial), - stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), - unk_2d0: U64(0x0), - unk_2d8: U64(0x0), - stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), - stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_partial), - stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_partial), - unk_2f8: Default::default(), - tib_blocks: cmdbuf.tib_blocks, - unk_30c: 0x0, - aux_fb_info: aux_fb_info, - tile_config: U64(tile_config), - unk_328_padding: Default::default(), - unk_partial_store_pipeline: - fw::fragment::raw::StorePipelineBinding::new( - cmdbuf.partial_store_pipeline_bind, - cmdbuf.partial_store_pipeline - ), - partial_store_pipeline: fw::fragment::raw::StorePipelineBinding::new( - cmdbuf.partial_store_pipeline_bind, - cmdbuf.partial_store_pipeline - ), - isp_bgobjdepth: cmdbuf.isp_bgobjdepth, - isp_bgobjvals: cmdbuf.isp_bgobjvals, - sample_size: cmdbuf.sample_size, - unk_37c: 0x0, - unk_380: U64(0x0), - unk_388: U64(0x0), - #[ver(V >= V13_0B4)] - unk_390_0: U64(0x0), - depth_dimensions: U64(cmdbuf.depth_dimensions as u64), + scissor_array: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.scissor_array), + unk_padding: U64(0), }, - unk_758_flag: 0, - unk_75c_flag: 0, - unk_buf: Default::default(), - busy_flag: 0, - tvb_overflow_count: 0, - unk_878: 0, - encoder_params: fw::job::raw::EncoderParams { - unk_8: (cmdbuf.flags - & uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 - != 0) as u32, - large_tib: large_tib as u32, - unk_10: 0x0, // fixed - encoder_id: cmdbuf.encoder_id, - unk_18: 0x0, // fixed - iogpu_compute_unk44: unks.frg_iogpucmp_unk44 as u32, - seq_buffer: U64(unks.frg_seq_buffer), - unk_28: U64(0x0), // fixed + visibility_result_buffer: U64(cmdbuf.visibility_result_buffer), + unk_118: U64(0x0), + unk_120: Default::default(), + unk_reload_pipeline: fw::fragment::raw::ClearPipelineBinding { + pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), + address: U64(cmdbuf.partial_reload_pipeline as u64), }, - process_empty_tiles: (cmdbuf.flags - & uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 - != 0) as u32, - no_clear_pipeline_textures: (cmdbuf.flags - & uapi::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES as u64 - != 0) as u32, - msaa_zs: (cmdbuf.flags & uapi::ASAHI_RENDER_MSAA_ZS as u64 != 0) as u32, - unk_pointee: 0, - #[ver(V >= V13_3)] - unk_v13_3: 0, - meta: fw::job::raw::JobMeta { - unk_0: 0, - unk_2: 0, - no_preemption: 0, - stamp: ev_frag.stamp_pointer, - fw_stamp: ev_frag.fw_stamp_pointer, - stamp_value: ev_frag.value.next(), - stamp_slot: ev_frag.slot, - evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, - uuid: uuid_3d, - event_seq: ev_frag.event_seq as u32, + unk_258: U64(0), + unk_260: U64(0), + unk_268: U64(0), + unk_270: U64(0), + reload_pipeline: fw::fragment::raw::ClearPipelineBinding { + pipeline_bind: U64(cmdbuf.partial_reload_pipeline_bind as u64), + address: U64(cmdbuf.partial_reload_pipeline as u64), }, - unk_after_meta: unk1.into(), - unk_buf_0: U64(0), - unk_buf_8: U64(0), - #[ver(G < G14X)] - unk_buf_10: U64(1), + zls_ctrl: U64(unks.reload_zlsctrl), #[ver(G >= G14X)] - unk_buf_10: U64(0), - cur_ts: U64(0), - start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), - end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), - unk_914: 0, - unk_918: U64(0), - unk_920: 0, - client_sequence: slot_client_seq, - pad_925: Default::default(), - unk_928: 0, - unk_92c: 0, - #[ver(V >= V13_0B4)] - unk_ts: U64(0), + unk_290: U64(unks.g14_unk), + #[ver(G < G14X)] + unk_290: U64(0x0), + depth_buffer_ptr1: U64(cmdbuf.depth_buffer_load), + unk_2a0: U64(0x0), + unk_2a8: U64(0x0), + depth_buffer_ptr2: U64(cmdbuf.depth_buffer_store), + depth_buffer_ptr3: U64(cmdbuf.depth_buffer_partial), + depth_meta_buffer_ptr3: U64(cmdbuf.depth_meta_buffer_partial), + stencil_buffer_ptr1: U64(cmdbuf.stencil_buffer_load), + unk_2d0: U64(0x0), + unk_2d8: U64(0x0), + stencil_buffer_ptr2: U64(cmdbuf.stencil_buffer_store), + stencil_buffer_ptr3: U64(cmdbuf.stencil_buffer_partial), + stencil_meta_buffer_ptr3: U64(cmdbuf.stencil_meta_buffer_partial), + unk_2f8: Default::default(), + tib_blocks: cmdbuf.tib_blocks, + unk_30c: 0x0, + aux_fb_info, + tile_config: U64(tile_config), + unk_328_padding: Default::default(), + unk_partial_store_pipeline: fw::fragment::raw::StorePipelineBinding::new( + cmdbuf.partial_store_pipeline_bind, + cmdbuf.partial_store_pipeline + ), + partial_store_pipeline: fw::fragment::raw::StorePipelineBinding::new( + cmdbuf.partial_store_pipeline_bind, + cmdbuf.partial_store_pipeline + ), + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + isp_bgobjvals: cmdbuf.isp_bgobjvals, + sample_size: cmdbuf.sample_size, + unk_37c: 0x0, + unk_380: U64(0x0), + unk_388: U64(0x0), #[ver(V >= V13_0B4)] - unk_92d_8: Default::default(), - } - )) + unk_390_0: U64(0x0), + depth_dimensions: U64(cmdbuf.depth_dimensions as u64), + }), + unk_758_flag: 0, + unk_75c_flag: 0, + unk_buf: Default::default(), + busy_flag: 0, + tvb_overflow_count: 0, + unk_878: 0, + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: (cmdbuf.flags & uapi::ASAHI_RENDER_SET_WHEN_RELOADING_Z_OR_S as u64 + != 0) as u32, + large_tib: large_tib as u32, + unk_10: 0x0, // fixed + encoder_id: cmdbuf.encoder_id, + unk_18: 0x0, // fixed + iogpu_compute_unk44: unks.frg_iogpucmp_unk44 as u32, + seq_buffer: U64(unks.frg_seq_buffer), + unk_28: U64(0x0), // fixed + }), + process_empty_tiles: (cmdbuf.flags + & uapi::ASAHI_RENDER_PROCESS_EMPTY_TILES as u64 + != 0) as u32, + no_clear_pipeline_textures: (cmdbuf.flags + & uapi::ASAHI_RENDER_NO_CLEAR_PIPELINE_TEXTURES as u64 + != 0) as u32, + msaa_zs: (cmdbuf.flags & uapi::ASAHI_RENDER_MSAA_ZS as u64 != 0) as u32, + unk_pointee: 0, + #[ver(V >= V13_3)] + unk_v13_3: 0, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: 0, + stamp: ev_frag.stamp_pointer, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + stamp_slot: ev_frag.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_3d, + event_seq: ev_frag.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + #[ver(G < G14X)] + unk_buf_10: U64(1), + #[ver(G >= G14X)] + unk_buf_10: U64(0), + cur_ts: U64(0), + start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), + end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), + unk_914: 0, + unk_918: U64(0), + unk_920: 0, + client_sequence: slot_client_seq, + pad_925: Default::default(), + unk_928: 0, + unk_92c: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_92d_8: Default::default(), + }) }, )?; @@ -1080,23 +1080,22 @@ impl super::Queue::ver { if scene.rebind() || tvb_grown || tvb_autogrown { mod_dev_dbg!(self.dev, "[Submission {}] Create Bind Buffer\n", id); - let bind_buffer = kalloc.private.new_inplace( - fw::buffer::InitBuffer::ver { - scene: scene.clone(), + let bind_buffer = kalloc.private.new_init( + { + let scene = scene.clone(); + try_init!(fw::buffer::InitBuffer::ver { scene }) }, - |inner, ptr: &mut MaybeUninit>| { - Ok(place!( - ptr, - fw::buffer::raw::InitBuffer::ver { - tag: fw::workqueue::CommandType::InitBuffer, - vm_slot: vm_bind.slot(), - buffer_slot: inner.scene.slot(), - unk_c: 0, - block_count: buffer.block_count(), - buffer: inner.scene.buffer_pointer(), - stamp_value: ev_vtx.value.next(), - } - )) + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::buffer::raw::InitBuffer::ver { + tag: fw::workqueue::CommandType::InitBuffer, + vm_slot, + buffer_slot: inner.scene.slot(), + unk_c: 0, + block_count: buffer.block_count(), + buffer: inner.scene.buffer_pointer(), + stamp_value: ev_vtx.value.next(), + }) }, )?; @@ -1105,387 +1104,386 @@ impl super::Queue::ver { } mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); - let vtx = GpuObject::new_prealloc( + let vtx = GpuObject::new_init_prealloc( kalloc.gpu_ro.alloc_object()?, |ptr: GpuWeakPointer| { - let mut builder = microseq::Builder::new(); + let has_result = vtx_result.is_some(); + let scene = scene.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::vertex::RunVertex::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), + stats + ); + + let start_vtx = builder.add(microseq::StartVertex::ver { + header: microseq::op::StartVertex::HEADER, + #[ver(G < G14X)] + tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + tiling_params: None, + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + buffer: scene.weak_buffer_pointer(), + scene: scene.weak_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_38: 1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + unk_44: 0, + event_seq: U64(ev_vtx.event_seq), + unk_50: 0, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + unk_64: 0x0, // fixed + unk_68: unk1.into(), + uuid: uuid_ta, + attachments: common::build_attachments( + cmdbuf.vertex_attachments, + cmdbuf.vertex_attachment_count, + )?, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + unk_178: 0x0, // padding? + })?; + + if has_result { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, start_ts), + work_queue: ev_vtx.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if has_result { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + cur_ts: inner_weak_ptr!(ptr, cur_ts), + start_ts: inner_weak_ptr!(ptr, start_ts), + update_ts: inner_weak_ptr!(ptr, end_ts), + work_queue: ev_vtx.info_ptr, + unk_24: U64(0), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } - let stats = inner_weak_ptr!( - gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), - stats - ); + let off = builder.offset_to(start_vtx); + builder.add(microseq::FinalizeVertex::ver { + header: microseq::op::FinalizeVertex::HEADER, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x0, // fixed + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_34: 0x0, // fixed + uuid: uuid_ta, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + unk_48: U64(0x0), // fixed + unk_50: 0x0, // fixed + unk_54: 0x0, // fixed + unk_58: U64(0x0), // fixed + unk_60: 0x0, // fixed + unk_64: 0x0, // fixed + unk_68: 0x0, // fixed + #[ver(G >= G14 && V < V13_0B4)] + unk_68_g14: U64(0), + restart_branch_offset: off, + has_attachments: (cmdbuf.vertex_attachment_count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_74: Default::default(), // Ventura + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(private)? + }, + notifier, + scene, + vm_bind, + timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + #[ver(G < G14)] + let core_masks = gpu.core_masks_packed(); - let start_vtx = builder.add(microseq::StartVertex::ver { - header: microseq::op::StartVertex::HEADER, + try_init!(fw::vertex::raw::RunVertex::ver { + tag: fw::workqueue::CommandType::RunVertex, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + vm_slot, + unk_8: 0, + notifier: inner.notifier.gpu_pointer(), + buffer_slot: inner.scene.slot(), + unk_1c: 0, + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + unk_34: 0, #[ver(G < G14X)] - tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), + job_params1 <- try_init!(fw::vertex::raw::JobParameters1::ver { + unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 + unk_8: f32!(1e-20), // fixed + unk_c: f32!(1e-20), // fixed + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + #[ver(G < G14)] + tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), + tpc: inner.scene.tpc_pointer(), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer().or(0x8000_0000_0000_0000), + iogpu_unk_54: U64(unks.iogpu_unk54), // fixed + iogpu_unk_56: U64(unks.iogpu_unk56), // fixed + #[ver(G < G14)] + tvb_cluster_meta1: inner + .scene + .meta_1_pointer() + .map(|x| x.or((tile_info.meta1_blocks as u64) << 50)), + utile_config, + unk_4c: 0, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed + tvb_heapmeta_2: inner.scene.tvb_heapmeta_pointer(), + #[ver(G < G14)] + unk_60: U64(0x0), // fixed + #[ver(G < G14)] + core_mask: Array::new([ + *core_masks.first().unwrap_or(&0), + *core_masks.get(1).unwrap_or(&0), + ]), + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + preempt_buf2: inner.scene.preempt_buf_2_pointer(), + unk_80: U64(0x1), // fixed + preempt_buf3: inner.scene.preempt_buf_3_pointer().or(0x4_0000_0000_0000), // check + encoder_addr: U64(cmdbuf.encoder_ptr), + #[ver(G < G14)] + tvb_cluster_meta2: inner.scene.meta_2_pointer(), + #[ver(G < G14)] + tvb_cluster_meta3: inner.scene.meta_3_pointer(), + #[ver(G < G14)] + tiling_control, + #[ver(G < G14)] + unk_ac: unks.tiling_control_2 as u32, // fixed + unk_b0: Default::default(), // fixed + pipeline_base: U64(0x11_00000000), + #[ver(G < G14)] + tvb_cluster_meta4: inner + .scene + .meta_4_pointer() + .map(|x| x.or(0x3000_0000_0000_0000)), + #[ver(G < G14)] + unk_f0: U64(unks.vtx_unk_f0), + unk_f8: U64(unks.vtx_unk_f8), // fixed + unk_100: Default::default(), // fixed + unk_118: unks.vtx_unk_118 as u32, // fixed + __pad: Default::default(), + }), #[ver(G < G14X)] - job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + tiling_params: tile_info.params, #[ver(G >= G14X)] - tiling_params: None, - #[ver(G >= G14X)] - job_params1: None, - #[ver(G >= G14X)] - registers: inner_weak_ptr!(ptr, registers), - buffer: scene.weak_buffer_pointer(), - scene: scene.weak_pointer(), - stats, - work_queue: ev_vtx.info_ptr, - vm_slot: vm_bind.slot(), - unk_38: 1, // fixed - event_generation: self.id as u32, - buffer_slot: scene.slot(), - unk_44: 0, - event_seq: U64(ev_vtx.event_seq), - unk_50: 0, - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), - unk_64: 0x0, // fixed - unk_68: unk1.into(), - uuid: uuid_ta, - attachments: common::build_attachments( - cmdbuf.vertex_attachments, - cmdbuf.vertex_attachment_count, - )?, - padding: 0, - #[ver(V >= V13_0B4)] - counter: U64(count_vtx), - #[ver(V >= V13_0B4)] - notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), - unk_178: 0x0, // padding? - })?; - - if vtx_result.is_some() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - cur_ts: inner_weak_ptr!(ptr, cur_ts), - start_ts: inner_weak_ptr!(ptr, start_ts), - update_ts: inner_weak_ptr!(ptr, start_ts), - work_queue: ev_vtx.info_ptr, - unk_24: U64(0), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } - - #[ver(G < G14X)] - builder.add(microseq::WaitForIdle { - header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), - })?; - #[ver(G >= G14X)] - builder.add(microseq::WaitForIdle2 { - header: microseq::op::WaitForIdle2::HEADER, - })?; - - if vtx_result.is_some() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - cur_ts: inner_weak_ptr!(ptr, cur_ts), - start_ts: inner_weak_ptr!(ptr, start_ts), - update_ts: inner_weak_ptr!(ptr, end_ts), - work_queue: ev_vtx.info_ptr, - unk_24: U64(0), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 + r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); + + let cl_tilemaps_ptr = inner + .scene + .cluster_tilemaps_pointer() + .map_or(0, |a| a.into()); + r.add(0x1c041, cl_tilemaps_ptr); + r.add(0x1c9d0, cl_tilemaps_ptr); + r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR - let off = builder.offset_to(start_vtx); - builder.add(microseq::FinalizeVertex::ver { - header: microseq::op::FinalizeVertex::HEADER, - scene: scene.weak_pointer(), - buffer: scene.weak_buffer_pointer(), - stats, - work_queue: ev_vtx.info_ptr, - vm_slot: vm_bind.slot(), - unk_28: 0x0, // fixed - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - unk_34: 0x0, // fixed - uuid: uuid_ta, - fw_stamp: ev_vtx.fw_stamp_pointer, - stamp_value: ev_vtx.value.next(), - unk_48: U64(0x0), // fixed - unk_50: 0x0, // fixed - unk_54: 0x0, // fixed - unk_58: U64(0x0), // fixed - unk_60: 0x0, // fixed - unk_64: 0x0, // fixed - unk_68: 0x0, // fixed - #[ver(G >= G14 && V < V13_0B4)] - unk_68_g14: U64(0), - restart_branch_offset: off, - has_attachments: (cmdbuf.vertex_attachment_count > 0) as u32, - #[ver(V >= V13_0B4)] - unk_74: Default::default(), // Ventura - })?; - - builder.add(microseq::RetireStamp { - header: microseq::op::RetireStamp::HEADER, - })?; - - Ok(box_in_place!(fw::vertex::RunVertex::ver { - notifier: notifier, - scene: scene.clone(), - micro_seq: builder.build(&mut kalloc.private)?, - vm_bind: vm_bind.clone(), - timestamps: timestamps, - })?) - }, - |inner, ptr, _gpu_ptr| { - #[ver(G < G14)] - let core_masks = gpu.core_masks_packed(); - Ok(place!( - ptr, - fw::vertex::raw::RunVertex::ver { - tag: fw::workqueue::CommandType::RunVertex, - #[ver(V >= V13_0B4)] - counter: U64(count_vtx), - vm_slot: vm_bind.slot(), - unk_8: 0, - notifier: inner.notifier.gpu_pointer(), - buffer_slot: inner.scene.slot(), - unk_1c: 0, - buffer: inner.scene.buffer_pointer(), - scene: inner.scene.gpu_pointer(), - unk_buffer_buf: inner.scene.kernel_buffer_pointer(), - unk_34: 0, - #[ver(G < G14X)] - job_params1: fw::vertex::raw::JobParameters1::ver { - unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 - unk_8: f32!(1e-20), // fixed - unk_c: f32!(1e-20), // fixed - tvb_tilemap: inner.scene.tvb_tilemap_pointer(), - #[ver(G < G14)] - tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), - tpc: inner.scene.tpc_pointer(), - tvb_heapmeta: inner + let tvb_heapmeta_ptr = inner .scene .tvb_heapmeta_pointer() - .or(0x8000_0000_0000_0000), - iogpu_unk_54: U64(unks.iogpu_unk54), // fixed - iogpu_unk_56: U64(unks.iogpu_unk56), // fixed - #[ver(G < G14)] - tvb_cluster_meta1: inner - .scene - .meta_1_pointer() - .map(|x| x.or((tile_info.meta1_blocks as u64) << 50)), - utile_config: utile_config, - unk_4c: 0, - ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed - tvb_heapmeta_2: inner.scene.tvb_heapmeta_pointer(), - #[ver(G < G14)] - unk_60: U64(0x0), // fixed - #[ver(G < G14)] - core_mask: Array::new([ - *core_masks.first().unwrap_or(&0), - *core_masks.get(1).unwrap_or(&0), - ]), - preempt_buf1: inner.scene.preempt_buf_1_pointer(), - preempt_buf2: inner.scene.preempt_buf_2_pointer(), - unk_80: U64(0x1), // fixed - preempt_buf3: inner - .scene - .preempt_buf_3_pointer() - .or(0x4_0000_0000_0000), // check - encoder_addr: U64(cmdbuf.encoder_ptr), - #[ver(G < G14)] - tvb_cluster_meta2: inner.scene.meta_2_pointer(), - #[ver(G < G14)] - tvb_cluster_meta3: inner.scene.meta_3_pointer(), - #[ver(G < G14)] - tiling_control: tiling_control, - #[ver(G < G14)] - unk_ac: unks.tiling_control_2 as u32, // fixed - unk_b0: Default::default(), // fixed - pipeline_base: U64(0x11_00000000), - #[ver(G < G14)] - tvb_cluster_meta4: inner - .scene - .meta_4_pointer() - .map(|x| x.or(0x3000_0000_0000_0000)), - #[ver(G < G14)] - unk_f0: U64(unks.vtx_unk_f0), - unk_f8: U64(unks.vtx_unk_f8), // fixed - unk_100: Default::default(), // fixed - unk_118: unks.vtx_unk_118 as u32, // fixed - __pad: Default::default(), - }, - #[ver(G < G14X)] - tiling_params: tile_info.params, - #[ver(G >= G14X)] - registers: fw::job::raw::RegisterArray::new( - inner_weak_ptr!(_gpu_ptr, registers.registers), - |r| { - r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 - r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); - r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); - - let cl_tilemaps_ptr = inner + .or(0x8000_0000_0000_0000) + .into(); + r.add(0x1c031, tvb_heapmeta_ptr); + r.add(0x1c9c0, tvb_heapmeta_ptr); + r.add(0x1c051, unks.iogpu_unk54); // iogpu_unk_54/55 + r.add(0x1c061, unks.iogpu_unk56); // iogpu_unk_56 + r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config + r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL + r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c958, 1); // s2.unk_80 + r.add( + 0x1c950, + inner .scene - .cluster_tilemaps_pointer() - .map_or(0, |a| a.into()); - r.add(0x1c041, cl_tilemaps_ptr); - r.add(0x1c9d0, cl_tilemaps_ptr); - r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR - - let tvb_heapmeta_ptr = inner - .scene - .tvb_heapmeta_pointer() - .or(0x8000_0000_0000_0000) - .into(); - r.add(0x1c031, tvb_heapmeta_ptr); - r.add(0x1c9c0, tvb_heapmeta_ptr); - r.add(0x1c051, unks.iogpu_unk54); // iogpu_unk_54/55 - r.add(0x1c061, unks.iogpu_unk56); // iogpu_unk_56 - r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config - r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL - r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); - r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); - r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); - r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); - r.add(0x1c958, 1); // s2.unk_80 - r.add( - 0x1c950, - inner - .scene - .preempt_buf_3_pointer() - .or(0x4_0000_0000_0000) - .into(), - ); - r.add(0x1c930, 0); // VCE related addr, lsb to enable - r.add(0x1c880, cmdbuf.encoder_ptr); // VDM_CTRL_STREAM_BASE - r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. - r.add( - 0x1c948, - inner.scene.meta_2_pointer().map_or(0, |a| a.into()), - ); // tvb_cluster_meta2 - r.add( - 0x1c888, - inner.scene.meta_3_pointer().map_or(0, |a| a.into()), - ); // tvb_cluster_meta3 - r.add(0x1c890, tiling_control.into()); // tvb_tiling_control - r.add(0x1c918, unks.tiling_control_2); - r.add(0x1c079, inner.scene.tvb_heapmeta_pointer().into()); - r.add(0x1c9d8, inner.scene.tvb_heapmeta_pointer().into()); - r.add(0x1c089, 0); - r.add(0x1c9e0, 0); - let cl_meta_4_pointer = - inner.scene.meta_4_pointer().map_or(0, |a| a.into()); - r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 - r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 - r.add(0x1c9a8, unks.vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? - r.add( - 0x1c920, - inner.scene.meta_1_pointer().map_or(0, |a| a.into()), - ); // ??? | meta1_blocks? - r.add(0x10151, 0); - r.add(0x1c199, 0); - r.add(0x1c1a1, 0); - r.add(0x1c1a9, 0); // 0x10151 bit 1 enables - r.add(0x1c1b1, 0); - r.add(0x1c1b9, 0); - r.add(0x10061, 0x11_00000000); // USC_EXEC_BASE_TA - r.add(0x11801, 0); // some shader? - r.add(0x11809, 0); // maybe arg? - r.add(0x11f71, 0); - r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG - r.add(0x1c850, tile_info.params.rgn_size.into()); - r.add(0x10131, tile_info.params.unk_4.into()); - r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL - r.add( - 0x10129, - tile_info.params.x_max as u64 - | ((tile_info.params.y_max as u64) << 16), - ); // PPP_SCREEN - r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN - r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 - r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 - r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE - r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC - r.add(0x10171, tile_info.params.unk_24.into()); - r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX - r.add(0x12099, unks.vtx_unk_118); - r.add(0x1c9e8, 0); - } - ), - tpc: inner.scene.tpc_pointer(), - tpc_size: U64(tile_info.tpc_size as u64), - microsequence: inner.micro_seq.gpu_pointer(), - microsequence_size: inner.micro_seq.len() as u32, - fragment_stamp_slot: ev_frag.slot, - fragment_stamp_value: ev_frag.value.next(), - unk_pointee: 0, - unk_pad: 0, - job_params2: fw::vertex::raw::JobParameters2 { - unk_480: Default::default(), // fixed - unk_498: U64(0x0), // fixed - unk_4a0: 0x0, // fixed - preempt_buf1: inner.scene.preempt_buf_1_pointer(), - unk_4ac: 0x0, // fixed - unk_4b0: U64(0x0), // fixed - unk_4b8: 0x0, // fixed - unk_4bc: U64(0x0), // fixed - unk_4c4_padding: Default::default(), - unk_50c: 0x0, // fixed - unk_510: U64(0x0), // fixed - unk_518: U64(0x0), // fixed - unk_520: U64(0x0), // fixed - }, - encoder_params: fw::job::raw::EncoderParams { - unk_8: 0x0, // fixed - large_tib: 0x0, // fixed - unk_10: 0x0, // fixed - encoder_id: cmdbuf.encoder_id, - unk_18: 0x0, // fixed - iogpu_compute_unk44: unks.vtx_iogpucmp_unk44 as u32, - seq_buffer: U64(unks.vtx_seq_buffer), - unk_28: U64(0x0), // fixed - }, - unk_55c: 0, - unk_560: 0, - memoryless_rts_used: (cmdbuf.flags - & uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 - != 0) as u32, - unk_568: 0, - unk_56c: 0, - meta: fw::job::raw::JobMeta { - unk_0: 0, - unk_2: 0, - no_preemption: 0, - stamp: ev_vtx.stamp_pointer, - fw_stamp: ev_vtx.fw_stamp_pointer, - stamp_value: ev_vtx.value.next(), - stamp_slot: ev_vtx.slot, - evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, - uuid: uuid_ta, - event_seq: ev_vtx.event_seq as u32, - }, - unk_after_meta: unk1.into(), - unk_buf_0: U64(0), - unk_buf_8: U64(0), - unk_buf_10: U64(0), - cur_ts: U64(0), - start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), - end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), - unk_5c4: 0, - unk_5c8: 0, - unk_5cc: 0, - unk_5d0: 0, - client_sequence: slot_client_seq, - pad_5d5: Default::default(), - unk_5d8: 0, - unk_5dc: 0, - #[ver(V >= V13_0B4)] - unk_ts: U64(0), - #[ver(V >= V13_0B4)] - unk_5dd_8: Default::default(), - } - )) + .preempt_buf_3_pointer() + .or(0x4_0000_0000_0000) + .into(), + ); + r.add(0x1c930, 0); // VCE related addr, lsb to enable + r.add(0x1c880, cmdbuf.encoder_ptr); // VDM_CTRL_STREAM_BASE + r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. + r.add( + 0x1c948, + inner.scene.meta_2_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta2 + r.add( + 0x1c888, + inner.scene.meta_3_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta3 + r.add(0x1c890, tiling_control.into()); // tvb_tiling_control + r.add(0x1c918, unks.tiling_control_2); + r.add(0x1c079, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x1c9d8, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x1c089, 0); + r.add(0x1c9e0, 0); + let cl_meta_4_pointer = + inner.scene.meta_4_pointer().map_or(0, |a| a.into()); + r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1c9a8, unks.vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? + r.add( + 0x1c920, + inner.scene.meta_1_pointer().map_or(0, |a| a.into()), + ); // ??? | meta1_blocks? + r.add(0x10151, 0); + r.add(0x1c199, 0); + r.add(0x1c1a1, 0); + r.add(0x1c1a9, 0); // 0x10151 bit 1 enables + r.add(0x1c1b1, 0); + r.add(0x1c1b9, 0); + r.add(0x10061, 0x11_00000000); // USC_EXEC_BASE_TA + r.add(0x11801, 0); // some shader? + r.add(0x11809, 0); // maybe arg? + r.add(0x11f71, 0); + r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG + r.add(0x1c850, tile_info.params.rgn_size.into()); + r.add(0x10131, tile_info.params.unk_4.into()); + r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL + r.add( + 0x10129, + tile_info.params.x_max as u64 + | ((tile_info.params.y_max as u64) << 16), + ); // PPP_SCREEN + r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN + r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 + r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 + r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE + r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC + r.add(0x10171, tile_info.params.unk_24.into()); + r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX + r.add(0x12099, unks.vtx_unk_118); + r.add(0x1c9e8, 0); + } + ), + tpc: inner.scene.tpc_pointer(), + tpc_size: U64(tile_info.tpc_size as u64), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + fragment_stamp_slot: ev_frag.slot, + fragment_stamp_value: ev_frag.value.next(), + unk_pointee: 0, + unk_pad: 0, + job_params2 <- try_init!(fw::vertex::raw::JobParameters2 { + unk_480: Default::default(), // fixed + unk_498: U64(0x0), // fixed + unk_4a0: 0x0, // fixed + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + unk_4ac: 0x0, // fixed + unk_4b0: U64(0x0), // fixed + unk_4b8: 0x0, // fixed + unk_4bc: U64(0x0), // fixed + unk_4c4_padding: Default::default(), + unk_50c: 0x0, // fixed + unk_510: U64(0x0), // fixed + unk_518: U64(0x0), // fixed + unk_520: U64(0x0), // fixed + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + large_tib: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: cmdbuf.encoder_id, + unk_18: 0x0, // fixed + iogpu_compute_unk44: unks.vtx_iogpucmp_unk44 as u32, + seq_buffer: U64(unks.vtx_seq_buffer), + unk_28: U64(0x0), // fixed + }), + unk_55c: 0, + unk_560: 0, + memoryless_rts_used: (cmdbuf.flags + & uapi::ASAHI_RENDER_MEMORYLESS_RTS_USED as u64 + != 0) as u32, + unk_568: 0, + unk_56c: 0, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: 0, + stamp: ev_vtx.stamp_pointer, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + stamp_slot: ev_vtx.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_ta, + event_seq: ev_vtx.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + unk_buf_10: U64(0), + cur_ts: U64(0), + start_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), + end_ts: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), + unk_5c4: 0, + unk_5c8: 0, + unk_5cc: 0, + unk_5d0: 0, + client_sequence: slot_client_seq, + pad_5d5: Default::default(), + unk_5d8: 0, + unk_5dc: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_5dd_8: Default::default(), + }) }, )?; From 071c6a11d8935a0b59fc22bc74835305cd392281 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 18:43:49 +0900 Subject: [PATCH 588/635] drm/asahi: object,alloc: Drop new_prealloc() All users are now converted to new_init_prealloc() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 18 ------------- drivers/gpu/drm/asahi/object.rs | 46 --------------------------------- 2 files changed, 64 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index 21fa83b2183ed9..ab1f02f31bdcd4 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -249,24 +249,6 @@ pub(crate) trait Allocator { GpuObject::>::new_default(self.alloc_object()?) } - /// Allocate a new GpuStruct object. See [`GpuObject::new_prealloc`]. - #[inline(never)] - fn new_prealloc( - &mut self, - inner_cb: impl FnOnce(GpuWeakPointer) -> Result>, - raw_cb: impl for<'a> FnOnce( - &'a T, - &'a mut MaybeUninit>, - GpuWeakPointer, - ) -> Result<&'a mut T::Raw<'a>>, - ) -> Result>> { - GpuObject::>::new_prealloc( - self.alloc_object()?, - inner_cb, - raw_cb, - ) - } - /// Allocate a new GpuStruct object. See [`GpuObject::new_init_prealloc`]. #[inline(never)] fn new_init_prealloc< diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index fb5dcc628d4fe9..c2876eaa362c77 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -334,52 +334,6 @@ impl> GpuObject { GpuObject::::new_boxed(alloc, Box::try_new(inner)?, callback) } - /// Create a new GpuObject given an allocator, with callback-based initialization. - /// - /// This is used when the construction of the `T` type requires knowing the GPU VA address of - /// the structure that is being constructed ahead of time. The first callback constructs a - /// `Box` given the pointer to the about-to-be-initialized GPU structure, and the second - /// callback initializes that structure as in `new_boxed`. - pub(crate) fn new_prealloc( - alloc: U, - inner_cb: impl FnOnce(GpuWeakPointer) -> Result>, - raw_cb: impl for<'a> FnOnce( - &'a T, - &'a mut MaybeUninit>, - GpuWeakPointer, - ) -> Result<&'a mut T::Raw<'a>>, - ) -> Result { - if alloc.size() < mem::size_of::>() { - return Err(ENOMEM); - } - let gpu_ptr = - GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); - mod_dev_dbg!( - alloc.device(), - "Allocating {} @ {:#x}\n", - core::any::type_name::(), - alloc.gpu_ptr() - ); - let inner = inner_cb(gpu_ptr)?; - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; - // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - let raw = raw_cb(&*inner, unsafe { &mut *p }, gpu_ptr)?; - if p as *mut T::Raw<'_> != raw as *mut _ { - dev_err!( - alloc.device(), - "Allocation callback returned a mismatched reference ({})\n", - core::any::type_name::(), - ); - return Err(EINVAL); - } - Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, - gpu_ptr, - alloc, - inner, - }) - } - /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing /// GpuStruct). /// From ba48e427e274bc1a9798533551fcbc9862630ca6 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 19:53:22 +0900 Subject: [PATCH 589/635] rust: init: Add chain() and pin_chain() functions These functinos allow users to chain post-processing code to an initializer, which can be used to mutate the (fully initialized) object in place as part of an initializer chain. Ideally this would be a trait function (e.g. init!(...).and_then(...)), but that requires return_position_impl_trait_in_trait... Signed-off-by: Asahi Lina --- rust/kernel/init.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 18bca4797a2a0b..35d8b25c99caa4 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1275,6 +1275,57 @@ pub unsafe trait Init: Sized { unsafe fn __init(self, slot: *mut T) -> Result<(), E>; } +/// Chains a closure to the initializer to be called on successful initialization. +/// +/// Returns a new initializer. If the closure returns `Err`, the object is +/// dropped. +// TODO: Once return_position_impl_trait_in_trait works, this should probably be +// a trait method and called `and_then()` or so. +pub fn chain( + this: impl Init, + f: impl FnOnce(&mut T) -> Result<(), E>, +) -> impl Init { + unsafe { + init_from_closure(|slot: *mut T| { + this.__init(slot)?; + + f(&mut *slot).map_err(|e| { + // SAFETY: The value was initialized above, and since we return + // `Err` here, the caller will consider the memory at `slot` to + // be uninitialized. + ptr::drop_in_place(slot); + e + }) + }) + } +} + +/// Chains a closure to the pinned initializer to be called on successful +/// initialization. +/// +/// Returns a new initializer. If the closure returns `Err`, the object is +/// dropped. +// TODO: Once return_position_impl_trait_in_trait works, this should probably be +// a trait method and called `and_then()` or so. +pub fn pin_chain( + this: impl PinInit, + f: impl FnOnce(&mut T) -> Result<(), E>, +) -> impl PinInit { + unsafe { + init_from_closure(|slot: *mut T| { + this.__pinned_init(slot)?; + + f(&mut *slot).map_err(|e| { + // SAFETY: The value was initialized above, and since we return + // `Err` here, the caller will consider the memory at `slot` to + // be uninitialized. + ptr::drop_in_place(slot); + e + }) + }) + } +} + // SAFETY: Every in-place initializer can also be used as a pin-initializer. unsafe impl PinInit for I where From 78aab9db281a2c18a8ad1ddf98c0ce51d8eaeaa5 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 19:36:42 +0900 Subject: [PATCH 590/635] drm/asahi: initdata: Port to init!() Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/fw/initdata.rs | 26 +- drivers/gpu/drm/asahi/initdata.rs | 1370 +++++++++++++------------- 2 files changed, 691 insertions(+), 705 deletions(-) diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs index 33c3d659fa30bf..b301b2cd99e648 100644 --- a/drivers/gpu/drm/asahi/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -156,7 +156,6 @@ pub(crate) mod raw { } default_zeroed!(HwDataA130Extra); - #[derive(Default)] #[repr(C)] pub(crate) struct T81xxData { pub(crate) unk_d8c: u32, @@ -177,6 +176,7 @@ pub(crate) mod raw { pub(crate) unk_dc8: u32, pub(crate) max_pstate_scaled: u32, } + default_zeroed!(T81xxData); #[versions(AGX)] #[derive(Default, Copy, Clone)] @@ -1262,22 +1262,10 @@ trivial_gpustruct!(GpuGlobalStatsFrag); trivial_gpustruct!(GpuStatsComp); #[versions(AGX)] -#[derive(Debug, Default)] -pub(crate) struct HwDataA {} +trivial_gpustruct!(HwDataA::ver); #[versions(AGX)] -impl GpuStruct for HwDataA::ver { - type Raw<'a> = raw::HwDataA::ver; -} - -#[versions(AGX)] -#[derive(Debug, Default)] -pub(crate) struct HwDataB {} - -#[versions(AGX)] -impl GpuStruct for HwDataB::ver { - type Raw<'a> = raw::HwDataB::ver; -} +trivial_gpustruct!(HwDataB::ver); #[versions(AGX)] #[derive(Debug)] @@ -1310,13 +1298,7 @@ impl GpuStruct for RuntimePointers::ver { } #[versions(AGX)] -#[derive(Debug, Default)] -pub(crate) struct Globals {} - -#[versions(AGX)] -impl GpuStruct for Globals::ver { - type Raw<'a> = raw::Globals::ver; -} +trivial_gpustruct!(Globals::ver); #[versions(AGX)] #[derive(Debug)] diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs index 3c3bd31b553130..423a60ae1f7380 100644 --- a/drivers/gpu/drm/asahi/initdata.rs +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -11,19 +11,20 @@ //! Many of these structures are poorly understood, so there are lots of hardcoded unknown values //! derived from observing the InitData structures that macOS generates. +use crate::f32; use crate::fw::initdata::*; use crate::fw::types::*; -use crate::{box_in_place, f32, place}; use crate::{gpu, hw, mmu}; use alloc::vec::Vec; -use kernel::error::Result; +use kernel::error::{Error, Result}; use kernel::macros::versions; +use kernel::{init, init::Init, try_init}; /// Builder helper for the global GPU InitData. #[versions(AGX)] pub(crate) struct InitDataBuilder<'a> { alloc: &'a mut gpu::KernelAllocators, - cfg: &'a hw::HwConfig, + cfg: &'static hw::HwConfig, dyncfg: &'a hw::DynConfig, } @@ -32,23 +33,26 @@ impl<'a> InitDataBuilder::ver<'a> { /// Create a new InitData builder pub(crate) fn new( alloc: &'a mut gpu::KernelAllocators, - cfg: &'a hw::HwConfig, + cfg: &'static hw::HwConfig, dyncfg: &'a hw::DynConfig, ) -> InitDataBuilder::ver<'a> { InitDataBuilder::ver { alloc, cfg, dyncfg } } /// Create the HwDataShared1 structure, which is used in two places in InitData. - #[inline(never)] - fn hw_shared1(cfg: &hw::HwConfig) -> raw::HwDataShared1 { - let mut ret = raw::HwDataShared1 { - unk_a4: cfg.shared1_a4, - ..Default::default() - }; - for (i, val) in cfg.shared1_tab.iter().enumerate() { - ret.table[i] = *val; - } - ret + fn hw_shared1(cfg: &'static hw::HwConfig) -> impl Init { + init::chain( + init!(raw::HwDataShared1 { + unk_a4: cfg.shared1_a4, + ..Zeroable::zeroed() + }), + |ret| { + for (i, val) in cfg.shared1_tab.iter().enumerate() { + ret.table[i] = *val; + } + Ok(()) + }, + ) } fn init_curve( @@ -75,691 +79,691 @@ impl<'a> InitDataBuilder::ver<'a> { } /// Create the HwDataShared2 structure, which is used in two places in InitData. - #[inline(never)] fn hw_shared2( - cfg: &hw::HwConfig, + cfg: &'static hw::HwConfig, dyncfg: &'a hw::DynConfig, - ) -> Result> { - let mut ret = box_in_place!(raw::HwDataShared2 { - unk_28: Array::new([0xff; 16]), - g14: Default::default(), - unk_508: cfg.shared2_unk_508, - ..Default::default() - })?; - - for (i, val) in cfg.shared2_tab.iter().enumerate() { - ret.table[i] = *val; - } + ) -> impl Init + 'a { + init::chain( + try_init!(raw::HwDataShared2 { + unk_28: Array::new([0xff; 16]), + g14: Default::default(), + unk_508: cfg.shared2_unk_508, + ..Zeroable::zeroed() + }), + |ret| { + for (i, val) in cfg.shared2_tab.iter().enumerate() { + ret.table[i] = *val; + } - let curve_cfg = match cfg.shared2_curves.as_ref() { - None => return Ok(ret), - Some(a) => a, - }; + let curve_cfg = match cfg.shared2_curves.as_ref() { + None => return Ok(()), + Some(a) => a, + }; - let mut t1 = Vec::new(); - let mut t3 = Vec::new(); + let mut t1 = Vec::new(); + let mut t3 = Vec::new(); - for _ in 0..curve_cfg.t3_scales.len() { - t3.try_push(Vec::new())?; - } + for _ in 0..curve_cfg.t3_scales.len() { + t3.try_push(Vec::new())?; + } - for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { - let t3_coef = curve_cfg.t3_coefs[i]; - if t3_coef == 0 { - t1.try_push(0xffff)?; - for j in t3.iter_mut() { - j.try_push(0x3ffffff)?; + for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { + let t3_coef = curve_cfg.t3_coefs[i]; + if t3_coef == 0 { + t1.try_push(0xffff)?; + for j in t3.iter_mut() { + j.try_push(0x3ffffff)?; + } + continue; + } + + let f_mhz = (ps.freq_hz / 1000) as u64; + let v_max = ps.max_volt_mv() as u64; + + t1.try_push( + (1000000000 * (curve_cfg.t1_coef as u64) / (f_mhz * v_max)) + .try_into() + .unwrap(), + )?; + + for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { + t3[j].try_push( + (t3_coef as u64 * 1000000000 * *scale as u64 / (f_mhz * v_max * 6)) + .try_into() + .unwrap(), + )?; + } } - continue; - } - let f_mhz = (ps.freq_hz / 1000) as u64; - let v_max = ps.max_volt_mv() as u64; - - t1.try_push( - (1000000000 * (curve_cfg.t1_coef as u64) / (f_mhz * v_max)) - .try_into() - .unwrap(), - )?; - - for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { - t3[j].try_push( - (t3_coef as u64 * 1000000000 * *scale as u64 / (f_mhz * v_max * 6)) - .try_into() - .unwrap(), - )?; - } - } + ret.g14.unk_14 = 0x6000000; + Self::init_curve( + &mut ret.g14.curve1, + 0, + 0x20000000, + &[0xffff], + &[0x0f07], + &[], + ); + Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); - ret.g14.unk_14 = 0x6000000; - Self::init_curve( - &mut ret.g14.curve1, - 0, - 0x20000000, - &[0xffff], - &[0x0f07], - &[], - ); - Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); - - Ok(ret) + Ok(()) + }, + ) } /// Create the HwDataShared3 structure, which is used in two places in InitData. - #[inline(never)] - fn hw_shared3(cfg: &hw::HwConfig) -> Result> { - let mut ret = box_in_place!(raw::HwDataShared3 { - ..Default::default() - })?; - - if !cfg.shared3_tab.is_empty() { - ret.unk_0 = 1; - ret.unk_4 = 500; - ret.unk_8 = cfg.shared3_unk; - ret.table.copy_from_slice(cfg.shared3_tab); - ret.unk_4c = 1; - } - - Ok(ret) + fn hw_shared3(cfg: &'static hw::HwConfig) -> impl Init { + init::chain(init::zeroed::(), |ret| { + if !cfg.shared3_tab.is_empty() { + ret.unk_0 = 1; + ret.unk_4 = 500; + ret.unk_8 = cfg.shared3_unk; + ret.table.copy_from_slice(cfg.shared3_tab); + ret.unk_4c = 1; + } + Ok(()) + }) } /// Create an unknown T81xx-specific data structure. - fn t81xx_data(dyncfg: &'a hw::DynConfig) -> raw::T81xxData { - raw::T81xxData { - unk_d8c: 0x80000000, - unk_d90: 4, - unk_d9c: f32!(0.6), - unk_da4: f32!(0.4), - unk_dac: f32!(0.38552), - unk_db8: f32!(65536.0), - unk_dbc: f32!(13.56), - max_pstate_scaled: 100 * dyncfg.pwr.perf_max_pstate, - ..Default::default() - } + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + init::chain(init::zeroed::(), move |ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + ret.unk_d8c = 0x80000000; + ret.unk_d90 = 4; + ret.unk_d9c = f32!(0.6); + ret.unk_da4 = f32!(0.4); + ret.unk_dac = f32!(0.38552); + ret.unk_db8 = f32!(65536.0); + ret.unk_dbc = f32!(13.56); + ret.max_pstate_scaled = 100 * perf_max_pstate; + } + _ => (), + } + Ok(()) + }) } /// Create the HwDataA structure. This mostly contains power-related configuration. - #[inline(never)] fn hwdata_a(&mut self) -> Result> { - self.alloc - .private - .new_inplace(Default::default(), |_inner, ptr| { - let pwr = &self.dyncfg.pwr; - let period_ms = pwr.power_sample_period; - let period_s = F32::from(period_ms) / f32!(1000.0); - let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; - #[ver(V >= V13_0B4)] - let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; - let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); - let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); - let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); - let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); - let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; - #[ver(V >= V13_0B4)] - let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; - let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); - let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); - - let base_ps = pwr.perf_base_pstate; - let base_ps_scaled = 100 * base_ps; - let max_ps = pwr.perf_max_pstate; - let max_ps_scaled = 100 * max_ps; - let boost_ps_count = max_ps - base_ps; - - #[allow(unused_variables)] - let base_clock_khz = self.cfg.base_clock_hz / 1000; - let clocks_per_period = pwr.pwr_sample_period_aic_clks; - - #[allow(unused_variables)] - let clocks_per_period_coarse = - self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; - - let raw = place!( - ptr, - raw::HwDataA::ver { - clocks_per_period: clocks_per_period, - #[ver(V >= V13_0B4)] - clocks_per_period_2: clocks_per_period, - pwr_status: AtomicU32::new(4), - unk_10: f32!(1.0), - actual_pstate: 1, - tgt_pstate: 1, - base_pstate_scaled: base_ps_scaled, - unk_40: 1, - max_pstate_scaled: max_ps_scaled, - min_pstate_scaled: 100, - unk_64c: 625, - pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, - pwr_filter_a: pwr_filter_a, - pwr_integral_gain: pwr.pwr_integral_gain, - pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), - max_power_1: pwr.max_power_mw.into(), - pwr_proportional_gain: pwr.pwr_proportional_gain, - pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), - pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - - max_ps_scaled as i32, - max_pstate_scaled_2: max_ps_scaled, - max_power_2: pwr.max_power_mw, - max_pstate_scaled_3: max_ps_scaled, - ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, - ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, - ppm_filter_a: ppm_filter_a, - ppm_ki_dt: pwr.ppm_ki * period_s, - unk_6fc: f32!(65536.0), - ppm_kp: pwr.ppm_kp, - pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, - max_pstate_scaled_4: max_ps_scaled, - unk_71c: f32!(0.0), - max_power_3: pwr.max_power_mw, - cur_power_mw_2: 0x0, - ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, - #[ver(V >= V13_0B4)] - ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, - perf_tgt_utilization: pwr.perf_tgt_utilization, - perf_boost_min_util: pwr.perf_boost_min_util, - perf_boost_ce_step: pwr.perf_boost_ce_step, - perf_reset_iters: pwr.perf_reset_iters, - unk_774: 6, - unk_778: 1, - perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, - perf_filter_a_neg: f32!(1.0) - perf_filter_a, - perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, - perf_filter_a: perf_filter_a, - perf_filter_a2: perf_filter_a2, - perf_ki: pwr.perf_integral_gain, - perf_ki2: pwr.perf_integral_gain2, - perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), - unk_79c: f32!(95.0), - perf_kp: pwr.perf_proportional_gain, - perf_kp2: pwr.perf_proportional_gain2, - boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), - base_pstate_scaled_2: base_ps_scaled, - max_pstate_scaled_5: max_ps_scaled, - base_pstate_scaled_3: base_ps_scaled, - perf_tgt_utilization_2: pwr.perf_tgt_utilization, - base_pstate_scaled_4: base_ps_scaled, - unk_7fc: f32!(65536.0), - pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), - max_pstate_scaled_6: max_ps_scaled.into(), - max_freq_mhz: pwr.max_freq_mhz, - pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, - min_pstate_scaled_4: f32!(100.0), - max_pstate_scaled_7: max_ps_scaled, - unk_alpha_neg: f32!(0.8), - unk_alpha: f32!(0.2), - fast_die0_sensor_mask: U64(self.cfg.fast_sensor_mask[0]), - fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, - unk_87c: self.cfg.da.unk_87c, - unk_880: 0x4, - unk_894: f32!(1.0), - - fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, - unk_8a8: f32!(65536.0), - fast_die0_kp: pwr.fast_die0_proportional_gain, - pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, - max_pstate_scaled_8: max_ps_scaled, - max_pstate_scaled_9: max_ps_scaled, - fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, - unk_8cc: self.cfg.da.unk_8cc, - max_pstate_scaled_10: max_ps_scaled, - max_pstate_scaled_11: max_ps_scaled, - unk_c2c: 1, - power_zone_count: pwr.power_zones.len() as u32, - max_power_4: pwr.max_power_mw, - max_power_5: pwr.max_power_mw, - max_power_6: pwr.max_power_mw, - avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, - avg_power_target_filter_a: avg_power_target_filter_a, - avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, - avg_power_target_filter_tc_xperiod: period_ms - * pwr.avg_power_target_filter_tc, - #[ver(V >= V13_0B4)] - avg_power_target_filter_tc_clks: period_ms - * pwr.avg_power_target_filter_tc - * base_clock_khz, - avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, - avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, - avg_power_filter_a: avg_power_filter_a, - avg_power_ki_dt: pwr.avg_power_ki_only * period_s, - unk_d20: f32!(65536.0), - avg_power_kp: pwr.avg_power_kp, - avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, - max_pstate_scaled_12: max_ps_scaled, - max_pstate_scaled_13: max_ps_scaled, - max_power_7: pwr.max_power_mw.into(), - max_power_8: pwr.max_power_mw, - avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, - #[ver(V >= V13_0B4)] - avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, - max_pstate_scaled_14: max_ps_scaled, - t81xx_data: match self.cfg.chip_id { - 0x8103 | 0x8112 => Self::t81xx_data(self.dyncfg), - _ => Default::default(), - }, - #[ver(V >= V13_0B4)] - unk_e10_0: { - let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); - let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); - raw::HwDataA130Extra { - unk_38: 4, - unk_3c: 8000, - gpu_se_inactive_threshold: pwr.se_inactive_threshold, - gpu_se_engagement_criteria: pwr.se_engagement_criteria, - gpu_se_reset_criteria: pwr.se_reset_criteria, - unk_54: 50, - unk_58: 0x1, - gpu_se_filter_a_neg: f32!(1.0) - filter_a, - gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, - gpu_se_filter_a: filter_a, - gpu_se_filter_1_a: filter_1_a, - gpu_se_ki_dt: pwr.se_ki * period_s, - gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, - unk_7c: f32!(65536.0), - gpu_se_kp: pwr.se_kp, - gpu_se_kp_1: pwr.se_kp_1, - - #[ver(V >= V13_3)] - unk_8c: 100, - #[ver(V < V13_3)] - unk_8c: 40, - - max_pstate_scaled_1: max_ps_scaled, - unk_9c: f32!(8000.0), - unk_a0: 1400, - gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant - * period_ms, - gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 - * period_ms, - gpu_se_filter_time_constant_clks: U64((pwr - .se_filter_time_constant - * clocks_per_period_coarse) - .into()), - gpu_se_filter_time_constant_1_clks: U64((pwr - .se_filter_time_constant_1 - * clocks_per_period_coarse) - .into()), - unk_c4: f32!(65536.0), - unk_114: f32!(65536.0), - unk_124: 40, - max_pstate_scaled_2: max_ps_scaled, - ..Default::default() - } - }, - fast_die0_sensor_mask_2: U64(self.cfg.fast_sensor_mask[0]), - unk_e24: self.cfg.da.unk_e24, - unk_e28: 1, - fast_die0_sensor_mask_alt: U64(self.cfg.fast_sensor_mask_alt[0]), - #[ver(V < V13_0B4)] - fast_die0_sensor_present: U64(self.cfg.fast_die0_sensor_present as u64), - unk_163c: 1, - unk_3644: 0, - hws1: Self::hw_shared1(self.cfg), - hws2: *Self::hw_shared2(self.cfg, self.dyncfg)?, - hws3: *Self::hw_shared3(self.cfg)?, - unk_3ce8: 1, - ..Default::default() + let pwr = &self.dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + #[ver(V >= V13_0B4)] + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + #[ver(V >= V13_0B4)] + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = max_ps - base_ps; + + #[allow(unused_variables)] + let base_clock_khz = self.cfg.base_clock_hz / 1000; + let clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + self.alloc.private.new_init(init::zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + init::chain( + try_init!(raw::HwDataA::ver { + clocks_per_period: clocks_per_period, + #[ver(V >= V13_0B4)] + clocks_per_period_2: clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + #[ver(V >= V13_0B4)] + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + #[ver(V >= V13_0B4)] + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + #[ver(V >= V13_0B4)] + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + #[ver(V >= V13_0B4)] + unk_e10_0 <- { + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + try_init!(raw::HwDataA130Extra { + unk_38: 4, + unk_3c: 8000, + gpu_se_inactive_threshold: pwr.se_inactive_threshold, + gpu_se_engagement_criteria: pwr.se_engagement_criteria, + gpu_se_reset_criteria: pwr.se_reset_criteria, + unk_54: 50, + unk_58: 0x1, + gpu_se_filter_a_neg: f32!(1.0) - filter_a, + gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, + gpu_se_filter_a: filter_a, + gpu_se_filter_1_a: filter_1_a, + gpu_se_ki_dt: pwr.se_ki * period_s, + gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, + unk_7c: f32!(65536.0), + gpu_se_kp: pwr.se_kp, + gpu_se_kp_1: pwr.se_kp_1, + + #[ver(V >= V13_3)] + unk_8c: 100, + #[ver(V < V13_3)] + unk_8c: 40, + + max_pstate_scaled_1: max_ps_scaled, + unk_9c: f32!(8000.0), + unk_a0: 1400, + gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant * period_ms, + gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 + * period_ms, + gpu_se_filter_time_constant_clks: U64((pwr.se_filter_time_constant + * clocks_per_period_coarse) + .into()), + gpu_se_filter_time_constant_1_clks: U64((pwr + .se_filter_time_constant_1 + * clocks_per_period_coarse) + .into()), + unk_c4: f32!(65536.0), + unk_114: f32!(65536.0), + unk_124: 40, + max_pstate_scaled_2: max_ps_scaled, + ..Zeroable::zeroed() + }) + }, + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + #[ver(V < V13_0B4)] + fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + unk_163c: 1, + unk_3644: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + unk_3ce8: 1, + ..Zeroable::zeroed() + }), + |raw| { + for i in 0..self.dyncfg.pwr.perf_states.len() { + raw.sram_k[i] = self.cfg.sram_k; } - ); - for i in 0..self.dyncfg.pwr.perf_states.len() { - raw.sram_k[i] = self.cfg.sram_k; - } + for (i, coef) in pwr.core_leak_coef.iter().enumerate() { + raw.core_leak_coef[i] = *coef; + } - for (i, coef) in pwr.core_leak_coef.iter().enumerate() { - raw.core_leak_coef[i] = *coef; - } + for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { + raw.sram_leak_coef[i] = *coef; + } - for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { - raw.sram_leak_coef[i] = *coef; - } + #[ver(V >= V13_0B4)] + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } - #[ver(V >= V13_0B4)] - if let Some(csafr) = pwr.csafr.as_ref() { - for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { - raw.aux_leak_coef.cs_1[i] = *coef; - raw.aux_leak_coef.cs_2[i] = *coef; + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } } - for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { - raw.aux_leak_coef.afr_1[i] = *coef; - raw.aux_leak_coef.afr_2[i] = *coef; + for i in 0..self.dyncfg.id.num_clusters as usize { + if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } } - } - for i in 0..self.dyncfg.id.num_clusters as usize { - if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { - (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); - (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); - } - if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { - (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); - (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + #[ver(V >= V13_0B4)] + raw.power_zones[i].unk_10 = 1320000000; } - } - for (i, pz) in pwr.power_zones.iter().enumerate() { - raw.power_zones[i].target = pz.target; - raw.power_zones[i].target_off = pz.target - pz.target_offset; - raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; - raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; - let filter_a = f32!(1.0) / pz.filter_tc.into(); - raw.power_zones[i].filter_a = filter_a; - raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; #[ver(V >= V13_0B4)] - raw.power_zones[i].unk_10 = 1320000000; - } - - #[ver(V >= V13_0B4)] - for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { - raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; - } + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } - Ok(raw) - }) + Ok(()) + }, + ) + }) } /// Create the HwDataB structure. This mostly contains GPU-related configuration. - #[inline(never)] fn hwdata_b(&mut self) -> Result> { - self.alloc - .private - .new_inplace(Default::default(), |_inner, ptr| { - let raw = place!( - ptr, - raw::HwDataB::ver { - // Userspace VA map related - #[ver(V < V13_0B4)] - unk_0: U64(0x13_00000000), - unk_8: U64(0x14_00000000), - #[ver(V < V13_0B4)] - unk_10: U64(0x1_00000000), - unk_18: U64(0xffc00000), - unk_20: U64(0x11_00000000), - unk_28: U64(0x11_00000000), - // userspace address? - unk_30: U64(0x6f_ffff8000), - // unmapped? - unkptr_38: U64(0xffffffa0_11800000), - // TODO: yuv matrices - chip_id: self.cfg.chip_id, - unk_454: 0x1, - unk_458: 0x1, - unk_460: 0x1, - unk_464: 0x1, - unk_468: 0x1, - unk_47c: 0x1, - unk_484: 0x1, - unk_48c: 0x1, - base_clock_khz: self.cfg.base_clock_hz / 1000, - power_sample_period: self.dyncfg.pwr.power_sample_period, - unk_49c: 0x1, - unk_4a0: 0x1, - unk_4a4: 0x1, - unk_4c0: 0x1f, - unk_4e0: U64(self.cfg.db.unk_4e0), - unk_4f0: 0x1, - unk_4f4: 0x1, - unk_504: 0x31, - unk_524: 0x1, // use_secure_cache_flush - unk_534: self.cfg.db.unk_534, - num_frags: self.dyncfg.id.num_frags * self.dyncfg.id.num_clusters, - unk_554: 0x1, - uat_ttb_base: U64(self.dyncfg.uat_ttb_base), - gpu_core_id: self.cfg.gpu_core as u32, - gpu_rev_id: self.dyncfg.id.gpu_rev_id as u32, - num_cores: self.dyncfg.id.num_cores * self.dyncfg.id.num_clusters, - max_pstate: self.dyncfg.pwr.perf_states.len() as u32 - 1, - #[ver(V < V13_0B4)] - num_pstates: self.dyncfg.pwr.perf_states.len() as u32, - #[ver(V < V13_0B4)] - min_sram_volt: self.dyncfg.pwr.min_sram_microvolt / 1000, - #[ver(V < V13_0B4)] - unk_ab8: self.cfg.db.unk_ab8, - #[ver(V < V13_0B4)] - unk_abc: self.cfg.db.unk_abc, - #[ver(V < V13_0B4)] - unk_ac0: 0x1020, - - #[ver(V >= V13_0B4)] - unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), - #[ver(V < V13_0B4)] - unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), - unk_b10: 0x1, - unk_b24: 0x1, - unk_b28: 0x1, - unk_b2c: 0x1, - unk_b30: self.cfg.db.unk_b30, - #[ver(V >= V13_0B4)] - unk_b38_0: 1, - #[ver(V >= V13_0B4)] - unk_b38_4: 1, - unk_b38: Array::new([0xffffffff; 12]), - #[ver(V >= V13_0B4 && V < V13_3)] - unk_c3c: 0x19, - #[ver(V >= V13_3)] - unk_c3c: 0x1a, - ..Default::default() - } - ); + self.alloc.private.new_init(init::zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + init::chain( + try_init!(raw::HwDataB::ver { + // Userspace VA map related + #[ver(V < V13_0B4)] + unk_0: U64(0x13_00000000), + unk_8: U64(0x14_00000000), + #[ver(V < V13_0B4)] + unk_10: U64(0x1_00000000), + unk_18: U64(0xffc00000), + unk_20: U64(0x11_00000000), + unk_28: U64(0x11_00000000), + // userspace address? + unk_30: U64(0x6f_ffff8000), + // unmapped? + unkptr_38: U64(0xffffffa0_11800000), + // TODO: yuv matrices + chip_id: cfg.chip_id, + unk_454: 0x1, + unk_458: 0x1, + unk_460: 0x1, + unk_464: 0x1, + unk_468: 0x1, + unk_47c: 0x1, + unk_484: 0x1, + unk_48c: 0x1, + base_clock_khz: cfg.base_clock_hz / 1000, + power_sample_period: dyncfg.pwr.power_sample_period, + unk_49c: 0x1, + unk_4a0: 0x1, + unk_4a4: 0x1, + unk_4c0: 0x1f, + unk_4e0: U64(cfg.db.unk_4e0), + unk_4f0: 0x1, + unk_4f4: 0x1, + unk_504: 0x31, + unk_524: 0x1, // use_secure_cache_flush + unk_534: cfg.db.unk_534, + num_frags: dyncfg.id.num_frags * dyncfg.id.num_clusters, + unk_554: 0x1, + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: dyncfg.pwr.perf_states.len() as u32 - 1, + #[ver(V < V13_0B4)] + num_pstates: dyncfg.pwr.perf_states.len() as u32, + #[ver(V < V13_0B4)] + min_sram_volt: dyncfg.pwr.min_sram_microvolt / 1000, + #[ver(V < V13_0B4)] + unk_ab8: cfg.db.unk_ab8, + #[ver(V < V13_0B4)] + unk_abc: cfg.db.unk_abc, + #[ver(V < V13_0B4)] + unk_ac0: 0x1020, - #[ver(V >= V13_3)] - for i in 0..16 { - raw.unk_arr_0[i] = i as u32; - } - - let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; - let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; - let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; - let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; - - for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { - raw.frequencies[i] = ps.freq_hz / 1000000; - for (j, mv) in ps.volt_mv.iter().enumerate() { - let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); - raw.voltages[i][j] = *mv; - raw.voltages_sram[i][j] = sram_mv; - } - for j in ps.volt_mv.len()..raw.voltages[i].len() { - raw.voltages[i][j] = raw.voltages[i][0]; - raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; + #[ver(V >= V13_0B4)] + unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), + #[ver(V < V13_0B4)] + unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), + unk_b10: 0x1, + unk_b24: 0x1, + unk_b28: 0x1, + unk_b2c: 0x1, + unk_b30: cfg.db.unk_b30, + #[ver(V >= V13_0B4)] + unk_b38_0: 1, + #[ver(V >= V13_0B4)] + unk_b38_4: 1, + unk_b38: Array::new([0xffffffff; 12]), + #[ver(V >= V13_0B4 && V < V13_3)] + unk_c3c: 0x19, + #[ver(V >= V13_3)] + unk_c3c: 0x1a, + ..Zeroable::zeroed() + }), + |raw| { + #[ver(V >= V13_3)] + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; } - raw.sram_k[i] = self.cfg.sram_k; - raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; - raw.rel_boost_freqs[i] = if i > base_ps { - (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) - } else { - 0 - }; - } - #[ver(V >= V13_0B4)] - if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { - let aux = &mut raw.aux_ps; - aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; - aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; + let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; + let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; + let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; - for (i, ps) in csafr.perf_states_cs.iter().enumerate() { - aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { + raw.frequencies[i] = ps.freq_hz / 1000000; for (j, mv) in ps.volt_mv.iter().enumerate() { - let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); - aux.cs_voltages[i][j] = *mv; - aux.cs_voltages_sram[i][j] = sram_mv; + let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); + raw.voltages[i][j] = *mv; + raw.voltages_sram[i][j] = sram_mv; } + for j in ps.volt_mv.len()..raw.voltages[i].len() { + raw.voltages[i][j] = raw.voltages[i][0]; + raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; + } + raw.sram_k[i] = self.cfg.sram_k; + raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; + raw.rel_boost_freqs[i] = if i > base_ps { + (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) + } else { + 0 + }; } - for (i, ps) in csafr.perf_states_afr.iter().enumerate() { - aux.afr_frequencies[i] = ps.freq_hz / 1000000; - for (j, mv) in ps.volt_mv.iter().enumerate() { - let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); - aux.afr_voltages[i][j] = *mv; - aux.afr_voltages_sram[i][j] = sram_mv; + #[ver(V >= V13_0B4)] + if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } } } - } - Ok(raw) - }) + Ok(()) + }, + ) + }) } /// Create the Globals structure, which contains global firmware config including more power /// configuration data and globals used to exchange state between the firmware and driver. - #[inline(never)] fn globals(&mut self) -> Result> { - self.alloc - .shared - .new_inplace(Default::default(), |_inner, ptr| { - let pwr = &self.dyncfg.pwr; - let period_ms = pwr.power_sample_period; - let period_s = F32::from(period_ms) / f32!(1000.0); - let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; - - let max_ps = pwr.perf_max_pstate; - let max_ps_scaled = 100 * max_ps; - - let raw = place!( - ptr, - raw::Globals::ver { - //ktrace_enable: 0xffffffff, - ktrace_enable: 0, - #[ver(V >= V13_2)] - unk_24_0: 3000, - unk_24: 0, - #[ver(V >= V13_0B4)] - debug: 0, - unk_28: 1, - #[ver(V >= V13_3)] - unk_2c_0: 1, - #[ver(V >= V13_0B4 && V < V13_3)] - unk_2c_0: 0, - unk_2c: 1, - unk_30: 0, - unk_34: 120, - sub: raw::GlobalsSub::ver { - unk_54: self.cfg.global_unk_54, - unk_56: 40, - unk_58: 0xffff, - unk_5e: U32(1), - unk_66: U32(1), - ..Default::default() - }, - unk_8900: 1, - pending_submissions: AtomicU32::new(0), - max_power: pwr.max_power_mw, - max_pstate_scaled: max_ps_scaled, - max_pstate_scaled_2: max_ps_scaled, - max_pstate_scaled_3: max_ps_scaled, - power_zone_count: pwr.power_zones.len() as u32, - avg_power_filter_tc_periods: avg_power_filter_tc_periods, - avg_power_ki_dt: pwr.avg_power_ki_only * period_s, - avg_power_kp: pwr.avg_power_kp, - avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, - avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, - unk_89bc: self.cfg.da.unk_8cc, - fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, - unk_89c4: self.cfg.da.unk_87c, - fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, - fast_die0_kp: pwr.fast_die0_proportional_gain, - fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, - unk_89e0: 1, - max_power_2: pwr.max_power_mw, - ppm_kp: pwr.ppm_kp, - ppm_ki_dt: pwr.ppm_ki * period_s, - #[ver(V >= V13_0B4)] - unk_89f4_8: 1, - unk_89f4: 0, - hws1: Self::hw_shared1(self.cfg), - hws2: *Self::hw_shared2(self.cfg, self.dyncfg)?, - hws3: *Self::hw_shared3(self.cfg)?, - #[ver(V >= V13_0B4)] - unk_hws2_0: self.cfg.unk_hws2_0, - #[ver(V >= V13_0B4)] - unk_hws2_4: self.cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), - #[ver(V >= V13_0B4)] - unk_hws2_24: self.cfg.unk_hws2_24, - unk_900c: 1, - #[ver(V >= V13_0B4)] - unk_9010_0: 1, - #[ver(V >= V13_0B4)] - unk_903c: 1, - #[ver(V < V13_0B4)] - unk_903c: 0, - fault_control: *crate::fault_control.read(), - do_init: 1, - unk_11020: 40, - unk_11024: 10, - unk_11028: 250, - #[ver(V >= V13_0B4)] - unk_1102c_0: 1, - #[ver(V >= V13_0B4)] - unk_1102c_4: 1, - #[ver(V >= V13_0B4)] - unk_1102c_8: 100, - #[ver(V >= V13_0B4)] - unk_1102c_c: 1, - idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), - fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, - fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, - unk_118e0: 40, - #[ver(V >= V13_0B4)] - unk_118e4_0: 50, - #[ver(V >= V13_0B4)] - unk_11edc: 0, - #[ver(V >= V13_0B4)] - unk_11efc: 0, - ..Default::default() + self.alloc.private.new_init(init::zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + + init::chain( + try_init!(raw::Globals::ver { + //ktrace_enable: 0xffffffff, + ktrace_enable: 0, + #[ver(V >= V13_2)] + unk_24_0: 3000, + unk_24: 0, + #[ver(V >= V13_0B4)] + debug: 0, + unk_28: 1, + #[ver(V >= V13_3)] + unk_2c_0: 1, + #[ver(V >= V13_0B4 && V < V13_3)] + unk_2c_0: 0, + unk_2c: 1, + unk_30: 0, + unk_34: 120, + sub <- try_init!(raw::GlobalsSub::ver { + unk_54: cfg.global_unk_54, + unk_56: 40, + unk_58: 0xffff, + unk_5e: U32(1), + unk_66: U32(1), + ..Zeroable::zeroed() + }), + unk_8900: 1, + pending_submissions: AtomicU32::new(0), + max_power: pwr.max_power_mw, + max_pstate_scaled: max_ps_scaled, + max_pstate_scaled_2: max_ps_scaled, + max_pstate_scaled_3: max_ps_scaled, + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: avg_power_filter_tc_periods, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + unk_89bc: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_89c4: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_89e0: 1, + max_power_2: pwr.max_power_mw, + ppm_kp: pwr.ppm_kp, + ppm_ki_dt: pwr.ppm_ki * period_s, + #[ver(V >= V13_0B4)] + unk_89f4_8: 1, + unk_89f4: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + #[ver(V >= V13_0B4)] + unk_hws2_0: cfg.unk_hws2_0, + #[ver(V >= V13_0B4)] + unk_hws2_4: cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), + #[ver(V >= V13_0B4)] + unk_hws2_24: cfg.unk_hws2_24, + unk_900c: 1, + #[ver(V >= V13_0B4)] + unk_9010_0: 1, + #[ver(V >= V13_0B4)] + unk_903c: 1, + #[ver(V < V13_0B4)] + unk_903c: 0, + fault_control: *crate::fault_control.read(), + do_init: 1, + unk_11020: 40, + unk_11024: 10, + unk_11028: 250, + #[ver(V >= V13_0B4)] + unk_1102c_0: 1, + #[ver(V >= V13_0B4)] + unk_1102c_4: 1, + #[ver(V >= V13_0B4)] + unk_1102c_8: 100, + #[ver(V >= V13_0B4)] + unk_1102c_c: 1, + idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), + fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + unk_118e0: 40, + #[ver(V >= V13_0B4)] + unk_118e4_0: 50, + #[ver(V >= V13_0B4)] + unk_11edc: 0, + #[ver(V >= V13_0B4)] + unk_11efc: 0, + ..Zeroable::zeroed() + }), + |raw| { + for (i, pz) in self.dyncfg.pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; } - ); - - for (i, pz) in pwr.power_zones.iter().enumerate() { - raw.power_zones[i].target = pz.target; - raw.power_zones[i].target_off = pz.target - pz.target_offset; - raw.power_zones[i].filter_tc = pz.filter_tc; - } - if let Some(tab) = self.cfg.global_tab.as_ref() { - for (i, x) in tab.iter().enumerate() { - raw.unk_118ec[i] = *x; + if let Some(tab) = self.cfg.global_tab.as_ref() { + for (i, x) in tab.iter().enumerate() { + raw.unk_118ec[i] = *x; + } + raw.unk_118e8 = 1; } - raw.unk_118e8 = 1; - } - - Ok(raw) - }) + Ok(()) + }, + ) + }) } /// Create the RuntimePointers structure, which contains pointers to most of the other /// structures including the ring buffer channels, statistics structures, and HwDataA/HwDataB. - #[inline(never)] fn runtime_pointers(&mut self) -> Result> { let hwa = self.hwdata_a()?; let hwb = self.hwdata_b()?; - let pointers: Box = box_in_place!(RuntimePointers::ver { - stats: Stats::ver { - vtx: self.alloc.private.new_default::()?, - frag: self.alloc.private.new_default::()?, - comp: self.alloc.private.new_default::()?, - }, + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointers::ver { + stats <- { + let alloc = &mut *alloc; + try_init!(Stats::ver { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_default::()?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + unkptr_190: alloc.private.array_empty(0x80)?, + unkptr_198: alloc.private.array_empty(0xc0)?, + hwdata_b: hwb, + + unkptr_1b8: alloc.private.array_empty(0x1000)?, + unkptr_1c0: alloc.private.array_empty(0x300)?, + unkptr_1c8: alloc.private.array_empty(0x1000)?, - hwdata_a: hwa, - unkptr_190: self.alloc.private.array_empty(0x80)?, - unkptr_198: self.alloc.private.array_empty(0xc0)?, - hwdata_b: hwb, - - unkptr_1b8: self.alloc.private.array_empty(0x1000)?, - unkptr_1c0: self.alloc.private.array_empty(0x300)?, - unkptr_1c8: self.alloc.private.array_empty(0x1000)?, - - #[ver(V < V13_3)] - buffer_mgr_ctl: self.alloc.gpu.array_empty(127)?, - #[ver(V >= V13_3)] - buffer_mgr_ctl: self.alloc.gpu_low.array_empty(127)?, - })?; - - self.alloc.private.new_boxed(pointers, |inner, ptr| { - Ok(place!( - ptr, - raw::RuntimePointers::ver { + #[ver(V < V13_3)] + buffer_mgr_ctl: alloc.gpu.array_empty(127)?, + #[ver(V >= V13_3)] + buffer_mgr_ctl: alloc.gpu_low.array_empty(127)?, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointers::ver { pipes: Default::default(), device_control: Default::default(), event: Default::default(), @@ -801,14 +805,13 @@ impl<'a> InitDataBuilder::ver<'a> { unk_6b38: 0xff, ..Default::default() }, - } - )) - }) + }) + }, + ) } /// Create the FwStatus structure, which is used to coordinate the firmware halt state between /// the firmware and the driver. - #[inline(never)] fn fw_status(&mut self) -> Result> { self.alloc .shared @@ -816,9 +819,8 @@ impl<'a> InitDataBuilder::ver<'a> { } /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. - #[inline(never)] fn uat_level_info( - cfg: &hw::HwConfig, + cfg: &'static hw::HwConfig, index_shift: usize, num_entries: usize, ) -> raw::UatLevelInfo { @@ -838,48 +840,50 @@ impl<'a> InitDataBuilder::ver<'a> { /// Build the top-level InitData object. #[inline(never)] pub(crate) fn build(&mut self) -> Result>> { - let inner: Box = box_in_place!(InitData::ver { - unk_buf: self.alloc.shared_ro.array_empty(0x4000)?, - runtime_pointers: self.runtime_pointers()?, - globals: self.globals()?, - fw_status: self.fw_status()?, - })?; - - Ok(Box::try_new(self.alloc.shared_ro.new_boxed( - inner, - |inner, ptr| { - Ok(place!( - ptr, - raw::InitData::ver { - #[ver(V == V13_3 && G == G13)] - ver_info: Array::new([1, 1, 16, 1]), // TODO - #[ver(V == V13_3 && G == G14)] - ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), - #[ver(V == V13_3 && G == G14X)] - ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), - unk_buf: inner.unk_buf.gpu_pointer(), - unk_8: 0, - unk_c: 0, - runtime_pointers: inner.runtime_pointers.gpu_pointer(), - globals: inner.globals.gpu_pointer(), - fw_status: inner.fw_status.gpu_pointer(), - uat_page_size: 0x4000, - uat_page_bits: 14, - uat_num_levels: 3, - uat_level_info: Array::new([ - Self::uat_level_info(self.cfg, 36, 8), - Self::uat_level_info(self.cfg, 25, 2048), - Self::uat_level_info(self.cfg, 14, 2048), - ]), - __pad0: Default::default(), - host_mapped_fw_allocations: 1, - unk_ac: 0, - unk_b0: 0, - unk_b4: 0, - unk_b8: 0, - } - )) + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + let fw_status = self.fw_status()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitData::ver { + unk_buf: shared_ro.array_empty(0x4000)?, + runtime_pointers, + globals, + fw_status, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitData::ver { + #[ver(V == V13_3 && G == G13)] + ver_info: Array::new([1, 1, 16, 1]), // TODO + #[ver(V == V13_3 && G == G14)] + ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), + #[ver(V == V13_3 && G == G14X)] + ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + fw_status: inner.fw_status.gpu_pointer(), + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new([ + Self::uat_level_info(cfg, 36, 8), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ]), + __pad0: Default::default(), + host_mapped_fw_allocations: 1, + unk_ac: 0, + unk_b0: 0, + unk_b4: 0, + unk_b8: 0, + }) }, - )?)?) + )?; + Ok(Box::try_new(obj)?) } } From daf2b378e7c34ee2305cd75ffe6e77e00c148719 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 19:38:12 +0900 Subject: [PATCH 591/635] drm/asahi: Delete place module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All usage has been replaced with init!() and friends! 🎉 Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/asahi.rs | 1 - drivers/gpu/drm/asahi/place.rs | 343 --------------------------------- 2 files changed, 344 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/place.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index e511d83f4cd150..5423c856abcea7 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -20,7 +20,6 @@ mod mem; mod microseq; mod mmu; mod object; -mod place; mod queue; mod regs; mod slotalloc; diff --git a/drivers/gpu/drm/asahi/place.rs b/drivers/gpu/drm/asahi/place.rs deleted file mode 100644 index 33df7199b6caa1..00000000000000 --- a/drivers/gpu/drm/asahi/place.rs +++ /dev/null @@ -1,343 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -//! "Placement new" macro -//! -//! This cursed abomination of a declarative macro is used to emulate a "placement new" feature, -//! which allows initializing objects directly in a user-provided memory region without first -//! going through the stack. -//! -//! This driver needs to manage several large GPU objects of a fixed layout. Linux kernel stacks are -//! very small, so it is impossible to create these objects on the stack. While the compiler can -//! sometimes optimize away the stack copy and directly instantiate in target memory, this is not -//! guaranteed and not reliable. Therefore, we need some mechanism to ergonomically initialize -//! complex structures directly in a pre-allocated piece of memory. -//! -//! This issue also affects some driver-internal structs which are large/complex enough to overflow -//! the stack. While this can be solved by breaking them up into pieces and using `Box` more -//! liberally, this has performance implications and still isn't very nice. This macro can also be -//! used to solve this issue. -//! -//! # Further reading -//! https://github.com/rust-lang/rust/issues/27779#issuecomment-378416911 -//! https://internals.rust-lang.org/t/removal-of-all-unstable-placement-features/7223 - -/// Initialize a `MaybeUninit` in-place, without constructing the value on the stack first. -/// -/// This macro is analogous to `MaybeUninit::write()`. In other words, -/// `place!(foo, bar)` is equivalent to `MaybeUninit::write(foo, bar)`, except that `bar` is not -/// constructed first, but rather its fields (if it is a structure constructor) are copied one by -/// one into the correct location in the `MaybeUninit`. -/// -/// The macro supports most Rust initialization syntax including type paths, generic arguments, -/// and nested structures. Nested structures are themselves initialized in-place field by field. -/// `..Default::default()` is supported, but this macro converts it to `..Zeroable::zeroed()`, as it -/// initializes those structs by zero-initializing the underlying memory. Usage of -/// `..Default::default()` with a type not implementing `Zeroable` will result in a compile error. -/// -/// Usage: -/// ``` -/// let mut buf = MaybeUninit::uninit(); -/// let mut_ref = place!(&mut buf, MyStruct { -/// b: true, -/// s: String::from("works"), -/// i: str::parse::("123").unwrap(), -/// v: vec![String::from("works")], -/// x: foo::MyOtherCoolStruct { -/// a: false, -/// b: String::from("Hello, world!"), -/// }, -/// y: foo::MyOtherCoolStruct { -/// a: false, -/// b: String::from("Hello, world!"), -/// }, -/// z: foo::MyCoolGenericStruct:: { -/// a: false, -/// b: String::from("Hello, world!"), -/// }, -/// }; -/// // `mut_ref` is now a mutable reference to the `buf`, which is now safely initialized. -/// ``` -/// -/// Based on https://crates.io/crates/place by DianaNites, with contributions by Joshua Barretto. -#[macro_export] -macro_rules! place { - // Top-level struct - (@STRUCT $ptr:ident, _TOP, $typ:path, {$($typ_init:tt)*} { $($fields:tt)* }) => {{ - place!(@STRUCT_ZERO $ptr, {$($typ_init)*} { $($fields)* }); - place!(@STRUCT_CHECK $ptr, {$($typ_init)*} { $($fields)* } { - place!(@FIELDS $ptr, $($fields)*); - }); - }}; - // Nested structure - (@STRUCT $ptr:ident, $f_struct:ident, $typ:path, {$($typ_init:tt)*} { $($fields:tt)* }) => {{ - use core::ptr::addr_of_mut; - let buf = unsafe { addr_of_mut!((*$ptr).$f_struct) }; - place!(@STRUCT_ZERO buf, {$($typ_init)*} { $($fields)* }); - place!(@STRUCT_CHECK $ptr, {$($typ_init)*} { $($fields)* } { - place!(@FIELDS buf, $($fields)*); - }); - }}; - - // Zero-initialize structure if the initializer ends in ..default::Default() - (@STRUCT_ZERO $ptr:ident, {$($typ_init:tt)*} { $($f:ident $(: $v:expr)?),* $(,)? }) => {}; - (@STRUCT_ZERO $ptr:ident, {$($typ_init:tt)*} { $($($f:ident $(: $v:expr)?),*,)? ..Default::default() }) => {{ - // Check that the structure actually implements Zeroable - const _: () = { - fn _check_default() { - let _ = $($typ_init)* { - ..Zeroable::zeroed() - }; - } - }; - use core::ptr; - unsafe { ptr::write_bytes($ptr, 0, 1) }; - - }}; - - // Check that all fields are specified - (@STRUCT_CHECK $ptr:ident, {$($typ_init:tt)*} { $($($f:ident $(: $v:expr)?),*,)? ..Default::default() } {$($body:tt)*}) => { - if false { - #[allow(clippy::redundant_field_names)] - let _x = $($typ_init)* { - $($( - $f $(: $v)? - ),* - ,)? - ..Zeroable::zeroed() - }; - } else { - {$($body)*} - } - }; - (@STRUCT_CHECK $ptr:ident, {$($typ_init:tt)*} { $($f:ident $(: $v:expr)?),* $(,)? } {$($body:tt)*}) => { - if false { - #[allow(clippy::redundant_field_names)] - let _x = $($typ_init)* { - $( - $f $(: $v)? - ),* - }; - } else { - {$($body)*} - } - }; - // Top-level scalar - (@SCALAR $ptr:ident, _TOP, $val:expr) => { - let tmp = $val; - unsafe { $ptr.write(tmp); } - }; - // Regular field - (@SCALAR $ptr:ident, $f:ident, $val:expr) => {{ - use core::ptr::addr_of_mut; - let tmp = $val; - unsafe { addr_of_mut!((*$ptr).$f).write(tmp); } - }}; - // Type-like name followed by braces is a nested structure - (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {{ $($fields:tt)* } $($tail:tt)*}) => { - place!(@STRUCT $ptr, $f, $($head)*, {$($head)*} { $($fields)* }); - place!(@FIELDS $ptr $($tail)*) - }; - // Type-like name followed by ::ident, append to head - (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {::$id:ident $($tail:tt)*}) => { - place!(@PARTIAL $ptr, $f, {$($head)* :: $id}, {$($tail)*}); - }; - // Type-like name followed by ::, append to head - (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {::<$($gen:ty),*> $($tail:tt)*}) => { - place!(@PARTIAL $ptr, $f, {$($head)* :: <$($gen),*>}, {$($tail)*}); - }; - // Type-like name followed by ::<'lifetime>, append to head - (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {::<$li:lifetime> $($tail:tt)*}) => { - place!(@PARTIAL $ptr, $f, {$($head)* :: <$li>}, {$($tail)*}); - }; - // Anything else, parse it as an expression - (@PARTIAL $ptr:ident, $f:ident, {$($head:tt)*}, {$($tail:tt)*}) => { - place!(@EXPR $ptr, $f, $($head)* $($tail)*) - }; - // Expression followed by more fields - (@EXPR $ptr:ident, $f:ident, $val:expr, $($tail:tt)*) => { - place!(@SCALAR $ptr, $f, $val); - place!(@FIELDS $ptr, $($tail)*) - }; - // Last field expression, without a trailing comma - (@EXPR $ptr:ident, $f:ident, $val:expr) => { - place!(@SCALAR $ptr, $f, $val); - }; - // Field with a value starting with an ident, start incremental type parsing - (@FIELDS $ptr:ident, $f:ident : $id:ident $($tail:tt)*) => { - place!(@PARTIAL $ptr, $f, {$id}, {$($tail)*}); - }; - // Same, but starting with ::ident - (@FIELDS $ptr:ident, $f:ident : ::$id:ident $($tail:tt)*) => { - place!(@PARTIAL $ptr, $f, {::$id}, {$($tail)*}); - }; - // Otherwise, parse it as an expression - (@FIELDS $ptr:ident, $f:ident : $($tail:tt)*) => { - place!(@EXPR $ptr, $f, $($tail)*) - }; - // Default terminating case - (@FIELDS $ptr:ident, ..Default::default() ) => {}; - // Terminating case - (@FIELDS $ptr:ident $(,)? ) => {}; - ( - $buf:expr, - $($val:tt)* - ) => {{ - use core::mem::MaybeUninit; - // Ensures types are correct - let obj: &mut MaybeUninit<_> = $buf; - let top_ptr = obj.as_mut_ptr(); - place!(@FIELDS top_ptr, _TOP: $($val)*); - // SAFETY: All fields have been initialized above - // The compiler ensures that all fields were used, all types were correct, - // and that size and alignment are correct. - unsafe { obj.assume_init_mut() } - }}; -} - -/// Helper macro to get the struct type part of a struct initialization expression. -#[macro_export] -#[doc(hidden)] -macro_rules! get_type { - ($t:ty { $($val:tt)* }) => { - $t - }; -} - -/// Like `Box::try_new(...)`, but with in-place initialization. -#[macro_export] -macro_rules! box_in_place { - ($($val:tt)*) => {{ - use $crate::place; - let b = Box::<$crate::get_type!($($val)*)>::try_new_uninit(); - match b { - Ok(mut p) => { - place!((&mut *p), $($val)*); - Ok(unsafe { p.assume_init() }) - } - Err(e) => Err(e) - } - }}; -} - -// TODO: figure out how to make this run -#[cfg(test)] -mod tests { - use super::*; - use core::mem::MaybeUninit; - - #[derive(Debug, PartialEq)] - struct MyCoolStruct { - b: bool, - s: String, - i: i32, - v: Vec, - x: MyOtherCoolStruct, - y: MyOtherCoolStruct, - z: foo::MyCoolGenericStruct, - } - - #[derive(Debug, PartialEq)] - struct MyDefaultStruct { - b: bool, - i: i32, - j: i16, - } - default_zeroed!(MyDefaultStruct); - - mod foo { - #[derive(Debug, PartialEq)] - pub struct MyOtherCoolStruct { - pub a: bool, - pub b: String, - } - #[derive(Debug, PartialEq)] - pub struct MyCoolGenericStruct { - pub a: T, - pub b: U, - } - } - - use foo::MyOtherCoolStruct; - - #[test] - fn test_initialized() { - let mut buf: MaybeUninit = MaybeUninit::uninit(); - - let x: &mut MyCoolStruct = place!( - &mut buf, - MyCoolStruct { - b: true, - s: String::from("works"), - i: str::parse::("123").unwrap(), - v: vec![String::from("works")], - x: MyOtherCoolStruct { - a: false, - b: String::from("Hello, world!"), - }, - y: foo::MyOtherCoolStruct { - a: false, - b: String::from("Hello, world!"), - }, - z: foo::MyCoolGenericStruct:: { - a: false, - b: String::from("Hello, world!"), - } - } - ); - //dbg!(x); - - assert_eq!( - x, - &MyCoolStruct { - b: true, - s: String::from("works"), - i: str::parse::("123").unwrap(), - v: vec![String::from("works")], - x: foo::MyOtherCoolStruct { - a: false, - b: String::from("Hello, world!"), - }, - y: foo::MyOtherCoolStruct { - a: false, - b: String::from("Hello, world!"), - }, - z: foo::MyCoolGenericStruct:: { - a: false, - b: String::from("Hello, world!"), - }, - }, - ); - } - - #[test] - fn test_default() { - let mut buf: MaybeUninit = MaybeUninit::uninit(); - - let x: &mut MyDefaultStruct = place!( - &mut buf, - MyDefaultStruct { - b: true, - i: 1, - ..Default::default() - } - ); - - assert_eq!( - x, - &MyDefaultStruct { - b: true, - i: 1, - j: 0, - }, - ); - } - - #[test] - fn test_scalar() { - let mut buf: MaybeUninit = MaybeUninit::uninit(); - - let x: &mut u32 = place!(&mut buf, 1234); - - assert_eq!(x, &mut 1234u32); - } -} From 338382c58ba40e68d4b9d840eb0b9537064bdc4b Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 22:21:10 +0900 Subject: [PATCH 592/635] rust: drm: device: Convert Device to AlwaysRefCounted Switch from being a refcount wrapper itself to a transparent wrapper around `bindings::drm_device`. The refcounted type then becomes ARef>. Signed-off-by: Asahi Lina --- rust/kernel/drm/device.rs | 71 ++++++++++++++++-------------------- rust/kernel/drm/drv.rs | 19 +++++----- rust/kernel/drm/file.rs | 4 +- rust/kernel/drm/gem/mod.rs | 16 ++++---- rust/kernel/drm/gem/shmem.rs | 19 +++++----- rust/kernel/drm/ioctl.rs | 10 ++--- 6 files changed, 64 insertions(+), 75 deletions(-) diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 6007f941137a2c..3a42effbb2b426 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -4,73 +4,66 @@ //! //! C header: [`include/linux/drm/drm_device.h`](../../../../include/linux/drm/drm_device.h) -use crate::{bindings, device, drm, types::ForeignOwnable}; +use crate::{ + bindings, device, drm, + types::{ARef, AlwaysRefCounted, ForeignOwnable}, +}; +use core::cell::UnsafeCell; use core::marker::PhantomData; +use core::ptr::NonNull; -/// Represents a reference to a DRM device. The device is reference-counted and is guaranteed to -/// not be dropped while this object is alive. +/// A typed DRM device with a specific driver. The device is always reference-counted. +#[repr(transparent)] pub struct Device { - // Type invariant: ptr must be a valid and initialized drm_device, - // and this value must either own a reference to it or the caller - // must ensure that it is never dropped if the reference is borrowed. - pub(super) ptr: *mut bindings::drm_device, + pub(super) drm: UnsafeCell, _p: PhantomData, } impl Device { - // Not intended to be called externally, except via declare_drm_ioctls!() - #[doc(hidden)] - pub unsafe fn from_raw(raw: *mut bindings::drm_device) -> Device { - Device { - ptr: raw, - _p: PhantomData, - } - } - - #[allow(dead_code)] - pub(crate) fn raw(&self) -> *const bindings::drm_device { - self.ptr + #[allow(dead_code, clippy::mut_from_ref)] + pub(crate) unsafe fn raw_mut(&self) -> &mut bindings::drm_device { + unsafe { &mut *self.drm.get() } } - pub(crate) fn raw_mut(&mut self) -> *mut bindings::drm_device { - self.ptr + // Not intended to be called externally, except via declare_drm_ioctls!() + #[doc(hidden)] + pub unsafe fn borrow<'a>(raw: *const bindings::drm_device) -> &'a Self { + unsafe { &*(raw as *const Self) } } /// Returns a borrowed reference to the user data associated with this Device. pub fn data(&self) -> ::Borrowed<'_> { - unsafe { T::Data::borrow((*self.ptr).dev_private) } + // SAFETY: dev_private is guaranteed to be initialized for all + // Device objects exposed to users. + unsafe { T::Data::borrow((*self.drm.get()).dev_private) } } } -impl Drop for Device { - fn drop(&mut self) { - // SAFETY: By the type invariants, we know that `self` owns a reference, so it is safe to - // relinquish it now. - unsafe { bindings::drm_dev_put(self.ptr) }; +// SAFETY: DRM device objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + unsafe { bindings::drm_dev_get(&self.drm as *const _ as *mut _) }; } -} -impl Clone for Device { - fn clone(&self) -> Self { - // SAFETY: We get a new reference and then create a new owning object from the raw pointer - unsafe { - bindings::drm_dev_get(self.ptr); - Device::from_raw(self.ptr) - } + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The Device type has the same layout as drm_device, + // so we can just cast. + unsafe { bindings::drm_dev_put(obj.as_ptr() as *mut _) }; } } // SAFETY: `Device` only holds a pointer to a C device, which is safe to be used from any thread. -unsafe impl Send for Device {} +unsafe impl Send for ARef> {} // SAFETY: `Device` only holds a pointer to a C device, references to which are safe to be used // from any thread. -unsafe impl Sync for Device {} +unsafe impl Sync for ARef> {} // Make drm::Device work for dev_info!() and friends unsafe impl device::RawDevice for Device { fn raw_device(&self) -> *mut bindings::device { - // SAFETY: ptr must be valid per the type invariant - unsafe { (*self.ptr).dev } + // SAFETY: dev is initialized by C for all Device objects + unsafe { (*self.drm.get()).dev } } } diff --git a/rust/kernel/drm/drv.rs b/rust/kernel/drm/drv.rs index c9a2f68605a95f..b995e071151815 100644 --- a/rust/kernel/drm/drv.rs +++ b/rust/kernel/drm/drv.rs @@ -12,12 +12,13 @@ use crate::{ prelude::*, private::Sealed, str::CStr, - types::ForeignOwnable, + types::{ARef, ForeignOwnable}, ThisModule, }; use core::{ marker::{PhantomData, PhantomPinned}, pin::Pin, + ptr::NonNull, }; use macros::vtable; @@ -163,7 +164,7 @@ pub trait Driver { /// /// drm is always a valid pointer to an allocated drm_device pub struct Registration { - drm: drm::device::Device, + drm: ARef>, registered: bool, fops: bindings::file_operations, vtable: Pin>, @@ -253,11 +254,11 @@ impl Registration { pub fn new(parent: &dyn device::RawDevice) -> Result { let vtable = Pin::new(Box::try_new(Self::VTABLE)?); let raw_drm = unsafe { bindings::drm_dev_alloc(&*vtable, parent.raw_device()) }; - let raw_drm = from_err_ptr(raw_drm)?; + let raw_drm = NonNull::new(from_err_ptr(raw_drm)? as *mut _).ok_or(ENOMEM)?; // The reference count is one, and now we take ownership of that reference as a // drm::device::Device. - let drm = unsafe { drm::device::Device::from_raw(raw_drm) }; + let drm = unsafe { ARef::from_raw(raw_drm) }; Ok(Self { drm, @@ -287,10 +288,9 @@ impl Registration { // SAFETY: We never move out of `this`. let this = unsafe { self.get_unchecked_mut() }; let data_pointer = ::into_foreign(data); - // SAFETY: `drm` is valid per the type invariant - unsafe { - (*this.drm.raw_mut()).dev_private = data_pointer as *mut _; - } + // SAFETY: This is the only code touching dev_private, so it is safe to upgrade to a + // mutable reference. + unsafe { this.drm.raw_mut() }.dev_private = data_pointer as *mut _; this.fops.owner = module.0; this.vtable.fops = &this.fops; @@ -309,6 +309,7 @@ impl Registration { /// Returns a reference to the `Device` instance for this registration. pub fn device(&self) -> &drm::device::Device { + // TODO: rework this, ensure this only works after registration &self.drm } } @@ -329,7 +330,7 @@ impl Drop for Registration { if self.registered { // Get a pointer to the data stored in device before destroying it. // SAFETY: `drm` is valid per the type invariant - let data_pointer = unsafe { (*self.drm.raw_mut()).dev_private }; + let data_pointer = unsafe { self.drm.raw_mut().dev_private }; // SAFETY: Since `registered` is true, `self.drm` is both valid and registered. unsafe { bindings::drm_dev_unregister(self.drm.raw_mut()) }; diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index eb01f5833206d1..8f725ba17a0dcf 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -32,11 +32,11 @@ pub(super) unsafe extern "C" fn open_callback( raw_dev: *mut bindings::drm_device, raw_file: *mut bindings::drm_file, ) -> core::ffi::c_int { - let drm = core::mem::ManuallyDrop::new(unsafe { drm::device::Device::from_raw(raw_dev) }); + let drm = unsafe { drm::device::Device::borrow(raw_dev) }; // SAFETY: This reference won't escape this function let file = unsafe { &mut *raw_file }; - let inner = match T::open(&drm) { + let inner = match T::open(drm) { Err(e) => { return e.to_errno(); } diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index 658416b95818a5..31426ce03ab3d3 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -15,7 +15,7 @@ use crate::{ error::{to_result, Result}, prelude::*, }; -use core::{marker::PhantomPinned, mem, mem::ManuallyDrop, ops::Deref, ops::DerefMut}; +use core::{marker::PhantomPinned, mem, ops::Deref, ops::DerefMut}; /// GEM object functions, which must be implemented by drivers. pub trait BaseDriverObject: Sync + Send + Sized { @@ -211,9 +211,7 @@ impl BaseObject for T {} #[pin_data] pub struct Object { obj: bindings::drm_gem_object, - // The DRM core ensures the Device exists as long as its objects exist, so we don't need to - // manage the reference count here. - dev: ManuallyDrop>, + dev: *const bindings::drm_device, #[pin] inner: T, #[pin] @@ -251,14 +249,12 @@ impl Object { ..Default::default() }, inner <- T::new(dev, size), - // SAFETY: The drm subsystem guarantees that the drm_device will live as long as - // the GEM object lives, so we can conjure a reference out of thin air. - dev: ManuallyDrop::new(unsafe { device::Device::from_raw(dev.ptr) }), + dev: dev.drm.get(), _p: PhantomPinned }))?; to_result(unsafe { - bindings::drm_gem_object_init(dev.raw() as *mut _, &obj.obj as *const _ as *mut _, size) + bindings::drm_gem_object_init(dev.raw_mut(), &obj.obj as *const _ as *mut _, size) })?; // SAFETY: We never move out of self @@ -275,7 +271,9 @@ impl Object { /// Returns the `Device` that owns this GEM object. pub fn dev(&self) -> &device::Device { - &self.dev + // SAFETY: The drm subsystem guarantees that the drm_device will live as long as + // the GEM object lives, so we can just borrow from the raw pointer. + unsafe { device::Device::borrow(self.dev) } } } diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 17cfb51cf36449..782bd5dd85da22 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -12,7 +12,7 @@ use crate::{ use core::{ marker::{PhantomData, PhantomPinned}, mem, - mem::{ManuallyDrop, MaybeUninit}, + mem::MaybeUninit, ops::{Deref, DerefMut}, slice, }; @@ -71,7 +71,7 @@ pub struct Object { obj: bindings::drm_gem_shmem_object, // The DRM core ensures the Device exists as long as its objects exist, so we don't need to // manage the reference count here. - dev: ManuallyDrop>, + dev: *const bindings::drm_device, #[pin] inner: T, } @@ -80,13 +80,9 @@ pub struct Object { unsafe impl init::Zeroable for bindings::drm_gem_shmem_object {} unsafe extern "C" fn gem_create_object( - raw_dev: *mut bindings::drm_device, + dev: *mut bindings::drm_device, size: usize, ) -> *mut bindings::drm_gem_object { - // SAFETY: GEM ensures the device lives as long as its objects live, - // so we can conjure up a reference from thin air and never drop it. - let dev = ManuallyDrop::new(unsafe { device::Device::from_raw(raw_dev) }); - let p = unsafe { bindings::krealloc(core::ptr::null(), Object::::SIZE, bindings::GFP_KERNEL) as *mut Object @@ -98,7 +94,8 @@ unsafe extern "C" fn gem_create_object( let init = try_pin_init!(Object { obj <- init::zeroed(), - inner <- T::new(&*dev, size), + // SAFETY: GEM ensures the device lives as long as its objects live + inner <- T::new(unsafe { device::Device::borrow(dev)}, size), dev, }); @@ -163,7 +160,7 @@ impl Object { pub fn new(dev: &device::Device, size: usize) -> Result> { // SAFETY: This function can be called as long as the ALLOC_OPS are set properly // for this driver, and the gem_create_object is called. - let p = unsafe { bindings::drm_gem_shmem_create(dev.raw() as *mut _, size) }; + let p = unsafe { bindings::drm_gem_shmem_create(dev.raw_mut(), size) }; let p = crate::container_of!(p, Object, obj) as *mut _; // SAFETY: The gem_create_object callback ensures this is a valid Object, @@ -178,7 +175,9 @@ impl Object { /// Returns the `Device` that owns this GEM object. pub fn dev(&self) -> &device::Device { - &self.dev + // SAFETY: GEM ensures that the device outlives its objects, so we can + // just borrow here. + unsafe { device::Device::borrow(self.dev) } } /// Creates (if necessary) and returns a scatter-gather table of DMA pages for this object. diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index 3f6a93cf009c51..4a5f4d9ddb14d9 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -128,22 +128,20 @@ macro_rules! declare_drm_ioctls { raw_data: *mut ::core::ffi::c_void, raw_file_priv: *mut $crate::drm::ioctl::internal::drm_file, ) -> core::ffi::c_int { - // SAFETY: We never drop this, and the DRM core ensures the device lives - // while callbacks are being called. + // SAFETY: The DRM core ensures the device lives while callbacks are + // being called. // // FIXME: Currently there is nothing enforcing that the types of the // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. - let dev = ::core::mem::ManuallyDrop::new(unsafe { - $crate::drm::device::Device::from_raw(raw_dev) - }); + let dev = $crate::drm::device::Device::borrow(raw_dev); // SAFETY: This is just the ioctl argument, which hopefully has the right type // (we've done our best checking the size). let data = unsafe { &mut *(raw_data as *mut $crate::uapi::$struct) }; // SAFETY: This is just the DRM file structure let file = unsafe { $crate::drm::file::File::from_raw(raw_file_priv) }; - match $func(&*dev, data, &file) { + match $func(dev, data, &file) { Err(e) => e.to_errno(), Ok(i) => i.try_into().unwrap_or(ERANGE.to_errno()), } From d867cd1157ce2433aa6770d251c8af6724a2a886 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Fri, 19 May 2023 22:23:35 +0900 Subject: [PATCH 593/635] drm/asahi: Convert to ARef> DRM device API Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 18 +++++++++--------- drivers/gpu/drm/asahi/channel.rs | 30 +++++++++++++++--------------- drivers/gpu/drm/asahi/driver.rs | 2 ++ drivers/gpu/drm/asahi/gpu.rs | 6 +++--- drivers/gpu/drm/asahi/mmu.rs | 25 +++++++++---------------- drivers/gpu/drm/asahi/queue/mod.rs | 8 ++++---- drivers/gpu/drm/asahi/workqueue.rs | 8 ++++---- 7 files changed, 46 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index ab1f02f31bdcd4..3313676ae6a14f 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -15,7 +15,7 @@ use kernel::{c_str, drm::mm, error::Result, prelude::*, str::CString, sync::LockClassKey}; use crate::debug::*; -use crate::driver::AsahiDevice; +use crate::driver::{AsahiDevRef, AsahiDevice}; use crate::fw::types::Zeroable; use crate::mmu; use crate::object::{GpuArray, GpuObject, GpuOnlyArray, GpuStruct, GpuWeakPointer}; @@ -410,7 +410,7 @@ pub(crate) trait Allocator { /// `ptr` is either None or a valid, non-null pointer to the CPU view of the object. /// `gpu_ptr` is the GPU-side VA of the object. pub(crate) struct SimpleAllocation { - dev: AsahiDevice, + dev: AsahiDevRef, ptr: Option>, gpu_ptr: u64, size: usize, @@ -461,7 +461,7 @@ impl RawAllocation for SimpleAllocation { /// the guard page after the allocation, which can be useful to validate that the firmware's or /// GPU's idea of object size what we expect. pub(crate) struct SimpleAllocator { - dev: AsahiDevice, + dev: AsahiDevRef, start: u64, end: u64, prot: u32, @@ -490,7 +490,7 @@ impl SimpleAllocator { cpu_maps = true; } Ok(SimpleAllocator { - dev: dev.clone(), + dev: dev.into(), vm: vm.clone(), start, end, @@ -572,7 +572,7 @@ impl Allocator for SimpleAllocator { /// /// This is wrapped in an `mm::Node`. pub(crate) struct HeapAllocationInner { - dev: AsahiDevice, + dev: AsahiDevRef, ptr: Option>, real_size: usize, } @@ -664,7 +664,7 @@ impl RawAllocation for HeapAllocation { /// /// This is wrapped by an `mm::Allocator`. struct HeapAllocatorInner { - dev: AsahiDevice, + dev: AsahiDevRef, allocated: usize, backing_objects: Vec<(crate::gem::ObjectRef, u64)>, garbage: Option>>, @@ -678,7 +678,7 @@ struct HeapAllocatorInner { /// The heap is composed of a series of GEM objects. This implementation only ever grows the heap, /// never shrinks it. pub(crate) struct HeapAllocator { - dev: AsahiDevice, + dev: AsahiDevRef, start: u64, end: u64, top: u64, @@ -720,7 +720,7 @@ impl HeapAllocator { let name = CString::try_from_fmt(name)?; let inner = HeapAllocatorInner { - dev: dev.clone(), + dev: dev.into(), allocated: 0, backing_objects: Vec::new(), // TODO: This clearly needs a try_clone() or similar @@ -739,7 +739,7 @@ impl HeapAllocator { )?; Ok(HeapAllocator { - dev: dev.clone(), + dev: dev.into(), vm: vm.clone(), start, end, diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index b0069e5fb3a520..74632b51b4386a 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -9,7 +9,7 @@ //! implementation. use crate::debug::*; -use crate::driver::AsahiDevice; +use crate::driver::{AsahiDevRef, AsahiDevice}; use crate::fw::channels::*; use crate::fw::initdata::{raw, ChannelRing}; use crate::fw::types::*; @@ -185,7 +185,7 @@ where /// Device Control channel for global device management commands. #[versions(AGX)] pub(crate) struct DeviceControlChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: TxChannel, } @@ -199,7 +199,7 @@ impl DeviceControlChannel::ver { alloc: &mut gpu::KernelAllocators, ) -> Result { Ok(DeviceControlChannel::ver { - dev: dev.clone(), + dev: dev.into(), ch: TxChannel::::new(alloc, 0x100)?, }) } @@ -224,7 +224,7 @@ impl DeviceControlChannel::ver { /// Pipe channel to submit WorkQueue execution requests. #[versions(AGX)] pub(crate) struct PipeChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: TxChannel, } @@ -236,7 +236,7 @@ impl PipeChannel::ver { alloc: &mut gpu::KernelAllocators, ) -> Result { Ok(PipeChannel::ver { - dev: dev.clone(), + dev: dev.into(), ch: TxChannel::::new(alloc, 0x100)?, }) } @@ -255,7 +255,7 @@ impl PipeChannel::ver { /// Firmware Control channel, used for secure cache flush requests. pub(crate) struct FwCtlChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: TxChannel, } @@ -268,7 +268,7 @@ impl FwCtlChannel { alloc: &mut gpu::KernelAllocators, ) -> Result { Ok(FwCtlChannel { - dev: dev.clone(), + dev: dev.into(), ch: TxChannel::::new_uncached(alloc, 0x100)?, }) } @@ -293,7 +293,7 @@ impl FwCtlChannel { /// Event channel, used to notify the driver of command completions, GPU faults and errors, and /// other events. pub(crate) struct EventChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: RxChannel, mgr: Arc, gpu: Option>, @@ -307,7 +307,7 @@ impl EventChannel { mgr: Arc, ) -> Result { Ok(EventChannel { - dev: dev.clone(), + dev: dev.into(), ch: RxChannel::::new(alloc, 0x100)?, mgr, gpu: None, @@ -378,7 +378,7 @@ impl EventChannel { /// levels), and it also uses a side buffer to actually hold the log messages, only passing around /// pointers in the main buffer. pub(crate) struct FwLogChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: RxChannel, payload_buf: GpuArray, } @@ -393,7 +393,7 @@ impl FwLogChannel { alloc: &mut gpu::KernelAllocators, ) -> Result { Ok(FwLogChannel { - dev: dev.clone(), + dev: dev.into(), ch: RxChannel::::new(alloc, Self::RING_SIZE)?, payload_buf: alloc .shared @@ -467,7 +467,7 @@ impl FwLogChannel { } pub(crate) struct KTraceChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: RxChannel, } @@ -480,7 +480,7 @@ impl KTraceChannel { alloc: &mut gpu::KernelAllocators, ) -> Result { Ok(KTraceChannel { - dev: dev.clone(), + dev: dev.into(), ch: RxChannel::::new(alloc, 0x200)?, }) } @@ -502,7 +502,7 @@ impl KTraceChannel { /// Not really implemented other than debug logs yet... #[versions(AGX)] pub(crate) struct StatsChannel { - dev: AsahiDevice, + dev: AsahiDevRef, ch: RxChannel, } @@ -514,7 +514,7 @@ impl StatsChannel::ver { alloc: &mut gpu::KernelAllocators, ) -> Result { Ok(StatsChannel::ver { - dev: dev.clone(), + dev: dev.into(), ch: RxChannel::::new(alloc, 0x100)?, }) } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index dcf4e9ce3c0971..9ffb6905cd9c45 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -10,6 +10,7 @@ use crate::{debug, file, gem, gpu, hw, regs}; use kernel::device::RawDevice; use kernel::macros::vtable; +use kernel::types::ARef; /// Driver metadata const INFO: drv::DriverInfo = drv::DriverInfo { @@ -37,6 +38,7 @@ pub(crate) struct AsahiDriver; /// Convenience type alias for the DRM device type for this driver. pub(crate) type AsahiDevice = kernel::drm::device::Device; +pub(crate) type AsahiDevRef = ARef; /// DRM Driver implementation for `AsahiDriver`. #[vtable] diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a66330d45ab21e..77dac5e2b341cd 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -32,7 +32,7 @@ use kernel::{ use crate::alloc::Allocator; use crate::debug::*; -use crate::driver::AsahiDevice; +use crate::driver::{AsahiDevRef, AsahiDevice}; use crate::fw::channels::PipeType; use crate::fw::types::{U32, U64}; use crate::{ @@ -190,7 +190,7 @@ pub(crate) struct SequenceIDs { #[versions(AGX)] #[pin_data] pub(crate) struct GpuManager { - dev: AsahiDevice, + dev: AsahiDevRef, cfg: &'static hw::HwConfig, dyncfg: hw::DynConfig, pub(crate) initdata: fw::types::GpuObject, @@ -620,7 +620,7 @@ impl GpuManager::ver { }))?; let x = UniqueArc::pin_init(try_pin_init!(GpuManager::ver { - dev: dev.clone(), + dev: dev.into(), cfg, dyncfg: *dyncfg, initdata: *initdata, diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index a3e9cb633221c0..396683f34a08dc 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -181,7 +181,7 @@ const PAGETABLES_SIZE: usize = UAT_PGSZ; /// Inner data for a Vm instance. This is reference-counted by the outer Vm object. struct VmInner { - dev: driver::AsahiDevice, + dev: driver::AsahiDevRef, is_kernel: bool, min_va: usize, max_va: usize, @@ -618,7 +618,7 @@ impl UatInner { /// Top-level UAT manager object pub(crate) struct Uat { - dev: driver::AsahiDevice, + dev: driver::AsahiDevRef, cfg: &'static hw::HwConfig, pagetables_rgn: UatRegion, @@ -765,7 +765,7 @@ impl io_pgtable::FlushOps for Uat { impl Vm { /// Create a new virtual memory address space fn new( - dev: driver::AsahiDevice, + dev: &driver::AsahiDevice, uat_inner: Arc, cfg: &'static hw::HwConfig, is_kernel: bool, @@ -773,7 +773,7 @@ impl Vm { file_id: u64, ) -> Result { let page_table = AppleUAT::new( - &dev, + dev, io_pgtable::Config { pgsize_bitmap: UAT_PGSZ, ias: if is_kernel { UAT_IAS_KERN } else { UAT_IAS }, @@ -807,7 +807,7 @@ impl Vm { file_id, inner: Arc::pin_init(new_mutex!( VmInner { - dev, + dev: dev.into(), min_va, max_va, is_kernel, @@ -1168,14 +1168,7 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, file_id: u64) -> Result { - Vm::new( - self.dev.clone(), - self.inner.clone(), - self.cfg, - false, - id, - file_id, - ) + Vm::new(&self.dev, self.inner.clone(), self.cfg, false, id, file_id) } /// Creates the reference-counted inner data for a new `Uat` instance. @@ -1218,8 +1211,8 @@ impl Uat { let pagetables_rgn = Self::map_region(dev, c_str!("pagetables"), PAGETABLES_SIZE, true)?; dev_info!(dev, "MMU: Creating kernel page tables\n"); - let kernel_lower_vm = Vm::new(dev.clone(), inner.clone(), cfg, false, 1, 0)?; - let kernel_vm = Vm::new(dev.clone(), inner.clone(), cfg, true, 0, 0)?; + let kernel_lower_vm = Vm::new(dev, inner.clone(), cfg, false, 1, 0)?; + let kernel_vm = Vm::new(dev, inner.clone(), cfg, true, 0, 0)?; dev_info!(dev, "MMU: Kernel page tables created\n"); @@ -1227,7 +1220,7 @@ impl Uat { let ttb1 = kernel_vm.ttb(); let uat = Self { - dev: dev.clone(), + dev: dev.into(), cfg, pagetables_rgn, kernel_vm, diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index d5a1a197281c06..8df393fa34c127 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -18,7 +18,7 @@ use kernel::{ use crate::alloc::Allocator; use crate::debug::*; -use crate::driver::AsahiDevice; +use crate::driver::{AsahiDevRef, AsahiDevice}; use crate::fw::types::*; use crate::gpu::GpuManager; use crate::inner_weak_ptr; @@ -94,7 +94,7 @@ impl SubQueueJob::ver { #[versions(AGX)] pub(crate) struct Queue { - dev: AsahiDevice, + dev: AsahiDevRef, _sched: sched::Scheduler, entity: sched::Entity, vm: mmu::Vm, @@ -156,7 +156,7 @@ impl dma_fence::FenceOps for JobFence::ver { #[versions(AGX)] pub(crate) struct QueueJob { - dev: AsahiDevice, + dev: AsahiDevRef, vm_bind: mmu::VmBind, op_guard: Option, sj_vtx: Option, @@ -407,7 +407,7 @@ impl Queue::ver { }; let mut ret = Queue::ver { - dev: dev.clone(), + dev: dev.into(), _sched: sched, entity, vm, diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index 38ebe9deffd11d..b8b220da54650e 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -101,7 +101,7 @@ impl From for kernel::error::Error { /// A GPU context tracking structure, which must be explicitly invalidated when dropped. pub(crate) struct GpuContext { - dev: driver::AsahiDevice, + dev: driver::AsahiDevRef, data: Option>>, } no_debug!(GpuContext); @@ -114,7 +114,7 @@ impl GpuContext { buffer: Option>, ) -> Result { Ok(GpuContext { - dev: dev.clone(), + dev: dev.into(), data: Some(Box::try_new(alloc.shared.new_object( fw::workqueue::GpuContextData { _buffer: buffer }, |_inner| Default::default(), @@ -196,7 +196,7 @@ impl) + Send + Sync> Gen /// Inner data for managing a single work queue. #[versions(AGX)] struct WorkQueueInner { - dev: driver::AsahiDevice, + dev: driver::AsahiDevRef, event_manager: Arc, info: GpuObject, new: bool, @@ -564,7 +564,7 @@ impl WorkQueue::ver { let gpu_buf = alloc.private.array_empty(0x2c18)?; let shared = &mut alloc.shared; let inner = WorkQueueInner::ver { - dev: dev.clone(), + dev: dev.into(), event_manager, info: alloc.private.new_init( try_init!(QueueInfo::ver { From 2ec11c5a8db973e4af6529ebe4e299e52668d729 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 22:23:07 +0900 Subject: [PATCH 594/635] rust: kernel: str: Implement Debug for CString Signed-off-by: Asahi Lina --- rust/kernel/str.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index cd3d2a6cf1fc12..c7bfdcb0438df4 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -584,6 +584,12 @@ impl Deref for CString { } } +impl fmt::Debug for CString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + /// A convenience alias for [`core::format_args`]. #[macro_export] macro_rules! fmt { From 285b20e41ab95ba7bbd0395832b5084046e59b86 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 22:30:51 +0900 Subject: [PATCH 595/635] rust: types: Add Opaque::zeroed() Signed-off-by: Asahi Lina --- rust/kernel/types.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index c12a3bf1803bd6..64fe4f906faf70 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -237,6 +237,11 @@ impl Opaque { Self(MaybeUninit::uninit()) } + /// Creates a zeroed value. + pub fn zeroed() -> Self { + Self(MaybeUninit::zeroed()) + } + /// Creates a pin-initializer from the given initializer closure. /// /// The returned initializer calls the given closure with the pointer to the inner `T` of this From 2d25cd2e568844facaa756f525f70fc01c885342 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:13:42 +0900 Subject: [PATCH 596/635] rust: Use absolute paths to build Rust objects We want to use caller_location to uniquely identify callsites, to automatically create lockdep classes without macros. The location filename in local code uses the relative path passed to the compiler, but if that code is generic and instantiated from another crate, the path becomes absolute. To make this work and keep the paths consistent, always pass an absolute path to the compiler. Then the Location path is always identical regardless of how the code is being compiled. Signed-off-by: Asahi Lina --- rust/Makefile | 2 +- scripts/Makefile.build | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/Makefile b/rust/Makefile index be34d7653a554c..921c3147142e48 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -370,7 +370,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L --emit=dep-info=$(depfile) --emit=obj=$@ \ --emit=metadata=$(dir $@)$(patsubst %.o,lib%.rmeta,$(notdir $@)) \ --crate-type rlib -L$(objtree)/$(obj) \ - --crate-name $(patsubst %.o,%,$(notdir $@)) $< \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $(abspath $<) \ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) rust-analyzer: diff --git a/scripts/Makefile.build b/scripts/Makefile.build index ae0038a4f9b246..1871aa1941b4ed 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -298,27 +298,27 @@ rust_common_cmd = \ # would not match each other. quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $< + cmd_rustc_o_rs = $(rust_common_cmd) --emit=obj=$@ $(abspath $<) $(obj)/%.o: $(src)/%.rs FORCE $(call if_changed_dep,rustc_o_rs) quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ cmd_rustc_rsi_rs = \ - $(rust_common_cmd) -Zunpretty=expanded $< >$@; \ + $(rust_common_cmd) -Zunpretty=expanded $(abspath $<) >$@; \ command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@ $(obj)/%.rsi: $(src)/%.rs FORCE $(call if_changed_dep,rustc_rsi_rs) quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_s_rs = $(rust_common_cmd) --emit=asm=$@ $< + cmd_rustc_s_rs = $(rust_common_cmd) --emit=asm=$@ $(abspath $<) $(obj)/%.s: $(src)/%.rs FORCE $(call if_changed_dep,rustc_s_rs) quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ - cmd_rustc_ll_rs = $(rust_common_cmd) --emit=llvm-ir=$@ $< + cmd_rustc_ll_rs = $(rust_common_cmd) --emit=llvm-ir=$@ $(abspath $<) $(obj)/%.ll: $(src)/%.rs FORCE $(call if_changed_dep,rustc_ll_rs) From 824fadc7ebcacd39384e6f128ddf5675cd83d104 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:17:33 +0900 Subject: [PATCH 597/635] rust: kernel: Add simple siphash abstraction This allows Rust code to use the Hasher interface with the kernel siphash implementation. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 8 +++++++ rust/kernel/lib.rs | 1 + rust/kernel/siphash.rs | 39 +++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 rust/kernel/siphash.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index a752ca815b7e63..49a0d9b6550c03 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 3b128be210829c..77b9b870d6c808 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,13 @@ const char *rust_helper_errname(int err) } EXPORT_SYMBOL_GPL(rust_helper_errname); +u64 rust_helper_siphash(const void *data, size_t len, + const siphash_key_t *key) +{ + return siphash(data, len, key); +} +EXPORT_SYMBOL_GPL(rust_helper_siphash); + void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags) { xa_init_flags(xa, flags); diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 923d11dc5518c2..9aebff7b6b4e1d 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -57,6 +57,7 @@ pub mod of; pub mod platform; pub mod prelude; pub mod print; +pub mod siphash; pub mod soc; mod static_assert; #[doc(hidden)] diff --git a/rust/kernel/siphash.rs b/rust/kernel/siphash.rs new file mode 100644 index 00000000000000..e13a17cd5a9327 --- /dev/null +++ b/rust/kernel/siphash.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A core::hash::Hasher wrapper for the kernel siphash implementation. +//! +//! This module allows Rust code to use the kernel's siphash implementation +//! to hash Rust objects. + +use core::hash::Hasher; + +/// A Hasher implementation that uses the kernel siphash implementation. +#[derive(Default)] +pub struct SipHasher { + // SipHash state is 4xu64, but the Linux implementation + // doesn't expose incremental hashing so let's just chain + // individual SipHash calls for now, which return a u64 + // hash. + state: u64, +} + +impl SipHasher { + /// Create a new SipHasher with zeroed state. + pub fn new() -> Self { + SipHasher { state: 0 } + } +} + +impl Hasher for SipHasher { + fn finish(&self) -> u64 { + self.state + } + + fn write(&mut self, bytes: &[u8]) { + let key = bindings::siphash_key_t { + key: [self.state, 0], + }; + + self.state = unsafe { bindings::siphash(bytes.as_ptr() as *const _, bytes.len(), &key) }; + } +} From f27fbe68f226ea33ea0c9d0ae0bdae6b506ce534 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 18:40:11 +0900 Subject: [PATCH 598/635] rust: sync: Add dummy LockClassKey implementation for !CONFIG_LOCKDEP Lock classes aren't used without lockdep. The C side declares the key as an empty struct in that case, but let's make it an explicit ZST in Rust, implemented in a separate module. This allows us to more easily guarantee that the lockdep code will be trivially optimized out without CONFIG_LOCKDEP, including LockClassKey arguments that are passed around. Depending on whether CONFIG_LOCKDEP is enabled or not, we then import the real lockdep implementation or the dummy one. Signed-off-by: Asahi Lina --- rust/kernel/sync.rs | 29 ++++++++--------------------- rust/kernel/sync/lockdep.rs | 28 ++++++++++++++++++++++++++++ rust/kernel/sync/no_lockdep.rs | 19 +++++++++++++++++++ 3 files changed, 55 insertions(+), 21 deletions(-) create mode 100644 rust/kernel/sync/lockdep.rs create mode 100644 rust/kernel/sync/no_lockdep.rs diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index d219ee518eff15..352472c6b77a2d 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -5,37 +5,24 @@ //! This module contains the kernel APIs related to synchronisation that have been ported or //! wrapped for usage by Rust code in the kernel. -use crate::types::Opaque; - mod arc; mod condvar; pub mod lock; mod locked_by; +#[cfg(CONFIG_LOCKDEP)] +mod lockdep; +#[cfg(not(CONFIG_LOCKDEP))] +mod no_lockdep; +#[cfg(not(CONFIG_LOCKDEP))] +use no_lockdep as lockdep; + pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use condvar::CondVar; pub use lock::{mutex::Mutex, spinlock::SpinLock}; +pub use lockdep::LockClassKey; pub use locked_by::LockedBy; -/// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. -#[repr(transparent)] -pub struct LockClassKey(Opaque); - -// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and -// provides its own synchronization. -unsafe impl Sync for LockClassKey {} - -impl LockClassKey { - /// Creates a new lock class key. - pub const fn new() -> Self { - Self(Opaque::uninit()) - } - - pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { - self.0.get() - } -} - /// Defines a new static lock class and returns a pointer to it. #[doc(hidden)] #[macro_export] diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs new file mode 100644 index 00000000000000..d8cc24a2572e4b --- /dev/null +++ b/rust/kernel/sync/lockdep.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Lockdep utilities. +//! +//! This module abstracts the parts of the kernel lockdep API relevant to Rust +//! modules, including lock classes. + +use crate::types::Opaque; + +/// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. +#[repr(transparent)] +pub struct LockClassKey(Opaque); + +impl LockClassKey { + /// Creates a new lock class key. + pub const fn new() -> Self { + Self(Opaque::uninit()) + } + + pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { + self.0.get() + } +} + +// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and +// provides its own synchronization. +unsafe impl Sync for LockClassKey {} + diff --git a/rust/kernel/sync/no_lockdep.rs b/rust/kernel/sync/no_lockdep.rs new file mode 100644 index 00000000000000..69d42e8d980123 --- /dev/null +++ b/rust/kernel/sync/no_lockdep.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Dummy lockdep utilities. +//! +//! Takes the place of the `lockdep` module when lockdep is disabled. + +/// A dummy, zero-sized lock class. +pub struct LockClassKey(); + +impl LockClassKey { + /// Creates a new dummy lock class key. + pub const fn new() -> Self { + Self() + } + + pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { + core::ptr::null_mut() + } +} From bb12cfa3c13dcbfd1e0e4fe05982e5f5005914dd Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 19:38:57 +0900 Subject: [PATCH 599/635] rust: sync: Replace static LockClassKey refs with a pointer wrapper We want to be able to handle dynamic lock class creation and using pointers to things that aren't a real lock_class_key as lock classes. Doing this by casting around Rust references is difficult without accidentally invoking UB. Instead, switch LockClassKey to being a raw pointer wrapper around a lock_class_key, which means there is no UB possible on the Rust side just by creating and consuming these objects. The C code also should never actually dereference lock classes, only use their address (possibly with an offset). We still provide a dummy ZST version of this wrapper, to be used when lockdep is disabled. Signed-off-by: Asahi Lina --- rust/kernel/sync.rs | 6 +++--- rust/kernel/sync/condvar.rs | 2 +- rust/kernel/sync/lock.rs | 4 ++-- rust/kernel/sync/lockdep.rs | 28 ++++++++++++++++++++++------ rust/kernel/sync/no_lockdep.rs | 15 +++++++++++++-- rust/kernel/types.rs | 2 +- 6 files changed, 42 insertions(+), 15 deletions(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 352472c6b77a2d..49286c3e0ff328 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -20,7 +20,7 @@ use no_lockdep as lockdep; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use condvar::CondVar; pub use lock::{mutex::Mutex, spinlock::SpinLock}; -pub use lockdep::LockClassKey; +pub use lockdep::{LockClassKey, StaticLockClassKey}; pub use locked_by::LockedBy; /// Defines a new static lock class and returns a pointer to it. @@ -28,8 +28,8 @@ pub use locked_by::LockedBy; #[macro_export] macro_rules! static_lock_class { () => {{ - static CLASS: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new(); - &CLASS + static CLASS: $crate::sync::StaticLockClassKey = $crate::sync::StaticLockClassKey::new(); + CLASS.key() }}; } diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index ed353399c4e56f..3bccb2c6ef84be 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -92,7 +92,7 @@ unsafe impl Sync for CondVar {} impl CondVar { /// Constructs a new condvar initialiser. #[allow(clippy::new_ret_no_self)] - pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit { + pub fn new(name: &'static CStr, key: LockClassKey) -> impl PinInit { pin_init!(Self { _pin: PhantomPinned, // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index d493c5d191048b..8e71e7aa2cc1b6 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -103,7 +103,7 @@ unsafe impl Sync for Lock {} impl Lock { /// Constructs a new lock initialiser. #[allow(clippy::new_ret_no_self)] - pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit { + pub fn new(t: T, name: &'static CStr, key: LockClassKey) -> impl PinInit { pin_init!(Self { data: UnsafeCell::new(t), _pin: PhantomPinned, @@ -119,7 +119,7 @@ impl Lock { pub fn pin_init( t: impl PinInit, name: &'static CStr, - key: &'static LockClassKey, + key: LockClassKey, ) -> impl PinInit where E: core::convert::From, diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs index d8cc24a2572e4b..d8328f4275fb7b 100644 --- a/rust/kernel/sync/lockdep.rs +++ b/rust/kernel/sync/lockdep.rs @@ -9,20 +9,36 @@ use crate::types::Opaque; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] -pub struct LockClassKey(Opaque); +pub struct StaticLockClassKey(Opaque); -impl LockClassKey { +impl StaticLockClassKey { /// Creates a new lock class key. pub const fn new() -> Self { Self(Opaque::uninit()) } + /// Returns the lock class key reference for this static lock class. + pub const fn key(&self) -> LockClassKey { + LockClassKey(self.0.get()) + } +} + +// SAFETY: `bindings::lock_class_key` just represents an opaque memory location, and is never +// actually dereferenced. +unsafe impl Sync for StaticLockClassKey {} + +/// A reference to a lock class key. This is a raw pointer to a lock_class_key, +/// which is required to have a static lifetime. +#[derive(Copy, Clone)] +pub struct LockClassKey(*mut bindings::lock_class_key); + +impl LockClassKey { pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { - self.0.get() + self.0 } } -// SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and -// provides its own synchronization. +// SAFETY: `bindings::lock_class_key` just represents an opaque memory location, and is never +// actually dereferenced. +unsafe impl Send for LockClassKey {} unsafe impl Sync for LockClassKey {} - diff --git a/rust/kernel/sync/no_lockdep.rs b/rust/kernel/sync/no_lockdep.rs index 69d42e8d980123..518ec0bf9a7d1e 100644 --- a/rust/kernel/sync/no_lockdep.rs +++ b/rust/kernel/sync/no_lockdep.rs @@ -5,14 +5,25 @@ //! Takes the place of the `lockdep` module when lockdep is disabled. /// A dummy, zero-sized lock class. -pub struct LockClassKey(); +pub struct StaticLockClassKey(); -impl LockClassKey { +impl StaticLockClassKey { /// Creates a new dummy lock class key. pub const fn new() -> Self { Self() } + /// Returns the lock class key reference for this static lock class. + pub const fn key(&self) -> LockClassKey { + LockClassKey() + } +} + +/// A dummy reference to a lock class key. +#[derive(Copy, Clone)] +pub struct LockClassKey(); + +impl LockClassKey { pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { core::ptr::null_mut() } diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 64fe4f906faf70..1d6b8a165249bc 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -262,7 +262,7 @@ impl Opaque { } /// Returns a raw pointer to the opaque data. - pub fn get(&self) -> *mut T { + pub const fn get(&self) -> *mut T { UnsafeCell::raw_get(self.0.as_ptr()) } From 2f619b8c3fa3a3d1e85ea07eabef4c3c5cd83dfb Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 19:43:22 +0900 Subject: [PATCH 600/635] fixup! *RFL import: The rest of kernel::device (minus clk stuff) --- rust/kernel/device.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 14c551df0a41b2..d4712607282e9a 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -254,12 +254,12 @@ pub struct Data { #[macro_export] macro_rules! new_device_data { ($reg:expr, $res:expr, $gen:expr, $name:literal) => {{ - static CLASS1: $crate::sync::LockClassKey = $crate::sync::LockClassKey::new(); + static CLASS1: $crate::sync::LockClassKey = $crate::static_lock_class!(); let regs = $reg; let res = $res; let gen = $gen; let name = $crate::c_str!($name); - $crate::device::Data::try_new(regs, res, gen, name, &CLASS1) + $crate::device::Data::try_new(regs, res, gen, name, CLASS1) }}; } @@ -273,7 +273,7 @@ impl Data { resources: U, general: V, name: &'static CStr, - key1: &'static LockClassKey, + key1: LockClassKey, ) -> Result>> { let ret = UniqueArc::pin_init(pin_init!(Self { registrations <- Mutex::new(registrations, name, key1), From b62af476f010445c9847788e09dca53c2c65653a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 19:43:22 +0900 Subject: [PATCH 601/635] fixup! rust: dma_fence: Add DMA Fence abstraction --- rust/kernel/dma_fence.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rust/kernel/dma_fence.rs b/rust/kernel/dma_fence.rs index b67a67ed2d8a1f..0e488bc9bc7987 100644 --- a/rust/kernel/dma_fence.rs +++ b/rust/kernel/dma_fence.rs @@ -422,16 +422,12 @@ pub struct FenceContexts { count: u32, seqnos: Vec, lock_name: &'static CStr, - lock_key: &'static LockClassKey, + lock_key: LockClassKey, } impl FenceContexts { /// Create a new set of fence contexts. - pub fn new( - count: u32, - name: &'static CStr, - key: &'static LockClassKey, - ) -> Result { + pub fn new(count: u32, name: &'static CStr, key: LockClassKey) -> Result { let mut seqnos: Vec = Vec::new(); seqnos.try_reserve(count as usize)?; From d688e9274c22d45e2417abd1708980fba2fe7201 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 19:43:22 +0900 Subject: [PATCH 602/635] fixup! rust: drm: mm: Add DRM MM Range Allocator abstraction --- rust/kernel/drm/mm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs index 97e948becd6e1e..02d02c561fe0f5 100644 --- a/rust/kernel/drm/mm.rs +++ b/rust/kernel/drm/mm.rs @@ -169,7 +169,7 @@ impl, T> Allocator { size: u64, inner: A, name: &'static CStr, - lock_key: &'static LockClassKey, + lock_key: LockClassKey, ) -> Result> { // SAFETY: We call `Mutex::init_lock` below. let mm = UniqueArc::pin_init(Mutex::new( From e6b43d7cd58a36235dd18f70b6dd701012caa88a Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 19:43:40 +0900 Subject: [PATCH 603/635] drm/asahi: Lock class fixups Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 4 ++-- drivers/gpu/drm/asahi/queue/mod.rs | 4 ++-- drivers/gpu/drm/asahi/slotalloc.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index 3313676ae6a14f..ec82f69153543d 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -692,7 +692,7 @@ pub(crate) struct HeapAllocator { name: CString, } -static LOCK_KEY: LockClassKey = LockClassKey::new(); +static LOCK_KEY: LockClassKey = kernel::static_lock_class!(); impl HeapAllocator { /// Create a new HeapAllocator for a given `Vm` and address range. @@ -735,7 +735,7 @@ impl HeapAllocator { end - start + 1, inner, c_str!("HeapAllocator"), - &LOCK_KEY, + LOCK_KEY, )?; Ok(HeapAllocator { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 8df393fa34c127..21335e4c6c3868 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -341,7 +341,7 @@ impl ResultWriter { } static QUEUE_NAME: &CStr = c_str!("asahi_fence"); -static QUEUE_CLASS_KEY: kernel::sync::LockClassKey = kernel::sync::LockClassKey::new(); +static QUEUE_CLASS_KEY: kernel::sync::LockClassKey = kernel::static_lock_class!(); #[versions(AGX)] impl Queue::ver { @@ -424,7 +424,7 @@ impl Queue::ver { notifier_list: Arc::try_new(notifier_list)?, notifier, id, - fence_ctx: FenceContexts::new(1, QUEUE_NAME, &QUEUE_CLASS_KEY)?, + fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), }; diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index bd2da42245cbc4..e2897650c97e01 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -132,8 +132,8 @@ impl SlotAllocator { mut data: T::Data, mut constructor: impl FnMut(&mut T::Data, u32) -> T, name: &'static CStr, - lock_key1: &'static LockClassKey, - lock_key2: &'static LockClassKey, + lock_key1: LockClassKey, + lock_key2: LockClassKey, ) -> Result> { let mut slots = Vec::try_with_capacity(num_slots as usize)?; From f74738f5caec269b5139b89350f465e56f32424d Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:21:07 +0900 Subject: [PATCH 604/635] rust: sync: Implement dynamic lockdep class creation Using macros to create lock classes all over the place is unergonomic, and makes it impossible to add new features that require lock classes to code such as Arc<> without changing all callers. Rust has the ability to track the caller's identity by file/line/column number, and we can use that to dynamically generate lock classes instead. Signed-off-by: Asahi Lina --- rust/kernel/sync/lockdep.rs | 147 ++++++++++++++++++++++++++++++++- rust/kernel/sync/no_lockdep.rs | 8 ++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs index d8328f4275fb7b..fbf9f6ed403dda 100644 --- a/rust/kernel/sync/lockdep.rs +++ b/rust/kernel/sync/lockdep.rs @@ -5,7 +5,19 @@ //! This module abstracts the parts of the kernel lockdep API relevant to Rust //! modules, including lock classes. -use crate::types::Opaque; +use crate::{ + c_str, fmt, + init::InPlaceInit, + new_mutex, + prelude::{Box, Result, Vec}, + str::{CStr, CString}, + sync::Mutex, + types::Opaque, +}; + +use core::hash::{Hash, Hasher}; +use core::pin::Pin; +use core::sync::atomic::{AtomicPtr, Ordering}; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. #[repr(transparent)] @@ -42,3 +54,136 @@ impl LockClassKey { // actually dereferenced. unsafe impl Send for LockClassKey {} unsafe impl Sync for LockClassKey {} + +// Location is 'static but not really, since module unloads will +// invalidate existing static Locations within that module. +// To avoid breakage, we maintain our own location struct which is +// dynamically allocated on first reference. We store a hash of the +// whole location (including the filename string), as well as the +// line and column separately. The assumption is that this whole +// struct is highly unlikely to ever collide with a reasonable +// hash (this saves us from having to check the filename string +// itself). +#[derive(PartialEq, Debug)] +struct LocationKey { + hash: u64, + line: u32, + column: u32, +} + +struct DynLockClassKey { + key: Opaque, + loc: LocationKey, + name: CString, +} + +impl LocationKey { + fn new(loc: &'static core::panic::Location<'static>) -> Self { + let mut hasher = crate::siphash::SipHasher::new(); + loc.hash(&mut hasher); + + LocationKey { + hash: hasher.finish(), + line: loc.line(), + column: loc.column(), + } + } +} + +impl DynLockClassKey { + fn key(&'static self) -> LockClassKey { + LockClassKey(self.key.get()) + } + + fn name(&'static self) -> &CStr { + &self.name + } +} + +const LOCK_CLASS_BUCKETS: usize = 1024; + +#[track_caller] +fn caller_lock_class_inner() -> Result<&'static DynLockClassKey> { + // This is just a hack to make the below static array initialization work. + #[allow(clippy::declare_interior_mutable_const)] + const ATOMIC_PTR: AtomicPtr>> = + AtomicPtr::new(core::ptr::null_mut()); + + #[allow(clippy::complexity)] + static LOCK_CLASSES: [AtomicPtr>>; LOCK_CLASS_BUCKETS] = + [ATOMIC_PTR; LOCK_CLASS_BUCKETS]; + + let loc = core::panic::Location::caller(); + let loc_key = LocationKey::new(loc); + + let index = (loc_key.hash % (LOCK_CLASS_BUCKETS as u64)) as usize; + let slot = &LOCK_CLASSES[index]; + + let mut ptr = slot.load(Ordering::Relaxed); + if ptr.is_null() { + let new_element = Box::pin_init(new_mutex!(Vec::new()))?; + + if let Err(e) = slot.compare_exchange( + core::ptr::null_mut(), + // SAFETY: We never move out of this Box + Box::into_raw(unsafe { Pin::into_inner_unchecked(new_element) }), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + // SAFETY: We just got this pointer from `into_raw()` + unsafe { Box::from_raw(e) }; + } + + ptr = slot.load(Ordering::Relaxed); + assert!(!ptr.is_null()); + } + + // SAFETY: This mutex was either just created above or previously allocated, + // and we never free these objects so the pointer is guaranteed to be valid. + let mut guard = unsafe { (*ptr).lock() }; + + for i in guard.iter() { + if i.loc == loc_key { + return Ok(i); + } + } + + // We immediately leak the class, so it becomes 'static + let new_class = Box::leak(Box::try_new(DynLockClassKey { + key: Opaque::zeroed(), + loc: loc_key, + name: CString::try_from_fmt(fmt!("{}:{}:{}", loc.file(), loc.line(), loc.column()))?, + })?); + + // SAFETY: This is safe to call with a pointer to a dynamically allocated lockdep key, + // and we never free the objects so it is safe to never unregister the key. + unsafe { bindings::lockdep_register_key(new_class.key.get()) }; + + guard.try_push(new_class)?; + + Ok(new_class) +} + +#[track_caller] +pub(crate) fn caller_lock_class() -> (LockClassKey, &'static CStr) { + match caller_lock_class_inner() { + Ok(a) => (a.key(), a.name()), + Err(_) => { + crate::pr_err!( + "Failed to dynamically allocate lock class, lockdep may be unreliable.\n" + ); + + let loc = core::panic::Location::caller(); + // SAFETY: LockClassKey is opaque and the lockdep implementation only needs + // unique addresses for statically allocated keys, so it is safe to just cast + // the Location reference directly into a LockClassKey. However, this will + // result in multiple keys for the same callsite due to monomorphization, + // as well as spuriously destroyed keys when the static key is allocated in + // the wrong module, which is what makes this unreliable. + ( + LockClassKey(loc as *const _ as *mut _), + c_str!("fallback_lock_class"), + ) + } + } +} diff --git a/rust/kernel/sync/no_lockdep.rs b/rust/kernel/sync/no_lockdep.rs index 518ec0bf9a7d1e..de53c4de7fbe53 100644 --- a/rust/kernel/sync/no_lockdep.rs +++ b/rust/kernel/sync/no_lockdep.rs @@ -4,6 +4,8 @@ //! //! Takes the place of the `lockdep` module when lockdep is disabled. +use crate::{c_str, str::CStr}; + /// A dummy, zero-sized lock class. pub struct StaticLockClassKey(); @@ -28,3 +30,9 @@ impl LockClassKey { core::ptr::null_mut() } } + +pub(crate) fn caller_lock_class() -> (LockClassKey, &'static CStr) { + static DUMMY_LOCK_CLASS: StaticLockClassKey = StaticLockClassKey::new(); + + (DUMMY_LOCK_CLASS.key(), c_str!("dummy")) +} From 59007053c4cf1d3d2f41c91f2489ab5a16a3ba47 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:23:02 +0900 Subject: [PATCH 605/635] rust: sync: Classless Lock::new() and pin_init() Use the new automagic lock class code to remove the lock class and name parameters from Lock::new() and Lock::pin_init(). The old functions are renamed to new_with_class() and pin_init_with_class() respectively. Signed-off-by: Asahi Lina --- rust/kernel/sync/lock.rs | 42 ++++++++++++++++++++++++++++--- rust/kernel/sync/lock/mutex.rs | 4 +-- rust/kernel/sync/lock/spinlock.rs | 2 +- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 8e71e7aa2cc1b6..8849741c1d9ac8 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -5,7 +5,7 @@ //! It contains a generic Rust lock and guard that allow for different backends (e.g., mutexes, //! spinlocks, raw spinlocks) to be provided with minimal effort. -use super::LockClassKey; +use super::{lockdep::caller_lock_class, LockClassKey}; use crate::{ bindings, init::PinInit, pin_init, str::CStr, try_pin_init, types::Opaque, types::ScopeGuard, }; @@ -103,7 +103,40 @@ unsafe impl Sync for Lock {} impl Lock { /// Constructs a new lock initialiser. #[allow(clippy::new_ret_no_self)] - pub fn new(t: T, name: &'static CStr, key: LockClassKey) -> impl PinInit { + #[track_caller] + pub fn new(t: T) -> impl PinInit { + let (key, name) = caller_lock_class(); + Self::new_with_key(t, name, key) + } + + /// Constructs a new lock initialiser taking an initialiser/ + pub fn pin_init(t: impl PinInit) -> impl PinInit + where + E: core::convert::From, + { + let (key, name) = caller_lock_class(); + Self::pin_init_with_key(t, name, key) + } + + /// Constructs a new lock initialiser. + #[allow(clippy::new_ret_no_self)] + #[track_caller] + pub fn new_named(t: T, name: &'static CStr) -> impl PinInit { + let (key, _) = caller_lock_class(); + Self::new_with_key(t, name, key) + } + + /// Constructs a new lock initialiser taking an initialiser/ + pub fn pin_init_named(t: impl PinInit, name: &'static CStr) -> impl PinInit + where + E: core::convert::From, + { + let (key, _) = caller_lock_class(); + Self::pin_init_with_key(t, name, key) + } + + /// Constructs a new lock initialiser given a particular name and lock class key. + pub fn new_with_key(t: T, name: &'static CStr, key: LockClassKey) -> impl PinInit { pin_init!(Self { data: UnsafeCell::new(t), _pin: PhantomPinned, @@ -115,8 +148,9 @@ impl Lock { }) } - /// Constructs a new lock initialiser taking an initialiser. - pub fn pin_init( + /// Constructs a new lock initialiser taking an initialiser given a particular + /// name and lock class key. + pub fn pin_init_with_key( t: impl PinInit, name: &'static CStr, key: LockClassKey, diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 06fe685501b4e3..15ea70fa39334c 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -13,7 +13,7 @@ use crate::bindings; #[macro_export] macro_rules! new_mutex { ($inner:expr $(, $name:literal)? $(,)?) => { - $crate::sync::Mutex::new( + $crate::sync::Mutex::new_with_key( $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } @@ -26,7 +26,7 @@ macro_rules! new_mutex { #[macro_export] macro_rules! new_mutex_pinned { ($inner:expr $(, $name:literal)? $(,)?) => { - $crate::sync::Mutex::pin_init( + $crate::sync::Mutex::pin_init_with_key( $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 979b56464a4e9f..9f6137f047ee72 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -13,7 +13,7 @@ use crate::bindings; #[macro_export] macro_rules! new_spinlock { ($inner:expr $(, $name:literal)? $(,)?) => { - $crate::sync::SpinLock::new( + $crate::sync::SpinLock::new_with_class( $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } From b01b04ef5532af703d26b90f15f11af6e38adc7c Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:34:22 +0900 Subject: [PATCH 606/635] rust: init: Update documentation for new mutex init style Signed-off-by: Asahi Lina --- rust/kernel/init.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 35d8b25c99caa4..2b7fa770efa2cb 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -36,7 +36,7 @@ //! //! ```rust //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -//! use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! use kernel::{prelude::*, sync::Mutex}; //! # use core::pin::Pin; //! #[pin_data] //! struct Foo { @@ -46,7 +46,7 @@ //! } //! //! let foo = pin_init!(Foo { -//! a <- new_mutex!(42, "Foo::a"), +//! a <- Mutex::new_named(42, "Foo::a"), //! b: 24, //! }); //! ``` @@ -56,7 +56,7 @@ //! //! ```rust //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -//! # use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! # use kernel::{prelude::*, sync::Mutex}; //! # use core::pin::Pin; //! # #[pin_data] //! # struct Foo { @@ -65,7 +65,7 @@ //! # b: u32, //! # } //! # let foo = pin_init!(Foo { -//! # a <- new_mutex!(42, "Foo::a"), +//! # a <- Mutex::new_named(42, "Foo::a"), //! # b: 24, //! # }); //! let foo: Result>> = Box::pin_init(foo); @@ -98,7 +98,7 @@ //! impl DriverData { //! fn new() -> impl PinInit { //! try_pin_init!(Self { -//! status <- new_mutex!(0, "DriverData::status"), +//! status <- Mutex::new_named(0, "DriverData::status"), //! buffer: Box::init(kernel::init::zeroed())?, //! }) //! } @@ -240,7 +240,7 @@ pub mod macros; /// } /// /// stack_pin_init!(let foo = pin_init!(Foo { -/// a <- new_mutex!(42), +/// a <- Mutex::new(42), /// b: Bar { /// x: 64, /// }, @@ -290,7 +290,7 @@ macro_rules! stack_pin_init { /// } /// /// stack_try_pin_init!(let foo: Result, AllocError> = pin_init!(Foo { -/// a <- new_mutex!(42), +/// a <- Mutex::new(42), /// b: Box::try_new(Bar { /// x: 64, /// })?, @@ -316,7 +316,7 @@ macro_rules! stack_pin_init { /// } /// /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { -/// a <- new_mutex!(42), +/// a <- Mutex::new(42), /// b: Box::try_new(Bar { /// x: 64, /// })?, @@ -1435,7 +1435,7 @@ where /// /// ```rust /// let array: Arc<[Mutex; 1000_000_000]>= -/// Arc::pin_init(init_array_from_fn(|i| new_mutex!(i))).unwrap(); +/// Arc::pin_init(init_array_from_fn(|i| Mutex::new(i))).unwrap(); /// println!("{array:?}"); /// ``` pub fn pin_init_array_from_fn( From 20e8da04fe594a58e694d46f2b6f3d0a23128d73 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:25:20 +0900 Subject: [PATCH 607/635] fixup! *RFL import: The rest of kernel::device (minus clk stuff) --- rust/kernel/device.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index d4712607282e9a..769a4e223bce09 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -276,7 +276,7 @@ impl Data { key1: LockClassKey, ) -> Result>> { let ret = UniqueArc::pin_init(pin_init!(Self { - registrations <- Mutex::new(registrations, name, key1), + registrations <- Mutex::new_with_key(registrations, name, key1), resources, general, }))?; From 0e1433227ec1f85af9262f8617add4cd2ff211c7 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:25:20 +0900 Subject: [PATCH 608/635] fixup! rust: drm: mm: Add DRM MM Range Allocator abstraction --- rust/kernel/drm/mm.rs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs index 02d02c561fe0f5..20afe1b5dc7930 100644 --- a/rust/kernel/drm/mm.rs +++ b/rust/kernel/drm/mm.rs @@ -7,8 +7,7 @@ use crate::{ bindings, error::{to_result, Result}, - str::CStr, - sync::{Arc, LockClassKey, Mutex, UniqueArc}, + sync::{Arc, Mutex, UniqueArc}, types::Opaque, }; @@ -164,19 +163,10 @@ impl, T> Allocator { /// /// The user may optionally provide an inner object representing allocator state, which will /// be protected by the same lock. If not required, `()` can be used. - pub fn new( - start: u64, - size: u64, - inner: A, - name: &'static CStr, - lock_key: LockClassKey, - ) -> Result> { + #[track_caller] + pub fn new(start: u64, size: u64, inner: A) -> Result> { // SAFETY: We call `Mutex::init_lock` below. - let mm = UniqueArc::pin_init(Mutex::new( - MmInner(Opaque::uninit(), inner, PhantomData), - name, - lock_key, - ))?; + let mm = UniqueArc::pin_init(Mutex::new(MmInner(Opaque::uninit(), inner, PhantomData)))?; unsafe { // SAFETY: The Opaque instance provides a valid pointer, and it is initialized after From ff212440b8d395108f699457db929c49bc83d4c8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:25:38 +0900 Subject: [PATCH 609/635] drm/asahi: Classless mutex etc Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/alloc.rs | 12 ++---------- drivers/gpu/drm/asahi/mmu.rs | 8 +------- drivers/gpu/drm/asahi/slotalloc.rs | 2 +- rust/kernel/sync/lock.rs | 3 ++- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs index ec82f69153543d..a6a60da4914616 100644 --- a/drivers/gpu/drm/asahi/alloc.rs +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -12,7 +12,7 @@ //! debugging with a GPU memory snapshot, since it makes it easier to identify use-after-free and //! caching issues. -use kernel::{c_str, drm::mm, error::Result, prelude::*, str::CString, sync::LockClassKey}; +use kernel::{drm::mm, error::Result, prelude::*, str::CString}; use crate::debug::*; use crate::driver::{AsahiDevRef, AsahiDevice}; @@ -692,8 +692,6 @@ pub(crate) struct HeapAllocator { name: CString, } -static LOCK_KEY: LockClassKey = kernel::static_lock_class!(); - impl HeapAllocator { /// Create a new HeapAllocator for a given `Vm` and address range. #[allow(dead_code)] @@ -730,13 +728,7 @@ impl HeapAllocator { total_garbage: 0, }; - let mm = mm::Allocator::new( - start, - end - start + 1, - inner, - c_str!("HeapAllocator"), - LOCK_KEY, - )?; + let mm = mm::Allocator::new(start, end - start + 1, inner)?; Ok(HeapAllocator { dev: dev.into(), diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 396683f34a08dc..379560e83a577d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -794,13 +794,7 @@ impl Vm { IOVA_USER_TOP }; - let mm = mm::Allocator::new( - min_va as u64, - (max_va - min_va + 1) as u64, - (), - c_str!("asahi Vm"), - static_lock_class!(), - )?; + let mm = mm::Allocator::new(min_va as u64, (max_va - min_va + 1) as u64, ())?; Ok(Vm { id, diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index e2897650c97e01..42e1f7089ce6d8 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -156,7 +156,7 @@ impl SlotAllocator { let alloc = Arc::pin_init(pin_init!(SlotAllocatorOuter { // SAFETY: `mutex_init!` is called below. - inner <- Mutex::new(inner, name, lock_key1), + inner <- Mutex::new_with_key(inner, name, lock_key1), // SAFETY: `condvar_init!` is called below. cond <- CondVar::new(name, lock_key2), }))?; diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 8849741c1d9ac8..f6185cb2f86a24 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -110,6 +110,7 @@ impl Lock { } /// Constructs a new lock initialiser taking an initialiser/ + #[track_caller] pub fn pin_init(t: impl PinInit) -> impl PinInit where E: core::convert::From, @@ -119,7 +120,6 @@ impl Lock { } /// Constructs a new lock initialiser. - #[allow(clippy::new_ret_no_self)] #[track_caller] pub fn new_named(t: T, name: &'static CStr) -> impl PinInit { let (key, _) = caller_lock_class(); @@ -127,6 +127,7 @@ impl Lock { } /// Constructs a new lock initialiser taking an initialiser/ + #[track_caller] pub fn pin_init_named(t: impl PinInit, name: &'static CStr) -> impl PinInit where E: core::convert::From, From c430a6c683429f9e47ab662b7e236f26f9642bd8 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 20:35:09 +0900 Subject: [PATCH 610/635] drm/asahi: New Mutex style Signed-off-by: Asahi Lina --- drivers/gpu/drm/asahi/buffer.rs | 4 ++-- drivers/gpu/drm/asahi/file.rs | 8 ++++---- drivers/gpu/drm/asahi/gem.rs | 3 +-- drivers/gpu/drm/asahi/gpu.rs | 28 ++++++++++++++-------------- drivers/gpu/drm/asahi/mmu.rs | 11 +++++------ drivers/gpu/drm/asahi/workqueue.rs | 7 ++++--- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs index cb15ea40141a45..48604faf9aee2e 100644 --- a/drivers/gpu/drm/asahi/buffer.rs +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -40,7 +40,7 @@ use crate::{alloc, fw, gpu, hw, mmu, slotalloc}; use core::sync::atomic::Ordering; use kernel::prelude::*; use kernel::sync::{Arc, Mutex}; -use kernel::{c_str, new_mutex, static_lock_class}; +use kernel::{c_str, static_lock_class}; const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; @@ -394,7 +394,7 @@ impl Buffer::ver { })?; Ok(Buffer::ver { - inner: Arc::pin_init(new_mutex!(BufferInner::ver { + inner: Arc::pin_init(Mutex::new(BufferInner::ver { info, ualloc, ualloc_priv, diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 6ea4d45c9d0176..d1e1ef335ffe95 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -17,7 +17,7 @@ use kernel::io_buffer::{IoBufferReader, IoBufferWriter}; use kernel::prelude::*; use kernel::sync::{Arc, Mutex}; use kernel::user_ptr::UserSlicePtr; -use kernel::{dma_fence, drm, new_mutex, uapi, xarray}; +use kernel::{dma_fence, drm, uapi, xarray}; const DEBUG_CLASS: DebugFlags = DebugFlags::File; @@ -287,7 +287,7 @@ impl File { file_id, id ); - let ualloc = Arc::pin_init(new_mutex!(alloc::DefaultAllocator::new( + let ualloc = Arc::pin_init(Mutex::new(alloc::DefaultAllocator::new( device, &vm, VM_DRV_GPU_START, @@ -299,7 +299,7 @@ impl File { fmt!("File {} VM {} GPU Shared", file_id, id), false, )?))?; - let ualloc_priv = Arc::pin_init(new_mutex!(alloc::DefaultAllocator::new( + let ualloc_priv = Arc::pin_init(Mutex::new(alloc::DefaultAllocator::new( device, &vm, VM_DRV_GPUFW_START, @@ -605,7 +605,7 @@ impl File { .new_queue(vm, ualloc, ualloc_priv, data.priority, data.queue_caps)?; data.queue_id = resv.index().try_into()?; - resv.store(Arc::pin_init(new_mutex!(queue))?)?; + resv.store(Arc::pin_init(Mutex::new(queue))?)?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 649368ba6c41e6..80d480951d980a 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -10,7 +10,6 @@ use kernel::{ drm::{gem, gem::shmem}, error::Result, - new_mutex, prelude::*, soc::apple::rtkit, sync::Mutex, @@ -271,7 +270,7 @@ impl gem::BaseDriverObject for DriverObject { kernel: false, flags: 0, vm_id: None, - mappings <- new_mutex!(Vec::new()), + mappings <- Mutex::new(Vec::new()), id, }) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 77dac5e2b341cd..e1b447d52642ed 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -16,10 +16,10 @@ use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use core::time::Duration; use kernel::{ + c_str, delay::coarse_sleep, error::code::*, macros::versions, - new_mutex, prelude::*, soc::apple::rtkit, sync::{ @@ -589,17 +589,17 @@ impl GpuManager::ver { }; for _i in 0..=NUM_PIPES - 1 { - pipes.vtx.try_push(Box::pin_init(new_mutex!( + pipes.vtx.try_push(Box::pin_init(Mutex::new_named( channel::PipeChannel::ver::new(dev, &mut alloc)?, - "pipe_vtx", + c_str!("pipe_vtx"), ))?)?; - pipes.frag.try_push(Box::pin_init(new_mutex!( + pipes.frag.try_push(Box::pin_init(Mutex::new_named( channel::PipeChannel::ver::new(dev, &mut alloc)?, - "pipe_frag", + c_str!("pipe_frag"), ))?)?; - pipes.comp.try_push(Box::pin_init(new_mutex!( + pipes.comp.try_push(Box::pin_init(Mutex::new_named( channel::PipeChannel::ver::new(dev, &mut alloc)?, - "pipe_comp", + c_str!("pipe_comp"), ))?)?; } @@ -626,18 +626,18 @@ impl GpuManager::ver { initdata: *initdata, uat: *uat, io_mappings: Vec::new(), - rtkit <- new_mutex!(None, "rtkit"), + rtkit <- Mutex::new_named(None, c_str!("rtkit")), crashed: AtomicBool::new(false), event_manager, - alloc <- new_mutex!(alloc, "alloc"), - fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), - rx_channels <- new_mutex!(*rx_channels, "rx_channels"), - tx_channels <- new_mutex!(*tx_channels, "tx_channels"), + alloc <- Mutex::new_named(alloc, c_str!("alloc")), + fwctl_channel <- Mutex::new_named(fwctl_channel, c_str!("fwctl_channel")), + rx_channels <- Mutex::new_named(*rx_channels, c_str!("rx_channels")), + tx_channels <- Mutex::new_named(*tx_channels, c_str!("tx_channels")), pipes, buffer_mgr: buffer::BufferManager::new()?, ids: Default::default(), - garbage_work <- new_mutex!(Vec::new(), "garbage_work"), - garbage_contexts <- new_mutex!(Vec::new(), "garbage_contexts"), + garbage_work <- Mutex::new_named(Vec::new(), c_str!("garbage_work")), + garbage_contexts <- Mutex::new_named(Vec::new(), c_str!("garbage_contexts")), }))?; Ok(x) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 379560e83a577d..3d21e35cabb024 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -22,7 +22,6 @@ use kernel::{ error::{to_result, Result}, io_pgtable, io_pgtable::{prot, AppleUAT, IoPageTable}, - new_mutex, prelude::*, static_lock_class, sync::{ @@ -799,7 +798,7 @@ impl Vm { Ok(Vm { id, file_id, - inner: Arc::pin_init(new_mutex!( + inner: Arc::pin_init(Mutex::new_named( VmInner { dev: dev.into(), min_va, @@ -813,7 +812,7 @@ impl Vm { active_users: 0, id, }, - "VmInner" + c_str!("VmInner"), ))?, }) } @@ -1177,16 +1176,16 @@ impl Uat { Arc::pin_init(try_pin_init!(UatInner { handoff_flush <- init::pin_init_array_from_fn(|i| { - new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + Mutex::new_named(HandoffFlush(&handoff.flush[i]), c_str!("handoff_flush")) }), - shared <- new_mutex!( + shared <- Mutex::new_named( UatShared { kernel_ttb1: 0, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, }, - "uat_shared" + c_str!("uat_shared") ), })) } diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs index b8b220da54650e..7109efbf2e2572 100644 --- a/drivers/gpu/drm/asahi/workqueue.rs +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -24,6 +24,7 @@ use crate::{channel, driver, event, fw, gpu, object, regs}; use core::num::NonZeroU64; use core::sync::atomic::Ordering; use kernel::{ + c_str, error::code::*, prelude::*, sync::{ @@ -632,9 +633,9 @@ impl WorkQueue::ver { let info_pointer = inner.info.weak_pointer(); let mutex_init = match pipe_type { - PipeType::Vertex => kernel::new_mutex!(inner, "WorkQueue::inner (Vertex)"), - PipeType::Fragment => kernel::new_mutex!(inner, "WorkQueue::inner (Fragment)"), - PipeType::Compute => kernel::new_mutex!(inner, "WorkQueue::inner (Compute)"), + PipeType::Vertex => Mutex::new_named(inner, c_str!("WorkQueue::inner (Vertex)")), + PipeType::Fragment => Mutex::new_named(inner, c_str!("WorkQueue::inner (Fragment)")), + PipeType::Compute => Mutex::new_named(inner, c_str!("WorkQueue::inner (Compute)")), }; Arc::pin_init(pin_init!(Self { From 81c0186126b24a841fbc7077840e2fde57bb1697 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 21:21:06 +0900 Subject: [PATCH 611/635] rust: sync: Add LockdepMap abstraction This allows Rust code to explicitly integrate types with lockdep. Signed-off-by: Asahi Lina --- rust/bindings/bindings_helper.h | 1 + rust/helpers.c | 16 +++++++++++++ rust/kernel/sync/lockdep.rs | 40 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 49a0d9b6550c03..ae094a8db77f7f 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers.c b/rust/helpers.c index 77b9b870d6c808..04ebdb69e0af7c 100644 --- a/rust/helpers.c +++ b/rust/helpers.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #include #include #include @@ -159,6 +161,20 @@ u64 rust_helper_siphash(const void *data, size_t len, } EXPORT_SYMBOL_GPL(rust_helper_siphash); +void rust_helper_lock_acquire_ret(struct lockdep_map *lock, unsigned int subclass, + int trylock, int read, int check, + struct lockdep_map *nest_lock) +{ + lock_acquire(lock, subclass, trylock, read, check, nest_lock, _RET_IP_); +} +EXPORT_SYMBOL_GPL(rust_helper_lock_acquire_ret); + +void rust_helper_lock_release_ret(struct lockdep_map *lock) +{ + lock_release(lock, _RET_IP_); +} +EXPORT_SYMBOL_GPL(rust_helper_lock_release_ret); + void rust_helper_xa_init_flags(struct xarray *xa, gfp_t flags) { xa_init_flags(xa, flags); diff --git a/rust/kernel/sync/lockdep.rs b/rust/kernel/sync/lockdep.rs index fbf9f6ed403dda..ca32aa833e1076 100644 --- a/rust/kernel/sync/lockdep.rs +++ b/rust/kernel/sync/lockdep.rs @@ -187,3 +187,43 @@ pub(crate) fn caller_lock_class() -> (LockClassKey, &'static CStr) { } } } + +pub(crate) struct LockdepMap(Opaque); +pub(crate) struct LockdepGuard<'a>(&'a LockdepMap); + +#[allow(dead_code)] +impl LockdepMap { + #[track_caller] + pub(crate) fn new() -> Self { + let map = Opaque::uninit(); + let (key, name) = caller_lock_class(); + + unsafe { + bindings::lockdep_init_map_type( + map.get(), + name.as_char_ptr(), + key.as_ptr(), + 0, + bindings::lockdep_wait_type_LD_WAIT_INV as _, + bindings::lockdep_wait_type_LD_WAIT_INV as _, + bindings::lockdep_lock_type_LD_LOCK_NORMAL as _, + ) + }; + + LockdepMap(map) + } + + #[inline(always)] + pub(crate) fn lock(&self) -> LockdepGuard<'_> { + unsafe { bindings::lock_acquire_ret(self.0.get(), 0, 0, 1, 1, core::ptr::null_mut()) }; + + LockdepGuard(self) + } +} + +impl<'a> Drop for LockdepGuard<'a> { + #[inline(always)] + fn drop(&mut self) { + unsafe { bindings::lock_release_ret(self.0 .0.get()) }; + } +} From 088cf4cc64f4e0eec3c9b99e36c9f5e9a86b1173 Mon Sep 17 00:00:00 2001 From: Asahi Lina Date: Sat, 27 May 2023 21:22:03 +0900 Subject: [PATCH 612/635] rust: sync: arc: Add lockdep integration Signed-off-by: Asahi Lina --- lib/Kconfig.debug | 8 +++++ rust/kernel/init.rs | 6 ++++ rust/kernel/sync/arc.rs | 71 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 39d1d93164bd09..5e5919cfa1cab3 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2880,6 +2880,14 @@ config RUST_BUILD_ASSERT_ALLOW If unsure, say N. +config RUST_EXTRA_LOCKDEP + bool "Extra lockdep checking" + depends on RUST && PROVE_LOCKING + help + Enabled additional lockdep integration with certain Rust types. + + If unsure, say N. + endmenu # "Rust" endmenu # Kernel hacking diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 2b7fa770efa2cb..b2bee8660704e5 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -1489,6 +1489,7 @@ pub trait InPlaceInit: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[track_caller] fn try_pin_init(init: impl PinInit) -> Result, E> where E: From; @@ -1497,6 +1498,7 @@ pub trait InPlaceInit: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. + #[track_caller] fn pin_init(init: impl PinInit) -> error::Result> where Error: From, @@ -1509,11 +1511,13 @@ pub trait InPlaceInit: Sized { } /// Use the given initializer to in-place initialize a `T`. + #[track_caller] fn try_init(init: impl Init) -> Result where E: From; /// Use the given initializer to in-place initialize a `T`. + #[track_caller] fn init(init: impl Init) -> error::Result where Error: From, @@ -1558,6 +1562,7 @@ impl InPlaceInit for Box { impl InPlaceInit for UniqueArc { #[inline] + #[track_caller] fn try_pin_init(init: impl PinInit) -> Result, E> where E: From, @@ -1572,6 +1577,7 @@ impl InPlaceInit for UniqueArc { } #[inline] + #[track_caller] fn try_init(init: impl Init) -> Result where E: From, diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index e6d20624246529..379f742638355a 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -34,6 +34,9 @@ use core::{ }; use macros::pin_data; +#[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] +use crate::sync::lockdep::LockdepMap; + mod std_vendor; /// A reference-counted pointer to an instance of `T`. @@ -127,6 +130,17 @@ pub struct Arc { _p: PhantomData>, } +#[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] +#[pin_data] +#[repr(C)] +struct ArcInner { + refcount: Opaque, + lockdep_map: LockdepMap, + data: T, +} + +// FIXME: pin_data does not work well with cfg attributes within the struct definition. +#[cfg(not(CONFIG_RUST_EXTRA_LOCKDEP))] #[pin_data] #[repr(C)] struct ArcInner { @@ -157,11 +171,14 @@ unsafe impl Sync for Arc {} impl Arc { /// Constructs a new reference counted instance of `T`. + #[track_caller] pub fn try_new(contents: T) -> Result { // INVARIANT: The refcount is initialised to a non-zero value. let value = ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), + #[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] + lockdep_map: LockdepMap::new(), data: contents, }; @@ -176,6 +193,7 @@ impl Arc { /// /// If `T: !Unpin` it will not be able to move afterwards. #[inline] + #[track_caller] pub fn pin_init(init: impl PinInit) -> error::Result where Error: From, @@ -187,6 +205,7 @@ impl Arc { /// /// This is equivalent to [`pin_init`], since an [`Arc`] is always pinned. #[inline] + #[track_caller] pub fn init(init: impl Init) -> error::Result where Error: From, @@ -279,15 +298,46 @@ impl Drop for Arc { // freed/invalid memory as long as it is never dereferenced. let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); + // SAFETY: By the type invariant, there is necessarily a reference to the object. + // We cannot hold the map lock across the reference decrement, as we might race + // another thread. Therefore, we lock and immediately drop the guard here. This + // only serves to inform lockdep of the dependency up the call stack. + #[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] + unsafe { self.ptr.as_ref() }.lockdep_map.lock(); + // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and // this instance is being dropped, so the broken invariant is not observable. // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; + if is_zero { // The count reached zero, we must free the memory. - // - // SAFETY: The pointer was initialised from the result of `Box::leak`. - unsafe { Box::from_raw(self.ptr.as_ptr()) }; + + // SAFETY: If we get this far, we had the last reference to the object. + // That means we are responsible for freeing it, so we can safely lock + // the fake lock again. This wraps dropping the inner object, which + // informs lockdep of the dependencies down the call stack. + #[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] + let guard = unsafe { self.ptr.as_ref() }.lockdep_map.lock(); + + // SAFETY: The pointer was initialised from the result of `Box::leak`, + // and the value is valid. + unsafe { core::ptr::drop_in_place(&mut self.ptr.as_mut().data) }; + + // We need to drop the lock guard before freeing the LockdepMap itself + #[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] + core::mem::drop(guard); + + // SAFETY: The pointer was initialised from the result of `Box::leak`, + // and the lockdep map is valid. + #[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] + unsafe { + core::ptr::drop_in_place(&mut self.ptr.as_mut().lockdep_map) + }; + + // SAFETY: The pointer was initialised from the result of `Box::leak`, and + // a ManuallyDrop is compatible. We already dropped the contents above. + unsafe { Box::from_raw(self.ptr.as_ptr() as *mut ManuallyDrop>) }; } } } @@ -499,6 +549,7 @@ pub struct UniqueArc { impl UniqueArc { /// Tries to allocate a new [`UniqueArc`] instance. + #[track_caller] pub fn try_new(value: T) -> Result { Ok(Self { // INVARIANT: The newly-created object has a ref-count of 1. @@ -507,13 +558,27 @@ impl UniqueArc { } /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. + #[track_caller] pub fn try_new_uninit() -> Result>, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. + #[cfg(CONFIG_RUST_EXTRA_LOCKDEP)] + let inner = { + let map = LockdepMap::new(); + Box::try_init::(try_init!(ArcInner { + // SAFETY: There are no safety requirements for this FFI call. + refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), + lockdep_map: map, + data <- init::uninit::(), + }? AllocError))? + }; + // FIXME: try_init!() does not work with cfg attributes. + #[cfg(not(CONFIG_RUST_EXTRA_LOCKDEP))] let inner = Box::try_init::(try_init!(ArcInner { // SAFETY: There are no safety requirements for this FFI call. refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), data <- init::uninit::(), }? AllocError))?; + Ok(UniqueArc { // INVARIANT: The newly-created object has a ref-count of 1. // SAFETY: The pointer from the `Box` is valid. From a12264a879ddc9198951cb04bd1a44793a6de69a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 30 May 2023 20:56:14 +0900 Subject: [PATCH 613/635] drivers/perf: apple_m1: Force 63bit counters for M2 CPUs Sidharth reports that on M2, the PMU never generates any interrupt when using 'perf record', which is a annoying as you get no sample. I'm temped to say "no sample, no problem", but others may have a different opinion. Upon investigation, it appears that the counters on M2 are significantly different from the ones on M1, as they count on 64 bits instead of 48. Which of course, in the fine M1 tradition, means that we can only use 63 bits, as the top bit is used to signal the interrupt... This results in having to introduce yet another flag to indicate yet another odd counter width. Who knows what the next crazy implementation will do... With this, perf can work out the correct offset, and 'perf record' works as intended. Tested on M2 and M2-Pro CPUs. Cc: Janne Grunau Cc: Hector Martin Cc: Mark Rutland Cc: Will Deacon Fixes: 7d0bfb7c9977 ("drivers/perf: apple_m1: Add Apple M2 support") Reported-by: Sidharth Kshatriya Signed-off-by: Marc Zyngier --- drivers/perf/apple_m1_cpu_pmu.c | 38 ++++++++++++++++++++++++--------- drivers/perf/arm_pmu.c | 2 ++ include/linux/perf/arm_pmu.h | 2 ++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/drivers/perf/apple_m1_cpu_pmu.c b/drivers/perf/apple_m1_cpu_pmu.c index 03e1e81b94b9bd..89067a4726488c 100644 --- a/drivers/perf/apple_m1_cpu_pmu.c +++ b/drivers/perf/apple_m1_cpu_pmu.c @@ -493,6 +493,17 @@ static int m1_pmu_map_event(struct perf_event *event) return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); } +static int m2_pmu_map_event(struct perf_event *event) +{ + /* + * Same deal as the above, except that M2 has 64bit counters. + * Which, as far as we're concerned, actually means 63 bits. + * Yes, this is getting awkward. + */ + event->hw.flags |= ARMPMU_EVT_63BIT; + return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT); +} + static void m1_pmu_reset(void *info) { int i; @@ -525,7 +536,7 @@ static int m1_pmu_set_event_filter(struct hw_perf_event *event, return 0; } -static int m1_pmu_init(struct arm_pmu *cpu_pmu) +static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags) { cpu_pmu->handle_irq = m1_pmu_handle_irq; cpu_pmu->enable = m1_pmu_enable_event; @@ -536,7 +547,14 @@ static int m1_pmu_init(struct arm_pmu *cpu_pmu) cpu_pmu->clear_event_idx = m1_pmu_clear_event_idx; cpu_pmu->start = m1_pmu_start; cpu_pmu->stop = m1_pmu_stop; - cpu_pmu->map_event = m1_pmu_map_event; + + if (flags & ARMPMU_EVT_47BIT) + cpu_pmu->map_event = m1_pmu_map_event; + else if (flags & ARMPMU_EVT_63BIT) + cpu_pmu->map_event = m2_pmu_map_event; + else + return WARN_ON(-EINVAL); + cpu_pmu->reset = m1_pmu_reset; cpu_pmu->set_event_filter = m1_pmu_set_event_filter; @@ -550,25 +568,25 @@ static int m1_pmu_init(struct arm_pmu *cpu_pmu) static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu) { cpu_pmu->name = "apple_icestorm_pmu"; - return m1_pmu_init(cpu_pmu); + return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); } static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu) { cpu_pmu->name = "apple_firestorm_pmu"; - return m1_pmu_init(cpu_pmu); + return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT); } -static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) +static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) { - cpu_pmu->name = "apple_blizzard_pmu"; - return m1_pmu_init(cpu_pmu); + cpu_pmu->name = "apple_avalanche_pmu"; + return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); } -static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu) +static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu) { - cpu_pmu->name = "apple_avalanche_pmu"; - return m1_pmu_init(cpu_pmu); + cpu_pmu->name = "apple_blizzard_pmu"; + return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT); } static const struct of_device_id m1_pmu_of_device_ids[] = { diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index 15bd1e34a88eae..277e29fbd504f8 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -109,6 +109,8 @@ static inline u64 arm_pmu_event_max_period(struct perf_event *event) { if (event->hw.flags & ARMPMU_EVT_64BIT) return GENMASK_ULL(63, 0); + else if (event->hw.flags & ARMPMU_EVT_63BIT) + return GENMASK_ULL(62, 0); else if (event->hw.flags & ARMPMU_EVT_47BIT) return GENMASK_ULL(46, 0); else diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 525b5d64e3948c..c0e4baf940dce6 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -26,9 +26,11 @@ */ #define ARMPMU_EVT_64BIT 0x00001 /* Event uses a 64bit counter */ #define ARMPMU_EVT_47BIT 0x00002 /* Event uses a 47bit counter */ +#define ARMPMU_EVT_63BIT 0x00004 /* Event uses a 63bit counter */ static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_64BIT) == ARMPMU_EVT_64BIT); static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_47BIT) == ARMPMU_EVT_47BIT); +static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_63BIT) == ARMPMU_EVT_63BIT); #define HW_OP_UNSUPPORTED 0xFFFF #define C(_x) PERF_COUNT_HW_CACHE_##_x From 53fe89e862032857ada6aaab51f2f27675e00c98 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 30 May 2023 21:33:57 +0900 Subject: [PATCH 614/635] fixup! arm64: dts: apple: Add MTP nodes to t6020x --- arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 415379b028fddb..bcdb6850607529 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -136,8 +136,10 @@ }; &mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; + mtp_mt: multi-touch { - apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; }; keyboard: keyboard { @@ -146,7 +148,6 @@ }; stm { - apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; }; actuator { From b312855079fdc84a7c11ff9e66b37452a25d4e3e Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 30 May 2023 21:33:57 +0900 Subject: [PATCH 615/635] fixup! arm64: dts: apple: t8112: Add mtp device nodes for j413/j493 --- arch/arm64/boot/dts/apple/t8112-j413.dts | 5 +++-- arch/arm64/boot/dts/apple/t8112-j493.dts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 1b95c320247573..d4724ffd6a82c4 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -210,8 +210,10 @@ status = "okay"; }; &mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + multi-touch { - apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; firmware-name = "apple/tpmtfw-j413.bin"; }; @@ -221,7 +223,6 @@ }; stm { - apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; }; actuator { diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index 50d0b4be3fd32b..ac29b7bc53a087 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -198,8 +198,10 @@ status = "okay"; }; &mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + multi-touch { - apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; firmware-name = "apple/tpmtfw-j493.bin"; }; @@ -209,7 +211,6 @@ }; stm { - apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; }; actuator { From 107ed86e21d1522d5d5d9d629f1d9c371e37df7f Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 30 May 2023 19:52:09 +0900 Subject: [PATCH 616/635] power: supply: macsmc_power: Add CHWA charge thresholds This is a hardcoded charge threshold feature present in firmware 13.0 or newer. Userspace settings are rounded to one of the two possible behaviors. Signed-off-by: Hector Martin --- drivers/power/supply/macsmc_power.c | 67 +++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/drivers/power/supply/macsmc_power.c b/drivers/power/supply/macsmc_power.c index d928c127dd7dc5..5ca3f7c4cb6282 100644 --- a/drivers/power/supply/macsmc_power.c +++ b/drivers/power/supply/macsmc_power.c @@ -20,11 +20,13 @@ struct macsmc_power { struct device *dev; struct apple_smc *smc; + struct power_supply_desc batt_desc; struct power_supply *batt; char model_name[MAX_STRING_LENGTH]; char serial_number[MAX_STRING_LENGTH]; char mfg_date[MAX_STRING_LENGTH]; + bool has_chwa; struct power_supply *ac; @@ -62,9 +64,11 @@ static struct macsmc_power *g_power; #define CH0R_LOWER_FLAGS GENMASK(15, 0) #define CH0R_NOAC_CH0I BIT(0) +#define CH0R_NOAC_DISCONNECTED BIT(4) #define CH0R_NOAC_CH0J BIT(5) #define CH0R_BMS_BUSY BIT(8) #define CH0R_NOAC_CH0K BIT(9) +#define CH0R_NOAC_CHWA BIT(11) #define CH0X_CH0C BIT(0) #define CH0X_CH0B BIT(1) @@ -72,6 +76,10 @@ static struct macsmc_power *g_power; #define ACSt_CAN_BOOT_AP BIT(2) #define ACSt_CAN_BOOT_IBOOT BIT(1) +#define CHWA_FIXED_START_THRESHOLD 75 +#define CHWA_FIXED_END_THRESHOLD 80 +#define CHWA_PROP_WRITE_THRESHOLD 95 + static void macsmc_do_dbg(struct macsmc_power *power) { int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; @@ -107,6 +115,7 @@ static int macsmc_battery_get_status(struct macsmc_power *power) u64 nocharge_flags; u32 nopower_flags; u16 ac_current; + bool chwa_limit = false; int ret; /* @@ -153,14 +162,29 @@ static int macsmc_battery_get_status(struct macsmc_power *power) else if (ret) return POWER_SUPPLY_STATUS_FULL; + /* + * If we have charge limits supported and enabled and the SoC is > 75%, + * that means we are not charging for that reason (if not charging). + */ + if (power->has_chwa && apple_smc_read_flag(power->smc, SMC_KEY(CHWA)) == 1) { + u8 buic = 0; + + if (apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &buic) >= 0 && + buic >= CHWA_FIXED_START_THRESHOLD) + chwa_limit = true; + } + /* If there are reasons we aren't charging... */ ret = apple_smc_read_u64(power->smc, SMC_KEY(CHNC), &nocharge_flags); if (!ret) { /* Perhaps the battery is full after all */ if (nocharge_flags & CHNC_BATTERY_FULL) return POWER_SUPPLY_STATUS_FULL; - /* Or maybe the BMS is just busy doing something, if so call it charging anyway */ - else if (nocharge_flags == CHNC_BMS_BUSY) + /* + * Or maybe the BMS is just busy doing something, if so call it charging anyway. + * But CHWA limits show up as this, so exclude those. + */ + else if (nocharge_flags == CHNC_BMS_BUSY && !chwa_limit) return POWER_SUPPLY_STATUS_CHARGING; /* If we have other reasons we aren't charging, say we aren't */ else if (nocharge_flags) @@ -393,6 +417,16 @@ static int macsmc_battery_get_property(struct power_supply *psy, case POWER_SUPPLY_PROP_MANUFACTURE_DAY: ret = macsmc_battery_get_date(&power->mfg_date[4], &val->intval); break; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA)); + val->intval = ret == 1 ? CHWA_FIXED_START_THRESHOLD : 100; + ret = ret < 0 ? ret : 0; + break; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA)); + val->intval = ret == 1 ? CHWA_FIXED_END_THRESHOLD : 100; + ret = ret < 0 ? ret : 0; + break; default: return -EINVAL; } @@ -409,6 +443,16 @@ static int macsmc_battery_set_property(struct power_supply *psy, switch (psp) { case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: return macsmc_battery_set_charge_behaviour(power, val->intval); + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + /* + * Ignore, we allow writes so userspace isn't confused but this is + * not configurable independently, it always is 75 or 100 depending + * on the end_threshold boolean setting. + */ + return 0; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + return apple_smc_write_flag(power->smc, SMC_KEY(CHWA), + val->intval <= CHWA_PROP_WRITE_THRESHOLD); default: return -EINVAL; } @@ -417,15 +461,20 @@ static int macsmc_battery_set_property(struct power_supply *psy, static int macsmc_battery_property_is_writeable(struct power_supply *psy, enum power_supply_property psp) { + struct macsmc_power *power = power_supply_get_drvdata(psy); + switch (psp) { case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: return true; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + return power->has_chwa; default: return false; } } -static enum power_supply_property macsmc_battery_props[] = { +static const enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_PRESENT, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, @@ -454,6 +503,8 @@ static enum power_supply_property macsmc_battery_props[] = { POWER_SUPPLY_PROP_MANUFACTURE_YEAR, POWER_SUPPLY_PROP_MANUFACTURE_MONTH, POWER_SUPPLY_PROP_MANUFACTURE_DAY, + POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD, + POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD }; static const struct power_supply_desc macsmc_battery_desc = { @@ -651,6 +702,7 @@ static int macsmc_power_probe(struct platform_device *pdev) power->dev = &pdev->dev; power->smc = smc; + power->batt_desc = macsmc_battery_desc; dev_set_drvdata(&pdev->dev, power); /* Ignore devices without a charger/battery */ @@ -666,11 +718,18 @@ static int macsmc_power_probe(struct platform_device *pdev) apple_smc_write_u8(power->smc, SMC_KEY(CH0K), 0); apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); + if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA)) >= 0) { + power->has_chwa = true; + } else { + /* Remove the last 2 properties that control the charge threshold */ + power->batt_desc.num_properties -= 2; + } + /* Doing one read of this flag enables critical shutdown notifications */ apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val); psy_cfg.drv_data = power; - power->batt = devm_power_supply_register(&pdev->dev, &macsmc_battery_desc, &psy_cfg); + power->batt = devm_power_supply_register(&pdev->dev, &power->batt_desc, &psy_cfg); if (IS_ERR(power->batt)) { dev_err(&pdev->dev, "Failed to register battery\n"); ret = PTR_ERR(power->batt); From d8ff4371eb354b0e1165849b92806215619d45e6 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Fri, 8 Jul 2022 02:11:21 +0900 Subject: [PATCH 617/635] hid: Add Apple DockChannel HID transport driver Apple M2 devices have an MTP coprocessor embedded in the SoC that handles HID for the integrated touchpad/keyboard, and communicates over the DockChannel interface. This driver implements this new interface. Signed-off-by: Hector Martin --- drivers/hid/Kconfig | 2 + drivers/hid/Makefile | 2 + drivers/hid/dockchannel-hid/Kconfig | 14 + drivers/hid/dockchannel-hid/Makefile | 6 + drivers/hid/dockchannel-hid/dockchannel-hid.c | 1216 +++++++++++++++++ 5 files changed, 1240 insertions(+) create mode 100644 drivers/hid/dockchannel-hid/Kconfig create mode 100644 drivers/hid/dockchannel-hid/Makefile create mode 100644 drivers/hid/dockchannel-hid/dockchannel-hid.c diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index b611db62e75897..d3a821dcad2c65 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -1320,4 +1320,6 @@ source "drivers/hid/surface-hid/Kconfig" source "drivers/hid/spi-hid/Kconfig" +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 6214a3631b5119..556bed470e0704 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -169,3 +169,5 @@ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig new file mode 100644 index 00000000000000..8a81d551a83d51 --- /dev/null +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + default ARCH_APPLE + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile new file mode 100644 index 00000000000000..7dba766b047fcc --- /dev/null +++ b/drivers/hid/dockchannel-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c new file mode 100644 index 00000000000000..6589917664c4df --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -0,0 +1,1216 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define GROUP_INPUT 0 +#define GROUP_OUTPUT 1 +#define GROUP_CMD 2 + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, GROUP_CMD, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, GROUP_CMD, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, GROUP_OUTPUT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, GROUP_OUTPUT, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strlcpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%d > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case GROUP_INPUT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%d > %ld)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static int dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); + return 0; +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); From f68d94e9f909348a2c396dae332b9ef611b5647b Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 3 Jul 2022 23:33:37 +0900 Subject: [PATCH 618/635] soc: apple: Add RTKit helper driver This driver can be used for coprocessors that do some background task or communicate out-of-band, and do not do any mailbox I/O beyond the standard RTKit initialization. Signed-off-by: Hector Martin --- drivers/soc/apple/Kconfig | 14 +++ drivers/soc/apple/Makefile | 3 + drivers/soc/apple/rtkit-helper.c | 152 +++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 drivers/soc/apple/rtkit-helper.c diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 9e93df0ac82582..694a44968f7612 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -30,6 +30,20 @@ config APPLE_RTKIT Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index aa4203e0246288..07a95a883d3df3 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -4,6 +4,9 @@ obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += apple-pmgr-pwrstate.o obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c new file mode 100644 index 00000000000000..46bf0f0c1c724d --- /dev/null +++ b/drivers/soc/apple/rtkit-helper.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static int apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); + + return 0; +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); From 0f755bd14f8f4d705757ad3c87987ee5d02e3f25 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 20:08:45 +0100 Subject: [PATCH 619/635] HID: transport: spi: Check status message after transmits Probably pointless but might be helpful to debug issues. Gets rid of 'spi_hid_apple_status_ok' is unused compiler warning. Signed-off-by: Janne Grunau --- drivers/hid/spi-hid/spi-hid-apple-core.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index cd018df4f38715..29317a824c7270 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -248,7 +248,17 @@ static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, offsetof(struct spihid_transfer_packet, crc16))); + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } mutex_unlock(&spihid->tx_lock); if (err < 0) return err; From 21305be878516797bedb5b4675a9859d4a4213b3 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 22:56:16 +0100 Subject: [PATCH 620/635] HID: magicmouse: Add .reset_resume for SPI trackpads The trackpad has to request multi touch reports during resume. Signed-off-by: Janne Grunau --- drivers/hid/hid-magicmouse.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index b22ab0f320c05d..414744400f21fe 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -1268,6 +1268,16 @@ static const struct hid_device_id magic_mice[] = { }; MODULE_DEVICE_TABLE(hid, magic_mice); +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + if (hdev->bus == BUS_SPI) + return magicmouse_enable_multitouch(hdev); + + return 0; +} +#endif + static struct hid_driver magicmouse_driver = { .name = "magicmouse", .id_table = magic_mice, @@ -1278,6 +1288,10 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif + }; module_hid_driver(magicmouse_driver); From 35b64ace0dc7a88c01ab3d075ae602cbf1e7a6f3 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 11 Dec 2022 23:24:41 +0100 Subject: [PATCH 621/635] HID: transport: spi: Add suspend support Working suspend and resume. The keyboard can not yet used as wakeup source. Most likely caused by missing irq_set_wake support in the gpio/pinctrl driver. Signed-off-by: Janne Grunau --- drivers/hid/spi-hid/spi-hid-apple-core.c | 89 ++++++++++++++++++++++++ drivers/hid/spi-hid/spi-hid-apple-of.c | 18 ++++- drivers/hid/spi-hid/spi-hid-apple.h | 4 ++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c index 29317a824c7270..787dcd9ceb4ffd 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-core.c +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -115,6 +115,10 @@ struct spihid_apple { /* state tracking flags */ bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif }; /** @@ -1035,6 +1039,91 @@ void spihid_apple_core_shutdown(struct spi_device *spi) } EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + MODULE_DESCRIPTION("Apple SPI HID transport driver"); MODULE_AUTHOR("Janne Grunau "); MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c index f1380bfc52672e..6531f15e2516e9 100644 --- a/drivers/hid/spi-hid/spi-hid-apple-of.c +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -65,6 +65,20 @@ static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) return 0; } +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + static int spihid_apple_of_probe(struct spi_device *spi) { struct device *dev = &spi->dev; @@ -79,6 +93,8 @@ static int spihid_apple_of_probe(struct spi_device *spi) spihid_of->ops.power_off = spihid_apple_of_power_off; spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); if (IS_ERR(spihid_of->enable_gpio)) { @@ -120,7 +136,7 @@ MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); static struct spi_driver spihid_apple_of_driver = { .driver = { .name = "spi-hid-apple-of", - //.pm = &spi_hid_apple_of_pm, + .pm = &spihid_apple_core_pm, .owner = THIS_MODULE, .of_match_table = of_match_ptr(spihid_apple_of_match), }, diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h index 2d9554e8a5f819..9abecd1ba78028 100644 --- a/drivers/hid/spi-hid/spi-hid-apple.h +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -20,6 +20,8 @@ struct spihid_apple_ops { int (*power_off)(struct spihid_apple_ops *ops); int (*enable_irq)(struct spihid_apple_ops *ops); int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); }; irqreturn_t spihid_apple_core_irq(int irq, void *data); @@ -28,4 +30,6 @@ int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops void spihid_apple_core_remove(struct spi_device *spi); void spihid_apple_core_shutdown(struct spi_device *spi); +extern const struct dev_pm_ops spihid_apple_core_pm; + #endif /* SPI_HID_APPLE_H */ From f7d8fa78bed7a860d429937d1b635c0f8bd96d59 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Mon, 10 Apr 2023 22:44:44 +0900 Subject: [PATCH 622/635] HID: Bump maximum report size to 16384 This maximum is arbitrary. Recent Apple devices have some vendor-defined reports with 16384 here which fail to parse without this, so let's bump it to that. This value is used as follows: report->size += parser->global.report_size * parser->global.report_count; [...] /* Total size check: Allow for possible report index byte */ if (report->size > (max_buffer_size - 1) << 3) { hid_err(parser->device, "report is too long\n"); return -1; } All of these fields are unsigned integers, and report_count is bounded by HID_MAX_USAGES (12288). Therefore, as long as the respective maximums do not overflow an unsigned integer (let's say a signed integer just in case), we're safe. This holds for 16384. Signed-off-by: Hector Martin --- drivers/hid/hid-core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 368bb290443fbe..01fcafd907fe1b 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -436,7 +436,10 @@ static int hid_parser_global(struct hid_parser *parser, struct hid_item *item) case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; From 1a0d1e79fe3a7f77bdba929b789d1ca63566edba Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 30 Apr 2023 23:48:45 +0900 Subject: [PATCH 623/635] HID: magicmouse: Handle touch controller resets on SPI devices On at least some SPI devices (e.g. recent Apple Silicon machines), the Broadcom touch controller is prone to crashing. When this happens, the STM eventually notices and resets it. It then notifies the driver via HID report 0x60, and the driver needs to re-enable MT mode to make things work again. This poses an additional issue: the hidinput core will close the low-level transport while the device is closed, which can cause us to miss a reset notification. To fix this, override the input open/close callbacks and send the MT enable every time the HID device is opened, instead of only once on probe. This should increase general robustness, even if the reset mechanism doesn't work for some reason, so it's worth doing it for USB devices too. MTP devices are exempt since they do not require the MT enable at all. Signed-off-by: Hector Martin --- drivers/hid/hid-magicmouse.c | 203 +++++++++++++++++++++++------------ 1 file changed, 133 insertions(+), 70 deletions(-) diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 414744400f21fe..54206faa31a141 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -60,6 +60,7 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 #define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 #define MTP_REPORT_ID 0x75 #define USB_BATTERY_TIMEOUT_MS 60000 @@ -170,6 +171,97 @@ struct magicmouse_sc { struct magicmouse_input_ops input_ops; }; +static int magicmouse_enable_multitouch(struct hid_device *hdev) +{ + const u8 *feature; + const u8 feature_mt[] = { 0xD7, 0x01 }; + const u8 feature_mt_mouse2[] = { 0xF1, 0x02, 0x01 }; + const u8 feature_mt_trackpad2_usb[] = { 0x02, 0x01 }; + const u8 feature_mt_trackpad2_bt[] = { 0xF1, 0x02, 0x01 }; + u8 *buf; + int ret; + int feature_size; + + if (hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2) { + if (hdev->vendor == BT_VENDOR_ID_APPLE) { + feature_size = sizeof(feature_mt_trackpad2_bt); + feature = feature_mt_trackpad2_bt; + } else { /* USB_VENDOR_ID_APPLE */ + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + } + } else if (hdev->vendor == SPI_VENDOR_ID_APPLE) { + feature_size = sizeof(feature_mt_trackpad2_usb); + feature = feature_mt_trackpad2_usb; + } else if (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { + feature_size = sizeof(feature_mt_mouse2); + feature = feature_mt_mouse2; + } else { + feature_size = sizeof(feature_mt); + feature = feature_mt; + } + + buf = kmemdup(feature, feature_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = hid_hw_raw_request(hdev, buf[0], buf, feature_size, + HID_FEATURE_REPORT, HID_REQ_SET_REPORT); + kfree(buf); + return ret; +} + +static void magicmouse_enable_mt_work(struct work_struct *work) +{ + struct magicmouse_sc *msc = + container_of(work, struct magicmouse_sc, work.work); + int ret; + + ret = magicmouse_enable_multitouch(msc->hdev); + if (ret < 0) + hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + */ + ret = magicmouse_enable_multitouch(hdev); + if (ret == -EIO && hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + return 0; + } + if (ret < 0) + hid_err(hdev, "unable to request touch data (%d)\n", ret); + + /* + * MT enable is usually not required after the first time, so don't + * consider it fatal. + */ + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -691,12 +783,19 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, static int magicmouse_raw_event_spi(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { + struct magicmouse_sc *msc = hid_get_drvdata(hdev); const size_t hdr_sz = sizeof(struct tp_mouse_report); - if (size < hdr_sz) + if (!size) return 0; - if (data[0] != TRACKPAD2_USB_REPORT_ID) + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != TRACKPAD2_USB_REPORT_ID || size < hdr_sz) return 0; return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); @@ -878,10 +977,17 @@ static int magicmouse_setup_input_usb(struct input_dev *input, */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; return 0; } -static int magicmouse_setup_input_spi(struct input_dev *input, +static int magicmouse_setup_input_mtp(struct input_dev *input, struct hid_device *hdev) { int error; @@ -954,6 +1060,25 @@ static int magicmouse_setup_input_spi(struct input_dev *input, return 0; } +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_mtp(input, hdev); + if (ret) + return ret; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + + return 0; +} + static int magicmouse_input_mapping(struct hid_device *hdev, struct hid_input *hi, struct hid_field *field, struct hid_usage *usage, unsigned long **bit, int *max) @@ -990,57 +1115,6 @@ static int magicmouse_input_configured(struct hid_device *hdev, return 0; } -static int magicmouse_enable_multitouch(struct hid_device *hdev) -{ - const u8 *feature; - const u8 feature_mt[] = { 0xD7, 0x01 }; - const u8 feature_mt_mouse2[] = { 0xF1, 0x02, 0x01 }; - const u8 feature_mt_trackpad2_usb[] = { 0x02, 0x01 }; - const u8 feature_mt_trackpad2_bt[] = { 0xF1, 0x02, 0x01 }; - u8 *buf; - int ret; - int feature_size; - - if (hdev->product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2) { - if (hdev->vendor == BT_VENDOR_ID_APPLE) { - feature_size = sizeof(feature_mt_trackpad2_bt); - feature = feature_mt_trackpad2_bt; - } else { /* USB_VENDOR_ID_APPLE */ - feature_size = sizeof(feature_mt_trackpad2_usb); - feature = feature_mt_trackpad2_usb; - } - } else if (hdev->vendor == SPI_VENDOR_ID_APPLE) { - feature_size = sizeof(feature_mt_trackpad2_usb); - feature = feature_mt_trackpad2_usb; - } else if (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { - feature_size = sizeof(feature_mt_mouse2); - feature = feature_mt_mouse2; - } else { - feature_size = sizeof(feature_mt); - feature = feature_mt; - } - - buf = kmemdup(feature, feature_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - ret = hid_hw_raw_request(hdev, buf[0], buf, feature_size, - HID_FEATURE_REPORT, HID_REQ_SET_REPORT); - kfree(buf); - return ret; -} - -static void magicmouse_enable_mt_work(struct work_struct *work) -{ - struct magicmouse_sc *msc = - container_of(work, struct magicmouse_sc, work.work); - int ret; - - ret = magicmouse_enable_multitouch(msc->hdev); - if (ret < 0) - hid_err(msc->hdev, "unable to request touch data (%d)\n", ret); -} - static int magicmouse_fetch_battery(struct hid_device *hdev) { #ifdef CONFIG_HID_BATTERY_STRENGTH @@ -1100,7 +1174,7 @@ static int magicmouse_probe(struct hid_device *hdev, // conflicts with the report ID. if (id->bus == BUS_HOST) { msc->input_ops.raw_event = magicmouse_raw_event_mtp; - msc->input_ops.setup_input = magicmouse_setup_input_spi; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; } else if (id->bus == BUS_SPI) { msc->input_ops.raw_event = magicmouse_raw_event_spi; msc->input_ops.setup_input = magicmouse_setup_input_spi; @@ -1180,21 +1254,10 @@ static int magicmouse_probe(struct hid_device *hdev, if (id->bus == BUS_HOST) return 0; - /* - * Some devices repond with 'invalid report id' when feature - * report switching it into multitouch mode is sent to it. - * - * This results in -EIO from the _raw low-level transport callback, - * but there seems to be no other way of switching the mode. - * Thus the super-ugly hacky success check below. - */ - ret = magicmouse_enable_multitouch(hdev); - if (ret != -EIO && ret < 0) { - hid_err(hdev, "unable to request touch data (%d)\n", ret); - goto err_stop_hw; - } - if (ret == -EIO && id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2) { - schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + /* SPI devices need to watch for reset events to re-send the MT enable */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_RESET_REPORT_ID, 0); + report->size = 2; } return 0; From 8bad6a3027f8f281867a06f8fa01642cd5f7a98b Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Thu, 27 Apr 2023 09:02:11 +1000 Subject: [PATCH 624/635] xfs: fix livelock in delayed allocation at ENOSPC On a filesystem with a non-zero stripe unit and a large sequential write, delayed allocation will set a minimum allocation length of the stripe unit. If allocation fails because there are no extents long enough for an aligned minlen allocation, it is supposed to fall back to unaligned allocation which allows single block extents to be allocated. When the allocator code was rewritting in the 6.3 cycle, this fallback was broken - the old code used args->fsbno as the both the allocation target and the allocation result, the new code passes the target as a separate parameter. The conversion didn't handle the aligned->unaligned fallback path correctly - it reset args->fsbno to the target fsbno on failure which broke allocation failure detection in the high level code and so it never fell back to unaligned allocations. This resulted in a loop in writeback trying to allocate an aligned block, getting a false positive success, trying to insert the result in the BMBT. This did nothing because the extent already was in the BMBT (merge results in an unchanged extent) and so it returned the prior extent to the conversion code as the current iomap. Because the iomap returned didn't cover the offset we tried to map, xfs_convert_blocks() then retries the allocation, which fails in the same way and now we have a livelock. Reported-and-tested-by: Brian Foster Fixes: 85843327094f ("xfs: factor xfs_bmap_btalloc()") Signed-off-by: Dave Chinner Reviewed-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 34de6e6898c480..3ce43068daa13b 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3505,7 +3505,6 @@ xfs_bmap_btalloc_at_eof( * original non-aligned state so the caller can proceed on allocation * failure as if this function was never called. */ - args->fsbno = ap->blkno; args->alignment = 1; return 0; } From 544c9c8a46f536ff31c94f3abcf4e41a2eb27782 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 18 Apr 2023 21:50:43 +0900 Subject: [PATCH 625/635] cpufreq: apple-soc: Drop setting the PS2 field on M2+ Newer devices don't use this. We still don't know what it does, but let's keep to the same behavior macOS has here. Signed-off-by: Hector Martin --- drivers/cpufreq/apple-soc-cpufreq.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 021f423705e1b1..51a71322f07ac8 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -25,7 +25,7 @@ #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) #define APPLE_DVFS_CMD_SET BIT(25) -#define APPLE_DVFS_CMD_PS2 GENMASK(16, 12) +#define APPLE_DVFS_CMD_PS2 GENMASK(15, 12) #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) /* Same timebase as CPU counter (24MHz) */ @@ -55,6 +55,7 @@ #define APPLE_DVFS_TRANSITION_TIMEOUT 100 struct apple_soc_cpufreq_info { + bool has_ps2; u64 max_pstate; u64 cur_pstate_mask; u64 cur_pstate_shift; @@ -69,18 +70,21 @@ struct apple_cpu_priv { static struct cpufreq_driver apple_soc_cpufreq_driver; static const struct apple_soc_cpufreq_info soc_t8103_info = { + .has_ps2 = true, .max_pstate = 15, .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, }; static const struct apple_soc_cpufreq_info soc_t8112_info = { + .has_ps2 = false, .max_pstate = 31, .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, }; static const struct apple_soc_cpufreq_info soc_default_info = { + .has_ps2 = false, .max_pstate = 15, .cur_pstate_mask = 0, /* fallback */ }; @@ -148,9 +152,12 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, return -EIO; } - reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2); + reg &= ~APPLE_DVFS_CMD_PS1; reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate); - reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); + if (priv->info->has_ps2) { + reg &= ~APPLE_DVFS_CMD_PS2; + reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); + } reg |= APPLE_DVFS_CMD_SET; writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); From e2bf6cd8340abe9dbb54689313217ffff44fe946 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 6 Jun 2023 15:53:40 +0900 Subject: [PATCH 626/635] fixup! xhci-pci: asmedia: Add a firmware loader for ASM2214a chips --- drivers/usb/host/xhci-pci-asmedia.c | 112 ++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 15 deletions(-) diff --git a/drivers/usb/host/xhci-pci-asmedia.c b/drivers/usb/host/xhci-pci-asmedia.c index b52412e46bfcf4..80f1b3eabd6deb 100644 --- a/drivers/usb/host/xhci-pci-asmedia.c +++ b/drivers/usb/host/xhci-pci-asmedia.c @@ -38,13 +38,27 @@ /* BAR0 registers */ #define ASMT_REG_ADDR 0x3000 -#define ASMT_REG_DATA 0x3004 +#define ASMT_REG_WDATA 0x3004 +#define ASMT_REG_RDATA 0x3008 #define ASMT_REG_STATUS 0x3009 #define ASMT_REG_STATUS_BUSY BIT(7) -#define ASMT_REG_WDATA 0x3010 -#define ASMT_REG_RDATA 0x3018 +#define ASMT_REG_CODE_WDATA 0x3010 +#define ASMT_REG_CODE_RDATA 0x3018 + +#define ASMT_MMIO_CPU_MISC 0x500e +#define ASMT_MMIO_CPU_MISC_CODE_RAM_WR BIT(0) + +#define ASMT_MMIO_CPU_MODE_NEXT 0x5040 +#define ASMT_MMIO_CPU_MODE_CUR 0x5041 + +#define ASMT_MMIO_CPU_MODE_RAM BIT(0) +#define ASMT_MMIO_CPU_MODE_HALFSPEED BIT(1) + +#define ASMT_MMIO_CPU_EXEC_CTRL 0x5042 +#define ASMT_MMIO_CPU_EXEC_CTRL_RESET BIT(0) +#define ASMT_MMIO_CPU_EXEC_CTRL_HALT BIT(1) #define TIMEOUT_USEC 10000 #define RESET_TIMEOUT_USEC 500000 @@ -178,11 +192,41 @@ static int asmedia_wait_reset(struct pci_dev *pdev) return ret; } -static void asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data) { +static u8 asmedia_read_reg(struct usb_hcd *hcd, u16 addr) { void __iomem *regs = hcd->regs; u8 status; int ret; + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, + status, !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg wait timed out ([%04x])\n", addr); + return ~0; + } + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, + status, !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg addr timed out ([%04x])\n", addr); + return ~0; + } + + return readb_relaxed(regs + ASMT_REG_RDATA); +} + +static void asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data, bool wait) { + void __iomem *regs = hcd->regs; + u8 status; + int ret, i; + writew_relaxed(addr, regs + ASMT_REG_ADDR); ret = readb_poll_timeout(regs + ASMT_REG_STATUS, @@ -191,10 +235,10 @@ static void asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data) { if (ret) dev_err(hcd->self.controller, - "Write addr timed out ([%04x] = %02x)\n", + "Write reg addr timed out ([%04x] = %02x)\n", addr, data); - writeb_relaxed(data, regs + ASMT_REG_DATA); + writeb_relaxed(data, regs + ASMT_REG_WDATA); ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, !(status & ASMT_REG_STATUS_BUSY), @@ -202,8 +246,22 @@ static void asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data) { if (ret) dev_err(hcd->self.controller, - "Write data timed out ([%04x] = %02x)\n", + "Write reg data timed out ([%04x] = %02x)\n", + addr, data); + + if (!wait) + return; + + for (i = 0; i < TIMEOUT_USEC; i++) { + if (asmedia_read_reg(hcd, addr) == data) + break; + } + + if (i >= TIMEOUT_USEC) { + dev_err(hcd->self.controller, + "Verify register timed out ([%04x] = %02x)\n", addr, data); + } } static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) @@ -211,16 +269,20 @@ static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) struct usb_hcd *hcd; void __iomem *regs; const u16 *fw_data = (const u16 *)fw->data; + u16 raddr; u32 data; size_t index = 0, addr = 0; size_t words = fw->size >> 1; - int ret; + int ret, i; hcd = dev_get_drvdata(&pdev->dev); regs = hcd->regs; - asmedia_write_reg(hcd, 0x5040, 2); - asmedia_write_reg(hcd, 0x5042, 1); + asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + + asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_RESET, false); ret = asmedia_wait_reset(pdev); if (ret) { @@ -228,7 +290,11 @@ static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) return ret; } - asmedia_write_reg(hcd, 0x500e, 1); + asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_HALT, false); + + asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, + ASMT_MMIO_CPU_MISC_CODE_RAM_WR, true); pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, ASMT_CFG_SRAM_ACCESS_ENABLE); @@ -243,18 +309,34 @@ static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) pci_write_config_word(pdev, ASMT_CFG_SRAM_ADDR, addr); - writel_relaxed(data, regs + ASMT_REG_WDATA); + writel_relaxed(data, regs + ASMT_REG_CODE_WDATA); + + for (i = 0; i < TIMEOUT_USEC; i++) { + pci_read_config_word(pdev, ASMT_CFG_SRAM_ADDR, &raddr); + if (raddr != addr) + break; + udelay(1); + } + + if (raddr == addr) { + dev_err(hcd->self.controller, "Word write timed out\n"); + return -ETIMEDOUT; + } if (++index & 0x4000) index += 0x4000; addr += 2; } - asmedia_write_reg(hcd, 0x5040, 3); - pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); - asmedia_write_reg(hcd, 0x500e, 0); + asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, 0, true); + + asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_RAM | + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + + asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, 0, false); ret = asmedia_wait_reset(pdev); if (ret) { From 9ee0e4cec33dc2bfffb5e7a1dc247a7b7f857556 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 6 Jun 2023 15:31:01 +0900 Subject: [PATCH 627/635] PCI: apple: Do not power down devices on port setup If a device is already powered, leave it powered. Otherwise port setup done by u-boot breaks. Signed-off-by: Hector Martin --- drivers/pci/controller/pcie-apple.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 7d7d8973d9a7e2..b309d9a15e9f08 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -598,7 +598,7 @@ static int apple_pcie_probe_port(struct device_node *np) gpiod_put(gd); gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "pwren", 0, - GPIOD_OUT_LOW, "PWREN"); + GPIOD_ASIS, "PWREN"); if (IS_ERR(gd)) { if (PTR_ERR(gd) != -ENOENT) return PTR_ERR(gd); @@ -623,7 +623,7 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, return PTR_ERR(reset); pwren = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "pwren", - GPIOD_OUT_LOW, "PWREN"); + GPIOD_ASIS, "PWREN"); if (IS_ERR(pwren)) { if (PTR_ERR(pwren) == -ENOENT) pwren = NULL; From fa7596809aee310363aa3e4ab680c8d1366065c1 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 6 Jun 2023 15:53:23 +0900 Subject: [PATCH 628/635] wifi: brcmfmac: Do not service msgbuf IRQs until ready in MSI mode This is the counterpart to b50255c83b. In MSI mode we can still get MSIs even with IRQs disabled, so add an explicit gate for it. Signed-off-by: Hector Martin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 1dca969c608dea..c1726fe5ae296e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -407,6 +407,7 @@ struct brcmf_pciedev_info { wait_queue_head_t mbdata_resp_wait; bool mbdata_completed; bool irq_allocated; + bool irq_ready; bool have_msi; bool wowl_enabled; u8 dma_idx_sz; @@ -987,6 +988,8 @@ static void brcmf_pcie_bus_console_read(struct brcmf_pciedev_info *devinfo, static void brcmf_pcie_intr_disable(struct brcmf_pciedev_info *devinfo) { brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, 0); + + devinfo->irq_ready = false; } @@ -995,6 +998,8 @@ static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, devinfo->reginfo->int_d2h_db | devinfo->reginfo->int_fn0); + + devinfo->irq_ready = true; } static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) @@ -1040,7 +1045,7 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) brcmf_pcie_poll_mb_data(devinfo); } if (devinfo->have_msi || status & devinfo->reginfo->int_d2h_db) { - if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) + if (devinfo->state == BRCMFMAC_PCIE_STATE_UP && devinfo->irq_ready) brcmf_proto_msgbuf_rx_trigger(&devinfo->pdev->dev); } From 4427f8da930cf1ae027422d1ca6c736e21dd0348 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Tue, 18 Apr 2023 22:59:53 +0300 Subject: [PATCH 629/635] gpu: drm: adp: Add Apple Display Pipe driver This display controller is present on M-series chips and is used to drive the touchbar display. Signed-off-by: Sasha Finkelstein Signed-off-by: Janne Grunau --- drivers/gpu/drm/Kconfig | 2 + drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/adp/Kconfig | 12 + drivers/gpu/drm/adp/Makefile | 5 + drivers/gpu/drm/adp/adp_drv.c | 762 ++++++++++++++++++++++++++++++++++ 5 files changed, 782 insertions(+) create mode 100644 drivers/gpu/drm/adp/Kconfig create mode 100644 drivers/gpu/drm/adp/Makefile create mode 100644 drivers/gpu/drm/adp/adp_drv.c diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index bda2b6400f835c..d6bfe617f53189 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -371,6 +371,8 @@ source "drivers/gpu/drm/sprd/Kconfig" source "drivers/gpu/drm/apple/Kconfig" +source "drivers/gpu/drm/adp/Kconfig" + config DRM_HYPERV tristate "DRM Support for Hyper-V synthetic video device" depends on DRM && PCI && MMU && HYPERV diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index dad51ccb32e21c..495cae121eb51d 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -183,6 +183,7 @@ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ obj-$(CONFIG_DRM_LIMA) += lima/ obj-$(CONFIG_DRM_PANFROST) += panfrost/ obj-$(CONFIG_DRM_APPLE) += apple/ +obj-$(CONFIG_DRM_ADP) += adp/ obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/adp/Kconfig b/drivers/gpu/drm/adp/Kconfig new file mode 100644 index 00000000000000..739029bde31919 --- /dev/null +++ b/drivers/gpu/drm/adp/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +config DRM_ADP + tristate "DRM Support for pre-DCP Apple display controllers" + depends on DRM && OF && ARM64 + depends on ARCH_APPLE || COMPILE_TEST + select DRM_KMS_HELPER + select DRM_KMS_DMA_HELPER + select DRM_GEM_DMA_HELPER + select VIDEOMODE_HELPERS + select DRM_MIPI_DSI + help + Say Y if you have an Apple Arm laptop with a touchbar. diff --git a/drivers/gpu/drm/adp/Makefile b/drivers/gpu/drm/adp/Makefile new file mode 100644 index 00000000000000..28a5d4b4a267f3 --- /dev/null +++ b/drivers/gpu/drm/adp/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +adpdrm-y := adp_drv.o +obj-$(CONFIG_DRM_ADP) += adpdrm.o +obj-$(CONFIG_DRM_ADP) += panel-summit.o diff --git a/drivers/gpu/drm/adp/adp_drv.c b/drivers/gpu/drm/adp/adp_drv.c new file mode 100644 index 00000000000000..b26caae66c5894 --- /dev/null +++ b/drivers/gpu/drm/adp/adp_drv.c @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ADP_INT_STATUS 0x34 +#define ADP_INT_STATUS_INT_MASK 0x7 +#define ADP_INT_STATUS_VBLANK 0x1 +#define ADP_CTRL 0x100 +#define ADP_CTRL_VBLANK_ON 0x12 +#define ADP_CTRL_FIFO_ON 0x601 +#define ADP_SCREEN_SIZE 0x0c + +#define ADBE_FIFO 0x10c0 +#define ADBE_FIFO_SYNC 0xc0000000 + +#define ADBE_BLEND_BYPASS 0x2020 +#define ADBE_BLEND_EN1 0x2028 +#define ADBE_BLEND_EN2 0x2074 +#define ADBE_BLEND_EN3 0x202c +#define ADBE_BLEND_EN4 0x2034 +#define ADBE_MASK_BUF 0x2200 + +#define ADBE_SRC_START 0x4040 +#define ADBE_SRC_SIZE 0x4048 +#define ADBE_DST_START 0x4050 +#define ADBE_DST_SIZE 0x4054 +#define ADBE_STRIDE 0x4038 +#define ADBE_FB_BASE 0x4030 + +#define ADBE_LAYER_EN1 0x4020 +#define ADBE_LAYER_EN2 0x4068 +#define ADBE_LAYER_EN3 0x40b4 +#define ADBE_LAYER_EN4 0x40f4 +#define ADBE_SCALE_CTL 0x40ac +#define ADBE_SCALE_CTL_BYPASS 0x100000 + +#define ADBE_LAYER_CTL 0x1038 +#define ADBE_LAYER_CTL_ENABLE 0x10000 + +#define ADBE_PIX_FMT 0x402c +#define ADBE_PIX_FMT_XRGB32 0x53e4001 + +#define DSI_GEN_HDR 0x6c +#define DSI_GEN_PLD_DATA 0x70 + +#define DSI_CMD_PKT_STATUS 0x74 + +#define GEN_PLD_R_EMPTY BIT(4) +#define GEN_PLD_W_FULL BIT(3) +#define GEN_PLD_W_EMPTY BIT(2) +#define GEN_CMD_FULL BIT(1) +#define GEN_CMD_EMPTY BIT(0) +#define GEN_RD_CMD_BUSY BIT(6) +#define CMD_PKT_STATUS_TIMEOUT_US 20000 + +DEFINE_DRM_GEM_DMA_FOPS(adp_fops); + +static int adp_drm_gem_dumb_create(struct drm_file *file_priv, + struct drm_device *drm, + struct drm_mode_create_dumb *args) +{ + args->height = ALIGN(args->height, 64); + args->size = args->pitch * args->height; + + return drm_gem_dma_dumb_create_internal(file_priv, drm, args); +} + +static const struct drm_driver adp_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .fops = &adp_fops, + DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(adp_drm_gem_dumb_create), + .name = "adp", + .desc = "Apple Display Pipe DRM Driver", + .date = "20230412", + .major = 0, + .minor = 1, +}; + +struct adp_drv_private { + struct drm_device drm; + struct drm_crtc crtc; + struct drm_encoder encoder; + struct drm_connector connector; + struct mipi_dsi_host dsi; + void __iomem *be; + void __iomem *fe; + void __iomem *mipi; + u32 *mask_buf; + u64 mask_buf_size; + dma_addr_t mask_iova; + int be_irq; + int fe_irq; + spinlock_t irq_lock; + struct drm_pending_vblank_event *event; +}; + +struct adp_plane { + struct drm_plane base_plane; + u8 id; +}; + +#define to_adp(x) container_of(x, struct adp_drv_private, drm) +#define crtc_to_adp(x) container_of(x, struct adp_drv_private, crtc) +#define conn_to_adp(x) container_of(x, struct adp_drv_private, connector) +#define mipi_to_adp(x) container_of(x, struct adp_drv_private, dsi) + +static int adp_plane_atomic_check(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *new_plane_state; + struct drm_crtc_state *crtc_state; + + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + + if (!new_plane_state->crtc) + return 0; + + crtc_state = drm_atomic_get_crtc_state(state, new_plane_state->crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + return drm_atomic_helper_check_plane_state(new_plane_state, + crtc_state, + DRM_PLANE_NO_SCALING, + DRM_PLANE_NO_SCALING, + true, true); +} + +static void adp_plane_atomic_update(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct adp_drv_private *adp; + struct drm_rect src_rect; + struct drm_gem_dma_object *obj; + struct drm_framebuffer *fb; + struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane); + u32 src_pos, src_size, dst_pos, dst_size; + if (!plane || !new_state) + return; + + fb = new_state->fb; + if (!fb) + return; + adp = to_adp(plane->dev); + + drm_rect_fp_to_int(&src_rect, &new_state->src); + src_pos = src_rect.x1 << 16 | src_rect.y1; + dst_pos = new_state->dst.x1 << 16 | new_state->dst.y1; + src_size = drm_rect_width(&src_rect) << 16 | drm_rect_height(&src_rect); + dst_size = drm_rect_width(&new_state->dst) << 16 | + drm_rect_height(&new_state->dst); + writel(src_pos, adp->be + ADBE_SRC_START); + writel(src_size, adp->be + ADBE_SRC_SIZE); + writel(dst_pos, adp->be + ADBE_DST_START); + writel(dst_size, adp->be + ADBE_DST_SIZE); + writel(fb->pitches[0], adp->be + ADBE_STRIDE); + obj = drm_fb_dma_get_gem_obj(fb, 0); + if (obj) + writel(obj->dma_addr + fb->offsets[0], adp->be + ADBE_FB_BASE); + + writel(0x1, adp->be + ADBE_LAYER_EN1); + writel(0x1, adp->be + ADBE_LAYER_EN2); + writel(0x1, adp->be + ADBE_LAYER_EN3); + writel(0x1, adp->be + ADBE_LAYER_EN4); + writel(ADBE_SCALE_CTL_BYPASS, adp->be + ADBE_SCALE_CTL); + writel(ADBE_LAYER_CTL_ENABLE | 0x1, adp->be + ADBE_LAYER_CTL); + writel(ADBE_PIX_FMT_XRGB32, adp->be + ADBE_PIX_FMT); + +} + +static void adp_plane_atomic_disable(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct adp_drv_private *adp = to_adp(plane->dev); + writel(0x0, adp->be + ADBE_LAYER_EN1); + writel(0x0, adp->be + ADBE_LAYER_EN2); + writel(0x0, adp->be + ADBE_LAYER_EN3); + writel(0x0, adp->be + ADBE_LAYER_EN4); + writel(ADBE_LAYER_CTL_ENABLE, adp->be + ADBE_LAYER_CTL); +} + +static const struct drm_plane_helper_funcs adp_plane_helper_funcs = { + .atomic_check = adp_plane_atomic_check, + .atomic_update = adp_plane_atomic_update, + .atomic_disable = adp_plane_atomic_disable, + DRM_GEM_SHADOW_PLANE_HELPER_FUNCS +}; + +static const struct drm_plane_funcs adp_plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + DRM_GEM_SHADOW_PLANE_FUNCS +}; + +static const u32 plane_formats[] = { + DRM_FORMAT_XRGB8888, +}; + +#define ALL_CRTCS 1 + +struct adp_plane *adp_plane_new(struct adp_drv_private *adp, u8 id) +{ + struct drm_device *drm = &adp->drm; + struct adp_plane *plane; + enum drm_plane_type plane_type; + + plane_type = (id == 0) ? DRM_PLANE_TYPE_PRIMARY : + DRM_PLANE_TYPE_OVERLAY; + + plane = drmm_universal_plane_alloc(drm, struct adp_plane, base_plane, + ALL_CRTCS, &adp_plane_funcs, + plane_formats, ARRAY_SIZE(plane_formats), + NULL, plane_type, "plane %d", id); + if (!plane) { + drm_err(drm, "failed to allocate plane"); + return ERR_PTR(-ENOMEM); + } + plane->id = id; + + drm_plane_helper_add(&plane->base_plane, &adp_plane_helper_funcs); + return plane; +} + +static void adp_enable_vblank(struct adp_drv_private *adp) +{ + u32 cur_ctrl; + + writel(ADP_INT_STATUS_INT_MASK, adp->fe + ADP_INT_STATUS); + + cur_ctrl = readl(adp->fe + ADP_CTRL); + writel(cur_ctrl | ADP_CTRL_VBLANK_ON, adp->fe + ADP_CTRL); +} + +static int adp_crtc_enable_vblank(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct adp_drv_private *adp = to_adp(dev); + adp_enable_vblank(adp); + + return 0; +} + +static void adp_disable_vblank(struct adp_drv_private *adp) +{ + u32 cur_ctrl; + + cur_ctrl = readl(adp->fe + ADP_CTRL); + writel(cur_ctrl & ~ADP_CTRL_VBLANK_ON, adp->fe + ADP_CTRL); + writel(ADP_INT_STATUS_INT_MASK, adp->fe + ADP_INT_STATUS); +} + +static void adp_crtc_disable_vblank(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct adp_drv_private *adp = to_adp(dev); + + adp_disable_vblank(adp); +} + + +static void adp_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct adp_drv_private *adp = crtc_to_adp(crtc); + writel(0x1, adp->be + ADBE_BLEND_EN2); + writel(0x10, adp->be + ADBE_BLEND_EN1); + writel(0x1, adp->be + ADBE_BLEND_EN3); + writel(0x1, adp->be + ADBE_BLEND_BYPASS); + writel(0x1, adp->be + ADBE_BLEND_EN4); +} + +static void adp_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct adp_drv_private *adp = crtc_to_adp(crtc); + struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state, crtc); + + drm_atomic_helper_disable_planes_on_crtc(old_state, false); + + writel(0x0, adp->be + ADBE_BLEND_EN2); + writel(0x0, adp->be + ADBE_BLEND_EN1); + writel(0x0, adp->be + ADBE_BLEND_EN3); + writel(0x0, adp->be + ADBE_BLEND_BYPASS); + writel(0x0, adp->be + ADBE_BLEND_EN4); + drm_crtc_vblank_off(crtc); +} + +static void adp_crtc_atomic_flush(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + u32 frame_num = 1; + struct adp_drv_private *adp = crtc_to_adp(crtc); + struct drm_crtc_state *new_state = drm_atomic_get_new_crtc_state(state, crtc); + u64 new_size = ALIGN(new_state->mode.hdisplay * + new_state->mode.vdisplay * 4, PAGE_SIZE); + + if (new_size != adp->mask_buf_size) { + if (adp->mask_buf) + dma_free_coherent(crtc->dev->dev, adp->mask_buf_size, + adp->mask_buf, adp->mask_iova); + adp->mask_buf = NULL; + if (new_size != 0) { + adp->mask_buf = dma_alloc_coherent(crtc->dev->dev, new_size, + &adp->mask_iova, GFP_KERNEL); + memset(adp->mask_buf, 0xFF, new_size); + writel(adp->mask_iova, adp->be + ADBE_MASK_BUF); + } + adp->mask_buf_size = new_size; + } + writel(ADBE_FIFO_SYNC | frame_num, adp->be + ADBE_FIFO); + //FIXME: use adbe flush interrupt + spin_lock_irq(&crtc->dev->event_lock); + if (crtc->state->event) { + drm_crtc_vblank_get(crtc); + adp->event = crtc->state->event; + } + crtc->state->event = NULL; + spin_unlock_irq(&crtc->dev->event_lock); +} + +static const struct drm_crtc_funcs adp_crtc_funcs = { + .destroy = drm_crtc_cleanup, + .set_config = drm_atomic_helper_set_config, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .enable_vblank = adp_crtc_enable_vblank, + .disable_vblank = adp_crtc_disable_vblank, +}; + + +static const struct drm_crtc_helper_funcs adp_crtc_helper_funcs = { + .atomic_enable = adp_crtc_atomic_enable, + .atomic_disable = adp_crtc_atomic_disable, + .atomic_flush = adp_crtc_atomic_flush, +}; + +static int adp_setup_crtc(struct adp_drv_private *adp) +{ + struct drm_device *drm = &adp->drm; + struct adp_plane *primary; + int ret; + + primary = adp_plane_new(adp, 0); + if (IS_ERR(primary)) + return PTR_ERR(primary); + + ret = drm_crtc_init_with_planes(drm, &adp->crtc, &primary->base_plane, + NULL, &adp_crtc_funcs, NULL); + if (ret) + return ret; + + drm_crtc_helper_add(&adp->crtc, &adp_crtc_helper_funcs); + return 0; +} + +static int adp_get_modes(struct drm_connector *connector) +{ + struct adp_drv_private *adp = conn_to_adp(connector); + struct drm_display_mode *mode; + u32 size; + + size = readl(adp->fe + ADP_SCREEN_SIZE); + mode = drm_mode_create(connector->dev); + + mode->vdisplay = size >> 16; + mode->hdisplay = size & 0xFFFF; + mode->hsync_start = mode->hdisplay + 8; + mode->hsync_end = mode->hsync_start + 80; + mode->htotal = mode->hsync_end + 40; + mode->vsync_start = mode->vdisplay + 1; + mode->vsync_end = mode->vsync_start + 15; + mode->vtotal = mode->vsync_end + 6; + mode->clock = (mode->vtotal * mode->htotal * 60) / 1000; + mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED; + mode->flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC; + drm_mode_set_name(mode); + drm_mode_probed_add(connector, mode); + return 1; +} + +int adp_detect_ctx(struct drm_connector *connector, + struct drm_modeset_acquire_ctx *ctx, + bool force) { + connector->display_info.non_desktop = true; + drm_object_property_set_value(&connector->base, + connector->dev->mode_config.non_desktop_property, + connector->display_info.non_desktop); + return connector_status_connected; +} + +static const struct drm_connector_funcs adp_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = drm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, +}; + +static const struct drm_connector_helper_funcs adp_connector_helper_funcs = { + .get_modes = adp_get_modes, + .detect_ctx = adp_detect_ctx, +}; + +static const struct drm_mode_config_funcs adp_mode_config_funcs = { + .fb_create = drm_gem_fb_create_with_dirty, + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, +}; + +static int adp_setup_mode_config(struct adp_drv_private *adp) +{ + struct drm_device *drm = &adp->drm; + int ret; + + ret = drmm_mode_config_init(drm); + if (ret) + return ret; + + drm->mode_config.min_width = 32; + drm->mode_config.min_height = 32; + drm->mode_config.max_width = 16384; + drm->mode_config.max_height = 16384; + drm->mode_config.preferred_depth = 24; + drm->mode_config.prefer_shadow = 0; + drm->mode_config.funcs = &adp_mode_config_funcs; + + ret = adp_setup_crtc(adp); + if (ret) { + drm_err(drm, "failed to create crtc"); + return ret; + } + + adp->encoder.possible_crtcs = ALL_CRTCS; + ret = drm_simple_encoder_init(drm, &adp->encoder, DRM_MODE_ENCODER_DSI); + if (ret) { + drm_err(drm, "failed to init encoder"); + return ret; + } + drm_connector_helper_add(&adp->connector, + &adp_connector_helper_funcs); + ret = drm_connector_init(drm, &adp->connector, &adp_connector_funcs, + DRM_MODE_CONNECTOR_DSI); + if (ret) + return ret; + + drm_connector_attach_encoder(&adp->connector, &adp->encoder); + + ret = drm_vblank_init(drm, drm->mode_config.num_crtc); + if (ret < 0) { + drm_err(drm, "failed to initialize vblank"); + return ret; + } + + drm_mode_config_reset(drm); + + return 0; +} + +static int adp_parse_of(struct platform_device *pdev, struct adp_drv_private *adp) +{ + adp->be = devm_platform_ioremap_resource_byname(pdev, "be"); + if (IS_ERR(adp->be)) { + dev_err(&pdev->dev, "failed to map display backend mmio"); + return PTR_ERR(adp->be); + } + + adp->fe = devm_platform_ioremap_resource_byname(pdev, "fe"); + if (IS_ERR(adp->fe)) { + dev_err(&pdev->dev, "failed to map display pipe mmio"); + return PTR_ERR(adp->fe); + } + + adp->mipi = devm_platform_ioremap_resource_byname(pdev, "mipi"); + if (IS_ERR(adp->mipi)) { + dev_err(&pdev->dev, "failed to map mipi mmio"); + return PTR_ERR(adp->mipi); + } + + adp->be_irq = platform_get_irq_byname(pdev, "be"); + if (adp->be_irq < 0) { + dev_err(&pdev->dev, "failed to find be irq"); + return adp->be_irq; + } + + adp->fe_irq = platform_get_irq_byname(pdev, "fe"); + if (adp->fe_irq < 0) { + dev_err(&pdev->dev, "failed to find fe irq"); + return adp->fe_irq; + } + return 0; +} + + +static int adp_dsi_gen_pkt_hdr_write(struct adp_drv_private *adp, u32 hdr_val) +{ + int ret; + u32 val, mask; + + ret = readl_poll_timeout(adp->mipi + DSI_CMD_PKT_STATUS, + val, !(val & GEN_CMD_FULL), 1000, + CMD_PKT_STATUS_TIMEOUT_US); + if (ret) { + dev_err(adp->drm.dev, "failed to get available command FIFO\n"); + return ret; + } + + writel(hdr_val, adp->mipi + DSI_GEN_HDR); + + mask = GEN_CMD_EMPTY | GEN_PLD_W_EMPTY; + ret = readl_poll_timeout(adp->mipi + DSI_CMD_PKT_STATUS, + val, (val & mask) == mask, + 1000, CMD_PKT_STATUS_TIMEOUT_US); + if (ret) { + dev_err(adp->drm.dev, "failed to write command FIFO\n"); + return ret; + } + + return 0; +} + +static int adp_dsi_write(struct adp_drv_private *adp, + const struct mipi_dsi_packet *packet) +{ + const u8 *tx_buf = packet->payload; + int len = packet->payload_length, pld_data_bytes = sizeof(u32), ret; + __le32 word; + u32 val; + + while (len) { + if (len < pld_data_bytes) { + word = 0; + memcpy(&word, tx_buf, len); + writel(le32_to_cpu(word), adp->mipi + DSI_GEN_PLD_DATA); + len = 0; + } else { + memcpy(&word, tx_buf, pld_data_bytes); + writel(le32_to_cpu(word), adp->mipi + DSI_GEN_PLD_DATA); + tx_buf += pld_data_bytes; + len -= pld_data_bytes; + } + + ret = readl_poll_timeout(adp->mipi + DSI_CMD_PKT_STATUS, + val, !(val & GEN_PLD_W_FULL), 1000, + CMD_PKT_STATUS_TIMEOUT_US); + if (ret) { + dev_err(adp->drm.dev, + "failed to get available write payload FIFO\n"); + return ret; + } + } + + word = 0; + memcpy(&word, packet->header, sizeof(packet->header)); + return adp_dsi_gen_pkt_hdr_write(adp, le32_to_cpu(word)); +} + +static int adp_dsi_read(struct adp_drv_private *adp, + const struct mipi_dsi_msg *msg) +{ + int i, j, ret, len = msg->rx_len; + u8 *buf = msg->rx_buf; + u32 val; + + /* Wait end of the read operation */ + ret = readl_poll_timeout(adp->mipi + DSI_CMD_PKT_STATUS, + val, !(val & GEN_RD_CMD_BUSY), + 1000, CMD_PKT_STATUS_TIMEOUT_US); + if (ret) { + dev_err(adp->drm.dev, "Timeout during read operation\n"); + return ret; + } + + for (i = 0; i < len; i += 4) { + /* Read fifo must not be empty before all bytes are read */ + ret = readl_poll_timeout(adp->mipi + DSI_CMD_PKT_STATUS, + val, !(val & GEN_PLD_R_EMPTY), + 1000, CMD_PKT_STATUS_TIMEOUT_US); + if (ret) { + dev_err(adp->drm.dev, "Read payload FIFO is empty\n"); + return ret; + } + + val = readl(adp->mipi + DSI_GEN_PLD_DATA); + for (j = 0; j < 4 && j + i < len; j++) + buf[i + j] = val >> (8 * j); + } + + return ret; +} + +static ssize_t adp_dsi_host_transfer(struct mipi_dsi_host *host, + const struct mipi_dsi_msg *msg) +{ + struct adp_drv_private *adp = mipi_to_adp(host); + struct mipi_dsi_packet packet; + int ret, nb_bytes; + + ret = mipi_dsi_create_packet(&packet, msg); + if (ret) { + dev_err(adp->drm.dev, "failed to create packet: %d\n", ret); + return ret; + } + + ret = adp_dsi_write(adp, &packet); + if (ret) + return ret; + + if (msg->rx_buf && msg->rx_len) { + ret = adp_dsi_read(adp, msg); + if (ret) + return ret; + nb_bytes = msg->rx_len; + } else { + nb_bytes = packet.size; + } + + return nb_bytes; +} + +static int adp_dsi_host_attach(struct mipi_dsi_host *host, + struct mipi_dsi_device *dev) +{ + return 0; +} + +static int adp_dsi_host_detach(struct mipi_dsi_host *host, + struct mipi_dsi_device *dev) +{ + return 0; +} + +static const struct mipi_dsi_host_ops adp_dsi_host_ops = { + .transfer = adp_dsi_host_transfer, + .attach = adp_dsi_host_attach, + .detach = adp_dsi_host_detach, +}; + +static irqreturn_t adp_fe_irq(int irq, void *arg) +{ + struct adp_drv_private *adp = (struct adp_drv_private *)arg; + u32 int_status; + u32 int_ctl; + + spin_lock(&adp->irq_lock); + + int_status = readl(adp->fe + ADP_INT_STATUS); + if (int_status & ADP_INT_STATUS_VBLANK) { + drm_crtc_handle_vblank(&adp->crtc); + spin_lock(&adp->crtc.dev->event_lock); + if (adp->event) { + int_ctl = readl(adp->fe + ADP_CTRL); + if ((int_ctl & 0xF00) == 0x600) { + drm_crtc_send_vblank_event(&adp->crtc, adp->event); + adp->event = NULL; + drm_crtc_vblank_put(&adp->crtc); + } + } + spin_unlock(&adp->crtc.dev->event_lock); + } + + writel(int_status, adp->fe + ADP_INT_STATUS); + + spin_unlock(&adp->irq_lock); + + return IRQ_HANDLED; +} + +static int adp_probe(struct platform_device *pdev) +{ + struct adp_drv_private *adp; + int err; + + adp = devm_drm_dev_alloc(&pdev->dev, &adp_driver, struct adp_drv_private, drm); + if (IS_ERR(adp)) + return PTR_ERR(adp); + + spin_lock_init(&adp->irq_lock); + + dev_set_drvdata(&pdev->dev, &adp->drm); + + err = adp_parse_of(pdev, adp); + if (err < 0) + return err; + + adp->dsi.dev = &pdev->dev; + adp->dsi.ops = &adp_dsi_host_ops; + err = mipi_dsi_host_register(&adp->dsi); + if (err < 0) + return err; + + adp_disable_vblank(adp); + writel(ADP_CTRL_FIFO_ON | ADP_CTRL_VBLANK_ON, adp->fe + ADP_CTRL); + + err = adp_setup_mode_config(adp); + if (err < 0) + return err; + + err = devm_request_irq(&pdev->dev, adp->fe_irq, adp_fe_irq, 0, + "adp-fe", adp); + if (err) + return err; + + err = drm_dev_register(&adp->drm, 0); + if (err) + return err; + return 0; +} + +static int adp_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct drm_device *drm = dev_get_drvdata(dev); + struct adp_drv_private *adp = to_adp(drm); + + adp_disable_vblank(adp); + mipi_dsi_host_unregister(&adp->dsi); + drm_dev_unregister(drm); + dev_set_drvdata(dev, NULL); + drm_atomic_helper_shutdown(drm); + return 0; +} + +static const struct of_device_id adp_of_match[] = { + { .compatible = "apple,h7-display-pipe", }, + { }, +}; +MODULE_DEVICE_TABLE(of, adp_of_match); + +static struct platform_driver adp_platform_driver = { + .driver = { + .name = "adp", + .of_match_table = adp_of_match, + }, + .probe = adp_probe, + .remove = adp_remove, +}; + +module_platform_driver(adp_platform_driver); + +MODULE_DESCRIPTION("Apple Display Pipe DRM driver"); +MODULE_LICENSE("GPL v2"); From fec303e3e98c52c14b41c1742cbfe22cb519e663 Mon Sep 17 00:00:00 2001 From: Sasha Finkelstein Date: Tue, 18 Apr 2023 23:03:15 +0300 Subject: [PATCH 630/635] gpu: drm: adp: Add a backlight driver for the Summit LCD This is the display panel used for the touchbar on laptops that have it. Signed-off-by: Sasha Finkelstein --- drivers/gpu/drm/adp/panel-summit.c | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 drivers/gpu/drm/adp/panel-summit.c diff --git a/drivers/gpu/drm/adp/panel-summit.c b/drivers/gpu/drm/adp/panel-summit.c new file mode 100644 index 00000000000000..03d46565a6f606 --- /dev/null +++ b/drivers/gpu/drm/adp/panel-summit.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include