-
Notifications
You must be signed in to change notification settings - Fork 653
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
util: Use enhanced s390 PCI instructions
In the existing MMIO implementation s390 relies on special syscalls to access PCI memory. This was necessary as s390 originally only had special privileged instructions for accessing PCI memory. With z15 however comes support for 4 new PCI memory I/O (MIO) instructions which operate on virtually mapped PCI memory spaces. While these are still special PCI access instructions instead of real MMIO they behave much more like standard MMIO access. There is a load like instruction pcilgi, a store like instruction pcistgi a block store variant for efficient memcpy pcistbi and a write barrier instruction pciwb. The load and store variants always operate on a 64 bit register but only load/store the right most bytes of the register controlled by a length value in a paired register (even numbered register rN + odd numbered register r(N+1)). As the previous PCI instructions did not operate on virtual memory mappings at all a kernel using them does not setup virtual memory mappings and thus can't support user-space using the new instructions. Also as use of PCI MIO instructions can be disabled via the pci=nomio kernel parameter we can't solely rely on hardware support and kernel version. Instead Linux exposes whether PCI MIO instructions are enabled via an ELF hardware capability. With this patch we check for this capability and if enabled use the newly introduced PCI MIO instructions for MMIO access and barriers. Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
- Loading branch information
Showing
7 changed files
with
233 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #ifndef _FIXUP_SYS_AUXV_H | ||
| #define _FIXUP_SYS_AUXV_H | ||
| #if defined(__s390x__) | ||
|
|
||
| #include_next <sys/auxv.h> | ||
|
|
||
| #define HWCAP_S390_PCI_MIO 2097152 | ||
|
|
||
| #endif | ||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| /* GPLv2 or OpenIB.org BSD (MIT) See COPYING file */ | ||
| #ifndef __S390_UTIL_MMIO_H | ||
| #define __S390_UTIL_MMIO_H | ||
| #ifdef __s390x__ | ||
| #include <stdbool.h> | ||
| #include <stdint.h> | ||
| #include <endian.h> | ||
| #include <unistd.h> | ||
| #include <sys/syscall.h> | ||
| #include <sys/auxv.h> | ||
|
|
||
| #include <util/compiler.h> | ||
|
|
||
| /* s390 requires special instructions to access IO memory. Originally there | ||
| were only privileged IO instructions that are exposed via special syscalls. | ||
| Starting with z15 there are also non-privileged memory IO (MIO) instructions | ||
| we can execute in user-space. Despite the hardware support this requires | ||
| support in the kernel. If MIO instructions are available is indicated in an | ||
| ELF hardware capability. | ||
| */ | ||
| extern bool s390_is_mio_supported; | ||
|
|
||
| union register_pair { | ||
| unsigned __int128 pair; | ||
| struct { | ||
| uint64_t even; | ||
| uint64_t odd; | ||
| }; | ||
| }; | ||
|
|
||
| /* The following pcilgi and pcistgi instructions allow IO memory access from | ||
| user-space but are only available on z15 and newer. | ||
| */ | ||
| static inline uint64_t s390_pcilgi(const void *ioaddr, size_t len) | ||
| { | ||
| union register_pair ioaddr_len = {.even = (uint64_t)ioaddr, .odd = len}; | ||
| uint64_t val; | ||
| int cc; | ||
|
|
||
| asm volatile ( | ||
| /* pcilgi */ | ||
| ".insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" | ||
| "ipm %[cc]\n" | ||
| "srl %[cc],28\n" | ||
| : [cc] "=d" (cc), [val] "=d" (val), | ||
| [ioaddr_len] "+&d" (ioaddr_len.pair) :: "cc"); | ||
| if (unlikely(cc)) | ||
| val = -1ULL; | ||
|
|
||
| return val; | ||
| } | ||
|
|
||
| static inline void s390_pcistgi(void *ioaddr, uint64_t val, size_t len) | ||
| { | ||
| union register_pair ioaddr_len = {.even = (uint64_t)ioaddr, .odd = len}; | ||
|
|
||
| asm volatile ( | ||
| /* pcistgi */ | ||
| ".insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" | ||
| : [ioaddr_len] "+&d" (ioaddr_len.pair) | ||
| : [val] "d" (val) | ||
| : "cc", "memory"); | ||
| } | ||
|
|
||
| /* This is the block store variant of unprivileged IO access instructions */ | ||
| static inline void s390_pcistbi(void *ioaddr, const void *data, size_t len) | ||
| { | ||
| const uint8_t *src = data; | ||
|
|
||
| asm volatile ( | ||
| /* pcistbi */ | ||
| ".insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" | ||
| : [len] "+d" (len) | ||
| : [ioaddr] "d" ((uint64_t *)ioaddr), | ||
| [src] "Q" (*src) | ||
| : "cc"); | ||
| } | ||
|
|
||
| static inline void s390_pciwb(void) | ||
| { | ||
| if (s390_is_mio_supported) | ||
| asm volatile (".insn rre,0xb9d50000,0,0\n"); /* pciwb */ | ||
| else | ||
| asm volatile("" ::: "memory"); | ||
| } | ||
|
|
||
| static inline void s390_mmio_write_syscall(void *mmio_addr, const void *val, | ||
| size_t length) | ||
| { | ||
| syscall(__NR_s390_pci_mmio_write, mmio_addr, val, length); | ||
| } | ||
|
|
||
| static inline void s390_mmio_read_syscall(const void *mmio_addr, void *val, | ||
| size_t length) | ||
| { | ||
| syscall(__NR_s390_pci_mmio_read, mmio_addr, val, length); | ||
| } | ||
|
|
||
| #endif /* __s390x__ */ | ||
| #endif /* __S390_UTIL_MMIO_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters