6.18: PCIe GPU testing (AMD and Intel Xe)

arch/arm64/Kconfig

@@ -1696,6 +1696,32 @@ config ARM64_TAGGED_ADDR_ABI
 	  to system calls as pointer arguments. For details, see
 	  Documentation/arch/arm64/tagged-address-abi.rst.
 
+config ARM64_FORCE_PCIE_MMIO_DEVICE_MAPPINGS
+	bool "Force Device memory mappings for PCIe MMIO space"
+	default y
+	help
+	  PCIe device drivers may map MMIO space as Normal non-cacheable,
+	  for the purpose of enabling write combining or unaligned accesses.
+
+	  On many platforms (e.g. Ampere Altra, RK35xx), the PCIe interface
+	  cannot support unaligned outbound transactions. This may lead to
+	  data corruption, for instance, when a regular memcpy is performed by
+	  an application on a GPU's VRAM BAR.
+
+	  This option forces all software that maps PCIe MMIO space as Normal
+	  non-cacheable memory to use Device-nGnRE instead. If the strict alignment
+	  is not met, the CPU will raise alignment faults that can be further
+	  handled by the kernel by enabling CONFIG_ARM64_ALIGNMENT_FIXUPS.
+
+config ARM64_ALIGNMENT_FIXUPS
+	bool "Fix up misaligned multi-word loads and stores in 64-bit kernel/user space"
+	default y
+	help
+	  This option enables kernel/user space code to perform unaligned accesses
+	  to memory regions that do not normally support them (e.g. device mappings)
+	  by trapping alignment faults on common load/store instructions and breaking
+	  up the offending accesses into properly aligned ones.
+
 menuconfig COMPAT
 	bool "Kernel support for 32-bit EL0"
 	depends on ARM64_4K_PAGES || EXPERT

arch/arm64/configs/bcm2711_defconfig

@@ -1070,6 +1070,15 @@ CONFIG_AUXDISPLAY=y
 CONFIG_HD44780=m
 CONFIG_DRM=m
 CONFIG_DRM_LOAD_EDID_FIRMWARE=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_NOUVEAU=m
+CONFIG_NOUVEAU_DEBUG_MMU=y
+CONFIG_NOUVEAU_DEBUG_PUSH=y
+CONFIG_DRM_XE=m
+CONFIG_DRM_XE_FORCE_PROBE="*"
 CONFIG_DRM_UDL=m
 CONFIG_DRM_PANEL_LVDS=m
 CONFIG_DRM_PANEL_ILITEK_IL79600A=m
@@ -1133,6 +1142,9 @@ CONFIG_SND_SERIAL_U16550=m
 CONFIG_SND_MPU401=m
 CONFIG_SND_PIMIDI=m
 CONFIG_SND_PISOUND_MICRO=m
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_GENERIC=m
+CONFIG_SND_HDA_CODEC_HDMI=m
 CONFIG_SND_USB_AUDIO=m
 CONFIG_SND_USB_UA101=m
 CONFIG_SND_USB_CAIAQ=m

arch/arm64/configs/bcm2712_defconfig

@@ -1072,6 +1072,12 @@ CONFIG_AUXDISPLAY=y
 CONFIG_HD44780=m
 CONFIG_DRM=m
 CONFIG_DRM_LOAD_EDID_FIRMWARE=y
+CONFIG_DRM_RADEON=m
+CONFIG_DRM_AMDGPU=m
+CONFIG_DRM_AMDGPU_SI=y
+CONFIG_DRM_AMDGPU_CIK=y
+CONFIG_DRM_XE=m
+CONFIG_DRM_XE_FORCE_PROBE="*"
 CONFIG_DRM_UDL=m
 CONFIG_DRM_PANEL_LVDS=m
 CONFIG_DRM_PANEL_ILITEK_IL79600A=m
@@ -1135,6 +1141,9 @@ CONFIG_SND_SERIAL_U16550=m
 CONFIG_SND_MPU401=m
 CONFIG_SND_PIMIDI=m
 CONFIG_SND_PISOUND_MICRO=m
+CONFIG_SND_HDA_INTEL=m
+CONFIG_SND_HDA_GENERIC=m
+CONFIG_SND_HDA_CODEC_HDMI=m
 CONFIG_SND_USB_AUDIO=m
 CONFIG_SND_USB_UA101=m
 CONFIG_SND_USB_CAIAQ=m

arch/arm64/include/asm/exception.h

@@ -82,6 +82,7 @@ void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs)
 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
 void do_el0_cp15(unsigned long esr, struct pt_regs *regs);
 int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
+int do_alignment_fixup(unsigned long addr, unsigned int esr, struct pt_regs *regs);
 void do_el0_svc(struct pt_regs *regs);
 void do_el0_svc_compat(struct pt_regs *regs);
 void do_el0_fpac(struct pt_regs *regs, unsigned long esr);

arch/arm64/include/asm/pgtable.h

@@ -338,11 +338,6 @@ static inline pte_t pte_mkyoung(pte_t pte)
 	return set_pte_bit(pte, __pgprot(PTE_AF));
 }
 
-static inline pte_t pte_mkspecial(pte_t pte)
-{
-	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
-}
-
 static inline pte_t pte_mkcont(pte_t pte)
 {
 	return set_pte_bit(pte, __pgprot(PTE_CONT));
@@ -802,6 +797,21 @@ static inline void __set_puds(struct mm_struct *mm,
 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
 			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
 
+extern bool range_is_pci(phys_addr_t phys_addr, size_t size);
+
+static inline pte_t pte_mkspecial(pte_t pte)
+{
+#ifdef CONFIG_ARM64_FORCE_PCIE_MMIO_DEVICE_MAPPINGS
+	phys_addr_t phys = __pte_to_phys(pte);
+	pgprot_t prot = __pgprot(pte_val(pte) & ~__phys_to_pte_val(__pte_to_phys(__pte(~0ull))));
+
+	if ((pgprot_val(prot) != pgprot_val(pgprot_device(prot))) &&
+	    range_is_pci(phys, PAGE_SIZE))
+		pte = __pte(__phys_to_pte_val(phys) | pgprot_val(pgprot_device(prot)));
+#endif
+	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
+}
+
 #define __HAVE_PHYS_MEM_ACCESS_PROT
 struct file;
 extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,

arch/arm64/kernel/Makefile

@@ -36,6 +36,7 @@ obj-y			:= debug-monitors.o entry.o irq.o fpsimd.o		\
 			   syscall.o proton-pack.o idle.o patching.o pi/	\
 			   rsi.o jump_label.o
 
+obj-$(CONFIG_ARM64_ALIGNMENT_FIXUPS)	+= alignment.o
 obj-$(CONFIG_COMPAT)			+= sys32.o signal32.o			\
 					   sys_compat.o
 obj-$(CONFIG_COMPAT)			+= sigreturn32.o