x86/mtrr: Enable Rd/WrDram mod in AMD fixed MTRRs

AMD's fixed MTRRs have RdDram and WrDram bits that route memory
accesses to DRAM vs. MMIO.  These are typically hidden for normal
operation by clearing SYS_CFG[19] (MtrrFixDramModEn).  According to
BKDGs and AMD64 Programmer's Manual vol 2, this bit is clear at
reset, should be set for configuration during POST, then cleared for
normal operation.

Attempting to modify the RdDram and WrDram settings without unhiding
them causes a General Protection Fault.  Add functions to enable and
disable MtrrFixDramModEn.  Unhide/hide as necessary when copying or
writing the fixed MTRRs.

Finally, modify sipi_vector.S to enable the bits prior to writing
the fixed MTRRs and disable when complete.

This functionality is compiled out on non-AMD platforms.

BUG=b:68019051
TEST=Boot Kahlee, check steps with HDT

Change-Id: Ie195131ff752400eb886dfccc39b314b4fa6b3f3
Signed-off-by: Marshall Dawson <marshalldawson3rd@gmail.com>
Reviewed-on: https://review.coreboot.org/23722
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Aaron Durbin <adurbin@chromium.org>

src/cpu/x86/mp_init.c

@@ -299,6 +299,8 @@ static int save_bsp_msrs(char *start, int size)
 		return -1;
 	}
 
+	fixed_mtrrs_expose_amd_rwdram();
+
 	msr_entry = (void *)start;
 	for (i = 0; i < NUM_FIXED_MTRRS; i++)
 		msr_entry = save_msr(fixed_mtrrs[i], msr_entry);
@@ -310,6 +312,8 @@ static int save_bsp_msrs(char *start, int size)
 
 	msr_entry = save_msr(MTRR_DEF_TYPE_MSR, msr_entry);
 
+	fixed_mtrrs_hide_amd_rwdram();
+
 	return msr_count;
 }
 

src/cpu/x86/mtrr/mtrr.c

@@ -36,8 +36,8 @@
 #include <arch/cpu.h>
 #include <arch/acpi.h>
 #include <memrange.h>
-#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
 #include <cpu/amd/mtrr.h>
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
 #define MTRR_FIXED_WRBACK_BITS (MTRR_READ_MEM | MTRR_WRITE_MEM)
 #else
 #define MTRR_FIXED_WRBACK_BITS 0
@@ -83,6 +83,30 @@ void enable_fixed_mtrr(void)
 	wrmsr(MTRR_DEF_TYPE_MSR, msr);
 }
 
+void fixed_mtrrs_expose_amd_rwdram(void)
+{
+	msr_t syscfg;
+
+	if (!IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS))
+		return;
+
+	syscfg = rdmsr(SYSCFG_MSR);
+	syscfg.lo |= SYSCFG_MSR_MtrrFixDramModEn;
+	wrmsr(SYSCFG_MSR, syscfg);
+}
+
+void fixed_mtrrs_hide_amd_rwdram(void)
+{
+	msr_t syscfg;
+
+	if (!IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS))
+		return;
+
+	syscfg = rdmsr(SYSCFG_MSR);
+	syscfg.lo &= ~SYSCFG_MSR_MtrrFixDramModEn;
+	wrmsr(SYSCFG_MSR, syscfg);
+}
+
 static void enable_var_mtrr(unsigned char deftype)
 {
 	msr_t msr;
@@ -310,6 +334,8 @@ static void commit_fixed_mtrrs(void)
 	msr_t fixed_msrs[NUM_FIXED_MTRRS];
 	unsigned long msr_index[NUM_FIXED_MTRRS];
 
+	fixed_mtrrs_expose_amd_rwdram();
+
 	memset(&fixed_msrs, 0, sizeof(fixed_msrs));
 
 	msr_num = 0;
@@ -351,6 +377,8 @@ static void commit_fixed_mtrrs(void)
 	for (i = 0; i < ARRAY_SIZE(fixed_msrs); i++)
 		wrmsr(msr_index[i], fixed_msrs[i]);
 	enable_cache();
+	fixed_mtrrs_hide_amd_rwdram();
+
 }
 
 void x86_setup_fixed_mtrrs_no_enable(void)

src/cpu/x86/sipi_vector.S

@@ -15,6 +15,7 @@
  */
 
 #include <cpu/x86/cr.h>
+#include <cpu/amd/mtrr.h>
 
 /* The SIPI vector is responsible for initializing the APs in the sytem. It
  * loads microcode, sets up MSRs, and enables caching before calling into
@@ -172,6 +173,15 @@ microcode_done:
 	mov	msr_count, %ebx
 	test	%ebx, %ebx
 	jz	1f
+
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
+	/* Allow modification of RdDram and WrDram bits */
+	mov	$SYSCFG_MSR, %ecx
+	rdmsr
+	or	$SYSCFG_MSR_MtrrFixDramModEn, %eax
+	wrmsr
+#endif
+
 load_msr:
 	mov	(%edi), %ecx
 	mov	4(%edi), %eax
@@ -181,6 +191,13 @@ load_msr:
 	dec	%ebx
 	jnz	load_msr
 
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
+	mov	$SYSCFG_MSR, %ecx
+	rdmsr
+	and	$~SYSCFG_MSR_MtrrFixDramModEn, %eax
+	wrmsr
+#endif
+
 1:
 	/* Enable caching. */
 	mov	%cr0, %eax

src/include/cpu/x86/mtrr.h

@@ -76,6 +76,10 @@ void x86_setup_mtrrs_with_detect(void);
  */
 void x86_setup_var_mtrrs(unsigned int address_bits, unsigned int above4gb);
 void enable_fixed_mtrr(void);
+/* Unhide Rd/WrDram bits and allow modification for AMD. */
+void fixed_mtrrs_expose_amd_rwdram(void);
+/* Hide Rd/WrDram bits and allow modification for AMD. */
+void fixed_mtrrs_hide_amd_rwdram(void);
 void x86_setup_fixed_mtrrs(void);
 /* Set up fixed MTRRs but do not enable them. */
 void x86_setup_fixed_mtrrs_no_enable(void);