diff --git a/src/cpu/x86/64bit/Makefile.inc b/src/cpu/x86/64bit/Makefile.inc
new file mode 100644
index 0000000000..721e62044d
--- /dev/null
+++ b/src/cpu/x86/64bit/Makefile.inc
@@ -0,0 +1,7 @@
+bootblock-y += mode_switch.S
+ifneq ($(CONFIG_VBOOT_STARTS_BEFORE_BOOTBLOCK),y)
+verstage-y += mode_switch.S
+endif
+romstage-y += mode_switch.S
+postcar-y += mode_switch.S
+ramstage-y += mode_switch.S
diff --git a/src/cpu/x86/Makefile.inc b/src/cpu/x86/Makefile.inc
index b9705286f8..dffc2b1cb9 100644
--- a/src/cpu/x86/Makefile.inc
+++ b/src/cpu/x86/Makefile.inc
@@ -3,8 +3,8 @@ subdirs-y += mtrr
 subdirs-y += pae
 subdirs-$(CONFIG_HAVE_SMI_HANDLER) += smm
 subdirs-$(CONFIG_UDELAY_TSC) += tsc
-
-all-$(CONFIG_ARCH_ALL_STAGES_X86_64) += 64bit/mode_switch.S
+# Use ARCH_BOOTBLOCK_X86_64 as a proxy for knowing if 64bit is going to be used
+subdirs-$(CONFIG_ARCH_BOOTBLOCK_X86_64) += 64bit
 
 subdirs-$(CONFIG_PARALLEL_MP) += name
 ramstage-$(CONFIG_PARALLEL_MP) += mp_init.c