diff --git a/payloads/Kconfig b/payloads/Kconfig
index c7a7ba6e1d..d0f8a44080 100644
--- a/payloads/Kconfig
+++ b/payloads/Kconfig
@@ -100,6 +100,7 @@ config PAYLOAD_FIT_SUPPORT
 	bool "FIT support"
 	default n
 	default y if PAYLOAD_LINUX && (ARCH_ARM || ARCH_ARM64)
+	depends on ARCH_ARM64
 	select FLATTENED_DEVICE_TREE
 	help
 	  Select this option if your payload is of type FIT.