diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index ff601ea..15a6233 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -466,12 +466,14 @@ config ARCH_MX7
 	select CPU_V7
 	select SYS_FSL_HAS_SEC if SECURE_BOOT
 	select SYS_FSL_SEC_COMPAT_4
+	select SYS_FSL_SEC_LE
 
 config ARCH_MX6
 	bool "Freescale MX6"
 	select CPU_V7
 	select SYS_FSL_HAS_SEC if SECURE_BOOT
 	select SYS_FSL_SEC_COMPAT_4
+	select SYS_FSL_SEC_LE
 
 config ARCH_MX5
 	bool "Freescale MX5"
diff --git a/arch/arm/cpu/armv7/ls102xa/Kconfig b/arch/arm/cpu/armv7/ls102xa/Kconfig
index e233aa4..d154f7b 100644
--- a/arch/arm/cpu/armv7/ls102xa/Kconfig
+++ b/arch/arm/cpu/armv7/ls102xa/Kconfig
@@ -7,6 +7,7 @@ config ARCH_LS1021A
 	select SYS_FSL_DDR_VER_50
 	select SYS_FSL_HAS_SEC
 	select SYS_FSL_SEC_COMPAT_5
+	select SYS_FSL_SEC_LE
 
 menu "LS102xA architecture"
 	depends on ARCH_LS1021A
diff --git a/arch/arm/cpu/armv8/fsl-layerscape/Kconfig b/arch/arm/cpu/armv8/fsl-layerscape/Kconfig
index 17b470d..a1f781e 100644
--- a/arch/arm/cpu/armv8/fsl-layerscape/Kconfig
+++ b/arch/arm/cpu/armv8/fsl-layerscape/Kconfig
@@ -31,12 +31,14 @@ config ARCH_LS2080A
 	select SYS_FSL_HAS_DP_DDR
 	select SYS_FSL_HAS_SEC
 	select SYS_FSL_SEC_COMPAT_5
+	select SYS_FSL_SEC_LE
 	select SYS_FSL_SRDS_2
 
 config FSL_LSCH2
 	bool
 	select SYS_FSL_HAS_SEC
 	select SYS_FSL_SEC_COMPAT_5
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SRDS_1
 	select SYS_HAS_SERDES
 
diff --git a/arch/arm/include/asm/arch-fsl-layerscape/config.h b/arch/arm/include/asm/arch-fsl-layerscape/config.h
index f4f9eaa..29fc33d 100644
--- a/arch/arm/include/asm/arch-fsl-layerscape/config.h
+++ b/arch/arm/include/asm/arch-fsl-layerscape/config.h
@@ -55,9 +55,6 @@
 #define CONFIG_SYS_FSL_SFP_LE
 #define CONFIG_SYS_FSL_SRK_LE
 
-/* SEC */
-#define CONFIG_SYS_FSL_SEC_LE
-
 /* Security Monitor */
 #define CONFIG_SYS_FSL_SEC_MON_LE
 
@@ -144,7 +141,6 @@
 #define CONFIG_SYS_FSL_QSPI_BE
 #define CONFIG_SYS_FSL_CCSR_GUR_BE
 #define CONFIG_SYS_FSL_PEX_LUT_BE
-#define CONFIG_SYS_FSL_SEC_BE
 
 /* SoC related */
 #ifdef CONFIG_LS1043A
diff --git a/arch/arm/include/asm/arch-ls102xa/config.h b/arch/arm/include/asm/arch-ls102xa/config.h
index 97c69e9..1c5158b 100644
--- a/arch/arm/include/asm/arch-ls102xa/config.h
+++ b/arch/arm/include/asm/arch-ls102xa/config.h
@@ -106,7 +106,6 @@
 #define CONFIG_SYS_FSL_QSPI_BE
 #define CONFIG_SYS_FSL_DCU_BE
 #define CONFIG_SYS_FSL_SEC_MON_LE
-#define CONFIG_SYS_FSL_SEC_LE
 #define CONFIG_SYS_FSL_SFP_VER_3_2
 #define CONFIG_SYS_FSL_SFP_BE
 #define CONFIG_SYS_FSL_SRK_LE
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 9fc1d5c..853e265 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -24,6 +24,7 @@ config MPC83xx
 	bool "MPC83xx"
 	select CREATE_ARCH_SYMLINK
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 
 config MPC85xx
diff --git a/arch/powerpc/cpu/mpc85xx/Kconfig b/arch/powerpc/cpu/mpc85xx/Kconfig
index 1287ab6..f36114a 100644
--- a/arch/powerpc/cpu/mpc85xx/Kconfig
+++ b/arch/powerpc/cpu/mpc85xx/Kconfig
@@ -326,6 +326,7 @@ config ARCH_B4420
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_B4860
@@ -333,18 +334,21 @@ config ARCH_B4860
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_BSC9131
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_BSC9132
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -352,6 +356,7 @@ config ARCH_C29X
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_6
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -359,6 +364,7 @@ config ARCH_MPC8536
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -370,12 +376,14 @@ config ARCH_MPC8541
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 
 config ARCH_MPC8544
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -383,6 +391,7 @@ config ARCH_MPC8548
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -390,6 +399,7 @@ config ARCH_MPC8555
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 
 config ARCH_MPC8560
@@ -400,24 +410,28 @@ config ARCH_MPC8568
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 
 config ARCH_MPC8569
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 
 config ARCH_MPC8572
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 
 config ARCH_P1010
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -425,6 +439,7 @@ config ARCH_P1011
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -432,6 +447,7 @@ config ARCH_P1020
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -439,6 +455,7 @@ config ARCH_P1021
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -446,6 +463,7 @@ config ARCH_P1022
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -453,12 +471,14 @@ config ARCH_P1023
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_P1024
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -466,6 +486,7 @@ config ARCH_P1025
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -473,6 +494,7 @@ config ARCH_P2020
 	bool
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_2
 	select SYS_PPC_E500_USE_DEBUG_TLB
 
@@ -481,6 +503,7 @@ config ARCH_P2041
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_P3041
@@ -488,6 +511,7 @@ config ARCH_P3041
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_P4080
@@ -495,6 +519,7 @@ config ARCH_P4080
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_P5020
@@ -502,6 +527,7 @@ config ARCH_P5020
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_P5040
@@ -509,6 +535,7 @@ config ARCH_P5040
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_QEMU_E500
@@ -519,6 +546,7 @@ config ARCH_T1023
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_5
 
 config ARCH_T1024
@@ -526,6 +554,7 @@ config ARCH_T1024
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_5
 
 config ARCH_T1040
@@ -533,6 +562,7 @@ config ARCH_T1040
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_5
 
 config ARCH_T1042
@@ -540,6 +570,7 @@ config ARCH_T1042
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_5
 
 config ARCH_T2080
@@ -547,6 +578,7 @@ config ARCH_T2080
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_T2081
@@ -554,6 +586,7 @@ config ARCH_T2081
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_T4160
@@ -561,6 +594,7 @@ config ARCH_T4160
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config ARCH_T4240
@@ -568,6 +602,7 @@ config ARCH_T4240
 	select E500MC
 	select FSL_LAW
 	select SYS_FSL_HAS_SEC
+	select SYS_FSL_SEC_BE
 	select SYS_FSL_SEC_COMPAT_4
 
 config BOOKE
diff --git a/arch/powerpc/include/asm/config.h b/arch/powerpc/include/asm/config.h
index 9b7bcb0..d4f05d1 100644
--- a/arch/powerpc/include/asm/config.h
+++ b/arch/powerpc/include/asm/config.h
@@ -67,13 +67,6 @@
 #endif
 #endif
 
-/*
- * SEC (crypto unit) major compatible version determination
- */
-#if defined(CONFIG_MPC83xx)
-#define CONFIG_SYS_FSL_SEC_BE
-#endif
-
 /* Since so many PPC SOCs have a semi-common LBC, define this here */
 #if defined(CONFIG_MPC85xx) || defined(CONFIG_MPC86xx) || \
 	defined(CONFIG_MPC83xx)
diff --git a/arch/powerpc/include/asm/config_mpc85xx.h b/arch/powerpc/include/asm/config_mpc85xx.h
index 7131b61..8bae577 100644
--- a/arch/powerpc/include/asm/config_mpc85xx.h
+++ b/arch/powerpc/include/asm/config_mpc85xx.h
@@ -20,7 +20,6 @@
 
 /* IP endianness */
 #define CONFIG_SYS_FSL_IFC_BE
-#define CONFIG_SYS_FSL_SEC_BE
 #define CONFIG_SYS_FSL_SFP_BE
 #define CONFIG_SYS_FSL_SEC_MON_BE
 
diff --git a/drivers/crypto/fsl/Kconfig b/drivers/crypto/fsl/Kconfig
index 510a108..3188959 100644
--- a/drivers/crypto/fsl/Kconfig
+++ b/drivers/crypto/fsl/Kconfig
@@ -30,6 +30,9 @@ config SYS_FSL_SEC_COMPAT_6
 	help
 		Secure boot and trust architecture compatible version 6
 
+config SYS_FSL_SEC_BE
+	bool "Big-endian access to Freescale Secure Boot"
+
 config SYS_FSL_SEC_COMPAT
 	int "Freescale Secure Boot compatibility"
 	depends on SYS_FSL_HAS_SEC
@@ -37,3 +40,6 @@ config SYS_FSL_SEC_COMPAT
 	default 4 if SYS_FSL_SEC_COMPAT_4
 	default 5 if SYS_FSL_SEC_COMPAT_5
 	default 6 if SYS_FSL_SEC_COMPAT_6
+
+config SYS_FSL_SEC_LE
+	bool "Little-endian access to Freescale Secure Boot"
diff --git a/include/configs/mx6_common.h b/include/configs/mx6_common.h
index 8ee7aaf..74b0a97 100644
--- a/include/configs/mx6_common.h
+++ b/include/configs/mx6_common.h
@@ -96,7 +96,6 @@
 #define CONFIG_CSF_SIZE			0x2000
 #define CONFIG_FSL_CAAM
 #define CONFIG_CMD_DEKBLOB
-#define CONFIG_SYS_FSL_SEC_LE
 #endif
 
 #endif
diff --git a/include/configs/mx7_common.h b/include/configs/mx7_common.h
index bd98925..f111fc5 100644
--- a/include/configs/mx7_common.h
+++ b/include/configs/mx7_common.h
@@ -77,7 +77,6 @@
 #define CONFIG_CSF_SIZE			0x2000
 #define CONFIG_FSL_CAAM
 #define CONFIG_CMD_DEKBLOB
-#define CONFIG_SYS_FSL_SEC_LE
 #endif
 
 #endif
diff --git a/include/fsl_sec.h b/include/fsl_sec.h
index e6080d4..61c671d 100644
--- a/include/fsl_sec.h
+++ b/include/fsl_sec.h
@@ -24,7 +24,7 @@
 #define sec_in16(a)       in_be16(a)
 #define sec_clrbits32     clrbits_be32
 #define sec_setbits32     setbits_be32
-#else
+#elif defined(CONFIG_SYS_FSL_HAS_SEC)
 #error Neither CONFIG_SYS_FSL_SEC_LE nor CONFIG_SYS_FSL_SEC_BE is defined
 #endif
 
diff --git a/scripts/config_whitelist.txt b/scripts/config_whitelist.txt
index fa54921..fcae43a 100644
--- a/scripts/config_whitelist.txt
+++ b/scripts/config_whitelist.txt
@@ -5547,9 +5547,7 @@ CONFIG_SYS_FSL_SCFG_OFFSET
 CONFIG_SYS_FSL_SCFG_PIXCLKCR_OFFSET
 CONFIG_SYS_FSL_SCFG_PIXCLK_ADDR
 CONFIG_SYS_FSL_SEC_ADDR
-CONFIG_SYS_FSL_SEC_BE
 CONFIG_SYS_FSL_SEC_IDX_OFFSET
-CONFIG_SYS_FSL_SEC_LE
 CONFIG_SYS_FSL_SEC_MON_BE
 CONFIG_SYS_FSL_SEC_MON_LE
 CONFIG_SYS_FSL_SEC_OFFSET
