diff --git a/arch/xtensa/include/irq.h b/arch/xtensa/include/irq.h
index 5dc3e54b6ca..39ae9149f03 100644
--- a/arch/xtensa/include/irq.h
+++ b/arch/xtensa/include/irq.h
@@ -189,6 +189,7 @@ struct xcpt_syscall_s
 
 struct xcptcontext
 {
+#ifdef CONFIG_ENABLE_ALL_SIGNALS
   /* These are saved copies of registers used during signal processing.
    *
    * REVISIT:  Because there is only one copy of these save areas,
@@ -199,10 +200,6 @@ struct xcptcontext
 
   uint32_t *saved_regs;
 
-  /* Register save area */
-
-  uint32_t *regs;
-
 #ifndef CONFIG_BUILD_FLAT
   /* This is the saved address to use when returning from a user-space
    * signal handler.
@@ -210,6 +207,10 @@ struct xcptcontext
 
   uintptr_t sigreturn;
 #endif
+#endif /* CONFIG_ENABLE_ALL_SIGNALS */
+  /* Register save area */
+
+  uint32_t *regs;
 
 #ifdef CONFIG_LIB_SYSCALL
   /* The following array holds the return address and the exc_return value
diff --git a/arch/xtensa/src/common/CMakeLists.txt b/arch/xtensa/src/common/CMakeLists.txt
index 089976c79e6..60042af46f4 100644
--- a/arch/xtensa/src/common/CMakeLists.txt
+++ b/arch/xtensa/src/common/CMakeLists.txt
@@ -69,14 +69,16 @@ list(
   xtensa_perf.c
   xtensa_releasestack.c
   xtensa_registerdump.c
-  xtensa_sigdeliver.c
   xtensa_swint.c
   xtensa_stackframe.c
   xtensa_saveusercontext.c
-  xtensa_schedsigaction.c
   xtensa_usestack.c
   xtensa_tcbinfo.c)
 
+if(CONFIG_ENABLE_ALL_SIGNALS)
+  list(APPEND SRCS xtensa_schedsigaction.c xtensa_sigdeliver.c)
+endif()
+
 # Configuration-dependent common Xtensa files
 
 if(CONFIG_ARCH_USE_TEXT_HEAP)
@@ -106,8 +108,10 @@ endif()
 if(CONFIG_BUILD_PROTECTED)
   target_sources(arch_interface PRIVATE xtensa_signal_handler.S)
   list(APPEND SRCS xtensa_dispatch_syscall.S)
-  list(APPEND SRCS xtensa_signal_dispatch.c xtensa_task_start.c
-       xtensa_pthread_start.c)
+  list(APPEND SRCS xtensa_task_start.c xtensa_pthread_start.c)
+  if(CONFIG_ENABLE_ALL_SIGNALS)
+    list(APPEND SRCS xtensa_signal_dispatch.c)
+  endif()
 endif()
 
 target_sources(arch PRIVATE ${SRCS})
diff --git a/arch/xtensa/src/common/Make.defs b/arch/xtensa/src/common/Make.defs
index 7a6fe24ae9c..3e1ea13a785 100644
--- a/arch/xtensa/src/common/Make.defs
+++ b/arch/xtensa/src/common/Make.defs
@@ -37,13 +37,17 @@ CMN_CSRCS += xtensa_getintstack.c xtensa_initialize.c xtensa_initialstate.c
 CMN_CSRCS += xtensa_irqdispatch.c xtensa_lowputs.c
 CMN_CSRCS += xtensa_modifyreg8.c xtensa_modifyreg16.c xtensa_modifyreg32.c
 CMN_CSRCS += xtensa_mpu.c xtensa_nputs.c xtensa_oneshot.c xtensa_perf.c
-CMN_CSRCS += xtensa_releasestack.c xtensa_registerdump.c xtensa_sigdeliver.c
+CMN_CSRCS += xtensa_releasestack.c xtensa_registerdump.c
 CMN_CSRCS += xtensa_swint.c xtensa_stackframe.c
-CMN_CSRCS += xtensa_saveusercontext.c xtensa_schedsigaction.c
+CMN_CSRCS += xtensa_saveusercontext.c
 CMN_CSRCS += xtensa_usestack.c xtensa_tcbinfo.c
 
 # Configuration-dependent common Xtensa files
 
+ifeq ($(CONFIG_ENABLE_ALL_SIGNALS),y)
+CMN_CSRCS += xtensa_schedsigaction.c xtensa_sigdeliver.c
+endif
+
 ifeq ($(CONFIG_ARCH_USE_TEXT_HEAP),y)
   CMN_ASRCS += xtensa_loadstore.S
 endif
@@ -80,5 +84,8 @@ endif
 ifeq ($(CONFIG_BUILD_PROTECTED),y)
   CMN_UASRCS += xtensa_signal_handler.S
   CMN_ASRCS  += xtensa_dispatch_syscall.S
-  CMN_CSRCS  += xtensa_signal_dispatch.c xtensa_task_start.c xtensa_pthread_start.c
+  CMN_CSRCS  += xtensa_task_start.c xtensa_pthread_start.c
+ifeq ($(CONFIG_ENABLE_ALL_SIGNALS),y)
+  CMN_CSRCS += xtensa_signal_dispatch.c
+endif
 endif
diff --git a/arch/xtensa/src/common/xtensa_int_handlers.S b/arch/xtensa/src/common/xtensa_int_handlers.S
index 3c4073b478f..8e60754e8b4 100644
--- a/arch/xtensa/src/common/xtensa_int_handlers.S
+++ b/arch/xtensa/src/common/xtensa_int_handlers.S
@@ -179,7 +179,7 @@ g_intstacktop:
 	 * signal context (ref: xtensa_schedulesigaction.c):
 	 */
 
-#if CONFIG_ARCH_INTERRUPTSTACK < 15
+#if (CONFIG_ARCH_INTERRUPTSTACK < 15) && defined(CONFIG_ENABLE_ALL_SIGNALS)
 	addi	sp, sp, -XCPTCONTEXT_SIZE
 #endif
 
@@ -266,7 +266,7 @@ g_intstacktop:
 
 	addi	sp, sp, XCHAL_NUM_INTERRUPTS / 8
 
-#if CONFIG_ARCH_INTERRUPTSTACK < 15
+#if (CONFIG_ARCH_INTERRUPTSTACK < 15) && defined(CONFIG_ENABLE_ALL_SIGNALS)
 	addi	sp, sp, XCPTCONTEXT_SIZE
 #endif
 	.endm
diff --git a/arch/xtensa/src/common/xtensa_signal_dispatch.c b/arch/xtensa/src/common/xtensa_signal_dispatch.c
index dd0fdeb2da5..f446738b76c 100644
--- a/arch/xtensa/src/common/xtensa_signal_dispatch.c
+++ b/arch/xtensa/src/common/xtensa_signal_dispatch.c
@@ -30,7 +30,7 @@
 
 #include <arch/syscall.h>
 
-#if !defined(CONFIG_BUILD_FLAT) && defined(__KERNEL__)
+#ifdef __KERNEL__
 
 /****************************************************************************
  * Public Functions
@@ -74,4 +74,4 @@ void up_signal_dispatch(_sa_sigaction_t sighand, int signo,
             (uintptr_t)info, (uintptr_t)ucontext);
 }
 
-#endif /* !CONFIG_BUILD_FLAT && __KERNEL__ */
+#endif /* __KERNEL__ */