From 4b94dc3092d5ea11e512b1d5653eeb5b16509206 Mon Sep 17 00:00:00 2001
From: chao an <anchao@xiaomi.com>
Date: Thu, 20 Jul 2023 17:07:55 +0800
Subject: [PATCH] toolchain/gcc: fix linker error if enable STACK_CANARIES/LTO
 at same time

If -fstack-protector-all is enabled, gcc linker will need GCC
SSP(Stack Smashing Protector) support, Since the implement of SSP
is related to the OS, most of embedded toolchain does not provide
ssp support, so an error will be reported when linking:

enable CONFIG_LTO_FULL && CONFIG_STACK_CANARIES

arm-none-eabi/bin/ld: cannot find -lssp_nonshared: No such file or directory
arm-none-eabi/bin/ld: cannot find -lssp: No such file or directory

https://github.com/gcc-mirror/gcc/blob/master/gcc/gcc.cc#L983-L985

Since nuttx has already implemented SSP related hook functions,
so in this PR, we filter out this option in the link phase to ensure that
the implementation of lssp/lssp_nonshared will not be referenced

Signed-off-by: chao an <anchao@xiaomi.com>
---
 arch/arm/src/Makefile           | 10 +++++++++-
 arch/arm64/src/Makefile         | 10 +++++++++-
 arch/risc-v/src/Makefile        | 10 +++++++++-
 arch/xtensa/src/Makefile        | 10 +++++++++-
 libs/libc/assert/CMakeLists.txt |  3 +++
 libs/libc/assert/Make.defs      |  3 +++
 6 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/arch/arm/src/Makefile b/arch/arm/src/Makefile
index 1a4b5c272f5..48500803129 100644
--- a/arch/arm/src/Makefile
+++ b/arch/arm/src/Makefile
@@ -105,10 +105,18 @@ EXTRA_LIBS += $(wildcard $(APPDIR)$(DELIM)staging$(DELIM)*$(LIBEXT))
 
 ifeq ($(CONFIG_ARM_TOOLCHAIN_ARMCLANG),)
   ifeq ($(LD),$(CC))
+    ifeq ($(CONFIG_STACK_CANARIES),y)
+      # filter out ssp(Stack Smashing Protector) related flags:
+      #  -fstack-protector
+      #  -fstack-protector-all
+      #  -fstack-protector-strong
+      #  -fstack-protector-explicit
+      STRIPCFLAGS = $(filter -fstack-protector%,$(CFLAGS))
+    endif
     LDSTARTGROUP ?= -Wl,--start-group
     LDENDGROUP   ?= -Wl,--end-group
     LDFLAGS      := $(addprefix -Xlinker ,$(LDFLAGS))
-    LDFLAGS      += $(CFLAGS)
+    LDFLAGS      += $(filter-out $(STRIPCFLAGS),$(CFLAGS))
   else
     LDSTARTGROUP ?= --start-group
     LDENDGROUP   ?= --end-group
diff --git a/arch/arm64/src/Makefile b/arch/arm64/src/Makefile
index 35fa194f2e5..cebecfe6af5 100644
--- a/arch/arm64/src/Makefile
+++ b/arch/arm64/src/Makefile
@@ -87,10 +87,18 @@ EXTRA_LIBS += $(wildcard $(APPDIR)$(DELIM)staging$(DELIM)*$(LIBEXT))
 # Override in Make.defs if linker is not 'ld'
 
 ifeq ($(LD),$(CC))
+  ifeq ($(CONFIG_STACK_CANARIES),y)
+    # filter out ssp(Stack Smashing Protector) related flags:
+    #  -fstack-protector
+    #  -fstack-protector-all
+    #  -fstack-protector-strong
+    #  -fstack-protector-explicit
+    STRIPCFLAGS = $(filter -fstack-protector%,$(CFLAGS))
+  endif
   LDSTARTGROUP ?= -Wl,--start-group
   LDENDGROUP   ?= -Wl,--end-group
   LDFLAGS      := $(addprefix -Xlinker ,$(LDFLAGS))
-  LDFLAGS      += $(CFLAGS)
+  LDFLAGS      += $(filter-out $(STRIPCFLAGS),$(CFLAGS))
 else
   LDSTARTGROUP ?= --start-group
   LDENDGROUP   ?= --end-group
diff --git a/arch/risc-v/src/Makefile b/arch/risc-v/src/Makefile
index 4118775cf72..0b23888be0e 100644
--- a/arch/risc-v/src/Makefile
+++ b/arch/risc-v/src/Makefile
@@ -99,10 +99,18 @@ ARCHSCRIPT := $(call CONVERT_PATH,$(ARCHSCRIPT))
 LDFLAGS += $(addprefix -T,$(addsuffix .tmp,$(ARCHSCRIPT))) $(EXTRALINKCMDS)
 
 ifeq ($(LD),$(CC))
+  ifeq ($(CONFIG_STACK_CANARIES),y)
+    # filter out ssp(Stack Smashing Protector) related flags:
+    #  -fstack-protector
+    #  -fstack-protector-all
+    #  -fstack-protector-strong
+    #  -fstack-protector-explicit
+    STRIPCFLAGS = $(filter -fstack-protector%,$(CFLAGS))
+  endif
   LDSTARTGROUP ?= -Wl,--start-group
   LDENDGROUP   ?= -Wl,--end-group
   LDFLAGS      := $(addprefix -Xlinker ,$(LDFLAGS))
-  LDFLAGS      += $(CFLAGS)
+  LDFLAGS      += $(filter-out $(STRIPCFLAGS),$(CFLAGS))
 else
   LDSTARTGROUP ?= --start-group
   LDENDGROUP   ?= --end-group
diff --git a/arch/xtensa/src/Makefile b/arch/xtensa/src/Makefile
index 132475b1468..3ca6f2336d2 100644
--- a/arch/xtensa/src/Makefile
+++ b/arch/xtensa/src/Makefile
@@ -98,10 +98,18 @@ EXTRA_LIBS := $(filter-out $(NAMEFULL_LIBS) $(NAMESPEC_LIBS),$(EXTRA_LIBS))
 EXTRA_LIBS += $(wildcard $(APPDIR)$(DELIM)staging$(DELIM)*$(LIBEXT))
 
 ifeq ($(LD),$(CC))
+  ifeq ($(CONFIG_STACK_CANARIES),y)
+    # filter out ssp(Stack Smashing Protector) related flags:
+    #  -fstack-protector
+    #  -fstack-protector-all
+    #  -fstack-protector-strong
+    #  -fstack-protector-explicit
+    STRIPCFLAGS = $(filter -fstack-protector%,$(CFLAGS))
+  endif
   LDSTARTGROUP ?= -Wl,--start-group
   LDENDGROUP   ?= -Wl,--end-group
   LDFLAGS      := $(addprefix -Xlinker ,$(LDFLAGS))
-  LDFLAGS      += $(CFLAGS)
+  LDFLAGS      += $(filter-out $(STRIPCFLAGS),$(CFLAGS))
 else
   LDSTARTGROUP ?= --start-group
   LDENDGROUP   ?= --end-group
diff --git a/libs/libc/assert/CMakeLists.txt b/libs/libc/assert/CMakeLists.txt
index 790509c04f1..09c4b50b220 100644
--- a/libs/libc/assert/CMakeLists.txt
+++ b/libs/libc/assert/CMakeLists.txt
@@ -24,4 +24,7 @@ if(CONFIG_STACK_CANARIES)
   list(APPEND SRCS lib_stackchk.c)
 endif()
 
+set_source_files_properties(lib_assert.c PROPERTIES COMPILE_FLAGS -fno-lto)
+set_source_files_properties(lib_stackchk.c PROPERTIES COMPILE_FLAGS -fno-lto)
+
 target_sources(c PRIVATE ${SRCS})
diff --git a/libs/libc/assert/Make.defs b/libs/libc/assert/Make.defs
index dfe45759d41..830a50c05fa 100644
--- a/libs/libc/assert/Make.defs
+++ b/libs/libc/assert/Make.defs
@@ -24,6 +24,9 @@ ifeq ($(CONFIG_STACK_CANARIES),y)
 CSRCS += lib_stackchk.c
 endif
 
+assert/lib_assert.c_CFLAGS += -fno-lto
+assert/lib_stackchk.c_CFLAGS += -fno-lto
+
 # Add the assert directory to the build
 
 DEPPATH += --dep-path assert