CI: Improvement to speed up compilation and reduce download errors.

The simple improvement is designed to speed up compilation and reduce download errors on github and local.

Added a folder nxtmpdir for storing third-party packages

nuttxworkspace
|
|- nuttx
|- apps
|- nxtmpdir

tools/Unix.mk:
added export NXTMPDIR := $(WSDIR)/nxtmpdir

tools/configure.sh:
added option -S creates the nxtmpdir folder for third-party packages.

tools/Config.mk:
added macro
CLONE - Git clone repository.
CHECK_COMMITSHA - Check if the branch contains the commit SHA-1.

tools/testbuild.sh:
added option -S

For now I added in the folder this package

ESP_HAL_3RDPARTY_URL = https://github.com/espressif/esp-hal-3rdparty.git

ARCH
arch/xtensa/src/esp32/Make.defs
arch/xtensa/src/esp32s2/Make.defs
arch/xtensa/src/esp32s3/Make.defs
arch/risc-v/src/common/espressif/Make.defs
arch/risc-v/src/esp32c3-legacy/Make.defs

but you can also add other packages (maybe also of apps)

tools/Config.mk

@@ -560,6 +560,18 @@ define COPYFILE
 endef
 endif
 
+# COPYDIR - Copy one directory
+
+ifeq ($(CONFIG_WINDOWS_NATIVE),y)
+define COPYDIR
+	$(Q) if exist $1 (xcopy /c /q /s /e /y /i $1 $2)
+endef
+else
+define COPYDIR
+	$(Q) cp -fr $1 $2
+endef
+endif
+
 # CATFILE - Cat a list of files
 #
 # USAGE: $(call CATFILE,dest,src1,src2,src3,...)
@@ -608,6 +620,43 @@ define DOWNLOAD
 	$(ECHO_END)
 endef
 
+# CLONE - Git clone repository. Initializes a new Git repository in the 
+#         folder on your local machine and populates it with the contents
+#         of the central repository.
+#         The third argument is an storage path. The second argument is used
+#         if it is not provided or is empty.
+# Example: $(call CLONE,$(URL_BASE),$(PATH),$(STORAGE_FOLDER))
+
+define CLONE
+	$(ECHO_BEGIN)"Clone: $(if $3,$3,$2) "
+	if [ -z $3 ]; then \
+		git clone --quiet $1 $2; \
+	else \
+		if [ ! -d $3 ]; then \
+			git clone --quiet $1 $3; \
+		fi; \
+		cp -fr $3 $2; \
+	fi
+	$(ECHO_END)
+endef
+
+# CHECK_COMMITSHA - Check if the branch contains the commit SHA-1.
+#         Remove the folder if the commit is not present in the branch.
+#         The first argument is the repository folder on the local machine.
+#         The second argument is a unique SHA-1 hash value.
+# Example: $(call CHECK_COMMITSHA,$(GIT_FOLDER),$(COMMIT_SHA-1))
+
+define CHECK_COMMITSHA
+	$(ECHO_BEGIN)"COMMIT SHA-1: $2 "
+	if [ -d $1 ]; then \
+		if ! git -C $1 branch --contains $2 > /dev/null 2>&1; then \
+			echo "Commit is not present removed folder $1 "; \
+			rm -rf $1; \
+		fi \
+	fi
+	$(ECHO_END)
+endef
+
 # CLEAN - Default clean target
 
 ifeq ($(CONFIG_ARCH_COVERAGE),y)

tools/Unix.mk

@@ -21,6 +21,9 @@
 ############################################################################
 
 export TOPDIR := ${shell echo $(CURDIR) | sed -e 's/ /\\ /g'}
+WSDIR := ${shell cd "${TOPDIR}"/.. && pwd -P}
+
+export NXTMPDIR := $(WSDIR)/nxtmpdir
 
 ifeq ($(V),)
   MAKE := $(MAKE) -s --no-print-directory

tools/configure.sh

@@ -23,14 +23,16 @@ set -e
 
 WD=`test -d ${0%/*} && cd ${0%/*}; pwd`
 TOPDIR="${WD}/.."
+WSDIR=`cd "${TOPDIR}/.." && pwd -P`
 MAKECMD="make"
 USAGE="
 
-USAGE: ${0} [-E] [-e] [-l|m|c|g|n|B] [-L [boardname]] [-a <app-dir>] <board-selection> [make-opts]
+USAGE: ${0} [-E] [-e] [-S] [-l|m|c|g|n|B] [-L [boardname]] [-a <app-dir>] <board-selection> [make-opts]
 
 Where:
   -E enforces distclean if already configured.
   -e performs distclean if configuration changed.
+  -S adds the nxtmpdir folder for third-party packages.
   -l selects the Linux (l) host environment.
   -m selects the macOS (m) host environment.
   -c selects the Windows host and Cygwin (c) environment.
@@ -70,6 +72,7 @@ unset appdir
 unset host
 unset enforce_distclean
 unset distclean
+unset store_nxtmpdir
 
 function dumpcfgs
 {
@@ -122,6 +125,9 @@ while [ ! -z "$1" ]; do
     dumpcfgs $1
     exit 0
     ;;
+  -S )
+    store_nxtmpdir=y
+    ;;
   *)
     boardconfig=$1
     shift
@@ -217,6 +223,19 @@ if [ -r ${dest_config} ]; then
   fi
 fi
 
+if [ "X${store_nxtmpdir}" = "Xy" ]; then
+  if [ ! -d "${WSDIR}/nxtmpdir" ]; then
+    mkdir -p "${WSDIR}/nxtmpdir"
+    echo "Folder ${WSDIR}/nxtmpdir created."
+  fi
+else
+  if [ -d "${WSDIR}/nxtmpdir" ]; then
+    rm -rf "${WSDIR}/nxtmpdir"
+    echo "Folder ${WSDIR}/nxtmpdir clean."
+  fi
+fi
+
+
 # Okay... Everything looks good.  Setup the configuration
 
 echo "  Copy files"
@@ -289,8 +308,8 @@ if [ -z "${appdir}" ]; then
     if [ -d "${TOPDIR}/../apps-${CONFIG_VERSION_STRING}" ]; then
       appdir="../apps-${CONFIG_VERSION_STRING}"
     else
-        echo "ERROR: Could not find the path to the appdir"
-        exit 7
+      echo "ERROR: Could not find the path to the appdir"
+      exit 7
     fi
   fi
 fi

tools/testbuild.sh

@@ -33,6 +33,7 @@ EXTRA_FLAGS="EXTRAFLAGS="
 MAKE=make
 unset testfile
 unset HOPTION
+unset STORE
 unset JOPTION
 PRINTLISTONLY=0
 GITCLEAN=0
@@ -65,7 +66,7 @@ esac
 function showusage {
   echo ""
   echo "USAGE: $progname -h [-l|m|c|g|n] [-d] [-e <extraflags>] [-x] [-j <ncpus>] [-a <appsdir>] [-t <topdir>] [-p]"
-  echo "       [-A] [-C] [-G] [-N] [-R] [--codechecker] <testlist-file>"
+  echo "       [-A] [-C] [-G] [-N] [-R] [-S] [--codechecker] <testlist-file>"
   echo ""
   echo "Where:"
   echo "  -h will show this help test and terminate"
@@ -88,6 +89,7 @@ function showusage {
   echo "         as well."
   echo "  -N Use CMake with Ninja as the backend."
   echo "  -R execute \"run\" script in the config directories if exists."
+  echo "  -S Adds the nxtmpdir folder for third-party packages."
   echo "  --codechecker enables CodeChecker statically analyze the code."
   echo "  <testlist-file> selects the list of configurations to test.  No default"
   echo ""
@@ -145,6 +147,9 @@ while [ ! -z "$1" ]; do
   -R )
     RUN=1
     ;;
+  -S )
+    STORE+=" $1"
+    ;;
   --codechecker )
     CODECHECKER=1
     ;;
@@ -314,7 +319,7 @@ function distclean {
 # Configure for the next build
 
 function configure_default {
-  if ! ./tools/configure.sh ${HOPTION} $config ${JOPTION} 1>/dev/null; then
+  if ! ./tools/configure.sh ${HOPTION} ${STORE} $config ${JOPTION} 1>/dev/null; then
     fail=1
   fi