Refactor UTF-8 code to common static library.

.gitignore

@@ -111,14 +111,14 @@ test/lib/c/*.test
 test/lib/cpp/*.test
 
 test/unit/broker/bridge_topic_test
-test/unit/coverage.info
 test/unit/broker/keepalive_test
-test/unit/lib/lib_test
 test/unit/broker/persist_read_test
 test/unit/broker/persist_write_test
 test/unit/broker/subs_test
+test/unit/coverage.info
+test/unit/lib/lib_test
+test/unit/libcommon/libcommon_test
 test/unit/tls_test
-test/unit/out/
 
 www/cache/
 __pycache__

CMakeLists.txt

@@ -134,6 +134,7 @@ if(WITH_TLS)
 endif()
 
 
+add_subdirectory(libcommon)
 add_subdirectory(lib)
 if(WITH_CLIENTS)
 	add_subdirectory(client)

Makefile

@@ -1,6 +1,6 @@
 include config.mk
 
-DIRS=lib apps client plugins src
+DIRS=libcommon lib apps client plugins src
 DOCDIRS=man
 DISTDIRS=man
 DISTFILES= \

apps/db_dump/CMakeLists.txt

@@ -6,7 +6,6 @@ add_executable(mosquitto_db_dump
 	../../lib/memory_mosq.c
 	../../lib/packet_datatypes.c
 	../../lib/property_mosq.c
-	../../common/utf8_mosq.c
 
 	../../src/memory_public.c
 	../../src/persist_read.c
@@ -39,6 +38,8 @@ if(WITH_TLS)
 	target_link_libraries(mosquitto_db_dump PRIVATE config-header OpenSSL::SSL)
 endif()
 
+target_link_libraries(mosquitto_db_dump PRIVATE libmosquitto_common)
+
 install(TARGETS mosquitto_db_dump
 	RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
 )

apps/db_dump/Makefile

@@ -4,7 +4,7 @@ include ${R}/config.mk
 LOCAL_CFLAGS+=
 LOCAL_CPPFLAGS+=-I${R}/lib -I${R}/src -I${R}/common -DWITH_BROKER
 LOCAL_LDFLAGS+=
-LOCAL_LDADD+=-lcjson -lm
+LOCAL_LDADD+=-lcjson -lm ${LIBMOSQ_COMMON}
 
 # ------------------------------------------
 #  Compile time options
@@ -28,8 +28,7 @@ BROKER_OBJS = \
 	${R}/src/persist_read_v234.o \
 	${R}/src/persist_read_v5.o \
 	${R}/src/property_mosq.o \
-	${R}/src/topic_tok.o \
-	${R}/src/utf8_mosq.o
+	${R}/src/topic_tok.o
 
 .PHONY: all clean reallyclean
 

apps/mosquitto_passwd/Makefile

@@ -4,7 +4,7 @@ include ${R}/config.mk
 LOCAL_CFLAGS+=
 LOCAL_CPPFLAGS+=-I${R}/lib
 LOCAL_LDFLAGS+=
-LOCAL_LDADD+=-lcrypto ${LIB_ARGON2}
+LOCAL_LDADD+=-lcrypto ${LIB_ARGON2} ${LIBMOSQ_COMMON}
 
 .PHONY: all install uninstall clean reallyclean
 
@@ -17,8 +17,7 @@ OBJS_EXTERNAL= \
 	memory_mosq.o \
 	memory_public.o \
 	misc_mosq.o \
-	password_mosq.o \
-	utf8_mosq.o
+	password_mosq.o
 
 
 ifeq ($(WITH_TLS),yes)
@@ -55,9 +54,6 @@ misc_mosq.o : ${R}/common/misc_mosq.c ${R}/common/misc_mosq.h
 password_mosq.o : ${R}/common/password_mosq.c ${R}/common/password_mosq.h
 	${CROSS_COMPILE}${CC} ${LOCAL_CPPFLAGS} $(LOCAL_CFLAGS) -c $< -o $@
 
-utf8_mosq.o : ${R}/common/utf8_mosq.c
-	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
-
 install : all
 ifeq ($(WITH_TLS),yes)
 	$(INSTALL) -d "${DESTDIR}$(prefix)/bin"

config.mk

@@ -221,6 +221,7 @@ ifeq ($(WITH_SHARED_LIBRARIES),yes)
 else
 	LIBMOSQ:=${R}/lib/libmosquitto.a
 endif
+LIBMOSQ_COMMON:=-Wl,--whole-archive ${R}/libcommon/libmosquitto_common.a -Wl,--no-whole-archive
 
 ifeq ($(WITH_TLS),yes)
 	LOCAL_CPPFLAGS+=-DWITH_TLS

lib/CMakeLists.txt

@@ -50,7 +50,6 @@ set(C_SRC
 	thread_mosq.c
 	../common/time_mosq.c ../common/time_mosq.h
 	tls_mosq.c
-	../common/utf8_mosq.c
 	util_mosq.c util_topic.c util_mosq.h
 	will_mosq.c will_mosq.h)
 
@@ -58,7 +57,7 @@ if (WITH_THREADING AND WIN32)
 	list(APPEND C_SRC "../common/winthread_mosq.c" "../common/winthread_mosq.h")
 endif()
 
-set(LIBRARIES common-options)
+set(LIBRARIES common-options libmosquitto_common)
 
 if(WITH_TLS)
 	set (LIBRARIES ${LIBRARIES} OpenSSL::SSL)

lib/Makefile

@@ -4,7 +4,7 @@ include ${R}/config.mk
 LOCAL_CFLAGS+=-fPIC
 LOCAL_CPPFLAGS+=
 LOCAL_LDFLAGS+=-Wl,--version-script=linker.version -Wl,-soname,libmosquitto.so.$(SOVERSION) -fPIC -shared
-LOCAL_LIBADD+=-lcjson -lc ${LIB_ARGON2}
+LOCAL_LIBADD+=-lcjson -lc ${LIB_ARGON2} ${LIBMOSQ_COMMON}
 STATIC_LIB_DEPS:=
 
 # ------------------------------------------
@@ -89,8 +89,7 @@ OBJS_EXTERNAL= \
 	base64_mosq.o \
 	misc_mosq.o \
 	password_mosq.o \
-	time_mosq.o \
-	utf8_mosq.o
+	time_mosq.o
 
 ifeq ($(WITH_WEBSOCKETS),yes)
 	OBJS_EXTERNAL+=${R}/deps/picohttpparser/picohttpparser.o
@@ -160,8 +159,5 @@ password_mosq.o : ${R}/common/password_mosq.c net_mosq.h
 time_mosq.o : ${R}/common/time_mosq.c ${R}/common/time_mosq.h
 	${CROSS_COMPILE}$(CC) $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
 
-utf8_mosq.o : ${R}/common/utf8_mosq.c
-	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
-
 ${R}/deps/picohttpparser/picohttpparser.o : ${R}/deps/picohttpparser/picohttpparser.c
 	${CROSS_COMPILE}$(CC) $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@

libcommon/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(C_SRC
+	utf8_common.c
+)
+
+add_library(libmosquitto_common OBJECT
+	${C_SRC}
+)
+
+target_include_directories(libmosquitto_common
+	PUBLIC
+		"${mosquitto_SOURCE_DIR}/"
+		"${mosquitto_SOURCE_DIR}/include"
+)
+
+set_target_properties(libmosquitto_common PROPERTIES
+	OUTPUT_NAME mosquitto_common
+	VERSION ${VERSION}
+	SOVERSION 1
+	POSITION_INDEPENDENT_CODE 1
+)

libcommon/Makefile

@@ -0,0 +1,35 @@
+R=..
+include ${R}/config.mk
+
+LOCAL_CFLAGS+=-fPIC
+LOCAL_CPPFLAGS+=
+LOCAL_LDFLAGS+=-fPIC
+LOCAL_LIBADD+=
+
+# ------------------------------------------
+#  Targets
+# ------------------------------------------
+.PHONY : really clean install
+
+OBJS= \
+	utf8_common.o
+
+all : libmosquitto_common.a
+
+install :
+
+uninstall :
+
+reallyclean : clean
+
+clean :
+	-rm -f ${OBJS} libmosquitto_common.so.${SOVERSION} libmosquitto_common.so libmosquitto_common.a *.gcno *.gcda
+
+libmosquitto_common.so.${SOVERSION} : ${OBJS}
+	${CROSS_COMPILE}$(CC) $(LOCAL_LDFLAGS) $^ -o $@ ${LOCAL_LIBADD}
+
+libmosquitto_common.a : ${OBJS}
+	${CROSS_COMPILE}$(AR) cr $@ $^
+
+${OBJS} : %.o: %.c
+	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@

src/CMakeLists.txt

@@ -77,7 +77,6 @@ set (MOSQ_SRCS
 	../lib/tls_mosq.c
 	topic_tok.c
 	../lib/util_mosq.c ../lib/util_topic.c ../lib/util_mosq.h
-	../common/utf8_mosq.c
 	websockets.c
 	will_delay.c
 	../lib/will_mosq.c ../lib/will_mosq.h
@@ -248,6 +247,7 @@ endif()
 target_link_libraries(mosquitto
 	PUBLIC
 		config-header
+		libmosquitto_common
 	PRIVATE
 		common-options
 		${MOSQ_LIBS}

src/Makefile

@@ -6,7 +6,7 @@ include ${R}/config.mk
 LOCAL_CFLAGS+=
 LOCAL_CPPFLAGS+=-DWITH_BROKER -I${R}/lib
 LOCAL_LDFLAGS+=
-LOCAL_LDADD+=-lcjson -lm ${LIB_ARGON2}
+LOCAL_LDADD+=-lcjson -lm ${LIB_ARGON2} ${LIBMOSQ_COMMON}
 
 # ------------------------------------------
 #  Platform specific
@@ -139,7 +139,6 @@ OBJS_EXTERNAL= \
 		strings_mosq.o \
 		time_mosq.o \
 		tls_mosq.o \
-		utf8_mosq.o \
 		util_mosq.o \
 		util_topic.o \
 		will_mosq.o
@@ -247,9 +246,6 @@ util_mosq.o : ${R}/lib/util_mosq.c ${R}/lib/util_mosq.h
 util_topic.o : ${R}/lib/util_topic.c ${R}/lib/util_mosq.h
 	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
 
-utf8_mosq.o : ${R}/common/utf8_mosq.c
-	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
-
 will_mosq.o : ${R}/lib/will_mosq.c ${R}/lib/will_mosq.h
 	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
 

test/unit/CMakeLists.txt

@@ -17,5 +17,6 @@ target_link_libraries(common-unit-test-header
         CUnit::CUnit
 )
 
+add_subdirectory(libcommon)
 add_subdirectory(broker)
 add_subdirectory(lib)

test/unit/Makefile

@@ -6,6 +6,7 @@ include ${R}/config.mk
 all :
 
 test-compile:
+	$(MAKE) -C libcommon $@
 	$(MAKE) -C broker $@
 	$(MAKE) -C lib $@
 
@@ -13,10 +14,12 @@ check : test
 
 ptest : test
 test : test-compile
+	$(MAKE) -C libcommon $@
 	$(MAKE) -C broker $@
 	$(MAKE) -C lib $@
 
 reallyclean : clean
 clean :
+	$(MAKE) -C libcommon $@
 	$(MAKE) -C broker $@
 	$(MAKE) -C lib $@

test/unit/broker/CMakeLists.txt

@@ -43,7 +43,6 @@ add_library(persistence-read-obj
         ../../../src/persist_read.c
         ../../../src/retain.c
         ../../../src/topic_tok.c
-        ../../../common/utf8_mosq.c
 )
 target_compile_definitions(persistence-read-obj PRIVATE WITH_PERSISTENCE WITH_BROKER)
 target_link_libraries(persistence-read-obj PUBLIC common-unit-test-header OpenSSL::SSL)
@@ -56,7 +55,7 @@ add_executable(persist-read-test
     ../../../lib/util_mosq.c
 )
 target_compile_definitions(persist-read-test PRIVATE TEST_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" WITH_PERSISTENCE WITH_BROKER)
-target_link_libraries(persist-read-test PRIVATE persistence-read-obj)
+target_link_libraries(persist-read-test PRIVATE persistence-read-obj libmosquitto_common)
 add_test(NAME unit-persist-read-test COMMAND persist-read-test)
 
 # persist-write-test
@@ -74,7 +73,6 @@ add_library(persistence-write-obj
         ../../../src/retain.c
         ../../../src/subs.c
         ../../../src/topic_tok.c
-        ../../../common/utf8_mosq.c
 )
 target_compile_definitions(persistence-write-obj PRIVATE WITH_PERSISTENCE WITH_BROKER)
 target_link_libraries(persistence-write-obj PUBLIC common-unit-test-header)
@@ -88,7 +86,7 @@ add_executable(persist-write-test
     ../../../lib/packet_mosq.c
 )
 target_compile_definitions(persist-write-test PRIVATE TEST_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" WITH_PERSISTENCE WITH_BROKER WITH_SYS_TREE)
-target_link_libraries(persist-write-test PRIVATE persistence-write-obj OpenSSL::SSL)
+target_link_libraries(persist-write-test PRIVATE persistence-write-obj OpenSSL::SSL libmosquitto_common)
 add_test(NAME unit-persist-write-test COMMAND persist-write-test)
 
 # subs-test
@@ -101,7 +99,6 @@ add_library(subs-obj
         ../../../src/database.c
         ../../../src/subs.c
         ../../../src/topic_tok.c
-        ../../../common/utf8_mosq.c
 )
 target_compile_definitions(subs-obj PRIVATE WITH_BROKER)
 target_link_libraries(subs-obj PUBLIC common-unit-test-header)
@@ -112,5 +109,5 @@ add_executable(subs-test
 )
 
 target_compile_definitions(subs-test PRIVATE WITH_PERSISTENCE WITH_BROKER WITH_SYS_TREE)
-target_link_libraries(subs-test PRIVATE common-unit-test-header subs-obj)
+target_link_libraries(subs-test PRIVATE common-unit-test-header subs-obj libmosquitto_common)
 add_test(NAME unit-subs-test COMMAND subs-test)

test/unit/broker/Makefile

@@ -7,7 +7,7 @@ include ${R}/make/broker.mk
 LOCAL_CFLAGS+=-coverage
 LOCAL_CPPFLAGS+=-DWITH_BROKER -I${R}/src -I${R}/test -I${R}/lib -DTEST_SOURCE_DIR='"$(realpath .)"' -I${R}/lib
 LOCAL_LDFLAGS+=-coverage
-LOCAL_LDADD+=-lcunit
+LOCAL_LDADD+=-lcunit ${LIBMOSQ_COMMON}
 
 ifeq ($(WITH_TLS),yes)
 	LOCAL_LDADD+=-lssl -lcrypto
@@ -22,8 +22,7 @@ BRIDGE_TOPIC_OBJS = \
 		${R}/src/memory_mosq.o \
 		${R}/src/packet_datatypes.o \
 		${R}/src/property_mosq.o \
-		${R}/src/util_topic.o \
-		${R}/src/utf8_mosq.o
+		${R}/src/util_topic.o
 
 KEEPALIVE_TEST_OBJS = \
 		keepalive_stubs.o \
@@ -47,8 +46,7 @@ PERSIST_READ_OBJS = \
 		${R}/src/property_mosq.o \
 		${R}/src/retain.o \
 		${R}/src/topic_tok.o \
-		${R}/src/util_mosq.o \
-		${R}/src/utf8_mosq.o
+		${R}/src/util_mosq.o
 
 PERSIST_WRITE_TEST_OBJS = \
 		persist_write_test.o \
@@ -70,8 +68,7 @@ PERSIST_WRITE_OBJS = \
 		${R}/src/retain.o \
 		${R}/src/subs.o \
 		${R}/src/topic_tok.o \
-		${R}/src/util_mosq.o \
-		${R}/src/utf8_mosq.o
+		${R}/src/util_mosq.o
 
 SUBS_TEST_OBJS = \
 		subs_test.o \
@@ -84,8 +81,7 @@ SUBS_OBJS = \
 		${R}/src/packet_datatypes.o \
 		${R}/src/property_mosq.o \
 		${R}/src/subs.o \
-		${R}/src/topic_tok.o \
-		${R}/src/utf8_mosq.o
+		${R}/src/topic_tok.o
 
 all : test-compile
 
@@ -177,9 +173,6 @@ ${R}/src/util_mosq.o : ${R}/lib/util_mosq.c
 ${R}/src/util_topic.o : ${R}/lib/util_topic.c
 	$(MAKE) -C ${R}/src/ util_topic.o
 
-${R}/src/utf8_mosq.o : ${R}/common/utf8_mosq.c
-	$(MAKE) -C ${R}/src/ utf8_mosq.o
-
 build : bridge_topic_test keepalive_test persist_read_test persist_write_test subs_test
 
 test : build

test/unit/lib/CMakeLists.txt

@@ -9,7 +9,6 @@ add_executable(lib-test
 	property_value.c
 	strings_test.c
 	stubs.c
-	utf8.c
 	util_topic_test.c
 	# main test files
 	test.c
@@ -21,8 +20,7 @@ add_executable(lib-test
 	../../../lib/strings_mosq.c
 	../../../lib/util_mosq.c
 	../../../lib/util_topic.c
-	../../../common/utf8_mosq.c
 )
 
-target_link_libraries(lib-test PRIVATE common-unit-test-header OpenSSL::SSL)
+target_link_libraries(lib-test PRIVATE common-unit-test-header OpenSSL::SSL libmosquitto_common)
 add_test(NAME unit-lib-test COMMAND lib-test)

test/unit/lib/Makefile

@@ -6,7 +6,7 @@ include ${R}/config.mk
 LOCAL_CFLAGS+=-coverage
 LOCAL_CPPFLAGS+=-I${R}/src -I${R}/test -I${R}/lib -DTEST_SOURCE_DIR='"$(realpath .)"'
 LOCAL_LDFLAGS+=-coverage
-LOCAL_LDADD+=-lcunit
+LOCAL_LDADD+=-lcunit ${LIBMOSQ_COMMON}
 
 ifeq ($(WITH_TLS),yes)
 	LOCAL_LDADD+=-lssl -lcrypto
@@ -24,7 +24,6 @@ TEST_OBJS = \
 	property_value.o \
 	strings_test.o \
 	stubs.o \
-	utf8.o \
 	util_topic_test.o
 
 LIB_OBJS = \
@@ -35,8 +34,7 @@ LIB_OBJS = \
 	${R}/lib/property_mosq.o \
 	${R}/lib/strings_mosq.o \
 	${R}/lib/util_mosq.o \
-	${R}/lib/util_topic.o \
-	${R}/lib/utf8_mosq.o
+	${R}/lib/util_topic.o
 
 
 all : test-compile

test/unit/lib/test.c

@@ -12,7 +12,6 @@ int init_property_user_read_tests(void);
 int init_property_write_tests(void);
 int init_property_value_tests(void);
 int init_strings_tests(void);
-int init_utf8_tests(void);
 int init_util_topic_tests(void);
 int init_misc_trim_tests(void);
 
@@ -29,7 +28,6 @@ int main(int argc, char *argv[])
     }
 
     if(0
-			|| init_utf8_tests()
 			|| init_datatype_read_tests()
 			|| init_datatype_write_tests()
 			|| init_property_add_tests()

test/unit/libcommon/CMakeLists.txt

@@ -0,0 +1,12 @@
+add_executable(libcommon-test
+	utf8.c
+	test.c
+)
+
+target_link_libraries(libcommon-test
+	PRIVATE
+		common-unit-test-header
+		OpenSSL::SSL
+		libmosquitto_common
+)
+add_test(NAME unit-libcommon-test COMMAND libcommon-test)

test/unit/libcommon/Makefile

@@ -0,0 +1,44 @@
+R=../../..
+include ${R}/config.mk
+
+.PHONY: all check test test-compile clean coverage
+
+LOCAL_CFLAGS+=-coverage
+LOCAL_CPPFLAGS+=-I${R}/src -I${R}/test -I${R}/lib -DTEST_SOURCE_DIR='"$(realpath .)"'
+LOCAL_LDFLAGS+=-coverage
+LOCAL_LDADD+=-lcunit ${LIBMOSQ_COMMON}
+
+ifeq ($(WITH_TLS),yes)
+	LOCAL_LDADD+=-lssl -lcrypto
+endif
+
+TEST_OBJS = \
+	test.o \
+	utf8.o \
+
+LIB_OBJS =
+
+
+all : test-compile
+
+check : test
+
+libcommon_test : ${TEST_OBJS} ${LIB_OBJS}
+	$(CROSS_COMPILE)$(CC) $(LOCAL_LDFLAGS) -o $@ $^ $(LOCAL_LDADD)
+
+${TEST_OBJS} : %.o: %.c
+	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
+
+lib_stubs.o : stubs.c
+	${CROSS_COMPILE}$(CC) $(LIB_LOCAL_CPPFLAGS) $(LIB_LOCAL_CFLAGS) $(CFLAGS) $(CPPFLAGS) -c $< -o $@
+
+build : libcommon_test
+
+test : build
+	./libcommon_test
+
+test-compile: build
+
+clean :
+	-rm -rf libcommon_test
+	-rm -rf *.o *.gcda *.gcno coverage.info

test/unit/libcommon/test.c

@@ -0,0 +1,36 @@
+#include "config.h"
+#include <stdio.h>
+
+#include <CUnit/CUnit.h>
+#include <CUnit/Basic.h>
+
+int init_utf8_tests(void);
+
+int main(int argc, char *argv[])
+{
+	unsigned int fails;
+
+	UNUSED(argc);
+	UNUSED(argv);
+
+    if(CU_initialize_registry() != CUE_SUCCESS){
+        printf("Error initializing CUnit registry.\n");
+        return 1;
+    }
+
+    if(0
+			|| init_utf8_tests()
+			){
+
+        CU_cleanup_registry();
+        return 1;
+    }
+
+    CU_basic_set_mode(CU_BRM_NORMAL);
+    CU_basic_run_tests();
+	fails = CU_get_number_of_failures();
+    CU_cleanup_registry();
+
+    return (int)fails;
+}
+