component_information: switch from gzip to xz

Improves compression, e.g. current params file: 62KB to 51KB

There's also a PRESET_EXTREME option, which reduces by another 2KB.
We can revisit that once needed, as it increases mem usage as well.

Jenkinsfile

@@ -112,8 +112,8 @@ pipeline {
             sh 'make distclean'
             sh 'make parameters_metadata'
             dir('build/px4_sitl_default/docs') {
-              archiveArtifacts(artifacts: 'parameters.md, parameters.xml, params.json.gz')
+              archiveArtifacts(artifacts: 'parameters.md, parameters.xml, params.json.xz')
-              stash includes: 'parameters.md, parameters.xml, params.json.gz', name: 'metadata_parameters'
+              stash includes: 'parameters.md, parameters.xml, params.json.xz', name: 'metadata_parameters'
             }
           }
           post {
@@ -344,7 +344,7 @@ pipeline {
             withAWS(credentials: 'px4_aws_s3_key', region: 'us-east-1') {
               s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'airframes.xml', path: 'Firmware/master/')
               s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'parameters.xml', path: 'Firmware/master/')
-              s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'params.json.gz', path: 'Firmware/master/')
+              s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'params.json.xz', path: 'Firmware/master/')
             }
           }
           when {

Tools/px_romfs_pruner.py

@@ -83,7 +83,7 @@ def main():
 
             # only prune text files
             if ".zip" in file or ".bin" in file or ".swp" in file \
-                    or ".gz" in file or ".bz2" in file \
+                    or ".gz" in file or ".xz" in file or ".bz2" in file \
                     or ".data" in file or ".DS_Store" in file:
                 continue
 

cmake/px4_add_board.cmake

@@ -209,13 +209,13 @@ function(px4_add_board)
 	endif()
 	foreach(metadata ${EMBEDDED_METADATA})
 		if(${metadata} STREQUAL "parameters")
-			list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.gz)
+			list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.xz)
 			list(APPEND romfs_extra_dependencies parameters_xml)
 		else()
 			message(FATAL_ERROR "invalid value for EMBEDDED_METADATA: ${metadata}")
 		endif()
 	endforeach()
-	list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.gz)
+	list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.xz)
 	list(APPEND romfs_extra_dependencies component_version_json)
 	set(config_romfs_extra_files ${romfs_extra_files} CACHE INTERNAL "extra ROMFS files" FORCE)
 	set(config_romfs_extra_dependencies ${romfs_extra_dependencies} CACHE INTERNAL "extra ROMFS deps" FORCE)

src/lib/component_information/CMakeLists.txt

@@ -40,7 +40,7 @@ if (${index} EQUAL -1)
 endif()
 
 set(component_version_json ${PX4_BINARY_DIR}/component_version.json)
-add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.gz
+add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.xz
 	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_version.py
 		${component_version_json}
 		--compress
@@ -55,13 +55,13 @@ add_custom_target(component_version_json DEPENDS ${component_version_json})
 set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/hashes.h)
 add_custom_command(OUTPUT ${component_information_header}
 	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_hashes.py
-		${PX4_BINARY_DIR}/params.json.gz
+		${PX4_BINARY_DIR}/params.json.xz
 		${component_version_json}
 		--output ${component_information_header}
 	DEPENDS
 		generate_hashes.py
 		parameters_xml
-		${PX4_BINARY_DIR}/params.json.gz
+		${PX4_BINARY_DIR}/params.json.xz
 		${component_version_json}
 	COMMENT "Generating component_information/hashes.h"
 )

src/lib/component_information/generate_component_version.py

@@ -2,7 +2,7 @@
 
 import argparse
 import json
-import gzip
+import lzma #to create .xz file
 
 parser = argparse.ArgumentParser(description="""Generate the COMPONENT_VERSION json file""")
 parser.add_argument('filename', metavar='component_version.json', help='JSON output file')
@@ -14,9 +14,9 @@ filename = args.filename
 compress = args.compress
 
 def save_compressed(filename):
-    #create gz compressed version
+    #create lzma compressed version
-    gz_filename=filename+'.gz'
+    xz_filename=filename+'.xz'
-    with gzip.open(gz_filename, 'wt') as f:
+    with lzma.open(xz_filename, 'wt', preset=9) as f:
         with open(filename, 'r') as content_file:
             f.write(content_file.read())
 

src/lib/parameters/CMakeLists.txt

@@ -89,7 +89,7 @@ add_custom_command(OUTPUT ${generated_serial_params_file}
 set(parameters_xml ${PX4_BINARY_DIR}/parameters.xml)
 set(parameters_json ${PX4_BINARY_DIR}/params.json) # file name needs to be kept short to fit into url (metadata_uri)
 file(GLOB_RECURSE param_src_files ${PX4_SOURCE_DIR}/src/*params.c)
-add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.gz
+add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.xz
 	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/px_process_params.py
 		--src-path ${module_list} ${generated_params_dir}
 		--xml ${parameters_xml}

src/lib/parameters/px_process_params.py

@@ -49,15 +49,15 @@ import os
 import argparse
 from px4params import srcscanner, srcparser, injectxmlparams, xmlout, markdownout, jsonout
 
-import gzip #to create .gz file
+import lzma #to create .xz file
 import re
 import json
 import codecs
 
-def SaveCompressed(filename):
+def save_compressed(filename):
-    #create gz compressed version
+    #create lzma compressed version
-    gz_filename=filename+'.gz'
+    xz_filename=filename+'.xz'
-    with gzip.open(gz_filename, 'wt') as f:
+    with lzma.open(xz_filename, 'wt', preset=9) as f:
         with open(filename, 'r') as content_file:
             f.write(content_file.read())
 
@@ -185,7 +185,7 @@ def main():
         for f in output_files:
             if args.verbose:
                 print("Compressing file " + f)
-            SaveCompressed(f)
+            save_compressed(f)
             
 
 if __name__ == "__main__":

src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp

@@ -69,12 +69,12 @@ public:
 		switch (type) {
 		case COMP_METADATA_TYPE_VERSION:
 			component_info.metadata_uid = component_information::component_version_hash;
-			handled = get_component_information("component_version.json.gz", component_info);
+			handled = get_component_information("component_version.json.xz", component_info);
 			break;
 
 		case COMP_METADATA_TYPE_PARAMETER:
 			component_info.metadata_uid = component_information::params_hash;
-			handled = get_component_information("params.json.gz", component_info);
+			handled = get_component_information("params.json.xz", component_info);
 			break;
 
 		case COMP_METADATA_TYPE_COMMANDS: