component_information: rework to reflect mavlink message changes

Jenkinsfile

@@ -110,8 +110,8 @@ pipeline {
             sh 'make distclean'
             sh 'make parameters_metadata'
             dir('build/px4_sitl_default/docs') {
-              archiveArtifacts(artifacts: 'parameters.md, parameters.xml, params.json.xz')
+              archiveArtifacts(artifacts: 'parameters.md, parameters.xml, parameters.json.xz')
-              stash includes: 'parameters.md, parameters.xml, params.json.xz', name: 'metadata_parameters'
+              stash includes: 'parameters.md, parameters.xml, parameters.json.xz', name: 'metadata_parameters'
             }
           }
           post {
@@ -342,7 +342,7 @@ pipeline {
             withAWS(credentials: 'px4_aws_s3_key', region: 'us-east-1') {
               s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'airframes.xml', path: 'Firmware/master/')
               s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'parameters.xml', path: 'Firmware/master/')
-              s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'params.json.xz', path: 'Firmware/master/')
+              s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'parameters.json.xz', path: 'Firmware/master/')
             }
           }
           when {

cmake/metadata.cmake

@@ -59,7 +59,7 @@ add_custom_target(metadata_parameters
 	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py
 		--src-path `find ${PX4_SOURCE_DIR}/src -maxdepth 4 -type d`
 		--inject-xml ${PX4_SOURCE_DIR}/src/lib/parameters/parameters_injected.xml
-		--json ${PX4_BINARY_DIR}/docs/params.json
+		--json ${PX4_BINARY_DIR}/docs/parameters.json
 		--compress
 
 	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py

cmake/px4_add_board.cmake

@@ -209,14 +209,14 @@ function(px4_add_board)
 	set(config_romfs_extra_dependencies)
 	foreach(metadata ${EMBEDDED_METADATA})
 		if(${metadata} STREQUAL "parameters")
-			list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.xz)
+			list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/parameters.json.xz)
 			list(APPEND romfs_extra_dependencies parameters_xml)
 		else()
 			message(FATAL_ERROR "invalid value for EMBEDDED_METADATA: ${metadata}")
 		endif()
 	endforeach()
-	list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.xz)
+	list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_general.json.xz)
-	list(APPEND romfs_extra_dependencies component_version_json)
+	list(APPEND romfs_extra_dependencies component_general_json)
 	set(config_romfs_extra_files ${romfs_extra_files} CACHE INTERNAL "extra ROMFS files" FORCE)
 	set(config_romfs_extra_dependencies ${romfs_extra_dependencies} CACHE INTERNAL "extra ROMFS deps" FORCE)
 

src/lib/component_information/CMakeLists.txt

@@ -31,39 +31,55 @@
 #
 ############################################################################
 
-set(component_version_extra_args)
+# types:
+# 0: COMP_METADATA_TYPE_GENERAL
+# 1: COMP_METADATA_TYPE_PARAMETER
+# 2: COMP_METADATA_TYPE_COMMANDS
+# 3: COMP_METADATA_TYPE_PERIPHERALS
+set(comp_metadata_types)
 # We disable parameter support if the metadata is not in the ROMFS. This can be
 # removed once it is stored on a server
 list(FIND config_romfs_extra_dependencies "parameters_xml" index)
+set(comp_metadata_board "${PX4_BOARD_VENDOR}_${PX4_BOARD_MODEL}_${PX4_BOARD_LABEL}")
+set(s3_url "https://px4-travis.s3.amazonaws.com")
+set(comp_metadata_param_uri_board "${s3_url}/Firmware/{version}/${comp_metadata_board}/parameters.json.xz")
 if (${index} EQUAL -1)
-	set(component_version_extra_args "--no-parameters")
+	set(comp_metadata_param_uri ${comp_metadata_param_uri_board})
+	# use generic URL as fallback
+	set(comp_metadata_param_uri_fallback "${s3_url}/Firmware/{version}/_general/parameters.json.xz")
+else()
+	set(comp_metadata_param_uri "mftp://etc/extras/parameters.json.xz")
+	set(comp_metadata_param_uri_fallback ${comp_metadata_param_uri_board})
 endif()
+# arguments: type, metadata file, uri, fallback uri, optional translation uri
+list(APPEND comp_metadata_types "--type" "1,${PX4_BINARY_DIR}/parameters.json.xz,${comp_metadata_param_uri},${comp_metadata_param_uri_fallback},")
 
-set(component_version_json ${PX4_BINARY_DIR}/component_version.json)
+set(component_general_json ${PX4_BINARY_DIR}/component_general.json)
-add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.xz
+add_custom_command(OUTPUT ${component_general_json} ${component_general_json}.xz
-	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_version.py
+	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_general.py
-		${component_version_json}
+		${component_general_json}
 		--compress
-		${component_version_extra_args}
+		${comp_metadata_types}
+		--version-file ${PX4_BINARY_DIR}/src/lib/version/build_git_version.h
 	DEPENDS
-		generate_component_version.py
+		generate_component_general.py
-	COMMENT "Generating component_version.json"
+		${PX4_BINARY_DIR}/parameters.json.xz
+		parameters_xml
+		ver_gen
+	COMMENT "Generating component_general.json"
 )
-add_custom_target(component_version_json DEPENDS ${component_version_json})
+add_custom_target(component_general_json DEPENDS ${component_general_json})
 
 
-set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/hashes.h)
+set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/checksums.h)
 add_custom_command(OUTPUT ${component_information_header}
-	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_hashes.py
+	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_crc.py
-		${PX4_BINARY_DIR}/params.json.xz
+		${component_general_json}
-		${component_version_json}
 		--output ${component_information_header}
 	DEPENDS
-		generate_hashes.py
+		generate_crc.py
-		parameters_xml
+		${component_general_json}
-		${PX4_BINARY_DIR}/params.json.xz
+	COMMENT "Generating component_information/checksums.h"
-		${component_version_json}
-	COMMENT "Generating component_information/hashes.h"
 )
 add_custom_target(component_information_header DEPENDS ${component_information_header})
 

src/lib/component_information/generate_component_general.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import lzma #to create .xz file
+import re
+
+parser = argparse.ArgumentParser(description="""Generate the COMPONENT_GENERAL json file""")
+parser.add_argument('filename', metavar='component_general.json', help='JSON output file')
+parser.add_argument('--compress', action='store_true', help='Add a compressed output file')
+parser.add_argument('--type', metavar='type', action="append", default=[],
+        help='Metadata type (<type>,<metadata file>,<uri>,<fallback uri>,[translation uri])')
+parser.add_argument('--version-file', metavar='build_git_version.h', help='Git version file')
+
+args = parser.parse_args()
+filename = args.filename
+compress = args.compress
+version_file = args.version_file
+
+version_dir = ''
+if version_file is not None:
+    for line in open(version_file, "r"):
+        version_search = re.search('PX4_GIT_TAG_OR_BRANCH_NAME\s+"(.+)"', line)
+        if version_search:
+            version_dir = version_search.group(1)
+            break
+
+
+def save_compressed(filename):
+    #create lzma compressed version
+    xz_filename=filename+'.xz'
+    with lzma.open(xz_filename, 'wt', preset=9) as f:
+        with open(filename, 'r') as content_file:
+            f.write(content_file.read())
+
+component_general = {}
+component_general['version'] = 1
+
+def create_table():
+    a = []
+    for i in range(256):
+        k = i
+        for j in range(8):
+            if k & 1:
+                k ^= 0x1db710640
+            k >>= 1
+        a.append(k)
+    return a
+
+# MAVLink's CRC32
+def crc_update(buf, crc_table, crc):
+    for k in buf:
+        crc = (crc >> 8) ^ crc_table[(crc & 0xff) ^ k]
+    return crc
+
+crc_table = create_table()
+
+metadata_types = []
+for metadata_type_tuple in args.type:
+    type_id, metadata_file, uri, fallback_uri, translation_uri = metadata_type_tuple.split(',')
+    file_crc = 0
+    for line in open(metadata_file, "rb"):
+        file_crc = crc_update(line, crc_table, file_crc)
+    json_type = {
+            'type': int(type_id),
+            'uri': uri.replace('{version}', version_dir),
+            'fileCrc': file_crc,
+            'uriFallback': fallback_uri.replace('{version}', version_dir),
+            }
+    if len(translation_uri) > 0:
+        json_type['translationUri'] = translation_uri
+    metadata_types.append(json_type)
+component_general['metadataTypes'] = metadata_types
+
+with open(filename, 'w') as outfile:
+    json.dump(component_general, outfile)
+
+if compress:
+    save_compressed(filename)

src/lib/component_information/generate_component_version.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import json
-import lzma #to create .xz file
-
-parser = argparse.ArgumentParser(description="""Generate the COMPONENT_VERSION json file""")
-parser.add_argument('filename', metavar='component_version.json', help='JSON output file')
-parser.add_argument('--compress', action='store_true', help='Add a compressed output file')
-parser.add_argument('--no-parameters', action='store_true', help='disable parameter support')
-
-args = parser.parse_args()
-filename = args.filename
-compress = args.compress
-
-def save_compressed(filename):
-    #create lzma compressed version
-    xz_filename=filename+'.xz'
-    with lzma.open(xz_filename, 'wt', preset=9) as f:
-        with open(filename, 'r') as content_file:
-            f.write(content_file.read())
-
-component_version = {}
-component_version['version'] = 1
-# types:
-# 0: COMP_METADATA_TYPE_VERSION
-# 1: COMP_METADATA_TYPE_PARAMETER
-# 2: COMP_METADATA_TYPE_COMMANDS
-supported_types = [0]
-if not args.no_parameters: supported_types.append(1)
-component_version['supportedCompMetadataTypes'] = supported_types
-
-with open(filename, 'w') as outfile:
-    json.dump(component_version, outfile)
-
-if compress:
-    save_compressed(filename)

src/lib/component_information/generate_crc.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+import argparse
+import os
+
+parser = argparse.ArgumentParser(description="""Generate the COMPONENT_INFORMATION checksums (CRC32) header file""")
+parser.add_argument('--output', metavar='checksums.h', help='output file')
+parser.add_argument('files', nargs='+', metavar='input_file.json', help='files to generate CRC from')
+
+args = parser.parse_args()
+filename = args.output
+files = args.files
+
+def create_table():
+    a = []
+    for i in range(256):
+        k = i
+        for j in range(8):
+            if k & 1:
+                k ^= 0x1db710640
+            k >>= 1
+        a.append(k)
+    return a
+
+# MAVLink's CRC32
+def crc_update(buf, crc_table, crc):
+    for k in buf:
+        crc = (crc >> 8) ^ crc_table[(crc & 0xff) ^ k]
+    return crc
+
+crc_table = create_table()
+
+with open(filename, 'w') as outfile:
+    outfile.write("#include <stdint.h>\n")
+    outfile.write("namespace component_information {\n")
+    for filename in files:
+        file_crc = 0
+        for line in open(filename, "rb"):
+            file_crc = crc_update(line, crc_table, file_crc)
+
+        basename = os.path.basename(filename)
+        identifier = basename.split('.')[0]
+        outfile.write("static constexpr uint32_t {:}_crc = {:};\n".format(identifier, file_crc))
+
+
+    outfile.write("}\n")
+
+

src/lib/component_information/generate_hashes.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import zlib
-import os
-
-parser = argparse.ArgumentParser(description="""Generate the COMPONENT_INFORMATION hashes header file""")
-parser.add_argument('--output', metavar='hashes.h', help='output file')
-parser.add_argument('files', nargs='+', metavar='hashes.json', help='files to generate hashes from')
-
-args = parser.parse_args()
-filename = args.output
-files = args.files
-
-with open(filename, 'w') as outfile:
-    outfile.write("#include <stdint.h>\n")
-    outfile.write("namespace component_information {\n")
-    for filename in files:
-        # get CRC
-        file_hash = 0
-        for line in open(filename, "rb"):
-            file_hash = zlib.crc32(line, file_hash)
-
-        basename = os.path.basename(filename)
-        identifier = basename.split('.')[0]
-        outfile.write("static constexpr uint32_t {:}_hash = {:};\n".format(identifier, file_hash))
-
-
-    outfile.write("}\n")
-
-

src/lib/parameters/CMakeLists.txt

@@ -92,7 +92,7 @@ add_custom_command(OUTPUT ${generated_serial_params_file}
 )
 
 set(parameters_xml ${PX4_BINARY_DIR}/parameters.xml)
-set(parameters_json ${PX4_BINARY_DIR}/params.json) # file name needs to be kept short to fit into url (metadata_uri)
+set(parameters_json ${PX4_BINARY_DIR}/parameters.json)
 file(GLOB_RECURSE param_src_files ${PX4_SOURCE_DIR}/src/*params.c)
 add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.xz
 	COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/px_process_params.py

src/lib/parameters/px4params/jsonout.py

@@ -8,7 +8,6 @@ class JsonOutput():
     def __init__(self, groups, board, inject_xml_file_name):
         all_json=dict()
         all_json['version']=1
-        all_json['scope']="Firmware"
         all_params=[]
         all_json['parameters']=all_params
 

src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp

@@ -36,7 +36,7 @@
 
 #include "../mavlink_stream.h"
 
-#include <component_information/hashes.h>
+#include <component_information/checksums.h>
 
 #include <px4_platform_common/defines.h>
 
@@ -61,34 +61,17 @@ public:
 	bool request_message(float param2, float param3, float param4,
 			     float param5, float param6, float param7) override
 	{
-		int type = (int)roundf(param2);
 		mavlink_component_information_t component_info{};
-		bool handled = false;
+		PX4_DEBUG("COMPONENT_INFORMATION request");
-		PX4_DEBUG("COMPONENT_INFORMATION request type %i", type);
 
-		switch (type) {
+		strncpy(component_info.general_metadata_uri, "mftp://etc/extras/component_general.json.xz",
-		case COMP_METADATA_TYPE_VERSION:
+			sizeof(component_info.general_metadata_uri) - 1);
-			component_info.metadata_uid = component_information::component_version_hash;
+		component_info.general_metadata_file_crc = component_information::component_general_crc;
-			handled = get_component_information("component_version.json.xz", component_info);
-			break;
 
-		case COMP_METADATA_TYPE_PARAMETER:
+		component_info.time_boot_ms = hrt_absolute_time() / 1000;
-			component_info.metadata_uid = component_information::params_hash;
+		mavlink_msg_component_information_send_struct(_mavlink->get_channel(), &component_info);
-			handled = get_component_information("params.json.xz", component_info);
-			break;
 
-		case COMP_METADATA_TYPE_COMMANDS:
+		return true;
-			// TODO
-			break;
-		}
-
-		if (handled) {
-			component_info.metadata_type = type;
-			component_info.time_boot_ms = hrt_absolute_time() / 1000;
-			mavlink_msg_component_information_send_struct(_mavlink->get_channel(), &component_info);
-		}
-
-		return handled;
 	}
 private:
 	explicit MavlinkStreamComponentInformation(Mavlink *mavlink) : MavlinkStream(mavlink) {}
@@ -97,40 +80,6 @@ private:
 	{
 		return false;
 	}
-
-	bool get_component_information(const char *file, mavlink_component_information_t &component_info)
-	{
-		char full_path[64];
-		snprintf(full_path, sizeof(full_path), "%s/etc/extras/%s", PX4_ROOTFSDIR, file);
-		full_path[sizeof(full_path) - 1] = '\0';
-
-		if (file_exist(full_path)) {
-			if (snprintf(component_info.metadata_uri, sizeof(component_info.metadata_uri), "mftp://etc/extras/%s", file)
-			    >= (int)sizeof(component_info.metadata_uri)) {
-				PX4_ERR("path too long (%s)", file);
-				return false;
-			}
-
-		} else {
-			// TODO:
-			// - check for tagged version, use per-version files
-			// - generate & use board-specific file
-			if (snprintf(component_info.metadata_uri, sizeof(component_info.metadata_uri),
-				     "https://px4-travis.s3.amazonaws.com/Firmware/master/%s", file)
-			    >= (int)sizeof(component_info.metadata_uri)) {
-				PX4_ERR("url too long (%s)", file);
-				return false;
-			}
-		}
-
-		return true;
-	}
-
-	bool file_exist(const char *filename)
-	{
-		struct stat buffer;
-		return stat(filename, &buffer) == 0;
-	}
 };
 
 #endif // COMPONENT_INFORMATION_HPP