From 1f389c684b12e11edc9cf13ce8647241790640ae Mon Sep 17 00:00:00 2001 From: liang yongxiang Date: Fri, 3 Aug 2018 11:09:01 +0800 Subject: [PATCH 1/5] [tools] add scons --dist-strip support --- tools/building.py | 30 +++++-------- tools/mkdist.py | 109 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 23 deletions(-) diff --git a/tools/building.py b/tools/building.py index e73815de22..527da03103 100644 --- a/tools/building.py +++ b/tools/building.py @@ -115,7 +115,7 @@ class Win32Spawn: # generate cconfig.h file def GenCconfigFile(env, BuildOptions): import rtconfig - + if rtconfig.PLATFORM == 'gcc': contents = '' if not os.path.isfile('cconfig.h'): @@ -147,21 +147,16 @@ def PrepareBuilding(env, root_directory, has_libcpu=False, remove_components = [ global Rtt_Root # ===== Add option to SCons ===== - AddOption('--copy', - dest = 'copy', - action = 'store_true', - default = False, - help = 'copy rt-thread directory to local.') - AddOption('--copy-header', - dest = 'copy-header', - action = 'store_true', - default = False, - help = 'copy header of rt-thread directory to local.') AddOption('--dist', dest = 'make-dist', action = 'store_true', default = False, help = 'make distribution') + AddOption('--dist-strip', + dest = 'make-dist-strip', + action = 'store_true', + default = False, + help = 'make distribution and strip useless files') AddOption('--cscope', dest = 'cscope', action = 'store_true', @@ -713,7 +708,7 @@ def DoBuilding(target, objects): program = Env.Program(target, objects) EndBuilding(target, program) - + def GenTargetProject(program = None): if GetOption('target') == 'mdk': @@ -789,17 +784,12 @@ def EndBuilding(target, program = None): GenTargetProject(program) BSP_ROOT = Dir('#').abspath - if GetOption('copy') and program != None: - from mkdist import MakeCopy - MakeCopy(program, BSP_ROOT, Rtt_Root, Env) - need_exit = True - if GetOption('copy-header') and program != None: - from mkdist import MakeCopyHeader - MakeCopyHeader(program, BSP_ROOT, Rtt_Root, Env) - need_exit = True if GetOption('make-dist') and program != None: from mkdist import MkDist MkDist(program, BSP_ROOT, Rtt_Root, Env) + if GetOption('make-dist-strip') and program != None: + from mkdist import MkDist_Strip + MkDist_Strip(program, BSP_ROOT, Rtt_Root, Env) need_exit = True if GetOption('cscope'): from cscope import CscopeDatabase diff --git a/tools/mkdist.py b/tools/mkdist.py index 466439caf7..c232aeee00 100644 --- a/tools/mkdist.py +++ b/tools/mkdist.py @@ -89,7 +89,12 @@ def walk_kconfig(RTT_ROOT, source_list): pathfile = os.path.join(parent, 'KConfig') source_list.append(pathfile) -def bsp_update_sconstruct(dist_dir): +def bsp_copy_files(bsp_root, dist_dir): + # copy BSP files + do_copy_folder(os.path.join(bsp_root), dist_dir, + ignore_patterns('build', 'dist', '*.pyc', '*.old', '*.map', 'rtthread.bin', '.sconsign.dblite', '*.elf', '*.axf', 'cconfig.h')) + +def bsp_update_sconstruct(dist_dir): with open(os.path.join(dist_dir, 'SConstruct'), "r") as f: data = f.readlines() with open(os.path.join(dist_dir, 'SConstruct'), "w") as f: @@ -149,6 +154,104 @@ def zip_dist(bsp_root, dist_dir, dist_name): zip.close() +def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): + global source_list + + print("make distribution and strip useless files....") + + dist_name = os.path.basename(BSP_ROOT) + dist_dir = os.path.join(BSP_ROOT, 'dist', dist_name) + target_path = os.path.join(dist_dir, 'rt-thread') + + bsp_copy_files(BSP_ROOT, dist_dir): + + # get all source files from program + for item in program: + walk_children(item) + source_list.sort() + + # copy the source files without libcpu and components/libc in RT-Thread + target_list = [] + libcpu_dir = os.path.join(RTT_ROOT, 'libcpu').lower() + libc_dir = os.path.join(RTT_ROOT, 'components', 'libc').lower() + for src in source_list: + if src.lower().startswith(BSP_ROOT.lower()): + continue + + # skip libc and libcpu dir + if src.lower().startswith(libcpu_dir): + continue + if src.lower().startswith(libc_dir): + continue + + if src.lower().startswith(RTT_ROOT.lower()): + target_list.append(src) + source_list = target_list + + # get source directory + src_dir = [] + for src in source_list: + src = src.replace(RTT_ROOT, '') + if src[0] == os.sep or src[0] == '/': + src = src[1:] + + path = os.path.dirname(src) + sub_path = path.split(os.sep) + full_path = RTT_ROOT + for item in sub_path: + full_path = os.path.join(full_path, item) + if full_path not in src_dir: + src_dir.append(full_path) + + # add all of SConscript files + for item in src_dir: + source_list.append(os.path.join(item, 'SConscript')) + + # add all of Kconfig files + walk_kconfig(RTT_ROOT, source_list) + + # copy all files to target directory + source_list.sort() + for src in source_list: + dst = src.replace(RTT_ROOT, '') + if dst[0] == os.sep or dst[0] == '/': + dst = dst[1:] + + print('=> %s' % dst) + dst = os.path.join(target_path, dst) + do_copy_file(src, dst) + + # copy tools directory + print("=> tools") + do_copy_folder(os.path.join(RTT_ROOT, "tools"), os.path.join(target_path, "tools"), ignore_patterns('*.pyc')) + do_copy_file(os.path.join(RTT_ROOT, 'Kconfig'), os.path.join(target_path, 'Kconfig')) + do_copy_file(os.path.join(RTT_ROOT, 'AUTHORS'), os.path.join(target_path, 'AUTHORS')) + do_copy_file(os.path.join(RTT_ROOT, 'COPYING'), os.path.join(target_path, 'COPYING')) + do_copy_file(os.path.join(RTT_ROOT, 'README.md'), os.path.join(target_path, 'README.md')) + do_copy_file(os.path.join(RTT_ROOT, 'README_zh.md'), os.path.join(target_path, 'README_zh.md')) + + print('=> libc') + do_copy_folder(os.path.join(RTT_ROOT, "components", 'libc', 'compilers'), os.path.join(target_path, "components", 'libc', 'compilers')) + + # copy all libcpu/ARCH directory + print('=> libcpu') + import rtconfig + do_copy_folder(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH), os.path.join(target_path, 'libcpu', rtconfig.ARCH)) + do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'Kconfig'), os.path.join(target_path, 'libcpu', 'Kconfig')) + do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'SConscript'), os.path.join(target_path, 'libcpu', 'SConscript')) + + # change RTT_ROOT in SConstruct + bsp_update_sconstruct(dist_dir) + # change RTT_ROOT in Kconfig + bsp_update_kconfig(dist_dir) + # update all project files + bs_update_ide_project(dist_dir, target_path) + + # make zip package + zip_dist(BSP_ROOT, dist_dir, dist_name) + + print('done!') + def MkDist(program, BSP_ROOT, RTT_ROOT, Env): print("make distribution....") @@ -159,8 +262,7 @@ def MkDist(program, BSP_ROOT, RTT_ROOT, Env): # copy BSP files print("=> %s" % os.path.basename(BSP_ROOT)) - do_copy_folder(os.path.join(BSP_ROOT), dist_dir, - ignore_patterns('build', 'dist', '*.pyc', '*.old', '*.map', 'rtthread.bin', '.sconsign.dblite', '*.elf', '*.axf', 'cconfig.h')) + bsp_copy_files(BSP_ROOT, dist_dir): # copy tools directory print("=> components") @@ -205,3 +307,4 @@ def MkDist(program, BSP_ROOT, RTT_ROOT, Env): zip_dist(BSP_ROOT, dist_dir, dist_name) print('done!') + From 12eafd541d39b2d8f8ab3ef31be06cdab1c5411b Mon Sep 17 00:00:00 2001 From: liang yongxiang Date: Fri, 3 Aug 2018 11:51:52 +0800 Subject: [PATCH 2/5] [tools] Reduce copied files --- tools/mkdist.py | 62 +++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/tools/mkdist.py b/tools/mkdist.py index c232aeee00..78c2edc36a 100644 --- a/tools/mkdist.py +++ b/tools/mkdist.py @@ -53,7 +53,7 @@ def do_copy_folder(src_dir, dst_dir, ignore=None): shutil.copytree(src_dir, dst_dir, ignore = ignore) -source_ext = ["c", "h", "s", "S", "cpp", "xpm"] +source_ext = ['c', 'h', 's', 'S', 'cpp', 'xpm'] source_list = [] def walk_children(child): @@ -95,21 +95,24 @@ def bsp_copy_files(bsp_root, dist_dir): ignore_patterns('build', 'dist', '*.pyc', '*.old', '*.map', 'rtthread.bin', '.sconsign.dblite', '*.elf', '*.axf', 'cconfig.h')) def bsp_update_sconstruct(dist_dir): - with open(os.path.join(dist_dir, 'SConstruct'), "r") as f: + with open(os.path.join(dist_dir, 'SConstruct'), 'r') as f: data = f.readlines() - with open(os.path.join(dist_dir, 'SConstruct'), "w") as f: + with open(os.path.join(dist_dir, 'SConstruct'), 'w') as f: for line in data: if line.find('RTT_ROOT') != -1: if line.find('sys.path') != -1: f.write('# set RTT_ROOT\n') - f.write("if not os.getenv('RTT_ROOT'): \n RTT_ROOT='rt-thread'\n\n") + f.write('if not os.getenv("RTT_ROOT"): \n RTT_ROOT="rt-thread"\n\n') f.write(line) def bsp_update_kconfig(dist_dir): # change RTT_ROOT in Kconfig - with open(os.path.join(dist_dir, 'Kconfig'), "r") as f: + if not os.path.isfile(os.path.join(dist_dir, 'Kconfig')): + return + + with open(os.path.join(dist_dir, 'Kconfig'), 'r') as f: data = f.readlines() - with open(os.path.join(dist_dir, 'Kconfig'), "w") as f: + with open(os.path.join(dist_dir, 'Kconfig'), 'w') as f: found = 0 for line in data: if line.find('RTT_ROOT') != -1: @@ -131,19 +134,19 @@ def bs_update_ide_project(bsp_root, rtt_root): 'cdk':('gcc', 'gcc')} scons_env = os.environ.copy() - scons_env["RTT_ROOT"] = rtt_root + scons_env['RTT_ROOT'] = rtt_root for item in tgt_dict: child = subprocess.Popen('scons --target=' + item, cwd=bsp_root, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) stdout, stderr = child.communicate() if child.returncode == 0: - print("update %s project" % item) + print('update %s project' % item) def zip_dist(bsp_root, dist_dir, dist_name): import zipfile zip_filename = os.path.join(bsp_root, 'dist', dist_name) - zip = zipfile.ZipFile(zip_filename + ".zip", 'w') + zip = zipfile.ZipFile(zip_filename + '.zip', 'w') pre_len = len(os.path.dirname(dist_dir)) for parent, dirnames, filenames in os.walk(dist_dir): @@ -157,13 +160,14 @@ def zip_dist(bsp_root, dist_dir, dist_name): def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): global source_list - print("make distribution and strip useless files....") + print('make distribution and strip useless files....') dist_name = os.path.basename(BSP_ROOT) dist_dir = os.path.join(BSP_ROOT, 'dist', dist_name) target_path = os.path.join(dist_dir, 'rt-thread') - bsp_copy_files(BSP_ROOT, dist_dir): + print('=> %s' % os.path.basename(BSP_ROOT)) + bsp_copy_files(BSP_ROOT, dist_dir) # get all source files from program for item in program: @@ -173,7 +177,7 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): # copy the source files without libcpu and components/libc in RT-Thread target_list = [] libcpu_dir = os.path.join(RTT_ROOT, 'libcpu').lower() - libc_dir = os.path.join(RTT_ROOT, 'components', 'libc').lower() + libc_dir = os.path.join(RTT_ROOT, 'components', 'libc', 'compilers').lower() for src in source_list: if src.lower().startswith(BSP_ROOT.lower()): continue @@ -222,21 +226,23 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): do_copy_file(src, dst) # copy tools directory - print("=> tools") - do_copy_folder(os.path.join(RTT_ROOT, "tools"), os.path.join(target_path, "tools"), ignore_patterns('*.pyc')) + print('=> tools') + do_copy_folder(os.path.join(RTT_ROOT, 'tools'), os.path.join(target_path, 'tools'), ignore_patterns('*.pyc')) do_copy_file(os.path.join(RTT_ROOT, 'Kconfig'), os.path.join(target_path, 'Kconfig')) do_copy_file(os.path.join(RTT_ROOT, 'AUTHORS'), os.path.join(target_path, 'AUTHORS')) do_copy_file(os.path.join(RTT_ROOT, 'COPYING'), os.path.join(target_path, 'COPYING')) do_copy_file(os.path.join(RTT_ROOT, 'README.md'), os.path.join(target_path, 'README.md')) do_copy_file(os.path.join(RTT_ROOT, 'README_zh.md'), os.path.join(target_path, 'README_zh.md')) - print('=> libc') - do_copy_folder(os.path.join(RTT_ROOT, "components", 'libc', 'compilers'), os.path.join(target_path, "components", 'libc', 'compilers')) + print('=> %s' % os.path.join('components', 'libc', 'components')) + do_copy_folder(os.path.join(RTT_ROOT, 'rcomponents', 'libc', 'compilers'), os.path.join(target_path, 'components', 'libc', 'compilers')) # copy all libcpu/ARCH directory print('=> libcpu') import rtconfig - do_copy_folder(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH), os.path.join(target_path, 'libcpu', rtconfig.ARCH)) + do_copy_folder(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH, rtconfig.CPU), os.path.join(target_path, 'libcpu', rtconfig.ARCH, rtconfig.CPU)) + if os.path.exists(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH, 'common')): + do_copy_folder(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH, 'common'), os.path.join(target_path, 'libcpu', rtconfig.ARCH, 'common')) do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'Kconfig'), os.path.join(target_path, 'libcpu', 'Kconfig')) do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'SConscript'), os.path.join(target_path, 'libcpu', 'SConscript')) @@ -253,7 +259,7 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): print('done!') def MkDist(program, BSP_ROOT, RTT_ROOT, Env): - print("make distribution....") + print('make distribution....') dist_name = os.path.basename(BSP_ROOT) dist_dir = os.path.join(BSP_ROOT, 'dist', dist_name) @@ -261,19 +267,19 @@ def MkDist(program, BSP_ROOT, RTT_ROOT, Env): target_path = os.path.join(dist_dir, 'rt-thread') # copy BSP files - print("=> %s" % os.path.basename(BSP_ROOT)) - bsp_copy_files(BSP_ROOT, dist_dir): + print('=> %s' % os.path.basename(BSP_ROOT)) + bsp_copy_files(BSP_ROOT, dist_dir) # copy tools directory - print("=> components") - do_copy_folder(os.path.join(RTT_ROOT, "components"), os.path.join(target_path, "components")) + print('=> components') + do_copy_folder(os.path.join(RTT_ROOT, 'components'), os.path.join(target_path, 'components')) # skip documentation directory # skip examples # copy include directory - print("=> include") - do_copy_folder(os.path.join(RTT_ROOT, "include"), os.path.join(target_path, "include")) + print('=> include') + do_copy_folder(os.path.join(RTT_ROOT, 'include'), os.path.join(target_path, 'include')) # copy all libcpu/ARCH directory print('=> libcpu') @@ -283,12 +289,12 @@ def MkDist(program, BSP_ROOT, RTT_ROOT, Env): do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'SConscript'), os.path.join(target_path, 'libcpu', 'SConscript')) # copy src directory - print("=> src") - do_copy_folder(os.path.join(RTT_ROOT, "src"), os.path.join(target_path, "src")) + print('=> src') + do_copy_folder(os.path.join(RTT_ROOT, 'src'), os.path.join(target_path, 'src')) # copy tools directory - print("=> tools") - do_copy_folder(os.path.join(RTT_ROOT, "tools"), os.path.join(target_path, "tools"), ignore_patterns('*.pyc')) + print('=> tools') + do_copy_folder(os.path.join(RTT_ROOT, 'tools'), os.path.join(target_path, 'tools'), ignore_patterns('*.pyc')) do_copy_file(os.path.join(RTT_ROOT, 'Kconfig'), os.path.join(target_path, 'Kconfig')) do_copy_file(os.path.join(RTT_ROOT, 'AUTHORS'), os.path.join(target_path, 'AUTHORS')) From 09a844f7296b1d2cd319e1d0c126de8f6f472971 Mon Sep 17 00:00:00 2001 From: liang yongxiang Date: Fri, 3 Aug 2018 16:18:27 +0800 Subject: [PATCH 3/5] [tools] The output directory of "scons --dist-strip" is changed from current-bsp/dist to current-bsp/dist-strip --- tools/mkdist.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/mkdist.py b/tools/mkdist.py index 78c2edc36a..beb01fecb1 100644 --- a/tools/mkdist.py +++ b/tools/mkdist.py @@ -142,10 +142,10 @@ def bs_update_ide_project(bsp_root, rtt_root): if child.returncode == 0: print('update %s project' % item) -def zip_dist(bsp_root, dist_dir, dist_name): +def zip_dist(dist_dir, dist_name): import zipfile - zip_filename = os.path.join(bsp_root, 'dist', dist_name) + zip_filename = os.path.join(dist_dir) zip = zipfile.ZipFile(zip_filename + '.zip', 'w') pre_len = len(os.path.dirname(dist_dir)) @@ -163,7 +163,7 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): print('make distribution and strip useless files....') dist_name = os.path.basename(BSP_ROOT) - dist_dir = os.path.join(BSP_ROOT, 'dist', dist_name) + dist_dir = os.path.join(BSP_ROOT, 'dist-strip', dist_name) target_path = os.path.join(dist_dir, 'rt-thread') print('=> %s' % os.path.basename(BSP_ROOT)) @@ -254,7 +254,7 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): bs_update_ide_project(dist_dir, target_path) # make zip package - zip_dist(BSP_ROOT, dist_dir, dist_name) + zip_dist(dist_dir, dist_name) print('done!') @@ -310,7 +310,7 @@ def MkDist(program, BSP_ROOT, RTT_ROOT, Env): bs_update_ide_project(dist_dir, target_path) # make zip package - zip_dist(BSP_ROOT, dist_dir, dist_name) + zip_dist(dist_dir, dist_name) print('done!') From 981afddee307ce039bbb1fdcd876418655f99412 Mon Sep 17 00:00:00 2001 From: liang yongxiang Date: Fri, 3 Aug 2018 16:29:13 +0800 Subject: [PATCH 4/5] [tools] fixed typo in MkDist_Strip --- tools/mkdist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/mkdist.py b/tools/mkdist.py index beb01fecb1..40a3928b13 100644 --- a/tools/mkdist.py +++ b/tools/mkdist.py @@ -234,8 +234,8 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): do_copy_file(os.path.join(RTT_ROOT, 'README.md'), os.path.join(target_path, 'README.md')) do_copy_file(os.path.join(RTT_ROOT, 'README_zh.md'), os.path.join(target_path, 'README_zh.md')) - print('=> %s' % os.path.join('components', 'libc', 'components')) - do_copy_folder(os.path.join(RTT_ROOT, 'rcomponents', 'libc', 'compilers'), os.path.join(target_path, 'components', 'libc', 'compilers')) + print('=> %s' % os.path.join('components', 'libc', 'compilers')) + do_copy_folder(os.path.join(RTT_ROOT, 'components', 'libc', 'compilers'), os.path.join(target_path, 'components', 'libc', 'compilers')) # copy all libcpu/ARCH directory print('=> libcpu') From b2efa8eeff27db7ec7169ea9160ed3df2908bd5a Mon Sep 17 00:00:00 2001 From: liang yongxiang Date: Fri, 3 Aug 2018 19:47:24 +0800 Subject: [PATCH 5/5] [tools] update 'scons --dist-strip': copy all sal_socket and add more info to libcpu --- tools/mkdist.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/mkdist.py b/tools/mkdist.py index 40a3928b13..eef6bfd118 100644 --- a/tools/mkdist.py +++ b/tools/mkdist.py @@ -178,6 +178,8 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): target_list = [] libcpu_dir = os.path.join(RTT_ROOT, 'libcpu').lower() libc_dir = os.path.join(RTT_ROOT, 'components', 'libc', 'compilers').lower() + sal_dir = os.path.join(RTT_ROOT, 'components', 'net', 'sal_socket').lower() + sources_include_sal = False for src in source_list: if src.lower().startswith(BSP_ROOT.lower()): continue @@ -187,6 +189,9 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): continue if src.lower().startswith(libc_dir): continue + if src.lower().startswith(sal_dir): + sources_include_sal = True + continue if src.lower().startswith(RTT_ROOT.lower()): target_list.append(src) @@ -237,11 +242,16 @@ def MkDist_Strip(program, BSP_ROOT, RTT_ROOT, Env): print('=> %s' % os.path.join('components', 'libc', 'compilers')) do_copy_folder(os.path.join(RTT_ROOT, 'components', 'libc', 'compilers'), os.path.join(target_path, 'components', 'libc', 'compilers')) + if sources_include_sal: + print('=> %s' % os.path.join('components', 'net', 'sal_socket')) + do_copy_folder(os.path.join(RTT_ROOT, 'components', 'net', 'sal_socket'), os.path.join(target_path, 'components', 'net', 'sal_socket')) + # copy all libcpu/ARCH directory - print('=> libcpu') import rtconfig + print('=> %s' % (os.path.join('libcpu', rtconfig.ARCH, rtconfig.CPU))) do_copy_folder(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH, rtconfig.CPU), os.path.join(target_path, 'libcpu', rtconfig.ARCH, rtconfig.CPU)) if os.path.exists(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH, 'common')): + print('=> %s' % (os.path.join('libcpu', rtconfig.ARCH, 'common'))) do_copy_folder(os.path.join(RTT_ROOT, 'libcpu', rtconfig.ARCH, 'common'), os.path.join(target_path, 'libcpu', rtconfig.ARCH, 'common')) do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'Kconfig'), os.path.join(target_path, 'libcpu', 'Kconfig')) do_copy_file(os.path.join(RTT_ROOT, 'libcpu', 'SConscript'), os.path.join(target_path, 'libcpu', 'SConscript'))