From 10d8b43c6138078c464cb50d56f6b896592f5545 Mon Sep 17 00:00:00 2001 From: Vincent Wei Date: Thu, 7 Mar 2019 09:55:23 +0800 Subject: [PATCH] update emoji data tables --- src/font/unicode-emoji-table.h | 263 +++++++----------------------- src/font/unicode-emoji.c | 9 +- src/font/utils/fetch-emoji.sh | 20 +++ src/font/utils/fetch-ucd.sh | 20 +++ src/font/utils/gen-emoji-table.py | 24 +-- 5 files changed, 117 insertions(+), 219 deletions(-) create mode 100755 src/font/utils/fetch-emoji.sh create mode 100755 src/font/utils/fetch-ucd.sh diff --git a/src/font/unicode-emoji-table.h b/src/font/unicode-emoji-table.h index 47f11192..38a515de 100644 --- a/src/font/unicode-emoji-table.h +++ b/src/font/unicode-emoji-table.h @@ -2,18 +2,18 @@ /* * The following tables are generated by running: * - * ./gen-emoji-table.py emoji-data.txt + * ./gen-emoji-table.py emoji/emoji-data.txt * * on file with this header: * * # emoji-data.txt - * # Date: 2018-02-07, 07:55:18 GMT - * # © 2018 Unicode®, Inc. + * # Date: 2019-01-15, 12:10:05 GMT + * # © 2019 Unicode®, Inc. * # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. * # For terms of use, see http://www.unicode.org/terms_of_use.html * # * # Emoji Data for UTS #51 - * # Version: 11.0 + * # Version: 12.0 * # * # For documentation and usage, see http://www.unicode.org/reports/tr51 */ @@ -22,7 +22,7 @@ #define UNICODE_EMOJI_TABLE_H struct Interval { - Uchar32 start, end; + Uchar32 start, end; }; static const struct Interval _unicode_emoji_table[] = @@ -159,21 +159,25 @@ static const struct Interval _unicode_emoji_table[] = {0x1F5FA, 0x1F64F}, {0x1F680, 0x1F6C5}, {0x1F6CB, 0x1F6D2}, + {0x1F6D5, 0x1F6D5}, {0x1F6E0, 0x1F6E5}, {0x1F6E9, 0x1F6E9}, {0x1F6EB, 0x1F6EC}, {0x1F6F0, 0x1F6F0}, - {0x1F6F3, 0x1F6F9}, - {0x1F910, 0x1F93A}, - {0x1F93C, 0x1F93E}, - {0x1F940, 0x1F945}, - {0x1F947, 0x1F970}, + {0x1F6F3, 0x1F6FA}, + {0x1F7E0, 0x1F7EB}, + {0x1F90D, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1F971}, {0x1F973, 0x1F976}, - {0x1F97A, 0x1F97A}, - {0x1F97C, 0x1F9A2}, - {0x1F9B0, 0x1F9B9}, - {0x1F9C0, 0x1F9C2}, - {0x1F9D0, 0x1F9FF}, + {0x1F97A, 0x1F9A2}, + {0x1F9A5, 0x1F9AA}, + {0x1F9AE, 0x1F9CA}, + {0x1F9CD, 0x1F9FF}, + {0x1FA70, 0x1FA73}, + {0x1FA78, 0x1FA7A}, + {0x1FA80, 0x1FA82}, + {0x1FA90, 0x1FA95}, }; static const struct Interval _unicode_emoji_presentation_table[] = @@ -243,18 +247,22 @@ static const struct Interval _unicode_emoji_presentation_table[] = {0x1F680, 0x1F6C5}, {0x1F6CC, 0x1F6CC}, {0x1F6D0, 0x1F6D2}, + {0x1F6D5, 0x1F6D5}, {0x1F6EB, 0x1F6EC}, - {0x1F6F4, 0x1F6F9}, - {0x1F910, 0x1F93A}, - {0x1F93C, 0x1F93E}, - {0x1F940, 0x1F945}, - {0x1F947, 0x1F970}, + {0x1F6F4, 0x1F6FA}, + {0x1F7E0, 0x1F7EB}, + {0x1F90D, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1F971}, {0x1F973, 0x1F976}, - {0x1F97A, 0x1F97A}, - {0x1F97C, 0x1F9A2}, - {0x1F9B0, 0x1F9B9}, - {0x1F9C0, 0x1F9C2}, - {0x1F9D0, 0x1F9FF}, + {0x1F97A, 0x1F9A2}, + {0x1F9A5, 0x1F9AA}, + {0x1F9AE, 0x1F9CA}, + {0x1F9CD, 0x1F9FF}, + {0x1FA70, 0x1FA73}, + {0x1FA78, 0x1FA7A}, + {0x1FA80, 0x1FA82}, + {0x1FA90, 0x1FA95}, }; static const struct Interval _unicode_emoji_modifier_table[] = @@ -273,12 +281,12 @@ static const struct Interval _unicode_emoji_modifier_base_table[] = {0x1F3CA, 0x1F3CC}, {0x1F442, 0x1F443}, {0x1F446, 0x1F450}, - {0x1F466, 0x1F469}, - {0x1F46E, 0x1F46E}, - {0x1F470, 0x1F478}, + {0x1F466, 0x1F478}, {0x1F47C, 0x1F47C}, {0x1F481, 0x1F483}, {0x1F485, 0x1F487}, + {0x1F48F, 0x1F48F}, + {0x1F491, 0x1F491}, {0x1F4AA, 0x1F4AA}, {0x1F574, 0x1F575}, {0x1F57A, 0x1F57A}, @@ -290,17 +298,19 @@ static const struct Interval _unicode_emoji_modifier_base_table[] = {0x1F6B4, 0x1F6B6}, {0x1F6C0, 0x1F6C0}, {0x1F6CC, 0x1F6CC}, - {0x1F918, 0x1F91C}, - {0x1F91E, 0x1F91F}, + {0x1F90F, 0x1F90F}, + {0x1F918, 0x1F91F}, {0x1F926, 0x1F926}, {0x1F930, 0x1F939}, - {0x1F93D, 0x1F93E}, + {0x1F93C, 0x1F93E}, {0x1F9B5, 0x1F9B6}, {0x1F9B8, 0x1F9B9}, + {0x1F9BB, 0x1F9BB}, + {0x1F9CD, 0x1F9CF}, {0x1F9D1, 0x1F9DD}, }; -static const struct Interval _unicode_emoji_extended_pictographic_table[] = +static const struct Interval _unicode_extended_pictographic_table[] = { {0x00A9, 0x00A9}, {0x00AE, 0x00AE}, @@ -323,38 +333,9 @@ static const struct Interval _unicode_emoji_extended_pictographic_table[] = {0x25FB, 0x25FE}, {0x2600, 0x2605}, {0x2607, 0x2612}, - {0x2614, 0x2615}, - {0x2616, 0x2617}, - {0x2618, 0x2618}, - {0x2619, 0x2619}, - {0x261A, 0x266F}, - {0x2670, 0x2671}, - {0x2672, 0x267D}, - {0x267E, 0x267F}, - {0x2680, 0x2685}, - {0x2690, 0x2691}, - {0x2692, 0x269C}, - {0x269D, 0x269D}, - {0x269E, 0x269F}, - {0x26A0, 0x26A1}, - {0x26A2, 0x26B1}, - {0x26B2, 0x26B2}, - {0x26B3, 0x26BC}, - {0x26BD, 0x26BF}, - {0x26C0, 0x26C3}, - {0x26C4, 0x26CD}, - {0x26CE, 0x26CE}, - {0x26CF, 0x26E1}, - {0x26E2, 0x26E2}, - {0x26E3, 0x26E3}, - {0x26E4, 0x26E7}, - {0x26E8, 0x26FF}, - {0x2700, 0x2700}, - {0x2701, 0x2704}, - {0x2705, 0x2705}, - {0x2708, 0x2709}, - {0x270A, 0x270B}, - {0x270C, 0x2712}, + {0x2614, 0x2685}, + {0x2690, 0x2705}, + {0x2708, 0x2712}, {0x2714, 0x2714}, {0x2716, 0x2716}, {0x271D, 0x271D}, @@ -381,163 +362,33 @@ static const struct Interval _unicode_emoji_extended_pictographic_table[] = {0x303D, 0x303D}, {0x3297, 0x3297}, {0x3299, 0x3299}, - {0x1F000, 0x1F02B}, - {0x1F02C, 0x1F02F}, - {0x1F030, 0x1F093}, - {0x1F094, 0x1F09F}, - {0x1F0A0, 0x1F0AE}, - {0x1F0AF, 0x1F0B0}, - {0x1F0B1, 0x1F0BE}, - {0x1F0BF, 0x1F0BF}, - {0x1F0C0, 0x1F0C0}, - {0x1F0C1, 0x1F0CF}, - {0x1F0D0, 0x1F0D0}, - {0x1F0D1, 0x1F0DF}, - {0x1F0E0, 0x1F0F5}, - {0x1F0F6, 0x1F0FF}, + {0x1F000, 0x1F0FF}, {0x1F10D, 0x1F10F}, {0x1F12F, 0x1F12F}, - {0x1F16C, 0x1F16F}, - {0x1F170, 0x1F171}, - {0x1F17E, 0x1F17E}, - {0x1F17F, 0x1F17F}, + {0x1F16C, 0x1F171}, + {0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, {0x1F1AD, 0x1F1E5}, - {0x1F201, 0x1F202}, - {0x1F203, 0x1F20F}, + {0x1F201, 0x1F20F}, {0x1F21A, 0x1F21A}, {0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A}, {0x1F23C, 0x1F23F}, - {0x1F249, 0x1F24F}, - {0x1F250, 0x1F251}, - {0x1F252, 0x1F25F}, - {0x1F260, 0x1F265}, - {0x1F266, 0x1F2FF}, - {0x1F300, 0x1F320}, - {0x1F321, 0x1F32C}, - {0x1F32D, 0x1F32F}, - {0x1F330, 0x1F335}, - {0x1F336, 0x1F336}, - {0x1F337, 0x1F37C}, - {0x1F37D, 0x1F37D}, - {0x1F37E, 0x1F37F}, - {0x1F380, 0x1F393}, - {0x1F394, 0x1F39F}, - {0x1F3A0, 0x1F3C4}, - {0x1F3C5, 0x1F3C5}, - {0x1F3C6, 0x1F3CA}, - {0x1F3CB, 0x1F3CE}, - {0x1F3CF, 0x1F3D3}, - {0x1F3D4, 0x1F3DF}, - {0x1F3E0, 0x1F3F0}, - {0x1F3F1, 0x1F3F7}, - {0x1F3F8, 0x1F3FA}, - {0x1F400, 0x1F43E}, - {0x1F43F, 0x1F43F}, - {0x1F440, 0x1F440}, - {0x1F441, 0x1F441}, - {0x1F442, 0x1F4F7}, - {0x1F4F8, 0x1F4F8}, - {0x1F4F9, 0x1F4FC}, - {0x1F4FD, 0x1F4FE}, - {0x1F4FF, 0x1F4FF}, - {0x1F500, 0x1F53D}, - {0x1F546, 0x1F54A}, - {0x1F54B, 0x1F54F}, - {0x1F550, 0x1F567}, - {0x1F568, 0x1F579}, - {0x1F57A, 0x1F57A}, - {0x1F57B, 0x1F5A3}, - {0x1F5A4, 0x1F5A4}, - {0x1F5A5, 0x1F5FA}, - {0x1F5FB, 0x1F5FF}, - {0x1F600, 0x1F600}, - {0x1F601, 0x1F610}, - {0x1F611, 0x1F611}, - {0x1F612, 0x1F614}, - {0x1F615, 0x1F615}, - {0x1F616, 0x1F616}, - {0x1F617, 0x1F617}, - {0x1F618, 0x1F618}, - {0x1F619, 0x1F619}, - {0x1F61A, 0x1F61A}, - {0x1F61B, 0x1F61B}, - {0x1F61C, 0x1F61E}, - {0x1F61F, 0x1F61F}, - {0x1F620, 0x1F625}, - {0x1F626, 0x1F627}, - {0x1F628, 0x1F62B}, - {0x1F62C, 0x1F62C}, - {0x1F62D, 0x1F62D}, - {0x1F62E, 0x1F62F}, - {0x1F630, 0x1F633}, - {0x1F634, 0x1F634}, - {0x1F635, 0x1F640}, - {0x1F641, 0x1F642}, - {0x1F643, 0x1F644}, - {0x1F645, 0x1F64F}, - {0x1F680, 0x1F6C5}, - {0x1F6C6, 0x1F6CF}, - {0x1F6D0, 0x1F6D0}, - {0x1F6D1, 0x1F6D2}, - {0x1F6D3, 0x1F6D4}, - {0x1F6D5, 0x1F6DF}, - {0x1F6E0, 0x1F6EC}, - {0x1F6ED, 0x1F6EF}, - {0x1F6F0, 0x1F6F3}, - {0x1F6F4, 0x1F6F6}, - {0x1F6F7, 0x1F6F8}, - {0x1F6F9, 0x1F6F9}, - {0x1F6FA, 0x1F6FF}, + {0x1F249, 0x1F3FA}, + {0x1F400, 0x1F53D}, + {0x1F546, 0x1F64F}, + {0x1F680, 0x1F6FF}, {0x1F774, 0x1F77F}, - {0x1F7D5, 0x1F7D8}, - {0x1F7D9, 0x1F7FF}, + {0x1F7D5, 0x1F7FF}, {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, {0x1F85A, 0x1F85F}, {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F8FF}, - {0x1F90C, 0x1F90F}, - {0x1F910, 0x1F918}, - {0x1F919, 0x1F91E}, - {0x1F91F, 0x1F91F}, - {0x1F920, 0x1F927}, - {0x1F928, 0x1F92F}, - {0x1F930, 0x1F930}, - {0x1F931, 0x1F932}, - {0x1F933, 0x1F93A}, - {0x1F93C, 0x1F93E}, - {0x1F93F, 0x1F93F}, - {0x1F940, 0x1F945}, - {0x1F947, 0x1F94B}, - {0x1F94C, 0x1F94C}, - {0x1F94D, 0x1F94F}, - {0x1F950, 0x1F95E}, - {0x1F95F, 0x1F96B}, - {0x1F96C, 0x1F970}, - {0x1F971, 0x1F972}, - {0x1F973, 0x1F976}, - {0x1F977, 0x1F979}, - {0x1F97A, 0x1F97A}, - {0x1F97B, 0x1F97B}, - {0x1F97C, 0x1F97F}, - {0x1F980, 0x1F984}, - {0x1F985, 0x1F991}, - {0x1F992, 0x1F997}, - {0x1F998, 0x1F9A2}, - {0x1F9A3, 0x1F9AF}, - {0x1F9B0, 0x1F9B9}, - {0x1F9BA, 0x1F9BF}, - {0x1F9C0, 0x1F9C0}, - {0x1F9C1, 0x1F9C2}, - {0x1F9C3, 0x1F9CF}, - {0x1F9D0, 0x1F9E6}, - {0x1F9E7, 0x1F9FF}, - {0x1FA00, 0x1FA5F}, - {0x1FA60, 0x1FA6D}, - {0x1FA6E, 0x1FFFD}, + {0x1F90C, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1FFFD}, }; #endif /* UNICODE_EMOJI_TABLE_H */ diff --git a/src/font/unicode-emoji.c b/src/font/unicode-emoji.c index 7e3fae6a..08682d42 100644 --- a/src/font/unicode-emoji.c +++ b/src/font/unicode-emoji.c @@ -34,9 +34,12 @@ /* ** Some operators to check Emoji. -** Implementation of the functions is based on some code from LGPL'd Pango. +** Implementation of the functions is based on some code from HarfBuzz +** (licensed under MIT).: ** -** Created by WEI Yongming at 2019/02/26 +** https://github.com/harfbuzz/harfbuzz +** +** Created by WEI Yongming at 2019/03/01 */ #include @@ -82,7 +85,7 @@ DEFINE_unicode_is_(emoji) DEFINE_unicode_is_(emoji_presentation) DEFINE_unicode_is_(emoji_modifier) DEFINE_unicode_is_(emoji_modifier_base) -DEFINE_unicode_is_(emoji_extended_pictographic) +DEFINE_unicode_is_(extended_pictographic) BOOL _unicode_is_emoji_text_default(Uchar32 ch) { diff --git a/src/font/utils/fetch-emoji.sh b/src/font/utils/fetch-emoji.sh new file mode 100755 index 00000000..eec739b8 --- /dev/null +++ b/src/font/utils/fetch-emoji.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +url='http://www.unicode.org/Public/emoji/12.0/' +files='emoji-data.txt emoji-sequences.txt emoji-variation-sequences.txt emoji-zwj-sequences.txt' + +if test ! -d emoji; then + mkdir emoji +fi + +cd emoji/ + +for file in $files +do + if [ -f $file ]; then + echo "$file exists, skipping" + else + echo "Fetching $url$file" + wget $url$file + fi +done diff --git a/src/font/utils/fetch-ucd.sh b/src/font/utils/fetch-ucd.sh new file mode 100755 index 00000000..aaaa7279 --- /dev/null +++ b/src/font/utils/fetch-ucd.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +url='https://www.unicode.org/Public/11.0.0/ucd/' +files='UnicodeData.txt IndicPositionalCategory.txt IndicSyllabicCategory.txt ArabicShaping.txt BidiBrackets.txt BidiMirroring.txt Blocks.txt' + +if test ! -d ucd; then + mkdir ucd +fi + +cd ucd/ + +for file in $files +do + if [ -f $file ]; then + echo "$file exists, skipping" + else + echo "Fetching $url$file" + wget $url$file + fi +done diff --git a/src/font/utils/gen-emoji-table.py b/src/font/utils/gen-emoji-table.py index 21cf9721..00c6ed6f 100755 --- a/src/font/utils/gen-emoji-table.py +++ b/src/font/utils/gen-emoji-table.py @@ -1,12 +1,18 @@ #!/usr/bin/python +# Copied from HarfBuzz +# https://github.com/harfbuzz/harfbuzz +# On 2019-03-07 +# +# Revised by Vincent Wei for MiniGUI 3.4 + from __future__ import print_function, division, absolute_import import sys import os.path from collections import OrderedDict if len (sys.argv) != 2: - print("usage: ./gen-emoji-table.py emoji-data.txt", file=sys.stderr) + print("usage: ./gen-emoji-table.py emoji/emoji-data.txt", file=sys.stderr) sys.exit (1) f = open(sys.argv[1]) @@ -32,13 +38,11 @@ for line in f.readlines(): else: ranges[typ].append((start, end)) - - print("/* == Start of generated table == */") print("/*") print(" * The following tables are generated by running:") print(" *") -print(" * ./gen-emoji-table.py emoji-data.txt") +print(" * ./gen-emoji-table.py emoji/emoji-data.txt") print(" *") print(" * on file with this header:") print(" *") @@ -52,13 +56,13 @@ print() print("struct Interval {\n Uchar32 start, end;\n};") for typ,s in ranges.items(): - if typ not in ['emoji', - 'emoji_presentation', - 'emoji_modifier', - 'emoji_modifier_base', - 'extended_pictographic']: continue + if typ not in ['Emoji', + 'Emoji_Presentation', + 'Emoji_Modifier', + 'Emoji_Modifier_Base', + 'Extended_Pictographic']: continue print() - print("static const struct Interval _unicode_%s_table[] =" % typ) + print("static const struct Interval _unicode_%s_table[] =" % typ.lower()) print("{") for pair in sorted(s): print(" {0x%04X, 0x%04X}," % pair)