update emoji data tables

src/font/unicode-emoji-table.h

@@ -2,18 +2,18 @@
 /*
  * The following tables are generated by running:
  *
- *   ./gen-emoji-table.py emoji-data.txt
+ *   ./gen-emoji-table.py emoji/emoji-data.txt
  *
  * on file with this header:
  *
  * # emoji-data.txt
- * # Date: 2018-02-07, 07:55:18 GMT
- * # © 2018 Unicode®, Inc.
+ * # Date: 2019-01-15, 12:10:05 GMT
+ * # © 2019 Unicode®, Inc.
  * # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
  * # For terms of use, see http://www.unicode.org/terms_of_use.html
  * #
  * # Emoji Data for UTS #51
- * # Version: 11.0
+ * # Version: 12.0
  * #
  * # For documentation and usage, see http://www.unicode.org/reports/tr51
  */
@@ -22,7 +22,7 @@
 #define UNICODE_EMOJI_TABLE_H
 
 struct Interval {
-    Uchar32 start, end;
+  Uchar32 start, end;
 };
 
 static const struct Interval _unicode_emoji_table[] =
@@ -159,21 +159,25 @@ static const struct Interval _unicode_emoji_table[] =
   {0x1F5FA, 0x1F64F},
   {0x1F680, 0x1F6C5},
   {0x1F6CB, 0x1F6D2},
+  {0x1F6D5, 0x1F6D5},
   {0x1F6E0, 0x1F6E5},
   {0x1F6E9, 0x1F6E9},
   {0x1F6EB, 0x1F6EC},
   {0x1F6F0, 0x1F6F0},
-  {0x1F6F3, 0x1F6F9},
-  {0x1F910, 0x1F93A},
-  {0x1F93C, 0x1F93E},
-  {0x1F940, 0x1F945},
-  {0x1F947, 0x1F970},
+  {0x1F6F3, 0x1F6FA},
+  {0x1F7E0, 0x1F7EB},
+  {0x1F90D, 0x1F93A},
+  {0x1F93C, 0x1F945},
+  {0x1F947, 0x1F971},
   {0x1F973, 0x1F976},
-  {0x1F97A, 0x1F97A},
-  {0x1F97C, 0x1F9A2},
-  {0x1F9B0, 0x1F9B9},
-  {0x1F9C0, 0x1F9C2},
-  {0x1F9D0, 0x1F9FF},
+  {0x1F97A, 0x1F9A2},
+  {0x1F9A5, 0x1F9AA},
+  {0x1F9AE, 0x1F9CA},
+  {0x1F9CD, 0x1F9FF},
+  {0x1FA70, 0x1FA73},
+  {0x1FA78, 0x1FA7A},
+  {0x1FA80, 0x1FA82},
+  {0x1FA90, 0x1FA95},
 };
 
 static const struct Interval _unicode_emoji_presentation_table[] =
@@ -243,18 +247,22 @@ static const struct Interval _unicode_emoji_presentation_table[] =
   {0x1F680, 0x1F6C5},
   {0x1F6CC, 0x1F6CC},
   {0x1F6D0, 0x1F6D2},
+  {0x1F6D5, 0x1F6D5},
   {0x1F6EB, 0x1F6EC},
-  {0x1F6F4, 0x1F6F9},
-  {0x1F910, 0x1F93A},
-  {0x1F93C, 0x1F93E},
-  {0x1F940, 0x1F945},
-  {0x1F947, 0x1F970},
+  {0x1F6F4, 0x1F6FA},
+  {0x1F7E0, 0x1F7EB},
+  {0x1F90D, 0x1F93A},
+  {0x1F93C, 0x1F945},
+  {0x1F947, 0x1F971},
   {0x1F973, 0x1F976},
-  {0x1F97A, 0x1F97A},
-  {0x1F97C, 0x1F9A2},
-  {0x1F9B0, 0x1F9B9},
-  {0x1F9C0, 0x1F9C2},
-  {0x1F9D0, 0x1F9FF},
+  {0x1F97A, 0x1F9A2},
+  {0x1F9A5, 0x1F9AA},
+  {0x1F9AE, 0x1F9CA},
+  {0x1F9CD, 0x1F9FF},
+  {0x1FA70, 0x1FA73},
+  {0x1FA78, 0x1FA7A},
+  {0x1FA80, 0x1FA82},
+  {0x1FA90, 0x1FA95},
 };
 
 static const struct Interval _unicode_emoji_modifier_table[] =
@@ -273,12 +281,12 @@ static const struct Interval _unicode_emoji_modifier_base_table[] =
   {0x1F3CA, 0x1F3CC},
   {0x1F442, 0x1F443},
   {0x1F446, 0x1F450},
-  {0x1F466, 0x1F469},
-  {0x1F46E, 0x1F46E},
-  {0x1F470, 0x1F478},
+  {0x1F466, 0x1F478},
   {0x1F47C, 0x1F47C},
   {0x1F481, 0x1F483},
   {0x1F485, 0x1F487},
+  {0x1F48F, 0x1F48F},
+  {0x1F491, 0x1F491},
   {0x1F4AA, 0x1F4AA},
   {0x1F574, 0x1F575},
   {0x1F57A, 0x1F57A},
@@ -290,17 +298,19 @@ static const struct Interval _unicode_emoji_modifier_base_table[] =
   {0x1F6B4, 0x1F6B6},
   {0x1F6C0, 0x1F6C0},
   {0x1F6CC, 0x1F6CC},
-  {0x1F918, 0x1F91C},
-  {0x1F91E, 0x1F91F},
+  {0x1F90F, 0x1F90F},
+  {0x1F918, 0x1F91F},
   {0x1F926, 0x1F926},
   {0x1F930, 0x1F939},
-  {0x1F93D, 0x1F93E},
+  {0x1F93C, 0x1F93E},
   {0x1F9B5, 0x1F9B6},
   {0x1F9B8, 0x1F9B9},
+  {0x1F9BB, 0x1F9BB},
+  {0x1F9CD, 0x1F9CF},
   {0x1F9D1, 0x1F9DD},
 };
 
-static const struct Interval _unicode_emoji_extended_pictographic_table[] =
+static const struct Interval _unicode_extended_pictographic_table[] =
 {
   {0x00A9, 0x00A9},
   {0x00AE, 0x00AE},
@@ -323,38 +333,9 @@ static const struct Interval _unicode_emoji_extended_pictographic_table[] =
   {0x25FB, 0x25FE},
   {0x2600, 0x2605},
   {0x2607, 0x2612},
-  {0x2614, 0x2615},
-  {0x2616, 0x2617},
-  {0x2618, 0x2618},
-  {0x2619, 0x2619},
-  {0x261A, 0x266F},
-  {0x2670, 0x2671},
-  {0x2672, 0x267D},
-  {0x267E, 0x267F},
-  {0x2680, 0x2685},
-  {0x2690, 0x2691},
-  {0x2692, 0x269C},
-  {0x269D, 0x269D},
-  {0x269E, 0x269F},
-  {0x26A0, 0x26A1},
-  {0x26A2, 0x26B1},
-  {0x26B2, 0x26B2},
-  {0x26B3, 0x26BC},
-  {0x26BD, 0x26BF},
-  {0x26C0, 0x26C3},
-  {0x26C4, 0x26CD},
-  {0x26CE, 0x26CE},
-  {0x26CF, 0x26E1},
-  {0x26E2, 0x26E2},
-  {0x26E3, 0x26E3},
-  {0x26E4, 0x26E7},
-  {0x26E8, 0x26FF},
-  {0x2700, 0x2700},
-  {0x2701, 0x2704},
-  {0x2705, 0x2705},
-  {0x2708, 0x2709},
-  {0x270A, 0x270B},
-  {0x270C, 0x2712},
+  {0x2614, 0x2685},
+  {0x2690, 0x2705},
+  {0x2708, 0x2712},
   {0x2714, 0x2714},
   {0x2716, 0x2716},
   {0x271D, 0x271D},
@@ -381,163 +362,33 @@ static const struct Interval _unicode_emoji_extended_pictographic_table[] =
   {0x303D, 0x303D},
   {0x3297, 0x3297},
   {0x3299, 0x3299},
-  {0x1F000, 0x1F02B},
-  {0x1F02C, 0x1F02F},
-  {0x1F030, 0x1F093},
-  {0x1F094, 0x1F09F},
-  {0x1F0A0, 0x1F0AE},
-  {0x1F0AF, 0x1F0B0},
-  {0x1F0B1, 0x1F0BE},
-  {0x1F0BF, 0x1F0BF},
-  {0x1F0C0, 0x1F0C0},
-  {0x1F0C1, 0x1F0CF},
-  {0x1F0D0, 0x1F0D0},
-  {0x1F0D1, 0x1F0DF},
-  {0x1F0E0, 0x1F0F5},
-  {0x1F0F6, 0x1F0FF},
+  {0x1F000, 0x1F0FF},
   {0x1F10D, 0x1F10F},
   {0x1F12F, 0x1F12F},
-  {0x1F16C, 0x1F16F},
-  {0x1F170, 0x1F171},
-  {0x1F17E, 0x1F17E},
-  {0x1F17F, 0x1F17F},
+  {0x1F16C, 0x1F171},
+  {0x1F17E, 0x1F17F},
   {0x1F18E, 0x1F18E},
   {0x1F191, 0x1F19A},
   {0x1F1AD, 0x1F1E5},
-  {0x1F201, 0x1F202},
-  {0x1F203, 0x1F20F},
+  {0x1F201, 0x1F20F},
   {0x1F21A, 0x1F21A},
   {0x1F22F, 0x1F22F},
   {0x1F232, 0x1F23A},
   {0x1F23C, 0x1F23F},
-  {0x1F249, 0x1F24F},
-  {0x1F250, 0x1F251},
-  {0x1F252, 0x1F25F},
-  {0x1F260, 0x1F265},
-  {0x1F266, 0x1F2FF},
-  {0x1F300, 0x1F320},
-  {0x1F321, 0x1F32C},
-  {0x1F32D, 0x1F32F},
-  {0x1F330, 0x1F335},
-  {0x1F336, 0x1F336},
-  {0x1F337, 0x1F37C},
-  {0x1F37D, 0x1F37D},
-  {0x1F37E, 0x1F37F},
-  {0x1F380, 0x1F393},
-  {0x1F394, 0x1F39F},
-  {0x1F3A0, 0x1F3C4},
-  {0x1F3C5, 0x1F3C5},
-  {0x1F3C6, 0x1F3CA},
-  {0x1F3CB, 0x1F3CE},
-  {0x1F3CF, 0x1F3D3},
-  {0x1F3D4, 0x1F3DF},
-  {0x1F3E0, 0x1F3F0},
-  {0x1F3F1, 0x1F3F7},
-  {0x1F3F8, 0x1F3FA},
-  {0x1F400, 0x1F43E},
-  {0x1F43F, 0x1F43F},
-  {0x1F440, 0x1F440},
-  {0x1F441, 0x1F441},
-  {0x1F442, 0x1F4F7},
-  {0x1F4F8, 0x1F4F8},
-  {0x1F4F9, 0x1F4FC},
-  {0x1F4FD, 0x1F4FE},
-  {0x1F4FF, 0x1F4FF},
-  {0x1F500, 0x1F53D},
-  {0x1F546, 0x1F54A},
-  {0x1F54B, 0x1F54F},
-  {0x1F550, 0x1F567},
-  {0x1F568, 0x1F579},
-  {0x1F57A, 0x1F57A},
-  {0x1F57B, 0x1F5A3},
-  {0x1F5A4, 0x1F5A4},
-  {0x1F5A5, 0x1F5FA},
-  {0x1F5FB, 0x1F5FF},
-  {0x1F600, 0x1F600},
-  {0x1F601, 0x1F610},
-  {0x1F611, 0x1F611},
-  {0x1F612, 0x1F614},
-  {0x1F615, 0x1F615},
-  {0x1F616, 0x1F616},
-  {0x1F617, 0x1F617},
-  {0x1F618, 0x1F618},
-  {0x1F619, 0x1F619},
-  {0x1F61A, 0x1F61A},
-  {0x1F61B, 0x1F61B},
-  {0x1F61C, 0x1F61E},
-  {0x1F61F, 0x1F61F},
-  {0x1F620, 0x1F625},
-  {0x1F626, 0x1F627},
-  {0x1F628, 0x1F62B},
-  {0x1F62C, 0x1F62C},
-  {0x1F62D, 0x1F62D},
-  {0x1F62E, 0x1F62F},
-  {0x1F630, 0x1F633},
-  {0x1F634, 0x1F634},
-  {0x1F635, 0x1F640},
-  {0x1F641, 0x1F642},
-  {0x1F643, 0x1F644},
-  {0x1F645, 0x1F64F},
-  {0x1F680, 0x1F6C5},
-  {0x1F6C6, 0x1F6CF},
-  {0x1F6D0, 0x1F6D0},
-  {0x1F6D1, 0x1F6D2},
-  {0x1F6D3, 0x1F6D4},
-  {0x1F6D5, 0x1F6DF},
-  {0x1F6E0, 0x1F6EC},
-  {0x1F6ED, 0x1F6EF},
-  {0x1F6F0, 0x1F6F3},
-  {0x1F6F4, 0x1F6F6},
-  {0x1F6F7, 0x1F6F8},
-  {0x1F6F9, 0x1F6F9},
-  {0x1F6FA, 0x1F6FF},
+  {0x1F249, 0x1F3FA},
+  {0x1F400, 0x1F53D},
+  {0x1F546, 0x1F64F},
+  {0x1F680, 0x1F6FF},
   {0x1F774, 0x1F77F},
-  {0x1F7D5, 0x1F7D8},
-  {0x1F7D9, 0x1F7FF},
+  {0x1F7D5, 0x1F7FF},
   {0x1F80C, 0x1F80F},
   {0x1F848, 0x1F84F},
   {0x1F85A, 0x1F85F},
   {0x1F888, 0x1F88F},
   {0x1F8AE, 0x1F8FF},
-  {0x1F90C, 0x1F90F},
-  {0x1F910, 0x1F918},
-  {0x1F919, 0x1F91E},
-  {0x1F91F, 0x1F91F},
-  {0x1F920, 0x1F927},
-  {0x1F928, 0x1F92F},
-  {0x1F930, 0x1F930},
-  {0x1F931, 0x1F932},
-  {0x1F933, 0x1F93A},
-  {0x1F93C, 0x1F93E},
-  {0x1F93F, 0x1F93F},
-  {0x1F940, 0x1F945},
-  {0x1F947, 0x1F94B},
-  {0x1F94C, 0x1F94C},
-  {0x1F94D, 0x1F94F},
-  {0x1F950, 0x1F95E},
-  {0x1F95F, 0x1F96B},
-  {0x1F96C, 0x1F970},
-  {0x1F971, 0x1F972},
-  {0x1F973, 0x1F976},
-  {0x1F977, 0x1F979},
-  {0x1F97A, 0x1F97A},
-  {0x1F97B, 0x1F97B},
-  {0x1F97C, 0x1F97F},
-  {0x1F980, 0x1F984},
-  {0x1F985, 0x1F991},
-  {0x1F992, 0x1F997},
-  {0x1F998, 0x1F9A2},
-  {0x1F9A3, 0x1F9AF},
-  {0x1F9B0, 0x1F9B9},
-  {0x1F9BA, 0x1F9BF},
-  {0x1F9C0, 0x1F9C0},
-  {0x1F9C1, 0x1F9C2},
-  {0x1F9C3, 0x1F9CF},
-  {0x1F9D0, 0x1F9E6},
-  {0x1F9E7, 0x1F9FF},
-  {0x1FA00, 0x1FA5F},
-  {0x1FA60, 0x1FA6D},
-  {0x1FA6E, 0x1FFFD},
+  {0x1F90C, 0x1F93A},
+  {0x1F93C, 0x1F945},
+  {0x1F947, 0x1FFFD},
 };
 
 #endif /* UNICODE_EMOJI_TABLE_H */

src/font/unicode-emoji.c

@@ -34,9 +34,12 @@
 
 /*
 ** Some operators to check Emoji.
-** Implementation of the functions is based on some code from LGPL'd Pango.
+** Implementation of the functions is based on some code from HarfBuzz
+** (licensed under MIT).:
 **
-** Created by WEI Yongming at 2019/02/26
+**  https://github.com/harfbuzz/harfbuzz
+**
+** Created by WEI Yongming at 2019/03/01
 */
 
 #include <stdio.h>
@@ -82,7 +85,7 @@ DEFINE_unicode_is_(emoji)
 DEFINE_unicode_is_(emoji_presentation)
 DEFINE_unicode_is_(emoji_modifier)
 DEFINE_unicode_is_(emoji_modifier_base)
-DEFINE_unicode_is_(emoji_extended_pictographic)
+DEFINE_unicode_is_(extended_pictographic)
 
 BOOL _unicode_is_emoji_text_default(Uchar32 ch)
 {

src/font/utils/fetch-emoji.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+url='http://www.unicode.org/Public/emoji/12.0/'
+files='emoji-data.txt emoji-sequences.txt emoji-variation-sequences.txt emoji-zwj-sequences.txt'
+
+if test ! -d emoji; then
+    mkdir emoji
+fi
+
+cd emoji/
+
+for file in $files
+do
+    if [ -f $file ]; then
+        echo "$file exists, skipping"
+    else
+        echo "Fetching $url$file"
+        wget $url$file
+    fi
+done

src/font/utils/fetch-ucd.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+url='https://www.unicode.org/Public/11.0.0/ucd/'
+files='UnicodeData.txt IndicPositionalCategory.txt IndicSyllabicCategory.txt ArabicShaping.txt BidiBrackets.txt BidiMirroring.txt Blocks.txt'
+
+if test ! -d ucd; then
+    mkdir ucd
+fi
+
+cd ucd/
+
+for file in $files
+do
+    if [ -f $file ]; then
+        echo "$file exists, skipping"
+    else
+        echo "Fetching $url$file"
+        wget $url$file
+    fi
+done

src/font/utils/gen-emoji-table.py

@@ -1,12 +1,18 @@
 #!/usr/bin/python
 
+# Copied from HarfBuzz
+#   https://github.com/harfbuzz/harfbuzz
+# On 2019-03-07
+#
+# Revised by Vincent Wei for MiniGUI 3.4
+
 from __future__ import print_function, division, absolute_import
 import sys
 import os.path
 from collections import OrderedDict
 
 if len (sys.argv) != 2:
-	print("usage: ./gen-emoji-table.py emoji-data.txt", file=sys.stderr)
+	print("usage: ./gen-emoji-table.py emoji/emoji-data.txt", file=sys.stderr)
 	sys.exit (1)
 
 f = open(sys.argv[1])
@@ -32,13 +38,11 @@ for line in f.readlines():
 	else:
 		ranges[typ].append((start, end))
 
-
-
 print("/* == Start of generated table == */")
 print("/*")
 print(" * The following tables are generated by running:")
 print(" *")
-print(" *   ./gen-emoji-table.py emoji-data.txt")
+print(" *   ./gen-emoji-table.py emoji/emoji-data.txt")
 print(" *")
 print(" * on file with this header:")
 print(" *")
@@ -52,13 +56,13 @@ print()
 print("struct Interval {\n  Uchar32 start, end;\n};")
 
 for typ,s in ranges.items():
-	if typ not in ['emoji',
-		       'emoji_presentation',
-		       'emoji_modifier',
-		       'emoji_modifier_base',
-		       'extended_pictographic']: continue
+	if typ not in ['Emoji',
+		       'Emoji_Presentation',
+		       'Emoji_Modifier',
+		       'Emoji_Modifier_Base',
+		       'Extended_Pictographic']: continue
 	print()
-	print("static const struct Interval _unicode_%s_table[] =" % typ)
+	print("static const struct Interval _unicode_%s_table[] =" % typ.lower())
 	print("{")
 	for pair in sorted(s):
 		print("  {0x%04X, 0x%04X}," % pair)