[translations] Resolve issue with lines over characters for some languages, and some wrong characters used (#1268)

* Resolve a problem in font generation when y_offset is negative.

* Change to use full dynamic detection of special chars, generalized for all languages including cjk.

* Isolated colorlcd and bwlcd translation encoding method.

* Fixed the cz char mapping.

* Only cz requires subset lowercase handling.

radio/src/translations/CMakeLists.txt

@@ -1,5 +1,13 @@
 file(GLOB translations *.h.txt)
 
+  if(FONTS_TARGET STREQUAL taranis_fonts)
+    set(bwlcd T)
+  elseif(FONTS_TARGET STREQUAL 9x_fonts_1bit)
+    set(bwlcd T)
+  else()
+    set(bwlcd F)
+  endif()
+
 foreach(translation ${translations})
   get_filename_component(language ${translation} NAME_WE)
   get_filename_component(path ${translation} DIRECTORY)
@@ -7,7 +15,7 @@ foreach(translation ${translations})
   if(PYTHONINTERP_FOUND)
     add_custom_command(
       OUTPUT ${target}
-      COMMAND ${PYTHON_EXECUTABLE} ${TOOLS_DIR}/encode-translation.py ${translation} ${target} ${language}
+      COMMAND ${PYTHON_EXECUTABLE} ${TOOLS_DIR}/encode-translation.py ${translation} ${target} ${language} ${bwlcd}
       DEPENDS ${translation} ${TOOLS_DIR}/encode-translation.py ${TOOLS_DIR}/charset.py
       )
   endif()

tools/build-font-bitmap.py

@@ -5,7 +5,7 @@ import argparse
 import os
 import sys
 from PIL import Image, ImageDraw, ImageFont
-from charset import get_chars, special_chars, extra_chars, standard_chars, is_cjk_char
+from charset import get_chars, special_chars, extra_chars, standard_chars
 
 EXTRA_BITMAP_MAX_WIDTH = 297
 
@@ -50,7 +50,7 @@ class FontBitmap:
 
         return (text_width, text_height)
 
-    def draw_char(self, draw, x, c):
+    def draw_char(self, draw, x, y, c):
 
         # default width for space
         width = 4
@@ -64,7 +64,7 @@ class FontBitmap:
               width = self.font.getmask(c).getbbox()[2]
               _, (offset_x, offset_y) = self.font.font.getsize(c)
 
-            draw.text((x - offset_x, 0), c, fill=self.foreground, font=self.font)
+            draw.text((x - offset_x, y), c, fill=self.foreground, font=self.font)
 
         return width
 
@@ -73,6 +73,11 @@ class FontBitmap:
 
         (_,baseline), (offset_x, offset_y) = self.font.font.getsize(self.chars)
         (text_width, text_height) = self.get_text_dimensions(self.chars)
+        y_shifts = 0
+        if offset_y < 0:
+            y_shifts = -offset_y
+            text_height += y_shifts
+            offset_y = 0
 
         img_width = text_width + self.extra_bitmap_width
         image = Image.new("RGB", (img_width, text_height), self.background)
@@ -89,7 +94,7 @@ class FontBitmap:
                     if self.extra_bitmap:
 
                         # copy extra_bitmap at once
-                        image.paste(self.extra_bitmap, (width, offset_y))
+                        image.paste(self.extra_bitmap, (width, offset_y + y_shifts))
 
                         # append width for extra_bitmap symbols
                         for coord in [14, 14, 12, 12, 13, 13, 13, 13, 13] + [15] * 12:
@@ -106,7 +111,7 @@ class FontBitmap:
                 continue
 
             # normal characters + CJK
-            w = self.draw_char(draw, width, c)
+            w = self.draw_char(draw, width, y_shifts, c)
 
             coords.append(width)
             width += w

tools/charset.py

@@ -9,62 +9,32 @@ standard_chars = """ !"#$%&'()*+,-./0123456789:;<=>?°ABCDEFGHIJKLMNOPQRSTUVWXYZ
 
 extra_chars = "".join([chr(0x10000+i) for i in range(21)])
 
-def is_cjk_char(c):
-    return 0x4E00 <= ord(c) <= 0x9FFF
+def is_special_char(c):
+    return 192 <= ord(c) <= 383 or 0x4E00 <= ord(c) <= 0x9FFF
 
-def cjk_chars(lang):
+def get_special_chars():
+  result = {}
+  for lang in["en", "fr", "de", "cz", "nl", "es", "fi", "it", "pl", "pt", "se", "cn", "tw"]:
     charset = set()
     tools_path = os.path.dirname(os.path.realpath(__file__))
     with open(os.path.join(tools_path, "../radio/src/translations/%s.h.txt" % lang), encoding='utf-8') as f:
         data = f.read()
         for c in data:
-            if is_cjk_char(c):
+            if is_special_char(c):
                 charset.add(c)
-                # print(ord(c))
-    result = list(charset)
-    result.sort()
+    data = list(charset)
+    data.sort()
+    result[lang] = data
+
   return result
 
-
-special_chars = {
-    "en": "",
-    "fr": "éèàîç",
-    "de": "ÄäÖöÜüß",
-    "cz": "ěščřžýáíéňóůúďťĚŠČŘŽÝÁÍÉŇÓÚŮĎŤ",
-    "nl": "",
-    "es": "Ññ",
-    "fi": "åäöÅÄÖ",
-    "it": "àù",
-    "pl": "ąćęłńóśżźĄĆĘŁŃÓŚŻŹ",
-    "pt": "ÁáÂâÃãÀàÇçÉéÊêÍíÓóÔôÕõÚú",
-    "se": "åäöÅÄÖ",
-    "cn": "".join(cjk_chars("cn")),
-    "tw": "".join(cjk_chars("tw")),
-}
-
-subset_lowercase = {
-    "Č": "č",
-    "Ě": "ě",
-    "Š": "š",
-    "Ú": "ú",
-    "Ů": "ů",
-    "Ž": "ž"
-}
-
-# print("CN charset: %d symbols" % len(special_chars["cn"]))
-
+special_chars = get_special_chars()
 
 def get_chars(subset):
     result = standard_chars + extra_chars
-    if False: # subset == "all":
-        for key, chars in special_chars.items():
-            result += "".join([char for char in chars if char not in result])
-    else:
-        if subset in special_chars:
-            result += "".join([char for char in special_chars[subset] if char not in subset_lowercase])
+    result += "".join([char for char in special_chars[subset]])
     return result
 
-
 def get_chars_encoding(subset):
     result = {}
     if subset in ("cn", "tw"):
@@ -83,7 +53,51 @@ def get_chars_encoding(subset):
         for char in chars:
             if char not in standard_chars:
                 result[char] = "\\%03o" % (offset + chars.index(char))
-        for upper, lower in subset_lowercase.items():
+    return result
+
+special_chars_BW = {
+    "en": "",
+    "fr": "éèàîç",
+    "de": "ÄäÖöÜüß",
+    "cz": "áčéěíóřšúůýÁÍŘÝžÉ",
+    "nl": "",
+    "es": "ÑñÁáÉéÍíÓóÚú",
+    "fi": "åäöÅÄÖ",
+    "it": "àù",
+    "pl": "ąćęłńóśżźĄĆĘŁŃÓŚŻŹ",
+    "pt": "ÁáÂâÃãÀàÇçÉéÊêÍíÓóÔôÕõÚú",
+    "se": "åäöÅÄÖ",
+    "cn": "",
+    "tw": "",
+}
+
+subset_lowercase_BW = {
+    "Č": "č",
+    "Ě": "ě",
+    "Š": "š",
+    "Ú": "ú",
+    "Ů": "ů",
+    "Ž": "ž"
+}
+
+def get_chars_BW(subset):
+    result = standard_chars + extra_chars
+    if subset in special_chars_BW:
+        if (subset == "cz"):
+            result += "".join([char for char in special_chars_BW[subset] if char not in subset_lowercase_BW])
+        else:
+            result += "".join([char for char in special_chars_BW[subset]])
+    return result
+
+def get_chars_encoding_BW(subset):
+    result = {}
+    offset = 128 - len(standard_chars)
+    chars = get_chars_BW(subset)
+    for char in chars:
+        if char not in standard_chars:
+            result[char] = "\\%03o" % (offset + chars.index(char))
+    if (subset == "cz"):            
+        for upper, lower in subset_lowercase_BW.items():
             if lower in result:
                 result[upper] = result[lower]
     return result

tools/encode-translation.py

@@ -4,20 +4,25 @@
 import argparse
 import codecs
 import sys
-from charset import special_chars, get_chars_encoding
-
+from charset import special_chars, get_chars_encoding, special_chars_BW, get_chars_encoding_BW
 
 def main():
     parser = argparse.ArgumentParser(description='Encoder for OpenTX translations')
     parser.add_argument('input', action="store", help="Input file name")
     parser.add_argument('output', action="store", help="Output file name")
     parser.add_argument('language', action="store", help="Two letter language identifier", default=None)
+    parser.add_argument('bwlcd', action="store", help="Black White LCD", default="F")
     parser.add_argument("--reverse", help="Reversed char conversion (from number to char)", action="store_true")
     args = parser.parse_args()
 
+    if args.bwlcd == "F":
       if args.language not in special_chars:
           parser.error(args.language + ' is not a supported language. Try one of the supported ones: %s' % list(special_chars.keys()))
           sys.exit()
+    else:
+      if args.language not in special_chars_BW:
+          parser.error(args.language + ' is not a supported language. Try one of the supported ones: %s' % list(special_chars_BW.keys()))
+          sys.exit()
 
     # if args.reverse:
     #     for translation in special_chars:
@@ -31,8 +36,13 @@ def main():
 
     for line in in_file.readlines():
         # Do the special chars replacements
+        if args.bwlcd == "F":        
             for before, after in get_chars_encoding(args.language).items():
                 line = line.replace(before, after)
+        else:
+            for before, after in get_chars_encoding_BW(args.language).items():
+                line = line.replace(before, after)
+
         if line.startswith("#define ZSTR_"):
             before = line[32:-2]
             after = ""