152 lines · python
1#!/usr/bin/env python2"""3Unicode case folding database conversion utility4 5Parses the database and generates a C++ function which implements the case6folding algorithm. The database entries are of the form:7 8 <code>; <status>; <mapping>; # <name>9 10<status> can be one of four characters:11 C - Common mappings12 S - mappings for Simple case folding13 F - mappings for Full case folding14 T - special case for Turkish I characters15 16Right now this generates a function which implements simple case folding (C+S17entries).18"""19 20from __future__ import print_function21 22import sys23import re24from urllib.request import urlopen25 26 27# This variable will body of the mappings function28body = ""29 30# Reads file line-by-line, extracts Common and Simple case fold mappings and31# returns a (from_char, to_char, from_name) tuple.32def mappings(f):33 previous_from = -134 expr = re.compile(r"^(.*); [CS]; (.*); # (.*)")35 for line in f:36 m = expr.match(line)37 if not m:38 continue39 from_char = int(m.group(1), 16)40 to_char = int(m.group(2), 16)41 from_name = m.group(3)42 43 if from_char <= previous_from:44 raise Exception("Duplicate or unsorted characters in input")45 yield from_char, to_char, from_name46 previous_from = from_char47 48 49# Computes the shift (to_char - from_char) in a mapping.50def shift(mapping):51 return mapping[1] - mapping[0]52 53 54# Computes the stride (from_char2 - from_char1) of two mappings.55def stride2(mapping1, mapping2):56 return mapping2[0] - mapping1[0]57 58 59# Computes the stride of a list of mappings. The list should have at least two60# mappings. All mappings in the list are assumed to have the same stride.61def stride(block):62 return stride2(block[0], block[1])63 64 65# b is a list of mappings. All the mappings are assumed to have the same66# shift and the stride between adjecant mappings (if any) is constant.67def dump_block(b):68 global body69 70 if len(b) == 1:71 # Special case for handling blocks of length 1. We don't even need to72 # emit the "if (C < X) return C" check below as all characters in this73 # range will be caught by the "C < X" check emitted by the first74 # non-trivial block.75 body += " // {2}\n if (C == {0:#06x})\n return {1:#06x};\n".format(*b[0])76 return77 78 first = b[0][0]79 last = first + stride(b) * (len(b) - 1)80 modulo = first % stride(b)81 82 # All characters before this block map to themselves.83 body += " if (C < {0:#06x})\n return C;\n".format(first)84 body += " // {0} characters\n".format(len(b))85 86 # Generic pattern: check upper bound (lower bound is checked by the "if"87 # above) and modulo of C, return C+shift.88 pattern = " if (C <= {0:#06x} && C % {1} == {2})\n return C + {3};\n"89 90 if stride(b) == 2 and shift(b[0]) == 1 and modulo == 0:91 # Special case:92 # We can elide the modulo-check because the expression "C|1" will map93 # the intervening characters to themselves.94 pattern = " if (C <= {0:#06x})\n return C | 1;\n"95 elif stride(b) == 1:96 # Another special case: X % 1 is always zero, so don't emit the97 # modulo-check.98 pattern = " if (C <= {0:#06x})\n return C + {3};\n"99 100 body += pattern.format(last, stride(b), modulo, shift(b[0]))101 102 103current_block = []104f = urlopen(sys.argv[1])105for m in mappings(f):106 if len(current_block) == 0:107 current_block.append(m)108 continue109 110 if shift(current_block[0]) != shift(m):111 # Incompatible shift, start a new block.112 dump_block(current_block)113 current_block = [m]114 continue115 116 if len(current_block) == 1 or stride(current_block) == stride2(117 current_block[-1], m118 ):119 current_block.append(m)120 continue121 122 # Incompatible stride, start a new block.123 dump_block(current_block)124 current_block = [m]125f.close()126 127dump_block(current_block)128 129print(130 "//===---------- Support/UnicodeCaseFold.cpp -------------------------------===//"131)132print("//")133print("// This file was generated by utils/unicode-case-fold.py from the Unicode")134print("// case folding database at")135print("// ", sys.argv[1])136print("//")137print("// To regenerate this file, run:")138print("// utils/unicode-case-fold.py \\")139print('// "{}" \\'.format(sys.argv[1]))140print("// > lib/Support/UnicodeCaseFold.cpp")141print("//")142print(143 "//===----------------------------------------------------------------------===//"144)145print("")146print('#include "llvm/Support/Unicode.h"')147print("")148print("int llvm::sys::unicode::foldCharSimple(int C) {")149print(body)150print(" return C;")151print("}")152