1 import re, unicodedata, sys

     2 

     3 if sys.maxunicode == 65535:

     4     raise RuntimeError, "need UCS-4 Python"

     5 

     6 def gen_category(cats):

     7     for i in range(0, 0x110000):

     8         if unicodedata.category(unichr(i)) in cats:

     9             yield(i)

    10 

    11 def gen_bidirectional(cats):

    12     for i in range(0, 0x110000):

    13         if unicodedata.bidirectional(unichr(i)) in cats:

    14             yield(i)

    15 

    16 def compact_set(l):

    17     single = []

    18     tuple = []

    19     prev = None

    20     span = 0

    21     for e in l:

    22         if prev is None:

    23             prev = e

    24             span = 0

    25             continue

    26         if prev+span+1 != e:

    27             if span > 2:

    28                 tuple.append((prev,prev+span+1))

    29             else:

    30                 for i in range(prev, prev+span+1):

    31                     single.append(i)

    32             prev = e

    33             span = 0

    34         else:

    35             span += 1

    36     if span:

    37         tuple.append((prev,prev+span+1))

    38     else:

    39         single.append(prev)

    40     tuple = " + ".join(["range(%d,%d)" % t for t in tuple])

    41     if not single:

    42         return "set(%s)" % tuple

    43     if not tuple:

    44         return "set(%s)" % repr(single)

    45     return "set(%s + %s)" % (repr(single),tuple)

    46 

    47 ############## Read the tables in the RFC #######################

    48 

    49 data = open("rfc3454.txt").readlines()

    50 

    51 tables = []

    52 curname = None

    53 for l in data:

    54     l = l.strip()

    55     if not l:

    56         continue

    57     # Skip RFC page breaks

    58     if l.startswith("Hoffman & Blanchet") or\

    59        l.startswith("RFC 3454"):

    60         continue

    61     # Find start/end lines

    62     m = re.match("----- (Start|End) Table ([A-Z](.[0-9])+) -----", l)

    63     if m:

    64         if m.group(1) == "Start":

    65             if curname:

    66                 raise "Double Start",(curname, l)

    67             curname = m.group(2)

    68             table = {}

    69             tables.append((curname, table))

    70             continue

    71         else:

    72             if not curname:

    73                 raise "End without start", l

    74             curname = None

    75             continue

    76     if not curname:

    77         continue

    78     # Now we are in a table

    79     fields = l.split(";")

    80     if len(fields) > 1:

    81         # Drop comment field

    82         fields = fields[:-1]

    83     if len(fields) == 1:

    84         fields = fields[0].split("-")

    85         if len(fields) > 1:

    86             # range

    87             try:

    88                 start, end = fields

    89             except ValueError:

    90                 raise "Unpacking problem", l

    91         else:

    92             start = end = fields[0]

    93         start = int(start, 16)

    94         end = int(end, 16)

    95         for i in range(start, end+1):

    96             table[i] = i

    97     else:

    98         code, value = fields

    99         value = value.strip()

   100         if value:

   101             value = [int(v, 16) for v in value.split(" ")]

   102         else:

   103             # table B.1

   104             value = None

   105         table[int(code, 16)] = value

   106 

   107 ########### Generate compact Python versions of the tables #############

   108 

   109 print """# This file is generated by mkstringprep.py. DO NOT EDIT.

   110 \"\"\"Library that exposes various tables found in the StringPrep RFC 3454.

   111 

   112 There are two kinds of tables: sets, for which a member test is provided,

   113 and mappings, for which a mapping function is provided.

   114 \"\"\"

   115 

   116 import unicodedata

   117 """

   118 

   119 print "assert unicodedata.unidata_version == %s" % repr(unicodedata.unidata_version)

   120 

   121 # A.1 is the table of unassigned characters

   122 # XXX Plane 15 PUA is listed as unassigned in Python.

   123 name, table = tables[0]

   124 del tables[0]

   125 assert name == "A.1"

   126 table = set(table.keys())

   127 Cn = set(gen_category(["Cn"]))

   128 

   129 # FDD0..FDEF are process internal codes

   130 Cn -= set(range(0xFDD0, 0xFDF0))

   131 # not a character

   132 Cn -= set(range(0xFFFE, 0x110000, 0x10000))

   133 Cn -= set(range(0xFFFF, 0x110000, 0x10000))

   134 

   135 # assert table == Cn

   136 

   137 print """

   138 def in_table_a1(code):

   139     if unicodedata.category(code) != 'Cn': return False

   140     c = ord(code)

   141     if 0xFDD0 <= c < 0xFDF0: return False

   142     return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)

   143 """

   144 

   145 # B.1 cannot easily be derived

   146 name, table = tables[0]

   147 del tables[0]

   148 assert name == "B.1"

   149 table = table.keys()

   150 table.sort()

   151 print """

   152 b1_set = """ + compact_set(table) + """

   153 def in_table_b1(code):

   154     return ord(code) in b1_set

   155 """

   156 

   157 # B.2 and B.3 is case folding.

   158 # It takes CaseFolding.txt into account, which is

   159 # not available in the Python database. Since

   160 # B.2 is derived from B.3, we process B.3 first.

   161 # B.3 supposedly *is* CaseFolding-3.2.0.txt.

   162 

   163 name, table_b2 = tables[0]

   164 del tables[0]

   165 assert name == "B.2"

   166 

   167 name, table_b3 = tables[0]

   168 del tables[0]

   169 assert name == "B.3"

   170 

   171 # B.3 is mostly Python's .lower, except for a number

   172 # of special cases, e.g. considering canonical forms.

   173 

   174 b3_exceptions = {}

   175 

   176 for k,v in table_b2.items():

   177     if map(ord, unichr(k).lower()) != v:

   178         b3_exceptions[k] = u"".join(map(unichr,v))

   179 

   180 b3 = b3_exceptions.items()

   181 b3.sort()

   182 

   183 print """

   184 b3_exceptions = {"""

   185 for i,(k,v) in enumerate(b3):

   186     print "0x%x:%s," % (k, repr(v)),

   187     if i % 4 == 3:

   188         print

   189 print "}"

   190 

   191 print """

   192 def map_table_b3(code):

   193     r = b3_exceptions.get(ord(code))

   194     if r is not None: return r

   195     return code.lower()

   196 """

   197 

   198 def map_table_b3(code):

   199     r = b3_exceptions.get(ord(code))

   200     if r is not None: return r

   201     return code.lower()

   202 

   203 # B.2 is case folding for NFKC. This is the same as B.3,

   204 # except where NormalizeWithKC(Fold(a)) !=

   205 # NormalizeWithKC(Fold(NormalizeWithKC(Fold(a))))

   206 

   207 def map_table_b2(a):

   208     al = map_table_b3(a)

   209     b = unicodedata.normalize("NFKC", al)

   210     bl = u"".join([map_table_b3(ch) for ch in b])

   211     c = unicodedata.normalize("NFKC", bl)

   212     if b != c:

   213         return c

   214     else:

   215         return al

   216 

   217 specials = {}

   218 for k,v in table_b2.items():

   219     if map(ord, map_table_b2(unichr(k))) != v:

   220         specials[k] = v

   221 

   222 # B.3 should not add any additional special cases

   223 assert specials == {}

   224 

   225 print """

   226 def map_table_b2(a):

   227     al = map_table_b3(a)

   228     b = unicodedata.normalize("NFKC", al)

   229     bl = u"".join([map_table_b3(ch) for ch in b])

   230     c = unicodedata.normalize("NFKC", bl)

   231     if b != c:

   232         return c

   233     else:

   234         return al

   235 """

   236 

   237 # C.1.1 is a table with a single character

   238 name, table = tables[0]

   239 del tables[0]

   240 assert name == "C.1.1"

   241 assert table == {0x20:0x20}

   242 

   243 print """

   244 def in_table_c11(code):

   245     return code == u" "

   246 """

   247 

   248 # C.1.2 is the rest of all space characters

   249 name, table = tables[0]

   250 del tables[0]

   251 assert name == "C.1.2"

   252 

   253 # table = set(table.keys())

   254 # Zs = set(gen_category(["Zs"])) - set([0x20])

   255 # assert Zs == table

   256 

   257 print """

   258 def in_table_c12(code):

   259     return unicodedata.category(code) == "Zs" and code != u" "

   260 

   261 def in_table_c11_c12(code):

   262     return unicodedata.category(code) == "Zs"

   263 """

   264 

   265 # C.2.1 ASCII control characters

   266 name, table_c21 = tables[0]

   267 del tables[0]

   268 assert name == "C.2.1"

   269 

   270 Cc = set(gen_category(["Cc"]))

   271 Cc_ascii = Cc & set(range(128))

   272 table_c21 = set(table_c21.keys())

   273 assert Cc_ascii == table_c21

   274 

   275 print """

   276 def in_table_c21(code):

   277     return ord(code) < 128 and unicodedata.category(code) == "Cc"

   278 """

   279 

   280 # C.2.2 Non-ASCII control characters. It also includes

   281 # a number of characters in category Cf.

   282 name, table_c22 = tables[0]

   283 del tables[0]

   284 assert name == "C.2.2"

   285 

   286 Cc_nonascii = Cc - Cc_ascii

   287 table_c22 = set(table_c22.keys())

   288 assert len(Cc_nonascii - table_c22) == 0

   289 

   290 specials = list(table_c22 - Cc_nonascii)

   291 specials.sort()

   292 

   293 print """c22_specials = """ + compact_set(specials) + """

   294 def in_table_c22(code):

   295     c = ord(code)

   296     if c < 128: return False

   297     if unicodedata.category(code) == "Cc": return True

   298     return c in c22_specials

   299 

   300 def in_table_c21_c22(code):

   301     return unicodedata.category(code) == "Cc" or \\

   302            ord(code) in c22_specials

   303 """

   304 

   305 # C.3 Private use

   306 name, table = tables[0]

   307 del tables[0]

   308 assert name == "C.3"

   309 

   310 Co = set(gen_category(["Co"]))

   311 assert set(table.keys()) == Co

   312 

   313 print """

   314 def in_table_c3(code):

   315     return unicodedata.category(code) == "Co"

   316 """

   317 

   318 # C.4 Non-character code points, xFFFE, xFFFF

   319 # plus process internal codes

   320 name, table = tables[0]

   321 del tables[0]

   322 assert name == "C.4"

   323 

   324 nonchar = set(range(0xFDD0,0xFDF0) +

   325               range(0xFFFE,0x110000,0x10000) +

   326               range(0xFFFF,0x110000,0x10000))

   327 table = set(table.keys())

   328 assert table == nonchar

   329 

   330 print """

   331 def in_table_c4(code):

   332     c = ord(code)

   333     if c < 0xFDD0: return False

   334     if c < 0xFDF0: return True

   335     return (ord(code) & 0xFFFF) in (0xFFFE, 0xFFFF)

   336 """

   337 

   338 # C.5 Surrogate codes

   339 name, table = tables[0]

   340 del tables[0]

   341 assert name == "C.5"

   342 

   343 Cs = set(gen_category(["Cs"]))

   344 assert set(table.keys()) == Cs

   345 

   346 print """

   347 def in_table_c5(code):

   348     return unicodedata.category(code) == "Cs"

   349 """

   350 

   351 # C.6 Inappropriate for plain text

   352 name, table = tables[0]

   353 del tables[0]

   354 assert name == "C.6"

   355 

   356 table = table.keys()

   357 table.sort()

   358 

   359 print """

   360 c6_set = """ + compact_set(table) + """

   361 def in_table_c6(code):

   362     return ord(code) in c6_set

   363 """

   364 

   365 # C.7 Inappropriate for canonical representation

   366 name, table = tables[0]

   367 del tables[0]

   368 assert name == "C.7"

   369 

   370 table = table.keys()

   371 table.sort()

   372 

   373 print """

   374 c7_set = """ + compact_set(table) + """

   375 def in_table_c7(code):

   376     return ord(code) in c7_set

   377 """

   378 

   379 # C.8 Change display properties or are deprecated

   380 name, table = tables[0]

   381 del tables[0]

   382 assert name == "C.8"

   383 

   384 table = table.keys()

   385 table.sort()

   386 

   387 print """

   388 c8_set = """ + compact_set(table) + """

   389 def in_table_c8(code):

   390     return ord(code) in c8_set

   391 """

   392 

   393 # C.9 Tagging characters

   394 name, table = tables[0]

   395 del tables[0]

   396 assert name == "C.9"

   397 

   398 table = table.keys()

   399 table.sort()

   400 

   401 print """

   402 c9_set = """ + compact_set(table) + """

   403 def in_table_c9(code):

   404     return ord(code) in c9_set

   405 """

   406 

   407 # D.1 Characters with bidirectional property "R" or "AL"

   408 name, table = tables[0]

   409 del tables[0]

   410 assert name == "D.1"

   411 

   412 RandAL = set(gen_bidirectional(["R","AL"]))

   413 assert set(table.keys()) == RandAL

   414 

   415 print """

   416 def in_table_d1(code):

   417     return unicodedata.bidirectional(code) in ("R","AL")

   418 """

   419 

   420 # D.2 Characters with bidirectional property "L"

   421 name, table = tables[0]

   422 del tables[0]

   423 assert name == "D.2"

   424 

   425 L = set(gen_bidirectional(["L"]))

   426 assert set(table.keys()) == L

   427 

   428 print """

   429 def in_table_d2(code):

   430     return unicodedata.bidirectional(code) == "L"

   431 """