gen-unicode.py

#!/usr/bin/env python

# Copyright JS Foundation and other contributors, http://js.foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import argparse
import csv
import itertools
import os
import re
import warnings

from gen_c_source import LICENSE, format_code
from settings import PROJECT_DIR


UNICODE_DATA_FILE = 'UnicodeData.txt'
SPECIAL_CASING_FILE = 'SpecialCasing.txt'
DERIVED_PROPS_FILE = 'DerivedCoreProperties.txt'
PROP_LIST_FILE = 'PropList.txt'
CASE_FOLDING_FILE = 'CaseFolding.txt'

RANGES_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges.inc.h')
RANGES_SUP_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-ranges-sup.inc.h')
CONVERSIONS_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h')
CONVERSIONS_SUP_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions-sup.inc.h')
FOLDING_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-folding.inc.h')
FOLDING_SUP_C_SOURCE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-folding-sup.inc.h')

UNICODE_PLANE_TYPE_BASIC = 0
UNICODE_PLANE_TYPE_SUPPLEMENTARY = 1

# common code generation

class UnicodeBasicSource(object):
    # pylint: disable=too-many-instance-attributes
    def __init__(self, filepath, character_type="uint16_t", length_type="uint8_t"):
        self._filepath = filepath
        self._header = [LICENSE, ""]
        self._data = []
        self._table_name_suffix = ""
        self.character_type = character_type
        self.length_type = length_type

        self._range_table_types = [self.character_type,
                                   self.length_type,
                                   self.character_type]
        self._range_table_names = ["interval_starts",
                                   "interval_lengths",
                                   "chars"]
        self._range_table_descriptions = ["Character interval starting points for",
                                          "Character interval lengths for",
                                          "Non-interval characters for"]

        self._conversion_range_types = [self.character_type,
                                        self.length_type]
        self._conversion_range_names = ["ranges",
                                        "range_lengths"]

    def complete_header(self, completion):
        self._header.append(completion)
        self._header.append("")  # for an extra empty line

    def add_whitepace_range(self, category, categorizer, units):
        self.add_range(category, categorizer.create_tables(units))

    def add_range(self, category, tables):
        idx = 0
        for table in tables:
            self.add_table(table,
                           "/**\n * %s %s.\n */" % (self._range_table_descriptions[idx], category),
                           self._range_table_types[idx],
                           category,
                           self._range_table_names[idx])
            idx += 1

    def add_conversion_range(self, category, tables, descriptions):
        self.add_named_conversion_range(category, tables, self._conversion_range_names, descriptions)

    def add_named_conversion_range(self, category, tables, table_names, descriptions):
        idx = 0
        for table in tables:
            self.add_table(table,
                           descriptions[idx],
                           self._conversion_range_types[idx],
                           category,
                           table_names[idx])
            idx += 1

    def add_table(self, table, description, table_type, category, table_name):
        if table and sum(table) != 0:
            self._data.append(description)
            self._data.append("static const %s lit_unicode_%s%s%s[] JERRY_ATTR_CONST_DATA ="
                              % (table_type,
                                 category.lower(),
                                 "_" + table_name if table_name else "",
                                 self._table_name_suffix))
            self._data.append("{")
            self._data.append(format_code(table, 1, 6 if self._table_name_suffix else 4))
            self._data.append("};")
            self._data.append("")  # for an extra empty line

    def generate(self):
        with open(self._filepath, 'w') as generated_source:
            generated_source.write("\n".join(self._header))
            generated_source.write("\n".join(self._data))


class UnicodeSupplementarySource(UnicodeBasicSource):
    def __init__(self, filepath):
        UnicodeBasicSource.__init__(self, filepath, "uint32_t", "uint16_t")
        self._table_name_suffix = "_sup"

    def add_whitepace_range(self, category, categorizer, units):
        self.add_range(category, categorizer.create_tables(units))

class UnicodeBasicCategorizer(object):
    def __init__(self):
        self._length_limit = 0xff
        self.extra_id_continue_units = set([0x200C, 0x200D])

    #pylint: disable=no-self-use
    def in_range(self, i):
        return i >= 0x80 and i < 0x10000

    def _group_ranges(self, units):
        """
        Convert an increasing list of integers into a range list
        :return: List of ranges.
        """
        for _, group in itertools.groupby(enumerate(units), lambda q: (q[1] - q[0])):
            group = list(group)
            yield group[0][1], group[-1][1]

    def create_tables(self, units):
        """
        Split list of ranges into intervals and single char lists.
        :return: A tuple containing the following info:
            - list of interval starting points
            - list of interval lengths
            - list of single chars
        """

        interval_sps = []
        interval_lengths = []
        chars = []

        for element in self._group_ranges(units):
            interval_length = element[1] - element[0]
            if interval_length == 0:
                chars.append(element[0])
            elif interval_length > self._length_limit:
                for i in range(element[0], element[1], self._length_limit + 1):
                    length = min(self._length_limit, element[1] - i)
                    interval_sps.append(i)
                    interval_lengths.append(length)
            else:
                interval_sps.append(element[0])
                interval_lengths.append(interval_length)

        return interval_sps, interval_lengths, chars

    def read_units(self, file_path, categories, subcategories=None):
        """
        Read the Unicode Derived Core Properties file and extract the ranges
        for the given categories.

        :param file_path: Path to the Unicode "DerivedCoreProperties.txt" file.
        :param categories: A list of category strings to extract from the Unicode file.
        :param subcategories: A list of subcategory strings to restrict categories.
        :return: A dictionary each string from the :param categories: is a key and for each
                key list of code points are stored.
        """
        # Create a dictionary in the format: { category[0]: [ ], ..., category[N]: [ ] }
        units = {}
        for category in categories:
            units[category] = []

        # Formats to match:
        #  <HEX>     ; <category> #
        #  <HEX>..<HEX>     ; <category> # <subcategory>
        matcher = r"(?P<start>[\dA-F]+)(?:\.\.(?P<end>[\dA-F]+))?\s+; (?P<category>[\w]+) # (?P<subcategory>[\w&]{2})"

        with open(file_path, "r") as src_file:
            for line in src_file:
                match = re.match(matcher, line)

                if (match
                        and match.group("category") in categories
                        and (not subcategories or match.group("subcategory") in subcategories)):
                    start = int(match.group("start"), 16)
                    # if no "end" found use the "start"
                    end = int(match.group("end") or match.group("start"), 16)

                    matching_code_points = [
                        code_point for code_point in range(start, end + 1) if self.in_range(code_point)
                    ]

                    units[match.group("category")].extend(matching_code_points)

        return units

    def read_case_mappings(self, unicode_data_file, special_casing_file):
        """
        Read the corresponding unicode values of lower and upper case letters and store these in tables.

        :param unicode_data_file: Contains the default case mappings (one-to-one mappings).
        :param special_casing_file: Contains additional informative case mappings that are either not one-to-one
                                    or which are context-sensitive.
        :return: Upper and lower case mappings.
        """

        lower_case_mapping = {}
        upper_case_mapping = {}

        # Add one-to-one mappings
        with open(unicode_data_file) as unicode_data:
            reader = csv.reader(unicode_data, delimiter=';')

            for line in reader:
                letter_id = int(line[0], 16)

                if not self.in_range(letter_id):
                    continue

                capital_letter = line[12]
                small_letter = line[13]

                if capital_letter:
                    upper_case_mapping[letter_id] = parse_unicode_sequence(capital_letter)

                if small_letter:
                    lower_case_mapping[letter_id] = parse_unicode_sequence(small_letter)

        # Update the conversion tables with the special cases
        with open(special_casing_file) as special_casing:
            reader = csv.reader(special_casing, delimiter=';')

            for line in reader:
                # Skip comment sections and empty lines
                if not line or line[0].startswith('#'):
                    continue

                # Replace '#' character with empty string
                for idx, fragment in enumerate(line):
                    if fragment.find('#') >= 0:
                        line[idx] = ''

                letter_id = int(line[0], 16)
                condition_list = line[4]

                if not self.in_range(letter_id) or condition_list:
                    continue

                original_letter = parse_unicode_sequence(line[0])
                small_letter = parse_unicode_sequence(line[1])
                capital_letter = parse_unicode_sequence(line[3])

                if small_letter != original_letter:
                    lower_case_mapping[letter_id] = small_letter
                if capital_letter != original_letter:
                    upper_case_mapping[letter_id] = capital_letter

        return lower_case_mapping, upper_case_mapping

class UnicodeSupplementaryCategorizer(UnicodeBasicCategorizer):
    def __init__(self):
        UnicodeBasicCategorizer.__init__(self)
        self._length_limit = 0xffff
        self.extra_id_continue_units = set()

    def in_range(self, i):
        return i >= 0x10000

def generate_ranges(script_args, plane_type):
    if plane_type == UNICODE_PLANE_TYPE_SUPPLEMENTARY:
        c_source = UnicodeSupplementarySource(RANGES_SUP_C_SOURCE)
        categorizer = UnicodeSupplementaryCategorizer()
    else:
        c_source = UnicodeBasicSource(RANGES_C_SOURCE)
        categorizer = UnicodeBasicCategorizer()

    header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
                         " * from %s. Do not edit! */" % (DERIVED_PROPS_FILE),
                         ""]

    c_source.complete_header("\n".join(header_completion))

    derived_props_path = os.path.join(script_args.unicode_dir, DERIVED_PROPS_FILE)
    units = categorizer.read_units(derived_props_path, ["ID_Start", "ID_Continue"])

    units["ID_Continue"] = sorted(set(units["ID_Continue"]).union(categorizer.extra_id_continue_units)
                                  - set(units["ID_Start"]))

    for category, unit in units.items():
        c_source.add_range(category, categorizer.create_tables(unit))

    prop_list_path = os.path.join(script_args.unicode_dir, PROP_LIST_FILE)

    white_space_units = categorizer.read_units(prop_list_path, ["White_Space"], ["Zs"])["White_Space"]

    c_source.add_whitepace_range("White_Space", categorizer, white_space_units)

    c_source.generate()


# functions for unicode conversions

def make_char(hex_val):
    """
    Create a unicode character from a hex value

    :param hex_val: Hex value of the character.
    :return: Unicode character corresponding to the value.
    """

    try:
        return unichr(hex_val)
    except NameError:
        return chr(hex_val)


def parse_unicode_sequence(raw_data):
    """
    Parse unicode sequence from raw data.

    :param raw_data: Contains the unicode sequence which needs to parse.
    :return: The parsed unicode sequence.
    """

    result = ''

    for unicode_char in raw_data.split(' '):
        if unicode_char == '':
            continue

        # Convert it to unicode code point (from hex value without 0x prefix)
        hex_val = int(unicode_char, 16)
        result += make_char(hex_val)

    return result

def extract_ranges(letter_case, reverse_letter_case=None):
    """
    Extract ranges from case mappings
    (the second param is optional, if it's not empty, a range will contains bidirectional conversions only).

    :param letter_id: An integer, representing the unicode code point of the character.
    :param letter_case: case mappings dictionary which contains the conversions.
    :param reverse_letter_case: Comparable case mapping table which contains the return direction of the conversion.
    :return: A table with the start points and their mapped value, and another table with the lengths of the ranges.
    """

    in_range = False
    range_position = -1
    ranges = []
    range_lengths = []

    for letter_id in sorted(letter_case.keys()):
        prev_letter_id = letter_id - 1

        # One-way conversions
        if reverse_letter_case is None:
            if len(letter_case[letter_id]) > 1:
                in_range = False
                continue

            if prev_letter_id not in letter_case or len(letter_case[prev_letter_id]) > 1:
                in_range = False
                continue

        # Two way conversions
        else:
            if not is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
                in_range = False
                continue

            if not is_bidirectional_conversion(prev_letter_id, letter_case, reverse_letter_case):
                in_range = False
                continue

        conv_distance = calculate_conversion_distance(letter_case, letter_id)
        prev_conv_distance = calculate_conversion_distance(letter_case, prev_letter_id)

        if conv_distance != prev_conv_distance:
            in_range = False
            continue

        if in_range:
            range_lengths[range_position] += 1
        else:
            in_range = True
            range_position += 1

            # Add the start point of the range and its mapped value
            ranges.extend([prev_letter_id, ord(letter_case[prev_letter_id])])
            range_lengths.append(2)

    # Remove all ranges from the case mapping table.
    for idx in range(0, len(ranges), 2):
        range_length = range_lengths[idx // 2]

        for incr in range(range_length):
            del letter_case[ranges[idx] + incr]
            if reverse_letter_case is not None:
                del reverse_letter_case[ranges[idx + 1] + incr]

    return ranges, range_lengths


def extract_character_pair_ranges(letter_case, reverse_letter_case):
    """
    Extract two or more character pairs from the case mapping tables.

    :param letter_case: case mappings dictionary which contains the conversions.
    :param reverse_letter_case: Comparable case mapping table which contains the return direction of the conversion.
    :return: A table with the start points, and another table with the lengths of the ranges.
    """

    start_points = []
    lengths = []
    in_range = False
    element_counter = -1

    for letter_id in sorted(letter_case.keys()):
        # Only extract character pairs
        if not is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
            in_range = False
            continue

        if ord(letter_case[letter_id]) == letter_id + 1:
            prev_letter_id = letter_id - 2

            if not is_bidirectional_conversion(prev_letter_id, letter_case, reverse_letter_case):
                in_range = False

            if in_range:
                lengths[element_counter] += 2
            else:
                element_counter += 1
                start_points.append(letter_id)
                lengths.append(2)
                in_range = True

        else:
            in_range = False

    # Remove all found case mapping from the conversion tables after the scanning method
    for idx, letter_id in enumerate(start_points):
        conv_length = lengths[idx]

        for incr in range(0, conv_length, 2):
            del letter_case[letter_id + incr]
            del reverse_letter_case[letter_id + 1 + incr]

    return start_points, lengths


def extract_character_pairs(letter_case, reverse_letter_case):
    """
    Extract character pairs. Check that two unicode value are also a mapping value of each other.

    :param letter_case: case mappings dictionary which contains the conversions.
    :param reverse_letter_case: Comparable case mapping table which contains the return direction of the conversion.
    :return: A table with character pairs.
    """

    character_pairs = []

    for letter_id in sorted(letter_case.keys()):
        if is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
            mapped_value = letter_case[letter_id]
            character_pairs.extend([letter_id, ord(mapped_value)])

            # Remove character pairs from case mapping tables
            del letter_case[letter_id]
            del reverse_letter_case[ord(mapped_value)]

    return character_pairs


def extract_special_ranges(letter_case):
    """
    Extract special ranges. It contains start points of one-to-two letter case ranges
    where the second character is always the same.

    :param letter_case: case mappings dictionary which contains the conversions.

    :return: A table with the start points and their mapped values, and a table with the lengths of the ranges.
    """

    special_ranges = []
    special_range_lengths = []

    range_position = -1

    for letter_id in sorted(letter_case.keys()):
        mapped_value = letter_case[letter_id]

        if len(mapped_value) != 2:
            continue

        prev_letter_id = letter_id - 1

        if prev_letter_id not in letter_case:
            in_range = False
            continue

        prev_mapped_value = letter_case[prev_letter_id]

        if len(prev_mapped_value) != 2:
            continue

        if prev_mapped_value[1] != mapped_value[1]:
            continue

        if (ord(prev_mapped_value[0]) - prev_letter_id) != (ord(mapped_value[0]) - letter_id):
            in_range = False
            continue

        if in_range:
            special_range_lengths[range_position] += 1
        else:
            range_position += 1
            in_range = True

            special_ranges.extend([prev_letter_id, ord(prev_mapped_value[0]), ord(prev_mapped_value[1])])
            special_range_lengths.append(1)

    # Remove special ranges from the conversion table
    for idx in range(0, len(special_ranges), 3):
        range_length = special_range_lengths[idx // 3]
        letter_id = special_ranges[idx]

        for incr in range(range_length):
            del letter_case[special_ranges[idx] + incr]

    return special_ranges, special_range_lengths


def extract_conversions(letter_case):
    """
    Extract conversions. It provide the full (or remained) case mappings from the table.
    The counter table contains the information of how much one-to-one, one-to-two or one-to-three mappings
    exists successively in the conversion table.

    :return: A table with conversions, and a table with counters.
    """

    unicodes = [[], [], []]
    unicode_lengths = [0, 0, 0]

    # 1 to 1 byte
    for letter_id in sorted(letter_case.keys()):
        mapped_value = letter_case[letter_id]

        if len(mapped_value) != 1:
            continue

        unicodes[0].extend([letter_id, ord(mapped_value)])
        del letter_case[letter_id]

    # 1 to 2 bytes
    for letter_id in sorted(letter_case.keys()):
        mapped_value = letter_case[letter_id]

        if len(mapped_value) != 2:
            continue

        unicodes[1].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1])])
        del letter_case[letter_id]

    # 1 to 3 bytes
    for letter_id in sorted(letter_case.keys()):
        mapped_value = letter_case[letter_id]

        if len(mapped_value) != 3:
            continue

        unicodes[2].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1]), ord(mapped_value[2])])
        del letter_case[letter_id]

    unicode_lengths = [int(len(unicodes[0]) / 2), int(len(unicodes[1]) / 3), int(len(unicodes[2]) / 4)]

    return list(itertools.chain.from_iterable(unicodes)), unicode_lengths


def is_bidirectional_conversion(letter_id, letter_case, reverse_letter_case):
    """
    Check that two unicode value are also a mapping value of each other.

    :param letter_id: An integer, representing the unicode code point of the character.
    :param other_case_mapping: Comparable case mapping table which possible contains
                               the return direction of the conversion.
    :return: True, if it's a reverible conversion, false otherwise.
    """

    if letter_id not in letter_case:
        return False

    # Check one-to-one mapping
    mapped_value = letter_case[letter_id]
    if len(mapped_value) > 1:
        return False

    # Check two way conversions
    mapped_value_id = ord(mapped_value)

    if mapped_value_id not in reverse_letter_case or len(reverse_letter_case[mapped_value_id]) > 1:
        return False

    if ord(reverse_letter_case[mapped_value_id]) != letter_id:
        return False

    return True


def calculate_conversion_distance(letter_case, letter_id):
    """
    Calculate the distance between the unicode character and its mapped value
    (only needs and works with one-to-one mappings).

    :param letter_case: case mappings dictionary which contains the conversions.
    :param letter_id: An integer, representing the unicode code point of the character.
    :return: The conversion distance.
    """

    if letter_id not in letter_case or len(letter_case[letter_id]) > 1:
        return None

    return ord(letter_case[letter_id]) - letter_id


def generate_conversions(script_args, plane_type):
    if plane_type == UNICODE_PLANE_TYPE_SUPPLEMENTARY:
        c_source = UnicodeSupplementarySource(CONVERSIONS_SUP_C_SOURCE)
        categorizer = UnicodeSupplementaryCategorizer()
    else:
        c_source = UnicodeBasicSource(CONVERSIONS_C_SOURCE)
        categorizer = UnicodeBasicCategorizer()

    header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
                         " * from %s and %s files. Do not edit! */" % (UNICODE_DATA_FILE, SPECIAL_CASING_FILE),
                         ""]

    c_source.complete_header("\n".join(header_completion))

    unicode_data_path = os.path.join(script_args.unicode_dir, UNICODE_DATA_FILE)
    special_casing_path = os.path.join(script_args.unicode_dir, SPECIAL_CASING_FILE)

    # Read the corresponding unicode values of lower and upper case letters and store these in tables
    lower_case, upper_case = categorizer.read_case_mappings(unicode_data_path, special_casing_path)

    c_source.add_conversion_range("character_case",
                                  extract_ranges(lower_case, upper_case),
                                  [("/* Contains start points of character case ranges "
                                    "(these are bidirectional conversions). */"),
                                   "/* Interval lengths of start points in `character_case_ranges` table. */"])
    c_source.add_conversion_range("character_pair",
                                  extract_character_pair_ranges(lower_case, upper_case),
                                  ["/* Contains the start points of bidirectional conversion ranges. */",
                                   "/* Interval lengths of start points in `character_pair_ranges` table. */"])

    c_source.add_table(extract_character_pairs(lower_case, upper_case),
                       "/* Contains lower/upper case bidirectional conversion pairs. */",
                       c_source.character_type,
                       "character_pairs",
                       "")

    c_source.add_conversion_range("upper_case_special",
                                  extract_special_ranges(upper_case),
                                  [("/* Contains start points of one-to-two uppercase ranges where the "
                                    "second character\n"
                                    " * is always the same.\n"
                                    " */"),
                                   "/* Interval lengths for start points in `upper_case_special_ranges` table. */"])

    c_source.add_conversion_range("lower_case",
                                  extract_ranges(lower_case),
                                  ["/* Contains start points of lowercase ranges. */",
                                   "/* Interval lengths for start points in `lower_case_ranges` table. */"])

    c_source.add_named_conversion_range("lower_case",
                                        extract_conversions(lower_case),
                                        ["conversions", "conversion_counters"],
                                        [("/* The remaining lowercase conversions. The lowercase variant can "
                                          "be one-to-three character long. */"),
                                         ("/* Number of one-to-one, one-to-two, and one-to-three lowercase "
                                          "conversions. */")])

    c_source.add_named_conversion_range("upper_case",
                                        extract_conversions(upper_case),
                                        ["conversions", "conversion_counters"],
                                        [("/* The remaining uppercase conversions. The uppercase variant can "
                                          "be one-to-three character long. */"),
                                         ("/* Number of one-to-one, one-to-two, and one-to-three uppercase "
                                          "conversions. */")])

    if lower_case:
        warnings.warn('Not all elements extracted from the lowercase table!')
    if upper_case:
        warnings.warn('Not all elements extracted from the uppercase table!')

    c_source.generate()


def generate_folding(script_args, plane_type):
    if plane_type == UNICODE_PLANE_TYPE_SUPPLEMENTARY:
        c_source = UnicodeSupplementarySource(FOLDING_SUP_C_SOURCE)
        categorizer = UnicodeSupplementaryCategorizer()
    else:
        c_source = UnicodeBasicSource(FOLDING_C_SOURCE)
        categorizer = UnicodeBasicCategorizer()

    header_completion = ["/* This file is automatically generated by the %s script" % os.path.basename(__file__),
                         " * from the %s file. Do not edit! */" % (CASE_FOLDING_FILE),
                         ""]

    c_source.complete_header("\n".join(header_completion))

    unicode_data_path = os.path.join(script_args.unicode_dir, UNICODE_DATA_FILE)
    special_casing_path = os.path.join(script_args.unicode_dir, SPECIAL_CASING_FILE)
    case_folding_path = os.path.join(script_args.unicode_dir, CASE_FOLDING_FILE)

    # Read the corresponding unicode values of lower and upper case letters and store these in tables
    lower_case, upper_case = categorizer.read_case_mappings(unicode_data_path, special_casing_path)

    folding = {}

    with open(case_folding_path, 'r') as case_folding:
        case_folding_re = re.compile(r'(?P<code_point>[^;]*);\s*(?P<type>[^;]*);\s*(?P<folding>[^;]*);')
        for line in case_folding:
            match = case_folding_re.match(line)
            if match and match.group('type') in ('S', 'C'):
                code_point = int(match.group('code_point'), 16)

                if categorizer.in_range(code_point):
                    folding[code_point] = parse_unicode_sequence(match.group('folding'))

    should_to_upper = []
    should_skip_to_lower = []

    for code_point in lower_case:
        if code_point not in folding:
            should_skip_to_lower.append(code_point)

    for code_point, folded in folding.items():
        if lower_case.get(code_point, make_char(code_point)) != folded:
            should_to_upper.append(code_point)

            if upper_case.get(code_point, '') == folded:
                should_skip_to_lower.append(code_point)

    c_source.add_range('folding_skip_to_lower', categorizer.create_tables(should_skip_to_lower))
    c_source.add_range('folding_to_upper', categorizer.create_tables(should_to_upper))

    c_source.generate()


# entry point


def main():
    parser = argparse.ArgumentParser(description='lit-unicode-{conversions,ranges}-{sup}.inc.h generator',
                                     epilog='''
                                        The input data must be retrieved from
                                        http://www.unicode.org/Public/<VERSION>/ucd/UCD.zip.
                                        The last known good version is 13.0.0.
                                        ''')
    def check_dir(path):
        if not os.path.isdir(path) or not os.access(path, os.R_OK):
            raise argparse.ArgumentTypeError('The %s directory does not exist or is not readable!' % path)
        return path

    parser.add_argument('--unicode-dir', metavar='DIR', action='store', required=True,
                        type=check_dir, help='specify the unicode data directory')

    script_args = parser.parse_args()

    generate_ranges(script_args, UNICODE_PLANE_TYPE_BASIC)
    generate_ranges(script_args, UNICODE_PLANE_TYPE_SUPPLEMENTARY)
    generate_conversions(script_args, UNICODE_PLANE_TYPE_BASIC)
    generate_conversions(script_args, UNICODE_PLANE_TYPE_SUPPLEMENTARY)
    generate_folding(script_args, UNICODE_PLANE_TYPE_BASIC)
    # There are currently no code points in the supplementary planes that require special folding
    # generate_folding(script_args, UNICODE_PLANE_TYPE_SUPPLEMENTARY)


if __name__ == "__main__":
    main()