#########################################################################
#                                                                       #
#                                                                       #
#   copyright 2002 Paul Henry Tremblay                                  #
#                                                                       #
#   This program is distributed in the hope that it will be useful,     #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of      #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU    #
#   General Public License for more details.                            #
#                                                                       #
#                                                                       #
#########################################################################
import os
import re

from calibre.ebooks.rtf2xml import check_brackets, copy
from calibre.ptempfile import better_mktemp

from . import open_for_read, open_for_write


class ProcessTokens:
    '''
    Process each token on a line and add information that will be useful for
    later processing. Information will be put on one line, delimited by "<"
    for main fields, and ">" for sub fields
    '''

    def __init__(
        self,
        in_file,
        exception_handler,
        bug_handler,
        copy=None,
        run_level=1,
    ):
        self.__file = in_file
        self.__bug_handler = bug_handler
        self.__copy = copy
        self.__run_level = run_level
        self.__write_to = better_mktemp()
        self.initiate_token_dict()
        # self.initiate_token_actions()
        self.compile_expressions()
        self.__bracket_count = 0
        self.__exception_handler = exception_handler
        self.__bug_handler = bug_handler

    def compile_expressions(self):
        self.__num_exp = re.compile(r'([a-zA-Z]+)(.*)')
        self.__utf_exp = re.compile(r'(&.*?;)')

    def initiate_token_dict(self):
        self.__return_code = 0
        self.dict_token = {
            # unicode
            'mshex': ('nu', '__________', self.__ms_hex_func),
            # brackets
            '{': ('nu', '{', self.ob_func),
            '}': ('nu', '}', self.cb_func),
            # microsoft characters
            'ldblquote': ('mc', 'ldblquote', self.ms_sub_func),
            'rdblquote': ('mc', 'rdblquote', self.ms_sub_func),
            'rquote': ('mc', 'rquote', self.ms_sub_func),
            'lquote': ('mc', 'lquote', self.ms_sub_func),
            'emdash': ('mc', 'emdash', self.ms_sub_func),
            'endash': ('mc', 'endash', self.ms_sub_func),
            'bullet': ('mc', 'bullet', self.ms_sub_func),
            '~': ('mc', '~', self.ms_sub_func),
            'tab': ('mc', 'tab', self.ms_sub_func),
            '_': ('mc', '_', self.ms_sub_func),
            ';': ('mc', ';', self.ms_sub_func),
            # this must be wrong
            '-': ('mc', '-', self.ms_sub_func),
            'line': ('mi', 'hardline-break', self.direct_conv_func),  # calibre
            # misc => ml
            '*': ('ml', 'asterisk__', self.default_func),
            ':': ('ml', 'colon_____', self.default_func),
            # text
            'backslash': ('nu', '\\', self.text_func),
            'ob': ('nu', '{', self.text_func),
            'cb': ('nu', '}', self.text_func),
            # paragraph formatting => pf
            'page': ('pf', 'page-break', self.default_func),
            'par': ('pf', 'par-end___', self.default_func),
            'pard': ('pf', 'par-def___', self.default_func),
            'keepn': ('pf', 'keep-w-nex', self.bool_st_func),
            'widctlpar': ('pf', 'widow-cntl', self.bool_st_func),
            'adjustright': ('pf', 'adjust-rgt', self.bool_st_func),
            'lang': ('pf', 'language__', self.__language_func),
            'ri': ('pf', 'right-inde', self.divide_by_20),
            'fi': ('pf', 'fir-ln-ind', self.divide_by_20),
            'li': ('pf', 'left-inden', self.divide_by_20),
            'sb': ('pf', 'space-befo', self.divide_by_20),
            'sa': ('pf', 'space-afte', self.divide_by_20),
            'sl': ('pf', 'line-space', self.divide_by_20),
            'deftab': ('pf', 'default-ta', self.divide_by_20),
            'ql': ('pf', 'align_____<left', self.two_part_func),
            'qc': ('pf', 'align_____<cent', self.two_part_func),
            'qj': ('pf', 'align_____<just', self.two_part_func),
            'qr': ('pf', 'align_____<right', self.two_part_func),
            'nowidctlpar': ('pf', 'widow-cntr<false', self.two_part_func),
            'tx': ('pf', 'tab-stop__', self.divide_by_20),
            'tb': ('pf', 'tab-bar-st', self.divide_by_20),
            'tqr': ('pf', 'tab-right_', self.default_func),
            'tqdec': ('pf', 'tab-dec___', self.default_func),
            'tqc': ('pf', 'tab-center', self.default_func),
            'tlul': ('pf', 'leader-und', self.default_func),
            'tlhyph': ('pf', 'leader-hyp', self.default_func),
            'tldot': ('pf', 'leader-dot', self.default_func),
            # stylesheet = > ss
            'stylesheet': ('ss', 'style-shet', self.default_func),
            'sbasedon': ('ss', 'based-on__', self.default_func),
            'snext': ('ss', 'next-style', self.default_func),
            'cs': ('ss', 'char-style', self.default_func),
            's': ('ss', 'para-style', self.default_func),
            # graphics => gr
            'pict': ('gr', 'picture___', self.default_func),
            'objclass': ('gr', 'obj-class_', self.default_func),
            'macpict': ('gr', 'mac-pic___', self.default_func),
            # section => sc
            'sect': ('sc', 'section___', self.default_func),
            'sectd': ('sc', 'sect-defin', self.default_func),
            'endhere': ('sc', 'sect-note_', self.default_func),
            # list=> ls
            'pntext': ('ls', 'list-text_', self.default_func),
            # this line must be wrong because it duplicates an earlier one
            'listtext': ('ls', 'list-text_', self.default_func),
            'pn': ('ls', 'list______', self.default_func),
            'pnseclvl': ('ls', 'list-level', self.default_func),
            'pncard': ('ls', 'list-cardi', self.bool_st_func),
            'pndec': ('ls', 'list-decim', self.bool_st_func),
            'pnucltr': ('ls', 'list-up-al', self.bool_st_func),
            'pnucrm': ('ls', 'list-up-ro', self.bool_st_func),
            'pnord': ('ls', 'list-ord__', self.bool_st_func),
            'pnordt': ('ls', 'list-ordte', self.bool_st_func),
            'pnlvlblt': ('ls', 'list-bulli', self.bool_st_func),
            'pnlvlbody': ('ls', 'list-simpi', self.bool_st_func),
            'pnlvlcont': ('ls', 'list-conti', self.bool_st_func),
            'pnhang': ('ls', 'list-hang_', self.bool_st_func),
            'pntxtb': ('ls', 'list-tebef', self.bool_st_func),
            'ilvl': ('ls', 'list-level', self.default_func),
            'ls': ('ls', 'list-id___', self.default_func),
            'pnstart': ('ls', 'list-start', self.default_func),
            'itap': ('ls', 'nest-level', self.default_func),
            'leveltext': ('ls', 'level-text', self.default_func),
            'levelnumbers': ('ls', 'level-numb', self.default_func),
            'list': ('ls', 'list-in-tb', self.default_func),
            'listlevel': ('ls', 'list-tb-le', self.default_func),
            'listname': ('ls', 'list-name_', self.default_func),
            'listtemplateid': ('ls', 'ls-tem-id_', self.default_func),
            'leveltemplateid': ('ls', 'lv-tem-id_', self.default_func),
            'listhybrid': ('ls', 'list-hybri', self.default_func),
            'levelstartat': ('ls', 'level-star', self.default_func),
            'levelspace': ('ls', 'level-spac', self.divide_by_20),
            'levelindent': ('ls', 'level-inde', self.default_func),
            'levelnfc': ('ls', 'level-type', self.__list_type_func),
            'levelnfcn': ('ls', 'level-type', self.__list_type_func),
            'listid': ('ls', 'lis-tbl-id', self.default_func),
            'listoverride': ('ls', 'lis-overid', self.default_func),
            # duplicate
            'pnlvl': ('ls', 'list-level', self.default_func),
            # root info => ri
            'rtf': ('ri', 'rtf_______', self.default_func),
            'deff': ('ri', 'deflt-font', self.default_func),
            'mac': ('ri', 'macintosh_', self.default_func),
            'pc': ('ri', 'pc________', self.default_func),
            'pca': ('ri', 'pca_______', self.default_func),
            'ansi': ('ri', 'ansi______', self.default_func),
            'ansicpg': ('ri', 'ansi-codpg', self.default_func),
            # notes => nt
            'footnote': ('nt', 'footnote__', self.default_func),
            'ftnalt': ('nt', 'type______<endnote', self.two_part_func),
            # anchor => an
            'tc': ('an', 'toc_______', self.default_func),
            'bkmkstt': ('an', 'book-mk-st', self.default_func),
            'bkmkstart': ('an', 'book-mk-st', self.default_func),
            'bkmkend': ('an', 'book-mk-en', self.default_func),
            'xe': ('an', 'index-mark', self.default_func),
            'rxe': ('an', 'place_____', self.default_func),
            # index => in
            'bxe': ('in', 'index-bold', self.default_func),
            'ixe': ('in', 'index-ital', self.default_func),
            'txe': ('in', 'index-see_', self.default_func),
            # table of contents => tc
            'tcl': ('tc', 'toc-level_', self.default_func),
            'tcn': ('tc', 'toc-sup-nu', self.default_func),
            # field => fd
            'field': ('fd', 'field_____', self.default_func),
            'fldinst': ('fd', 'field-inst', self.default_func),
            'fldrslt': ('fd', 'field-rslt', self.default_func),
            'datafield': ('fd', 'datafield_', self.default_func),
            # info-tables => it
            'fonttbl': ('it', 'font-table', self.default_func),
            'colortbl': ('it', 'colr-table', self.default_func),
            'listoverridetable': ('it', 'lovr-table', self.default_func),
            'listtable': ('it', 'listtable_', self.default_func),
            'revtbl': ('it', 'revi-table', self.default_func),
            # character info => ci
            'b': ('ci', 'bold______', self.bool_st_func),
            'blue': ('ci', 'blue______', self.color_func),
            'caps': ('ci', 'caps______', self.bool_st_func),
            'cf': ('ci', 'font-color', self.colorz_func),
            'chftn': ('ci', 'footnot-mk', self.bool_st_func),
            'dn': ('ci', 'font-down_', self.divide_by_2),
            'embo': ('ci', 'emboss____', self.bool_st_func),
            'f': ('ci', 'font-style', self.default_func),
            'fs': ('ci', 'font-size_', self.divide_by_2),
            'green': ('ci', 'green_____', self.color_func),
            'i': ('ci', 'italics___', self.bool_st_func),
            'impr': ('ci', 'engrave___', self.bool_st_func),
            'outl': ('ci', 'outline___', self.bool_st_func),
            'plain': ('ci', 'plain_____', self.bool_st_func),
            'red': ('ci', 'red_______', self.color_func),
            'scaps': ('ci', 'small-caps', self.bool_st_func),
            'shad': ('ci', 'shadow____', self.bool_st_func),
            'strike': ('ci', 'strike-thr', self.bool_st_func),
            'striked': ('ci', 'dbl-strike', self.bool_st_func),
            'sub': ('ci', 'subscript_', self.bool_st_func),
            'super': ('ci', 'superscrip', self.bool_st_func),
            'nosupersub': ('ci', 'no-su-supe', self.__no_sup_sub_func),
            'up': ('ci', 'font-up___', self.divide_by_2),
            'v': ('ci', 'hidden____', self.default_func),
            # underline
            # can't see why it isn't a char info: 'ul'=>'ci'
            'ul': ('ci', 'underlined<continous', self.two_part_func),
            'uld': ('ci', 'underlined<dotted', self.two_part_func),
            'uldash': ('ci', 'underlined<dash', self.two_part_func),
            'uldashd': ('ci', 'underlined<dash-dot', self.two_part_func),
            'uldashdd': ('ci', 'underlined<dash-dot-dot', self.two_part_func),
            'uldb': ('ci', 'underlined<double', self.two_part_func),
            'ulhwave': ('ci', 'underlined<heavy-wave', self.two_part_func),
            'ulldash': ('ci', 'underlined<long-dash', self.two_part_func),
            'ulth': ('ci', 'underlined<thich', self.two_part_func),
            'ulthd': ('ci', 'underlined<thick-dotted', self.two_part_func),
            'ulthdash': ('ci', 'underlined<thick-dash', self.two_part_func),
            'ulthdashd': ('ci', 'underlined<thick-dash-dot', self.two_part_func),
            'ulthdashdd': ('ci', 'underlined<thick-dash-dot-dot', self.two_part_func),
            'ulthldash': ('ci', 'underlined<thick-long-dash', self.two_part_func),
            'ululdbwave': ('ci', 'underlined<double-wave', self.two_part_func),
            'ulw': ('ci', 'underlined<word', self.two_part_func),
            'ulwave': ('ci', 'underlined<wave', self.two_part_func),
            'ulnone': ('ci', 'underlined<false', self.two_part_func),
            # table => tb
            'trowd': ('tb', 'row-def___', self.default_func),
            'cell': ('tb', 'cell______', self.default_func),
            'row': ('tb', 'row_______', self.default_func),
            'intbl': ('tb', 'in-table__', self.default_func),
            'cols': ('tb', 'columns___', self.default_func),
            'trleft': ('tb', 'row-pos-le', self.divide_by_20),
            'cellx': ('tb', 'cell-posit', self.divide_by_20),
            'trhdr': ('tb', 'row-header', self.default_func),
            # preamble => pr
            # document information => di
            # TODO integrate \userprops
            'info': ('di', 'doc-info__', self.default_func),
            'title': ('di', 'title_____', self.default_func),
            'author': ('di', 'author____', self.default_func),
            'operator': ('di', 'operator__', self.default_func),
            'manager': ('di', 'manager___', self.default_func),
            'company': ('di', 'company___', self.default_func),
            'keywords': ('di', 'keywords__', self.default_func),
            'category': ('di', 'category__', self.default_func),
            'doccomm': ('di', 'doc-notes_', self.default_func),
            'comment': ('di', 'doc-notes_', self.default_func),
            'subject': ('di', 'subject___', self.default_func),
            'creatim': ('di', 'create-tim', self.default_func),
            'yr': ('di', 'year______', self.default_func),
            'mo': ('di', 'month_____', self.default_func),
            'dy': ('di', 'day_______', self.default_func),
            'min': ('di', 'minute____', self.default_func),
            'sec': ('di', 'second____', self.default_func),
            'revtim': ('di', 'revis-time', self.default_func),
            'edmins': ('di', 'edit-time_', self.default_func),
            'printim': ('di', 'print-time', self.default_func),
            'buptim': ('di', 'backuptime', self.default_func),
            'nofwords': ('di', 'num-of-wor', self.default_func),
            'nofchars': ('di', 'num-of-chr', self.default_func),
            'nofcharsws': ('di', 'numofchrws', self.default_func),
            'nofpages': ('di', 'num-of-pag', self.default_func),
            'version': ('di', 'version___', self.default_func),
            'vern': ('di', 'intern-ver', self.default_func),
            'hlinkbase': ('di', 'linkbase__', self.default_func),
            'id': ('di', 'internalID', self.default_func),
            # headers and footers => hf
            'headerf': ('hf', 'head-first', self.default_func),
            'headerl': ('hf', 'head-left_', self.default_func),
            'headerr': ('hf', 'head-right', self.default_func),
            'footerf': ('hf', 'foot-first', self.default_func),
            'footerl': ('hf', 'foot-left_', self.default_func),
            'footerr': ('hf', 'foot-right', self.default_func),
            'header': ('hf', 'header____', self.default_func),
            'footer': ('hf', 'footer____', self.default_func),
            # page => pa
            'margl': ('pa', 'margin-lef', self.divide_by_20),
            'margr': ('pa', 'margin-rig', self.divide_by_20),
            'margb': ('pa', 'margin-bot', self.divide_by_20),
            'margt': ('pa', 'margin-top', self.divide_by_20),
            'gutter': ('pa', 'gutter____', self.divide_by_20),
            'paperw': ('pa', 'paper-widt', self.divide_by_20),
            'paperh': ('pa', 'paper-hght', self.divide_by_20),
            # annotation => an
            'annotation': ('an', 'annotation', self.default_func),
            # border => bd
            'trbrdrh': ('bd', 'bor-t-r-hi', self.default_func),
            'trbrdrv': ('bd', 'bor-t-r-vi', self.default_func),
            'trbrdrt': ('bd', 'bor-t-r-to', self.default_func),
            'trbrdrl': ('bd', 'bor-t-r-le', self.default_func),
            'trbrdrb': ('bd', 'bor-t-r-bo', self.default_func),
            'trbrdrr': ('bd', 'bor-t-r-ri', self.default_func),
            'clbrdrb': ('bd', 'bor-cel-bo', self.default_func),
            'clbrdrt': ('bd', 'bor-cel-to', self.default_func),
            'clbrdrl': ('bd', 'bor-cel-le', self.default_func),
            'clbrdrr': ('bd', 'bor-cel-ri', self.default_func),
            'brdrb': ('bd', 'bor-par-bo', self.default_func),
            'brdrt': ('bd', 'bor-par-to', self.default_func),
            'brdrl': ('bd', 'bor-par-le', self.default_func),
            'brdrr': ('bd', 'bor-par-ri', self.default_func),
            'box': ('bd', 'bor-par-bx', self.default_func),
            'chbrdr': ('bd', 'bor-par-bo', self.default_func),
            'brdrbtw': ('bd', 'bor-for-ev', self.default_func),
            'brdrbar': ('bd', 'bor-outsid', self.default_func),
            'brdrnone': ('bd', 'bor-none__<false', self.two_part_func),
            # border type => bt
            'brdrs': ('bt', 'bdr-single', self.default_func),
            'brdrth': ('bt', 'bdr-doubtb', self.default_func),
            'brdrsh': ('bt', 'bdr-shadow', self.default_func),
            'brdrdb': ('bt', 'bdr-double', self.default_func),
            'brdrdot': ('bt', 'bdr-dotted', self.default_func),
            'brdrdash': ('bt', 'bdr-dashed', self.default_func),
            'brdrhair': ('bt', 'bdr-hair__', self.default_func),
            'brdrinset': ('bt', 'bdr-inset_', self.default_func),
            'brdrdashsm': ('bt', 'bdr-das-sm', self.default_func),
            'brdrdashd': ('bt', 'bdr-dot-sm', self.default_func),
            'brdrdashdd': ('bt', 'bdr-dot-do', self.default_func),
            'brdroutset': ('bt', 'bdr-outset', self.default_func),
            'brdrtriple': ('bt', 'bdr-trippl', self.default_func),
            'brdrtnthsg': ('bt', 'bdr-thsm__', self.default_func),
            'brdrthtnsg': ('bt', 'bdr-htsm__', self.default_func),
            'brdrtnthtnsg': ('bt', 'bdr-hthsm_', self.default_func),
            'brdrtnthmg': ('bt', 'bdr-thm___', self.default_func),
            'brdrthtnmg': ('bt', 'bdr-htm___', self.default_func),
            'brdrtnthtnmg': ('bt', 'bdr-hthm__', self.default_func),
            'brdrtnthlg': ('bt', 'bdr-thl___', self.default_func),
            'brdrtnthtnlg': ('bt', 'bdr-hthl__', self.default_func),
            'brdrwavy': ('bt', 'bdr-wavy__', self.default_func),
            'brdrwavydb': ('bt', 'bdr-d-wav_', self.default_func),
            'brdrdashdotstr': ('bt', 'bdr-strip_', self.default_func),
            'brdremboss': ('bt', 'bdr-embos_', self.default_func),
            'brdrengrave': ('bt', 'bdr-engra_', self.default_func),
            'brdrframe': ('bt', 'bdr-frame_', self.default_func),
            'brdrw': ('bt', 'bdr-li-wid', self.divide_by_20),
            'brsp': ('bt', 'bdr-sp-wid', self.divide_by_20),
            'brdrcf': ('bt', 'bdr-color_', self.default_func),
            # comments
            # 'comment'            : ('cm', 'comment___', self.default_func),
        }
        self.__number_type_dict = {
            0: 'Arabic',
            1: 'uppercase Roman numeral',
            2: 'lowercase Roman numeral',
            3: 'uppercase letter',
            4: 'lowercase letter',
            5: 'ordinal number',
            6: 'cardianl text number',
            7: 'ordinal text number',
            10: 'Kanji numbering without the digit character',
            11: 'Kanji numbering with the digit character',
            1246: 'phonetic Katakana characters in aiueo order',
            1346: 'phonetic katakana characters in iroha order',
            14: 'double byte character',
            15: 'single byte character',
            16: 'Kanji numbering 3',
            17: 'Kanji numbering 4',
            18: 'Circle numbering',
            19: 'double-byte Arabic numbering',
            2046: 'phonetic double-byte Katakana characters',
            2146: 'phonetic double-byte katakana characters',
            22: 'Arabic with leading zero',
            23: 'bullet',
            24: 'Korean numbering 2',
            25: 'Korean numbering 1',
            26: 'Chinese numbering 1',
            27: 'Chinese numbering 2',
            28: 'Chinese numbering 3',
            29: 'Chinese numbering 4',
            30: 'Chinese Zodiac numbering 1',
            31: 'Chinese Zodiac numbering 2',
            32: 'Chinese Zodiac numbering 3',
            33: 'Taiwanese double-byte numbering 1',
            34: 'Taiwanese double-byte numbering 2',
            35: 'Taiwanese double-byte numbering 3',
            36: 'Taiwanese double-byte numbering 4',
            37: 'Chinese double-byte numbering 1',
            38: 'Chinese double-byte numbering 2',
            39: 'Chinese double-byte numbering 3',
            40: 'Chinese double-byte numbering 4',
            41: 'Korean double-byte numbering 1',
            42: 'Korean double-byte numbering 2',
            43: 'Korean double-byte numbering 3',
            44: 'Korean double-byte numbering 4',
            45: 'Hebrew non-standard decimal',
            46: 'Arabic Alif Ba Tah',
            47: 'Hebrew Biblical standard',
            48: 'Arabic Abjad style',
            255: 'No number',
        }
        self.__language_dict = {
            1078: 'Afrikaans',
            1052: 'Albanian',
            1025: 'Arabic',
            5121: 'Arabic Algeria',
            15361: 'Arabic Bahrain',
            3073: 'Arabic Egypt',
            1: 'Arabic General',
            2049: 'Arabic Iraq',
            11265: 'Arabic Jordan',
            13313: 'Arabic Kuwait',
            12289: 'Arabic Lebanon',
            4097: 'Arabic Libya',
            6145: 'Arabic Morocco',
            8193: 'Arabic Oman',
            16385: 'Arabic Qatar',
            10241: 'Arabic Syria',
            7169: 'Arabic Tunisia',
            14337: 'Arabic U.A.E.',
            9217: 'Arabic Yemen',
            1067: 'Armenian',
            1101: 'Assamese',
            2092: 'Azeri Cyrillic',
            1068: 'Azeri Latin',
            1069: 'Basque',
            1093: 'Bengali',
            4122: 'Bosnia Herzegovina',
            1026: 'Bulgarian',
            1109: 'Burmese',
            1059: 'Byelorussian',
            1027: 'Catalan',
            2052: 'Chinese China',
            4: 'Chinese General',
            3076: 'Chinese Hong Kong',
            4100: 'Chinese Singapore',
            1028: 'Chinese Taiwan',
            1050: 'Croatian',
            1029: 'Czech',
            1030: 'Danish',
            2067: 'Dutch Belgium',
            1043: 'Dutch Standard',
            3081: 'English Australia',
            10249: 'English Belize',
            2057: 'English British',
            4105: 'English Canada',
            9225: 'English Caribbean',
            9: 'English General',
            6153: 'English Ireland',
            8201: 'English Jamaica',
            5129: 'English New Zealand',
            13321: 'English Philippines',
            7177: 'English South Africa',
            11273: 'English Trinidad',
            1033: 'English United States',
            1061: 'Estonian',
            1080: 'Faerose',
            1065: 'Farsi',
            1035: 'Finnish',
            1036: 'French',
            2060: 'French Belgium',
            11276: 'French Cameroon',
            3084: 'French Canada',
            12300: "French Cote d'Ivoire",
            5132: 'French Luxembourg',
            13324: 'French Mali',
            6156: 'French Monaco',
            8204: 'French Reunion',
            10252: 'French Senegal',
            4108: 'French Swiss',
            7180: 'French West Indies',
            9228: 'French Democratic Republic of the Congo',
            1122: 'Frisian',
            1084: 'Gaelic',
            2108: 'Gaelic Ireland',
            1110: 'Galician',
            1079: 'Georgian',
            1031: 'German',
            3079: 'German Austrian',
            5127: 'German Liechtenstein',
            4103: 'German Luxembourg',
            2055: 'German Switzerland',
            1032: 'Greek',
            1095: 'Gujarati',
            1037: 'Hebrew',
            1081: 'Hindi',
            1038: 'Hungarian',
            1039: 'Icelandic',
            1057: 'Indonesian',
            1040: 'Italian',
            2064: 'Italian Switzerland',
            1041: 'Japanese',
            1099: 'Kannada',
            1120: 'Kashmiri',
            2144: 'Kashmiri India',
            1087: 'Kazakh',
            1107: 'Khmer',
            1088: 'Kirghiz',
            1111: 'Konkani',
            1042: 'Korean',
            2066: 'Korean Johab',
            1108: 'Lao',
            1062: 'Latvian',
            1063: 'Lithuanian',
            2087: 'Lithuanian Classic',
            1086: 'Malay',
            2110: 'Malay Brunei Darussalam',
            1100: 'Malayalam',
            1082: 'Maltese',
            1112: 'Manipuri',
            1102: 'Marathi',
            1104: 'Mongolian',
            1121: 'Nepali',
            2145: 'Nepali India',
            1044: 'Norwegian Bokmal',
            2068: 'Norwegian Nynorsk',
            1096: 'Oriya',
            1045: 'Polish',
            1046: 'Portuguese (Brazil)',
            2070: 'Portuguese (Portugal)',
            1094: 'Punjabi',
            1047: 'Rhaeto-Romanic',
            1048: 'Romanian',
            2072: 'Romanian Moldova',
            1049: 'Russian',
            2073: 'Russian Moldova',
            1083: 'Sami Lappish',
            1103: 'Sanskrit',
            3098: 'Serbian Cyrillic',
            2074: 'Serbian Latin',
            1113: 'Sindhi',
            1051: 'Slovak',
            1060: 'Slovenian',
            1070: 'Sorbian',
            11274: 'Spanish Argentina',
            16394: 'Spanish Bolivia',
            13322: 'Spanish Chile',
            9226: 'Spanish Colombia',
            5130: 'Spanish Costa Rica',
            7178: 'Spanish Dominican Republic',
            12298: 'Spanish Ecuador',
            17418: 'Spanish El Salvador',
            4106: 'Spanish Guatemala',
            18442: 'Spanish Honduras',
            2058: 'Spanish Mexico',
            3082: 'Spanish Modern',
            19466: 'Spanish Nicaragua',
            6154: 'Spanish Panama',
            15370: 'Spanish Paraguay',
            10250: 'Spanish Peru',
            20490: 'Spanish Puerto Rico',
            1034: 'Spanish Traditional',
            14346: 'Spanish Uruguay',
            8202: 'Spanish Venezuela',
            1072: 'Sutu',
            1089: 'Swahili',
            1053: 'Swedish',
            2077: 'Swedish Finland',
            1064: 'Tajik',
            1097: 'Tamil',
            1092: 'Tatar',
            1098: 'Telugu',
            1054: 'Thai',
            1105: 'Tibetan',
            1073: 'Tsonga',
            1074: 'Tswana',
            1055: 'Turkish',
            1090: 'Turkmen',
            1058: 'Ukranian',
            1056: 'Urdu',
            2080: 'Urdu India',
            2115: 'Uzbek Cyrillic',
            1091: 'Uzbek Latin',
            1075: 'Venda',
            1066: 'Vietnamese',
            1106: 'Welsh',
            1076: 'Xhosa',
            1085: 'Yiddish',
            1077: 'Zulu',
            1024: 'Unkown',
            255: 'Unkown',
        }
        # unknown
        # These must get passed on because they occurred after \\*
        # 'do'                :   ('un', 'unknown___', self.default_func),
        # 'company'           :   ('un', 'company___', self.default_func),
        # 'shpinst'           :   ('un', 'unknown___', self.default_func),
        # 'panose'            :   ('un', 'unknown___', self.default_func),
        # 'falt'              :   ('un', 'unknown___', self.default_func),
        # 'listoverridetable' :   ('un', 'unknown___', self.default_func),
        # 'category'          :   ('un', 'unknown___', self.default_func),
        # 'template'          :   ('un', 'unknown___', self.default_func),
        # 'ud'                :   ('un', 'unknown___', self.default_func),
        # 'formfield'         :   ('un', 'unknown___', self.default_func),
        # 'ts'                :   ('un', 'unknown___', self.default_func),
        # 'rsidtbl'           :   ('un', 'unknown___', self.default_func),
        # 'generator'         :   ('un', 'unknown___', self.default_func),
        # 'ftnsep'            :   ('un', 'unknown___', self.default_func),
        # 'aftnsep'           :   ('un', 'unknown___', self.default_func),
        # 'aftnsepc'          :   ('un', 'unknown___', self.default_func),
        # 'aftncn'            :   ('un', 'unknown___', self.default_func),
        # 'objclass'          :   ('un', 'unknown___', self.default_func),
        # 'objdata'           :   ('un', 'unknown___', self.default_func),
        # 'picprop'           :   ('un', 'unknown___', self.default_func),
        # 'blipuid'           :   ('un', 'unknown___', self.default_func),

    def __ms_hex_func(self, pre, token, num):
        num = num[1:]  # chop off leading 0, which I added
        num = num.upper()  # the mappings store hex in caps
        return f"tx<hx<__________<'{num}\n"  # add an ' for the mappings

    def ms_sub_func(self, pre, token, num):
        return f'tx<mc<__________<{token}\n'

    def direct_conv_func(self, pre, token, num):
        return f'mi<tg<empty_____<{token}\n'

    def default_func(self, pre, token, num):
        if num is None:
            num = 'true'
        return f'cw<{pre}<{token}<nu<{num}\n'

    def colorz_func(self, pre, token, num):
        if num is None:
            num = '0'
        return f'cw<{pre}<{token}<nu<{num}\n'

    def __list_type_func(self, pre, token, num):
        type = 'arabic'
        if num is None:
            type = 'Arabic'
        else:
            try:
                num = int(num)
            except ValueError:
                if self.__run_level > 3:
                    msg = f'Number "{num}" cannot be converted to integer\n'
                    raise self.__bug_handler(msg)
            type = self.__number_type_dict.get(num)
            if type is None:
                if self.__run_level > 3:
                    msg = 'No type for "%s" in self.__number_type_dict\n'
                    raise self.__bug_handler
                type = 'Arabic'
        return f'cw<{pre}<{token}<nu<{type}\n'

    def __language_func(self, pre, token, num):
        _lang_m = re.search(r'[0-9]+', num)
        assert _lang_m is not None
        lang_name = self.__language_dict.get(int(_lang_m.group()))
        if not lang_name:
            lang_name = 'not defined'
            if self.__run_level > 3:
                msg = f'No entry for number "{num}"'
                raise self.__bug_handler(msg)
        return f'cw<{pre}<{token}<nu<{lang_name}\n'

    def two_part_func(self, pre, token, num):
        list = token.split('<')
        token = list[0]
        num = list[1]
        return f'cw<{pre}<{token}<nu<{num}\n'
        # return 'cw<nu<nu<nu<%s>num<%s\n' % (token, num)

    def divide_by_2(self, pre, token, num):
        num = self.divide_num(num, 2)
        return f'cw<{pre}<{token}<nu<{num}\n'
        # return 'cw<nu<nu<nu<%s>%s<%s\n' % (token, num, token)

    def divide_by_20(self, pre, token, num):
        num = self.divide_num(num, 20)
        return f'cw<{pre}<{token}<nu<{num}\n'
        # return 'cw<nu<nu<nu<%s>%s<%s\n' % (token, num, token)

    def text_func(self, pre, token, num=None):
        return f'tx<nu<__________<{token}\n'

    def ob_func(self, pre, token, num=None):
        self.__bracket_count += 1
        return f'ob<nu<open-brack<{self.__bracket_count:04}\n'

    def cb_func(self, pre, token, num=None):
        line = f'cb<nu<clos-brack<{self.__bracket_count:04}\n'
        self.__bracket_count -= 1
        return line

    def color_func(self, pre, token, num):
        third_field = 'nu'
        if num[-1] == ';':
            num = num[:-1]
            third_field = 'en'
        num = f'{int(num):X}'
        if len(num) != 2:
            num = '0' + num
        return f'cw<{pre}<{token}<{third_field}<{num}\n'
        # return 'cw<cl<%s<nu<nu<%s>%s<%s\n' % (third_field, token, num, token)

    def bool_st_func(self, pre, token, num):
        if num is None or num in {'', '1'}:
            return f'cw<{pre}<{token}<nu<true\n'
            # return 'cw<nu<nu<nu<%s>true<%s\n' % (token, token)
        elif num == '0':
            return f'cw<{pre}<{token}<nu<false\n'
            # return 'cw<nu<nu<nu<%s>false<%s\n' % (token, token)
        else:
            msg = f"boolean should have some value module process tokens\ntoken is {token}\n'{num}'\n"
            raise self.__bug_handler(msg)

    def __no_sup_sub_func(self, pre, token, num):
        the_string = 'cw<ci<subscript_<nu<false\n'
        the_string += 'cw<ci<superscrip<nu<false\n'
        return the_string

    def divide_num(self, numerator, denominator):
        try:
            # calibre why ignore negative number? Wrong in case of \fi
            _num_m = re.search(r'[0-9.\-]+', numerator)
            assert _num_m is not None
            numerator = float(_num_m.group())
        except TypeError:
            if self.__run_level > 3:
                msg = (
                    'No number to process?\nthis indicates that the token \\(\\li\\)'
                    'should have a number and does not\nnumerator is'
                    f'"{numerator}"\ndenominator is "{denominator}"\n'
                )
                raise self.__bug_handler(msg)
            self.__return_code = max(self.__return_code, 5)
            return 0
        num = f'{round(numerator / denominator, 2):0.2f}'
        return num
        string_num = str(num)
        if string_num[-2:] == '.0':
            string_num = string_num[:-2]
        return string_num

    def split_let_num(self, token):
        match_obj = re.search(self.__num_exp, token)
        if match_obj is not None:
            first = match_obj.group(1)
            second = match_obj.group(2)
            if not second:
                if self.__run_level > 3:
                    msg = f"token is '{token}' \n"
                    raise self.__bug_handler(msg)
                return first, 0
        else:
            if self.__run_level > 3:
                msg = f"token is '{token}' \n"
                raise self.__bug_handler
            return token, 0
        return first, second

    def convert_to_hex(self, number):
        """Convert a string to uppercase hexadecimal"""
        num = int(number)
        try:
            hex_num = f'{num:X}'
            return hex_num
        except Exception:
            raise self.__bug_handler

    def process_cw(self, token):
        """Change the value of the control word by determining what dictionary
        it belongs to"""
        special = ['*', ':', '}', '{', '~', '_', '-', ';']
        # if token != "{" or token != "}":
        token = token[1:]  # strip off leading \
        token = token.replace(' ', '')
        # if not token: return
        only_alpha = token.isalpha()
        num = None
        if not only_alpha and token not in special:
            token, num = self.split_let_num(token)
        pre, token, action = self.dict_token.get(token, (None, None, None))
        if action:
            return action(pre, token, num)

    def __check_brackets(self, in_file):
        self.__check_brack_obj = check_brackets.CheckBrackets(file=in_file)
        good_br = self.__check_brack_obj.check_brackets()[0]
        if not good_br:
            return 1

    def process_tokens(self):
        """Main method for handling other methods."""
        line_count = 0
        with open_for_read(self.__file) as read_obj:
            with open_for_write(self.__write_to) as write_obj:
                for line in read_obj:
                    token = line.replace('\n', '')
                    line_count += 1
                    if line_count == 1 and token != '\\{':
                        msg = "\nInvalid RTF: document doesn't start with {\n"
                        raise self.__exception_handler(msg)
                    elif line_count == 2 and token[0:4] != '\\rtf':
                        msg = "\nInvalid RTF: document doesn't start with \\rtf \n"
                        raise self.__exception_handler(msg)

                    the_index = token.find('\\ ')
                    if token is not None and the_index > -1:
                        msg = f'\nInvalid RTF: token "\\ " not valid.\nError at line {line_count}'
                        raise self.__exception_handler(msg)
                    elif token[:1] == '\\':
                        line = self.process_cw(token)
                        if line is not None:
                            write_obj.write(line)
                    else:
                        fields = re.split(self.__utf_exp, token)
                        for field in fields:
                            if not field:
                                continue
                            if field[0:1] == '&':
                                write_obj.write(f'tx<ut<__________<{field}\n')
                            else:
                                write_obj.write(f'tx<nu<__________<{field}\n')

        if not line_count:
            msg = '\nInvalid RTF: file appears to be empty.\n'
            raise self.__exception_handler(msg)

        copy_obj = copy.Copy(bug_handler=self.__bug_handler)
        if self.__copy:
            copy_obj.copy_file(self.__write_to, 'processed_tokens.data')
        copy_obj.rename(self.__write_to, self.__file)
        os.remove(self.__write_to)

        bad_brackets = self.__check_brackets(self.__file)
        if bad_brackets:
            msg = '\nInvalid RTF: document does not have matching brackets.\n'
            raise self.__exception_handler(msg)
        else:
            return self.__return_code
