56 lines
2.3 KiB
Python
56 lines
2.3 KiB
Python
|
import re
|
||
|
|
from typing import Iterator
|
||
|
|
|
||
|
|
from latex2mathml import commands
|
||
|
|
from latex2mathml.symbols_parser import convert_symbol
|
||
|
|
|
||
|
|
UNITS = ("in", "mm", "cm", "pt", "em", "ex", "pc", "bp", "dd", "cc", "sp", "mu")
|
||
|
|
|
||
|
|
PATTERN = re.compile(
|
||
|
|
rf"""
|
||
|
|
(%[^\n]+) | # comment
|
||
|
|
(a-zA-Z) | # letter
|
||
|
|
([_^])(\d) | # number succeeding an underscore or a caret
|
||
|
|
(-?\d+(?:\.\d+)?\s*(?:{'|'.join(UNITS)})) | # dimension
|
||
|
|
(\d+(?:\.\d+)?) | # integer/decimal
|
||
|
|
(\.\d*) | # dot (.) or decimal can start with just a dot
|
||
|
|
(\\[\\\[\]{{}}\s!,:>;|_%#$&]) | # escaped characters
|
||
|
|
(\\(?:begin|end)\s*{{[a-zA-Z]+\*?}}) | # begin or end
|
||
|
|
(\\operatorname\s*{{[a-zA-Z\s*]+\*?\s*}}) | # operatorname
|
||
|
|
# color, fbox, href, hbox, mbox, style, text, textbf, textit, textrm, textsf, texttt
|
||
|
|
(\\(?:color|fbox|hbox|href|mbox|style|text|textbf|textit|textrm|textsf|texttt))\s*{{([^}}]*)}} |
|
||
|
|
(\\[cdt]?frac)\s*([.\d])\s*([.\d])? | # fractions
|
||
|
|
(\\math[a-z]+)({{)([a-zA-Z])(}}) | # commands starting with math
|
||
|
|
(\\[a-zA-Z]+) | # other commands
|
||
|
|
(\S) # non-space character
|
||
|
|
""",
|
||
|
|
re.VERBOSE,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def tokenize(latex_string: str, skip_comments: bool = True) -> Iterator[str]:
|
||
|
|
"""
|
||
|
|
Converts Latex string into tokens.
|
||
|
|
|
||
|
|
:param latex_string: Latex string.
|
||
|
|
:param skip_comments: Flag to skip comments (default=True).
|
||
|
|
"""
|
||
|
|
for match in PATTERN.finditer(latex_string):
|
||
|
|
tokens = tuple(filter(lambda x: x is not None, match.groups()))
|
||
|
|
if tokens[0].startswith(commands.MATH):
|
||
|
|
full_math = "".join(tokens)
|
||
|
|
symbol = convert_symbol(full_math)
|
||
|
|
if symbol:
|
||
|
|
yield f"&#x{symbol};"
|
||
|
|
continue
|
||
|
|
for captured in tokens:
|
||
|
|
if skip_comments and captured.startswith("%"):
|
||
|
|
break
|
||
|
|
if captured.endswith(UNITS):
|
||
|
|
yield captured.replace(" ", "")
|
||
|
|
continue
|
||
|
|
if captured.startswith((commands.BEGIN, commands.END, commands.OPERATORNAME)):
|
||
|
|
yield "".join(captured.split(" "))
|
||
|
|
continue
|
||
|
|
yield captured
|