% \iffalse meta-comment
%
%% File: l3unicode.dtx
%
% Copyright (C) 2018-2025 The LaTeX Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version.  The latest version
% of this license is in the file
%
%    https://www.latex-project.org/lppl.txt
%
% This file is part of the "l3kernel bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
%    https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver>
\documentclass[full,kernel]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
%   The \pkg{l3unicode} module\\ Unicode support functions^^A
% }
%
% \author{^^A
%  The \LaTeX{} Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released 2025-10-09}
%
% \maketitle
%
% \begin{documentation}
%
% This module provides Unicode-specific functions along with loading data
% from a range of Unicode Consortium files. Most of the code here is
% internal, but there are a small set of public functions. These work with
% Unicode \meta{codepoints} and are designed to give usable results with
% both Unicode-aware and $8$-bit engines.
%
% \begin{function}[EXP, added = 2022-10-09, updated = 2022-11-09]
%   {\codepoint_generate:nn}
%   \begin{syntax}
%      \cs{codepoint_generate:nn} \Arg{codepoint} \Arg{catcode}
%   \end{syntax}
%   Generates one or more character tokens representing the \meta{codepoint}.
%   With Unicode engines, exactly one character token will be generated, and
%   this will have the \meta{catcode} specified as the second argument:
%   \begin{itemize}
%     \item $1$ (begin group)
%     \item $2$ (end group)
%     \item $3$ (math toggle)
%     \item $4$ (alignment)
%     \item $6$ (parameter)
%     \item $7$ (math superscript)
%     \item $8$ (math subscript)
%     \item $10$ (space)
%     \item $11$ (letter)
%     \item $12$ (other)
%     \item $13$ (active)
%   \end{itemize}
%   For $8$-bit engines, between one and four character tokens will be
%   produced: these will be the bytes of the UTF-8 representation of the
%   \meta{codepoint}. For all codepoints outside of the classical ASCII
%   range, the generated character tokens will be active (category code
%   $13$); for codepoints in the ASCII range, the given \meta{catcode}
%   will be used. To allow the result of this function to be used
%   inside an expansion context, the result is protected by \cs{exp_not:n}.
%
%   \begin{texnote}
%     Users of (u)p\TeX{} note that these engines are treated as $8$-bit in
%     this context. In particular, for up\TeX{}, irrespective of the
%     \tn{kcatcode} of the \meta{codepoint}, any value outside the ASCII range
%     will result in a series of active bytes being generated.
%   \end{texnote}
% \end{function}
%
% \begin{function}[EXP, added = 2022-10-09]
%   {\codepoint_str_generate:n}
%   \begin{syntax}
%      \cs{codepoint_str_generate:n} \Arg{codepoint}
%   \end{syntax}
%   Generates one or more character tokens representing the \meta{codepoint}.
%   With Unicode engines, exactly one character token will be generated.
%   For $8$-bit engines, between one and four character tokens will be
%   produced: these will be the bytes of the UTF-8 representation of the
%   \meta{codepoint}. All of the generated character tokens will be of
%   category code $12$, except any spaces (codepoint $32$), which will be
%   category code $10$.
% \end{function}
%
% \begin{function}[added = 2023-06-19, EXP]{\codepoint_to_category:n}
%   \begin{syntax}
%     \cs{codepoint_to_category:n} \Arg{codepoint}
%   \end{syntax}
%   Expands to the Unicode general category identifier of the \meta{codepoint}.
%   The general category identifier is a string made up of two letter
%   characters, the first uppercase and the second lowercase. The uppercase
%   letters divide codepoints into broader groups, which are then refined
%   by the lowercase letter. For example, codepoints representing letters
%   all have identifiers starting \texttt{L}, for example \texttt{Lu}
%   (uppercase letter), \texttt{Lt} (titlecase letter), etc.
%   Full details are available in the documentation provided by the Unicode
%   Consortium: see
%   \url{https://www.unicode.org/reports/tr44/#General_Category_Values}
% \end{function}
%
% \begin{function}[added = 2022-10-09, EXP]{\codepoint_to_nfd:n}
%   \begin{syntax}
%     \cs{codepoint_to_nfd:n} \Arg{codepoint}
%   \end{syntax}
%   Converts the \meta{codepoint} to the Unicode Normalization
%   Form Canonical Decomposition. The generated character(s) will have
%   the current category code as they would if typed in directly for Unicode
%   engines; for $8$-bit engines, active characters are used for all codepoints
%   outside of the ASCII range.
% \end{function}
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3unicode} implementation}
%
%    \begin{macrocode}
%<*code>
%    \end{macrocode}
%
%    \begin{macrocode}
%<@@=codepoint>
%    \end{macrocode}
%
% \subsection{User functions}
%
% \begin{macro}[EXP]{\codepoint_str_generate:n}
% \begin{macro}[EXP]{\@@_str_generate:nnnn}
% \begin{macro}[EXP]{\codepoint_generate:nn}
% \begin{macro}[EXP]{\@@_generate:nnnn}
% \begin{macro}[EXP]{\@@_generate:n}
%   Conversion of a codepoint to a character (Unicode engines) or to one
%   or more bytes ($8$-bit engines) is required. For loading the data,
%   all that is needed is the form which creates strings: these are outside
%   the group as they will also be used when looking up data in the hash
%   table storage at point-of-use. Later, we will also need functions that
%   can generate character tokens for document use: those are defined below,
%   in the data recovery setup.
%    \begin{macrocode}
\sys_if_engine_opentype:TF
  {
    \cs_new:Npn \codepoint_str_generate:n #1
      {
        \int_compare:nNnTF {#1} = { `\  }
          { ~ }
          { \char_generate:nn {#1} { 12 } }
      }
    \cs_new:Npn \codepoint_generate:nn #1#2
      {
        \int_compare:nNnTF {#1} = { `\  }
          { ~ }
          {
            \__kernel_exp_not:w \exp_after:wN \exp_after:wN \exp_after:wN
              { \char_generate:nn {#1} {#2} }
          }
      }
  }
  {
    \cs_new:Npn \codepoint_str_generate:n #1
      {
        \int_compare:nNnTF {#1} = { `\  }
          { ~ }
          {
            \use:e
              {
                \exp_not:N \@@_str_generate:nnnn
                  \__kernel_codepoint_to_bytes:n {#1}
              }
          }
      }
    \cs_new:Npn \@@_str_generate:nnnn #1#2#3#4
      {
        \char_generate:nn {#1} { 12 }
        \tl_if_blank:nF {#2}
          {
            \char_generate:nn {#2} { 12 }
            \tl_if_blank:nF {#3}
              {
                \char_generate:nn {#3} { 12 }
                \tl_if_blank:nF {#4}
                  { \char_generate:nn {#4} { 12 } }
              }
          }
      }
    \cs_new:Npn \codepoint_generate:nn #1#2
      {
        \int_compare:nNnTF {#1} = { `\  }
          { ~ }
          {
            \int_compare:nNnTF {#1} < { "80 }
              {
                \__kernel_exp_not:w \exp_after:wN \exp_after:wN \exp_after:wN
                  { \char_generate:nn {#1} {#2} }
              }
              {
                \use:e
                  {
                    \exp_not:N \@@_generate:nnnn
                      \__kernel_codepoint_to_bytes:n {#1}
                  }
              }
          }
      }
    \cs_new:Npn \@@_generate:nnnn #1#2#3#4
      {
        \__kernel_exp_not:w \exp_after:wN
          {
            \tex_expanded:D
              {
                \@@_generate:n {#1}
                \@@_generate:n {#2}
                \tl_if_blank:nF {#3}
                  {
                    \@@_generate:n {#3}
                    \tl_if_blank:nF {#4}
                      { \@@_generate:n {#4} }
                  }
              }
          }
      }
    \cs_new:Npn \@@_generate:n #1
      {
        \__kernel_exp_not:w \exp_after:wN \exp_after:wN \exp_after:wN
          { \char_generate:nn {#1} { 13 } }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]{\__kernel_codepoint_to_bytes:n}
% \begin{macro}[EXP]{\@@_to_bytes_auxi:n}
% \begin{macro}[EXP]{\@@_to_bytes_auxii:Nnn}
% \begin{macro}[EXP]{\@@_to_bytes_auxiii:n}
% \begin{macro}[EXP]
%   {
%     \@@_to_bytes_outputi:nw   ,
%     \@@_to_bytes_outputii:nw  ,
%     \@@_to_bytes_outputiii:nw ,
%     \@@_to_bytes_outputiv:nw
%   }
% \begin{macro}[EXP]
%   {\@@_to_bytes_output:nnn, \@@_to_bytes_output:fnn}
% \begin{macro}[EXP]{\@@_to_bytes_end:}
%   This code converts a codepoint into the correct UTF-8 representation.
%   In terms of the algorithm itself, see
%   \url{https://en.wikipedia.org/wiki/UTF-8} for the octet pattern.
%    \begin{macrocode}
\cs_new:Npn \__kernel_codepoint_to_bytes:n #1
  {
    \exp_args:Nf \@@_to_bytes_auxi:n
      { \int_eval:n {#1} }
  }
\cs_new:Npn \@@_to_bytes_auxi:n #1
  {
    \if_int_compare:w #1 > "80 \exp_stop_f:
      \if_int_compare:w #1 < "800 \exp_stop_f:
        \@@_to_bytes_outputi:nw
          { \@@_to_bytes_auxii:Nnn C {#1} { 64 } }
        \@@_to_bytes_outputii:nw
          { \@@_to_bytes_auxiii:n {#1} }
      \else:
        \if_int_compare:w #1 < "10000 \exp_stop_f:
          \@@_to_bytes_outputi:nw
            { \@@_to_bytes_auxii:Nnn E {#1} { 64 * 64 } }
          \@@_to_bytes_outputii:nw
            {
              \@@_to_bytes_auxiii:n
                { \int_div_truncate:nn {#1} { 64 } }
            }
          \@@_to_bytes_outputiii:nw
            { \@@_to_bytes_auxiii:n {#1} }
        \else:
          \@@_to_bytes_outputi:nw
            {
              \@@_to_bytes_auxii:Nnn F
                {#1} { 64 * 64 * 64 }
            }
          \@@_to_bytes_outputii:nw
            {
              \@@_to_bytes_auxiii:n
                { \int_div_truncate:nn {#1} { 64 * 64 } }
            }
          \@@_to_bytes_outputiii:nw
            {
              \@@_to_bytes_auxiii:n
                { \int_div_truncate:nn {#1} { 64 } }
            }
          \@@_to_bytes_outputiv:nw
            { \@@_to_bytes_auxiii:n {#1} }
        \fi:
      \fi:
    \else:
      \@@_to_bytes_outputi:nw {#1}
    \fi:
    \@@_to_bytes_end: { } { } { } { }
  }
\cs_new:Npn \@@_to_bytes_auxii:Nnn #1#2#3
  {  "#10 + \int_div_truncate:nn {#2} {#3} }
\cs_new:Npn \@@_to_bytes_auxiii:n #1
  { \int_mod:nn {#1} { 64 } + 128 }
\cs_new:Npn \@@_to_bytes_outputi:nw
  #1 #2 \@@_to_bytes_end: #3
  { \@@_to_bytes_output:fnn { \int_eval:n {#1} } { } {#2} }
\cs_new:Npn \@@_to_bytes_outputii:nw
  #1 #2 \@@_to_bytes_end: #3#4
  { \@@_to_bytes_output:fnn { \int_eval:n {#1} } { {#3} } {#2} }
\cs_new:Npn \@@_to_bytes_outputiii:nw
  #1 #2 \@@_to_bytes_end: #3#4#5
  {
    \@@_to_bytes_output:fnn
      { \int_eval:n {#1} } { {#3} {#4} } {#2}
  }
\cs_new:Npn \@@_to_bytes_outputiv:nw
  #1 #2 \@@_to_bytes_end: #3#4#5#6
  {
    \@@_to_bytes_output:fnn
      { \int_eval:n {#1} } { {#3} {#4} {#5} } {#2}
  }
\cs_new:Npn \@@_to_bytes_output:nnn #1#2#3
  {
    #3
    \@@_to_bytes_end: #2 {#1}
  }
\cs_generate_variant:Nn \@@_to_bytes_output:nnn { f }
\cs_new:Npn \@@_to_bytes_end: { }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]{\codepoint_to_category:n}
%   Get the value and convert back to the string.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \cs_new:Npn \codepoint_to_category:n #1
      {
        \__kernel_codepoint_data:wn \int_eval:n { #1 } { category }
      }
  }
  {
    \cs_new:Npn \codepoint_to_category:n #1
      {
        \cs:w
          c_@@_category_
          \tex_romannumeral:D 
            \__kernel_codepoint_data:nn { category } {#1}
          _str
        \cs_end:
      }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[EXP]{\codepoint_to_nfd:n, \@@_to_nfd:n}
% \begin{macro}[EXP]{\@@_to_nfd:nn}
% \begin{macro}[EXP]{\@@_to_nfd:nnn}
% \begin{macro}[EXP]{\@@_to_nfd:nnnn}
%   Converted to NFD is a potentially-recursive process: the key is to
%   check if we get the input codepoint back again. As far as possible,
%   we use the same path for all engines.
%    \begin{macrocode}
\cs_new:Npn \codepoint_to_nfd:n #1
  { \exp_args:Ne \@@_to_nfd:n { \int_eval:n {#1} } }
\cs_new:Npn \@@_to_nfd:n #1
  { \@@_to_nfd:nn {#1} { \char_value_catcode:n {#1} } }
\sys_if_engine_opentype:F
  {
    \cs_gset:Npn \@@_to_nfd:n #1
      {
        \int_compare:nNnTF {#1} > { "80 }
          { \@@_to_nfd:nn {#1} { 12 } }
          { \@@_to_nfd:nn {#1} { \char_value_catcode:n {#1} } }
      }
  }
\cs_new:Npn \@@_to_nfd:nn #1#2
  {
    \exp_args:Ne \@@_to_nfd:nnn
      { \@@_nfd:n {#1} } {#1} {#2}
  }
\cs_new:Npn \@@_to_nfd:nnn #1#2#3 { \@@_to_nfd:nnnn #1 {#2} {#3} }
\cs_new:Npn \@@_to_nfd:nnnn #1#2#3#4
  {
    \int_compare:nNnTF {#1} = {#3}
      { \codepoint_generate:nn {#1} {#4} }
      {
        \@@_to_nfd:nn {#1} {#4}
        \tl_if_blank:nF {#2}
          { \@@_to_nfd:nn {#2} {#4} }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Data loader}
%
% Text operations requires data from the Unicode Consortium. Data read into
% Unicode engine formats is at best a small part of what we need, so there
% is a loader here to set  up the appropriate data structures.
%
% Where we need data for most or all of the Unicode range, we use the two-stage
% table approach recommended by the Unicode Consortium and demonstrated in a
% model implementation in Python in
% \url{https://www.strchr.com/multi-stage_tables}. This approach uses the
% \texttt{intarray} (\texttt{fontdimen}-based) data type as it is fast for
% random access and avoids significant hash table usage. In contrast, where
% only a small subset of codepoints are required, storage as macros is
% preferable. There is also some consideration of the effort needed to load
% data: see for example the grapheme breaking information, which would be 
% problematic to convert into a two-stage table but which can be used with
% reasonable performance in a small number of comma lists (at the cost that
% breaking at higher codepoint Hangul characters will be slightly slow).
%
% \begin{variable}{\c_@@_block_size_int}
%   Choosing the block size for the blocks in the two-stage approach is
%   non-trivial: depending on the data stored, the optimal size for
%   memory usage will vary. At the same time, for us there is also the
%   question of load-time: larger blocks require longer comma lists
%   as intermediates, so are slower. As this is going to be needed
%   to use the data, we set it up outside of the group for clarity.
%    \begin{macrocode}
\int_const:Nn \c_@@_block_size_int { 64 }
%    \end{macrocode}
% \end{variable}
%
% Parsing the data files can be the same way for all engines, but where they
% are stored as character tokens, the construction method depends on whether
% they are Unicode or $8$-bit internally. Parsing is therefore done by common
% functions, with some data storage using engine-specific auxiliaries.
%
% As only the data needs to remain at the end of this process, everything
% is set up inside a group. The only thing that is outside is creating a
% stream: they are global anyway and it is best to force a stream for
% all engines.
%
% Lua\TeX{} does not use the parsing into multi stage tables. While
% it still stores the data in this format, this is handled by a faster
% Lua implementation in \pkg{lua-uni-algos} and reused here.
%
% \begin{variable}{\g_@@_data_ior}
%    \begin{macrocode}
\ior_new:N \g_@@_data_ior
%    \end{macrocode}
% \end{variable}
%
% We need some setup for the two-part table approach. The number of blocks we
% need will be variable, but the resulting size of the stage one table
% is predictable. For performance reasons, we therefore create the stage one 
% tables now so they can be used immediately, and will later rename them as a
% constant tables. For each two-stage table construction, we need a comma
% list to hold the partial block and a couple of integers to track where
% we are up to. To avoid burning registers, the latter are stored in macros
% and are \enquote{fake} integers. We also avoid any \texttt{new} functions,
% keeping as much as possible local.
%
% As we need both positive and negative values, case data requires one
% two-stage table for each transformation. In contrasts, general Unicode
% properties could be stored in one table with appropriate combination rules:
% that is not done at present but is likely to be added over time. Here, all
% that is needed is additional entries into the comma-list to create the
% structures.
%
% Notice that in the standard \pkg{expl3} way we are indexes position not
% offset: that does mean a little work later.
%
% For integer values, everything is done using token lists to avoid losing
% registers or making global names.
%    \begin{macrocode}
\sys_if_engine_luatex:F
  {
    \group_begin:
      \clist_map_inline:nn
        { category , grapheme , lowercase , uppercase , wordbreak }
        {
          \cs_set_nopar:cpn { l_@@_ #1 _block_clist } { }
          \cs_set_nopar:cpn { l_@@_ #1 _block_tl } { 1 }
          \cs_set_nopar:cpn { l_@@_ #1 _pos_tl } { 0 }
          \cs_set_nopar:cpn { l_@@_ #1 _next_tl } { 0 }
          \intarray_new:cn { g_@@_ #1 _index_intarray }
            { \int_div_truncate:nn { "110000 } \c_@@_block_size_int }
        }
%    \end{macrocode}
% We also need to track the matched block: this is used dynamically so we
% only require one variable.
%    \begin{macrocode}
      \cs_set_nopar:Npn \l_@@_matched_block_tl { 0 }
%    \end{macrocode}
% For Unicode general category and the various breaking properties, there needs
% to be numerical representation of each possible value. As we need to go from
% string to number here, but the other way elsewhere, we set up fast mappings
% both ways, but one set local and the other as constants.
%    \begin{macrocode}
      \cs_set_nopar:Npn \l_@@_uppercase_default_tl { 0 }
      \cs_set_nopar:Npn \l_@@_lowercase_default_tl { 0 }
      \cs_set_protected:Npn \@@_data_auxi:w #1#2#3
        {
          \quark_if_recursion_tail_stop:n {#3}
          \cs_set_nopar:cpn { l_@@_ #2 _ #3 _tl } {#1}
          \str_const:cn { c_@@_ #2 _ \tex_romannumeral:D #1 _str } {#3}
          \exp_args:Ne \@@_data_auxi:w { \int_eval:n { #1 + 1 } } {#2}
        }
      \@@_data_auxi:w { 1 } { category }
        { Lu } { Ll } { Lt } { Lm } { Lo }
        { Mn } { Me } { Mc }
        { Nd } { Nl } { No }
        { Zs } { Zl } { Zp }
        { Cc } { Cf } { Co } { Cs } { Cn }
        { Pd } { Ps } { Pe } { Pc } { Po } { Pi } { Pf }
        { Sm } { Sc } { Sk } { So }
        \q_recursion_tail
        \q_recursion_stop
      \cs_set_eq:NN \l_@@_category_default_tl \l_@@_category_Cn_tl
      \@@_data_auxi:w { 1 } { grapheme }
        { Control }
        { CR } { LF } { ZWJ }
        { Extend }
        { L } { LV } { LVT } { T } { V }
        { Prepend }
        { Regional_Indicator }
        { SpacingMark }
        { Other }
        \q_recursion_tail
        \q_recursion_stop
      \cs_set_eq:NN \l_@@_grapheme_default_tl \l_@@_grapheme_Other_tl
      \@@_data_auxi:w { 1 } { wordbreak }
        { Double_Quote } { Single_Quote }
        { CR } { LF } { Newline }
        { WSegSpace } { ZWJ }
        { Extend } { ExtendNumLet }
        { Regional_Indicator }
        { Format }
        { Katakana }
        { ALetter } { MidLetter } { Hebrew_Letter }
        { Numeric }{ MidNum } { MidNumLet }
        { Other }
        \q_recursion_tail
        \q_recursion_stop
      \cs_set_eq:NN \l_@@_wordbreak_default_tl \l_@@_wordbreak_Other_tl
%    \end{macrocode}
%  For the the property fields, we need to use a two-step procedure
%  as the file is ordered by class not codepoint. First, we parse the content
%  into the hash table locally: there are around $1400$ codepoints to handle,
%  which is workable. Reading this is quite easy: we store any end-of-range
%  codepoint and the class that applies. Conversion to a two-stage table
%  is deferred to later.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ~ #3 \q_stop
        { \@@_data_auxii:w #1 .. \q_stop {#2} }
      \cs_set_protected:Npn \@@_data_auxii:w #1 .. #2 \q_stop
        { \@@_data_auxiii:w #1 ~ .. #2 ~ \q_stop }
      \cs_set_protected:Npn \@@_data_auxiii:w #1 ~ #2 .. #3 ~ #4 \q_stop #5#6
        {
          \cs_set_nopar:cpe { l_@@_ #6 _ \tex_romannumeral:D "#1 _tl }
            {
              {#3}
              { \use:c { l_@@_ #6 _ #5 _tl } }
            }
        }
      \cs_set_protected:Npn \@@_data_auxvi:w #1#2
        {
          \ior_open:Nn \g_@@_data_ior {#1}
          \ior_str_map_inline:Nn \g_@@_data_ior
            {
              \str_if_eq:eeF { \tl_head:w ##1 \c_hash_str \q_stop } { \c_hash_str }
                {
                  \tl_if_blank:nF {##1}
                    { \@@_data_auxi:w ##1 \q_stop {#2} }
                }
            }
          \ior_close:N \g_@@_data_ior
        }
      \@@_data_auxvi:w { GraphemeBreakProperty.txt } { grapheme }
      \@@_data_auxvi:w { WordBreakProperty.txt } { wordbreak }
%    \end{macrocode}
%  The set up for adding property data is now sorted.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_property:nnnn #1#2#3#4
        {
          \int_compare:nNnT {#3} > { \use:c { l_@@_ #4 _next_tl } }
            {
              \@@_range:nnv {#3} {#4}
                { l_@@_ #4 _default_tl }
            }
          \@@_add:nn {#4} {#2}
          \tl_set:ce { l_@@_ #4 _next_tl } { \int_eval:n { #3 + 1 } }
          \tl_if_blank:nF {#1}
            {
              \@@_range:nnn {"#1} {#4} {#2}
              \@@_add:nn {#4} {#2}
              \tl_set:ce { l_@@_ #4 _next_tl } { \int_eval:n { "#1 + 1 } }
            }
        }
%    \end{macrocode}
% Parse the main Unicode data file and pull out the NFD and case changing
% data. The NFD data is stored on using the hash table approach and can yield
% a predictable number of codepoints: one or two. We also need the case data,
% which will be modified further below. To allow for finding ranges, the
% description of the codepoint needs to be carried forward.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_auxi:w
        #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ;
        {
          \tl_if_blank:nF {#6}
            {
              \tl_if_head_eq_charcode:nNF {#6}  < % >
                { \@@_data_auxii:w #1 ; #6 ~ \q_stop }
            }
          \@@_data_auxiii:w #1 ; #2 ; #3 ;
        }
      \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop
        {
          \tl_const:ce
            { c_@@_nfd_ \codepoint_str_generate:n {"#1} _tl }
            {
              {"#2}
              { \tl_if_blank:nF {#3} {"#3} }
            }
        }
%    \end{macrocode}
% The category data needs to be converted from a string to the numerical
% equivalent: a simple operation.
% The case data is going to be stored as an offset from the parent character,
% rather than an absolute value. We therefore deal with that plus the situation
% where a codepoint has no mapping data in one shot.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_auxiii:w
        #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ~ \q_stop
        {
          \use:e
            {
              \@@_data_auxiv:w
                #1 ; #2 ;
                \@@_data_category:n {#3} ;
                \@@_data_offset:nn {#1} {#7} ;
                \@@_data_offset:nn {#1} {#8} ;
                #9;
            }
        }
      \cs_set:Npn \@@_data_category:n #1
        { \use:c { l_@@_category_ #1 _tl } }
      \cs_set:Npn \@@_data_offset:nn #1#2
        {
          \tl_if_blank:nTF {#2}
            { 0 }
            { \int_eval:n { "#2 - "#1 } }
        }
%    \end{macrocode}
% To deal with ranges, we track the position of the next codepoint expected.
% If there is a gap, we deal with that separately: it could be a range or
% an unused part of the Unicode space. As such, we deal with the current
% codepoint here whether or not there is range to fill in. Upper- and 
% lowercase data go into the two-stage table, any titlecase exception is
% just stored in a macro. The data for the codepoint is added to the current
% block, and if that is now complete we move on to save the block. The
% case exceptions are all stored as codepoints, with a fixed number of
% balanced text as we know that there are never more than three.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_auxiv:w #1 ; #2 ; #3 ; #4 ; #5 ; #6 ;
        {
          \int_compare:nNnT {"#1} > \l_@@_category_next_tl
            {
              \@@_data_auxv:nnnnw {#1} {#3} {#4} {#5}
                #2 Last> \q_stop
            }
          \@@_add:nn { category } {#3}
          \@@_add:nn { uppercase } {#4}
          \@@_add:nn { lowercase } {#5}
          \int_compare:nNnF {#4} = { \@@_data_offset:nn {#1} {#6} }
            {
              \tl_const:ce
                { c_@@_titlecase_ \codepoint_str_generate:n {"#1} _tl }
                { {"#6} { } { } }
            }
          \@@_data_auxvi:nn { grapheme } {"#1}
          \@@_data_auxvi:nn { wordbreak } {"#1}
%    \end{macrocode}
% To deal with the property data, we need to recover stored information and
% build the table. Generally we can assume that all codepoints in these
% files are in |UnicodeData.txt|, but there is one place
% a bit more work is needed. For Hangul syllables, the main file lists a range
% but within that we have different classifications for breaking: that needs to
% be handled with a second loop. The values for this range are hard-coded,
% skipping the end-of-range as that will be mopped up by the main loop.
%    \begin{macrocode}
          \int_compare:nNnT {"#1} = { "AC00 }
            {
              \int_step_inline:nnn { "AC01 } { "D7A2 }
                { \@@_data_auxvi:nn { grapheme } {##1} }
            }
          \tl_set:Ne \l_@@_category_next_tl
            { \int_eval:n { "#1 + 1 } }
          \tl_set_eq:NN \l_@@_lowercase_next_tl \l_@@_category_next_tl
          \tl_set_eq:NN \l_@@_uppercase_next_tl \l_@@_category_next_tl
        }
      \cs_set_protected:Npn \@@_add:nn #1#2
        {
          \clist_put_right:cn { l_@@_ #1 _block_clist } {#2}
          \int_compare:nNnT { \clist_count:c { l_@@_ #1 _block_clist } }
            = \c_@@_block_size_int
            { \@@_save_blocks:nn {#1} { 1 } }
        }
%    \end{macrocode}
%  Distinguish between a range and a gap, and pass on the appropriate value(s).
%  The general category for unassigned characters is \texttt{Cn}, so we
%  find the correct value once and then use that.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_auxv:nnnnw #1#2#3#4#5 Last> #6 \q_stop
        {
          \tl_if_blank:nTF {#6}
            {
              \@@_range:nno {"#1} { category }
                \l_@@_category_default_tl
              \@@_range:nnn {"#1} { uppercase } { 0 }
              \@@_range:nnn {"#1} { lowercase } { 0 }
            }
            {
              \@@_range:nnn {"#1} { category } {#2}
              \@@_range:nnn {"#1} { uppercase } {#3}
              \@@_range:nnn {"#1} { lowercase } {#4}
            }      
        }
%    \end{macrocode}
%  Recover and process grapheme data.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_data_auxvi:nn #1#2
        {
          \cs_if_exist:cT
            { l_@@_ #1 _ \tex_romannumeral:D #2 _tl }
            {
              \exp_after:wN \exp_after:wN \exp_after:wN \@@_data_property:nnnn
                \cs:w
                  l_@@_ #1 _ \tex_romannumeral:D #2 _tl
                    \cs_end: {#2} {#1}
            }
        }
%    \end{macrocode}
%  Calculated the length of the range and the space remaining in the current
%  block.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_range:nnn #1#2
        {
          \exp_args:Nf \@@_range_aux:nnn
            { \int_eval:n { #1 - \use:c { l_@@_ #2 _next_tl } } }
            {#2}
        }
      \cs_set_protected:Npn \@@_range:nno { \exp_args:Nnno \@@_range:nnn }
      \cs_set_protected:Npn \@@_range:nnv { \exp_args:Nnnv \@@_range:nnn }
      \cs_set_protected:Npn \@@_range_aux:nnn #1#2
        {
          \exp_args:Nf \@@_range:nnnn
            {
              \int_min:nn
                {#1}
                {
                  \c_@@_block_size_int 
                  - \clist_count:c { l_@@_ #2 _block_clist }
                }
            }
            {#1} {#2}
        }
%    \end{macrocode}
%   Here we want to do three things: add to and possibly complete the current
%   block, add complete blocks quickly, then finish up the range in a final
%   open block. We need to avoid as far as possible avoid dealing with every
%   single codepoint, so the middle step is optimized.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_range:nnnn #1#2#3#4
        {  
          \prg_replicate:nn {#1}
            { \clist_put_right:cn { l_@@_ #3 _block_clist } {#4} }
        \int_compare:nNnT { \clist_count:c { l_@@_ #3 _block_clist } }
          = \c_@@_block_size_int
          { \@@_save_blocks:nn {#3} { 1 } }
        \int_compare:nNnF
          { \int_div_truncate:nn { #2 - #1 } \c_@@_block_size_int } = 0
          {
            \tl_set:ce { l_@@_ #3 _block_clist }
              {
                \exp_args:NNe \use:nn \use_none:n
                  { \prg_replicate:nn { \c_@@_block_size_int } { , #4 } }
              }
            \@@_save_blocks:nn {#3}
              { \int_div_truncate:nn { (#2 - #1) } \c_@@_block_size_int }
          }
        \prg_replicate:nn
          { \int_mod:nn { #2 - #1 } \c_@@_block_size_int }
          { \clist_put_right:ce { l_@@_ #3 _block_clist } {#4} }
        }
%    \end{macrocode}
%   To allow rapid comparison, each completed block is stored locally as a
%   comma list: once all of the blocks have been created, they are converted
%   into an \texttt{intarray} in one step. The aim here is to check the current
%   block against those we've already used, and either match to an existing
%   block or save a new block.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_save_blocks:nn #1#2
        {
          \tl_set_eq:Nc \l_@@_matched_block_tl { l_@@_ #1 _block_tl }
          \int_step_inline:nn { \tl_use:c { l_@@_ #1 _block_tl } - 1 }
            {
              \tl_if_eq:ccT { l_@@_ #1 _block_clist }
                { l_@@_ #1 _block_ ##1 _clist }
                { \tl_set:Nn \l_@@_matched_block_tl {##1} }
            }
          \int_compare:nNnT
            { \tl_use:c { l_@@_ #1 _block_tl } } = \l_@@_matched_block_tl
              {
                \clist_set_eq:cc
                  {
                    l_@@_ #1 _block_
                    \tl_use:c { l_@@_ #1 _block_tl } _clist
                  }
                  { l_@@_ #1 _block_clist }
                \tl_set:ce { l_@@_ #1 _block_tl }
                  { \int_eval:n { \tl_use:c { l_@@_ #1 _block_tl } + 1 } }
              }
%    \end{macrocode}
% Here, we avoid \cs{prg_replicate:nn} as the number of tokens generated would be
% high: that shows in the format dump (although \TeX{} recovers memory during
% the subsequent runs).
%    \begin{macrocode}
            \int_step_inline:nnn
              { \tl_use:c { l_@@_ #1 _pos_tl } + 1 }
              { \tl_use:c { l_@@_ #1 _pos_tl } + #2 }
              {
                \exp_args:Nc \__kernel_intarray_gset:Nnn
                  { g_@@_ #1 _index_intarray }
                  {##1}
                  \l_@@_matched_block_tl
              }
            \tl_set:ce { l_@@_ #1 _pos_tl }
              { \int_eval:n { \tl_use:c { l_@@_ #1 _pos_tl } + #2 } }
          \clist_clear:c { l_@@_ #1 _block_clist }
        }
%    \end{macrocode}
% Close out the final block, rename the first stage table, then combine all
% of the block comma-lists into one large second-stage table with offsets.
% As we use an index not an offset, there is a little back-and-forward to do.
%    \begin{macrocode}
      \cs_set_protected:Npn \@@_finalize_blocks:n #1
        {
          \clist_map_inline:nn {#1}
            {
              \exp_args:Nnnv \@@_range:nnn { "110000 } {##1}
                { l_@@_ ##1 _default_tl }
              \@@_finalize_blocks_aux:n {##1}
            }
        }
      \cs_set_protected:Npn \@@_finalize_blocks_aux:n #1
        {
          \cs_gset_eq:cc { c_@@_ #1 _index_intarray } { g_@@_ #1 _index_intarray }
          \cs_undefine:c { g_@@_ #1 _index_intarray }
          \intarray_new:cn { g_@@_ #1 _blocks_intarray }
            { ( \tl_use:c { l_@@_ #1 _block_tl } - 1 ) * \c_@@_block_size_int }
          \int_step_inline:nn { \tl_use:c { l_@@_ #1 _block_tl } - 1 }
            {
              \exp_args:Nv \@@_finalize_blocks:nnn
                { l_@@_ #1 _block_ ##1 _clist }
                {##1} {#1}
            }
          \cs_gset_eq:cc { c_@@_ #1 _blocks_intarray }
            { g_@@_ #1 _blocks_intarray }
          \cs_undefine:c { g_@@_ #1 _blocks_intarray }
        }
      \cs_set_protected:Npn \@@_finalize_blocks:nnn #1#2#3
        {
          \exp_args:Nnf \@@_finalize_blocks:nnnw { 1 }
            { \int_eval:n { ( #2 - 1 ) * \c_@@_block_size_int } }
            {#3}
            #1 , \q_recursion_tail , \q_recursion_stop
        }
      \cs_set_protected:Npn \@@_finalize_blocks:nnnw #1#2#3#4 ,
        {
          \quark_if_recursion_tail_stop:n {#4}
          \intarray_gset:cnn { g_@@_ #3 _blocks_intarray }
            { #1 + #2 }
            {#4}
          \exp_args:Nf \@@_finalize_blocks:nnnw
            { \int_eval:n { #1 + 1 } } {#2} {#3}
        }
%    \end{macrocode}
%  With the setup done, read the main data file: it's easiest to do that as
%  a token list with spaces retained.
%    \begin{macrocode}
      \ior_open:Nn \g_@@_data_ior { UnicodeData.txt }
      \char_set_catcode_space:n { `\  }%
      \ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl
        {%
          \if_meaning:w \l_@@_tmpa_tl \c_space_tl
            \exp_after:wN \ior_map_break:
          \fi:
          \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop
        }%
      \char_set_catcode_ignore:n { `\  }%
%    \end{macrocode}
%  Blocks need to be finalized once they have been read as this relies on
%  knowing the last real codepoint. So the tables for |UnicodeData.txt|
%  are closed out before reading additional files.
%    \begin{macrocode}
      \@@_finalize_blocks:n
        { category , grapheme , lowercase , uppercase , wordbreak }
    \group_end:
%    \end{macrocode}
%
% \begin{macro}[EXP]{\__kernel_codepoint_data:nn}
% \begin{macro}[EXP]{\@@_data:nnn}
%   Recover data from a two-stage table: entirely generic as this applies to
%   all tables (as we use the same block size for all of them). Notice that
%   as we use indices not offsets we have to shuffle out-by-one issues. This
%   function is needed \emph{before} loading the special casing data, as there
%   we need to be able to check the standard case mappings.
%
%   This macro is not defined in Lua\TeX{} where the data is provided in a more
%   preprocessed form in \cs\__kernel_codepoint_data:wn.
%    \begin{macrocode}
    \cs_new:Npn \__kernel_codepoint_data:nn #1#2
      {
        \exp_args:Nf \@@_data:nnn
          {
            \int_eval:n
              {
                \c_@@_block_size_int *
                  (
                    \intarray_item:cn { c_@@_ #1 _index_intarray }
                      {
                        \int_div_truncate:nn {#2}
                          \c_@@_block_size_int
                        + 1
                      }
                      - 1
                  )
              }
          }
          {#2} {#1}
      }
    \cs_new:Npn \@@_data:nnn #1#2#3
      {
        \intarray_item:cn { c_@@_ #3 _blocks_intarray }
          { #1 + \int_mod:nn {#2} \c_@@_block_size_int + 1 }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% The remaining files use the \TeX{} based parser also in Lua\TeX{}.
%
% The other data files all use C-style comments so we have to worry about
% |#| tokens (and reading as strings). The set up for case folding is in two
% parts. For the basic (core) mappings, folding is the same as lower casing in
% most positions so only store the differences. For the more complex foldings,
% always store the result, splitting up the two or three code points in the input
% as required.
%    \begin{macrocode}
\group_begin:
  \ior_open:Nn \g_@@_data_ior { CaseFolding.txt }
  \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop
    {
      \if:w \tl_head:n { #2 ? } C
        \reverse_if:N \if_int_compare:w
          \@@_lowercase_base_mapping:n { "#1 }
            = "#3 ~
          \tl_const:ce
            { c_@@_casefold_ \codepoint_str_generate:n {"#1} _tl }
            { {"#3} { } { } }
        \fi:
      \else:
        \if:w \tl_head:n { #2 ? } F
          \@@_data_auxii:w #1 ~ #3 ~ \q_stop
        \fi:
      \fi:
    }
  \sys_if_engine_luatex:TF
    {
      \cs_set:Npn \@@_lowercase_base_mapping:n #1
        {
          \__kernel_codepoint_data:wn #1 { lowercase }
        }
    }
    {
      \cs_set:Npn \@@_lowercase_base_mapping:n #1
        {
          \int_eval:n { \__kernel_codepoint_data:nn { lowercase } {#1} + #1 }
        }
    }
%    \end{macrocode}
% Here, |#4| can have a trailing space, so we tidy up a bit at the cost of
% speed for these small number of cases it applies to.
%    \begin{macrocode}
  \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop
    {
      \tl_const:ce { c_@@_casefold_ \codepoint_str_generate:n {"#1} _tl }
        {
          {"#2}
          {"#3}
          { \tl_if_blank:nF {#4} { " \int_to_Hex:n {"#4} } }
        }
    }
  \ior_str_map_inline:Nn \g_@@_data_ior
    {
      \reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop
        \@@_data_auxi:w #1 \q_stop
      \fi:
    }
  \ior_close:N \g_@@_data_ior
%    \end{macrocode}
% For upper- and lowercasing special situations, there is a bit more to
% do as we also have titlecasing to consider, plus we need to stop part-way
% through the file.
%    \begin{macrocode}
  \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt }
  \cs_set_protected:Npn \@@_data_auxi:w
    #1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop
    {
      \use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop
      \use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop
      \str_if_eq:nnF {#3} {#4}
        { \use:n { \@@_data_auxii:w #1 ~ title ~ #3 ~ } ~ \q_stop }
    }
  \cs_set_protected:Npn \@@_data_auxii:w
    #1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop
    {
      \tl_if_empty:nF {#4}
        {
          \tl_const:ce { c_@@_ #2 case_ \codepoint_str_generate:n {"#1} _tl }
            {
              {"#3}
              {"#4}
              { \tl_if_blank:nF {#5} {"#5} }
            }
        }
    }
  \ior_str_map_inline:Nn \g_@@_data_ior
    {
      \str_if_eq:eeTF { \tl_head:w #1 \c_hash_str \q_stop } { \c_hash_str }
        {
          \str_if_eq:eeT
            {#1}
            { \c_hash_str \c_space_tl Conditional~Mappings }
            { \ior_map_break: }
        }
        { \@@_data_auxi:w #1 \q_stop }
    }
  \ior_close:N \g_@@_data_ior
\group_end:
%    \end{macrocode}
%
% \begin{macro}[EXP]{\__kernel_codepoint_case:nn}
% \begin{macro}[EXP]{\@@_case:nnn}
% \begin{macro}[EXP]
%   {\@@_uppercase:n, \@@_lowercase:n, \@@_titlecase:n, \@@_casefold:n}
% \begin{macro}[EXP]{\@@_case:nn}
%   With the core data files loaded, there is now a need to provide access to
%   this information for other modules. That is done here such that case
%   folding can also be covered. At this level, all that needs to be returned
%   is the
%    \begin{macrocode}
\cs_new:Npn \__kernel_codepoint_case:nn #1#2
  {
    \exp_args:Ne \@@_case:nnn
      { \codepoint_str_generate:n {#2} } {#1} {#2}
  }
\cs_new:Npn \@@_case:nnn #1#2#3
  {
    \cs_if_exist:cTF { c_@@_ #2 _ #1 _tl }
      {
        \tl_use:c
          { c_@@_ #2 _ #1 _tl }
      }
      { \use:c { @@_ #2 :n } {#3} }
  }
\cs_new:Npn \@@_uppercase:n { \@@_case:nn { uppercase } }
\cs_new:Npn \@@_lowercase:n { \@@_case:nn { lowercase } }
\sys_if_engine_luatex:TF
  {
    \cs_new:Npn \@@_titlecase:n { \@@_case:nn { titlecase } }
  }
  {
    \cs_new:Npn \@@_titlecase:n { \@@_case:nn { uppercase } }
  }
\cs_new:Npn \@@_casefold:n  { \@@_case:nn { lowercase } }
\sys_if_engine_luatex:TF
  {
    \cs_new:Npn \@@_case:nn #1#2
      {
        { \__kernel_codepoint_data:wn \int_eval:n {#2} {#1} }
        { }
        { }
      }
  }
  {
    \cs_new:Npn \@@_case:nn #1#2
      {
        { \int_eval:n { \__kernel_codepoint_data:nn {#1} {#2} + #2 } }
        { }
        { }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]
%   {
%     \__kernel_codepoint_to_grapheme_class:n,
%     \__kernel_codepoint_to_wordbreak_class:n
%   }
%   Get the value and convert back to the string: not currently
%   public but otherwise very similar to \cs{codepoint_to_category:n}.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \cs_new:Npn \__kernel_codepoint_to_grapheme_class:n #1
      {
        \__kernel_codepoint_data:wn \int_eval:n {#1} { grapheme }
      }
    \cs_new:Npn \__kernel_codepoint_to_wordbreak_class:n #1
      {
        \__kernel_codepoint_data:wn \int_eval:n {#1} { wordbreak }
      }
  }
  {
    \cs_new:Npn \__kernel_codepoint_to_grapheme_class:n #1
      {
        \cs:w
          c_@@_grapheme_
          \tex_romannumeral:D 
            \__kernel_codepoint_data:nn { grapheme } {#1}
          _str
        \cs_end:
      }
    \cs_new:Npn \__kernel_codepoint_to_wordbreak_class:n #1
      {
        \cs:w
          c_@@_wordbreak_
          \tex_romannumeral:D 
            \__kernel_codepoint_data:nn { wordbreak } {#1}
          _str
        \cs_end:
      }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_nfd:n}
% \begin{macro}[EXP]{\@@_nfd:nn}
%   A simple interface.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \cs_new:Npn \@@_nfd:n #1
      { \@@_nfd:w \__int_eval:w #1 \__int_eval_end: }
  }
  {
    \cs_new:Npn \@@_nfd:n #1
      { \exp_args:Ne \@@_nfd:nn { \codepoint_str_generate:n {#1} } {#1} }
    \cs_new:Npn \@@_nfd:nn #1#2
      {
        \tl_if_exist:cTF { c_@@_nfd_ #1 _tl }
          { \tl_use:c { c_@@_nfd_ #1 _tl } }
          { {#2} { } }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%    \begin{macrocode}
%</code>
%    \end{macrocode}
%
% \end{implementation}
%
% \PrintIndex
