catc-ini.mkiv

%D \module
%D   [       file=catc-ini,
%D        version=2006.09.18,
%D          title=\CONTEXT\ System Macros,
%D       subtitle=Catcode Handling,
%D         author=Hans Hagen,
%D           date=\currentdate,
%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
%C
%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

%D We've split the functionality of syst-cat.* over more files
%D now so that we can load more selectively.

\registerctxluafile{catc-ini}{}

\unprotect

%D A long standing wish has been the availability of catcode arrays. Because
%D traditional \TEX\ does not provide this we implement a fake method in the
%D \MKII\ file. There is some overlap in code with \MKII\ but we take that
%D for granted. Also, in \MKIV\ less active characters are used.

% \normalprotected\def\setnewconstantfromchar#1%
%   {\expandafter\ifdefined\expandafter#1\expandafter
%         \let\expandafter#1\expandafter\undefined\expandafter\fi\expandafter
%         \newcount\expandafter#1\expandafter#1\the#1\relax}
%
% \normalprotected\def\setnewconstantfromchar#1%
%   {\begingroup
%    \scratchcounter#1%
%    \edef\!!stringa{\meaning#1}%
%    \chardef#1\scratchcounter
%    \edef\!!stringb{\meaning#1}%
%    \normalexpanded{\endgroup
%      \ifx\!!stringa\!!stringb
%        \let#1\noexpand\undefined
%        \newcount#1%
%      \fi
%      #1\the\scratchcounter\relax}}
%
% \normalprotected\def\setnewconstantfromchar#1%
%   {\begingroup
%    \edef\!!stringa{\meaning#1}%
%    \expandafter\chardef\expandafter#1\the#1%
%    \edef\!!stringb{\meaning#1}%
%    \normalexpanded{\endgroup
%      \ifx\!!stringa\!!stringb
%        \let#1\noexpand\undefined
%        \newcount#1%
%      \fi
%      #1\the#1\relax}}
%
% \normalprotected\def\setnewconstantfromchar#1%
%   {\scratchcounter#1\let#1\undefined\newcount#1#1\scratchcounter}

\def\promote#1{\scratchcounter#1\let#1\undefined\newcount#1#1\scratchcounter}

\promote\escapecatcode
\promote\begingroupcatcode
\promote\endgroupcatcode
\promote\mathshiftcatcode
\promote\alignmentcatcode
\promote\endoflinecatcode
\promote\parametercatcode
\promote\superscriptcatcode
\promote\subscriptcatcode
\promote\ignorecatcode
\promote\spacecatcode
\promote\lettercatcode
\promote\othercatcode
\promote\activecatcode
\promote\commentcatcode
\promote\invalidcatcode

\promote\tabasciicode
\promote\newlineasciicode
\promote\formfeedasciicode
\promote\endoflineasciicode
\promote\endoffileasciicode
\promote\spaceasciicode
\promote\exclamationmarkasciicode
\promote\doublequoteasciicode
\promote\hashasciicode
\promote\dollarasciicode
\promote\commentasciicode
\promote\ampersandasciicode
\promote\singlequoteasciicode
\promote\primeasciicode
\promote\hyphenasciicode
\promote\forwardslashasciicode
\promote\colonasciicode
\promote\lessthanasciicode
\promote\morethanasciicode
\promote\questionmarkasciicode
\promote\atsignasciicode
\promote\backslashasciicode
\promote\circumflexasciicode
\promote\underscoreasciicode
\promote\leftbraceasciicode
\promote\barasciicode
\promote\rightbraceasciicode
\promote\tildeasciicode
\promote\delasciicode

\let\promote\undefined

% \begingroup
%
%     \catcode\tabasciicode      \activecatcode
%     \catcode\formfeedasciicode \activecatcode
%     \catcode\endoflineasciicode\activecatcode
%
%     \letcharcode\tabasciicode      \relax
%     \letcharcode\newlineasciicode  \relax
%     \letcharcode\formfeedasciicode \relax
%     \letcharcode\endoflineasciicode\relax
%
%     \xdef\activetabtoken      {\Uchar\tabasciicode      } % \gdef\activetabtoken      {^^I}
%     \xdef\outputnewlinechar   {\Uchar\newlineasciicode  } % \gdef\outputnewlinechar   {^^J}
%     \xdef\activeformfeedtoken {\Uchar\formfeedasciicode } % \gdef\activeformfeedtoken {^^L}
%     \xdef\activeendoflinetoken{\Uchar\endoflineasciicode} % \gdef\activeendoflinetoken{^^M}
%
% \endgroup

\begingroup
    \letcharcode\newlineasciicode\relax \xdef\outputnewlinechar{\Uchar\newlineasciicode}
\endgroup

% \endlinechar = \endoflineasciicode % appended to input lines
% \newlinechar = \newlineasciicode   % can be used in write

% rather special and used in writing to file: \let\par\outputnewlinechar

% \normalprotected\def\initializenewlinechar % operating system dependent
%   {\begingroup
%    \newlinechar\newlineasciicode
%    \xdef\outputnewlinechar{^^J}%
%    \endgroup}

\normalprotected\def\initializenewlinechar % operating system dependent
  {\begingroup
   \letcharcode\newlineasciicode\relax
   \newlinechar\newlineasciicode
   \xdef\outputnewlinechar{\Uchar\newlineasciicode}%
   \endgroup}

%D We predefine some prefixes ahead of syst-aux and mult-sys.

% We reserve 8 slots for catcodes.
%
% \def\??catcodelet   {1>>} % let : \let
% \def\??catcodedef   {2>>} % def : \def
% \def\??catcodeued   {3>>} % ued : \unexpanded\def
% \def\??catcodeget   {4>>} %       \meaning
%
% \def\??catcodetablet{5>>}
% \def\??catcodetablen{6>>}

\installsystemnamespace {catcodelet} % let : \let
\installsystemnamespace {catcodedef} % def : \def
\installsystemnamespace {catcodeued} % ued : \unexpanded\def
\installsystemnamespace {catcodeget} %       \meaning

\installsystemnamespace {catcodetablet}
\installsystemnamespace {catcodetablen}

\newcount\c_syst_catcodes_n \c_syst_catcodes_n\zerocount % 0 = signal, so advance before allocate
\newcount\c_syst_catcodes_a
\newcount\c_syst_catcodes_b
\newcount\c_syst_catcodes_c

\normalprotected\def\newcatcodetable#1% we could move the cctdefcounter to lua
  {\global\advance\c_syst_catcodes_n\plusone
   \expandafter\xdef\csname\??catcodetablen\number\c_syst_catcodes_n\endcsname{\string#1}% logging
   \newconstant#1%
   #1\c_syst_catcodes_n
   \ctxcommand{registercatcodetable("\expandafter\gobbleoneargument\string#1",\number#1)}}

\newtoks \everysetdefaultcatcodes

\everysetdefaultcatcodes % this might get dropped
  {\catcode\backslashasciicode\othercatcode
   \catcode\endoflineasciicode\othercatcode
   \catcode\spaceasciicode    \othercatcode
   \catcode\commentasciicode  \othercatcode
   \catcode\delasciicode      \othercatcode}

\normalprotected\def\startcatcodetable#1#2\stopcatcodetable
  {\begingroup
   \catcodetable\inicatcodes
   \the\everysetdefaultcatcodes
   #2%
   \savecatcodetable#1\relax
   \endgroup}

\let\stopcatcodetable\relax

\normalprotected\def\startextendcatcodetable#1#2\stopextendcatcodetable
  {\begingroup
   \catcodetable#1\relax
   \globaldefs\plusone
   #2%
   \globaldefs\zerocount
   \endgroup}

\let\stopextendcatcodetable\relax

\normalprotected\def\permitcircumflexescape % to be used grouped
  {\catcode\circumflexasciicode\superscriptcatcode}

\let\permitcaretescape\permitcircumflexescape

% ==
%
% \normalprotected\def\startextendcatcodetable#1#2\stopextendcatcodetable
%   {\bgroup
%    \scratchcounter\the\catcodetable
%    \catcodetable #1 #2
%    \catcodetable\scratchcounter
%    \egroup}

%D The next command can be defined in a cleaner way in the MkIV way but we want
%D to have a fast one with a minimal chance for interference. Do we still need
%D this complex mechanism? Probably not. Future versions of \MKIV\ might only use
%D active characters for very special cases.

\setnewconstant\c_syst_catcodes_hack\tildeasciicode

%D Once a catcode is assigned, the next assignments will happen faster. However,
%D redefinitions probably happen seldom so it's sort of overkill.

\def\letcatcodecommand{\afterassignment\syst_catcodes_let_a\c_syst_catcodes_a}
\def\defcatcodecommand{\afterassignment\syst_catcodes_def_a\c_syst_catcodes_a}
\def\uedcatcodecommand{\afterassignment\syst_catcodes_ued_a\c_syst_catcodes_a}

\def\syst_catcodes_let_a{\afterassignment\syst_catcodes_let_b\c_syst_catcodes_b}
\def\syst_catcodes_def_a{\afterassignment\syst_catcodes_def_b\c_syst_catcodes_b}
\def\syst_catcodes_ued_a{\afterassignment\syst_catcodes_ued_b\c_syst_catcodes_b}

\def\syst_catcodes_let_b % each time
  {\ifcsname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
     \expandafter\lastnamedcs
   \else
     \expandafter\syst_catcodes_let_c
   \fi}

\def\syst_catcodes_def_b % each time
  {\ifcsname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
     \expandafter\lastnamedcs
   \else
     \expandafter\syst_catcodes_def_c
   \fi}

\def\syst_catcodes_ued_b % each time
  {\ifcsname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname
     \expandafter\lastnamedcs
   \else
     \expandafter\syst_catcodes_ued_c
   \fi}

\def\syst_catcodes_let_c % only first time
  {\expandafter\gdef\csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname\expandafter
     {\expandafter\let\csname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}%
   \syst_catcodes_reinstate_unexpanded
   \csname\??catcodelet\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}

\def\syst_catcodes_def_c % only first time (we could use \normalexpanded here)
  {\expandafter\gdef\csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
     \expandafter##\expandafter1\expandafter
       {\expandafter\def\csname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}%
   \syst_catcodes_reinstate_normal
   \csname\??catcodedef\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}

\def\syst_catcodes_ued_c % only first time
  {\expandafter\gdef\csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\expandafter\endcsname
     \expandafter##\expandafter1\expandafter
       {\expandafter\normalprotected\expandafter\def\csname\??catcodeget\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname{##1}}%
   \syst_catcodes_reinstate_unexpanded
   \csname\??catcodeued\number\c_syst_catcodes_a:\number\c_syst_catcodes_b\endcsname}

\def\reinstatecatcodecommand{\afterassignment\syst_catcodes_reinstate_normal\c_syst_catcodes_b}

\def\syst_catcodes_reinstate_normal % can be used when a direct definition has been done
  {\begingroup                      % and the selector has been lost
   \uccode\c_syst_catcodes_hack\c_syst_catcodes_b
   \catcode\uccode\c_syst_catcodes_hack\activecatcode
   \uppercase{\xdef~{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}}%
   \endgroup}

\def\syst_catcodes_reinstate_unexpanded % can be used when a direct definition has been done
  {\begingroup                          % and the selector has been lost
   \uccode\c_syst_catcodes_hack\c_syst_catcodes_b
   \catcode\uccode\c_syst_catcodes_hack\activecatcode
   \uppercase{\normalprotected\xdef~{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}}%
   \endgroup}

%D This can be used when a direct definition has been done and the selector has been
%D lost.

% problem: \next needs to be unique (as it gets bound)
%
% \def\syst_catcodes_reinstate_normal
%   {\begingroup
%    \edef\next{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}%
%    \global\letcharcode\c_syst_catcodes_b\next
%    \endgroup}
%
% \def\syst_catcodes_reinstate_unexpanded
%   {\begingroup
%    \normalprotected\edef\next{\noexpand\catcodecommand{\number\c_syst_catcodes_b}}%
%    \global\letcharcode\c_syst_catcodes_b\next
%    \endgroup}

\newconstant\defaultcatcodetable

\def\catcodecommand#1%
  {\csname\??catcodeget\number
     \ifcsname\??catcodeget\number\currentcatcodetable:\number#1\endcsname
       \currentcatcodetable \else \defaultcatcodetable
     \fi
   :\number#1\endcsname}

%D \macros
%D   {restorecatcodes,pushcatcodetable,popcatcodetable}
%D
%D We're not finished dealing \CATCODES\ yet. In \CONTEXT\ we use only one auxiliary
%D file, which deals with tables of contents, registers, two pass tracking, references
%D etc. This file, as well as files concerning graphics, is processed when needed,
%D which can be in the mid of typesetting verbatim. However, when reading in data in
%D verbatim mode, we should temporary restore the normal \CATCODES, and that's exactly
%D what the next macros do. Saving the catcodes can be disabled by saying \type
%D {\localcatcodestrue}. In \MKIV\ instead we can push and pop catcode tables and as
%D we keep track of used tables users seldom need to deal with this themselves.

\newcount\c_syst_catcodes_level

\normalprotected\def\pushcatcodetable
  {\advance\c_syst_catcodes_level\plusone
   \syst_catcodes_trace_push
   \expandafter\chardef\csname\??catcodetablet\number\c_syst_catcodes_level\endcsname\currentcatcodetable}

\normalprotected\def\popcatcodetable
  {\ifcase\c_syst_catcodes_level
     \syst_catcodes_trace_nesting_error
   \else
     \expandafter\catcodetable\csname\??catcodetablet\number\c_syst_catcodes_level\endcsname
     \syst_catcodes_trace_pop
     \advance\c_syst_catcodes_level\minusone
   \fi}

\normalprotected\def\syst_catcodes_trace_nesting_error
  {\immediate\write\statuswrite{}%
   \immediate\write\statuswrite{Fatal error: catcode push/pop mismatch. Fix this! (restore level: \number\c_syst_catcodes_level)}\wait\end
   \immediate\write\statuswrite{}}

\normalprotected\def\restorecatcodes % takes previous level
  {\ifnum\c_syst_catcodes_level>\plusone
     \expandafter\catcodetable\csname\??catcodetablet\number\numexpr\c_syst_catcodes_level-1\relax\endcsname
   \fi}

% \newtoks\everycatcodetable

\normalprotected\def\setcatcodetable#1%
  {\catcodetable#1%
%  \the\everycatcodetable
   \syst_catcodes_trace_set}

%D Handy for debugging:
%D
%D \starttyping
%D \tracecatcodetables
%D \stoptyping

\normalprotected\def\tracecatcodetables
  {\def\syst_catcodes_trace_set {\syst_catcodes_trace{set  \catcodetablename\space                                at \number\c_syst_catcodes_level}}%
   \def\syst_catcodes_trace_push{\syst_catcodes_trace{push \catcodetablename\space from \syst_catcodes_prev\space at \number\c_syst_catcodes_level}}%
   \def\syst_catcodes_trace_pop {\syst_catcodes_trace{pop  \catcodetablename\space to   \syst_catcodes_prev\space at \number\c_syst_catcodes_level}}}

\def\syst_catcodes_trace#1{\immediate\write\statuswrite{[#1]}}

\def\syst_catcodes_prev
  {\ifnum\numexpr\c_syst_catcodes_level-1\relax>\zerocount
     \csname\??catcodetablen\number\csname\??catcodetablet\number\numexpr\c_syst_catcodes_level-1\relax\endcsname\endcsname
   \else
     -%
   \fi}

\def\catcodetablename
  {\ifnum\currentcatcodetable>\zerocount
     \csname\??catcodetablen\number\currentcatcodetable\endcsname
   \else
     -%
   \fi}

\let\syst_catcodes_trace_set \empty
\let\syst_catcodes_trace_push\empty
\let\syst_catcodes_trace_pop \empty

% \tracecatcodetables

\protect

%D We still have to define these so let's do that now:

\newcatcodetable \inicatcodes
\initcatcodetable\inicatcodes

\let\currentcatcodetable\catcodetable

\endinput